mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 19:33:41 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b89d7e36bf |
No files matched your search
@@ -1,307 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regroup a GitHub release's generated notes by component.
|
||||
|
||||
GitHub's generated notes are one flat list of PRs in merge order. This keeps every
|
||||
line as GitHub wrote it, but sorts the PRs under component headings (S3 API, Filer,
|
||||
Mount, Volume Server, ...), with dependency bumps folded at the end.
|
||||
|
||||
The component comes from the PR title prefix, which nearly every PR has
|
||||
("s3api: ...", "fix(mount): ...", "filer.sync: ...", "rust volume: ..."). A PR with
|
||||
no recognised prefix is placed by the files it changes.
|
||||
|
||||
Usage:
|
||||
release_notes_by_component.py --tag 4.54 # print the regrouped notes
|
||||
release_notes_by_component.py --tag 4.54 --write # also update the release
|
||||
|
||||
Running it again on notes it already regrouped gives the same result.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# Display order. Each PR lands in exactly one section.
|
||||
SECTIONS = [
|
||||
"S3 API",
|
||||
"S3 Tables and Iceberg",
|
||||
"Vectors",
|
||||
"Filer",
|
||||
"Mount",
|
||||
"Volume Server",
|
||||
"Rust Volume Server",
|
||||
"Erasure Coding",
|
||||
"Master and Topology",
|
||||
"Admin UI and Workers",
|
||||
"Shell",
|
||||
"Security",
|
||||
"Message Queue",
|
||||
"Kubernetes, Helm and Deployment",
|
||||
"Core and CLI",
|
||||
"Documentation",
|
||||
"Tests and CI",
|
||||
"Other",
|
||||
"Dependencies",
|
||||
]
|
||||
|
||||
# Title scope -> section, first match wins. Matched against each lower-cased scope
|
||||
# token, e.g. "filer.sync", "s3 lifecycle", "weed/s3api" (a "weed/" prefix is
|
||||
# dropped first). Order matters: specific scopes come before the general ones
|
||||
# they start with (s3tables before s3, volume.fsck before volume).
|
||||
SCOPE_RULES = [
|
||||
(r"(build\(deps\)|deps|dependabot)\b", "Dependencies"),
|
||||
(r"(test|tests|e2e|integration)\b|.*\btests?$|test/", "Tests and CI"),
|
||||
(r"(rust[ -]?volume|seaweed-volume|volume\(rust\)|rust)\b", "Rust Volume Server"),
|
||||
(r"(s3 ?vectors?|vectors?|seaweed-vector|weed-vector|lance)\b", "Vectors"),
|
||||
(r"(s3 ?tables|iceberg)\b", "S3 Tables and Iceberg"),
|
||||
(r"(s3 sink|remote|remote_storage|filer|webdav|sftp|ftp|notification|replication|"
|
||||
r"redis\d*|leveldb\d*|backup)\b", "Filer"),
|
||||
(r"(s3|s3api|iam|oidc|sts|credential|sse|cors|policy|object ?lock)\b", "S3 API"),
|
||||
(r"(mount|fuse)\b", "Mount"),
|
||||
(r"(ec|ec_\w+|ec\.\w+|erasure[ _-]?coding|worker/ec)\b", "Erasure Coding"),
|
||||
(r"(master|topology|wdclient|cluster|raft|placement)\b", "Master and Topology"),
|
||||
(r"(admin|worker|weed-worker|plugin|plugin worker|maintenance)\b",
|
||||
"Admin UI and Workers"),
|
||||
(r"(shell|fsck|volume\.\w+|fs\.\w+|collection\.\w+|bucket\.\w+)", "Shell"),
|
||||
(r"(volume|volume_\w+|storage|vacuum|operation|compression|needle)\b", "Volume Server"),
|
||||
(r"(security|tls|mtls|jwt|grpc|kms)\b", "Security"),
|
||||
(r"(mq|kafka|messaging)\b", "Message Queue"),
|
||||
(r"(helm|k8s|kubernetes|operator|csi|docker|grafana|install\.sh|chart|snap|terraform)\b",
|
||||
"Kubernetes, Helm and Deployment"),
|
||||
(r"(docs?|readme|wiki)\b", "Documentation"),
|
||||
(r"(ci|release|build|github|workflow|actions)\b", "Tests and CI"),
|
||||
(r"(server|command|mini|weed-db|weed|util|pb|cmd|glog|stats|telemetry|query|config)\b",
|
||||
"Core and CLI"),
|
||||
]
|
||||
|
||||
# Changed path prefix -> section, for PRs whose title has no usable prefix.
|
||||
# Longest prefix wins.
|
||||
PATH_RULES = {
|
||||
"weed/s3api/s3tables/": "S3 Tables and Iceberg",
|
||||
"weed/s3api/iceberg/": "S3 Tables and Iceberg",
|
||||
"weed/s3api/s3vectors/": "Vectors",
|
||||
"weed/s3api/lance/": "Vectors",
|
||||
"seaweed-vector/": "Vectors",
|
||||
"weed/s3api/": "S3 API",
|
||||
"weed/iam/": "S3 API",
|
||||
"weed/iamapi/": "S3 API",
|
||||
"weed/credential/": "S3 API",
|
||||
"weed/filer/": "Filer",
|
||||
"weed/filer_client/": "Filer",
|
||||
"weed/server/filer": "Filer",
|
||||
"weed/server/webdav": "Filer",
|
||||
"weed/sftpd/": "Filer",
|
||||
"weed/replication/": "Filer",
|
||||
"weed/notification/": "Filer",
|
||||
"weed/remote_storage/": "Filer",
|
||||
"weed/mount/": "Mount",
|
||||
"weed/storage/erasure_coding/": "Erasure Coding",
|
||||
"weed/ec/": "Erasure Coding",
|
||||
"weed/worker/tasks/erasure_coding/": "Erasure Coding",
|
||||
"weed/storage/": "Volume Server",
|
||||
"weed/server/volume": "Volume Server",
|
||||
"weed/operation/": "Volume Server",
|
||||
"seaweed-volume/": "Rust Volume Server",
|
||||
"weed/topology/": "Master and Topology",
|
||||
"weed/server/master": "Master and Topology",
|
||||
"weed/wdclient/": "Master and Topology",
|
||||
"weed/cluster/": "Master and Topology",
|
||||
"weed/placement/": "Master and Topology",
|
||||
"weed/admin/": "Admin UI and Workers",
|
||||
"weed/worker/": "Admin UI and Workers",
|
||||
"weed/plugin/": "Admin UI and Workers",
|
||||
"seaweed-worker/": "Admin UI and Workers",
|
||||
"weed/shell/": "Shell",
|
||||
"weed/security/": "Security",
|
||||
"weed/kms/": "Security",
|
||||
"weed/mq/": "Message Queue",
|
||||
"k8s/": "Kubernetes, Helm and Deployment",
|
||||
"docker/": "Kubernetes, Helm and Deployment",
|
||||
"snap/": "Kubernetes, Helm and Deployment",
|
||||
"terraform/": "Kubernetes, Helm and Deployment",
|
||||
"weed/": "Core and CLI",
|
||||
"test/": "Tests and CI",
|
||||
".github/": "Tests and CI",
|
||||
}
|
||||
|
||||
PR_LINE = re.compile(r"^\* (?P<title>.+?) by @(?P<author>\S+) in https://github\.com/\S+/pull/(?P<num>\d+)\s*$")
|
||||
# Lines regroup() writes between the PRs, so a second run accepts them.
|
||||
SCAFFOLD = re.compile(r"^$|^### (%s)$|^<details><summary>Dependencies \(\d+ updates?\)</summary>$|^</details>$"
|
||||
% "|".join(re.escape(s) for s in SECTIONS))
|
||||
# "fix(s3tables,s3vectors): ...", "s3 lifecycle: ...", "[Volume] ...", "Fix/admin ..."
|
||||
CONVENTIONAL = re.compile(r"^(?P<type>[a-z]+)\((?P<scope>[^)]*)\)!?$")
|
||||
TYPE_ONLY = {"fix", "feat", "feature", "perf", "refactor", "chore", "style", "revert", "improvement"}
|
||||
|
||||
|
||||
def scope_of(title):
|
||||
"""Return the raw scope text of a PR title, or ""."""
|
||||
t = title.strip()
|
||||
m = re.match(r"^\[([^\]]{1,40})\]", t)
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = re.match(r"^(?:fix|feat|feature)/(\S+)", t, re.I)
|
||||
if m:
|
||||
return m.group(1)
|
||||
head, sep, _ = t.partition(":")
|
||||
if not sep or len(head) > 50:
|
||||
return ""
|
||||
head = head.strip()
|
||||
m = CONVENTIONAL.match(head.lower())
|
||||
if m:
|
||||
kind, scope = m.group("type"), m.group("scope")
|
||||
if kind in ("test", "tests", "ci"):
|
||||
return "test"
|
||||
if kind == "docs":
|
||||
return "docs"
|
||||
if kind == "build" and scope == "deps":
|
||||
return "build(deps)"
|
||||
return scope
|
||||
if head.lower() in TYPE_ONLY:
|
||||
return ""
|
||||
return head
|
||||
|
||||
|
||||
def section_by_scope(scope):
|
||||
if scope.strip().lower() == "build(deps)":
|
||||
return "Dependencies"
|
||||
for token in re.split(r"\s*[,+&]\s*|\s+and\s+", scope.lower()):
|
||||
token = token.strip()
|
||||
token = re.sub(r"^weed[/ ]", "", token)
|
||||
if not token:
|
||||
continue
|
||||
for pattern, section in SCOPE_RULES:
|
||||
if re.match(pattern, token):
|
||||
return section
|
||||
return ""
|
||||
|
||||
|
||||
def section_by_paths(paths):
|
||||
votes = {}
|
||||
code = [p for p in paths if not p.endswith(("_test.go", "_test.rs")) and not p.startswith("test/")]
|
||||
for p in code or paths:
|
||||
if p.endswith(".md"):
|
||||
section = "Documentation"
|
||||
else:
|
||||
best = max((pre for pre in PATH_RULES if p.startswith(pre)), key=len, default="")
|
||||
section = PATH_RULES.get(best, "Other")
|
||||
votes[section] = votes.get(section, 0) + 1
|
||||
if not votes:
|
||||
return "Other"
|
||||
# Most files wins; ties go to the earlier section in display order.
|
||||
return max(votes, key=lambda s: (votes[s], -SECTIONS.index(s)))
|
||||
|
||||
|
||||
def classify(title, author, pr_files):
|
||||
if author.startswith("dependabot") or title.lower().startswith("build(deps)"):
|
||||
return "Dependencies"
|
||||
section = section_by_scope(scope_of(title))
|
||||
if section:
|
||||
return section
|
||||
return section_by_paths(pr_files())
|
||||
|
||||
|
||||
def split_notes(body):
|
||||
"""Split generated notes into (PR lines, the text before them, the text after,
|
||||
the non-blank lines of What's Changed)."""
|
||||
lines = body.replace("\r\n", "\n").split("\n")
|
||||
start = next((i for i, l in enumerate(lines) if l.strip() == "## What's Changed"), None)
|
||||
if start is None:
|
||||
return [], body, "", []
|
||||
end = next((i for i in range(start + 1, len(lines))
|
||||
if lines[i].startswith(("## ", "**Full Changelog**"))), len(lines))
|
||||
prs, stray = [], []
|
||||
block = [l.rstrip() for l in lines[start + 1:end] if l.strip()]
|
||||
for line in lines[start + 1:end]:
|
||||
if PR_LINE.match(line):
|
||||
prs.append(line)
|
||||
elif not SCAFFOLD.match(line.strip()):
|
||||
stray.append(line)
|
||||
if prs and stray:
|
||||
raise ValueError("What's Changed has lines that are not PRs: %r" % stray[:3])
|
||||
before = "\n".join(lines[:start]).rstrip()
|
||||
after = "\n".join(lines[end:]).strip()
|
||||
return prs, before, after, block
|
||||
|
||||
|
||||
def regroup(body, pr_files):
|
||||
prs, before, after, block = split_notes(body)
|
||||
if not prs:
|
||||
return body
|
||||
groups = {s: [] for s in SECTIONS}
|
||||
for line in prs:
|
||||
m = PR_LINE.match(line)
|
||||
num = int(m.group("num"))
|
||||
groups[classify(m.group("title"), m.group("author"), lambda: pr_files(num))].append(line)
|
||||
|
||||
out = [before, ""] if before else []
|
||||
out.append("## What's Changed")
|
||||
for section in SECTIONS:
|
||||
items = groups[section]
|
||||
if not items:
|
||||
continue
|
||||
out.append("")
|
||||
if section == "Dependencies":
|
||||
out.append("<details><summary>Dependencies (%d update%s)</summary>" % (len(items), "" if len(items) == 1 else "s"))
|
||||
out.append("")
|
||||
out.extend(items)
|
||||
out.append("")
|
||||
out.append("</details>")
|
||||
else:
|
||||
out.append("### " + section)
|
||||
out.extend(items)
|
||||
if after:
|
||||
out.extend(["", after])
|
||||
notes = "\n".join(out).strip() + "\n"
|
||||
# Headings are accepted only as this script's own grouping: a hand-written
|
||||
# "### Security" over PRs of other components would otherwise be dropped.
|
||||
if len(block) != len(prs) and split_notes(notes)[3] != block:
|
||||
raise ValueError("What's Changed has headings that do not match the component grouping")
|
||||
return notes
|
||||
|
||||
|
||||
def gh(*args, stdin=None):
|
||||
return subprocess.run(["gh", *args], input=stdin, capture_output=True, text=True, check=True).stdout
|
||||
|
||||
|
||||
def pr_files(repo, num):
|
||||
return gh("pr", "view", str(num), "-R", repo, "--json", "files", "--jq", ".files[].path").splitlines()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--repo", default="seaweedfs/seaweedfs")
|
||||
ap.add_argument("--tag", required=True)
|
||||
ap.add_argument("--write", action="store_true", help="update the release notes on GitHub")
|
||||
args = ap.parse_args()
|
||||
|
||||
body = json.loads(gh("release", "view", args.tag, "-R", args.repo, "--json", "body"))["body"]
|
||||
|
||||
failed = []
|
||||
|
||||
def files_of(num):
|
||||
try:
|
||||
return pr_files(args.repo, num)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("warning: files of #%d: %s" % (num, e.stderr.strip()), file=sys.stderr)
|
||||
failed.append(num)
|
||||
return []
|
||||
|
||||
try:
|
||||
notes = regroup(body, files_of)
|
||||
except ValueError as e:
|
||||
sys.exit("release %s: %s" % (args.tag, e))
|
||||
sys.stdout.write(notes)
|
||||
if notes.strip() == body.replace("\r\n", "\n").strip():
|
||||
print("release %s: notes already grouped, nothing to change" % args.tag, file=sys.stderr)
|
||||
return
|
||||
if failed and args.write:
|
||||
sys.exit("release %s: not writing, could not read the files of %s"
|
||||
% (args.tag, ", ".join("#%d" % n for n in failed)))
|
||||
if args.write:
|
||||
gh("release", "edit", args.tag, "-R", args.repo, "--notes-file", "-", stdin=notes)
|
||||
print("release %s: notes updated" % args.tag, file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,190 +0,0 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
def load():
|
||||
path = Path(__file__).with_name("release_notes_by_component.py")
|
||||
spec = importlib.util.spec_from_file_location("release_notes_by_component", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
rn = load()
|
||||
|
||||
|
||||
def no_files(num):
|
||||
raise AssertionError("files of #%d should not be needed" % num)
|
||||
|
||||
|
||||
class ClassifyTest(unittest.TestCase):
|
||||
def check(self, title, section, author="someone", files=None):
|
||||
got = rn.classify(title, author, (lambda: files) if files is not None else (lambda: no_files(0)))
|
||||
self.assertEqual(got, section, title)
|
||||
|
||||
def test_title_prefixes(self):
|
||||
cases = {
|
||||
"s3: keep serving for -preStopSeconds after SIGTERM": "S3 API",
|
||||
"s3api: keep a multipart upload's state on the object's owner filer": "S3 API",
|
||||
"fix(s3api): reject an invalid storage class before uploading the body": "S3 API",
|
||||
"s3 lifecycle: count filer entries a walk reads but skips as progress": "S3 API",
|
||||
"iam/oidc: accept a list audience": "S3 API",
|
||||
"fix(s3tables,s3vectors): bind DeleteNamespace to the authorized records": "S3 Tables and Iceberg",
|
||||
"iceberg maintenance: expire snapshots": "S3 Tables and Iceberg",
|
||||
"s3vectors: support TagResource": "Vectors",
|
||||
"s3 vectors: page ListVectors": "Vectors",
|
||||
"vector: compact segments": "Vectors",
|
||||
"filer: fail health check when absent from lock ring": "Filer",
|
||||
"filer.sync: skip a rewritten source version": "Filer",
|
||||
"filer.remote.sync: retry": "Filer",
|
||||
"webdav: honor locks": "Filer",
|
||||
"remote.uncache, remote.meta.sync, remote.cache: keep local writes": "Filer",
|
||||
"mount: cache dir": "Mount",
|
||||
"fix(mount): flush on close": "Mount",
|
||||
"volume: keep reads over the download limit local": "Volume Server",
|
||||
"[Volume] Preserve request semantics in replica proxy reads": "Volume Server",
|
||||
"operation: pad the marginal gzip test": "Volume Server",
|
||||
"fix(storage): update deletion metrics": "Volume Server",
|
||||
"rust volume: bulk-load needle maps": "Rust Volume Server",
|
||||
"seaweed-volume: pin the status": "Rust Volume Server",
|
||||
"volume(rust): fix": "Rust Volume Server",
|
||||
"ec: mount shards through the encode's .vif": "Erasure Coding",
|
||||
"ec.rebuild: pick the right disk": "Erasure Coding",
|
||||
"ec_balance: spread shards": "Erasure Coding",
|
||||
"master: unregister a volume server whose heartbeats stop": "Master and Topology",
|
||||
"topology: never report negative free volume slots": "Master and Topology",
|
||||
"wdclient: let master-pushed adds replace locations": "Master and Topology",
|
||||
"admin: manage filer webhook targets": "Admin UI and Workers",
|
||||
"Fix/admin cluster submenu": "Admin UI and Workers",
|
||||
"weed-worker: retry": "Admin UI and Workers",
|
||||
"shell: fix s3.circuitBreaker help examples": "Shell",
|
||||
"volume.fsck: skip deleted": "Shell",
|
||||
"fix(fsck): inspect references to deleted volumes": "Shell",
|
||||
"security: optional dialed-host check": "Security",
|
||||
"helm: drop the 4.53 vector metrics render guard": "Kubernetes, Helm and Deployment",
|
||||
"install.sh: add --enterprise": "Kubernetes, Helm and Deployment",
|
||||
"mq/kafka: commit offsets": "Message Queue",
|
||||
"server: add -grpc.localSocketDir": "Core and CLI",
|
||||
"pb: recover when a local gRPC socket file is deleted": "Core and CLI",
|
||||
"docs: explain the 3-filer recommendation": "Documentation",
|
||||
"test(fuse): two mounts staging the same files": "Tests and CI",
|
||||
"test/s3vectors: a hardened cluster": "Tests and CI",
|
||||
"s3vectors test: cover tags": "Tests and CI",
|
||||
"ci: run the Python 3.9 jobs on 3.10": "Tests and CI",
|
||||
"build(deps): bump actions/setup-go from 6 to 7": "Dependencies",
|
||||
}
|
||||
for title, section in cases.items():
|
||||
self.check(title, section)
|
||||
|
||||
def test_dependabot_author(self):
|
||||
self.check("Bump x from 1 to 2", "Dependencies", author="dependabot[bot]")
|
||||
|
||||
def test_no_prefix_uses_files(self):
|
||||
self.check("Fix s3 cancelled read relookup", "S3 API",
|
||||
files=["weed/s3api/s3api_object_handlers.go", "weed/s3api/s3api_object_handlers_test.go"])
|
||||
self.check("Retry transient failures in star history generation", "Tests and CI",
|
||||
files=[".github/scripts/star_history.py"])
|
||||
self.check("fix: handle empty chunk list", "Filer",
|
||||
files=["weed/filer/filechunks.go", "weed/storage/needle/needle.go", "weed/filer/reader_at.go"])
|
||||
self.check("Improve everything", "Other", files=[])
|
||||
|
||||
def test_test_files_count_only_when_alone(self):
|
||||
self.check("Make it faster", "Mount",
|
||||
files=["weed/mount/weedfs.go", "test/fuse/a_test.go", "test/fuse/b_test.go"])
|
||||
self.check("Make it right", "S3 API",
|
||||
files=["weed/s3api/auth.go", "test/s3/iam/test_a.py", "test/s3/iam/test_b.py"])
|
||||
self.check("Cover more", "Tests and CI", files=["test/s3/iam/test_a.py"])
|
||||
|
||||
def test_path_with_space(self):
|
||||
with mock.patch.object(rn, "gh", return_value="test/s3/iam/policy cases.json\n"):
|
||||
files = rn.pr_files("seaweedfs/seaweedfs", 1)
|
||||
self.assertEqual(files, ["test/s3/iam/policy cases.json"])
|
||||
self.check("Cover more", "Tests and CI", files=files)
|
||||
|
||||
|
||||
NOTES = """## What's Changed
|
||||
* filer: fail health check when absent from lock ring by @a in https://github.com/seaweedfs/seaweedfs/pull/12709
|
||||
* s3: answer ClientDisconnected by @b in https://github.com/seaweedfs/seaweedfs/pull/12707
|
||||
* build(deps): bump actions/setup-go from 6 to 7 by @dependabot[bot] in https://github.com/seaweedfs/seaweedfs/pull/12685
|
||||
* filer.sync: skip a rewritten source version by @a in https://github.com/seaweedfs/seaweedfs/pull/12366
|
||||
* Fix s3 cancelled read relookup by @c in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
## New Contributors
|
||||
* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54
|
||||
"""
|
||||
|
||||
|
||||
class RegroupTest(unittest.TestCase):
|
||||
def files(self, num):
|
||||
self.assertEqual(num, 12600)
|
||||
return ["weed/s3api/s3api_object_handlers.go"]
|
||||
|
||||
def test_regroup(self):
|
||||
out = rn.regroup(NOTES, self.files)
|
||||
self.assertEqual(out, """## What's Changed
|
||||
|
||||
### S3 API
|
||||
* s3: answer ClientDisconnected by @b in https://github.com/seaweedfs/seaweedfs/pull/12707
|
||||
* Fix s3 cancelled read relookup by @c in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
### Filer
|
||||
* filer: fail health check when absent from lock ring by @a in https://github.com/seaweedfs/seaweedfs/pull/12709
|
||||
* filer.sync: skip a rewritten source version by @a in https://github.com/seaweedfs/seaweedfs/pull/12366
|
||||
|
||||
<details><summary>Dependencies (1 update)</summary>
|
||||
|
||||
* build(deps): bump actions/setup-go from 6 to 7 by @dependabot[bot] in https://github.com/seaweedfs/seaweedfs/pull/12685
|
||||
|
||||
</details>
|
||||
|
||||
## New Contributors
|
||||
* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54
|
||||
""")
|
||||
|
||||
def test_idempotent(self):
|
||||
once = rn.regroup(NOTES, self.files)
|
||||
self.assertEqual(rn.regroup(once, self.files), once)
|
||||
|
||||
def test_every_pr_kept(self):
|
||||
out = rn.regroup(NOTES, self.files)
|
||||
for n in (12709, 12707, 12685, 12366, 12600):
|
||||
self.assertEqual(out.count("/pull/%d\n" % n), 2 if n == 12600 else 1)
|
||||
|
||||
def test_changelog_kept_without_new_contributors(self):
|
||||
body = NOTES.replace(
|
||||
"## New Contributors\n* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600\n\n", "")
|
||||
out = rn.regroup(body, self.files)
|
||||
self.assertTrue(out.endswith("</details>\n\n**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54\n"), out)
|
||||
self.assertEqual(rn.regroup(out, self.files), out)
|
||||
|
||||
def test_unknown_line_refused(self):
|
||||
body = NOTES.replace("## New Contributors", "* a hand-written note\n\n## New Contributors")
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(body, self.files)
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(NOTES.replace("## What's Changed\n", "## What's Changed\n### Known issue\n"), self.files)
|
||||
|
||||
def test_hand_written_component_heading_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(NOTES.replace("## What's Changed\n", "## What's Changed\n### Security\n"), self.files)
|
||||
once = rn.regroup(NOTES, self.files)
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(once.replace("\n### Filer\n", "\n ### Filer\n"), self.files)
|
||||
|
||||
def test_author_from_the_by_clause(self):
|
||||
body = "## What's Changed\n* Bump a by @dependabot[bot] in docs by @alice in https://github.com/seaweedfs/seaweedfs/pull/7\n"
|
||||
self.assertIn("### Documentation", rn.regroup(body, lambda n: ["README.md"]))
|
||||
|
||||
def test_notes_without_prs_unchanged(self):
|
||||
body = "Hand-written notes\n\n**Full Changelog**: x"
|
||||
self.assertEqual(rn.regroup(body, no_files), body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -5,13 +5,11 @@ on:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/scripts/*release_notes_by_component.py'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/scripts/*release_notes_by_component.py'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
|
||||
concurrency:
|
||||
@@ -30,12 +28,3 @@ jobs:
|
||||
|
||||
- name: Run cleanup action unit tests
|
||||
run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py
|
||||
|
||||
release-notes-by-component:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run release notes grouping unit tests
|
||||
run: python3 .github/scripts/test_release_notes_by_component.py
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -104,51 +104,8 @@ jobs:
|
||||
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0"
|
||||
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate"
|
||||
grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
|
||||
test "$(grep -c -- '-preStopSeconds' /tmp/s3-rollout-defaults.yaml)" -eq 0
|
||||
echo "S3 rollout settings render correctly"
|
||||
|
||||
echo "=== Testing S3 preStopSeconds ==="
|
||||
if helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
|
||||
--set s3.enabled=true --set s3.preStopSeconds=20 > /tmp/s3-prestop-short.yaml 2>&1; then
|
||||
echo "FAIL: s3.preStopSeconds=20 rendered with the default 10s grace period"; exit 1
|
||||
fi
|
||||
grep -q "must be at least 35" /tmp/s3-prestop-short.yaml
|
||||
helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
|
||||
--set s3.enabled=true --set s3.preStopSeconds=20 \
|
||||
--set s3.terminationGracePeriodSeconds=35 > /tmp/s3-prestop.yaml
|
||||
grep -q -- "-preStopSeconds=20 " /tmp/s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 35$" /tmp/s3-prestop.yaml
|
||||
|
||||
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
|
||||
--set filer.s3.enabled=true > /tmp/filer-s3-prestop-defaults.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/filer-s3-prestop-defaults.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/filer-s3-prestop-defaults.yaml
|
||||
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
|
||||
--set filer.s3.enabled=true --set filer.s3.preStopSeconds=20 > /tmp/filer-s3-prestop.yaml
|
||||
grep -q -- "-s3.preStopSeconds=20 " /tmp/filer-s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 80$" /tmp/filer-s3-prestop.yaml
|
||||
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true > /tmp/aio-s3-prestop-defaults.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-defaults.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-defaults.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 > /tmp/aio-s3-prestop.yaml
|
||||
grep -q -- "-s3.preStopSeconds=20 " /tmp/aio-s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 80$" /tmp/aio-s3-prestop.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=0 > /tmp/aio-s3-prestop-off.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-off.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-off.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=25 > /tmp/aio-s3-prestop-own.yaml
|
||||
grep -q -- "-s3.preStopSeconds=25 " /tmp/aio-s3-prestop-own.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 85$" /tmp/aio-s3-prestop-own.yaml
|
||||
echo "S3 preStopSeconds renders correctly"
|
||||
|
||||
echo "=== Testing the S3 config checksum is stable across renders ==="
|
||||
# s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
|
||||
# Secret drew new keys for the checksum whenever lookup found nothing (on
|
||||
|
||||
@@ -4,9 +4,7 @@ name: "release: bump version and cut the release"
|
||||
# 1. bump MAJOR/MINOR in constants.go and the Helm Chart.yaml, commit to master
|
||||
# 2. push the <appVersion> tag, which fans out to the workflows that trigger on
|
||||
# `push: tags` (binaries_release*, container_release_unified, helm_manual_release)
|
||||
# 3. create the GitHub release, with GitHub's generated notes, then regroup those
|
||||
# notes by component (S3 API, Filer, Mount, ...) with
|
||||
# .github/scripts/release_notes_by_component.py
|
||||
# 3. create the GitHub release, with GitHub's generated notes
|
||||
# 4. dispatch "Prepare release" in seaweedfs-csi-driver and seaweedfs-operator,
|
||||
# which pick up the new master through `go get -u`, and wait for both
|
||||
#
|
||||
@@ -43,7 +41,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: read # the notes step reads a PR's files when its title has no component prefix
|
||||
outputs:
|
||||
app_version: ${{ steps.compute.outputs.app_version }}
|
||||
sha: ${{ steps.tag.outputs.sha }}
|
||||
@@ -177,19 +174,6 @@ jobs:
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
run: gh release create "$TAG" --title "$TAG" --generate-notes --verify-tag
|
||||
|
||||
# GitHub's generated notes are one flat list of PRs. Sort them under
|
||||
# component headings by title prefix (or changed files). The release is
|
||||
# already published, so a failure here only leaves the flat list; rerun
|
||||
# the script by hand to fix it:
|
||||
# python3 .github/scripts/release_notes_by_component.py --tag <tag> --write
|
||||
- name: Group the release notes by component
|
||||
if: ${{ !inputs.dry_run }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
run: python3 .github/scripts/release_notes_by_component.py --repo "$GITHUB_REPOSITORY" --tag "$TAG" --write > /dev/null
|
||||
|
||||
downstream:
|
||||
needs: release
|
||||
if: ${{ inputs.downstream && !inputs.dry_run }}
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v7
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- uses: actions/cache/restore@v6
|
||||
|
||||
@@ -168,6 +168,9 @@ jobs:
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
- name: Run Rust unit tests (redb experimental cursor)
|
||||
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
||||
|
||||
rust-unit-tests-windows:
|
||||
name: Rust Unit Tests (Windows)
|
||||
runs-on: windows-latest
|
||||
@@ -205,6 +208,9 @@ jobs:
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
- name: Run Rust unit tests (redb experimental cursor)
|
||||
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ['3.10', '3.11', '3.12']
|
||||
python-version: ['3.9', '3.11', '3.12']
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
@@ -410,7 +410,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
@@ -581,7 +581,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
@@ -810,7 +810,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
|
||||
@@ -67,8 +67,6 @@ Download the latest binary from the [releases](https://github.com/seaweedfs/seaw
|
||||
curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
```
|
||||
|
||||
For SeaweedFS Enterprise binaries, pass `--enterprise` to the script: `... | bash -s -- --enterprise`.
|
||||
|
||||
Then start a ready-to-use S3 object store:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -5,7 +5,7 @@ go 1.26.6
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.51.1
|
||||
cloud.google.com/go/storage v1.69.0
|
||||
cloud.google.com/go/storage v1.67.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -84,24 +84,24 @@ require (
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.7.3
|
||||
go.etcd.io/etcd/client/v3 v3.7.3
|
||||
go.etcd.io/etcd/api/v3 v3.7.1
|
||||
go.etcd.io/etcd/client/v3 v3.7.1
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.49.0
|
||||
gocloud.dev/pubsub/natspubsub v0.49.0
|
||||
gocloud.dev v0.47.0
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.47.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297
|
||||
golang.org/x/image v0.46.0
|
||||
golang.org/x/net v0.62.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
golang.org/x/tools v0.52.0 // indirect
|
||||
golang.org/x/tools v0.51.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.310.0
|
||||
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b // indirect
|
||||
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941 // indirect
|
||||
google.golang.org/grpc v1.86.0-dev.0.20261027044109-d619b5d02089
|
||||
google.golang.org/protobuf v1.36.12
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
@@ -123,10 +123,10 @@ require (
|
||||
github.com/apache/iceberg-go v0.7.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.7
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/cognusion/imaging v1.0.4
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
@@ -145,7 +145,7 @@ require (
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.32.0
|
||||
github.com/pkg/sftp v1.13.12
|
||||
github.com/rabbitmq/amqp091-go v1.16.0
|
||||
github.com/rabbitmq/amqp091-go v1.15.0
|
||||
github.com/rclone/rclone v1.75.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.23.0
|
||||
@@ -162,7 +162,6 @@ require (
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.155.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3
|
||||
go.uber.org/atomic v1.12.0
|
||||
go.yaml.in/yaml/v3 v3.0.6
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
@@ -185,7 +184,7 @@ require (
|
||||
github.com/anchore/go-lzo v0.1.1 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/thrift v0.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -248,7 +247,7 @@ require (
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/mschoch/smat v0.2.0 // indirect
|
||||
github.com/oklog/ulid/v2 v2.1.2 // indirect
|
||||
github.com/oklog/ulid/v2 v2.1.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
@@ -277,35 +276,37 @@ require (
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/log v1.47.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.1 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.6 // indirect
|
||||
golang.org/x/mod v0.42.0 // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.3 // indirect
|
||||
cloud.google.com/go/auth v0.24.1 // indirect
|
||||
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.10.0 // indirect
|
||||
cloud.google.com/go/iam v1.14.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.31.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.30.0 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.24.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
|
||||
@@ -319,22 +320,22 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6
|
||||
github.com/aws/smithy-go v1.28.9
|
||||
github.com/aws/smithy-go v1.28.8
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
@@ -369,14 +370,14 @@ require (
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.9 // indirect
|
||||
github.com/go-openapi/strfmt v0.27.2 // indirect
|
||||
github.com/go-openapi/errors v0.22.8 // indirect
|
||||
github.com/go-openapi/strfmt v0.27.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.3 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/goccy/go-json v0.11.2 // indirect
|
||||
github.com/goccy/go-json v0.10.6 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
@@ -386,7 +387,7 @@ require (
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
@@ -418,8 +419,8 @@ require (
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nats.go v1.49.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.15 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
@@ -433,7 +434,7 @@ require (
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.33
|
||||
github.com/pierrec/lz4/v4 v4.1.29
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
@@ -455,7 +456,7 @@ require (
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.2 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
@@ -480,9 +481,9 @@ require (
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.5.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
|
||||
go.opentelemetry.io/otel v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.47.0 // indirect
|
||||
@@ -492,7 +493,7 @@ require (
|
||||
go.uber.org/zap v1.28.0 // indirect
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/time v0.16.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
|
||||
@@ -4,8 +4,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
|
||||
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
|
||||
cloud.google.com/go/auth v0.24.1 h1:NTmd2iJb4DsGIav0KiqFMYI88nMshA2wnFMBatRmoCs=
|
||||
cloud.google.com/go/auth v0.24.1/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
|
||||
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a h1:jW1ZLFy9oa5ai8LOI40z6s+DL1cO/L1KSm9hVGnibn8=
|
||||
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.3.0 h1:FY8oSZpCYoUNv6QxVODuMjQz4IlSOVeiQtZ08vLPz88=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.3.0/go.mod h1:7+2uCm7++XFO+/lN06c2HXpDXb/NMNn2/UwyBPbTnkk=
|
||||
cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY=
|
||||
@@ -18,14 +18,14 @@ cloud.google.com/go/logging v1.20.0 h1:l7yzCB8/JSY5PHT9RsaqdT1jaBGbM6KlSJDX210Hl
|
||||
cloud.google.com/go/logging v1.20.0/go.mod h1:ALUaJQZM691Eeujy9kFx0xREq3AMPuGd+pTRZuTYv+w=
|
||||
cloud.google.com/go/longrunning v1.3.0 h1:lZ6c8SJJBmILp/Wnr3vGnBLa0q+cYfSHRCcUfVI2hpI=
|
||||
cloud.google.com/go/longrunning v1.3.0/go.mod h1:wpgLxCn+UA3/TZ1bj/fW39mfDpwVlMPu1xKJyoMt7V0=
|
||||
cloud.google.com/go/monitoring v1.31.0 h1:LCuAoYiq8snyL/eDAG/VC73uWFXeax8+YEfmoNGJUrY=
|
||||
cloud.google.com/go/monitoring v1.31.0/go.mod h1:7uk2dIJIf7ra64dI1VuQ0ttoFqvtKjdzq46u6zNP1hU=
|
||||
cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY=
|
||||
cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM=
|
||||
cloud.google.com/go/pubsub v1.51.1 h1:R3G1wCOxBO7jRpL8x2pdZMv1GAJDF6ax/m2zPOtvTNE=
|
||||
cloud.google.com/go/pubsub v1.51.1/go.mod h1:y2T0IKtW1iWwVvazYaRpqOAFO4gy2+O7dTDt9TWY/5U=
|
||||
cloud.google.com/go/pubsub/v2 v2.7.0 h1:MFrBTZZa6PDWZzCi4NJRsHKMm2w0a4oAaYNqwjgbQTE=
|
||||
cloud.google.com/go/pubsub/v2 v2.7.0/go.mod h1:JaFvWNVRk3Knoil/4M1ECeLOaI9D8drbmJWypQlK5aM=
|
||||
cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM=
|
||||
cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g=
|
||||
cloud.google.com/go/storage v1.67.0 h1:8xD3NvKuHE4c+b/kmSIVBdTc0BiOjplx+KecKGR9fr8=
|
||||
cloud.google.com/go/storage v1.67.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM=
|
||||
cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E=
|
||||
cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0=
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
@@ -48,8 +48,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3 h1:zoPQ7KXGgzqeUmNWl+b6IwVAqsb5lROEJhqAmqXQtm0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3/go.mod h1:ei/sDx/dwDIUtZE3PQP1u8+qrA+CRy4q7qW+A2MbgB4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 h1:cuiKf1UVyWHu+XSQghPZR/qEF43JIcuk2CDqMlPiT6M=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0/go.mod h1:9JSyvgXLPAOC4jfhgZg58XeU2FJHsGmxuYSclCFQ4ZY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4=
|
||||
@@ -60,8 +60,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 h1:0O6j18nQDoIff6/mtosT5LNU2WehEwp+LqfX6699oqY=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1/go.mod h1:xdYAf5bjkOpsd3auA8riiJW4vneBubt9caavL7626jQ=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 h1:MDT4FxAPve5FnYn6vOL1r7RCRDG+l9cI7a5LlCuHsqA=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
|
||||
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d/go.mod h1:RVnhzAX71far8Kc3TQeA0k/dcaEKUnTDSOyet/JCmGI=
|
||||
@@ -75,12 +75,14 @@ github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjw
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194 h1:dtOFxSTWWRpkmvXa6ycNiw8dVDu1wkgzcXyVV1VafNc=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194/go.mod h1:rl0WumSN9gSo775DgvQv+wMQ8rlb0ES/1hU5jkMtLXg=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 h1:5y2LR2pripbccOte6BpZ+guv5O5W0OQIfKvUYZrGaAQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0/go.mod h1:b3f7EifFRqkVt142USoGIP62uUeZm7ALCPrs1ZjWXYo=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0 h1:tRsXhJnn0w0TNwJpVrB94wsrimzjWKFzdQlUhRdNUzc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 h1:aPMa5qZU9dp4fU34o+jQgVIPFIOi20oxYg8TLigmZLg=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0 h1:rmzbFS2CkHq0y2L5LkMVWm+QXjjNEXxjmhrJqga/QGk=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0/go.mod h1:iIyrdhveU5Ow84ipRlxtjCOQnIM0DiqSMxuxm50xwZY=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 h1:BeC1Bub7Ttk33TF8v3ZeQ2L8acvz4ZLqtQuLyJAZquc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0/go.mod h1:UgG/Xn9+NXb1/ueSczIFxyuya3hs3srFDiZG2JrpJwY=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.61.0 h1:eSgSuKdcpY7RTN1oXSzC6S39rPBImn03ubLMsWK4RRg=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.61.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 h1:KhpL26/GviYITHMCEGGqT/l50siFaBCBBKrjv6Z0G/k=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1 h1:fxLusCG+GlGY1SM7BYAAiiSFhzKUs5zqs0P8knDqeFU=
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1/go.mod h1:yO+OQpByKDLTvpEcsFFexgzpeR8eRfCFWAYzxkAu4bk=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
|
||||
@@ -171,50 +173,50 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
|
||||
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0 h1:tFU7EJVm7MfyaXLsXdYTZ0Vd6vwt9OljCyqq0gqk0Mc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0/go.mod h1:oeKU5p+TQBnpeLmU42WZLWvzv+ORLn+74n4QVLnm/Z0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 h1:3dMP9eEmHVosFN1QiD24IJBKVY8m6iUfVYXqRCT118Q=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27/go.mod h1:KH4PKcqtjy1m0656eqQV2v53xE04ddJzuNloEKqlHkI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.7 h1:jBmtzRtGWVAzbfc4RBWo1dTte0ZZH8p9U42YJIw2+Bw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.7/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16 h1:q9YuNkYR/7lxflWCcEv3C0cecUt3Zvrh2hn9ZNxieNA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16/go.mod h1:3BSeJQF55QQI+dDFHx4WTIjTvX/Lvtk8Jaaf/SGAnBM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 h1:W0yUUzKBA8SofGQ0yZafEUuZq9UxfvijRgdc6vcRQPA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12/go.mod h1:xwKv6vwBNdpPFOxEm/ld4IODZK2s/1ksHvdBqLfH6Gs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 h1:lqSi8qvAnKC1vfosImdmOsuiJLr/jdybZX5299beB6Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12/go.mod h1:rqPQE2vqkW9YHx5+Gp55sx1wsTM4xtGQLl+TKMjxxrs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 h1:uvX2yaL+aMjkmnrLPPlR/zwjoKFRMoNhhJfeGjfk6qc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12/go.mod h1:dicT+2eNhHHqGkxHO7LoXw3QghIUGuzgdmFqaSTopgI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 h1:rKwkY3Z+fvt2W0msjdNs6uNewPmXzPayQ961oyuWhes=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25/go.mod h1:dBuAadQKd1F4Q4ZOg8gnFmuSHMRE40Pe3IfFwbtC4lE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 h1:U5HZ8FpTr6tyvulf1nT11Nw4KG1CAsXkuQGdUrIjZ/E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13/go.mod h1:dUfvCU1NaE+b3vqNENDiJncUQW7Djl5pmgUlcjesVqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 h1:wzAqMZ2O4+ij9HQqIrk+P9PBGacE0wOTuWB7Mo97//I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12/go.mod h1:/0TfyruByHsOlphzP26xDTRe2DKD0J1I1Uyy2TyP8EI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 h1:H0IaQayT3TLiGIHiNnleQZTbndDlx7LGZ0NWZ5mDwjA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12/go.mod h1:YuJJNxaOjnknLIrscBmkLhIpnmg9LzHd25sQRKUboGU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10 h1:WBJb2VxRzzKnuN94YWqZPiu5a5zc3IleFqgZnmVCyvU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10/go.mod h1:8NhAIBVYkVHq0hIbbzujdoou0VkKwMdG+De9e5oX6qI=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5 h1:4EezYQsIY0xPbD6u7TbQLfnpicSMQgA8n9paomVV4Zo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5/go.mod h1:Yr4VqXnyoyjWnOWJck8BOidkJtleQ1c8ypgd2n0epxE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 h1:D+Zmdt3T3P770MrqTmZvalbj6ADO7xA7n+qnNakGvrI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4/go.mod h1:dUYH3ia3qEO5ogaWDLp2tjtb4u4Y9xRbae7tOr1ClIA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 h1:9T6eO8/WXhij4PQaa2WaYsdekSCmT9Hw66Pw0S8do0Q=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0 h1:fD02zeo2Kc4hlFUvGX3FfA+gGOwd4BBcYgPDHDEzx9Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0/go.mod h1:/lB2ILjoT5/q0zSwtHnrJPjXBvjalnEIl0IgSEdvHec=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0 h1:FghdY9ebeZf9iIan9J0xfBEID8arYibNBeZ1XHvs4oA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0/go.mod h1:wu7uOJ/2nAdenjvhHikeF6j0al99kfPVG6ZDU4e76OY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
|
||||
github.com/aws/smithy-go v1.28.9 h1:zkZ058OpNX0iGTK24b+ZUEQSMc11CBA6oO7xtKZfzkY=
|
||||
github.com/aws/smithy-go v1.28.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ=
|
||||
github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -507,12 +509,12 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-openapi/errors v0.22.9 h1:HI9+SyVYiRzyeBQGv0CbWbQa+u2S0nny7G7AhY+UMiM=
|
||||
github.com/go-openapi/errors v0.22.9/go.mod h1:R73q66sXP2smzAGgFOI4nqtusRK2xN5pYCjsij+KX58=
|
||||
github.com/go-openapi/strfmt v0.27.2 h1:SG32SlbwNy92s0KJiVxt2joJeFdqIYHvwrA0OU6HqzQ=
|
||||
github.com/go-openapi/strfmt v0.27.2/go.mod h1:M4CKsMO0Fb8qR10+1Ra75wCKNNquy+Vj+4LWZrhTo2E=
|
||||
github.com/go-openapi/testify/v2 v2.8.0 h1:19QDx5b57p8KjO2E4sgs/woL4Akp+FGLRlVm2xn0Rho=
|
||||
github.com/go-openapi/testify/v2 v2.8.0/go.mod h1:4gUN8jC+RqOE2qjW/vDmfyTvl5BCht4qL7oca0odrak=
|
||||
github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I=
|
||||
github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w=
|
||||
github.com/go-openapi/strfmt v0.27.0 h1:kbcTeaD9TXuXD0hhMXzuYa1sdTo6+dWGvwjW93E80IM=
|
||||
github.com/go-openapi/strfmt v0.27.0/go.mod h1:s/qhDqfY72irigXUGJmtgid2Rm+3tnz3k8hZaRmvWYc=
|
||||
github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
|
||||
github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
@@ -544,8 +546,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
|
||||
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
|
||||
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
github.com/goccy/go-json v0.11.2 h1:jdZv93Tt4ioR8yW1CoNsvSxrcZlCXAUU1aZXN7gpXUA=
|
||||
github.com/goccy/go-json v0.11.2/go.mod h1:3NdmfEkZlB7YI5UFw/qdFKq8XN1aiWR0YyRPWZNQltY=
|
||||
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
|
||||
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
|
||||
@@ -561,8 +563,8 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFig
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=
|
||||
@@ -647,8 +649,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
@@ -913,10 +915,10 @@ github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.15 h1:StSf9TINInaZtr4oww2+kXmfwa9SkN//g/LwS19/UJ0=
|
||||
github.com/nats-io/nats-server/v2 v2.11.15/go.mod h1:zwhv8Y0PE3KHyKgznJc/9Xoai638SaJd83zzJ5GJn74=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
|
||||
github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
|
||||
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
|
||||
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
@@ -927,8 +929,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
|
||||
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec=
|
||||
github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
|
||||
github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
|
||||
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@@ -977,8 +979,8 @@ github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7c
|
||||
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014=
|
||||
github.com/pierrec/lz4/v4 v4.1.33/go.mod h1:7SE9MC2STkNtL4PIwGhjmyVwvILaGI9/COYQNBhKM/c=
|
||||
github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg=
|
||||
github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc=
|
||||
github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU=
|
||||
github.com/pierrre/geohash v1.0.0 h1:f/zfjdV4rVofTCz1FhP07T+EMQAvcMM2ioGZVt+zqjI=
|
||||
@@ -1049,8 +1051,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y=
|
||||
github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rabbitmq/amqp091-go v1.15.0 h1:LEQL4/yp48/Wigt6A6XOu18RQRo8ZHtB5I/KZJn+gkw=
|
||||
github.com/rabbitmq/amqp091-go v1.15.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.5 h1:K1++Qtk3PvgkiCCiv6Pahju1TMOzKY6VSwiwT7XLAVc=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.5/go.mod h1:vCeOPhlXzevN0AFojgh1zsjhetiShy/ArvJ/xkFUDWk=
|
||||
github.com/rclone/go-proton-api v1.0.4 h1:AJW0e9pB4j0hVK4WqyGErFwaI+5MUQWPCtj5FYYxtPg=
|
||||
@@ -1153,8 +1155,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0 h1:g5qTg3qIP1HXV+GzuyEfSJFVxUCpEP5Ui1OINVJHPB4=
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.2 h1:jUEsvCMD6fH25J8K/w3q/XnIx8W1lb8+YLaEEHIjHmc=
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.2/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
@@ -1349,36 +1351,36 @@ go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
|
||||
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
|
||||
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
|
||||
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
|
||||
go.etcd.io/etcd/api/v3 v3.7.3 h1:2tumepXe+wi7Wab64uUO71K7ISQyUWLGC1DJbbc59sY=
|
||||
go.etcd.io/etcd/api/v3 v3.7.3/go.mod h1:HOR9Qn4gb/1/hVAmYprmlLpvTe6ZgfgNS9Z/+Ty65po=
|
||||
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
|
||||
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3 h1:5zMwNu3ycrzCeA5NwOfQbxFRhqq974chq1DrYMua4QI=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3/go.mod h1:/OhzqYrF0U7d1NOk55VMyUQGbYD1WoIe4bJwxb47wOk=
|
||||
go.etcd.io/etcd/client/v3 v3.7.3 h1:UrXcetgkuSEocaBlQlulQRJlXLEnUAHuBRbHh0lue1Q=
|
||||
go.etcd.io/etcd/client/v3 v3.7.3/go.mod h1:DcPxpqcpV8WIaRi/2waPCx0/Uq8rQUweDJaDYZk1gEE=
|
||||
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
|
||||
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 h1:Ct6+aaiSO6Nk5SOb6PYcR7iWp+83YF/pL1OJcGqOmbg=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.47.0/go.mod h1:3N259ss0b1AVOs+fYw7lOO0mcUwNqSIpQ+Zitpuqx00=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.46.0/go.mod h1:nsrN5c/sOLoY2vsPxN/rQ0V0nvGrWJCqcW4UXLtqNG8=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0 h1:MCcYL7J6Vt/X0kjqbMZkekCmwsurbQRbL69vkiye2lk=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0/go.mod h1:3jnStNwSufK+f5ktjL4EPcwtig4rtd81NS70lqHuXl8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
|
||||
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
|
||||
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.47.0 h1:LaLVGJtIzHcvHjVs2OftmKcc+M9mdOqS8vZ1fBD1Yes=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.47.0/go.mod h1:vF6H5sNDuQBJWgSX00iZgbIqk2MZZoVMH0ar5t5bOpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 h1:RuynHbfU8JUEw7DyONgkVYg2SVtsoF28y0LGIr69jgA=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0/go.mod h1:qZF+/lBs71APw8mlnEZcqZHMzqrYrsFiJOv83lX1OGo=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=
|
||||
@@ -1398,8 +1400,8 @@ go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syod
|
||||
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
|
||||
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v1.11.1 h1:SCrPqH9NE5CmBrOqpjNCiT0X5mGwMyGERXdogfR7Yjw=
|
||||
go.opentelemetry.io/proto/otlp v1.11.1/go.mod h1:wq8W4aSf6bOy8RZyESOeQFr9Pu7wydWX/TY4sgvrTPg=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
@@ -1425,10 +1427,10 @@ go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
|
||||
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
gocloud.dev v0.49.0 h1:02y4IBHX12xiCBK3Nv80IzuCmZXLsOfgpKwMEKzR9O4=
|
||||
gocloud.dev v0.49.0/go.mod h1:uN+hz2x8ieyW5RjeMetvmpV/nRqFOr7DF0CKVY0Owp8=
|
||||
gocloud.dev/pubsub/natspubsub v0.49.0 h1:Jk1bywJ6LvqvBbxOQ0qs0q4aa+ORgg1FzjhTy+5O4cI=
|
||||
gocloud.dev/pubsub/natspubsub v0.49.0/go.mod h1:bmtxptoOPBNAgvTW9tWzoZvw9vzunDoNk6l4cHGd6nc=
|
||||
gocloud.dev v0.47.0 h1:qN4/0vtutn97f1X5ziisf3WXuaFL9AR0FIRnyJRYzgA=
|
||||
gocloud.dev v0.47.0/go.mod h1:OdMnyXQRhM17U0a0DwU57pe0c01qN3lJNmaeeRYW0D0=
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0 h1:cTmJnWOmhb3dYRzX7PvNkq2m5mQdBLFFxZbl/21pfjk=
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0/go.mod h1:hbEEsNt6mGQoonGQi39VOjHfx2f13DOmWouh4hGsfrU=
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.47.0 h1:iDSspbLlDbPL2avESbDfMNBK4r3B6H04Ejx+/hqh9JE=
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.47.0/go.mod h1:2tSZnGnGLPw2ayW4pVaUmnA1LCjP7rekpXPkaBUZt0o=
|
||||
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
|
||||
@@ -1449,8 +1451,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e h1:HgI6W1OolKK6tcV74is+rl9SMT0njsHlp0TbQiGXI/A=
|
||||
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e/go.mod h1:DBoxwWhjaW8DJJw/YeM4zpeOKMC1FMXSblsYy+1mFSw=
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY=
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk=
|
||||
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
|
||||
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -1606,8 +1608,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
|
||||
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
|
||||
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
|
||||
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -1629,10 +1631,10 @@ google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfG
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b h1:+t2qGqt6R8CO622uiX1JpnRvRvSOQoVRV0DiPzUjKXg=
|
||||
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b/go.mod h1:XRIfc1yr5xsbF8xFw5+g7SEmLJX/eQ7MMazTRbnMMvI=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b h1:nB3spRIaSPtxJpxb0k41yljROvm2Jrwfs2pSNdKS5y8=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b/go.mod h1:9N4vtjEwvgQKEdlFVQO0M44ErhecQ9DCSQfGoZ51M6w=
|
||||
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941 h1:CXq2XEi5aEyLjjPZIwqrmQH8ZY4ctUwK7uyAx3mjX/U=
|
||||
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941/go.mod h1:3gK0ehnhP0V7/x3ypGoLyT3d0n6TSi/EcsphIjHrcz4=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 h1:DS/w3N8rsfrwxwgGSj6WhMsPo7aNQGmDx74J81zd0NQ=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941/go.mod h1:AuGAdi9KnXss7xbLxIKhCvk+Ro/SxpET/iS45DNYN+M=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
|
||||
+31
-80
@@ -7,14 +7,12 @@
|
||||
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
|
||||
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
|
||||
# curl -fsSL ... | bash -s -- --enterprise --component all
|
||||
#
|
||||
# Options:
|
||||
# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed)
|
||||
# --version VER Release version tag (default: latest)
|
||||
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
|
||||
# --dir DIR Installation directory (default: /usr/local/bin)
|
||||
# --enterprise Install SeaweedFS Enterprise binaries (from seaweedfs/artifactory)
|
||||
# --help Show this help message
|
||||
|
||||
set -euo pipefail
|
||||
@@ -26,10 +24,6 @@ LARGE_DISK=false
|
||||
INSTALL_DIR="/usr/local/bin"
|
||||
WORKER_INSTALLED=false
|
||||
VECTOR_INSTALLED=false
|
||||
VOLUME_INSTALLED=false
|
||||
ENTERPRISE=false
|
||||
# Enterprise assets carry this after the binary name, e.g. weed-vector-enterprise_linux_amd64.tar.gz.
|
||||
EDITION=""
|
||||
|
||||
# Colors (if terminal supports them)
|
||||
if [ -t 1 ]; then
|
||||
@@ -59,7 +53,6 @@ while [ $# -gt 0 ]; do
|
||||
--version) VERSION="$2"; shift 2 ;;
|
||||
--large-disk) LARGE_DISK=true; shift ;;
|
||||
--dir) INSTALL_DIR="$2"; shift 2 ;;
|
||||
--enterprise) ENTERPRISE=true; REPO="seaweedfs/artifactory"; EDITION="-enterprise"; shift ;;
|
||||
--help|-h) usage ;;
|
||||
*) error "Unknown option: $1. Use --help for usage." ;;
|
||||
esac
|
||||
@@ -88,22 +81,15 @@ detect_platform() {
|
||||
}
|
||||
|
||||
# Get latest release tag from GitHub API
|
||||
# The Enterprise repo also releases other products (vfs-*, previews), which can
|
||||
# be its latest release, so take the newest plain version tag like 4.53.
|
||||
get_latest_version() {
|
||||
local url="https://api.github.com/repos/${REPO}/releases/latest"
|
||||
if [ "$ENTERPRISE" = true ]; then
|
||||
url="https://api.github.com/repos/${REPO}/releases?per_page=30"
|
||||
fi
|
||||
local body
|
||||
if command -v curl &>/dev/null; then
|
||||
body="$(curl -fsSL "$url")"
|
||||
curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
elif command -v wget &>/dev/null; then
|
||||
body="$(wget -qO- "$url")"
|
||||
wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
else
|
||||
error "Neither curl nor wget found. Please install one."
|
||||
fi
|
||||
echo "$body" | grep '"tag_name"' | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' | grep -E '^[0-9]+(\.[0-9]+)+$' | head -1 || true
|
||||
}
|
||||
|
||||
# Download a file
|
||||
@@ -124,22 +110,17 @@ go_asset_name() {
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
suffix="${suffix}_large_disk"
|
||||
fi
|
||||
if [ "$ENTERPRISE" = true ]; then
|
||||
suffix="weed-enterprise-${suffix}"
|
||||
fi
|
||||
if [ "$os" = "windows" ]; then
|
||||
echo "${suffix}.zip"
|
||||
else
|
||||
echo "${suffix}.tar.gz"
|
||||
fi
|
||||
echo "${suffix}.tar.gz"
|
||||
}
|
||||
|
||||
# Build Rust volume server asset name
|
||||
rust_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
local prefix="weed-volume${EDITION}"
|
||||
local prefix="weed-volume"
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
prefix="${prefix}_large_disk"
|
||||
prefix="weed-volume_large_disk"
|
||||
else
|
||||
prefix="weed-volume"
|
||||
fi
|
||||
local suffix="${os}_${arch}"
|
||||
if [ "$os" = "windows" ]; then
|
||||
@@ -149,19 +130,6 @@ rust_asset_name() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Enterprise publishes the Rust volume server for linux only.
|
||||
rust_volume_platform_supported() {
|
||||
case "$2" in
|
||||
amd64|arm64) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
case "$1" in
|
||||
linux) return 0 ;;
|
||||
darwin|windows) [ "$ENTERPRISE" != true ] ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Does a release carry this asset? Answers 0 for yes and 1 for a 404, and stops
|
||||
# the installer on anything else: a rate limit or a network blip must not read
|
||||
# as "this release predates the binary" and quietly skip it.
|
||||
@@ -186,23 +154,19 @@ asset_exists() {
|
||||
# maintains tables through the namespace and never opens a volume file.
|
||||
worker_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
echo "weed-worker${EDITION}_${os}_${arch}.tar.gz"
|
||||
echo "weed-worker_${os}_${arch}.tar.gz"
|
||||
}
|
||||
|
||||
# The vector engine behind the S3 Vectors API, built like the worker.
|
||||
# The vector engine behind the S3 Vectors API, published for linux only like
|
||||
# the worker.
|
||||
vector_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
if [ "$os" = "windows" ]; then
|
||||
echo "weed-vector${EDITION}_${os}_${arch}.zip"
|
||||
else
|
||||
echo "weed-vector${EDITION}_${os}_${arch}.tar.gz"
|
||||
fi
|
||||
echo "weed-vector_${os}_${arch}.tar.gz"
|
||||
}
|
||||
|
||||
vector_platform_supported() {
|
||||
case "$1/$2" in
|
||||
linux/amd64|linux/arm64) return 0 ;;
|
||||
darwin/amd64|darwin/arm64|windows/amd64) [ "$ENTERPRISE" != true ] ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
@@ -222,10 +186,7 @@ install_component() {
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
case "$asset_name" in
|
||||
*.zip) unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir" ;;
|
||||
*) tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" ;;
|
||||
esac
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
|
||||
# The Go release action puts the binary inside a directory
|
||||
local weed_bin
|
||||
@@ -238,16 +199,20 @@ install_component() {
|
||||
fi
|
||||
|
||||
chmod +x "$weed_bin"
|
||||
local dest_name="weed"
|
||||
if [ "$os" = "windows" ]; then
|
||||
dest_name="weed.exe"
|
||||
fi
|
||||
install_binary "$weed_bin" "$dest_name"
|
||||
ok "Installed weed to ${INSTALL_DIR}/${dest_name}"
|
||||
install_binary "$weed_bin" "weed"
|
||||
ok "Installed weed to ${INSTALL_DIR}/weed"
|
||||
;;
|
||||
|
||||
volume-rust)
|
||||
rust_volume_platform_supported "$os" "$arch" || error "The Rust volume server is not available for ${os}/${arch}"
|
||||
# Check platform support for Rust volume server
|
||||
case "$os" in
|
||||
linux|darwin|windows) ;;
|
||||
*) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;;
|
||||
esac
|
||||
case "$arch" in
|
||||
amd64|arm64) ;;
|
||||
*) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;;
|
||||
esac
|
||||
|
||||
asset_name="$(rust_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
@@ -279,7 +244,6 @@ install_component() {
|
||||
dest_name="weed-volume.exe"
|
||||
fi
|
||||
install_binary "$rust_bin" "$dest_name"
|
||||
VOLUME_INSTALLED=true
|
||||
ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}"
|
||||
;;
|
||||
|
||||
@@ -316,26 +280,19 @@ install_component() {
|
||||
;;
|
||||
|
||||
vector)
|
||||
vector_platform_supported "$os" "$arch" || error "The vector engine is not available for ${os}/${arch}"
|
||||
vector_platform_supported "$os" "$arch" || error "The vector engine is not available for ${os}/${arch}. Supported: linux/amd64, linux/arm64"
|
||||
|
||||
asset_name="$(vector_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
local vector_name="weed-vector"
|
||||
if [ "$os" = "windows" ]; then
|
||||
command -v unzip >/dev/null || error "unzip is required for Windows vector archives"
|
||||
unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir"
|
||||
vector_name="weed-vector.exe"
|
||||
else
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
fi
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
local vector_bin
|
||||
vector_bin="$(find "$tmpdir" -name "$vector_name" -type f | head -1)"
|
||||
[ -n "$vector_bin" ] || error "Could not find $vector_name binary in archive"
|
||||
vector_bin="$(find "$tmpdir" -name weed-vector -type f | head -1)"
|
||||
[ -n "$vector_bin" ] || error "Could not find weed-vector binary in archive"
|
||||
chmod +x "$vector_bin"
|
||||
install_binary "$vector_bin" "$vector_name"
|
||||
install_binary "$vector_bin" weed-vector
|
||||
VECTOR_INSTALLED=true
|
||||
ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector"
|
||||
;;
|
||||
@@ -392,11 +349,7 @@ main() {
|
||||
case "$COMPONENT" in
|
||||
all)
|
||||
install_component "weed" "$os" "$arch"
|
||||
if rust_volume_platform_supported "$os" "$arch"; then
|
||||
install_component "volume-rust" "$os" "$arch"
|
||||
else
|
||||
warn "Skipping the Rust volume server: no build for ${os}/${arch}"
|
||||
fi
|
||||
install_component "volume-rust" "$os" "$arch"
|
||||
# The worker is published for linux amd64/arm64 only, and only by
|
||||
# releases new enough to carry it; skip either case rather than fail
|
||||
# an install that has already put two binaries in place.
|
||||
@@ -425,7 +378,7 @@ main() {
|
||||
if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then
|
||||
info " weed: ${INSTALL_DIR}/weed"
|
||||
fi
|
||||
if [ "$VOLUME_INSTALLED" = true ]; then
|
||||
if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then
|
||||
info " weed-volume: ${INSTALL_DIR}/weed-volume"
|
||||
fi
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
@@ -438,9 +391,7 @@ main() {
|
||||
info "Quick start:"
|
||||
info " weed master # Start master server"
|
||||
info " weed volume -mserver=localhost:9333 # Start Go volume server"
|
||||
if [ "$VOLUME_INSTALLED" = true ]; then
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
fi
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker"
|
||||
fi
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.54"
|
||||
appVersion: "4.53"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.54.0
|
||||
version: 4.53.1
|
||||
@@ -718,7 +718,7 @@ The bundled engine images support Linux amd64 and arm64. Use `vector.imageOverri
|
||||
|
||||
`global.seaweedfs.enableSecurity` provisions and mounts the vector certificate and shared security configuration. With external certificates, provide the `<fullname>-vector-cert` secret (`my-seaweedfs-vector-cert` for release `my`). `fullname` follows `seaweedfs.fullname`, including `fullnameOverride`, `nameOverride`, and DNS-name truncation. Provide this secret alongside the existing CA and client secrets.
|
||||
|
||||
The default `vector.metricsPort: 0` keeps TCP probes. Set `vector.metricsPort: 9329` to serve `/metrics`, `/health`, and `/ready` from the engine; readiness then checks master registration. Images older than 4.54 do not accept the metrics flag, so leave metrics off when pinning `image.tag` or `vector.imageOverride` to one. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
|
||||
The default `vector.metricsPort: 0` and TCP probes support existing engine images. The default 4.53 image does not accept metrics flags; enabling metrics with it is rejected at render time. For engine builds with `/metrics`, `/health`, and `/ready`, set `vector.imageOverride` to that compatible image and `vector.metricsPort: 9329`; readiness then checks master registration. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
|
||||
|
||||
|
||||
To provision vector buckets during installation and upgrades, set:
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
{{- $s3PreStopSeconds := 0 }}
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }}
|
||||
{{- if kindIs "invalid" $s3PreStopSeconds }}
|
||||
{{- $s3PreStopSeconds = .Values.s3.preStopSeconds }}
|
||||
{{- end }}
|
||||
{{- $s3PreStopSeconds = $s3PreStopSeconds | int }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -72,7 +64,7 @@ spec:
|
||||
{{- tpl .Values.allInOne.tolerations . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
|
||||
terminationGracePeriodSeconds: 60
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.allInOne.priorityClassName }}
|
||||
priorityClassName: {{ .Values.allInOne.priorityClassName | quote }}
|
||||
@@ -268,9 +260,6 @@ spec:
|
||||
{{- if $auditLogConfig }}
|
||||
-s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $s3PreStopSeconds 0 }}
|
||||
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sftp.enabled }}
|
||||
-sftp \
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.filer.enabled }}
|
||||
{{- $s3PreStopSeconds := 0 }}
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
{{- $s3PreStopSeconds = .Values.filer.s3.preStopSeconds | int }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
@@ -73,7 +69,7 @@ spec:
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
serviceAccountName: {{ .Values.filer.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
|
||||
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
|
||||
terminationGracePeriodSeconds: 60
|
||||
{{- if .Values.filer.priorityClassName }}
|
||||
priorityClassName: {{ .Values.filer.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
@@ -282,9 +278,6 @@ spec:
|
||||
{{- if .Values.filer.s3.auditLogConfig }}
|
||||
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $s3PreStopSeconds 0 }}
|
||||
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
-master={{ include "seaweedfs.masterServerArg" . }} \
|
||||
{{- range .Values.filer.extraArgs }}
|
||||
|
||||
@@ -1,10 +1,5 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.s3.enabled }}
|
||||
{{- $preStopSeconds := .Values.s3.preStopSeconds | int }}
|
||||
{{- $minGracePeriod := add $preStopSeconds 15 }}
|
||||
{{- if and (gt $preStopSeconds 0) (lt (.Values.s3.terminationGracePeriodSeconds | int) $minGracePeriod) }}
|
||||
{{- fail (printf "s3.terminationGracePeriodSeconds (%v) must be at least %d: the gateway keeps serving for s3.preStopSeconds (%d) after SIGTERM, then gives in-flight requests up to 15s to finish; add the time any s3.lifecycle.preStop hook takes" .Values.s3.terminationGracePeriodSeconds $minGracePeriod $preStopSeconds) -}}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -159,9 +154,6 @@ spec:
|
||||
{{- if .Values.s3.auditLogConfig }}
|
||||
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $preStopSeconds 0 }}
|
||||
-preStopSeconds={{ $preStopSeconds }} \
|
||||
{{- end }}
|
||||
-filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ include "seaweedfs.namespace" . }}:{{ .Values.filer.port }} \
|
||||
{{- if .Values.s3.icebergPort }}
|
||||
-port.iceberg={{ .Values.s3.icebergPort }} \
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.vector.enabled }}
|
||||
{{- if and .Values.vector.metricsPort (not .Values.vector.imageOverride) (eq (toString (.Values.image.tag | default .Chart.AppVersion)) "4.53") }}
|
||||
{{- fail "vector.metricsPort requires vector.imageOverride selecting an engine with metrics support; the default 4.53 image does not support metrics" }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
|
||||
@@ -136,11 +136,13 @@ func TestVectorNetworkPolicyOverrides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorMetricsDefaultImage(t *testing.T) {
|
||||
output, _ := render(t, "vector.enabled=true", "vector.metricsPort=9329")
|
||||
for _, want := range []string{"--metrics-port=9329", "path: /ready", "path: /health"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("default image missing %q", want)
|
||||
}
|
||||
func TestVectorMetricsImageCompatibility(t *testing.T) {
|
||||
if _, err := exec.LookPath("helm"); err != nil {
|
||||
t.Skip("helm is not installed")
|
||||
}
|
||||
out, err := exec.Command("helm", "template", "vector-test", "..", "--set", "vector.enabled=true", "--set", "vector.metricsPort=9329").CombinedOutput()
|
||||
if err == nil || !strings.Contains(string(out), "default 4.53 image does not support metrics") {
|
||||
t.Fatalf("incompatible image accepted: %v %s", err, out)
|
||||
}
|
||||
render(t, "vector.enabled=true", "vector.metricsPort=9329", "vector.imageOverride=example/metrics-engine:test")
|
||||
}
|
||||
@@ -1070,10 +1070,6 @@ filer:
|
||||
httpsPort: 0
|
||||
# Suffix of the host name, {bucket}.{domainName}
|
||||
domainName: ""
|
||||
# Seconds the gateway keeps serving after SIGTERM while its health checks
|
||||
# return 503 (see s3.preStopSeconds). Added to the filer pod's 60s
|
||||
# termination grace period.
|
||||
preStopSeconds: 0
|
||||
# enable user & permission to s3 (need to inject to all services)
|
||||
enableAuth: false
|
||||
# set to the name of an existing kubernetes Secret with the s3 json config file
|
||||
@@ -1122,17 +1118,8 @@ s3:
|
||||
# maxUnavailable: 0
|
||||
# maxSurge: 25%
|
||||
updateStrategy: {}
|
||||
# Time the s3 pod is given to shut down, including any preStop hook.
|
||||
# With preStopSeconds set, it must be at least preStopSeconds + 15 (the
|
||||
# in-flight request drain), or the chart refuses to render. A lifecycle
|
||||
# preStop hook runs before SIGTERM, so add its duration on top; the chart
|
||||
# cannot tell how long an exec hook takes. preStopSeconds replaces a
|
||||
# preStop sleep hook.
|
||||
# Time the s3 pod is given to shut down, including any preStop hook
|
||||
terminationGracePeriodSeconds: 10
|
||||
# Seconds the gateway keeps serving after SIGTERM while its health checks
|
||||
# return 503, so the pod leaves the Service endpoints before it stops
|
||||
# listening. 0 stops at once.
|
||||
preStopSeconds: 0
|
||||
bindAddress: 0.0.0.0
|
||||
port: 8333
|
||||
# add additional https port
|
||||
@@ -1331,8 +1318,7 @@ s3:
|
||||
|
||||
# Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave
|
||||
# the Service endpoints before it receives SIGTERM; keep it shorter than
|
||||
# terminationGracePeriodSeconds. s3.preStopSeconds does the same from
|
||||
# inside the gateway and also fails its health checks.
|
||||
# terminationGracePeriodSeconds.
|
||||
# ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
|
||||
# Example:
|
||||
# lifecycle:
|
||||
@@ -2025,7 +2011,6 @@ allInOne:
|
||||
# The s3-secret.yaml template only reads from .Values.s3.credentials.
|
||||
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
|
||||
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
|
||||
preStopSeconds: null # Seconds to keep serving after SIGTERM, added to the pod's 60s grace period (null inherits from s3.preStopSeconds, 0 disables)
|
||||
trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35
|
||||
# You may specify buckets to be created during the install process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
|
||||
@@ -45,8 +45,6 @@ Rule of thumb: start with 3 masters, then grow volume servers with data.
|
||||
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
|
||||
- Run **3 filers** for HA
|
||||
- Filers are stateless relative to each other — state lives in the DB
|
||||
- No quorum among filers: 2 already survive one failure; the third keeps a spare while one is down for upgrades
|
||||
- Give clients every filer address (e.g. `-filer=f1:8888,f2:8888,f3:8888`) or a load balancer that forwards both the HTTP and gRPC ports (8888 and 18888)
|
||||
|
||||
**S3 Servers**
|
||||
|
||||
@@ -79,7 +77,7 @@ Keeps housekeeping off the hot path.
|
||||
|----------------|----------------------|-----------------------------------|
|
||||
| Master | 3 | Raft quorum, topology, volume map |
|
||||
| Volume Server | N (scale out) | Stores data needles |
|
||||
| Filer | 3 (2 minimum) | Metadata gateway, no quorum |
|
||||
| Filer | 3 | Metadata gateway |
|
||||
| Metadata DB | HA cluster | Persists filer metadata |
|
||||
| S3 Server | N (by throughput) | S3 API frontend |
|
||||
| Admin | 1 | Background job coordinator |
|
||||
|
||||
+1084
-1067
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 54 KiB After Width: | Height: | Size: 55 KiB |
Generated
+2
-2
@@ -3195,9 +3195,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redb"
|
||||
version = "4.6.0"
|
||||
version = "4.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "096b1dd5ec053c9c2a5a26a93aafeadf73ebcb710ea54dc3151f17a8e20535ef"
|
||||
checksum = "de6c3b63e007e90ce536ec2ae4690826136a20ec8dbbbb400daef1bb999d2e36"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -19,6 +19,9 @@ path = "src/main.rs"
|
||||
# Disable with --no-default-features for 4-byte offsets (32GB max volume size).
|
||||
default = ["5bytes"]
|
||||
5bytes = []
|
||||
# Unstable redb cursor bulk-load for full_rebuild. Off in production.
|
||||
# Pulls redb's experimental_cursor (and therefore experimental-api-5).
|
||||
redb-experimental-cursor = ["redb/experimental_cursor"]
|
||||
|
||||
[lints.clippy]
|
||||
# Protobuf message literals keep `..Default::default()` on purpose: it is
|
||||
|
||||
@@ -138,8 +138,7 @@ pub struct Cli {
|
||||
#[arg(long = "concurrentUploadLimitMB", default_value_t = 0)]
|
||||
pub concurrent_upload_limit_mb: u32,
|
||||
|
||||
/// Limit total concurrent download size in MB, 0 means unlimited; reads over
|
||||
/// the limit are sent to a replica unless readMode is local.
|
||||
/// Limit total concurrent download size in MB, 0 means unlimited.
|
||||
#[arg(long = "concurrentDownloadLimitMB", default_value_t = 0)]
|
||||
pub concurrent_download_limit_mb: u32,
|
||||
|
||||
|
||||
@@ -450,7 +450,6 @@ async fn run(
|
||||
|
||||
{
|
||||
let mut store = state.store.write().unwrap();
|
||||
store.store_dirs = config.folders.clone();
|
||||
for (i, dir) in config.folders.iter().enumerate() {
|
||||
let idx_dir = if config.idx_folder.is_empty() {
|
||||
dir.as_str()
|
||||
|
||||
@@ -913,6 +913,7 @@ async fn proxy_or_redirect_to_target(
|
||||
state: &VolumeServerState,
|
||||
info: ProxyRequestInfo,
|
||||
vid: VolumeId,
|
||||
allow_local_redirect: bool,
|
||||
) -> Response {
|
||||
// Look up volume locations from master
|
||||
let locations = match lookup_volume(
|
||||
@@ -960,7 +961,10 @@ async fn proxy_or_redirect_to_target(
|
||||
match state.read_mode {
|
||||
ReadMode::Proxy => proxy_request(state, &info, target).await,
|
||||
ReadMode::Redirect => redirect_request(&info, target, &state.outgoing_http_scheme),
|
||||
ReadMode::Local => StatusCode::NOT_FOUND.into_response(),
|
||||
ReadMode::Local if allow_local_redirect => {
|
||||
redirect_request(&info, target, &state.outgoing_http_scheme)
|
||||
}
|
||||
ReadMode::Local => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1427,7 +1431,7 @@ async fn proxy_missing_volume(
|
||||
None => return StatusCode::NOT_FOUND.into_response(),
|
||||
};
|
||||
|
||||
proxy_or_redirect_to_target(state, info, vid).await
|
||||
proxy_or_redirect_to_target(state, info, vid, false).await
|
||||
}
|
||||
|
||||
/// Go's `proxied` query param check (loop prevention).
|
||||
@@ -1460,10 +1464,8 @@ async fn check_download_limit(
|
||||
|
||||
// Go tries proxy to replica once before the wait loop
|
||||
// (checkDownloadLimit); it does not retry on each wakeup.
|
||||
let should_try_replica = state.read_mode != ReadMode::Local
|
||||
&& !is_proxied_query(&query_string)
|
||||
&& !state.current_master().await.is_empty()
|
||||
&& {
|
||||
let should_try_replica =
|
||||
!is_proxied_query(&query_string) && !state.current_master().await.is_empty() && {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
@@ -1473,24 +1475,25 @@ async fn check_download_limit(
|
||||
&& let Some(info) =
|
||||
build_proxy_request_info(path, request_headers, &query_string, method)
|
||||
{
|
||||
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid).await);
|
||||
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid, true).await);
|
||||
}
|
||||
|
||||
// Blocking wait loop (Go's waitForDownloadSlot)
|
||||
loop {
|
||||
let notified = state.download_notify.notified();
|
||||
if state.inflight_download_bytes.load(Ordering::Relaxed)
|
||||
<= state.concurrent_download_limit
|
||||
if tokio::time::timeout_at(deadline, state.download_notify.notified())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
if tokio::time::timeout_at(deadline, notified).await.is_err() {
|
||||
return ControlFlow::Break(json_error_with_query(
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"download limit exceeded",
|
||||
uri.query(),
|
||||
));
|
||||
}
|
||||
let current = state.inflight_download_bytes.load(Ordering::Relaxed);
|
||||
if current <= state.concurrent_download_limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
ControlFlow::Continue(true)
|
||||
@@ -6359,88 +6362,6 @@ mod tests {
|
||||
test_state_with_store(store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn download_limit_local_read_mode_waits_instead_of_proxying() {
|
||||
use crate::storage::super_block::ReplicaPlacement;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mut state = volume_test_state(&tmp);
|
||||
{
|
||||
let s = Arc::get_mut(&mut state).unwrap();
|
||||
s.read_mode = ReadMode::Local;
|
||||
s.concurrent_download_limit = 1;
|
||||
s.inflight_download_data_timeout = std::time::Duration::from_millis(50);
|
||||
}
|
||||
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
state.inflight_download_bytes.store(2, Ordering::Relaxed);
|
||||
state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1
|
||||
.set_replica_placement(ReplicaPlacement::from_string("001").unwrap())
|
||||
.unwrap();
|
||||
|
||||
let path = "/1,0123456789abcdef";
|
||||
let uri: axum::http::Uri = path.parse().unwrap();
|
||||
let flow = check_download_limit(
|
||||
&state,
|
||||
&uri,
|
||||
&HeaderMap::new(),
|
||||
path,
|
||||
VolumeId(1),
|
||||
&Method::GET,
|
||||
)
|
||||
.await;
|
||||
match flow {
|
||||
ControlFlow::Break(resp) => assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS),
|
||||
ControlFlow::Continue(_) => panic!("read over the download limit must not proceed"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn download_limit_wait_sees_release_before_waiting() {
|
||||
let mut state = streaming_test_state();
|
||||
{
|
||||
let s = Arc::get_mut(&mut state).unwrap();
|
||||
s.read_mode = ReadMode::Proxy;
|
||||
s.concurrent_download_limit = 1;
|
||||
s.inflight_download_data_timeout = std::time::Duration::from_secs(5);
|
||||
}
|
||||
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
state.inflight_download_bytes.store(2, Ordering::Relaxed);
|
||||
|
||||
let master_guard = state.current_master_url.write().await;
|
||||
let waiter = tokio::spawn({
|
||||
let state = state.clone();
|
||||
async move {
|
||||
let path = "/1,0123456789abcdef";
|
||||
let uri: axum::http::Uri = path.parse().unwrap();
|
||||
check_download_limit(
|
||||
&state,
|
||||
&uri,
|
||||
&HeaderMap::new(),
|
||||
path,
|
||||
VolumeId(1),
|
||||
&Method::GET,
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
state.inflight_download_bytes.store(0, Ordering::Relaxed);
|
||||
state.download_notify.notify_waiters();
|
||||
drop(master_guard);
|
||||
|
||||
let flow = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
|
||||
.await
|
||||
.expect("waiter missed the released download slot")
|
||||
.unwrap();
|
||||
assert!(matches!(flow, ControlFlow::Continue(true)));
|
||||
}
|
||||
|
||||
/// Write a needle to volume 1 and return its URL path.
|
||||
fn put_test_needle(state: &VolumeServerState, id: u64, data: &[u8]) -> String {
|
||||
put_test_needle_with(state, id, data, |_| {})
|
||||
|
||||
@@ -58,8 +58,6 @@ pub struct DiskLocation {
|
||||
/// skips a claimed vid outright and claims any other through its load,
|
||||
/// so a VolumeCopy's .note sweep cannot unlink files it still writes.
|
||||
pub(crate) load_guard: Option<Arc<Mutex<HashSet<VolumeId>>>>,
|
||||
/// The data directory of every disk on this server.
|
||||
pub(crate) store_dirs: Vec<String>,
|
||||
}
|
||||
|
||||
/// Keeps `vid` in the copy-claim set through one scan load so a VolumeCopy
|
||||
@@ -125,7 +123,6 @@ impl DiskLocation {
|
||||
disk_free_bytes: AtomicU64::new(0),
|
||||
min_free_space,
|
||||
load_guard: None,
|
||||
store_dirs: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -489,20 +486,8 @@ impl DiskLocation {
|
||||
|
||||
// Custom ratio comes from the volume's own .vif; the server holds no
|
||||
// cluster EC config in memory.
|
||||
let data_shards = ec_data_shards_from_vif(
|
||||
&self.directory,
|
||||
&self.idx_directory,
|
||||
&self.store_dirs,
|
||||
collection,
|
||||
vid,
|
||||
);
|
||||
if data_shards == 0 {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"sibling .vif files disagree on the encode; keeping EC shards"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
let data_shards =
|
||||
ec_data_shards_from_vif(&self.directory, &self.idx_directory, collection, vid);
|
||||
|
||||
// On-disk .dat: an empty <= superblock one is a stub; a transient stat
|
||||
// error keeps the shards rather than deleting.
|
||||
@@ -1035,9 +1020,7 @@ impl DiskLocation {
|
||||
// propagate the error to the caller.
|
||||
let created = !self.ec_volumes.contains_key(&vid);
|
||||
if created {
|
||||
let ec_vol =
|
||||
EcVolume::new_with_vif_dirs(&dir, idx_dir, &self.store_dirs, collection, vid)
|
||||
.map_err(VolumeError::Io)?;
|
||||
let ec_vol = EcVolume::new(&dir, idx_dir, collection, vid).map_err(VolumeError::Io)?;
|
||||
self.ec_volumes.insert(vid, ec_vol);
|
||||
}
|
||||
let ec_vol = self.ec_volumes.get_mut(&vid).expect("just inserted above");
|
||||
@@ -1458,7 +1441,6 @@ fn rm_if_present(path: String) -> io::Result<()> {
|
||||
fn ec_data_shards_from_vif(
|
||||
directory: &str,
|
||||
idx_directory: &str,
|
||||
store_dirs: &[String],
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> usize {
|
||||
@@ -1477,23 +1459,6 @@ fn ec_data_shards_from_vif(
|
||||
break;
|
||||
}
|
||||
}
|
||||
match crate::storage::erasure_coding::ec_volume::sibling_ec_config_vif(
|
||||
directory,
|
||||
idx_directory,
|
||||
store_dirs,
|
||||
collection,
|
||||
vid,
|
||||
) {
|
||||
Err(()) => return 0,
|
||||
Ok(Some(vi)) => {
|
||||
if let Some(ds) = vi.ec_shard_config.map(|c| c.data_shards as usize)
|
||||
&& ds > 0
|
||||
{
|
||||
return ds;
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
}
|
||||
DATA_SHARDS_COUNT
|
||||
}
|
||||
|
||||
@@ -1577,7 +1542,7 @@ pub(crate) fn vif_references_remote_file(vif_path: &str) -> bool {
|
||||
/// True when a `.vif` records an EC shard config, i.e. the volume was
|
||||
/// erasure-coded. Such a volume keeps no local `.dat`, so an empty `.dat`
|
||||
/// alongside it is a leftover stub safe to remove.
|
||||
pub(crate) fn vif_is_ec_volume(vif_path: &str) -> bool {
|
||||
fn vif_is_ec_volume(vif_path: &str) -> bool {
|
||||
fs::read_to_string(vif_path)
|
||||
.ok()
|
||||
.and_then(|s| serde_json::from_str::<VifVolumeInfo>(&s).ok())
|
||||
|
||||
@@ -272,96 +272,6 @@ fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: Volume
|
||||
data_vif
|
||||
}
|
||||
|
||||
/// [`locate_vif_path`], except that a data-dir `.vif` recording no EC config
|
||||
/// yields to the idx dir's when that one records it: a source volume's own
|
||||
/// `.vif` stays on a disk that later receives shards of the same volume id,
|
||||
/// while the encode's `.vif` lands with the index on a sibling.
|
||||
fn resolve_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String {
|
||||
let path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
let data_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir, collection, volume_id),
|
||||
);
|
||||
if dir_idx == dir || path != data_vif {
|
||||
return path;
|
||||
}
|
||||
let idx_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id),
|
||||
);
|
||||
prefer_ec_config_vif(path, idx_vif)
|
||||
}
|
||||
|
||||
fn read_vif_file(path: &str) -> Result<Option<crate::storage::volume::VifVolumeInfo>, ()> {
|
||||
match std::fs::read_to_string(path) {
|
||||
Ok(content) if content.trim().is_empty() => Ok(None),
|
||||
Ok(content) => serde_json::from_str(&content).map(Some).map_err(|_| ()),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(_) => Err(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `alt` when `local` records no EC config but `alt` does.
|
||||
fn prefer_ec_config_vif(local: String, alt: String) -> String {
|
||||
match read_vif_file(&local) {
|
||||
Ok(Some(info)) if info.ec_shard_config.is_some() => return local,
|
||||
Err(()) => return local,
|
||||
_ => {}
|
||||
}
|
||||
match read_vif_file(&alt) {
|
||||
Ok(Some(info)) if info.ec_shard_config.is_some() => alt,
|
||||
_ => local,
|
||||
}
|
||||
}
|
||||
|
||||
/// The `.vif` a disk mounting against a shared `-dir.idx` takes from a sibling
|
||||
/// data dir in `vif_dirs`: its own records no EC config, the index is not its
|
||||
/// own, and every sibling `.vif` recording an EC config names the same run.
|
||||
/// `Err` when a sibling `.vif` cannot be read or the siblings name different runs.
|
||||
pub(crate) fn sibling_ec_config_vif(
|
||||
dir: &str,
|
||||
dir_idx: &str,
|
||||
vif_dirs: &[String],
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> Result<Option<crate::storage::volume::VifVolumeInfo>, ()> {
|
||||
let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
|
||||
let data_vif = format!("{}.vif", data_base);
|
||||
if dir_idx == dir
|
||||
|| vif_dirs.iter().any(|d| d == dir_idx)
|
||||
|| matches!(ecx_file_size(&format!("{}.ecx", data_base)), Some(n) if n > 0)
|
||||
|| resolve_vif_path(dir, dir_idx, collection, volume_id) != data_vif
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
match read_vif_file(&data_vif) {
|
||||
Ok(Some(info)) if info.ec_shard_config.is_some() => return Ok(None),
|
||||
Err(()) => return Ok(None),
|
||||
_ => {}
|
||||
}
|
||||
let mut found: Option<crate::storage::volume::VifVolumeInfo> = None;
|
||||
for other in vif_dirs.iter().filter(|d| d.as_str() != dir) {
|
||||
let path = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(other, collection, volume_id)
|
||||
);
|
||||
let info = match read_vif_file(&path) {
|
||||
Ok(Some(info)) if info.ec_shard_config.is_some() => info,
|
||||
Ok(_) => continue,
|
||||
Err(()) => return Err(()),
|
||||
};
|
||||
let run = |v: &crate::storage::volume::VifVolumeInfo| {
|
||||
v.ec_shard_config.as_ref().map(|c| c.encode_ts_ns)
|
||||
};
|
||||
match &found {
|
||||
None => found = Some(info),
|
||||
Some(f) if run(f) != run(&info) => return Err(()),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
/// Load the volume's `.vif` (data dir first, then idx dir, then the active
|
||||
/// generation's versioned sidecar — see [`locate_vif_path`]). Absent
|
||||
/// everywhere is legal — legacy volumes predate the sidecar — and returns
|
||||
@@ -376,7 +286,7 @@ pub fn load_vif_info(
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<Option<crate::storage::volume::VifVolumeInfo>> {
|
||||
let vif_path = resolve_vif_path(dir, dir_idx, collection, volume_id);
|
||||
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
match std::fs::read_to_string(&vif_path) {
|
||||
Ok(content) if content.trim().is_empty() => Ok(None),
|
||||
Ok(content) => serde_json::from_str(&content).map(Some).map_err(|e| {
|
||||
@@ -426,16 +336,15 @@ pub fn load_vif_info_across_dirs(
|
||||
// pointed callers at the wrong disk whenever the vif lived with the
|
||||
// index — and a caller resolving the rest of the volume's metadata
|
||||
// against that answer would look in a directory holding none of it.
|
||||
let idx_vif = format!(
|
||||
let data_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id)
|
||||
crate::storage::volume::volume_file_name(dir, collection, volume_id)
|
||||
);
|
||||
let found_in =
|
||||
if dir_idx != dir && resolve_vif_path(dir, dir_idx, collection, volume_id) == idx_vif {
|
||||
dir_idx.to_string()
|
||||
} else {
|
||||
dir.to_string()
|
||||
};
|
||||
let found_in = if std::path::Path::new(&data_vif).exists() {
|
||||
dir.to_string()
|
||||
} else {
|
||||
dir_idx.to_string()
|
||||
};
|
||||
return Ok(Some((vif, found_in)));
|
||||
}
|
||||
for other in other_dirs {
|
||||
@@ -660,25 +569,7 @@ impl EcVolume {
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<Self> {
|
||||
Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL, &[])
|
||||
}
|
||||
|
||||
/// [`Self::new`] for a disk whose server's data directories are `vif_dirs`.
|
||||
pub fn new_with_vif_dirs(
|
||||
dir: &str,
|
||||
dir_idx: &str,
|
||||
vif_dirs: &[String],
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<Self> {
|
||||
Self::open_with(
|
||||
dir,
|
||||
dir_idx,
|
||||
collection,
|
||||
volume_id,
|
||||
EcjFsOps::REAL,
|
||||
vif_dirs,
|
||||
)
|
||||
Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL)
|
||||
}
|
||||
|
||||
/// [`Self::new`] with the journal-compaction filesystem steps supplied, so
|
||||
@@ -689,15 +580,10 @@ impl EcVolume {
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
ecj_ops: EcjFsOps,
|
||||
vif_dirs: &[String],
|
||||
) -> io::Result<Self> {
|
||||
// One load of the volume's `.vif`, used for both the shard config and
|
||||
// the version / dat-size fields below.
|
||||
let vif = load_vif_info(dir, dir_idx, collection, volume_id)?;
|
||||
let vif = sibling_ec_config_vif(dir, dir_idx, vif_dirs, collection, volume_id)
|
||||
.ok()
|
||||
.flatten()
|
||||
.or(vif);
|
||||
// Both directories: a split -dir/-dir.idx layout keeps the .ecsum with
|
||||
// the INDEX, and it is the only layout record a volume whose vif is
|
||||
// absent or config-free still has.
|
||||
@@ -3694,7 +3580,7 @@ mod tests {
|
||||
);
|
||||
let before = std::fs::read(&ecj_path).unwrap();
|
||||
(
|
||||
EcVolume::open_with(dir, dir, "", vid, ops, &[]),
|
||||
EcVolume::open_with(dir, dir, "", vid, ops),
|
||||
ecj_path,
|
||||
before,
|
||||
)
|
||||
@@ -3915,7 +3801,7 @@ mod tests {
|
||||
read_at: rewrite_after_last_read,
|
||||
..EcjFsOps::REAL
|
||||
};
|
||||
let vol = EcVolume::open_with(dir, dir, "", vid, ops, &[]).unwrap();
|
||||
let vol = EcVolume::open_with(dir, dir, "", vid, ops).unwrap();
|
||||
assert!(
|
||||
IN_PLACE_REWRITE.with(|r| r.borrow().is_none()),
|
||||
"the rewrite never ran"
|
||||
@@ -4222,143 +4108,6 @@ mod tests {
|
||||
assert_eq!(vol.block_size, 0, "legacy layout for a pre-record volume");
|
||||
}
|
||||
|
||||
/// Shards placed on the disk that still holds the source volume's `.vif`
|
||||
/// mount against an index on a sibling disk; the encode identity and
|
||||
/// layout come from the sibling's `.vif`, not the source's config-free one.
|
||||
#[test]
|
||||
fn test_ec_volume_prefers_idx_vif_over_config_free_data_vif() {
|
||||
let data_tmp = TempDir::new().unwrap();
|
||||
let idx_tmp = TempDir::new().unwrap();
|
||||
let dir = data_tmp.path().to_str().unwrap();
|
||||
let dir_idx = idx_tmp.path().to_str().unwrap();
|
||||
write_ecx_file(dir_idx, "pics", VolumeId(1), &[]);
|
||||
let data_base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", data_base),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
version: 3,
|
||||
dat_file_size: 4096,
|
||||
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
|
||||
data_shards: 12,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: 1_700_000_000_123_456_789,
|
||||
block_size: 3 * 1024 * 1024,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let idx_base = crate::storage::volume::volume_file_name(dir_idx, "pics", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", idx_base),
|
||||
serde_json::to_string_pretty(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap();
|
||||
assert_eq!(vol.encode_ts_ns, 1_700_000_000_123_456_789);
|
||||
assert_eq!((vol.data_shards, vol.parity_shards), (12, 4));
|
||||
assert_eq!(vol.block_size, 3 * 1024 * 1024);
|
||||
assert_eq!(vol.dat_file_size, 4096);
|
||||
let (_, found_in) = load_vif_info_across_dirs(dir, dir_idx, &[], "pics", VolumeId(1))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(found_in, dir_idx);
|
||||
}
|
||||
|
||||
/// A data-dir `.vif` that records an EC config stays authoritative.
|
||||
#[test]
|
||||
fn test_ec_volume_keeps_data_vif_with_ec_config() {
|
||||
let data_tmp = TempDir::new().unwrap();
|
||||
let idx_tmp = TempDir::new().unwrap();
|
||||
let dir = data_tmp.path().to_str().unwrap();
|
||||
let dir_idx = idx_tmp.path().to_str().unwrap();
|
||||
write_ecx_file(dir_idx, "pics", VolumeId(1), &[]);
|
||||
for (d, ts) in [(dir, 11), (dir_idx, 22)] {
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: ts,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let base = crate::storage::volume::volume_file_name(d, "pics", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", base),
|
||||
serde_json::to_string_pretty(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap();
|
||||
assert_eq!(vol.encode_ts_ns, 11);
|
||||
}
|
||||
|
||||
/// With a shared `-dir.idx` the encode's `.vif` sits in the data dir of the
|
||||
/// disk that received the index, so a disk holding a source volume's
|
||||
/// config-free `.vif` takes it from that sibling, but only when the
|
||||
/// siblings agree on one run.
|
||||
#[test]
|
||||
fn test_ec_volume_takes_sibling_vif_with_shared_index() {
|
||||
const RUN_A: i64 = 1_700_000_000_000_000_011;
|
||||
const RUN_B: i64 = 1_700_000_000_000_000_022;
|
||||
let cases: [(&str, &[i64], bool, i64); 4] = [
|
||||
("one sibling", &[RUN_A], false, RUN_A),
|
||||
("siblings agree", &[RUN_A, RUN_A], false, RUN_A),
|
||||
("siblings disagree", &[RUN_A, RUN_B], false, 0),
|
||||
("local index", &[RUN_A], true, 0),
|
||||
];
|
||||
for (name, sibling_ts, local_index, want) in cases {
|
||||
let data_tmp = TempDir::new().unwrap();
|
||||
let idx_tmp = TempDir::new().unwrap();
|
||||
let dir = data_tmp.path().to_str().unwrap();
|
||||
let dir_idx = idx_tmp.path().to_str().unwrap();
|
||||
let entry = [(NeedleId(1), Offset::from_actual_offset(8), Size(10))];
|
||||
write_ecx_file(dir_idx, "c", VolumeId(1), &entry);
|
||||
if local_index {
|
||||
write_ecx_file(dir, "c", VolumeId(1), &entry);
|
||||
}
|
||||
let data_base = crate::storage::volume::volume_file_name(dir, "c", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", data_base),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let mut dirs = vec![dir.to_string()];
|
||||
let mut siblings = Vec::new();
|
||||
for &ts in sibling_ts {
|
||||
let sibling = TempDir::new().unwrap();
|
||||
let sibling_dir = sibling.path().to_str().unwrap().to_string();
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
version: 3,
|
||||
dat_file_size: 4096,
|
||||
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: ts,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let base = crate::storage::volume::volume_file_name(&sibling_dir, "c", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", base),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
dirs.push(sibling_dir);
|
||||
siblings.push(sibling);
|
||||
}
|
||||
|
||||
let vol = EcVolume::new_with_vif_dirs(dir, dir_idx, &dirs, "c", VolumeId(1)).unwrap();
|
||||
assert_eq!(vol.encode_ts_ns, want, "{}", name);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ec_volume_wide_ratio_vif_config() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
|
||||
@@ -13,6 +13,7 @@ use std::io::{self, Read, Seek, Write};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
|
||||
|
||||
#[cfg(feature = "redb-experimental-cursor")]
|
||||
use std::ops::Bound;
|
||||
|
||||
mod compact_map;
|
||||
@@ -484,38 +485,6 @@ const NEEDLE_TABLE: TableDefinition<u64, &[u8]> = TableDefinition::new("needles"
|
||||
const META_TABLE: TableDefinition<&str, u64> = TableDefinition::new("meta");
|
||||
const META_IDX_SIZE: &str = "idx_size";
|
||||
|
||||
/// redb 4.6 starts a new file at 72 KiB: 64 KiB of pages plus the header
|
||||
/// and the region tracker.
|
||||
const REDB_SMALL_FILE_BYTES: u64 = 72 * 1024;
|
||||
/// redb 4.2 started at 1 MiB of pages plus that same 8 KiB.
|
||||
const REDB_OLD_FLOOR_BYTES: u64 = 1024 * 1024 + 8 * 1024;
|
||||
/// Meta value `1` means a compact of a file in the old-floor band already finished.
|
||||
const META_FLOOR_COMPACT: &str = "floor_compact";
|
||||
|
||||
/// True when a reused `.rdb` is still inside the old 4.2 starting size.
|
||||
/// A file one byte past it has grown.
|
||||
fn should_compact_old_floor(len: u64) -> bool {
|
||||
len > REDB_SMALL_FILE_BYTES && len <= REDB_OLD_FLOOR_BYTES
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum CompactFollowUp {
|
||||
/// Drop the `Database` and open the same path once. Do not compact again.
|
||||
ReopenSkipCompact,
|
||||
/// The handle can still begin a write. Do not record `floor_compact`.
|
||||
KeepHandle,
|
||||
}
|
||||
|
||||
fn compact_follow_up(err: &redb::CompactionError) -> CompactFollowUp {
|
||||
match err {
|
||||
redb::CompactionError::Storage(_) => CompactFollowUp::ReopenSkipCompact,
|
||||
redb::CompactionError::TransactionInProgress
|
||||
| redb::CompactionError::PersistentSavepointExists
|
||||
| redb::CompactionError::EphemeralSavepointExists => CompactFollowUp::KeepHandle,
|
||||
_ => CompactFollowUp::KeepHandle,
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes between two durable redb checkpoints. Every non-durable commit
|
||||
/// leaves an entry in redb's transaction tracker and pins the pages that
|
||||
/// were on disk at the last durable commit; only a durable commit clears
|
||||
@@ -679,6 +648,9 @@ impl RedbNeedleMap {
|
||||
let meta = txn
|
||||
.open_table(META_TABLE)
|
||||
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
|
||||
// ReadableTable::get guard borrows `meta`; edition 2024 drops the tail
|
||||
// expression's temporaries before `meta`, so no extra binding is needed.
|
||||
match meta.get(META_IDX_SIZE) {
|
||||
Ok(Some(guard)) => Ok(Some(guard.value())),
|
||||
Ok(None) => Ok(None),
|
||||
@@ -686,118 +658,6 @@ impl RedbNeedleMap {
|
||||
}
|
||||
}
|
||||
|
||||
fn rdb_file_len(&self) -> io::Result<u64> {
|
||||
Ok(std::fs::metadata(&self.rdb_path)?.len())
|
||||
}
|
||||
|
||||
fn read_floor_compact(&self) -> io::Result<Option<u64>> {
|
||||
let txn = self
|
||||
.db_or_err()?
|
||||
.begin_read()
|
||||
.map_err(|e| io::Error::other(format!("redb begin_read: {}", e)))?;
|
||||
let meta = txn
|
||||
.open_table(META_TABLE)
|
||||
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||
match meta.get(META_FLOOR_COMPACT) {
|
||||
Ok(Some(guard)) => Ok(Some(guard.value())),
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => Err(io::Error::other(format!("redb get meta: {}", e))),
|
||||
}
|
||||
}
|
||||
|
||||
fn save_floor_compact(&self) -> io::Result<()> {
|
||||
let txn = Self::begin_write_no_fsync(self.db_or_err()?)?;
|
||||
{
|
||||
let mut meta = txn
|
||||
.open_table(META_TABLE)
|
||||
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||
meta.insert(META_FLOOR_COMPACT, 1u64)
|
||||
.map_err(|e| io::Error::other(format!("redb insert meta: {}", e)))?;
|
||||
}
|
||||
txn.commit()
|
||||
.map_err(|e| io::Error::other(format!("redb commit meta: {}", e)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop the exclusive lock and open the same file. Does not compact.
|
||||
fn reopen_database(&mut self) -> io::Result<()> {
|
||||
self.db.take();
|
||||
let db = Database::builder()
|
||||
.set_cache_size(self.cache_bytes)
|
||||
.open(&self.rdb_path)
|
||||
.map_err(|e| io::Error::other(format!("redb reopen after compact: {}", e)))?;
|
||||
self.db = Some(db);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn record_floor_compact_if_still_in_band(&self) -> io::Result<()> {
|
||||
let len = match self.rdb_file_len() {
|
||||
Ok(len) => len,
|
||||
Err(e) => {
|
||||
tracing::warn!("redb floor compact flag skipped, metadata failed: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
if should_compact_old_floor(len) {
|
||||
self.save_floor_compact()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compact a reused file that is still the 4.2 starting allocation.
|
||||
///
|
||||
/// Called only from `try_reuse_rdb`, after the replay transaction has
|
||||
/// committed. `new` and `full_rebuild` already create a 72 KiB file.
|
||||
///
|
||||
/// An empty durable commit publishes a `Durability::None` tail before
|
||||
/// `compact`. A storage error inside `compact` would otherwise drop that
|
||||
/// tail when the handle is reopened. `checkpoint` is not used: during
|
||||
/// reuse `idx_file` is `None` and `idx_file_offset` is 0.
|
||||
fn shrink_old_floor(&mut self) -> io::Result<()> {
|
||||
let len = match self.rdb_file_len() {
|
||||
Ok(len) => len,
|
||||
Err(e) => {
|
||||
tracing::warn!("redb floor compact skipped, metadata failed: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
// The flag is irrelevant outside the band. A meta read error there
|
||||
// must not make `load_from_idx` rebuild a larger index.
|
||||
if !should_compact_old_floor(len) {
|
||||
return Ok(());
|
||||
}
|
||||
if self.read_floor_compact()? == Some(1) {
|
||||
return Ok(());
|
||||
}
|
||||
// Failure returns Err so `load_from_idx` full-rebuilds. Do not reopen
|
||||
// over a tail that this commit did not publish.
|
||||
let mut txn = self
|
||||
.db_or_err()?
|
||||
.begin_write()
|
||||
.map_err(|e| io::Error::other(format!("redb begin_write: {}", e)))?;
|
||||
txn.set_quick_repair(true);
|
||||
txn.commit()
|
||||
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
|
||||
let compact_result = self
|
||||
.db
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::other("redb database is closed"))?
|
||||
.compact();
|
||||
match compact_result {
|
||||
Ok(_) => self.record_floor_compact_if_still_in_band(),
|
||||
Err(e) => match compact_follow_up(&e) {
|
||||
CompactFollowUp::ReopenSkipCompact => {
|
||||
tracing::warn!("redb floor compact storage error, reopening: {}", e);
|
||||
self.reopen_database()
|
||||
}
|
||||
CompactFollowUp::KeepHandle => {
|
||||
tracing::warn!("redb floor compact skipped: {}", e);
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only read of META `idx_size` through the live handle. See
|
||||
/// [`test_support::live_meta_idx_size`] for why durability tests use
|
||||
/// this instead of copying the open `.rdb`.
|
||||
@@ -806,11 +666,6 @@ impl RedbNeedleMap {
|
||||
self.read_idx_size_meta().unwrap()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn live_floor_compact(&self) -> Option<u64> {
|
||||
self.read_floor_compact().unwrap()
|
||||
}
|
||||
|
||||
/// Load from an .idx file, reusing an existing .rdb if it is consistent.
|
||||
///
|
||||
/// Strategy:
|
||||
@@ -919,7 +774,6 @@ impl RedbNeedleMap {
|
||||
nm.save_idx_size_meta(idx_size)?;
|
||||
}
|
||||
|
||||
nm.shrink_old_floor()?;
|
||||
Ok(nm)
|
||||
}
|
||||
|
||||
@@ -942,8 +796,8 @@ impl RedbNeedleMap {
|
||||
/// `Database::create` opens an existing file (`truncate(false)`). Unlink
|
||||
/// is best-effort: if it fails (sticky bit, foreign owner) but the path is
|
||||
/// still writable, clear the leftover needles table before inserting.
|
||||
/// Live keys come from a `BTreeMap` so `insert_before` walks them in
|
||||
/// needle-id order and packs leaves, without a second Vec + sort scratch.
|
||||
/// Live keys come from a `BTreeMap` so inserts are in needle-id order and
|
||||
/// redb 4.2.0 packs leaves, without a second Vec + sort scratch.
|
||||
fn full_rebuild<R: Read + Seek>(
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
@@ -995,26 +849,43 @@ impl RedbNeedleMap {
|
||||
.map_err(|e| io::Error::other(format!("redb retain: {}", e)))?;
|
||||
}
|
||||
|
||||
let mut cursor = table
|
||||
.upper_bound_mut(Bound::<u64>::Unbounded)
|
||||
.map_err(|e| {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb upper_bound_mut: {}", e))
|
||||
#[cfg(not(feature = "redb-experimental-cursor"))]
|
||||
{
|
||||
for (key, nv) in &entries {
|
||||
let key_u64: u64 = (*key).into();
|
||||
let packed = pack_needle_value(nv);
|
||||
table
|
||||
.insert(key_u64, packed.as_slice())
|
||||
.map_err(|e| io::Error::other(format!("redb insert: {}", e)))?;
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "redb-experimental-cursor")]
|
||||
{
|
||||
let mut cursor =
|
||||
table
|
||||
.upper_bound_mut(Bound::<u64>::Unbounded)
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("redb upper_bound_mut: {}", e),
|
||||
)
|
||||
})?;
|
||||
for (key, nv) in &entries {
|
||||
let key_u64: u64 = (*key).into();
|
||||
let packed = pack_needle_value(nv);
|
||||
cursor
|
||||
.insert_before(key_u64, packed.as_slice())
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("redb insert_before: {}", e),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
cursor.close().map_err(|e| {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
|
||||
})?;
|
||||
for (key, nv) in &entries {
|
||||
let key_u64: u64 = (*key).into();
|
||||
let packed = pack_needle_value(nv);
|
||||
cursor
|
||||
.insert_before(key_u64, packed.as_slice())
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("redb insert_before: {}", e),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
cursor.close().map_err(|e| {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
|
||||
})?;
|
||||
}
|
||||
txn.commit()
|
||||
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
|
||||
@@ -1177,6 +1048,9 @@ impl RedbNeedleMap {
|
||||
let table = txn
|
||||
.open_table(NEEDLE_TABLE)
|
||||
.map_err(|e| io::Error::other(format!("redb open_table: {}", e)))?;
|
||||
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
|
||||
// ReadableTable::get guard borrows `table`; edition 2024 drops the tail
|
||||
// expression's temporaries before `table`, so no extra binding is needed.
|
||||
match table.get(key_u64) {
|
||||
Ok(Some(guard)) => Ok(packed_to_needle_value(guard.value())),
|
||||
Ok(None) => Ok(None),
|
||||
@@ -1307,7 +1181,7 @@ impl RedbNeedleMap {
|
||||
/// Reopen the database from `.idx` after a non-`TransactionPoisoned`
|
||||
/// commit error.
|
||||
///
|
||||
/// redb can make a `Durability::None` commit visible *before*
|
||||
/// redb 4.2.0 can make a `Durability::None` commit visible *before*
|
||||
/// returning `Err(CommitError::Storage(..))`. In that state the database
|
||||
/// refuses further write transactions, so the map cannot be used as-is.
|
||||
/// The `.idx` row that triggered the commit is **not** truncated — the
|
||||
@@ -1729,10 +1603,9 @@ pub(crate) mod test_support {
|
||||
/// every `put`/`delete` also commits non-durably, so before the first
|
||||
/// checkpoint this is the load-time value (`Some(0)` for a fresh map) —
|
||||
/// NOT the crash-durable `None` a copy of the open `.rdb` would show.
|
||||
/// A live read is the only portable observation: redb's default
|
||||
/// `ExclusiveWriter` takes an exclusive whole-file lock, which is
|
||||
/// advisory on Unix but mandatory on Windows, so copying the open
|
||||
/// `.rdb` fails there with OS error 33.
|
||||
/// A live read is the only portable observation: redb 4.2.0 takes an
|
||||
/// exclusive whole-file lock, which is advisory on Unix but mandatory
|
||||
/// on Windows, so copying the open `.rdb` fails there with OS error 33.
|
||||
///
|
||||
/// It still pins the property under test: the only *durable* META
|
||||
/// writer is `checkpoint`, so any value other than the load-time one
|
||||
@@ -1890,41 +1763,6 @@ mod tests {
|
||||
NeedleMapKind::Redb.redb_cache_bytes()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_new_file_is_at_most_72kib() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db_path = dir.path().join("fresh.rdb");
|
||||
let nm = RedbNeedleMap::new(db_path.to_str().unwrap(), redb_test_cache()).unwrap();
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
assert!(
|
||||
len <= REDB_SMALL_FILE_BYTES,
|
||||
"fresh redb file is {len} bytes, above the 72 KiB start"
|
||||
);
|
||||
drop(nm);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_compact_old_floor_bounds() {
|
||||
let small = REDB_SMALL_FILE_BYTES;
|
||||
let floor = REDB_OLD_FLOOR_BYTES;
|
||||
assert!(!should_compact_old_floor(small));
|
||||
assert!(should_compact_old_floor(small + 1));
|
||||
assert!(should_compact_old_floor(floor));
|
||||
assert!(!should_compact_old_floor(floor + 1));
|
||||
assert!(!should_compact_old_floor(0));
|
||||
}
|
||||
|
||||
fn reload_redb(db_path: &std::path::Path, idx_path: &std::path::Path) -> RedbNeedleMap {
|
||||
let mut idx = std::fs::File::open(idx_path).unwrap();
|
||||
RedbNeedleMap::load_from_idx(
|
||||
db_path.to_str().unwrap(),
|
||||
&mut idx,
|
||||
Version::current(),
|
||||
redb_test_cache(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Open a redb map on an empty .idx with an append writer attached, the
|
||||
/// way `Volume::load_index_redb` opens a writable volume.
|
||||
fn open_writable_redb(
|
||||
@@ -2057,6 +1895,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "redb-experimental-cursor")]
|
||||
#[test]
|
||||
fn test_redb_full_rebuild_insert_before_reads_back_thousands_of_shuffled_keys() {
|
||||
// Enough keys to split leaves. insert_before vs table.insert only
|
||||
@@ -2635,164 +2474,4 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_fresh_reuse_does_not_record_floor_compact() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
assert!(len <= REDB_SMALL_FILE_BYTES, "fresh file is {len} bytes");
|
||||
nm.close();
|
||||
drop(nm);
|
||||
let loaded = reload_redb(&db_path, &idx_path);
|
||||
assert_eq!(loaded.live_floor_compact(), None);
|
||||
assert!(loaded.get(NeedleId(1)).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_in_band_file_is_compacted_once() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||
let mut n = 0u64;
|
||||
loop {
|
||||
n += 1;
|
||||
nm.put(
|
||||
NeedleId(n),
|
||||
Offset::from_actual_offset((n as i64).saturating_mul(8)),
|
||||
Size(8),
|
||||
)
|
||||
.unwrap();
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
if should_compact_old_floor(len) {
|
||||
break;
|
||||
}
|
||||
assert!(
|
||||
n < 2_000_000,
|
||||
"file grew to {len} bytes without entering the old-floor band"
|
||||
);
|
||||
}
|
||||
nm.checkpoint(true).unwrap();
|
||||
nm.close();
|
||||
drop(nm);
|
||||
|
||||
let mut loaded = reload_redb(&db_path, &idx_path);
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
let flag = loaded.live_floor_compact();
|
||||
if should_compact_old_floor(len) {
|
||||
assert_eq!(flag, Some(1));
|
||||
} else {
|
||||
assert!(len <= REDB_SMALL_FILE_BYTES, "compacted file is {len} bytes");
|
||||
assert_eq!(flag, None);
|
||||
}
|
||||
let first = loaded.get(NeedleId(1)).unwrap();
|
||||
assert!(first.is_some());
|
||||
let last = loaded.get(NeedleId(n)).unwrap();
|
||||
assert!(last.is_some());
|
||||
// close checkpoints the non-durable floor_compact flag, then drop
|
||||
// releases the exclusive lock before the second open.
|
||||
loaded.close();
|
||||
drop(loaded);
|
||||
|
||||
let again = reload_redb(&db_path, &idx_path);
|
||||
assert_eq!(std::fs::metadata(&db_path).unwrap().len(), len);
|
||||
assert_eq!(again.live_floor_compact(), flag);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_sparse_old_floor_shrinks_on_reuse() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||
let mut n = 0u64;
|
||||
loop {
|
||||
n += 1;
|
||||
nm.put(
|
||||
NeedleId(n),
|
||||
Offset::from_actual_offset((n as i64).saturating_mul(8)),
|
||||
Size(8),
|
||||
)
|
||||
.unwrap();
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
if should_compact_old_floor(len) {
|
||||
break;
|
||||
}
|
||||
assert!(n < 2_000_000, "file is {len} bytes and never entered the band");
|
||||
}
|
||||
assert!(n > 32, "band was reached with only {n} keys");
|
||||
nm.checkpoint(true).unwrap();
|
||||
nm.close();
|
||||
drop(nm);
|
||||
|
||||
let keep = 32u64;
|
||||
let idx = std::fs::OpenOptions::new().write(true).open(&idx_path).unwrap();
|
||||
idx.set_len(keep * NEEDLE_MAP_ENTRY_SIZE as u64).unwrap();
|
||||
drop(idx);
|
||||
|
||||
let db = redb::Database::builder()
|
||||
.set_cache_size(redb_test_cache())
|
||||
.open(db_path.to_str().unwrap())
|
||||
.unwrap();
|
||||
let txn = db.begin_write().unwrap();
|
||||
{
|
||||
let mut table = txn.open_table(NEEDLE_TABLE).unwrap();
|
||||
for key in (keep + 1)..=n {
|
||||
table.remove(key).unwrap();
|
||||
}
|
||||
}
|
||||
{
|
||||
let mut meta = txn.open_table(META_TABLE).unwrap();
|
||||
meta.insert(META_IDX_SIZE, keep * NEEDLE_MAP_ENTRY_SIZE as u64)
|
||||
.unwrap();
|
||||
}
|
||||
txn.commit().unwrap();
|
||||
drop(db);
|
||||
|
||||
let before = std::fs::metadata(&db_path).unwrap().len();
|
||||
let loaded = reload_redb(&db_path, &idx_path);
|
||||
let after = std::fs::metadata(&db_path).unwrap().len();
|
||||
assert!(after < before, "reuse left the file at {after} bytes, was {before}");
|
||||
assert!(
|
||||
after <= REDB_SMALL_FILE_BYTES,
|
||||
"32 live keys still occupy {after} bytes"
|
||||
);
|
||||
assert!(loaded.get(NeedleId(1)).unwrap().is_some());
|
||||
assert!(loaded.get(NeedleId(keep)).unwrap().is_some());
|
||||
assert!(loaded.get(NeedleId(keep + 1)).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compact_follow_up_reopens_only_on_storage_error() {
|
||||
use redb::{CompactionError, StorageError};
|
||||
let storage = CompactionError::Storage(StorageError::Io(std::io::Error::other("sync")));
|
||||
assert_eq!(
|
||||
compact_follow_up(&storage),
|
||||
CompactFollowUp::ReopenSkipCompact
|
||||
);
|
||||
assert_eq!(
|
||||
compact_follow_up(&CompactionError::TransactionInProgress),
|
||||
CompactFollowUp::KeepHandle
|
||||
);
|
||||
assert_eq!(
|
||||
compact_follow_up(&CompactionError::PersistentSavepointExists),
|
||||
CompactFollowUp::KeepHandle
|
||||
);
|
||||
assert_eq!(
|
||||
compact_follow_up(&CompactionError::EphemeralSavepointExists),
|
||||
CompactFollowUp::KeepHandle
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_reopen_database_reads_committed_key() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut nm, _db_path, _idx_path) = open_writable_redb(dir.path());
|
||||
nm.put(NeedleId(7), Offset::from_actual_offset(64), Size(9))
|
||||
.unwrap();
|
||||
nm.reopen_database().unwrap();
|
||||
let nv = nm.get(NeedleId(7)).unwrap().unwrap();
|
||||
assert_eq!(nv.size, Size(9));
|
||||
nm.put(NeedleId(8), Offset::from_actual_offset(128), Size(10))
|
||||
.unwrap();
|
||||
let nv = nm.get(NeedleId(8)).unwrap().unwrap();
|
||||
assert_eq!(nv.size, Size(10));
|
||||
}
|
||||
}
|
||||
@@ -122,9 +122,6 @@ pub struct Store {
|
||||
/// The copy-claim set installed on every location by
|
||||
/// set_volume_load_guard, kept here so a later add_location gets it too.
|
||||
load_guard: Option<Arc<Mutex<HashSet<VolumeId>>>>,
|
||||
/// The data directory of every disk this server loads, set before the
|
||||
/// first `add_location` so a disk's EC mounts can see its siblings.
|
||||
pub store_dirs: Vec<String>,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
@@ -151,7 +148,6 @@ impl Store {
|
||||
disk_stats_of: super::disk_location::get_disk_stats,
|
||||
load_guard: None,
|
||||
preallocate_supported: PREALLOCATE_SUPPORTED,
|
||||
store_dirs: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,7 +170,6 @@ impl Store {
|
||||
tags,
|
||||
)?;
|
||||
loc.load_guard = self.load_guard.clone();
|
||||
loc.store_dirs = self.store_dirs.clone();
|
||||
loc.load_existing_volumes(self.needle_map_kind)?;
|
||||
|
||||
// Check for duplicate volume IDs across existing locations
|
||||
|
||||
@@ -8,9 +8,7 @@ use std::path::Path;
|
||||
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage::disk_location::{
|
||||
DiskLocation, parse_collection_volume_id_pub, vif_is_ec_volume,
|
||||
};
|
||||
use crate::storage::disk_location::{DiskLocation, parse_collection_volume_id_pub};
|
||||
use crate::storage::store::Store;
|
||||
use crate::storage::types::VolumeId;
|
||||
|
||||
@@ -209,33 +207,6 @@ fn mirror_sidecars_for_volume(
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> io::Result<usize> {
|
||||
// A local .vif with no EC config (a source volume's own) would shadow the
|
||||
// owner's once the index is copied next to it; keep mounting across disks.
|
||||
let dst_vif = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ".vif");
|
||||
if path_is_regular_file(&dst_vif)
|
||||
&& !vif_is_ec_volume(&dst_vif)
|
||||
&& (vif_is_ec_volume(&sidecar_dest_path(
|
||||
src_data_dir,
|
||||
src_idx_dir,
|
||||
collection,
|
||||
vid,
|
||||
".vif",
|
||||
)) || vif_is_ec_volume(&sidecar_dest_path(
|
||||
src_idx_dir,
|
||||
src_data_dir,
|
||||
collection,
|
||||
vid,
|
||||
".vif",
|
||||
)))
|
||||
{
|
||||
info!(
|
||||
volume_id = vid.0,
|
||||
collection = %collection,
|
||||
vif = %dst_vif,
|
||||
"local .vif records no EC config; not mirroring EC sidecars",
|
||||
);
|
||||
return Ok(0);
|
||||
}
|
||||
let mut copied = 0usize;
|
||||
for ext in EC_MIRRORED_SIDECARS {
|
||||
let dst = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ext);
|
||||
@@ -420,222 +391,6 @@ mod tests {
|
||||
assert_eq!(ecj_dst, ecj, ".ecj mirrored bytes differ from source");
|
||||
}
|
||||
|
||||
/// Shards on a disk that still holds the source volume's config-free
|
||||
/// `.vif` must mount with the owner's encode identity: mirroring the index
|
||||
/// next to that `.vif` would make the disk mount through it.
|
||||
#[test]
|
||||
fn mirror_skips_disk_with_config_free_vif() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
fs::create_dir_all(&dir0).unwrap();
|
||||
fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "c";
|
||||
let vid = 4122u32;
|
||||
plant_shard(&dir0, collection, vid, 2);
|
||||
plant_shard(&dir1, collection, vid, 0);
|
||||
fs::write(
|
||||
dir0.join(format!("{}_{}.vif", collection, vid)),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
plant_ecx(&dir1, collection, vid, &[0xA1u8; 20]);
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: 1_700_000_000_000_000_042,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
fs::write(
|
||||
dir1.join(format!("{}_{}.vif", collection, vid)),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
add_loc(&mut store, &dir0);
|
||||
add_loc(&mut store, &dir1);
|
||||
|
||||
assert!(
|
||||
!dir0.join(format!("{}_{}.ecx", collection, vid)).exists(),
|
||||
".ecx mirrored next to the config-free .vif"
|
||||
);
|
||||
let ec_vol = store.locations[0]
|
||||
.find_ec_volume(VolumeId(vid))
|
||||
.expect("dir0 ec volume mounted");
|
||||
assert_eq!(ec_vol.encode_ts_ns, 1_700_000_000_000_000_042);
|
||||
}
|
||||
|
||||
/// With a shared `-dir.idx` every disk sees the `.ecx`, and only the disk
|
||||
/// that received it holds the encode's `.vif`. A disk loaded first that
|
||||
/// still holds the source volume's config-free `.vif` must take that
|
||||
/// identity from its sibling.
|
||||
#[test]
|
||||
fn shared_index_disk_takes_sibling_vif_on_startup() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
let idx = tmp.path().join("idx");
|
||||
for d in [&dir0, &dir1, &idx] {
|
||||
fs::create_dir_all(d).unwrap();
|
||||
}
|
||||
|
||||
let collection = "c";
|
||||
let vid = 4123u32;
|
||||
plant_shard(&dir0, collection, vid, 2);
|
||||
plant_shard(&dir1, collection, vid, 0);
|
||||
fs::write(
|
||||
dir0.join(format!("{}_{}.vif", collection, vid)),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
plant_ecx(&idx, collection, vid, &[0xA1u8; 20]);
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: 1_700_000_000_000_000_043,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
fs::write(
|
||||
dir1.join(format!("{}_{}.vif", collection, vid)),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store.store_dirs = vec![
|
||||
dir0.to_str().unwrap().to_string(),
|
||||
dir1.to_str().unwrap().to_string(),
|
||||
];
|
||||
for dir in [&dir0, &dir1] {
|
||||
store
|
||||
.add_location(
|
||||
dir.to_str().unwrap(),
|
||||
idx.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for loc in &store.locations {
|
||||
let ec_vol = loc
|
||||
.find_ec_volume(VolumeId(vid))
|
||||
.unwrap_or_else(|| panic!("{} ec volume mounted", loc.directory));
|
||||
assert_eq!(
|
||||
ec_vol.encode_ts_ns, 1_700_000_000_000_000_043,
|
||||
"{}",
|
||||
loc.directory
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts a store whose first disk still holds the source `.dat`, its
|
||||
/// config-free `.vif` and one 12+4 shard, with the `.ecx` in a shared
|
||||
/// index dir and one sibling encode `.vif` per entry of `sibling_runs`.
|
||||
/// Asserts the source `.dat`, the shard and the `.ecx` survive startup.
|
||||
fn start_with_sibling_encode_vifs(sibling_runs: &[i64]) -> Store {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dirs: Vec<_> = (0..=sibling_runs.len())
|
||||
.map(|i| tmp.path().join(format!("data{}", i)))
|
||||
.collect();
|
||||
let idx = tmp.path().join("idx");
|
||||
for d in dirs.iter().chain([&idx]) {
|
||||
fs::create_dir_all(d).unwrap();
|
||||
}
|
||||
|
||||
let collection = "c";
|
||||
let vid = 4124u32;
|
||||
let base0 = dirs[0].join(format!("{}_{}", collection, vid));
|
||||
let dat = format!("{}.dat", base0.display());
|
||||
let shard = format!("{}.ec02", base0.display());
|
||||
fs::File::create(&dat)
|
||||
.unwrap()
|
||||
.set_len(11 * 1024 * 1024)
|
||||
.unwrap();
|
||||
fs::File::create(&shard)
|
||||
.unwrap()
|
||||
.set_len(1024 * 1024)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
format!("{}.vif", base0.display()),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
plant_ecx(&idx, collection, vid, &[0xA1u8; 20]);
|
||||
for (dir, run) in dirs[1..].iter().zip(sibling_runs) {
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards: 12,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: *run,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
fs::write(
|
||||
dir.join(format!("{}_{}.vif", collection, vid)),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store.store_dirs = dirs
|
||||
.iter()
|
||||
.map(|d| d.to_str().unwrap().to_string())
|
||||
.collect();
|
||||
for dir in &dirs {
|
||||
store
|
||||
.add_location(
|
||||
dir.to_str().unwrap(),
|
||||
idx.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let ecx = idx.join(format!("{}_{}.ecx", collection, vid));
|
||||
for path in [dat.as_str(), shard.as_str(), ecx.to_str().unwrap()] {
|
||||
assert!(Path::new(path).exists(), "{} removed at startup", path);
|
||||
}
|
||||
store
|
||||
}
|
||||
|
||||
/// A disk still holding the source `.dat` sizes its shards by the
|
||||
/// sibling's encode `.vif` before the startup check decides whether they
|
||||
/// are leftovers of an interrupted encode.
|
||||
#[test]
|
||||
fn shared_index_startup_validation_uses_sibling_vif_ratio() {
|
||||
let store = start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044]);
|
||||
let ec_vol = store.locations[0]
|
||||
.find_ec_volume(VolumeId(4124))
|
||||
.expect("dir0 ec volume mounted");
|
||||
assert!(ec_vol.has_shard(2), "shard 2 not mounted on dir0");
|
||||
}
|
||||
|
||||
/// Sibling `.vif` files naming different encode runs leave the ratio
|
||||
/// unknown, so the startup check keeps the shards.
|
||||
#[test]
|
||||
fn shared_index_startup_validation_keeps_shards_on_disagreeing_siblings() {
|
||||
start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044, 1_700_000_000_000_000_045]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_preserves_existing_destination_sidecars() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
|
||||
@@ -28,13 +28,11 @@ import (
|
||||
const filerGroup = "fuse-dlm-test"
|
||||
|
||||
// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
|
||||
// mounts for testing DLM-based cross-mount write coordination, and what two
|
||||
// mounts do without it.
|
||||
// mounts for testing DLM-based cross-mount write coordination.
|
||||
type dlmTestCluster struct {
|
||||
t testing.TB
|
||||
baseDir string
|
||||
weedBinary string
|
||||
dlm bool // the mounts run with -dlm
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
@@ -54,11 +52,6 @@ type dlmTestCluster struct {
|
||||
}
|
||||
|
||||
func startDLMTestCluster(t testing.TB) *dlmTestCluster {
|
||||
return startTestCluster(t, true)
|
||||
}
|
||||
|
||||
// startTestCluster starts the cluster; dlm says whether the mounts run with -dlm.
|
||||
func startTestCluster(t testing.TB, dlm bool) *dlmTestCluster {
|
||||
binary := findWeedBinary()
|
||||
if binary == "" {
|
||||
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
|
||||
@@ -71,7 +64,6 @@ func startTestCluster(t testing.TB, dlm bool) *dlmTestCluster {
|
||||
t: t,
|
||||
baseDir: baseDir,
|
||||
weedBinary: binary,
|
||||
dlm: dlm,
|
||||
}
|
||||
// Register cleanup early so processes are stopped even if a require fails below.
|
||||
t.Cleanup(c.Stop)
|
||||
@@ -232,20 +224,17 @@ func (c *dlmTestCluster) startMount(idx int, configDir string) error {
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return fmt.Errorf("create cache dir: %w", err)
|
||||
}
|
||||
args := []string{
|
||||
"-logdir=" + filepath.Join(c.baseDir, "logs"),
|
||||
c.mountCmds[idx] = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"mount",
|
||||
"-filer=" + c.filerAddress(0), // both mounts use filer0 for shared metadata
|
||||
"-dir=" + c.mountPoints[idx],
|
||||
"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
|
||||
"-dir="+c.mountPoints[idx],
|
||||
"-filer.path=/",
|
||||
"-dirAutoCreate",
|
||||
"-allowOthers=false",
|
||||
"-cacheDir=" + cacheDir,
|
||||
}
|
||||
if c.dlm {
|
||||
args = append(args, "-dlm")
|
||||
}
|
||||
c.mountCmds[idx] = exec.Command(c.weedBinary, args...)
|
||||
"-cacheDir="+cacheDir,
|
||||
"-dlm",
|
||||
)
|
||||
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
|
||||
}
|
||||
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// TestStageSameFilesWithoutDLM has two mounts without -dlm stage the same
|
||||
// files into one directory at the same time, the way two jobs extract the same
|
||||
// archive into a shared directory: a file that exists is skipped, otherwise it
|
||||
// is created, written and closed, then chmod'ed and given its mtime. Both
|
||||
// mounts often create the same file, and each then changes the attributes of
|
||||
// the copy it has cached.
|
||||
//
|
||||
// Whatever either mount is told, no entry may point at a chunk that is gone
|
||||
// from the volume server, and every file must read back whole. An update sent
|
||||
// from a cached copy after the other mount rewrote the file puts back the
|
||||
// chunks the rewrite dropped, which the filer deletes; reads of such a file
|
||||
// fail once the volume is vacuumed.
|
||||
func TestStageSameFilesWithoutDLM(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping FUSE integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startTestCluster(t, false)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
const (
|
||||
files = 400
|
||||
dir = "staged"
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
names := make([]string, files)
|
||||
content := make(map[string][]byte, files)
|
||||
for i := range names {
|
||||
names[i] = fmt.Sprintf("img-%04d.jpg", i)
|
||||
data := make([]byte, 1024+rng.Intn(63*1024))
|
||||
rng.Read(data)
|
||||
content[names[i]] = data
|
||||
}
|
||||
mtime := time.Date(2026, 10, 26, 4, 0, 0, 0, time.UTC)
|
||||
|
||||
// What the losing mount is told depends on the version (nothing, EIO,
|
||||
// EAGAIN); the files are what this test checks.
|
||||
failed := make([][]string, 2)
|
||||
setupErr := make([]error, 2)
|
||||
wrote := [2][]bool{make([]bool, files), make([]bool, files)}
|
||||
var ready, wg sync.WaitGroup
|
||||
ready.Add(2)
|
||||
for m := 0; m < 2; m++ {
|
||||
wg.Add(1)
|
||||
go func(m int) {
|
||||
defer wg.Done()
|
||||
stage := filepath.Join(cluster.mountPoints[m], dir)
|
||||
setupErr[m] = os.MkdirAll(stage, 0755)
|
||||
ready.Done()
|
||||
ready.Wait()
|
||||
if setupErr[m] != nil {
|
||||
return
|
||||
}
|
||||
for i, name := range names {
|
||||
p := filepath.Join(stage, name)
|
||||
if _, err := os.Lstat(p); err == nil {
|
||||
continue
|
||||
}
|
||||
wrote[m][i] = true
|
||||
if err := os.WriteFile(p, content[name], 0644); err != nil {
|
||||
failed[m] = append(failed[m], err.Error())
|
||||
continue
|
||||
}
|
||||
if err := os.Chmod(p, 0640); err != nil {
|
||||
failed[m] = append(failed[m], err.Error())
|
||||
continue
|
||||
}
|
||||
if err := os.Chtimes(p, mtime, mtime); err != nil {
|
||||
failed[m] = append(failed[m], err.Error())
|
||||
}
|
||||
}
|
||||
}(m)
|
||||
}
|
||||
wg.Wait()
|
||||
for m, f := range failed {
|
||||
require.NoError(t, setupErr[m], "mount%d", m)
|
||||
if len(f) > 0 {
|
||||
t.Logf("mount%d: %d calls failed, first: %s", m, len(f), f[0])
|
||||
}
|
||||
}
|
||||
both := 0
|
||||
for i := range names {
|
||||
if wrote[0][i] && wrote[1][i] {
|
||||
both++
|
||||
}
|
||||
}
|
||||
t.Logf("both mounts wrote %d of %d files", both, files)
|
||||
require.Positive(t, both, "the mounts never wrote the same file")
|
||||
|
||||
// The filer deletes the chunks an update dropped shortly after the update.
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
conn, err := grpc.NewClient(cluster.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
require.NoError(t, err)
|
||||
defer conn.Close()
|
||||
client := filer_pb.NewSeaweedFilerClient(conn)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
var broken []string
|
||||
for _, name := range names {
|
||||
if ctx.Err() != nil {
|
||||
broken = append(broken, "checks did not finish: "+ctx.Err().Error())
|
||||
break
|
||||
}
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: "/" + dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
broken = append(broken, fmt.Sprintf("%s: lookup: %v", name, err))
|
||||
continue
|
||||
}
|
||||
for _, chunk := range resp.Entry.GetChunks() {
|
||||
if !cluster.needleExists(ctx, t, chunk.GetFileIdString()) {
|
||||
broken = append(broken, fmt.Sprintf("%s: chunk %s is deleted", name, chunk.GetFileIdString()))
|
||||
}
|
||||
}
|
||||
if got, err := cluster.readFromFiler(ctx, "/"+dir+"/"+name); err != nil {
|
||||
broken = append(broken, fmt.Sprintf("%s: read: %v", name, err))
|
||||
} else if !bytes.Equal(got, content[name]) {
|
||||
broken = append(broken, fmt.Sprintf("%s: read %d bytes, want the %d written", name, len(got), len(content[name])))
|
||||
}
|
||||
}
|
||||
require.Empty(t, broken, "%d of %d staged files are broken", len(broken), files)
|
||||
}
|
||||
|
||||
// needleExists reports whether the volume server still has the needle; a
|
||||
// deleted one answers 404 (only readDeleted=true would serve it).
|
||||
func (c *dlmTestCluster) needleExists(ctx context.Context, t testing.TB, fid string) bool {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, fmt.Sprintf("http://127.0.0.1:%d/%s", c.volumePort, fid), nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode == http.StatusOK
|
||||
}
|
||||
|
||||
// readFromFiler reads a file whole through filer0's HTTP API.
|
||||
func (c *dlmTestCluster) readFromFiler(ctx context.Context, path string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d%s", c.filerPorts[0], path), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("status %s", resp.Status)
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
@@ -57,61 +57,3 @@ class VectorInstallerTest(unittest.TestCase):
|
||||
ARCHIVE=archive.as_posix(), DEST=dest.as_posix(), SYSTEM=system)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual((dest / name).read_bytes(), contents)
|
||||
|
||||
|
||||
class EnterpriseInstallerTest(unittest.TestCase):
|
||||
shell = VectorInstallerTest.shell
|
||||
|
||||
def enterprise(self, code, **env):
|
||||
return self.shell('ENTERPRISE=true; REPO=seaweedfs/artifactory; EDITION=-enterprise; ' + code, **env)
|
||||
|
||||
def test_asset_names(self):
|
||||
result = self.enterprise('go_asset_name linux amd64; go_asset_name windows amd64; '
|
||||
'rust_asset_name linux arm64; worker_asset_name linux amd64; '
|
||||
'vector_asset_name linux arm64; LARGE_DISK=true; '
|
||||
'go_asset_name darwin arm64; rust_asset_name linux amd64')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.split(), [
|
||||
'weed-enterprise-linux_amd64.tar.gz', 'weed-enterprise-windows_amd64.zip',
|
||||
'weed-volume-enterprise_linux_arm64.tar.gz', 'weed-worker-enterprise_linux_amd64.tar.gz',
|
||||
'weed-vector-enterprise_linux_arm64.tar.gz', 'weed-enterprise-darwin_arm64_large_disk.tar.gz',
|
||||
'weed-volume-enterprise_large_disk_linux_amd64.tar.gz',
|
||||
])
|
||||
|
||||
def test_open_source_go_asset_names(self):
|
||||
result = self.shell('go_asset_name linux amd64; go_asset_name windows amd64; '
|
||||
'LARGE_DISK=true; go_asset_name windows amd64')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.split(), [
|
||||
'linux_amd64.tar.gz', 'windows_amd64.zip', 'windows_amd64_large_disk.zip',
|
||||
])
|
||||
|
||||
def test_windows_weed_keeps_exe(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
directory = Path(directory)
|
||||
archive = directory / 'weed.zip'
|
||||
with zipfile.ZipFile(archive, 'w') as output:
|
||||
output.writestr('weed.exe', b'weed fixture')
|
||||
dest = directory / 'bin'
|
||||
result = self.shell('download() { cp "$ARCHIVE" "$2"; }; VERSION=test; '
|
||||
'INSTALL_DIR="$DEST"; install_component weed windows amd64',
|
||||
ARCHIVE=archive.as_posix(), DEST=dest.as_posix())
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual((dest / 'weed.exe').read_bytes(), b'weed fixture')
|
||||
|
||||
def test_rust_binaries_linux_only(self):
|
||||
for system, arch, supported in [
|
||||
('linux', 'amd64', True), ('linux', 'arm64', True),
|
||||
('darwin', 'arm64', False), ('windows', 'amd64', False),
|
||||
]:
|
||||
with self.subTest(system=system, arch=arch):
|
||||
for check in ['vector_platform_supported', 'rust_volume_platform_supported']:
|
||||
result = self.enterprise(f'{check} "$SYSTEM" "$ARCH"', SYSTEM=system, ARCH=arch)
|
||||
self.assertEqual(result.returncode == 0, supported, check)
|
||||
|
||||
def test_latest_version_skips_other_releases(self):
|
||||
tags = ['vfs-0.2.5', 'kvcache-filer-native-v0.1.0-preview.1', '4.54-rc1', '4.53', '4.52']
|
||||
listing = ''.join(f' "tag_name": "{tag}",\\n' for tag in tags)
|
||||
result = self.enterprise(f"curl() {{ printf '{listing}'; }}; get_latest_version")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.strip(), '4.53')
|
||||
@@ -104,9 +104,7 @@ for i in $(seq 1 60); do
|
||||
if mountpoint -q "${MOUNT_DIR}"; then
|
||||
break
|
||||
fi
|
||||
# Not kill -0: sudo briefly runs with a root real uid while starting, so
|
||||
# signalling it fails with EPERM even though it is alive.
|
||||
if ! ps -p "${mount_pid}" >/dev/null; then
|
||||
if ! kill -0 "${mount_pid}" 2>/dev/null; then
|
||||
echo "weed mount exited early; log tail:" >&2
|
||||
tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
|
||||
exit 1
|
||||
|
||||
@@ -239,7 +239,7 @@ The tests are automatically run in GitHub Actions on every push/PR that affects
|
||||
**Workflow**: `.github/workflows/s3-parquet-tests.yml`
|
||||
|
||||
**Test Matrix**:
|
||||
- Python versions: 3.10, 3.11, 3.12
|
||||
- Python versions: 3.9, 3.11, 3.12
|
||||
- PyArrow integration tests (s3fs): 20 test combinations
|
||||
- PyArrow native S3 tests: 6 test scenarios ✅
|
||||
- Cross-filesystem compatibility tests: 12 test scenarios ✅ **NEW**
|
||||
|
||||
@@ -256,17 +256,6 @@ var pluginConfigSections = []pluginConfigSection{
|
||||
"collection_filter": stringValue,
|
||||
},
|
||||
},
|
||||
{
|
||||
// s3_lifecycle has no worker config form; its knobs are admin values
|
||||
jobType: "s3_lifecycle",
|
||||
prefix: "maintenance.s3_lifecycle",
|
||||
adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
|
||||
"meta_log_retention_days": int64Value,
|
||||
"walker_interval_minutes": int64Value,
|
||||
"cluster_deletes_per_second": int64Value,
|
||||
"cluster_deletes_burst": int64Value,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func doubleValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
|
||||
|
||||
@@ -401,56 +401,3 @@ max_shard_transfers = 0
|
||||
t.Fatalf("max_shard_transfers = %d, want 0", got.GetInt64Value())
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPluginConfigFromTomlS3Lifecycle(t *testing.T) {
|
||||
s := newPluginServer(t)
|
||||
seed := &plugin_pb.PersistedJobTypeConfig{
|
||||
JobType: "s3_lifecycle",
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true, RetryLimit: 1},
|
||||
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
|
||||
"meta_log_retention_days": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
|
||||
},
|
||||
}
|
||||
if err := s.GetPlugin().SaveJobTypeConfig(seed); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
v := tomlConfig(t, `
|
||||
[maintenance.s3_lifecycle]
|
||||
meta_log_retention_days = 0
|
||||
walker_interval_minutes = 30
|
||||
cluster_deletes_per_second = 50
|
||||
cluster_deletes_burst = 200
|
||||
retry_limit = 3
|
||||
`)
|
||||
if err := s.ApplyPluginConfigFromToml(v); err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := s.GetPlugin().LoadJobTypeConfig("s3_lifecycle")
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("load: %v %v", cfg, err)
|
||||
}
|
||||
// an explicit 0 must replace the saved value, not read as unset
|
||||
if got, ok := cfg.AdminConfigValues["meta_log_retention_days"].GetKind().(*plugin_pb.ConfigValue_Int64Value); !ok || got.Int64Value != 0 {
|
||||
t.Errorf("meta_log_retention_days = %v, want int64 0", cfg.AdminConfigValues["meta_log_retention_days"])
|
||||
}
|
||||
if got := cfg.AdminConfigValues["walker_interval_minutes"].GetInt64Value(); got != 30 {
|
||||
t.Errorf("walker_interval_minutes = %d, want 30", got)
|
||||
}
|
||||
if got := cfg.AdminConfigValues["cluster_deletes_per_second"].GetInt64Value(); got != 50 {
|
||||
t.Errorf("cluster_deletes_per_second = %d, want 50", got)
|
||||
}
|
||||
if got := cfg.AdminConfigValues["cluster_deletes_burst"].GetInt64Value(); got != 200 {
|
||||
t.Errorf("cluster_deletes_burst = %d, want 200", got)
|
||||
}
|
||||
if cfg.AdminRuntime.RetryLimit != 3 {
|
||||
t.Errorf("retry_limit = %d, want 3", cfg.AdminRuntime.RetryLimit)
|
||||
}
|
||||
if !cfg.AdminRuntime.Enabled {
|
||||
t.Errorf("enabled flipped off by unrelated toml keys")
|
||||
}
|
||||
if cfg.UpdatedBy != "admin.toml" {
|
||||
t.Errorf("updated_by = %q, want admin.toml", cfg.UpdatedBy)
|
||||
}
|
||||
}
|
||||
@@ -1,192 +0,0 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/notification/webhook"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// NotificationWebhook is a stored webhook target as shown to the browser:
|
||||
// the bearer token never leaves the server.
|
||||
type NotificationWebhook struct {
|
||||
webhook.ManagedTarget
|
||||
HasBearerToken bool `json:"has_bearer_token"`
|
||||
}
|
||||
|
||||
type NotificationWebhooksData struct {
|
||||
Webhooks []NotificationWebhook `json:"webhooks"`
|
||||
ConfigPath string `json:"config_path"`
|
||||
}
|
||||
|
||||
// NotificationWebhookRequest creates or replaces a target. An empty
|
||||
// bearer_token keeps the stored one unless clear_bearer_token is set.
|
||||
type NotificationWebhookRequest struct {
|
||||
webhook.ManagedTarget
|
||||
ClearBearerToken bool `json:"clear_bearer_token"`
|
||||
}
|
||||
|
||||
var notificationWebhooksMu sync.Mutex
|
||||
|
||||
func (s *AdminServer) readNotificationWebhooks(ctx context.Context) (*webhook.ManagedConfig, error) {
|
||||
var conf *webhook.ManagedConfig
|
||||
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
var buf bytes.Buffer
|
||||
err := filer.ReadEntry(s.masterClient, client, filer.DirectoryEtcSeaweedFS, filer.NotificationWebhooksConfName, &buf)
|
||||
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
conf, err = webhook.ParseManagedConfig(buf.Bytes())
|
||||
return err
|
||||
})
|
||||
return conf, err
|
||||
}
|
||||
|
||||
func (s *AdminServer) saveNotificationWebhooks(ctx context.Context, conf *webhook.ManagedConfig) error {
|
||||
if err := conf.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
content, err := json.MarshalIndent(conf, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
return filer.SaveInsideFiler(ctx, client, filer.DirectoryEtcSeaweedFS, filer.NotificationWebhooksConfName, content)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetNotificationWebhooks(ctx context.Context) (*NotificationWebhooksData, error) {
|
||||
conf, err := s.readNotificationWebhooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data := &NotificationWebhooksData{
|
||||
Webhooks: make([]NotificationWebhook, 0, len(conf.Targets)),
|
||||
ConfigPath: filer.NotificationWebhooksConfPath,
|
||||
}
|
||||
for _, t := range conf.Targets {
|
||||
hasToken := t.BearerToken != ""
|
||||
t.BearerToken = ""
|
||||
data.Webhooks = append(data.Webhooks, NotificationWebhook{ManagedTarget: t, HasBearerToken: hasToken})
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetNotificationWebhooksAPI(w http.ResponseWriter, r *http.Request) {
|
||||
data, err := s.GetNotificationWebhooks(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) {
|
||||
s.putNotificationWebhook(w, r, "", true)
|
||||
}
|
||||
|
||||
func (s *AdminServer) UpdateNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) {
|
||||
s.putNotificationWebhook(w, r, mux.Vars(r)["name"], false)
|
||||
}
|
||||
|
||||
func (s *AdminServer) putNotificationWebhook(w http.ResponseWriter, r *http.Request, name string, create bool) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req NotificationWebhookRequest
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if create {
|
||||
name = req.Name
|
||||
} else if req.Name != "" && req.Name != name {
|
||||
writeJSONError(w, http.StatusBadRequest, "Webhook target name cannot be changed")
|
||||
return
|
||||
}
|
||||
target := req.ManagedTarget
|
||||
target.Name = name
|
||||
|
||||
notificationWebhooksMu.Lock()
|
||||
defer notificationWebhooksMu.Unlock()
|
||||
conf, err := s.readNotificationWebhooks(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
index := -1
|
||||
for i := range conf.Targets {
|
||||
if conf.Targets[i].Name == name {
|
||||
index = i
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case create && index >= 0:
|
||||
writeJSONError(w, http.StatusConflict, "Webhook target already exists")
|
||||
return
|
||||
case !create && index < 0:
|
||||
writeJSONError(w, http.StatusNotFound, "Webhook target not found")
|
||||
return
|
||||
}
|
||||
if index >= 0 && target.BearerToken == "" && !req.ClearBearerToken {
|
||||
target.BearerToken = conf.Targets[index].BearerToken
|
||||
}
|
||||
if err := target.Validate(); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if index >= 0 {
|
||||
conf.Targets[index] = target
|
||||
} else {
|
||||
conf.Targets = append(conf.Targets, target)
|
||||
}
|
||||
if err := s.saveNotificationWebhooks(r.Context(), conf); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to save webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
status := http.StatusOK
|
||||
if create {
|
||||
status = http.StatusCreated
|
||||
}
|
||||
hasToken := target.BearerToken != ""
|
||||
target.BearerToken = ""
|
||||
writeJSON(w, status, NotificationWebhook{ManagedTarget: target, HasBearerToken: hasToken})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
name := mux.Vars(r)["name"]
|
||||
|
||||
notificationWebhooksMu.Lock()
|
||||
defer notificationWebhooksMu.Unlock()
|
||||
conf, err := s.readNotificationWebhooks(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
kept := conf.Targets[:0]
|
||||
for _, t := range conf.Targets {
|
||||
if t.Name != name {
|
||||
kept = append(kept, t)
|
||||
}
|
||||
}
|
||||
if len(kept) == len(conf.Targets) {
|
||||
writeJSONError(w, http.StatusNotFound, "Webhook target not found")
|
||||
return
|
||||
}
|
||||
conf.Targets = kept
|
||||
if err := s.saveNotificationWebhooks(r.Context(), conf); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to save webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "Webhook target deleted"})
|
||||
}
|
||||
@@ -126,7 +126,6 @@ func (h *AdminHandlers) registerUIRoutes(r *mux.Router) {
|
||||
|
||||
// File browser routes
|
||||
r.HandleFunc("/files", h.fileBrowserHandlers.ShowFileBrowser).Methods(http.MethodGet)
|
||||
r.HandleFunc("/notifications/webhooks", h.ShowNotificationWebhooks).Methods(http.MethodGet)
|
||||
|
||||
// Cluster management routes
|
||||
r.HandleFunc("/cluster/masters", h.clusterHandlers.ShowClusterMasters).Methods(http.MethodGet)
|
||||
@@ -276,12 +275,6 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
|
||||
filesApi.HandleFunc("/metadata", h.fileBrowserHandlers.ExportMetadata).Methods(http.MethodGet)
|
||||
filesApi.HandleFunc("/list-folders", h.fileBrowserHandlers.ListFolders).Methods(http.MethodGet)
|
||||
|
||||
webhooksApi := api.PathPrefix("/notifications/webhooks").Subrouter()
|
||||
webhooksApi.HandleFunc("", h.adminServer.GetNotificationWebhooksAPI).Methods(http.MethodGet)
|
||||
webhooksApi.Handle("", wrapWrite(h.adminServer.CreateNotificationWebhookAPI)).Methods(http.MethodPost)
|
||||
webhooksApi.Handle("/{name}", wrapWrite(h.adminServer.UpdateNotificationWebhookAPI)).Methods(http.MethodPut)
|
||||
webhooksApi.Handle("/{name}", wrapWrite(h.adminServer.DeleteNotificationWebhookAPI)).Methods(http.MethodDelete)
|
||||
|
||||
volumeApi := api.PathPrefix("/volumes").Subrouter()
|
||||
volumeApi.HandleFunc("/export", h.clusterHandlers.ExportClusterVolumes).Methods(http.MethodGet)
|
||||
volumeApi.Handle("/{id}/{server}/vacuum", wrapWrite(h.clusterHandlers.VacuumVolume)).Methods(http.MethodPost)
|
||||
@@ -393,24 +386,6 @@ func (h *AdminHandlers) ShowS3TablesBuckets(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
// ShowNotificationWebhooks renders the filer webhook targets page
|
||||
func (h *AdminHandlers) ShowNotificationWebhooks(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
data, err := h.adminServer.GetNotificationWebhooks(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
component := app.NotificationWebhooks(*data)
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, component)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3VectorsBuckets renders the vector buckets page
|
||||
func (h *AdminHandlers) ShowS3VectorsBuckets(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
|
||||
@@ -62,20 +62,6 @@ func TestSetupRoutes_RegistersBucketLifecycleAPI_WithAuth(t *testing.T) {
|
||||
assertHasRoute(t, router, http.MethodDelete, "/api/s3/buckets/example/lifecycle")
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersNotificationWebhooks(t *testing.T) {
|
||||
for _, authRequired := range []bool{false, true} {
|
||||
router := mux.NewRouter()
|
||||
|
||||
newRouteTestAdminHandlers().SetupRoutes(router, authRequired, "admin", "password", "", "", true)
|
||||
|
||||
assertHasRoute(t, router, http.MethodGet, "/notifications/webhooks")
|
||||
assertHasRoute(t, router, http.MethodGet, "/api/notifications/webhooks")
|
||||
assertHasRoute(t, router, http.MethodPost, "/api/notifications/webhooks")
|
||||
assertHasRoute(t, router, http.MethodPut, "/api/notifications/webhooks/example")
|
||||
assertHasRoute(t, router, http.MethodDelete, "/api/notifications/webhooks/example")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersBucketPolicyAPI_NoAuth(t *testing.T) {
|
||||
router := mux.NewRouter()
|
||||
|
||||
|
||||
@@ -129,6 +129,21 @@ function setupSubmenuBehavior() {
|
||||
}
|
||||
}
|
||||
|
||||
// If we're on an object store page, expand the object store submenu
|
||||
if (currentPathStartsWith('/object-store/')) {
|
||||
const objectStoreSubmenu = document.getElementById('objectStoreSubmenu');
|
||||
if (objectStoreSubmenu) {
|
||||
objectStoreSubmenu.classList.add('show');
|
||||
|
||||
// Update the parent toggle button state
|
||||
const toggleButton = document.querySelector('[data-bs-target="#objectStoreSubmenu"]');
|
||||
if (toggleButton) {
|
||||
toggleButton.classList.remove('collapsed');
|
||||
toggleButton.setAttribute('aria-expanded', 'true');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prevent submenu from collapsing when clicking on submenu items
|
||||
const clusterSubmenuLinks = document.querySelectorAll('#clusterSubmenu .nav-link');
|
||||
clusterSubmenuLinks.forEach(function (link) {
|
||||
@@ -137,6 +152,61 @@ function setupSubmenuBehavior() {
|
||||
e.stopPropagation();
|
||||
});
|
||||
});
|
||||
|
||||
const objectStoreSubmenuLinks = document.querySelectorAll('#objectStoreSubmenu .nav-link');
|
||||
objectStoreSubmenuLinks.forEach(function (link) {
|
||||
link.addEventListener('click', function (e) {
|
||||
// Don't prevent the navigation, just stop the collapse behavior
|
||||
e.stopPropagation();
|
||||
});
|
||||
});
|
||||
|
||||
// Handle the main cluster toggle
|
||||
const clusterToggle = document.querySelector('[data-bs-target="#clusterSubmenu"]');
|
||||
if (clusterToggle) {
|
||||
clusterToggle.addEventListener('click', function (e) {
|
||||
e.preventDefault();
|
||||
|
||||
const submenu = document.getElementById('clusterSubmenu');
|
||||
const isExpanded = submenu.classList.contains('show');
|
||||
|
||||
if (isExpanded) {
|
||||
// Collapse
|
||||
submenu.classList.remove('show');
|
||||
this.classList.add('collapsed');
|
||||
this.setAttribute('aria-expanded', 'false');
|
||||
} else {
|
||||
// Expand
|
||||
submenu.classList.add('show');
|
||||
this.classList.remove('collapsed');
|
||||
this.setAttribute('aria-expanded', 'true');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Handle the main object store toggle
|
||||
const objectStoreToggle = document.querySelector('[data-bs-target="#objectStoreSubmenu"]');
|
||||
if (objectStoreToggle) {
|
||||
objectStoreToggle.addEventListener('click', function (e) {
|
||||
e.preventDefault();
|
||||
|
||||
const submenu = document.getElementById('objectStoreSubmenu');
|
||||
const isExpanded = submenu.classList.contains('show');
|
||||
|
||||
if (isExpanded) {
|
||||
// Collapse
|
||||
submenu.classList.remove('show');
|
||||
this.classList.add('collapsed');
|
||||
this.setAttribute('aria-expanded', 'false');
|
||||
} else {
|
||||
// Expand
|
||||
submenu.classList.add('show');
|
||||
this.classList.remove('collapsed');
|
||||
this.setAttribute('aria-expanded', 'true');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Mobile sidebar toggle and backdrop behavior
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
// Applies the saved color theme (auto, light or dark) before the page paints.
|
||||
(function () {
|
||||
var storageKey = 'admin-theme';
|
||||
var choices = ['auto', 'light', 'dark'];
|
||||
var media = window.matchMedia ? window.matchMedia('(prefers-color-scheme: dark)') : null;
|
||||
var root = document.documentElement;
|
||||
|
||||
function savedChoice() {
|
||||
try {
|
||||
var value = localStorage.getItem(storageKey);
|
||||
return choices.indexOf(value) >= 0 ? value : 'auto';
|
||||
} catch (e) {
|
||||
return 'auto';
|
||||
}
|
||||
}
|
||||
|
||||
function apply(choice) {
|
||||
var dark = choice === 'dark' || (choice === 'auto' && media && media.matches);
|
||||
root.setAttribute('data-bs-theme', dark ? 'dark' : 'light');
|
||||
root.setAttribute('data-theme-choice', choice);
|
||||
}
|
||||
|
||||
var current = savedChoice();
|
||||
apply(current);
|
||||
|
||||
if (media && media.addEventListener) {
|
||||
media.addEventListener('change', function () {
|
||||
apply(current);
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('click', function (event) {
|
||||
var toggle = event.target.closest && event.target.closest('[data-theme-toggle]');
|
||||
if (!toggle) {
|
||||
return;
|
||||
}
|
||||
current = choices[(choices.indexOf(current) + 1) % choices.length];
|
||||
try {
|
||||
localStorage.setItem(storageKey, current);
|
||||
} catch (e) {}
|
||||
apply(current);
|
||||
});
|
||||
})();
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,283 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
)
|
||||
|
||||
templ NotificationWebhooks(data dash.NotificationWebhooksData) {
|
||||
<div class="container-fluid">
|
||||
<div class="d-sm-flex align-items-center justify-content-between mb-4">
|
||||
<div>
|
||||
<h1 class="h3 mb-0 text-gray-800">
|
||||
<i class="bi bi-broadcast me-2"></i>Webhooks
|
||||
</h1>
|
||||
<p class="mb-0 text-muted">Send filer events to HTTP endpoints, filtered by path and event type</p>
|
||||
</div>
|
||||
<button type="button" class="btn btn-primary" onclick="openWebhookModal()">
|
||||
<i class="bi bi-plus me-1"></i>Add Webhook
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="alert alert-info">
|
||||
<i class="bi bi-info-circle me-1"></i>
|
||||
Targets are stored in the filer at <code>{ data.ConfigPath }</code> and apply to all filers without a restart.
|
||||
</div>
|
||||
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-body">
|
||||
if len(data.Webhooks) == 0 {
|
||||
<div class="text-center py-5 text-muted">
|
||||
<i class="bi bi-broadcast icon-3x mb-3"></i>
|
||||
<p>No webhook targets yet.</p>
|
||||
</div>
|
||||
} else {
|
||||
<div class="table-responsive">
|
||||
<table class="table table-bordered table-hover" id="webhooksTable" width="100%" cellspacing="0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Status</th>
|
||||
<th>Endpoint</th>
|
||||
<th>Events</th>
|
||||
<th>Path Prefixes</th>
|
||||
<th>Token</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, hook := range data.Webhooks {
|
||||
<tr>
|
||||
<td><strong>{ hook.Name }</strong></td>
|
||||
<td>
|
||||
if hook.Enabled {
|
||||
<span class="badge bg-success">Enabled</span>
|
||||
} else {
|
||||
<span class="badge bg-secondary">Disabled</span>
|
||||
}
|
||||
</td>
|
||||
<td><code>{ hook.Endpoint }</code></td>
|
||||
<td>{ webhookFilterLabel(hook.EventTypes) }</td>
|
||||
<td>{ webhookFilterLabel(hook.PathPrefixes) }</td>
|
||||
<td>
|
||||
if hook.HasBearerToken {
|
||||
<i class="bi bi-key text-success" title="Bearer token set"></i>
|
||||
} else {
|
||||
<span class="text-muted">-</span>
|
||||
}
|
||||
</td>
|
||||
<td>
|
||||
<div class="btn-group btn-group-sm">
|
||||
<button type="button" class="btn btn-outline-primary" title="Edit" data-name={ hook.Name } onclick="editWebhook(this.dataset.name)">
|
||||
<i class="bi bi-pencil"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-danger" title="Delete" data-name={ hook.Name } onclick="deleteWebhook(this.dataset.name)">
|
||||
<i class="bi bi-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="webhookModal" tabindex="-1" aria-labelledby="webhookModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="webhookModalLabel">Add Webhook</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<form id="webhookForm" onsubmit="saveWebhook(); return false;">
|
||||
<div class="row">
|
||||
<div class="col-md-4 mb-3">
|
||||
<label for="webhookName" class="form-label">Name</label>
|
||||
<input type="text" class="form-control" id="webhookName" required pattern="[A-Za-z0-9_\-]{1,64}" placeholder="ingest"/>
|
||||
</div>
|
||||
<div class="col-md-8 mb-3">
|
||||
<label for="webhookEndpoint" class="form-label">Endpoint</label>
|
||||
<input type="url" class="form-control" id="webhookEndpoint" required placeholder="https://example.com/hook"/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="webhookToken" class="form-label">Bearer Token</label>
|
||||
<input type="password" class="form-control" id="webhookToken" autocomplete="new-password"/>
|
||||
<div class="form-check mt-1 d-none" id="webhookClearTokenGroup">
|
||||
<input class="form-check-input" type="checkbox" id="webhookClearToken"/>
|
||||
<label class="form-check-label" for="webhookClearToken">Remove the stored token</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label class="form-label">Event Types</label>
|
||||
<div>
|
||||
for _, et := range []string{"create", "update", "delete", "rename"} {
|
||||
<div class="form-check form-check-inline">
|
||||
<input class="form-check-input webhook-event" type="checkbox" id={ "webhookEvent-" + et } value={ et }/>
|
||||
<label class="form-check-label" for={ "webhookEvent-" + et }>{ et }</label>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
<div class="form-text">None selected sends every event type.</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="webhookPaths" class="form-label">Path Prefixes</label>
|
||||
<textarea class="form-control" id="webhookPaths" rows="3" placeholder="/buckets/uploads"></textarea>
|
||||
<div class="form-text">One per line. Empty sends events for every path.</div>
|
||||
</div>
|
||||
<details class="mb-3">
|
||||
<summary>Delivery settings</summary>
|
||||
<div class="row mt-2">
|
||||
@webhookNumberInput("webhookTimeout", "Timeout (s)", "10")
|
||||
@webhookNumberInput("webhookRetries", "Max Retries", "3")
|
||||
@webhookNumberInput("webhookBackoff", "Backoff (s)", "3")
|
||||
@webhookNumberInput("webhookMaxBackoff", "Max Backoff (s)", "30")
|
||||
@webhookNumberInput("webhookWorkers", "Workers", "5")
|
||||
@webhookNumberInput("webhookBuffer", "Buffer Size", "10000")
|
||||
</div>
|
||||
</details>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" id="webhookEnabled" checked/>
|
||||
<label class="form-check-label" for="webhookEnabled">Enabled</label>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary" form="webhookForm">Save</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
let editingWebhook = '';
|
||||
|
||||
function webhookRequest(path, method, body) {
|
||||
const headers = { 'Content-Type': 'application/json' };
|
||||
const csrf = document.querySelector('meta[name="csrf-token"]');
|
||||
if (csrf && csrf.content) {
|
||||
headers['X-CSRF-Token'] = csrf.content;
|
||||
}
|
||||
return fetch(basePath('/api/notifications/webhooks' + path), {
|
||||
method: method,
|
||||
headers: headers,
|
||||
body: body ? JSON.stringify(body) : undefined
|
||||
});
|
||||
}
|
||||
|
||||
async function webhookResult(response, okMessage) {
|
||||
if (response.ok) {
|
||||
showAlert(okMessage, 'success');
|
||||
setTimeout(() => window.location.reload(), 800);
|
||||
return;
|
||||
}
|
||||
const error = await response.json().catch(() => ({}));
|
||||
showAlert(error.error || ('Request failed: ' + response.status), 'error');
|
||||
}
|
||||
|
||||
function fillWebhookForm(hook) {
|
||||
document.getElementById('webhookName').value = hook.name || '';
|
||||
document.getElementById('webhookName').disabled = !!editingWebhook;
|
||||
document.getElementById('webhookEndpoint').value = hook.endpoint || '';
|
||||
document.getElementById('webhookToken').value = '';
|
||||
document.getElementById('webhookToken').placeholder = hook.has_bearer_token ? 'Leave empty to keep the stored token' : '';
|
||||
document.getElementById('webhookClearToken').checked = false;
|
||||
document.getElementById('webhookClearTokenGroup').classList.toggle('d-none', !hook.has_bearer_token);
|
||||
document.querySelectorAll('.webhook-event').forEach(cb => {
|
||||
cb.checked = (hook.event_types || []).includes(cb.value);
|
||||
});
|
||||
document.getElementById('webhookPaths').value = (hook.path_prefixes || []).join('\n');
|
||||
document.getElementById('webhookTimeout').value = hook.timeout_seconds || '';
|
||||
document.getElementById('webhookRetries').value = hook.max_retries || '';
|
||||
document.getElementById('webhookBackoff').value = hook.backoff_seconds || '';
|
||||
document.getElementById('webhookMaxBackoff').value = hook.max_backoff_seconds || '';
|
||||
document.getElementById('webhookWorkers').value = hook.workers || '';
|
||||
document.getElementById('webhookBuffer').value = hook.buffer_size || '';
|
||||
document.getElementById('webhookEnabled').checked = hook.enabled !== false;
|
||||
document.getElementById('webhookModalLabel').textContent = editingWebhook ? 'Edit Webhook: ' + editingWebhook : 'Add Webhook';
|
||||
bootstrap.Modal.getOrCreateInstance(document.getElementById('webhookModal')).show();
|
||||
}
|
||||
|
||||
function openWebhookModal() {
|
||||
editingWebhook = '';
|
||||
fillWebhookForm({ enabled: true });
|
||||
}
|
||||
|
||||
async function editWebhook(name) {
|
||||
try {
|
||||
const response = await webhookRequest('', 'GET');
|
||||
const data = await response.json();
|
||||
const hook = (data.webhooks || []).find(h => h.name === name);
|
||||
if (!hook) {
|
||||
showAlert('Webhook target not found', 'error');
|
||||
return;
|
||||
}
|
||||
editingWebhook = name;
|
||||
fillWebhookForm(hook);
|
||||
} catch (error) {
|
||||
showAlert('Failed to load webhook target: ' + error.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
function webhookNumber(id) {
|
||||
const value = parseInt(document.getElementById(id).value, 10);
|
||||
return isNaN(value) ? 0 : value;
|
||||
}
|
||||
|
||||
async function saveWebhook() {
|
||||
const body = {
|
||||
name: document.getElementById('webhookName').value.trim(),
|
||||
enabled: document.getElementById('webhookEnabled').checked,
|
||||
endpoint: document.getElementById('webhookEndpoint').value.trim(),
|
||||
bearer_token: document.getElementById('webhookToken').value,
|
||||
clear_bearer_token: document.getElementById('webhookClearToken').checked,
|
||||
event_types: Array.from(document.querySelectorAll('.webhook-event:checked')).map(cb => cb.value),
|
||||
path_prefixes: document.getElementById('webhookPaths').value.split('\n').map(p => p.trim()).filter(p => p),
|
||||
timeout_seconds: webhookNumber('webhookTimeout'),
|
||||
max_retries: webhookNumber('webhookRetries'),
|
||||
backoff_seconds: webhookNumber('webhookBackoff'),
|
||||
max_backoff_seconds: webhookNumber('webhookMaxBackoff'),
|
||||
workers: webhookNumber('webhookWorkers'),
|
||||
buffer_size: webhookNumber('webhookBuffer')
|
||||
};
|
||||
try {
|
||||
const response = editingWebhook
|
||||
? await webhookRequest('/' + encodeURIComponent(editingWebhook), 'PUT', body)
|
||||
: await webhookRequest('', 'POST', body);
|
||||
await webhookResult(response, 'Webhook target saved');
|
||||
} catch (error) {
|
||||
showAlert('Failed to save webhook target: ' + error.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
function deleteWebhook(name) {
|
||||
showDeleteConfirm(name, async function() {
|
||||
try {
|
||||
await webhookResult(await webhookRequest('/' + encodeURIComponent(name), 'DELETE'), 'Webhook target deleted');
|
||||
} catch (error) {
|
||||
showAlert('Failed to delete webhook target: ' + error.message, 'error');
|
||||
}
|
||||
}, 'Are you sure you want to delete this webhook target?');
|
||||
}
|
||||
</script>
|
||||
}
|
||||
|
||||
templ webhookNumberInput(id string, label string, placeholder string) {
|
||||
<div class="col-md-4 mb-2">
|
||||
<label for={ id } class="form-label">{ label }</label>
|
||||
<input type="number" min="0" class="form-control" id={ id } placeholder={ placeholder }/>
|
||||
</div>
|
||||
}
|
||||
|
||||
func webhookFilterLabel(values []string) string {
|
||||
if len(values) == 0 {
|
||||
return "all"
|
||||
}
|
||||
return strings.Join(values, ", ")
|
||||
}
|
||||
@@ -1,362 +0,0 @@
|
||||
// Code generated by templ - DO NOT EDIT.
|
||||
|
||||
// templ: version: v0.3.1100
|
||||
|
||||
package app
|
||||
|
||||
//lint:file-ignore SA4006 This context is only used if a nested component is present.
|
||||
|
||||
import "github.com/a-h/templ"
|
||||
import templruntime "github.com/a-h/templ/runtime"
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func NotificationWebhooks(data dash.NotificationWebhooksData) templ.Component {
|
||||
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||
return templ_7745c5c3_CtxErr
|
||||
}
|
||||
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||
if !templ_7745c5c3_IsBuffer {
|
||||
defer func() {
|
||||
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err == nil {
|
||||
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||
}
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"container-fluid\"><div class=\"d-sm-flex align-items-center justify-content-between mb-4\"><div><h1 class=\"h3 mb-0 text-gray-800\"><i class=\"bi bi-broadcast me-2\"></i>Webhooks</h1><p class=\"mb-0 text-muted\">Send filer events to HTTP endpoints, filtered by path and event type</p></div><button type=\"button\" class=\"btn btn-primary\" onclick=\"openWebhookModal()\"><i class=\"bi bi-plus me-1\"></i>Add Webhook</button></div><div class=\"alert alert-info\"><i class=\"bi bi-info-circle me-1\"></i> Targets are stored in the filer at <code>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var2 string
|
||||
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.ConfigPath)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 24, Col: 70}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</code> and apply to all filers without a restart.</div><div class=\"card shadow mb-4\"><div class=\"card-body\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if len(data.Webhooks) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<div class=\"text-center py-5 text-muted\"><i class=\"bi bi-broadcast icon-3x mb-3\"></i><p>No webhook targets yet.</p></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "<div class=\"table-responsive\"><table class=\"table table-bordered table-hover\" id=\"webhooksTable\" width=\"100%\" cellspacing=\"0\"><thead><tr><th>Name</th><th>Status</th><th>Endpoint</th><th>Events</th><th>Path Prefixes</th><th>Token</th><th>Actions</th></tr></thead> <tbody>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, hook := range data.Webhooks {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "<tr><td><strong>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var3 string
|
||||
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(hook.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 51, Col: 63}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</strong></td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if hook.Enabled {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<span class=\"badge bg-success\">Enabled</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "<span class=\"badge bg-secondary\">Disabled</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</td><td><code>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var4 string
|
||||
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(hook.Endpoint)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 59, Col: 65}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</code></td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var5 string
|
||||
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(webhookFilterLabel(hook.EventTypes))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 60, Col: 81}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var6 string
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(webhookFilterLabel(hook.PathPrefixes))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 61, Col: 83}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if hook.HasBearerToken {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "<i class=\"bi bi-key text-success\" title=\"Bearer token set\"></i>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "<span class=\"text-muted\">-</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "</td><td><div class=\"btn-group btn-group-sm\"><button type=\"button\" class=\"btn btn-outline-primary\" title=\"Edit\" data-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var7 string
|
||||
templ_7745c5c3_Var7, templ_7745c5c3_Err = templruntime.NewAttributeExpression(hook.Name).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 71, Col: 136}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var7)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\" onclick=\"editWebhook(this.dataset.name)\"><i class=\"bi bi-pencil\"></i></button> <button type=\"button\" class=\"btn btn-outline-danger\" title=\"Delete\" data-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var8 string
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templruntime.NewAttributeExpression(hook.Name).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 74, Col: 137}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var8)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "\" onclick=\"deleteWebhook(this.dataset.name)\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</tbody></table></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</div></div></div><div class=\"modal fade\" id=\"webhookModal\" tabindex=\"-1\" aria-labelledby=\"webhookModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"webhookModalLabel\">Add Webhook</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><form id=\"webhookForm\" onsubmit=\"saveWebhook(); return false;\"><div class=\"row\"><div class=\"col-md-4 mb-3\"><label for=\"webhookName\" class=\"form-label\">Name</label> <input type=\"text\" class=\"form-control\" id=\"webhookName\" required pattern=\"[A-Za-z0-9_\\-]{1,64}\" placeholder=\"ingest\"></div><div class=\"col-md-8 mb-3\"><label for=\"webhookEndpoint\" class=\"form-label\">Endpoint</label> <input type=\"url\" class=\"form-control\" id=\"webhookEndpoint\" required placeholder=\"https://example.com/hook\"></div></div><div class=\"mb-3\"><label for=\"webhookToken\" class=\"form-label\">Bearer Token</label> <input type=\"password\" class=\"form-control\" id=\"webhookToken\" autocomplete=\"new-password\"><div class=\"form-check mt-1 d-none\" id=\"webhookClearTokenGroup\"><input class=\"form-check-input\" type=\"checkbox\" id=\"webhookClearToken\"> <label class=\"form-check-label\" for=\"webhookClearToken\">Remove the stored token</label></div></div><div class=\"mb-3\"><label class=\"form-label\">Event Types</label><div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, et := range []string{"create", "update", "delete", "rename"} {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "<div class=\"form-check form-check-inline\"><input class=\"form-check-input webhook-event\" type=\"checkbox\" id=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var9 string
|
||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templruntime.NewAttributeExpression("webhookEvent-" + et).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 121, Col: 127}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var9)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var10 string
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templruntime.NewAttributeExpression(et).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 121, Col: 140}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var10)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "\"> <label class=\"form-check-label\" for=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var11 string
|
||||
templ_7745c5c3_Var11, templ_7745c5c3_Err = templruntime.NewAttributeExpression("webhookEvent-" + et).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 122, Col: 98}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var11)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var12 string
|
||||
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(et)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 122, Col: 105}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</label></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "</div><div class=\"form-text\">None selected sends every event type.</div></div><div class=\"mb-3\"><label for=\"webhookPaths\" class=\"form-label\">Path Prefixes</label> <textarea class=\"form-control\" id=\"webhookPaths\" rows=\"3\" placeholder=\"/buckets/uploads\"></textarea><div class=\"form-text\">One per line. Empty sends events for every path.</div></div><details class=\"mb-3\"><summary>Delivery settings</summary><div class=\"row mt-2\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = webhookNumberInput("webhookTimeout", "Timeout (s)", "10").Render(ctx, templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = webhookNumberInput("webhookRetries", "Max Retries", "3").Render(ctx, templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = webhookNumberInput("webhookBackoff", "Backoff (s)", "3").Render(ctx, templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = webhookNumberInput("webhookMaxBackoff", "Max Backoff (s)", "30").Render(ctx, templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = webhookNumberInput("webhookWorkers", "Workers", "5").Render(ctx, templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = webhookNumberInput("webhookBuffer", "Buffer Size", "10000").Render(ctx, templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "</div></details><div class=\"form-check\"><input class=\"form-check-input\" type=\"checkbox\" id=\"webhookEnabled\" checked> <label class=\"form-check-label\" for=\"webhookEnabled\">Enabled</label></div></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\" form=\"webhookForm\">Save</button></div></div></div></div><script>\n let editingWebhook = '';\n\n function webhookRequest(path, method, body) {\n const headers = { 'Content-Type': 'application/json' };\n const csrf = document.querySelector('meta[name=\"csrf-token\"]');\n if (csrf && csrf.content) {\n headers['X-CSRF-Token'] = csrf.content;\n }\n return fetch(basePath('/api/notifications/webhooks' + path), {\n method: method,\n headers: headers,\n body: body ? JSON.stringify(body) : undefined\n });\n }\n\n async function webhookResult(response, okMessage) {\n if (response.ok) {\n showAlert(okMessage, 'success');\n setTimeout(() => window.location.reload(), 800);\n return;\n }\n const error = await response.json().catch(() => ({}));\n showAlert(error.error || ('Request failed: ' + response.status), 'error');\n }\n\n function fillWebhookForm(hook) {\n document.getElementById('webhookName').value = hook.name || '';\n document.getElementById('webhookName').disabled = !!editingWebhook;\n document.getElementById('webhookEndpoint').value = hook.endpoint || '';\n document.getElementById('webhookToken').value = '';\n document.getElementById('webhookToken').placeholder = hook.has_bearer_token ? 'Leave empty to keep the stored token' : '';\n document.getElementById('webhookClearToken').checked = false;\n document.getElementById('webhookClearTokenGroup').classList.toggle('d-none', !hook.has_bearer_token);\n document.querySelectorAll('.webhook-event').forEach(cb => {\n cb.checked = (hook.event_types || []).includes(cb.value);\n });\n document.getElementById('webhookPaths').value = (hook.path_prefixes || []).join('\\n');\n document.getElementById('webhookTimeout').value = hook.timeout_seconds || '';\n document.getElementById('webhookRetries').value = hook.max_retries || '';\n document.getElementById('webhookBackoff').value = hook.backoff_seconds || '';\n document.getElementById('webhookMaxBackoff').value = hook.max_backoff_seconds || '';\n document.getElementById('webhookWorkers').value = hook.workers || '';\n document.getElementById('webhookBuffer').value = hook.buffer_size || '';\n document.getElementById('webhookEnabled').checked = hook.enabled !== false;\n document.getElementById('webhookModalLabel').textContent = editingWebhook ? 'Edit Webhook: ' + editingWebhook : 'Add Webhook';\n bootstrap.Modal.getOrCreateInstance(document.getElementById('webhookModal')).show();\n }\n\n function openWebhookModal() {\n editingWebhook = '';\n fillWebhookForm({ enabled: true });\n }\n\n async function editWebhook(name) {\n try {\n const response = await webhookRequest('', 'GET');\n const data = await response.json();\n const hook = (data.webhooks || []).find(h => h.name === name);\n if (!hook) {\n showAlert('Webhook target not found', 'error');\n return;\n }\n editingWebhook = name;\n fillWebhookForm(hook);\n } catch (error) {\n showAlert('Failed to load webhook target: ' + error.message, 'error');\n }\n }\n\n function webhookNumber(id) {\n const value = parseInt(document.getElementById(id).value, 10);\n return isNaN(value) ? 0 : value;\n }\n\n async function saveWebhook() {\n const body = {\n name: document.getElementById('webhookName').value.trim(),\n enabled: document.getElementById('webhookEnabled').checked,\n endpoint: document.getElementById('webhookEndpoint').value.trim(),\n bearer_token: document.getElementById('webhookToken').value,\n clear_bearer_token: document.getElementById('webhookClearToken').checked,\n event_types: Array.from(document.querySelectorAll('.webhook-event:checked')).map(cb => cb.value),\n path_prefixes: document.getElementById('webhookPaths').value.split('\\n').map(p => p.trim())Line truncated
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func webhookNumberInput(id string, label string, placeholder string) templ.Component {
|
||||
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||
return templ_7745c5c3_CtxErr
|
||||
}
|
||||
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||
if !templ_7745c5c3_IsBuffer {
|
||||
defer func() {
|
||||
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err == nil {
|
||||
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||
}
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "<div class=\"col-md-4 mb-2\"><label for=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var14 string
|
||||
templ_7745c5c3_Var14, templ_7745c5c3_Err = templruntime.NewAttributeExpression(id).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 273, Col: 23}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var14)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "\" class=\"form-label\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var15 string
|
||||
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(label)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 273, Col: 52}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</label> <input type=\"number\" min=\"0\" class=\"form-control\" id=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var16 string
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templruntime.NewAttributeExpression(id).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 274, Col: 65}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var16)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "\" placeholder=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var17 string
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templruntime.NewAttributeExpression(placeholder).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 274, Col: 93}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var17)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "\"></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func webhookFilterLabel(values []string) string {
|
||||
if len(values) == 0 {
|
||||
return "all"
|
||||
}
|
||||
return strings.Join(values, ", ")
|
||||
}
|
||||
|
||||
var _ = templruntime.GeneratedTemplate
|
||||
@@ -12,56 +12,6 @@ script setBasePath(prefix string) {
|
||||
window.__BASE_PATH__ = prefix;
|
||||
}
|
||||
|
||||
// themeScript applies the saved color theme (auto, light or dark) inline, so
|
||||
// the root is themed before any external script or stylesheet is fetched.
|
||||
templ themeScript() {
|
||||
<script>
|
||||
(function () {
|
||||
var storageKey = 'admin-theme';
|
||||
var choices = ['auto', 'light', 'dark'];
|
||||
var media = window.matchMedia ? window.matchMedia('(prefers-color-scheme: dark)') : null;
|
||||
var root = document.documentElement;
|
||||
|
||||
function savedChoice() {
|
||||
try {
|
||||
var value = localStorage.getItem(storageKey);
|
||||
return choices.indexOf(value) >= 0 ? value : 'auto';
|
||||
} catch (e) {
|
||||
return 'auto';
|
||||
}
|
||||
}
|
||||
|
||||
function apply(choice) {
|
||||
var dark = choice === 'dark' || (choice === 'auto' && media && media.matches);
|
||||
root.setAttribute('data-bs-theme', dark ? 'dark' : 'light');
|
||||
root.setAttribute('data-theme-choice', choice);
|
||||
root.style.colorScheme = dark ? 'dark' : 'light';
|
||||
}
|
||||
|
||||
var current = savedChoice();
|
||||
apply(current);
|
||||
|
||||
if (media && media.addEventListener) {
|
||||
media.addEventListener('change', function () {
|
||||
apply(current);
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('click', function (event) {
|
||||
var toggle = event.target.closest && event.target.closest('[data-theme-toggle]');
|
||||
if (!toggle) {
|
||||
return;
|
||||
}
|
||||
current = choices[(choices.indexOf(current) + 1) % choices.length];
|
||||
try {
|
||||
localStorage.setItem(storageKey, current);
|
||||
} catch (e) {}
|
||||
apply(current);
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
}
|
||||
|
||||
templ Layout(view ViewContext, content templ.Component) {
|
||||
{{
|
||||
username := view.Username
|
||||
@@ -105,7 +55,7 @@ templ Layout(view ViewContext, content templ.Component) {
|
||||
<meta name="csrf-token" content={ csrfToken }>
|
||||
<link rel="icon" href={ string(view.P("/static/favicon.ico")) } type="image/x-icon">
|
||||
|
||||
@themeScript()
|
||||
<script src={ string(view.P("/static/js/theme.js")) }></script>
|
||||
<!-- Bootstrap CSS -->
|
||||
<link href={ string(view.P("/static/css/bootstrap.min.css")) } rel="stylesheet">
|
||||
<!-- Bootstrap Icons CSS -->
|
||||
@@ -288,11 +238,6 @@ templ Layout(view ViewContext, content templ.Component) {
|
||||
<i class="bi bi-folder me-2"></i>File Browser
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href={ view.P("/notifications/webhooks") }>
|
||||
<i class="bi bi-broadcast me-2"></i>Webhooks
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
if isMQPage {
|
||||
<a class="nav-link" href="#" data-bs-toggle="collapse" data-bs-target="#messageQueueSubmenu" aria-expanded="true" aria-controls="messageQueueSubmenu">
|
||||
@@ -459,7 +404,7 @@ templ LoginForm(title string, errorMessage string, csrfToken string) {
|
||||
<title>{title} - Login</title>
|
||||
<link rel="icon" href={ prefix + "/static/favicon.ico" } type="image/x-icon">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
@themeScript()
|
||||
<script src={ prefix + "/static/js/theme.js" }></script>
|
||||
<link href={ prefix + "/static/css/bootstrap.min.css" } rel="stylesheet">
|
||||
<link href={ prefix + "/static/css/bootstrap-icons.min.css" } rel="stylesheet">
|
||||
<link href={ prefix + "/static/css/admin.css" } rel="stylesheet">
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -54,8 +54,6 @@ const allowUntrustedRemoteEndpointsUsage = "if true, a read of a remote-only ent
|
||||
// that runs the embedded IAM API.
|
||||
const iamSelfServiceRequiresGrantUsage = "if true, an identity needs an explicit iam: grant to manage its own access keys (CreateAccessKey, DeleteAccessKey, UpdateAccessKey, ListAccessKeys, GetUser), as on AWS. Default allows it to every authenticated identity."
|
||||
|
||||
const s3PreStopSecondsUsage = "seconds to keep serving after SIGTERM, with /status, /healthz and /readyz returning 503, before the S3 gateway shuts down. Gives load balancers and Kubernetes Services time to stop routing to it. 0 shuts down at once"
|
||||
|
||||
// allowBackslashInKeyUsage documents the S3 flag keeping '\' a literal
|
||||
// object-key character instead of normalizing it to '/'. Repeated '//' in a
|
||||
// key still collapses: a filer entry name cannot contain an empty segment.
|
||||
@@ -200,7 +198,6 @@ func init() {
|
||||
filerS3Options.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
filerS3Options.allowBackslashInKey = cmdFiler.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
|
||||
filerS3Options.iamSelfServiceRequiresGrant = cmdFiler.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
filerS3Options.preStopSeconds = cmdFiler.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage)
|
||||
|
||||
// start webdav on filer
|
||||
filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// grpcLocalSocket is a service name and the resolved gRPC port it listens on.
|
||||
type grpcLocalSocket struct {
|
||||
name string
|
||||
port int
|
||||
}
|
||||
|
||||
// grpcLocalSocketPath returns the Unix socket path for local RPCs to a gRPC
|
||||
// service, inside dir.
|
||||
func grpcLocalSocketPath(dir, name string, grpcPort int) string {
|
||||
return filepath.Join(dir, fmt.Sprintf("seaweedfs-%s-grpc-%d.sock", name, grpcPort))
|
||||
}
|
||||
|
||||
// grpcLocalSocketPathLimit returns the longest Unix socket path the platform
|
||||
// accepts: sockaddr_un's sun_path less the terminating NUL.
|
||||
func grpcLocalSocketPathLimit() int {
|
||||
return len(syscall.RawSockaddrUnix{}.Path) - 1
|
||||
}
|
||||
|
||||
// checkGrpcLocalSocketDir rejects a socket folder that does not exist, is not
|
||||
// a directory, is too long for one of its socket paths, or refuses a socket
|
||||
// bind, since local RPCs keep dialing a registered socket that was never bound.
|
||||
//
|
||||
// The folder is unused, and not checked, on Windows (see
|
||||
// pb.RegisterLocalGrpcSocket) or when no service registers a socket.
|
||||
func checkGrpcLocalSocketDir(dir string, sockets []grpcLocalSocket) error {
|
||||
if runtime.GOOS == "windows" || len(sockets) == 0 {
|
||||
return nil
|
||||
}
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return errors.New("not a directory")
|
||||
}
|
||||
limit := grpcLocalSocketPathLimit()
|
||||
for _, socket := range sockets {
|
||||
path := grpcLocalSocketPath(dir, socket.name, socket.port)
|
||||
if len(path) > limit {
|
||||
return fmt.Errorf("socket path %q is %d bytes and does not fit the platform's %d-byte Unix socket path limit; use a shorter -grpc.localSocketDir", path, len(path), limit)
|
||||
}
|
||||
}
|
||||
return probeGrpcLocalSocketDir(dir)
|
||||
}
|
||||
|
||||
// probeGrpcLocalSocketDir binds and removes a randomly named Unix socket in
|
||||
// dir, so a file another user left in a shared folder cannot block startup.
|
||||
// The 23-byte name is shorter than any service socket name.
|
||||
func probeGrpcLocalSocketDir(dir string) error {
|
||||
const attempts = 5
|
||||
var lastErr error
|
||||
for i := 0; i < attempts; i++ {
|
||||
suffix := make([]byte, 4)
|
||||
if _, err := rand.Read(suffix); err != nil {
|
||||
return fmt.Errorf("cannot pick a Unix socket probe name: %v", err)
|
||||
}
|
||||
probePath := filepath.Join(dir, fmt.Sprintf("seaweedfs-%x.sock", suffix))
|
||||
listener, err := net.Listen("unix", probePath)
|
||||
if errors.Is(err, syscall.EADDRINUSE) {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create a Unix socket in it: %v; give the server's user write access or choose another -grpc.localSocketDir", err)
|
||||
}
|
||||
if err := listener.Close(); err != nil {
|
||||
return fmt.Errorf("cannot close the Unix socket probe %q: %v", probePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("cannot create a Unix socket in it: %d random probe names were all taken, last: %v", attempts, lastErr)
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestGrpcLocalSocketPath checks that -grpc.localSocketDir defaults to /tmp
|
||||
// for `weed server` and `weed mini` and that every role's socket lands in it.
|
||||
func TestGrpcLocalSocketPath(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
command string
|
||||
dir *string
|
||||
}{
|
||||
{"weed server", serverGrpcLocalSocketDir},
|
||||
{"weed mini", miniGrpcLocalSocketDir},
|
||||
} {
|
||||
t.Run(tc.command, func(t *testing.T) {
|
||||
if *tc.dir != "/tmp" {
|
||||
t.Fatalf("-grpc.localSocketDir default = %q, want /tmp", *tc.dir)
|
||||
}
|
||||
if got, want := grpcLocalSocketPath(*tc.dir, "volume", 17334), "/tmp/seaweedfs-volume-grpc-17334.sock"; got != want {
|
||||
t.Fatalf("default socket path = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
prev := *tc.dir
|
||||
*tc.dir = dir
|
||||
t.Cleanup(func() { *tc.dir = prev })
|
||||
|
||||
for _, role := range []string{"master", "volume", "filer", "s3", "admin"} {
|
||||
path := grpcLocalSocketPath(*tc.dir, role, 17334)
|
||||
if got := filepath.Dir(path); got != dir {
|
||||
t.Fatalf("%s socket path %q is not in %q", role, path, dir)
|
||||
}
|
||||
if got, want := filepath.Base(path), "seaweedfs-"+role+"-grpc-17334.sock"; got != want {
|
||||
t.Fatalf("%s socket file name = %q, want %q", role, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrpcLocalSocketDir checks that -grpc.localSocketDir is refused when one
|
||||
// of its socket paths cannot fit in the platform's sockaddr_un.
|
||||
func TestGrpcLocalSocketDir(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
|
||||
sockets := []grpcLocalSocket{{"volume", 17334}}
|
||||
limit := grpcLocalSocketPathLimit()
|
||||
|
||||
// /tmp is the historical default and short enough on every Unix for any
|
||||
// socket name; t.TempDir() is not, since macOS roots it under a long
|
||||
// /var/folders path.
|
||||
if err := checkGrpcLocalSocketDir("/tmp", sockets); err != nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir(%q): %v", "/tmp", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
for len(grpcLocalSocketPath(dir, "volume", 17334)) <= limit {
|
||||
dir = filepath.Join(dir, "abcdefghijklmnop")
|
||||
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||
t.Fatalf("create %q: %v", dir, err)
|
||||
}
|
||||
}
|
||||
err := checkGrpcLocalSocketDir(dir, sockets)
|
||||
if err == nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir accepted %q, whose socket path is longer than the %d-byte limit", dir, limit)
|
||||
}
|
||||
if path := grpcLocalSocketPath(dir, "volume", 17334); !strings.Contains(err.Error(), path) {
|
||||
t.Fatalf("error %q does not name the socket path %q", err, path)
|
||||
}
|
||||
if !strings.Contains(err.Error(), strconv.Itoa(limit)) {
|
||||
t.Fatalf("error %q does not name the %d-byte limit", err, limit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrpcLocalSocketDirAtLimit checks that a folder whose longest socket
|
||||
// path is exactly the platform limit is accepted and that the socket binds.
|
||||
func TestGrpcLocalSocketDirAtLimit(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
|
||||
base, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-limit-")
|
||||
if err != nil {
|
||||
t.Fatalf("create folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.RemoveAll(base); err != nil {
|
||||
t.Errorf("remove %q: %v", base, err)
|
||||
}
|
||||
})
|
||||
|
||||
sockets := []grpcLocalSocket{{"s3", 18333}, {"volume", 18080}}
|
||||
limit := grpcLocalSocketPathLimit()
|
||||
name := filepath.Base(grpcLocalSocketPath(base, "volume", 18080))
|
||||
dir := filepath.Join(base, strings.Repeat("d", limit-len(base)-len(name)-2))
|
||||
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||
t.Fatalf("create %q: %v", dir, err)
|
||||
}
|
||||
path := grpcLocalSocketPath(dir, "volume", 18080)
|
||||
if len(path) != limit {
|
||||
t.Fatalf("socket path %q is %d bytes, want %d", path, len(path), limit)
|
||||
}
|
||||
|
||||
if err := checkGrpcLocalSocketDir(dir, sockets); err != nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir(%q): %v", dir, err)
|
||||
}
|
||||
listener, err := net.Listen("unix", path)
|
||||
if err != nil {
|
||||
t.Fatalf("bind %q: %v", path, err)
|
||||
}
|
||||
if err := listener.Close(); err != nil {
|
||||
t.Fatalf("close %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrpcLocalSocketDirUnwritable checks that -grpc.localSocketDir is
|
||||
// refused when the server's user cannot create a socket in the folder.
|
||||
func TestGrpcLocalSocketDirUnwritable(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root bypasses folder mode bits")
|
||||
}
|
||||
|
||||
// A folder directly under /tmp keeps the probe socket path well inside the
|
||||
// sockaddr_un limit, which t.TempDir() does not on macOS.
|
||||
dir, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-unwritable-")
|
||||
if err != nil {
|
||||
t.Fatalf("create folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Chmod(dir, 0o755); err != nil {
|
||||
t.Errorf("restore mode of %q: %v", dir, err)
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("remove %q: %v", dir, err)
|
||||
}
|
||||
})
|
||||
if err := os.Chmod(dir, 0o555); err != nil {
|
||||
t.Fatalf("chmod %q: %v", dir, err)
|
||||
}
|
||||
|
||||
err = checkGrpcLocalSocketDir(dir, []grpcLocalSocket{{"volume", 17334}})
|
||||
if err == nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir accepted %q, where the server cannot create a socket", dir)
|
||||
}
|
||||
if !strings.Contains(err.Error(), dir) {
|
||||
t.Fatalf("error %q does not name the folder %q", err, dir)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot create a Unix socket") {
|
||||
t.Fatalf("error %q does not report the failed socket bind", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrpcLocalSocketDirProbeLeavesNothing checks that the startup probe
|
||||
// removes its socket, so repeated starts never fill a shared folder.
|
||||
func TestGrpcLocalSocketDirProbeLeavesNothing(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
|
||||
// See TestGrpcLocalSocketDirUnwritable for why the folder is under /tmp.
|
||||
dir, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-probe-")
|
||||
if err != nil {
|
||||
t.Fatalf("create folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("remove %q: %v", dir, err)
|
||||
}
|
||||
})
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := checkGrpcLocalSocketDir(dir, []grpcLocalSocket{{"volume", 17334}}); err != nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir(%q): %v", dir, err)
|
||||
}
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", dir, err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("probe left %d entries in %q, first %q", len(entries), dir, entries[0].Name())
|
||||
}
|
||||
}
|
||||
+6
-16
@@ -410,7 +410,6 @@ var (
|
||||
miniDataFolders = cmdMini.Flag.String("dir", ".", "directory to store data files")
|
||||
miniMetricsHttpPort = cmdMini.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
miniMetricsHttpIp = cmdMini.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
miniGrpcLocalSocketDir = cmdMini.Flag.String("grpc.localSocketDir", "/tmp", "directory for the local gRPC Unix sockets; default to /tmp. Point it at a directory a temp cleaner does not sweep, such as /run/seaweedfs")
|
||||
miniS3Config = cmdMini.Flag.String("s3.config", "", "path to the S3 config file")
|
||||
miniIamConfig = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3")
|
||||
miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
|
||||
@@ -518,7 +517,7 @@ func initMiniVolumeFlags() {
|
||||
miniOptions.v.fileSizeLimitMB = cmdMini.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
miniOptions.v.ldbTimeout = cmdMini.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb")
|
||||
miniOptions.v.concurrentUploadLimitMB = cmdMini.Flag.Int("volume.concurrentUploadLimitMB", 0, "limit total concurrent upload size")
|
||||
miniOptions.v.concurrentDownloadLimitMB = cmdMini.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size; reads over the limit are sent to a replica unless volume.readMode is local")
|
||||
miniOptions.v.concurrentDownloadLimitMB = cmdMini.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size")
|
||||
miniOptions.v.pprof = cmdMini.Flag.Bool("volume.pprof", false, "enable pprof http handlers")
|
||||
miniOptions.v.idxFolder = cmdMini.Flag.String("volume.dir.idx", "", "directory to store .idx files")
|
||||
miniOptions.v.inflightUploadDataTimeout = cmdMini.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout")
|
||||
@@ -572,7 +571,6 @@ func initMiniS3Flags() {
|
||||
miniS3Options.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
miniS3Options.allowBackslashInKey = cmdMini.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
|
||||
miniS3Options.iamSelfServiceRequiresGrant = cmdMini.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
miniS3Options.preStopSeconds = cmdMini.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage)
|
||||
// In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server
|
||||
miniS3Options.debug = new(bool) // explicitly false
|
||||
miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)")
|
||||
@@ -1570,21 +1568,13 @@ func runMini(cmd *Command, args []string) bool {
|
||||
|
||||
// Register Unix socket paths for gRPC services so local inter-service
|
||||
// communication goes through Unix sockets instead of TCP.
|
||||
localSockets := []grpcLocalSocket{
|
||||
{"master", *miniMasterOptions.portGrpc},
|
||||
{"volume", *miniOptions.v.portGrpc},
|
||||
{"filer", *miniFilerOptions.portGrpc},
|
||||
{"admin", *miniAdminOptions.grpcPort},
|
||||
}
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniMasterOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-master-grpc-%d.sock", *miniMasterOptions.portGrpc))
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniOptions.v.portGrpc, fmt.Sprintf("/tmp/seaweedfs-volume-grpc-%d.sock", *miniOptions.v.portGrpc))
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniFilerOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-filer-grpc-%d.sock", *miniFilerOptions.portGrpc))
|
||||
if *miniS3Options.portGrpc > 0 {
|
||||
localSockets = append(localSockets, grpcLocalSocket{"s3", *miniS3Options.portGrpc})
|
||||
}
|
||||
if err := checkGrpcLocalSocketDir(*miniGrpcLocalSocketDir, localSockets); err != nil {
|
||||
glog.Fatalf("Check gRPC local socket folder (-grpc.localSocketDir=%q): %v", *miniGrpcLocalSocketDir, err)
|
||||
}
|
||||
for _, socket := range localSockets {
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, socket.port, grpcLocalSocketPath(*miniGrpcLocalSocketDir, socket.name, socket.port))
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniS3Options.portGrpc, fmt.Sprintf("/tmp/seaweedfs-s3-grpc-%d.sock", *miniS3Options.portGrpc))
|
||||
}
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniAdminOptions.grpcPort, fmt.Sprintf("/tmp/seaweedfs-admin-grpc-%d.sock", *miniAdminOptions.grpcPort))
|
||||
|
||||
go stats_collect.StartMetricsServer(*miniMetricsHttpIp, *miniMetricsHttpPort)
|
||||
|
||||
|
||||
+13
-54
@@ -10,7 +10,6 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -89,7 +88,6 @@ type S3Options struct {
|
||||
allowUntrustedRemoteEndpoints *bool
|
||||
allowBackslashInKey *bool
|
||||
iamSelfServiceRequiresGrant *bool
|
||||
preStopSeconds *int
|
||||
// shutdownCtx, when non-nil, tells startS3Server to gracefully shut down
|
||||
// its HTTP/gRPC servers once the ctx is cancelled, in addition to on
|
||||
// interrupt. Used by weed mini to orchestrate an ordered shutdown.
|
||||
@@ -150,7 +148,6 @@ func init() {
|
||||
s3StandaloneOptions.allowUntrustedRemoteEndpoints = cmdS3.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
s3StandaloneOptions.allowBackslashInKey = cmdS3.Flag.Bool("allowBackslashInKey", false, allowBackslashInKeyUsage)
|
||||
s3StandaloneOptions.iamSelfServiceRequiresGrant = cmdS3.Flag.Bool("iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
s3StandaloneOptions.preStopSeconds = cmdS3.Flag.Int("preStopSeconds", 0, s3PreStopSecondsUsage)
|
||||
}
|
||||
|
||||
var cmdS3 = &Command{
|
||||
@@ -452,7 +449,17 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
// The Vectors API rides the main -port listener whenever a weed-vector
|
||||
// backend is reachable — /s3vectors/ in the SigV4 credential scope picks
|
||||
// the vectors router.
|
||||
vectorsHandler := s3Handler(router, s3opt.vectorsRouter(s3ApiServer, grpcDialOption, masterAddresses, filerGroup))
|
||||
vectorsHandler := http.Handler(router)
|
||||
if vectorsRouter := s3opt.vectorsRouter(s3ApiServer, grpcDialOption, masterAddresses, filerGroup); vectorsRouter != nil {
|
||||
vr, mainRouter := vectorsRouter, router
|
||||
vectorsHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if isVectorsRequest(r) {
|
||||
vr.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
mainRouter.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
localSocket := *s3opt.localSocket
|
||||
@@ -605,47 +612,16 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
func (s3opt *S3Options) newShutdown(stopGrpc func(), s3ApiServer *s3api.S3ApiServer, httpServers []*http.Server) func() {
|
||||
shutdown := newGracefulShutdown(stopGrpc, s3ApiServer.Shutdown, httpServers...)
|
||||
var cancelled <-chan struct{}
|
||||
grace.OnInterrupt(shutdown)
|
||||
if s3opt.shutdownCtx != nil {
|
||||
cancelled = s3opt.shutdownCtx.Done()
|
||||
go func() {
|
||||
<-cancelled
|
||||
<-s3opt.shutdownCtx.Done()
|
||||
shutdown()
|
||||
}()
|
||||
}
|
||||
preStop := sync.OnceFunc(preStopWait(s3ApiServer.StartDraining, s3opt.preStopDelay(), cancelled))
|
||||
// The wait has to finish before weed mini cancels shutdownCtx, and before
|
||||
// weed server shuts down the filer and volume server, whose hooks may be
|
||||
// registered after this one.
|
||||
beforeMiniClientsShutdown(preStop)
|
||||
grace.OnInterruptFirst(preStop)
|
||||
grace.OnInterrupt(shutdown)
|
||||
return shutdown
|
||||
}
|
||||
|
||||
func (s3opt *S3Options) preStopDelay() time.Duration {
|
||||
if s3opt.preStopSeconds == nil || *s3opt.preStopSeconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(*s3opt.preStopSeconds) * time.Second
|
||||
}
|
||||
|
||||
func preStopWait(startDraining func(), delay time.Duration, cancelled <-chan struct{}) func() {
|
||||
return func() {
|
||||
if delay <= 0 {
|
||||
return
|
||||
}
|
||||
startDraining()
|
||||
glog.V(0).Infof("S3 API Server failing health checks for %s before shutdown", delay)
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-cancelled:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// vendsCredentials reports whether -iceberg.credentialRole is set, which
|
||||
// both catalogs vend through, whichever of them runs.
|
||||
func (s3opt *S3Options) vendsCredentials() bool {
|
||||
@@ -792,23 +768,6 @@ func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOp
|
||||
return vectorsRouter
|
||||
}
|
||||
|
||||
// s3Handler serves router, and vectors for Vectors API requests when set. mux
|
||||
// matches Queries() before router middleware runs, so the escape wraps the
|
||||
// router for routing and authorization to read one query.
|
||||
func s3Handler(router, vectors *mux.Router) http.Handler {
|
||||
main := util_http.EscapeSemicolonsInQuery(router)
|
||||
if vectors == nil {
|
||||
return main
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if isVectorsRequest(r) {
|
||||
vectors.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
main.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// isVectorsRequest routes by the SigV4 credential scope — the s3vectors
|
||||
// service name lives there, not in the path.
|
||||
func isVectorsRequest(r *http.Request) bool {
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// A raw ';' pair must not select a Queries() route that the handler, reading
|
||||
// the escaped query, does not see.
|
||||
func TestS3HandlerEscapesSemicolonsBeforeRouting(t *testing.T) {
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
var served string
|
||||
var hasUploads bool
|
||||
router.Methods(http.MethodGet).Path("/{bucket}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
served, hasUploads = "uploads", r.URL.Query().Has("uploads")
|
||||
}).Queries("uploads", "")
|
||||
router.Methods(http.MethodGet).Path("/{bucket}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
served, hasUploads = "list", r.URL.Query().Has("uploads")
|
||||
})
|
||||
|
||||
for query, want := range map[string]string{"uploads": "uploads", "uploads;x=1": "list", "x=1;uploads": "list"} {
|
||||
s3Handler(router, nil).ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/bkt?"+query, nil))
|
||||
assert.Equal(t, want, served, query)
|
||||
assert.Equal(t, want == "uploads", hasUploads, query)
|
||||
}
|
||||
}
|
||||
@@ -79,109 +79,3 @@ func TestS3ShutdownDrainsActiveRequestBeforeServeReturns(t *testing.T) {
|
||||
t.Fatal("Serve exit path did not return after the request drained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3PreStopRunsBeforeMiniCancelsClients(t *testing.T) {
|
||||
savedClients, savedClusterCtx := miniClients, MiniClusterCtx
|
||||
t.Cleanup(func() { miniClients, MiniClusterCtx = savedClients, savedClusterCtx })
|
||||
clusterCtx, stopCluster := context.WithCancel(context.Background())
|
||||
t.Cleanup(stopCluster)
|
||||
MiniClusterCtx = clusterCtx
|
||||
resetMiniClients()
|
||||
|
||||
s3a := &s3api.S3ApiServer{}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/healthz", s3a.StatusHandler)
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
|
||||
server := httptest.NewServer(mux)
|
||||
t.Cleanup(server.Close)
|
||||
httpClosing := make(chan struct{})
|
||||
server.Config.RegisterOnShutdown(func() { close(httpClosing) })
|
||||
|
||||
get := func(path string) int {
|
||||
t.Helper()
|
||||
response, err := server.Client().Get(server.URL + path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response.Body.Close()
|
||||
return response.StatusCode
|
||||
}
|
||||
if code := get("/healthz"); code != http.StatusOK {
|
||||
t.Fatalf("probe before shutdown returned %d", code)
|
||||
}
|
||||
|
||||
preStopSeconds := 3600
|
||||
s3opt := &S3Options{shutdownCtx: miniClientsCtx(), preStopSeconds: &preStopSeconds}
|
||||
s3opt.newShutdown(func() {}, s3a, []*http.Server{server.Config})
|
||||
|
||||
// weed mini's interrupt hook, which runs before the S3 one.
|
||||
triggerDone := make(chan struct{})
|
||||
go func() { triggerMiniClientsShutdown(5 * time.Second); close(triggerDone) }()
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for get("/healthz") != http.StatusServiceUnavailable {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("probe did not start failing before mini cancelled its clients")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if code := get("/bucket/key"); code != http.StatusNoContent {
|
||||
t.Fatalf("request during the pre-stop wait returned %d", code)
|
||||
}
|
||||
select {
|
||||
case <-httpClosing:
|
||||
t.Fatal("HTTP drain started during the pre-stop wait")
|
||||
default:
|
||||
}
|
||||
|
||||
stopCluster()
|
||||
select {
|
||||
case <-httpClosing:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("HTTP drain did not start once the wait ended")
|
||||
}
|
||||
select {
|
||||
case <-triggerDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("mini client shutdown did not finish")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3PreStopWaitZeroDelay(t *testing.T) {
|
||||
drained := false
|
||||
preStopWait(func() { drained = true }, 0, nil)()
|
||||
if drained {
|
||||
t.Error("probes failed with no pre-stop delay")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3PreStopWaitElapses(t *testing.T) {
|
||||
const delay = 50 * time.Millisecond
|
||||
drained := false
|
||||
started := time.Now()
|
||||
preStopWait(func() { drained = true }, delay, nil)()
|
||||
if !drained {
|
||||
t.Error("probes did not fail during the pre-stop delay")
|
||||
}
|
||||
if elapsed := time.Since(started); elapsed < delay {
|
||||
t.Fatalf("wait returned after %s, want at least %s", elapsed, delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3PreStopDelay(t *testing.T) {
|
||||
seconds := func(n int) *int { return &n }
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
seconds *int
|
||||
want time.Duration
|
||||
}{
|
||||
{"unset", nil, 0},
|
||||
{"zero", seconds(0), 0},
|
||||
{"negative", seconds(-1), 0},
|
||||
{"set", seconds(20), 20 * time.Second},
|
||||
} {
|
||||
if got := (&S3Options{preStopSeconds: tc.seconds}).preStopDelay(); got != tc.want {
|
||||
t.Errorf("%s: preStopDelay() = %s, want %s", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,26 +76,3 @@
|
||||
# retry_limit = 1
|
||||
# seconds to wait between retry attempts
|
||||
# retry_backoff_seconds = 30
|
||||
|
||||
[maintenance.s3_lifecycle]
|
||||
# enabled = true
|
||||
# How far back the filer meta-log subscription can reliably reach, in days.
|
||||
# Rules whose TTL exceeds this window cannot be serviced by replay alone and
|
||||
# are partitioned into the walk set, which steady-state walks scan in full,
|
||||
# subject to walker_interval_minutes.
|
||||
# 0 means "no retention info supplied": the run then falls back to the longest
|
||||
# rule TTL, which keeps every replay rule in replay and promotes nothing.
|
||||
# Set this to the window the filer meta-log actually retains.
|
||||
# meta_log_retention_days = 0
|
||||
# Minimum minutes between steady-state bucket walks per shard. Cold-start and
|
||||
# recovery walks ignore it. 0 walks on every run.
|
||||
# walker_interval_minutes = 0
|
||||
# Cluster-wide delete budget in delete RPCs per second, divided across
|
||||
# execute-capable workers at dispatch time. 0 means unlimited.
|
||||
# cluster_deletes_per_second = 0
|
||||
# Token-bucket burst for the delete budget. 0 means 2 x per-second.
|
||||
# cluster_deletes_burst = 0
|
||||
# max retry attempts for a failed lifecycle run
|
||||
# retry_limit = 0
|
||||
# seconds to wait between retry attempts
|
||||
# retry_backoff_seconds = 5
|
||||
@@ -70,12 +70,6 @@ sub_url = "rabbit://myqueue"
|
||||
# BEST FOR: Low to moderate traffic (< 100 events/second sustained)
|
||||
# FOR HIGH TRAFFIC: Consider using Kafka, SQS, or pull-based event logs instead
|
||||
# Documentation: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Notification-Webhook
|
||||
#
|
||||
# Webhook targets edited in the admin UI (stored in the filer at
|
||||
# /etc/seaweedfs/notification_webhooks.json) always run and apply without a
|
||||
# filer restart, whether or not this section is enabled. They are sent after
|
||||
# the notification queue configured in this file, so a saturated target there
|
||||
# also delays them.
|
||||
enabled = false
|
||||
endpoint = "https://your-server.com/webhook" # required: HTTP endpoint URL
|
||||
bearer_token = "" # optional: bearer token for authentication
|
||||
|
||||
@@ -78,10 +78,7 @@ expires_after_seconds = 10 # seconds
|
||||
# gRPC mTLS configuration
|
||||
# All gRPC TLS authentications are mutual (mTLS)
|
||||
# The values for ca, cert, and key are paths to the certificate/key files
|
||||
# The host name is not checked, so the certificate files can be shared.
|
||||
# Set verify_server_host = true under [grpc] or a [grpc.<component>] section to
|
||||
# require that a server certificate is valid for the host or IP that component
|
||||
# dials, as the Rust volume server always does.
|
||||
# The host name is not checked, so the certificate files can be shared
|
||||
# Each [grpc.<component>] section also accepts optional client_cert/client_key,
|
||||
# presented when that component dials other servers. Set them when your CA
|
||||
# issues separate serverAuth-only and clientAuth-only certificates; when unset,
|
||||
@@ -92,7 +89,6 @@ expires_after_seconds = 10 # seconds
|
||||
# fatal: the component never falls back to plaintext.
|
||||
[grpc]
|
||||
ca = ""
|
||||
verify_server_host = false
|
||||
# Set wildcard domain for enable TLS authentication by common names
|
||||
allowed_wildcard_domain = "" # .mycompany.com
|
||||
# Optional allowlist a client checks the dialed server's certificate against:
|
||||
@@ -101,10 +97,9 @@ allowed_wildcard_domain = "" # .mycompany.com
|
||||
# With a per-workload CA, list the identities each server may present so one
|
||||
# workload cannot impersonate another component. Unset accepts any CA-signed
|
||||
# server certificate, as before. Any [grpc.<component>] section may set its own
|
||||
# allowed_server_commonNames to override this list. Only the filer scopes it
|
||||
# further per dialed target: [grpc.filer] master_, volume_ and
|
||||
# s3_allowed_server_commonNames. Other components dial every server with one
|
||||
# allowlist and refuse to start when per-target keys are their only allowlist.
|
||||
# allowed_server_commonNames to override this list, and scope it further per
|
||||
# dialed target, e.g. the filer reads [grpc.filer]
|
||||
# volume_allowed_server_commonNames for volume server calls.
|
||||
allowed_server_commonNames = "" # comma-separated server identities
|
||||
allowed_server_wildcard_domain = "" # .mycompany.com
|
||||
|
||||
|
||||
+3
-13
@@ -74,7 +74,6 @@ var (
|
||||
volumeMinFreeSpace = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
|
||||
serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
serverMetricsHttpIp = cmdServer.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
serverGrpcLocalSocketDir = cmdServer.Flag.String("grpc.localSocketDir", "/tmp", "directory for the local gRPC Unix sockets; default to /tmp. Point it at a directory a temp cleaner does not sweep, such as /run/seaweedfs")
|
||||
|
||||
// pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
|
||||
isStartingMasterServer = cmdServer.Flag.Bool("master", true, "whether to start master server")
|
||||
@@ -160,7 +159,7 @@ func init() {
|
||||
serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
serverOptions.v.ldbTimeout = cmdServer.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
|
||||
serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
|
||||
serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited; reads over the limit are sent to a replica unless volume.readMode is local")
|
||||
serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited")
|
||||
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
|
||||
serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
|
||||
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes -memprofile and -cpuprofile")
|
||||
@@ -212,7 +211,6 @@ func init() {
|
||||
s3Options.readerCacheSizeMB = cmdServer.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
|
||||
s3Options.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
s3Options.iamSelfServiceRequiresGrant = cmdServer.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
s3Options.preStopSeconds = cmdServer.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage)
|
||||
|
||||
sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port")
|
||||
sftpOptions.idleTimeout = cmdServer.Flag.Int("sftp.idleTimeout", 10, "connection idle seconds")
|
||||
@@ -435,7 +433,6 @@ func runServer(cmd *Command, args []string) bool {
|
||||
// Register Unix socket paths for gRPC services running in this process
|
||||
// so local inter-service communication uses Unix sockets instead of TCP.
|
||||
// Resolve gRPC ports early (same calculation each service does internally).
|
||||
var localSockets []grpcLocalSocket
|
||||
for _, svc := range []struct {
|
||||
starting *bool
|
||||
portGrpc *int
|
||||
@@ -451,15 +448,9 @@ func runServer(cmd *Command, args []string) bool {
|
||||
if *svc.portGrpc == 0 {
|
||||
*svc.portGrpc = 10000 + *svc.port
|
||||
}
|
||||
localSockets = append(localSockets, grpcLocalSocket{svc.name, *svc.portGrpc})
|
||||
pb.RegisterLocalGrpcSocket(*serverIp, *svc.portGrpc, fmt.Sprintf("/tmp/seaweedfs-%s-grpc-%d.sock", svc.name, *svc.portGrpc))
|
||||
}
|
||||
}
|
||||
if err := checkGrpcLocalSocketDir(*serverGrpcLocalSocketDir, localSockets); err != nil {
|
||||
glog.Fatalf("Check gRPC local socket folder (-grpc.localSocketDir=%q): %v", *serverGrpcLocalSocketDir, err)
|
||||
}
|
||||
for _, socket := range localSockets {
|
||||
pb.RegisterLocalGrpcSocket(*serverIp, socket.port, grpcLocalSocketPath(*serverGrpcLocalSocketDir, socket.name, socket.port))
|
||||
}
|
||||
|
||||
serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
|
||||
|
||||
@@ -565,8 +556,7 @@ func idleTimeoutSeconds(seconds *int) time.Duration {
|
||||
|
||||
func newHttpServer(h http.Handler, tlsConfig *tls.Config) *http.Server {
|
||||
s := &http.Server{
|
||||
Handler: h,
|
||||
ConnContext: util.ConnContext,
|
||||
Handler: h,
|
||||
}
|
||||
if tlsConfig != nil {
|
||||
s.TLSConfig = tlsConfig.Clone()
|
||||
|
||||
@@ -240,7 +240,7 @@ func init() {
|
||||
v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
v.ldbTimeout = cmdVolume.Flag.Int64("index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
|
||||
v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
|
||||
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited; reads over the limit are sent to a replica unless readMode is local")
|
||||
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited")
|
||||
v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes -memprofile and -cpuprofile")
|
||||
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
v.metricsHttpIp = cmdVolume.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
|
||||
@@ -80,7 +80,6 @@ type Filer struct {
|
||||
RemoteStorage *FilerRemoteStorage
|
||||
BuildGuardedRemoteClient RemoteStorageClientBuilder
|
||||
AllowUntrustedRemoteEndpoints bool
|
||||
webhooks managedWebhooks
|
||||
lazyFetchGroup singleflight.Group
|
||||
lazyListGroup singleflight.Group
|
||||
Dlm *lock_manager.DistributedLockManager
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/notification/webhook"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
NotificationWebhooksConfName = "notification_webhooks.json"
|
||||
notificationWebhooksDrain = 30 * time.Second
|
||||
)
|
||||
|
||||
var notificationWebhooksRetry = 5 * time.Second
|
||||
|
||||
var NotificationWebhooksConfPath = string(util.NewFullPath(DirectoryEtcSeaweedFS, NotificationWebhooksConfName))
|
||||
|
||||
// managedWebhooks runs the webhook targets edited from the admin UI, next to
|
||||
// whatever notification.toml configures.
|
||||
type managedWebhooks struct {
|
||||
reloadMu sync.Mutex
|
||||
applied []byte
|
||||
retry *time.Timer
|
||||
queue atomic.Pointer[webhook.Queue]
|
||||
}
|
||||
|
||||
func (f *Filer) LoadNotificationWebhooks() {
|
||||
f.reloadNotificationWebhooks()
|
||||
go func() {
|
||||
<-f.deletionQuit
|
||||
f.webhooks.reloadMu.Lock()
|
||||
defer f.webhooks.reloadMu.Unlock()
|
||||
if f.webhooks.retry != nil {
|
||||
f.webhooks.retry.Stop()
|
||||
}
|
||||
if q := f.webhooks.queue.Swap(nil); q != nil {
|
||||
go q.Close(notificationWebhooksDrain)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (f *Filer) maybeReloadNotificationWebhooks(event *filer_pb.SubscribeMetadataResponse) {
|
||||
if event == nil || event.EventNotification == nil {
|
||||
return
|
||||
}
|
||||
n := event.EventNotification
|
||||
touched := n.OldEntry != nil && n.OldEntry.Name == NotificationWebhooksConfName && event.Directory == DirectoryEtcSeaweedFS
|
||||
if n.NewEntry != nil && n.NewEntry.Name == NotificationWebhooksConfName && filer_pb.MetadataEventTargetDirectory(event) == DirectoryEtcSeaweedFS {
|
||||
touched = true
|
||||
}
|
||||
if touched {
|
||||
f.reloadNotificationWebhooks()
|
||||
}
|
||||
}
|
||||
|
||||
// reloadNotificationWebhooks reads the stored targets rather than the event's
|
||||
// copy, so concurrent change events cannot apply an older version last.
|
||||
func (f *Filer) reloadNotificationWebhooks() {
|
||||
f.webhooks.reloadMu.Lock()
|
||||
defer f.webhooks.reloadMu.Unlock()
|
||||
select {
|
||||
case <-f.deletionQuit:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
var content []byte
|
||||
entry, err := f.FindEntry(context.Background(), util.FullPath(NotificationWebhooksConfPath))
|
||||
switch {
|
||||
case err == filer_pb.ErrNotFound:
|
||||
case err != nil:
|
||||
glog.Errorf("read %s: %v", NotificationWebhooksConfPath, err)
|
||||
f.retryNotificationWebhooks()
|
||||
return
|
||||
case len(entry.Content) > 0:
|
||||
content = entry.Content
|
||||
default:
|
||||
if content, err = f.readEntry(entry.GetChunks(), entry.Size()); err != nil {
|
||||
glog.Errorf("read %s: %v", NotificationWebhooksConfPath, err)
|
||||
f.retryNotificationWebhooks()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if f.webhooks.applied != nil && bytes.Equal(content, f.webhooks.applied) {
|
||||
return
|
||||
}
|
||||
conf, err := webhook.ParseManagedConfig(content)
|
||||
if err != nil {
|
||||
glog.Errorf("keep current webhook targets, %s: %v", NotificationWebhooksConfPath, err)
|
||||
return
|
||||
}
|
||||
queue, err := webhook.NewManagedQueue(conf)
|
||||
if err != nil {
|
||||
glog.Errorf("keep current webhook targets, start %s: %v", NotificationWebhooksConfPath, err)
|
||||
return
|
||||
}
|
||||
glog.V(0).Infof("apply %d webhook targets from %s", len(conf.Targets), NotificationWebhooksConfPath)
|
||||
f.webhooks.applied = append([]byte{}, content...)
|
||||
if old := f.webhooks.queue.Swap(queue); old != nil {
|
||||
go old.Close(notificationWebhooksDrain)
|
||||
}
|
||||
}
|
||||
|
||||
// retryNotificationWebhooks keeps at most one pending retry, since no later
|
||||
// event replays the change whose read failed.
|
||||
func (f *Filer) retryNotificationWebhooks() {
|
||||
if f.webhooks.retry != nil {
|
||||
f.webhooks.retry.Stop()
|
||||
}
|
||||
f.webhooks.retry = time.AfterFunc(notificationWebhooksRetry, f.reloadNotificationWebhooks)
|
||||
}
|
||||
|
||||
// sendToNotificationWebhooks hands a send that raced a reload to the queue
|
||||
// that replaced the closing one.
|
||||
func (f *Filer) sendToNotificationWebhooks(key string, message *filer_pb.EventNotification) {
|
||||
for {
|
||||
q := f.webhooks.queue.Load()
|
||||
if q == nil {
|
||||
return
|
||||
}
|
||||
err := q.SendMessage(key, message)
|
||||
if errors.Is(err, webhook.ErrQueueClosed) && f.webhooks.queue.Load() != q {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// isNotificationWebhooksConf keeps the stored targets, bearer tokens
|
||||
// included, out of the events sent to notification queues.
|
||||
func isNotificationWebhooksConf(oldEntry, newEntry *Entry) bool {
|
||||
return (oldEntry != nil && string(oldEntry.FullPath) == NotificationWebhooksConfPath) ||
|
||||
(newEntry != nil && string(newEntry.FullPath) == NotificationWebhooksConfPath)
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func TestManagedNotificationWebhooks(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var keys []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct{ Key string }
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
mu.Lock()
|
||||
keys = append(keys, body.Key)
|
||||
mu.Unlock()
|
||||
}))
|
||||
defer server.Close()
|
||||
received := func() []string {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return append([]string(nil), keys...)
|
||||
}
|
||||
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, nil)
|
||||
t.Cleanup(func() {
|
||||
if q := f.webhooks.queue.Load(); q != nil {
|
||||
q.Close(time.Second)
|
||||
}
|
||||
})
|
||||
|
||||
saveConf := func(content string) {
|
||||
store.entries[NotificationWebhooksConfPath] = &Entry{FullPath: util.FullPath(NotificationWebhooksConfPath), Content: []byte(content)}
|
||||
}
|
||||
confEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: DirectoryEtcSeaweedFS,
|
||||
EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: NotificationWebhooksConfName}},
|
||||
}
|
||||
|
||||
f.LoadNotificationWebhooks()
|
||||
if f.webhooks.queue.Load() != nil {
|
||||
t.Fatal("expected no webhook queue without stored targets")
|
||||
}
|
||||
|
||||
saveConf(fmt.Sprintf(`{"targets":[{"name":"data","enabled":true,"endpoint":%q,"bearer_token":"tok","path_prefixes":["/"]}]}`, server.URL))
|
||||
f.maybeReloadNotificationWebhooks(confEvent)
|
||||
if f.webhooks.queue.Load() == nil {
|
||||
t.Fatal("expected a webhook queue after the targets were stored")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
f.NotifyUpdateEvent(ctx, nil, store.entries[NotificationWebhooksConfPath], false, false, nil)
|
||||
f.NotifyUpdateEvent(ctx, nil, &Entry{FullPath: "/data/a.txt"}, false, false, nil)
|
||||
waitForWebhook(t, func() bool { return len(received()) == 1 })
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if got := received(); len(got) != 1 || got[0] != "/data/a.txt" {
|
||||
t.Fatalf("expected only /data/a.txt to be delivered, got %v", got)
|
||||
}
|
||||
|
||||
running := f.webhooks.queue.Load()
|
||||
f.maybeReloadNotificationWebhooks(confEvent)
|
||||
if f.webhooks.queue.Load() != running {
|
||||
t.Fatal("unchanged targets must keep the running queue")
|
||||
}
|
||||
saveConf(`{"targets":[{"name":"bad name","enabled":true,"endpoint":"https://example.com"}]}`)
|
||||
f.maybeReloadNotificationWebhooks(confEvent)
|
||||
if f.webhooks.queue.Load() != running {
|
||||
t.Fatal("invalid targets must keep the running queue")
|
||||
}
|
||||
|
||||
delete(store.entries, NotificationWebhooksConfPath)
|
||||
f.maybeReloadNotificationWebhooks(&filer_pb.SubscribeMetadataResponse{
|
||||
Directory: DirectoryEtcSeaweedFS,
|
||||
EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: NotificationWebhooksConfName}},
|
||||
})
|
||||
if f.webhooks.queue.Load() != nil {
|
||||
t.Fatal("expected deleting the stored targets to stop the queue")
|
||||
}
|
||||
}
|
||||
|
||||
func waitForWebhook(t *testing.T, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("timed out")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedNotificationWebhooksRetryAfterReadError(t *testing.T) {
|
||||
retry := notificationWebhooksRetry
|
||||
t.Cleanup(func() { notificationWebhooksRetry = retry })
|
||||
notificationWebhooksRetry = 10 * time.Millisecond
|
||||
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, nil)
|
||||
t.Cleanup(func() {
|
||||
f.webhooks.reloadMu.Lock()
|
||||
if f.webhooks.retry != nil {
|
||||
f.webhooks.retry.Stop()
|
||||
}
|
||||
f.webhooks.reloadMu.Unlock()
|
||||
if q := f.webhooks.queue.Load(); q != nil {
|
||||
q.Close(time.Second)
|
||||
}
|
||||
})
|
||||
|
||||
store.mu.Lock()
|
||||
store.entries[NotificationWebhooksConfPath] = &Entry{
|
||||
FullPath: util.FullPath(NotificationWebhooksConfPath),
|
||||
Content: []byte(`{"targets":[{"name":"data","enabled":true,"endpoint":"http://127.0.0.1:1/hook"}]}`),
|
||||
}
|
||||
store.findErr = errors.New("store unavailable")
|
||||
store.mu.Unlock()
|
||||
|
||||
f.LoadNotificationWebhooks()
|
||||
if f.webhooks.queue.Load() != nil {
|
||||
t.Fatal("expected no webhook queue while the stored targets cannot be read")
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
store.findErr = nil
|
||||
store.mu.Unlock()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for f.webhooks.queue.Load() == nil {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("stored webhook targets were not applied after the read recovered")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedNotificationWebhooksStopOnShutdown(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, nil)
|
||||
store.entries[NotificationWebhooksConfPath] = &Entry{
|
||||
FullPath: util.FullPath(NotificationWebhooksConfPath),
|
||||
Content: []byte(`{"targets":[{"name":"data","enabled":true,"endpoint":"http://127.0.0.1:1/hook"}]}`),
|
||||
}
|
||||
|
||||
f.LoadNotificationWebhooks()
|
||||
if f.webhooks.queue.Load() == nil {
|
||||
t.Fatal("expected a webhook queue for the stored targets")
|
||||
}
|
||||
|
||||
close(f.deletionQuit)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for f.webhooks.queue.Load() != nil {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("webhook queue still installed after shutdown")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
store.entries[NotificationWebhooksConfPath] = &Entry{
|
||||
FullPath: util.FullPath(NotificationWebhooksConfPath),
|
||||
Content: []byte(`{"targets":[{"name":"late","enabled":true,"endpoint":"http://127.0.0.1:1/late"}]}`),
|
||||
}
|
||||
f.reloadNotificationWebhooks()
|
||||
if f.webhooks.queue.Load() != nil {
|
||||
t.Fatal("a reload after shutdown installed a webhook queue")
|
||||
}
|
||||
}
|
||||
@@ -68,15 +68,12 @@ func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
|
||||
defer f.metaLogInflight.done(event.TsNs)
|
||||
eventNotification := event.EventNotification
|
||||
|
||||
if !isNotificationWebhooksConf(oldEntry, newEntry) {
|
||||
if notification.Queue != nil {
|
||||
glog.V(3).Infof("notifying entry update %v", fullpath)
|
||||
if err := notification.Queue.SendMessage(fullpath, eventNotification); err != nil {
|
||||
// throw message
|
||||
glog.Error(err)
|
||||
}
|
||||
if notification.Queue != nil {
|
||||
glog.V(3).Infof("notifying entry update %v", fullpath)
|
||||
if err := notification.Queue.SendMessage(fullpath, eventNotification); err != nil {
|
||||
// throw message
|
||||
glog.Error(err)
|
||||
}
|
||||
f.sendToNotificationWebhooks(fullpath, eventNotification)
|
||||
}
|
||||
|
||||
f.logMetaEvent(ctx, event)
|
||||
@@ -291,14 +288,6 @@ func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFrom
|
||||
}
|
||||
}
|
||||
|
||||
// notifyRenameTarget wakes the subscribers of a move's destination directory,
|
||||
// which the partition key (the source directory) does not cover.
|
||||
func notifyRenameTarget(logBuffer *log_buffer.LogBuffer, event *filer_pb.SubscribeMetadataResponse) {
|
||||
if target := filer_pb.MetadataEventTargetDirectory(event); target != event.Directory {
|
||||
logBuffer.NotifySubscribers([]byte(target))
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) {
|
||||
data, err := proto.Marshal(event)
|
||||
if err != nil {
|
||||
@@ -311,7 +300,6 @@ func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetad
|
||||
glog.Errorf("failed to add data to log buffer for %s: %v", event.Directory, err)
|
||||
return
|
||||
}
|
||||
notifyRenameTarget(f.LocalMetaLogBuffer, event)
|
||||
// The in-memory copy now carries the position the stream assigns it, the
|
||||
// same value subscribers will see delivered.
|
||||
event.TsNs = tsNs
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
||||
)
|
||||
|
||||
func TestNotifyRenameTargetWakesDestinationSubscribers(t *testing.T) {
|
||||
logBuffer := log_buffer.NewLogBuffer("test", time.Minute, func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
|
||||
defer logBuffer.ShutdownLogBuffer()
|
||||
|
||||
logBuffer.SetSubscriberWakeFilter("dst", func(key []byte) bool { return bytes.Equal(key, []byte("/dst")) })
|
||||
dst := logBuffer.RegisterSubscriber("dst")
|
||||
defer logBuffer.UnregisterSubscriber("dst")
|
||||
|
||||
woken := func() bool {
|
||||
select {
|
||||
case <-dst:
|
||||
return true
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
notifyRenameTarget(logBuffer, &filer_pb.SubscribeMetadataResponse{Directory: "/src",
|
||||
EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "a"}, NewParentPath: "/src"}})
|
||||
if woken() {
|
||||
t.Fatal("an event that stays in its directory must not wake other directories")
|
||||
}
|
||||
|
||||
notifyRenameTarget(logBuffer, &filer_pb.SubscribeMetadataResponse{Directory: "/src",
|
||||
EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "a"}, NewEntry: &filer_pb.Entry{Name: "a"}, NewParentPath: "/dst"}})
|
||||
if !woken() {
|
||||
t.Fatal("a move must wake the destination directory's subscribers")
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
|
||||
func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
|
||||
f.maybeReloadFilerConfiguration(event)
|
||||
f.maybeReloadNotificationWebhooks(event)
|
||||
f.maybeReloadRemoteStorageConfigurationAndMapping(event)
|
||||
f.onBucketEvents(event)
|
||||
f.onEmptyFolderCleanupEvents(event)
|
||||
|
||||
@@ -68,10 +68,7 @@ type MetaAggregator struct {
|
||||
// the grace, but a rejoining peer still needs it to resume. Forgotten
|
||||
// peerResumeRetention after the removal so addresses that never return,
|
||||
// such as restarted pods' IPs, do not accumulate.
|
||||
peerResumeTsNs map[pb.ServerAddress]int64
|
||||
// newPeerStartTsNs is the newest start of a peer subscription that had no
|
||||
// resume point: the ring holds nothing that peer logged at or before it.
|
||||
newPeerStartTsNs int64
|
||||
peerResumeTsNs map[pb.ServerAddress]int64
|
||||
peerWatermarksLock sync.Mutex
|
||||
peerWg sync.WaitGroup
|
||||
// peerSubscribeLoop is replaceable so a test can observe the subscribe
|
||||
@@ -248,21 +245,9 @@ func (ma *MetaAggregator) peerResumeStartFrom(peer pb.ServerAddress, startFrom t
|
||||
if prev := ma.peerResumeTsNs[peer]; prev > 0 && prev < startFrom.UnixNano() {
|
||||
return time.Unix(0, prev)
|
||||
}
|
||||
if startTsNs := startFrom.UnixNano(); startTsNs > ma.newPeerStartTsNs {
|
||||
ma.newPeerStartTsNs = startTsNs
|
||||
}
|
||||
return startFrom
|
||||
}
|
||||
|
||||
// NewPeerStartTsNs returns the newest start of a peer subscription that had
|
||||
// no resume point. What that peer logged at or before it is only on disk, so
|
||||
// a cursor below it cannot be served from the ring alone.
|
||||
func (ma *MetaAggregator) NewPeerStartTsNs() int64 {
|
||||
ma.peerWatermarksLock.Lock()
|
||||
defer ma.peerWatermarksLock.Unlock()
|
||||
return ma.newPeerStartTsNs
|
||||
}
|
||||
|
||||
// advancePeerFlushWatermark records the peer's reported flush watermark.
|
||||
// Monotonic, and only for tracked peers (see advancePeerWatermark).
|
||||
func (ma *MetaAggregator) advancePeerFlushWatermark(peer pb.ServerAddress, tsNs int64) {
|
||||
@@ -643,7 +628,6 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
glog.Errorf("failed to add data to log buffer for %s: %v", dir, err)
|
||||
return err
|
||||
}
|
||||
notifyRenameTarget(ma.MetaLogBuffer, event)
|
||||
// The event now sits at tsNs in the local stream; offset tracking
|
||||
// still needs the peer's own position, so it is passed separately.
|
||||
event.TsNs = tsNs
|
||||
|
||||
@@ -99,9 +99,6 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) {
|
||||
if got := nextSubscribeStart(); got != firstStart.UnixNano() {
|
||||
t.Fatalf("expecting first subscribe at %d, got %d", firstStart.UnixNano(), got)
|
||||
}
|
||||
if got := ma.NewPeerStartTsNs(); got != firstStart.UnixNano() {
|
||||
t.Fatalf("expecting a first-seen peer to mark its start %d, got %d", firstStart.UnixNano(), got)
|
||||
}
|
||||
|
||||
deliveredThrough := time.Now().Add(-time.Hour).UnixNano()
|
||||
ma.advancePeerWatermark(peer, deliveredThrough)
|
||||
@@ -120,9 +117,6 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) {
|
||||
if got := nextSubscribeStart(); got != deliveredThrough {
|
||||
t.Fatalf("expecting re-added peer to resume at %d, got %d", deliveredThrough, got)
|
||||
}
|
||||
if got := ma.NewPeerStartTsNs(); got != firstStart.UnixNano() {
|
||||
t.Fatalf("expecting a resumed peer to leave the new-peer start at %d, got %d", firstStart.UnixNano(), got)
|
||||
}
|
||||
|
||||
// A peer gone past the retention is forgotten and starts afresh.
|
||||
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer)}, time.Now())
|
||||
@@ -139,9 +133,6 @@ func TestPeerResumeStartFromSurvivesRemoval(t *testing.T) {
|
||||
if got := nextSubscribeStart(); got != freshStart.UnixNano() {
|
||||
t.Fatalf("expecting forgotten peer to start at %d, got %d", freshStart.UnixNano(), got)
|
||||
}
|
||||
if got := ma.NewPeerStartTsNs(); got != freshStart.UnixNano() {
|
||||
t.Fatalf("expecting a forgotten peer to mark its start %d, got %d", freshStart.UnixNano(), got)
|
||||
}
|
||||
|
||||
// A first-seen peer, or a caller start already behind the recorded
|
||||
// position, keeps the caller's start.
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
)
|
||||
|
||||
@@ -19,13 +17,6 @@ func (ma *MetaAggregator) TrackPeerForTesting(peer pb.ServerAddress) {
|
||||
ma.initPeerWatermark(peer)
|
||||
}
|
||||
|
||||
// StartPeerForTesting is TrackPeerForTesting for a peer first followed from
|
||||
// startFrom, as OnPeerUpdate records it. Test support only.
|
||||
func (ma *MetaAggregator) StartPeerForTesting(peer pb.ServerAddress, startFrom time.Time) {
|
||||
ma.TrackPeerForTesting(peer)
|
||||
ma.peerResumeStartFrom(peer, startFrom)
|
||||
}
|
||||
|
||||
// ReportPeerWatermarksForTesting stands in for what a peer's stream reports:
|
||||
// its delivery watermark (events and idle heartbeats) and its flush
|
||||
// watermark. Test support only.
|
||||
|
||||
@@ -54,17 +54,19 @@ func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages {
|
||||
return dirtyPages
|
||||
}
|
||||
|
||||
func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) (n int, err error) {
|
||||
func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error {
|
||||
// A failed chunk upload already dropped its data, so this handle can never
|
||||
// flush cleanly. Reject further writes so the writer fails fast instead of
|
||||
// pushing the rest of the file through a pipeline that cannot persist it.
|
||||
if err := pages.LastError(); err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
pages.hasWrites = true
|
||||
|
||||
glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.fh, offset, offset+int64(len(data)))
|
||||
return pages.uploadPipeline.SaveDataAt(data, offset, isSequential, tsNs)
|
||||
_, err := pages.uploadPipeline.SaveDataAt(data, offset, isSequential, tsNs)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (pages *ChunkedDirtyPages) HasWrites() bool {
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestChunkedDirtyPagesFailWritesAfterUploadError(t *testing.T) {
|
||||
uploadErr := errors.New("assign volume failure: no writable volumes")
|
||||
pages.setLastError(uploadErr)
|
||||
|
||||
if _, err := pages.AddPage(0, []byte("x"), true, 1); !errors.Is(err, uploadErr) {
|
||||
if err := pages.AddPage(0, []byte("x"), true, 1); !errors.Is(err, uploadErr) {
|
||||
t.Fatalf("AddPage after upload failure = %v, want sticky %v", err, uploadErr)
|
||||
}
|
||||
if err := pages.FlushData(); !errors.Is(err, uploadErr) {
|
||||
|
||||
@@ -15,6 +15,10 @@ type PageWriter struct {
|
||||
randomWriter page_writer.DirtyPages
|
||||
}
|
||||
|
||||
var (
|
||||
_ = page_writer.DirtyPages(&PageWriter{})
|
||||
)
|
||||
|
||||
func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter {
|
||||
pw := &PageWriter{
|
||||
fh: fh,
|
||||
@@ -25,27 +29,23 @@ func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter {
|
||||
return pw
|
||||
}
|
||||
|
||||
// AddPage returns how many leading bytes of data it buffered; on error those
|
||||
// bytes stay buffered.
|
||||
func (pw *PageWriter) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) (buffered int64, err error) {
|
||||
func (pw *PageWriter) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error {
|
||||
|
||||
glog.V(4).Infof("%v AddPage [%d, %d)", pw.fh.fh, offset, offset+int64(len(data)))
|
||||
|
||||
chunkIndex := offset / pw.chunkSize
|
||||
for i := chunkIndex; len(data) > 0; i++ {
|
||||
writeSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
|
||||
n, err := pw.addToOneChunk(i, offset, data[:writeSize], isSequential, tsNs)
|
||||
buffered += int64(n)
|
||||
if err != nil {
|
||||
return buffered, err
|
||||
if err := pw.addToOneChunk(i, offset, data[:writeSize], isSequential, tsNs); err != nil {
|
||||
return err
|
||||
}
|
||||
offset += writeSize
|
||||
data = data[writeSize:]
|
||||
}
|
||||
return buffered, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte, isSequential bool, tsNs int64) (int, error) {
|
||||
func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte, isSequential bool, tsNs int64) error {
|
||||
return pw.randomWriter.AddPage(offset, data, isSequential, tsNs)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package page_writer
|
||||
|
||||
type DirtyPages interface {
|
||||
AddPage(offset int64, data []byte, isSequential bool, tsNs int64) (n int, err error)
|
||||
AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error
|
||||
FlushData() error
|
||||
HasWrites() bool
|
||||
ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64)
|
||||
|
||||
@@ -2,7 +2,6 @@ package page_writer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -145,15 +144,12 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsN
|
||||
pageChunk = NewMemChunk(logicChunkIndex, up.ChunkSize)
|
||||
// fmt.Printf(" create mem chunk %d\n", logicChunkIndex)
|
||||
} else {
|
||||
// Check the *SwapFileChunk before it becomes a PageChunk: a nil
|
||||
// pointer inside the interface would not compare equal to nil.
|
||||
swapChunk := up.swapFile.NewSwapFileChunk(logicChunkIndex)
|
||||
pageChunk = up.swapFile.NewSwapFileChunk(logicChunkIndex)
|
||||
// fmt.Printf(" create file chunk %d\n", logicChunkIndex)
|
||||
if swapChunk == nil {
|
||||
if pageChunk == nil {
|
||||
up.accountant.Release(up.ChunkSize)
|
||||
return 0, fmt.Errorf("failed to create swap file chunk")
|
||||
}
|
||||
pageChunk = swapChunk
|
||||
}
|
||||
up.writableChunks[logicChunkIndex] = pageChunk
|
||||
}
|
||||
@@ -165,9 +161,6 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsN
|
||||
//}
|
||||
n = pageChunk.WriteDataAt(p, off, tsNs)
|
||||
up.maybeMoveToSealed(pageChunk, logicChunkIndex)
|
||||
if n < len(p) {
|
||||
err = fmt.Errorf("chunk %d stored %d of %d bytes: %w", logicChunkIndex, n, len(p), io.ErrShortWrite)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
package page_writer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// When the swap file cannot be created, SaveDataAt must fail the write
|
||||
// instead of keeping a nil swap chunk and writing through it, and give back
|
||||
// the write budget it reserved for the chunk.
|
||||
func TestSaveDataAtFailsWhenSwapFileCannotBeCreated(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
accountant *WriteBufferAccountant
|
||||
}{
|
||||
{name: "no write budget"},
|
||||
{name: "write budget", accountant: NewWriteBufferAccountant(1 << 20)},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
notADir := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(notADir, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
up := NewUploadPipeline(nil, 1024, nil, 16, filepath.Join(notADir, "swap"), tc.accountant)
|
||||
|
||||
// A write that is not sequential goes to the swap file.
|
||||
n, err := up.SaveDataAt([]byte("data"), 0, false, 0)
|
||||
if err == nil {
|
||||
t.Fatalf("SaveDataAt = %d, nil; want an error", n)
|
||||
}
|
||||
if _, found := up.writableChunks[0]; found {
|
||||
t.Error("the failed write left a writable chunk behind")
|
||||
}
|
||||
if used := tc.accountant.Used(); used != 0 {
|
||||
t.Errorf("write budget still holds %d bytes", used)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A swap chunk whose file write fails stores nothing, so SaveDataAt must
|
||||
// report the short write instead of claiming the bytes.
|
||||
func TestSaveDataAtReportsFailedSwapWrite(t *testing.T) {
|
||||
up := NewUploadPipeline(nil, 1024, nil, 16, t.TempDir(), nil)
|
||||
defer up.Shutdown()
|
||||
if n, err := up.SaveDataAt([]byte("data"), 0, false, 0); err != nil || n != 4 {
|
||||
t.Fatalf("first write = %d, %v", n, err)
|
||||
}
|
||||
if err := up.swapFile.file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := up.SaveDataAt([]byte("more"), 4, false, 0); err == nil || n != 0 {
|
||||
t.Fatalf("write through a closed swap file = %d, %v; want 0 and an error", n, err)
|
||||
}
|
||||
}
|
||||
@@ -180,23 +180,19 @@ func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn)
|
||||
|
||||
// Perform the write
|
||||
fhOut.dirtyPages.writerPattern.MonitorWriteAt(offsetOut, int(numBytesRead))
|
||||
buffered, err := fhOut.dirtyPages.AddPage(
|
||||
if err := fhOut.dirtyPages.AddPage(
|
||||
offsetOut,
|
||||
buff[:numBytesRead],
|
||||
fhOut.dirtyPages.writerPattern.IsSequentialMode(),
|
||||
nowUnixNano)
|
||||
if buffered > 0 {
|
||||
fhOut.pageWrites.Add(1)
|
||||
totalCopied += buffered
|
||||
fhOut.dirtyContent = true
|
||||
}
|
||||
if err != nil {
|
||||
nowUnixNano); err != nil {
|
||||
glog.Errorf("AddPage error: %v", err)
|
||||
if totalCopied == 0 {
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
break
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
fhOut.pageWrites.Add(1)
|
||||
|
||||
// Accumulate for the next loop iteration
|
||||
totalCopied += numBytesRead
|
||||
fhOut.dirtyContent = true
|
||||
}
|
||||
|
||||
if totalCopied == 0 {
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package mount
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
@@ -410,47 +408,6 @@ func TestStreamingCopyMarksOnlyAcceptedContentDirty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Bytes the copy buffered before a page failed stay in the handle, so the
|
||||
// copy reports them and grows the destination over them.
|
||||
func TestStreamingCopyReportsPartlyBufferedCopy(t *testing.T) {
|
||||
wfs := newCopyRangeTestWFS()
|
||||
wfs.option.ConcurrentWriters = 1
|
||||
notADir := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(notADir, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wfs.option.uniqueCacheDirForWrite = filepath.Join(notADir, "swap")
|
||||
srcInode := wfs.inodeToPath.Lookup(util.FullPath("/src.txt"), 1, false, false, 0, true)
|
||||
dstInode := wfs.inodeToPath.Lookup(util.FullPath("/dst.txt"), 1, false, false, 0, true)
|
||||
srcHandle, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{
|
||||
Name: "src.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{FileMode: 0100644, FileSize: 1024, Inode: srcInode},
|
||||
Content: bytes.Repeat([]byte{'s'}, 1024),
|
||||
}, 0, 0)
|
||||
dstHandle, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{
|
||||
Name: "dst.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{FileMode: 0100644, Inode: dstInode},
|
||||
}, 0, 0)
|
||||
t.Cleanup(dstHandle.dirtyPages.Destroy)
|
||||
|
||||
// The destination range crosses into a second chunk, which needs the swap file.
|
||||
written, status := wfs.CopyFileRange(make(chan struct{}), &fuse.CopyFileRangeIn{
|
||||
FhIn: uint64(srcHandle.fh),
|
||||
FhOut: uint64(dstHandle.fh),
|
||||
OffOut: 512,
|
||||
Len: 1024,
|
||||
})
|
||||
if status != fuse.OK || written != 512 {
|
||||
t.Fatalf("CopyFileRange = %d, %v; want a short copy of 512", written, status)
|
||||
}
|
||||
if size := dstHandle.GetEntry().GetEntry().GetAttributes().GetFileSize(); size != 1024 {
|
||||
t.Fatalf("destination size = %d, want 1024", size)
|
||||
}
|
||||
if !dstHandle.dirtyContent || !dstHandle.dirtyMetadata {
|
||||
t.Fatal("the buffered prefix must leave the destination dirty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerSideCopySkipsFencedDestination(t *testing.T) {
|
||||
wfs := newCopyRangeTestWFS()
|
||||
wfs.option.WritebackCache = true
|
||||
|
||||
@@ -270,7 +270,7 @@ func TestTruncateEntryClearsDirtyPagesForOpenHandle(t *testing.T) {
|
||||
fh, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, entry, 0, 0)
|
||||
fh.RememberPath(fullPath)
|
||||
|
||||
if _, err := fh.dirtyPages.AddPage(0, []byte("hello"), true, time.Now().UnixNano()); err != nil {
|
||||
if err := fh.dirtyPages.AddPage(0, []byte("hello"), true, time.Now().UnixNano()); err != nil {
|
||||
t.Fatalf("AddPage: %v", err)
|
||||
}
|
||||
oldDirtyPages := fh.dirtyPages
|
||||
|
||||
@@ -68,21 +68,6 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
|
||||
return 0, fuse.OK
|
||||
}
|
||||
|
||||
// Size and times cover only the bytes buffered, or the next flush
|
||||
// publishes a hole where the data should be. A partly buffered write
|
||||
// is reported as a short write, or as an error to kernel writeback,
|
||||
// which ignores the count.
|
||||
offset := int64(in.Offset)
|
||||
buffered, err := fh.dirtyPages.AddPage(offset, data, fh.dirtyPages.writerPattern.IsSequentialMode(), tsNs)
|
||||
if err != nil {
|
||||
glog.Errorf("AddPage error: %v", err)
|
||||
if buffered == 0 {
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
data = data[:buffered]
|
||||
}
|
||||
fh.pageWrites.Add(1)
|
||||
|
||||
// Only an append past all locally owned bytes needs the stored EOF guard.
|
||||
if in.Offset > entry.Attributes.FileSize && wfs.isAppendWrite(in) {
|
||||
if !fh.pendingAppend || in.Offset < fh.pendingAppendOffset {
|
||||
@@ -91,6 +76,7 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
|
||||
fh.pendingAppend = true
|
||||
}
|
||||
entry.Content = nil
|
||||
offset := int64(in.Offset)
|
||||
oldFileSize := int64(entry.Attributes.FileSize)
|
||||
newFileSize := max(offset+int64(len(data)), oldFileSize)
|
||||
entry.Attributes.FileSize = uint64(newFileSize)
|
||||
@@ -110,6 +96,14 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
|
||||
wfs.AddUncommittedBytes(newFileSize - oldFileSize)
|
||||
}
|
||||
|
||||
// glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data))
|
||||
|
||||
if err := fh.dirtyPages.AddPage(offset, data, fh.dirtyPages.writerPattern.IsSequentialMode(), tsNs); err != nil {
|
||||
glog.Errorf("AddPage error: %v", err)
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
fh.pageWrites.Add(1)
|
||||
|
||||
written = uint32(len(data))
|
||||
|
||||
if offset == 0 {
|
||||
@@ -133,8 +127,5 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
|
||||
fh.mirrorFile.WriteAt(data, offset)
|
||||
}
|
||||
|
||||
if err != nil && in.WriteFlags&fuse.WRITE_CACHE != 0 {
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
return written, fuse.OK
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
//go:build !freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/go-fuse/v2/fuse"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// A write the swap file cannot buffer fails, and must not grow the file: the
|
||||
// next flush would publish that size with a hole where its data should be. A
|
||||
// write that buffers only its first chunk is a short write of those bytes, or
|
||||
// an error for kernel writeback, which would drop the rest silently.
|
||||
func TestFailedWriteDoesNotGrowFile(t *testing.T) {
|
||||
t.Run("short write", func(t *testing.T) { testFailedWriteDoesNotGrowFile(t, 0) })
|
||||
t.Run("kernel writeback", func(t *testing.T) { testFailedWriteDoesNotGrowFile(t, fuse.WRITE_CACHE) })
|
||||
}
|
||||
|
||||
func testFailedWriteDoesNotGrowFile(t *testing.T, writeFlags uint32) {
|
||||
base := attributeAppendEntry(1)
|
||||
base.CommitTsNs = 100
|
||||
peer := &g6ConditionalPeer{entry: proto.Clone(base).(*filer_pb.Entry)}
|
||||
peer.blobs = map[string][]byte{base.Chunks[0].GetFileIdString(): bytes.Repeat([]byte{'O'}, 256)}
|
||||
|
||||
w := newInvalidateTestWFS(t)
|
||||
notADir := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(notADir, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.option.uniqueCacheDirForWrite = filepath.Join(notADir, "swap")
|
||||
w.option.DefaultPermissions = true
|
||||
w.posixLocks = NewPosixLockTable()
|
||||
w.option.ConcurrentWriters = 1
|
||||
w.concurrentWriters = util.NewLimitedConcurrentExecutor(1)
|
||||
startFakeFiler(t, w, peer)
|
||||
w.inodeToPath.Lookup("/dir", 1, true, false, 0, false)
|
||||
w.inodeToPath.Lookup("/dir/file", 1, false, false, 42, true)
|
||||
w.inodeToPath.MarkChildrenCached("/dir")
|
||||
if err := w.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", proto.Clone(base).(*filer_pb.Entry)), 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := &fuse.OpenOut{}
|
||||
if code := w.Open(nil, &fuse.OpenIn{InHeader: fuse.InHeader{NodeId: 42}, Flags: syscall.O_WRONLY}, out); code != fuse.OK {
|
||||
t.Fatal(code)
|
||||
}
|
||||
fh := w.GetHandle(FileHandleId(out.Fh))
|
||||
g6AppendUploader(t, fh, 100, peer)
|
||||
|
||||
write := func(offset uint64, size int, value byte) (uint32, fuse.Status) {
|
||||
return w.Write(nil, &fuse.WriteIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: out.Fh, Offset: offset, Size: uint32(size), WriteFlags: writeFlags}, bytes.Repeat([]byte{value}, size))
|
||||
}
|
||||
if _, code := write(256, 256, 'A'); code != fuse.OK {
|
||||
t.Fatalf("buffered write: %v", code)
|
||||
}
|
||||
// The only writable chunk slot is taken, so this write needs the swap file.
|
||||
if _, code := write(4096, 256, 'B'); code == fuse.OK {
|
||||
t.Fatal("write through an unusable swap directory succeeded")
|
||||
}
|
||||
if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 512 {
|
||||
t.Fatalf("file size after the failed write = %d, want 512", size)
|
||||
}
|
||||
// The first 24 bytes fit the existing chunk; the next chunk needs the swap file.
|
||||
n, code := write(1000, 48, 'C')
|
||||
if writeFlags == 0 && (code != fuse.OK || n != 24) {
|
||||
t.Fatalf("boundary-crossing write = %d, %v; want a short write of 24", n, code)
|
||||
}
|
||||
if writeFlags != 0 && code == fuse.OK {
|
||||
t.Fatalf("boundary-crossing writeback = %d, OK; want an error", n)
|
||||
}
|
||||
if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 1024 {
|
||||
t.Fatalf("file size after the short write = %d, want 1024", size)
|
||||
}
|
||||
if code := w.Fsync(nil, &fuse.FsyncIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: out.Fh}); code != fuse.OK {
|
||||
t.Fatal(code)
|
||||
}
|
||||
|
||||
peer.mu.Lock()
|
||||
defer peer.mu.Unlock()
|
||||
want := append(bytes.Repeat([]byte{'O'}, 256), bytes.Repeat([]byte{'A'}, 256)...)
|
||||
want = append(want, make([]byte, 1000-512)...)
|
||||
want = append(want, bytes.Repeat([]byte{'C'}, 24)...)
|
||||
if got := g6StoredBytes(t, peer); peer.entry.Attributes.FileSize != 1024 || !bytes.Equal(got, want) {
|
||||
t.Fatalf("stored size %d, want 1024 with the original and buffered bytes", peer.entry.Attributes.FileSize)
|
||||
}
|
||||
}
|
||||
@@ -121,7 +121,7 @@ func newSlowDataFlush(t *testing.T, assignDelay, deadline time.Duration, stuckCo
|
||||
Chunks: stored,
|
||||
}, 0, 0)
|
||||
t.Cleanup(func() { wfs.ReleaseHandle(fh.fh) })
|
||||
if _, err := fh.dirtyPages.AddPage(offset, data, true, time.Now().UnixNano()); err != nil {
|
||||
if err := fh.dirtyPages.AddPage(offset, data, true, time.Now().UnixNano()); err != nil {
|
||||
t.Fatalf("AddPage: %v", err)
|
||||
}
|
||||
fh.dirtyMetadata = true
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ManagedTarget is a webhook target kept in the filer and edited from the
|
||||
// admin UI. Zero numeric settings take the same defaults as notification.toml.
|
||||
type ManagedTarget struct {
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
BearerToken string `json:"bearer_token,omitempty"`
|
||||
EventTypes []string `json:"event_types,omitempty"`
|
||||
PathPrefixes []string `json:"path_prefixes,omitempty"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
||||
MaxRetries int `json:"max_retries,omitempty"`
|
||||
BackoffSeconds int `json:"backoff_seconds,omitempty"`
|
||||
MaxBackoffSeconds int `json:"max_backoff_seconds,omitempty"`
|
||||
Workers int `json:"workers,omitempty"`
|
||||
BufferSize int `json:"buffer_size,omitempty"`
|
||||
}
|
||||
|
||||
type ManagedConfig struct {
|
||||
Targets []ManagedTarget `json:"targets"`
|
||||
}
|
||||
|
||||
var managedTargetName = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
||||
|
||||
func ParseManagedConfig(data []byte) (*ManagedConfig, error) {
|
||||
c := &ManagedConfig{}
|
||||
if len(strings.TrimSpace(string(data))) == 0 {
|
||||
return c, nil
|
||||
}
|
||||
if err := json.Unmarshal(data, c); err != nil {
|
||||
return nil, fmt.Errorf("parse webhook targets: %w", err)
|
||||
}
|
||||
if err := c.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *ManagedConfig) Validate() error {
|
||||
seen := make(map[string]bool, len(c.Targets))
|
||||
for i := range c.Targets {
|
||||
t := &c.Targets[i]
|
||||
if seen[t.Name] {
|
||||
return fmt.Errorf("duplicate webhook target %q", t.Name)
|
||||
}
|
||||
seen[t.Name] = true
|
||||
if err := t.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *ManagedTarget) Validate() error {
|
||||
if !managedTargetName.MatchString(t.Name) {
|
||||
return fmt.Errorf("webhook target name %q must be 1-64 letters, digits, '-' or '_'", t.Name)
|
||||
}
|
||||
u, err := url.Parse(t.Endpoint)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return fmt.Errorf("webhook target %q: endpoint must be an http or https URL", t.Name)
|
||||
}
|
||||
for _, et := range t.EventTypes {
|
||||
if !eventType(et).valid() {
|
||||
return fmt.Errorf("webhook target %q: invalid event type %q", t.Name, et)
|
||||
}
|
||||
}
|
||||
for _, p := range t.PathPrefixes {
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
return fmt.Errorf("webhook target %q: path prefix %q must start with '/'", t.Name, p)
|
||||
}
|
||||
}
|
||||
for field, v := range t.settings() {
|
||||
if v < 0 {
|
||||
return fmt.Errorf("webhook target %q: %s must not be negative", t.Name, field)
|
||||
}
|
||||
}
|
||||
if err := t.config().validate(); err != nil {
|
||||
return fmt.Errorf("webhook target %q: %w", t.Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *ManagedTarget) settings() map[string]int {
|
||||
return map[string]int{
|
||||
"timeout_seconds": t.TimeoutSeconds,
|
||||
"max_retries": t.MaxRetries,
|
||||
"backoff_seconds": t.BackoffSeconds,
|
||||
"max_backoff_seconds": t.MaxBackoffSeconds,
|
||||
"workers": t.Workers,
|
||||
"buffer_size": t.BufferSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ManagedTarget) config() *config {
|
||||
c := defaultConfig()
|
||||
c.endpoint = t.Endpoint
|
||||
c.authBearerToken = t.BearerToken
|
||||
c.eventTypes = t.EventTypes
|
||||
c.pathPrefixes = t.PathPrefixes
|
||||
for dst, v := range map[*int]int{
|
||||
&c.timeoutSeconds: t.TimeoutSeconds,
|
||||
&c.maxRetries: t.MaxRetries,
|
||||
&c.backoffSeconds: t.BackoffSeconds,
|
||||
&c.maxBackoffSeconds: t.MaxBackoffSeconds,
|
||||
&c.nWorkers: t.Workers,
|
||||
&c.bufferSize: t.BufferSize,
|
||||
} {
|
||||
if v > 0 {
|
||||
*dst = v
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// NewManagedQueue starts delivering to the enabled targets; it returns nil
|
||||
// when none is enabled.
|
||||
func NewManagedQueue(c *ManagedConfig) (*Queue, error) {
|
||||
if err := c.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var targets []*target
|
||||
for i := range c.Targets {
|
||||
if !c.Targets[i].Enabled {
|
||||
continue
|
||||
}
|
||||
t, err := newTarget(c.Targets[i].Name, c.Targets[i].config())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets = append(targets, t)
|
||||
}
|
||||
if len(targets) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
w := &Queue{managed: true}
|
||||
if err := w.initialize(targets); err != nil {
|
||||
w.cancel()
|
||||
return nil, err
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func TestParseManagedConfig(t *testing.T) {
|
||||
c, err := ParseManagedConfig(nil)
|
||||
if err != nil || len(c.Targets) != 0 {
|
||||
t.Fatalf("empty content: got %+v, %v", c, err)
|
||||
}
|
||||
|
||||
c, err = ParseManagedConfig([]byte(`{"targets":[{"name":"ingest","enabled":true,"endpoint":"https://example.com/hook","event_types":["create"],"path_prefixes":["/buckets/a"],"max_retries":5}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseManagedConfig: %v", err)
|
||||
}
|
||||
cfg := c.Targets[0].config()
|
||||
if cfg.maxRetries != 5 || cfg.timeoutSeconds != 10 || cfg.nWorkers != 5 || cfg.bufferSize != 10_000 {
|
||||
t.Fatalf("unexpected settings %+v", cfg)
|
||||
}
|
||||
|
||||
for name, body := range map[string]string{
|
||||
"bad name": `{"targets":[{"name":"a b","endpoint":"https://example.com"}]}`,
|
||||
"duplicate": `{"targets":[{"name":"a","endpoint":"https://example.com"},{"name":"a","endpoint":"https://example.com"}]}`,
|
||||
"no scheme": `{"targets":[{"name":"a","endpoint":"example.com/hook"}]}`,
|
||||
"ftp endpoint": `{"targets":[{"name":"a","endpoint":"ftp://example.com/hook"}]}`,
|
||||
"event type": `{"targets":[{"name":"a","endpoint":"https://example.com","event_types":["chmod"]}]}`,
|
||||
"relative path": `{"targets":[{"name":"a","endpoint":"https://example.com","path_prefixes":["buckets"]}]}`,
|
||||
"timeout range": `{"targets":[{"name":"a","endpoint":"https://example.com","timeout_seconds":301}]}`,
|
||||
"neg timeout": `{"targets":[{"name":"a","endpoint":"https://example.com","timeout_seconds":-1}]}`,
|
||||
"neg workers": `{"targets":[{"name":"a","endpoint":"https://example.com","workers":-1}]}`,
|
||||
"not json": `targets`,
|
||||
} {
|
||||
if _, err := ParseManagedConfig([]byte(body)); err == nil {
|
||||
t.Errorf("%s: expected an error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedQueueDeliversAndDrainsOnClose(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
var auth atomic.Value
|
||||
release := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
auth.Store(r.Header.Get("Authorization"))
|
||||
<-release
|
||||
hits.Add(1)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
|
||||
q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{
|
||||
{Name: "on", Enabled: true, Endpoint: server.URL, BearerToken: "secret", PathPrefixes: []string{"/data/"}},
|
||||
{Name: "off", Endpoint: "https://example.com/never"},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("NewManagedQueue: %v", err)
|
||||
}
|
||||
if len(q.targets) != 1 {
|
||||
t.Fatalf("expected only the enabled target, got %d", len(q.targets))
|
||||
}
|
||||
|
||||
msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}}
|
||||
for _, key := range []string{"/data/f.txt", "/other/f.txt"} {
|
||||
if err := q.SendMessage(key, msg); err != nil {
|
||||
t.Fatalf("SendMessage(%q): %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
closed := make(chan struct{})
|
||||
go func() {
|
||||
q.Close(10 * time.Second)
|
||||
close(closed)
|
||||
}()
|
||||
select {
|
||||
case <-closed:
|
||||
t.Fatal("Close returned before the in-flight delivery finished")
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Close did not return after the delivery finished")
|
||||
}
|
||||
|
||||
if got := hits.Load(); got != 1 {
|
||||
t.Fatalf("expected 1 delivery, got %d", got)
|
||||
}
|
||||
if got, _ := auth.Load().(string); !strings.HasSuffix(got, "secret") {
|
||||
t.Fatalf("expected the bearer token, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedQueueNoEnabledTargets(t *testing.T) {
|
||||
q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{{Name: "off", Endpoint: "https://example.com"}}})
|
||||
if err != nil || q != nil {
|
||||
t.Fatalf("expected no queue, got %v, %v", q, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedQueueCloseWaitsForBlockedSender(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
release := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
<-release
|
||||
hits.Add(1)
|
||||
}))
|
||||
defer server.Close()
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
|
||||
q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{
|
||||
{Name: "slow", Enabled: true, Endpoint: server.URL, Workers: 1, BufferSize: 100},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("NewManagedQueue: %v", err)
|
||||
}
|
||||
msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}}
|
||||
for i := 0; i < 100; i++ {
|
||||
if err := q.SendMessage("/a", msg); err != nil {
|
||||
t.Fatalf("SendMessage: %v", err)
|
||||
}
|
||||
}
|
||||
blocked := make(chan error, 1)
|
||||
go func() { blocked <- q.SendMessage("/b", msg) }()
|
||||
for q.senders.Load() == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
closed := make(chan struct{})
|
||||
go func() {
|
||||
q.Close(10 * time.Second)
|
||||
close(closed)
|
||||
}()
|
||||
for !q.closing.Load() {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if err := q.SendMessage("/c", msg); err != ErrQueueClosed {
|
||||
t.Fatalf("expected ErrQueueClosed after Close started, got %v", err)
|
||||
}
|
||||
|
||||
unblock()
|
||||
if err := <-blocked; err != nil {
|
||||
t.Fatalf("blocked sender: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Close did not return")
|
||||
}
|
||||
if got := hits.Load(); got != 101 {
|
||||
t.Fatalf("expected both accepted events delivered, got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -69,8 +69,10 @@ type config struct {
|
||||
pathPrefixes []string
|
||||
}
|
||||
|
||||
func defaultConfig() *config {
|
||||
return &config{
|
||||
func newConfigWithDefaults(configuration util.Configuration, prefix string) *config {
|
||||
c := &config{
|
||||
endpoint: configuration.GetString(prefix + "endpoint"),
|
||||
authBearerToken: configuration.GetString(prefix + "bearer_token"),
|
||||
timeoutSeconds: 10,
|
||||
maxRetries: 3,
|
||||
backoffSeconds: 3,
|
||||
@@ -78,12 +80,6 @@ func defaultConfig() *config {
|
||||
nWorkers: 5,
|
||||
bufferSize: 10_000,
|
||||
}
|
||||
}
|
||||
|
||||
func newConfigWithDefaults(configuration util.Configuration, prefix string) *config {
|
||||
c := defaultConfig()
|
||||
c.endpoint = configuration.GetString(prefix + "endpoint")
|
||||
c.authBearerToken = configuration.GetString(prefix + "bearer_token")
|
||||
|
||||
if bufferSize := configuration.GetInt(prefix + "buffer_size"); bufferSize > 0 {
|
||||
c.bufferSize = bufferSize
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ThreeDotsLabs/watermill"
|
||||
@@ -41,9 +40,9 @@ type target struct {
|
||||
|
||||
func newTarget(name string, cfg *config) (*target, error) {
|
||||
t := &target{
|
||||
name: name,
|
||||
config: cfg,
|
||||
filter: newFilter(cfg),
|
||||
name: name,
|
||||
config: cfg,
|
||||
filter: newFilter(cfg),
|
||||
sem: make(chan struct{}, cfg.nWorkers),
|
||||
inbox: make(chan *message.Message, cfg.bufferSize),
|
||||
pending: make(chan struct{}, cfg.bufferSize),
|
||||
@@ -75,18 +74,10 @@ type Queue struct {
|
||||
queueChannel *gochannel.GoChannel
|
||||
targets []*target
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
stopped chan struct{}
|
||||
// managed queues are replaced on reload and stopped with Close
|
||||
managed bool
|
||||
closing atomic.Bool
|
||||
senders atomic.Int64
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// ErrQueueClosed is returned by SendMessage once Close has started.
|
||||
var ErrQueueClosed = errors.New("webhook queue closed")
|
||||
|
||||
func (w *Queue) GetName() string {
|
||||
return queueName
|
||||
}
|
||||
@@ -147,11 +138,6 @@ func buildTargets(configuration util.Configuration, prefix string) ([]*target, e
|
||||
}
|
||||
|
||||
func (w *Queue) SendMessage(key string, msg proto.Message) error {
|
||||
w.senders.Add(1)
|
||||
defer w.senders.Add(-1)
|
||||
if w.closing.Load() {
|
||||
return ErrQueueClosed
|
||||
}
|
||||
eventNotification, ok := msg.(*filer_pb.EventNotification)
|
||||
if !ok {
|
||||
return nil
|
||||
@@ -239,7 +225,6 @@ func (w *Queue) Initialize(configuration util.Configuration, prefix string) erro
|
||||
|
||||
func (w *Queue) initialize(targets []*target) error {
|
||||
w.ctx, w.cancel = context.WithCancel(context.Background())
|
||||
w.stopped = make(chan struct{})
|
||||
w.targets = targets
|
||||
|
||||
if err := w.setupWatermillQueue(targets); err != nil {
|
||||
@@ -296,9 +281,7 @@ func (w *Queue) setupWatermillQueue(targets []*target) error {
|
||||
return fmt.Errorf("failed to create poison queue: %w", err)
|
||||
}
|
||||
|
||||
if !w.managed {
|
||||
router.AddPlugin(plugin.SignalsHandler)
|
||||
}
|
||||
router.AddPlugin(plugin.SignalsHandler)
|
||||
// outermost: the reservation SendMessage took releases once the whole
|
||||
// chain (retries, dead letter) is done, bounding in-flight deliveries.
|
||||
// poison wraps retry so the delivery is retried before dead-lettering.
|
||||
@@ -346,37 +329,11 @@ func (w *Queue) setupWatermillQueue(targets []*target) error {
|
||||
go func() {
|
||||
wg.Wait()
|
||||
w.cancel()
|
||||
close(w.stopped)
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close stops the queue once every sender already inside SendMessage has
|
||||
// returned and every target has delivered or dead-lettered what it holds, or
|
||||
// when drainTimeout passes.
|
||||
func (w *Queue) Close(drainTimeout time.Duration) {
|
||||
w.closing.Store(true)
|
||||
deadline := time.Now().Add(drainTimeout)
|
||||
for (w.senders.Load() > 0 || !w.drained()) && time.Now().Before(deadline) {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
w.cancel()
|
||||
<-w.stopped
|
||||
if err := w.queueChannel.Close(); err != nil {
|
||||
glog.Warningf("close webhook queue: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Queue) drained() bool {
|
||||
for _, t := range w.targets {
|
||||
if len(t.pending) > 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// forwardTargetMessages publishes each queued event to the target's topic in
|
||||
// order. Publish detaches its delivery, so the pending reservation is what
|
||||
// keeps a saturated target from queuing without bound.
|
||||
|
||||
@@ -57,54 +57,6 @@ func ScanMetadataEventSkeleton(data []byte) (skeleton *SubscribeMetadataResponse
|
||||
return skeleton, true
|
||||
}
|
||||
|
||||
// ScanMetadataEventDirectories extracts an event's directory and its
|
||||
// new_parent_path without allocating. ok=false means the data is malformed.
|
||||
func ScanMetadataEventDirectories(data []byte) (directory, newParentPath []byte, ok bool) {
|
||||
rest := data
|
||||
for len(rest) > 0 {
|
||||
num, typ, n := protowire.ConsumeTag(rest)
|
||||
if n < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest = rest[n:]
|
||||
if typ != protowire.BytesType || (num != 1 && num != 2) {
|
||||
m := protowire.ConsumeFieldValue(num, typ, rest)
|
||||
if m < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest = rest[m:]
|
||||
continue
|
||||
}
|
||||
v, m := protowire.ConsumeBytes(rest)
|
||||
if m < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest = rest[m:]
|
||||
if num == 1 { // directory
|
||||
directory = v
|
||||
continue
|
||||
}
|
||||
// event_notification: repeated occurrences merge, so the last
|
||||
// new_parent_path wins
|
||||
for len(v) > 0 {
|
||||
fnum, ftyp, fn := protowire.ConsumeTag(v)
|
||||
if fn < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
v = v[fn:]
|
||||
fm := protowire.ConsumeFieldValue(fnum, ftyp, v)
|
||||
if fm < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
if fnum == 4 && ftyp == protowire.BytesType { // new_parent_path
|
||||
newParentPath, _ = protowire.ConsumeBytes(v)
|
||||
}
|
||||
v = v[fm:]
|
||||
}
|
||||
}
|
||||
return directory, newParentPath, true
|
||||
}
|
||||
|
||||
func scanEventNotificationSkeleton(data []byte) (*EventNotification, bool) {
|
||||
notification := &EventNotification{}
|
||||
rest := data
|
||||
|
||||
@@ -110,86 +110,3 @@ func TestScanMetadataEventSkeletonEmptyPayload(t *testing.T) {
|
||||
t.Fatalf("empty payload: %+v ok=%v", skeleton, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataDirectoryMayMatchSubscriptionCoversMatches(t *testing.T) {
|
||||
dirs := []string{"/", "/data", "/data/pvc-7", "/data/pvc-70", "/data/pvc-7/sub", "/data/pvc-1", "/other", "/data/./pvc-7", "/data//pvc-7", "/data/pvc-1/../pvc-7"}
|
||||
names := []string{"pvc-7", "pvc-70", "a.log", "sub", ""}
|
||||
filters := []struct {
|
||||
prefix string
|
||||
prefixes []string
|
||||
dirs []string
|
||||
}{
|
||||
{prefix: "/data/pvc-7/"},
|
||||
{prefix: "/data/pvc-7"},
|
||||
{prefix: "/data/"},
|
||||
{prefix: "/"},
|
||||
{prefix: "/data/pvc-7/sub/deeper/"},
|
||||
{prefix: "/x", prefixes: []string{"/other/", "/data/pvc-7/"}},
|
||||
{prefix: "/x", dirs: []string{"/data/pvc-1"}},
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
for _, target := range append([]string{""}, dirs...) {
|
||||
for _, name := range names {
|
||||
event := &SubscribeMetadataResponse{Directory: dir, EventNotification: &EventNotification{
|
||||
OldEntry: &Entry{Name: name}, NewEntry: &Entry{Name: name}, NewParentPath: target}}
|
||||
for _, f := range filters {
|
||||
if !MetadataEventMatchesSubscription(event, f.prefix, f.prefixes, f.dirs) {
|
||||
continue
|
||||
}
|
||||
if !MetadataDirectoryMayMatchSubscription([]byte(dir), f.prefix, f.prefixes, f.dirs) &&
|
||||
(target == "" || !MetadataDirectoryMayMatchSubscription([]byte(target), f.prefix, f.prefixes, f.dirs)) {
|
||||
t.Errorf("event %q/%q -> %q matches %+v but neither directory may match", dir, name, target, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
dir, prefix string
|
||||
want bool
|
||||
}{
|
||||
{"/data/pvc-1", "/data/pvc-7/", false},
|
||||
{"/data/pvc-70", "/data/pvc-7/", false},
|
||||
{"/other", "/data/", false},
|
||||
{"/data/pvc-7", "/data/pvc-7/", true},
|
||||
{"/data/pvc-7/sub", "/data/pvc-7/", true},
|
||||
{"/data", "/data/pvc-7", true},
|
||||
{"/", "/data/", true},
|
||||
} {
|
||||
if got := MetadataDirectoryMayMatchSubscription([]byte(c.dir), c.prefix, nil, nil); got != c.want {
|
||||
t.Errorf("dir %q prefix %q: got %v, want %v", c.dir, c.prefix, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanMetadataEventDirectories(t *testing.T) {
|
||||
for _, event := range []*SubscribeMetadataResponse{
|
||||
{Directory: "/data/pvc-7", EventNotification: &EventNotification{NewEntry: bigEntry("a.log")}},
|
||||
{Directory: "/data/pvc-7", EventNotification: &EventNotification{OldEntry: bigEntry("a.log"), NewEntry: bigEntry("a.log"), NewParentPath: "/data/pvc-9"}},
|
||||
{Directory: "/data/pvc-7"},
|
||||
{},
|
||||
} {
|
||||
dir, target, ok := ScanMetadataEventDirectories(marshalEvent(t, event))
|
||||
if !ok || string(dir) != event.Directory || string(target) != event.GetEventNotification().GetNewParentPath() {
|
||||
t.Errorf("scan of %+v: dir %q target %q ok %v", event, dir, target, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// repeated event_notification fields merge: the last new_parent_path wins
|
||||
first := marshalEvent(t, &SubscribeMetadataResponse{Directory: "/a", EventNotification: &EventNotification{NewParentPath: "/b"}})
|
||||
second := marshalEvent(t, &SubscribeMetadataResponse{EventNotification: &EventNotification{NewEntry: &Entry{Name: "x"}, NewParentPath: "/c"}})
|
||||
merged := append(append([]byte{}, first...), second...)
|
||||
full := &SubscribeMetadataResponse{}
|
||||
if err := proto.Unmarshal(merged, full); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir, target, ok := ScanMetadataEventDirectories(merged)
|
||||
if !ok || string(dir) != full.Directory || string(target) != full.EventNotification.NewParentPath {
|
||||
t.Errorf("merged: dir %q target %q ok %v, full decode %q %q", dir, target, ok, full.Directory, full.EventNotification.NewParentPath)
|
||||
}
|
||||
|
||||
if _, _, ok := ScanMetadataEventDirectories([]byte{0xff, 0xff, 0xff}); ok {
|
||||
t.Error("malformed payload must not scan")
|
||||
}
|
||||
}
|
||||
@@ -414,44 +414,6 @@ func metadataEventMatchesPath(fullPath, dirPath, pathPrefix string, pathPrefixes
|
||||
return strings.HasPrefix(fullPath, pathPrefix)
|
||||
}
|
||||
|
||||
// MetadataDirectoryMayMatchSubscription reports whether an event in dir could
|
||||
// match the subscription, before the event is decoded. It errs toward true.
|
||||
func MetadataDirectoryMayMatchSubscription(dir []byte, pathPrefix string, pathPrefixes []string, directories []string) bool {
|
||||
if pathPrefix == "" || len(dir) == 0 || directoryMayHavePrefix(dir, pathPrefix) {
|
||||
return true
|
||||
}
|
||||
for _, p := range pathPrefixes {
|
||||
if directoryMayHavePrefix(dir, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, d := range directories {
|
||||
if string(dir) == d {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// matching cleans the joined path, so only a clean dir can be ruled out
|
||||
for i := 0; i+1 < len(dir); i++ {
|
||||
if dir[i] == '/' && (dir[i+1] == '/' || dir[i+1] == '.') {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// directoryMayHavePrefix reports whether some path dir/<name> can start with
|
||||
// prefix: dir+"/" and prefix agree on their common length.
|
||||
func directoryMayHavePrefix(dir []byte, prefix string) bool {
|
||||
n := min(len(dir), len(prefix))
|
||||
if string(dir[:n]) != prefix[:n] {
|
||||
return false
|
||||
}
|
||||
if len(prefix) <= len(dir) || dir[len(dir)-1] == '/' {
|
||||
return true
|
||||
}
|
||||
return prefix[len(dir)] == '/'
|
||||
}
|
||||
|
||||
func hasPrefixIn(text string, prefixes []string) bool {
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(text, p) {
|
||||
|
||||
@@ -167,58 +167,14 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
|
||||
if listener == nil {
|
||||
return
|
||||
}
|
||||
listener = &localSocketListener{Listener: listener}
|
||||
glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath)
|
||||
go serveGrpcOnLocalListener(grpcServer, socketPath, listener)
|
||||
}
|
||||
|
||||
// localSocketCheckInterval is how often a served socket path is checked for
|
||||
// removal by something else, such as a temp cleaner aging /tmp.
|
||||
var localSocketCheckInterval = time.Minute
|
||||
|
||||
// serveGrpcOnLocalListener serves the socket until the gRPC server stops,
|
||||
// rebinding it when the socket file is removed underneath the listener.
|
||||
func serveGrpcOnLocalListener(grpcServer *grpc.Server, socketPath string, listener net.Listener) {
|
||||
served := make(chan error, 1)
|
||||
serve := func(l net.Listener) {
|
||||
// A replaced listener must not unlink the path its successor binds;
|
||||
// every exit below removes the socket file itself.
|
||||
if unixListener, ok := l.(*net.UnixListener); ok {
|
||||
unixListener.SetUnlinkOnClose(false)
|
||||
go func() {
|
||||
if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped {
|
||||
glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err)
|
||||
}
|
||||
go func() { served <- grpcServer.Serve(&localSocketListener{Listener: l}) }()
|
||||
}
|
||||
serve(listener)
|
||||
ticker := time.NewTicker(localSocketCheckInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case err := <-served:
|
||||
if err != nil && err != grpc.ErrServerStopped {
|
||||
glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err)
|
||||
}
|
||||
os.Remove(socketPath)
|
||||
return
|
||||
case <-ticker.C:
|
||||
if _, err := os.Lstat(socketPath); !os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
// Bind before closing the old listener, so a failed bind is
|
||||
// retried on the next check and shutdown still ends this loop.
|
||||
replacement := ListenOnLocalSocket(socketPath)
|
||||
if replacement == nil {
|
||||
continue
|
||||
}
|
||||
glog.Warningf("gRPC Unix socket %s was removed and has been recreated", socketPath)
|
||||
listener.Close()
|
||||
if err := <-served; err == nil || err == grpc.ErrServerStopped {
|
||||
replacement.Close()
|
||||
os.Remove(socketPath)
|
||||
return
|
||||
}
|
||||
listener = replacement
|
||||
serve(listener)
|
||||
}
|
||||
}
|
||||
os.Remove(socketPath)
|
||||
}()
|
||||
}
|
||||
|
||||
// localSocketListener re-applies the buffer sizes to every accepted connection.
|
||||
@@ -277,21 +233,22 @@ func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
|
||||
func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
||||
var options []grpc.DialOption
|
||||
|
||||
// Route through Unix socket if one is registered for this address's port.
|
||||
// The socket file can vanish under a running server (temp cleaners age
|
||||
// /tmp), so a failed socket dial falls back to TCP.
|
||||
// Route through Unix socket if one is registered for this address's port
|
||||
if socketPath := resolveLocalGrpcSocket(address); socketPath != "" {
|
||||
options = append(options, grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
||||
options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
|
||||
d := net.Dialer{Control: setLocalSocketBuffers}
|
||||
conn, err := d.DialContext(ctx, "unix", socketPath)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
glog.V(1).Infof("dial unix %s: %v, dialing %s over TCP", socketPath, err, addr)
|
||||
return dialGrpcTcp(ctx, addr)
|
||||
}
|
||||
return conn, err
|
||||
return d.DialContext(ctx, "unix", socketPath)
|
||||
}))
|
||||
} else {
|
||||
options = append(options, grpc.WithContextDialer(dialGrpcTcp))
|
||||
// Always install so a conn cached before SetOutboundLocalIP binds once set;
|
||||
// the stock net.Dialer until then preserves gRPC's default dial behavior.
|
||||
options = append(options, grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
||||
if util.OutboundLocalAddr() == nil {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "tcp", addr)
|
||||
}
|
||||
return util.OutboundDialContext(ctx, "tcp", addr)
|
||||
}))
|
||||
}
|
||||
|
||||
options = append(options,
|
||||
@@ -315,17 +272,6 @@ func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...gr
|
||||
return grpc.DialContext(ctx, address, options...)
|
||||
}
|
||||
|
||||
// dialGrpcTcp is installed even before SetOutboundLocalIP so a cached conn
|
||||
// binds once it is set; the stock net.Dialer until then preserves gRPC's
|
||||
// default dial behavior.
|
||||
func dialGrpcTcp(ctx context.Context, addr string) (net.Conn, error) {
|
||||
if util.OutboundLocalAddr() == nil {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "tcp", addr)
|
||||
}
|
||||
return util.OutboundDialContext(ctx, "tcp", addr)
|
||||
}
|
||||
|
||||
func getOrCreateConnection(address string, waitForReady bool, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
|
||||
|
||||
grpcClientsLock.Lock()
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
package pb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/health"
|
||||
healthpb "google.golang.org/grpc/health/grpc_health_v1"
|
||||
)
|
||||
|
||||
func isolateLocalGrpcSockets(t *testing.T) {
|
||||
t.Helper()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
localGrpcSocketsLock.Lock()
|
||||
prevSockets, prevHosts := localGrpcSockets, localGrpcHosts
|
||||
localGrpcSockets = make(map[int]string)
|
||||
localGrpcHosts = make(map[int]map[string]struct{})
|
||||
localGrpcSocketsLock.Unlock()
|
||||
t.Cleanup(func() {
|
||||
localGrpcSocketsLock.Lock()
|
||||
localGrpcSockets, localGrpcHosts = prevSockets, prevHosts
|
||||
localGrpcSocketsLock.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
// localSocketDir is short because TMPDIR can push socket paths past the
|
||||
// sockaddr_un limit.
|
||||
func localSocketDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir, err := os.MkdirTemp("/tmp", "swsock")
|
||||
if err != nil {
|
||||
t.Fatalf("create socket dir: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
return dir
|
||||
}
|
||||
|
||||
func startHealthGrpcServer(t *testing.T) (*grpc.Server, int) {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
grpcServer := grpc.NewServer()
|
||||
healthpb.RegisterHealthServer(grpcServer, health.NewServer())
|
||||
go grpcServer.Serve(listener)
|
||||
t.Cleanup(grpcServer.Stop)
|
||||
return grpcServer, listener.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
func checkGrpcHealth(t *testing.T, address string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
conn, err := GrpcDial(ctx, address, false, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("dial %s: %v", address, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil {
|
||||
t.Fatalf("health check over %s: %v", address, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrpcDialFallsBackToTcpWhenLocalSocketIsMissing(t *testing.T) {
|
||||
isolateLocalGrpcSockets(t)
|
||||
_, port := startHealthGrpcServer(t)
|
||||
RegisterLocalGrpcSocket("127.0.0.1", port, filepath.Join(localSocketDir(t), "removed.sock"))
|
||||
|
||||
checkGrpcHealth(t, net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
|
||||
}
|
||||
|
||||
func TestServeGrpcOnLocalSocketRecreatesRemovedSocket(t *testing.T) {
|
||||
isolateLocalGrpcSockets(t)
|
||||
prevInterval := localSocketCheckInterval
|
||||
localSocketCheckInterval = 10 * time.Millisecond
|
||||
t.Cleanup(func() { localSocketCheckInterval = prevInterval })
|
||||
|
||||
grpcServer, port := startHealthGrpcServer(t)
|
||||
socketDir := localSocketDir(t)
|
||||
socketPath := filepath.Join(socketDir, "grpc.sock")
|
||||
RegisterLocalGrpcSocket("127.0.0.1", port, socketPath)
|
||||
ServeGrpcOnLocalSocket(grpcServer, port)
|
||||
waitForSocketFile(t, socketPath)
|
||||
|
||||
// Rebinding fails while the folder is gone and must be retried later.
|
||||
if err := os.RemoveAll(socketDir); err != nil {
|
||||
t.Fatalf("remove socket dir: %v", err)
|
||||
}
|
||||
time.Sleep(10 * localSocketCheckInterval)
|
||||
if err := os.Mkdir(socketDir, 0o700); err != nil {
|
||||
t.Fatalf("recreate socket dir: %v", err)
|
||||
}
|
||||
waitForSocketFile(t, socketPath)
|
||||
|
||||
conn, err := grpc.NewClient("unix:"+socketPath, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("dial recreated socket: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil {
|
||||
t.Fatalf("health check over the recreated socket: %v", err)
|
||||
}
|
||||
|
||||
grpcServer.Stop()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
if _, err := os.Lstat(socketPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("socket %s left behind after the server stopped", socketPath)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForSocketFile(t *testing.T, socketPath string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
if info, err := os.Lstat(socketPath); err == nil && info.Mode()&os.ModeSocket != 0 {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("socket %s was not created", socketPath)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -1948,14 +1948,6 @@ func isBucketListingRequest(r *http.Request, action Action) bool {
|
||||
return object == "" || object == "/"
|
||||
}
|
||||
|
||||
// addEmptyListPrefix reports a listing that names no prefix as s3:prefix "",
|
||||
// as AWS and the advanced IAM path do.
|
||||
func addEmptyListPrefix(conditions map[string][]string, r *http.Request, action Action) {
|
||||
if _, ok := conditions["s3:prefix"]; !ok && isBucketListingRequest(r, action) {
|
||||
conditions["s3:prefix"] = []string{""}
|
||||
}
|
||||
}
|
||||
|
||||
// multipartListingAction returns the scoped action for a request the router
|
||||
// sends to ListObjectPartsHandler or ListMultipartUploadsHandler, or "" for
|
||||
// any other request. It follows the route order in s3api_server.go and reads
|
||||
@@ -2760,7 +2752,6 @@ func (iam *IdentityAccessManagement) evaluateAttachedIAMPolicies(r *http.Request
|
||||
principal := buildPrincipalARN(identity, r)
|
||||
explicitAllow := false
|
||||
conditions := engine.ExtractConditionValuesFromRequest(r)
|
||||
addEmptyListPrefix(conditions, r, action)
|
||||
for key, values := range vectorIAMConditions(r) {
|
||||
conditions[key] = values
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
||||
// Prepare error handling outside callback scope
|
||||
var encryptionError error
|
||||
|
||||
if err := s3a.uploadFiler(*input.Bucket, *input.Key).mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
|
||||
if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
@@ -388,14 +388,14 @@ func applyMultipartSSES3HeadersFromUploadEntry(dst *filer_pb.Entry, sses3Info *m
|
||||
|
||||
func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload, uploadDirectory, entryName, dirName string, completedPartNumbers []int, completedPartMap map[int][]string, maxPartNo int) (*multipartCompletionState, *CompleteMultipartUploadResult, s3err.ErrorCode) {
|
||||
if output, _ := s3a.committedCompletion(r, input); output != nil {
|
||||
cleanupEntries, _, cleanupErr := s3a.uploadFiler(*input.Bucket, *input.Key).list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1)
|
||||
cleanupEntries, _, cleanupErr := s3a.list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1)
|
||||
if cleanupErr != nil && !errors.Is(cleanupErr, filer_pb.ErrNotFound) {
|
||||
glog.Warningf("completeMultipartUpload: failed to list stale upload directory %s for cleanup: %v", uploadDirectory, cleanupErr)
|
||||
}
|
||||
return &multipartCompletionState{deleteEntries: cleanupEntries, metadataOnlyCleanup: true}, output, s3err.ErrNone
|
||||
}
|
||||
|
||||
entries, _, err := s3a.uploadFiler(*input.Bucket, *input.Key).list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1)
|
||||
entries, _, err := s3a.list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1)
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err)
|
||||
if isFilerNotFound(err) {
|
||||
@@ -416,7 +416,7 @@ func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *
|
||||
return nil, nil, s3err.ErrNoSuchUpload
|
||||
}
|
||||
|
||||
pentry, err := s3a.uploadFiler(*input.Bucket, *input.Key).getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId)
|
||||
pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId)
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err)
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
@@ -796,7 +796,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
// The transaction removes the upload directory metadata-only,
|
||||
// so the part entries the object does not reference are freed
|
||||
// first or their chunks leak.
|
||||
if err := s3a.deleteUnusedPartEntries(r.Context(), uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState); err != nil {
|
||||
if err := s3a.deleteUnusedPartEntries(r.Context(), uploadDirectory, *input.Bucket, *input.UploadId, completionState); err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
@@ -807,7 +807,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
// Roll back only while the upload directory survives: once the
|
||||
// transaction removed it, the version file is the only record
|
||||
// left and deleting it would make a retry impossible.
|
||||
if exists, _ := s3a.uploadFiler(*input.Bucket, *input.Key).exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true); exists {
|
||||
if exists, _ := s3a.exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true); exists {
|
||||
if rollbackErr := s3a.rollbackMultipartVersion(versionDir, versionFileName); rollbackErr != nil {
|
||||
glog.Errorf("completeMultipartUpload: failed to rollback version %s for %s/%s after routed finalize error: %v", versionId, *input.Bucket, *input.Key, rollbackErr)
|
||||
completionState.manifestsReferenced = true
|
||||
@@ -849,7 +849,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
|
||||
if versioningState == s3_constants.VersioningSuspended {
|
||||
// For suspended versioning, add "null" version ID metadata and return "null" version ID
|
||||
removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState)
|
||||
removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.UploadId, completionState)
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
@@ -945,7 +945,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
}
|
||||
|
||||
// For non-versioned buckets, create main object file
|
||||
removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState)
|
||||
removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.UploadId, completionState)
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
@@ -1073,9 +1073,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
cleanupCtx := withFilerRetryBudget(context.Background(), filerRetryRequestBudget)
|
||||
// Keep the directory when an entry delete failed so its metadata
|
||||
// still references the chunks; removing it metadata-only orphans them.
|
||||
if err := s3a.deleteUnusedPartEntries(cleanupCtx, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState); err != nil {
|
||||
if err := s3a.deleteUnusedPartEntries(cleanupCtx, uploadDirectory, *input.Bucket, *input.UploadId, completionState); err != nil {
|
||||
glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err)
|
||||
} else if err := s3a.uploadFiler(*input.Bucket, *input.Key).rm(cleanupCtx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil {
|
||||
} else if err := s3a.rm(cleanupCtx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil {
|
||||
glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err)
|
||||
}
|
||||
}
|
||||
@@ -1091,10 +1091,10 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
// reference. A finalize that removes the whole upload directory metadata-only
|
||||
// frees them first, or their chunks leak; a failure here must abort the
|
||||
// completion, since the surviving entries keep the upload retriable.
|
||||
func (s3a *S3ApiServer) deleteUnusedPartEntries(ctx context.Context, uploadDirectory, bucket, object, uploadId string, completionState *multipartCompletionState) error {
|
||||
func (s3a *S3ApiServer) deleteUnusedPartEntries(ctx context.Context, uploadDirectory, bucket, uploadId string, completionState *multipartCompletionState) error {
|
||||
var lastErr error
|
||||
for _, deleteEntry := range completionState.deleteEntries {
|
||||
if err := s3a.uploadFiler(bucket, object).rm(ctx, uploadDirectory, deleteEntry.Name, !completionState.metadataOnlyCleanup, true); err != nil {
|
||||
if err := s3a.rm(ctx, uploadDirectory, deleteEntry.Name, !completionState.metadataOnlyCleanup, true); err != nil {
|
||||
glog.Warningf("completeMultipartUpload cleanup %s upload %s unused %s : %v", bucket, uploadId, deleteEntry.Name, err)
|
||||
lastErr = err
|
||||
}
|
||||
@@ -1182,7 +1182,7 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
|
||||
glog.V(2).Infof("abortMultipartUpload input %v", input)
|
||||
|
||||
uploadsFolder := s3a.genUploadsFolder(*input.Bucket)
|
||||
uploadEntry, err := s3a.uploadFiler(*input.Bucket, aws.StringValue(input.Key)).getEntry(uploadsFolder, *input.UploadId)
|
||||
uploadEntry, err := s3a.getEntry(uploadsFolder, *input.UploadId)
|
||||
// complete and abort both remove .uploads/<id>; a missing record is an upload
|
||||
// that does not exist (NoSuchUpload on AWS), not one this call discarded
|
||||
if err != nil {
|
||||
@@ -1215,7 +1215,7 @@ func (s3a *S3ApiServer) removeUploadDir(bucket, uploadId, object string) s3err.E
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
if completed {
|
||||
if err := s3a.uploadFiler(bucket, object).rm(context.Background(), s3a.genUploadsFolder(bucket), uploadId, false, true); err != nil {
|
||||
if err := s3a.rm(context.Background(), s3a.genUploadsFolder(bucket), uploadId, false, true); err != nil {
|
||||
glog.V(1).Infof("bucket %s remove upload %s: %v", bucket, uploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
@@ -1233,7 +1233,7 @@ func (s3a *S3ApiServer) removeUploadDir(bucket, uploadId, object string) s3err.E
|
||||
func (s3a *S3ApiServer) routedUploadDirDelete(bucket, uploadId, object string) s3err.ErrorCode {
|
||||
uploadsFolder := s3a.genUploadsFolder(bucket)
|
||||
rmUploadDir := func(isDeleteData bool) s3err.ErrorCode {
|
||||
if err := s3a.uploadFiler(bucket, object).rm(context.Background(), uploadsFolder, uploadId, isDeleteData, true); err != nil {
|
||||
if err := s3a.rm(context.Background(), uploadsFolder, uploadId, isDeleteData, true); err != nil {
|
||||
glog.V(1).Infof("bucket %s remove upload %s: %v", bucket, uploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
@@ -1588,7 +1588,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
|
||||
// directory as empty instead of erroring, so listing alone cannot tell a
|
||||
// completed upload (NoSuchUpload on AWS) from an open upload with no parts
|
||||
// yet (200 with an empty list). Probe the upload record instead.
|
||||
pentry, err := s3a.uploadFiler(*input.Bucket, *input.Key).getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId)
|
||||
pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil, s3err.ErrNoSuchUpload
|
||||
@@ -1618,7 +1618,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
|
||||
output.IsTruncated = aws.Bool(false)
|
||||
return output, s3err.ErrNone
|
||||
}
|
||||
entries, isLast, err := s3a.uploadFiler(*input.Bucket, *input.Key).list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d", *input.PartNumberMarker+1), true, uint32(*input.MaxParts))
|
||||
entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d", *input.PartNumberMarker+1), true, uint32(*input.MaxParts))
|
||||
if err != nil {
|
||||
// A store that reports the missing upload directory as not-found means
|
||||
// the upload is gone (completed or aborted), not a store error.
|
||||
|
||||
@@ -3,7 +3,6 @@ package s3api
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -86,50 +85,6 @@ func TestEvaluateIAMPolicies_ListBucketPrefixCondition(t *testing.T) {
|
||||
"prefix outside the s3:prefix condition must be denied")
|
||||
}
|
||||
|
||||
// A listing without ?prefix is the empty prefix, so the home-directory policy
|
||||
// that grants s3:prefix "" for the root level matches it in attached and
|
||||
// bucket policies alike.
|
||||
func TestListBucketWithoutPrefixMatchesEmptyPrefix(t *testing.T) {
|
||||
const bucket = "test-bucket"
|
||||
statement := map[string]any{
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:ListBucket",
|
||||
"Resource": "arn:aws:s3:::" + bucket,
|
||||
"Condition": map[string]any{"StringEquals": map[string]any{
|
||||
"s3:prefix": []string{"", "home/"},
|
||||
"s3:delimiter": "/",
|
||||
}},
|
||||
}
|
||||
iam := &IdentityAccessManagement{}
|
||||
require.NoError(t, iam.PutPolicy("home-root", mustPolicy(t, map[string]any{
|
||||
"Version": "2012-10-17", "Statement": []map[string]any{statement},
|
||||
})))
|
||||
identity := &Identity{Name: "carol", Account: &AccountAdmin, PolicyNames: []string{"home-root"}}
|
||||
|
||||
statement["Principal"] = "*"
|
||||
engine := NewBucketPolicyEngine()
|
||||
require.NoError(t, engine.engine.SetBucketPolicy(bucket, mustPolicy(t, map[string]any{
|
||||
"Version": "2012-10-17", "Statement": []map[string]any{statement},
|
||||
})))
|
||||
|
||||
for _, tt := range []struct {
|
||||
query string
|
||||
allowed bool
|
||||
}{
|
||||
{"list-type=2&delimiter=/", true},
|
||||
{"list-type=2&prefix=&delimiter=/", true},
|
||||
{"list-type=2&prefix=home/&delimiter=/", true},
|
||||
{"list-type=2&prefix=other/&delimiter=/", false},
|
||||
{"list-type=2", false},
|
||||
} {
|
||||
req := httptest.NewRequest("GET", "/"+bucket+"?"+tt.query, nil)
|
||||
require.Equal(t, tt.allowed, iam.evaluateIAMPolicies(req, identity, s3_constants.ACTION_LIST, bucket, ""), "attached policy, %s", tt.query)
|
||||
allowed, evaluated, err := engine.EvaluatePolicy(bucket, "", string(s3_constants.ACTION_LIST), "arn:aws:iam::000000000000:user/carol", req, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.allowed, evaluated && allowed, "bucket policy, %s", tt.query)
|
||||
}
|
||||
}
|
||||
|
||||
// Listing variants keep their specific action even when a prefix is promoted
|
||||
// into object: ?versions resolves to s3:ListBucketVersions, not s3:ListBucket.
|
||||
func TestEvaluateIAMPolicies_ListBucketVersionsWithPrefix(t *testing.T) {
|
||||
@@ -303,97 +258,6 @@ func TestAuthorizeAction_ListVersionsWithPromotedPrefix(t *testing.T) {
|
||||
"s3:GetObject must not satisfy the uploads listing")
|
||||
}
|
||||
|
||||
// Bucket-listing policies are evaluated by the IAM integration. Its authorizer
|
||||
// must expose prefix, delimiter and max-keys to conditions, and resolve the
|
||||
// listing variant (?versions, ?uploads) to its own action.
|
||||
func TestAuthorizeAction_BucketListingConditions(t *testing.T) {
|
||||
const bucket = "test-bucket"
|
||||
|
||||
ctx := context.Background()
|
||||
iamManager := integration.NewIAMManager()
|
||||
require.NoError(t, iamManager.Initialize(&integration.IAMConfig{
|
||||
STS: &sts.STSConfig{
|
||||
TokenDuration: sts.FlexibleDuration{Duration: time.Hour},
|
||||
MaxSessionLength: sts.FlexibleDuration{Duration: 12 * time.Hour},
|
||||
Issuer: "test-sts",
|
||||
SigningKey: []byte("test-signing-key-32-characters-long"),
|
||||
},
|
||||
Policy: &policy.PolicyEngineConfig{DefaultEffect: "Deny", StoreType: "memory"},
|
||||
Roles: &integration.RoleStoreConfig{StoreType: "memory"},
|
||||
}, func() string { return "localhost:8888" }))
|
||||
|
||||
allow := func(name, action string, condition map[string]map[string]interface{}) {
|
||||
t.Helper()
|
||||
require.NoError(t, iamManager.CreatePolicy(ctx, "", name, &policy.PolicyDocument{
|
||||
Version: "2012-10-17",
|
||||
Statement: []policy.Statement{{
|
||||
Effect: "Allow",
|
||||
Action: []string{action},
|
||||
Resource: []string{"arn:aws:s3:::" + bucket},
|
||||
Condition: condition,
|
||||
}},
|
||||
}))
|
||||
}
|
||||
// One-level listing of home/: a prefix alone is recursive, so the
|
||||
// delimiter is what narrows the grant.
|
||||
homeOneLevel := map[string]map[string]interface{}{"StringEquals": {
|
||||
"s3:prefix": "home/",
|
||||
"s3:delimiter": "/",
|
||||
}}
|
||||
allow("HomeOneLevel", "s3:ListBucket", homeOneLevel)
|
||||
allow("VersionsHomeOneLevel", "s3:ListBucketVersions", homeOneLevel)
|
||||
allow("UploadsHomeOneLevel", "s3:ListBucketMultipartUploads", homeOneLevel)
|
||||
allow("MaxKeysTen", "s3:ListBucket", map[string]map[string]interface{}{
|
||||
"NumericLessThanEquals": {"s3:max-keys": 10},
|
||||
})
|
||||
allow("VersionsMaxKeysTen", "s3:ListBucketVersions", map[string]map[string]interface{}{
|
||||
"NumericLessThanEquals": {"s3:max-keys": 10},
|
||||
})
|
||||
allow("UploadsMaxKeysTen", "s3:ListBucketMultipartUploads", map[string]map[string]interface{}{
|
||||
"NumericLessThanEquals": {"s3:max-keys": 10},
|
||||
})
|
||||
|
||||
s3iam := NewS3IAMIntegration(iamManager, "localhost:8888")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
policy string
|
||||
url string
|
||||
action Action
|
||||
object string
|
||||
expected s3err.ErrorCode
|
||||
}{
|
||||
{"ListObjectsV2 one level with the permitted delimiter", "HomeOneLevel", "/" + bucket + "?list-type=2&prefix=home/&delimiter=/", s3_constants.ACTION_LIST, "home/", s3err.ErrNone},
|
||||
{"recursive ListObjectsV2 without a delimiter is denied", "HomeOneLevel", "/" + bucket + "?list-type=2&prefix=home/", s3_constants.ACTION_LIST, "home/", s3err.ErrAccessDenied},
|
||||
{"nested prefix with the permitted delimiter is denied", "HomeOneLevel", "/" + bucket + "?list-type=2&prefix=home/bob/&delimiter=/", s3_constants.ACTION_LIST, "home/bob/", s3err.ErrAccessDenied},
|
||||
{"different delimiter is denied", "HomeOneLevel", "/" + bucket + "?list-type=2&prefix=home/&delimiter=.", s3_constants.ACTION_LIST, "home/", s3err.ErrAccessDenied},
|
||||
{"bucket root listing is denied", "HomeOneLevel", "/" + bucket + "?list-type=2&delimiter=/", s3_constants.ACTION_LIST, "", s3err.ErrAccessDenied},
|
||||
{"ListObjectVersions one level is allowed", "VersionsHomeOneLevel", "/" + bucket + "?versions&prefix=home/&delimiter=/", s3_constants.ACTION_LIST, "home/", s3err.ErrNone},
|
||||
{"ListObjectVersions without a delimiter is denied", "VersionsHomeOneLevel", "/" + bucket + "?versions&prefix=home/", s3_constants.ACTION_LIST, "home/", s3err.ErrAccessDenied},
|
||||
{"ListMultipartUploads one level is allowed", "UploadsHomeOneLevel", "/" + bucket + "?uploads&prefix=home/&delimiter=/", s3_constants.ACTION_READ, "", s3err.ErrNone},
|
||||
{"ListMultipartUploads without a delimiter is denied", "UploadsHomeOneLevel", "/" + bucket + "?uploads&prefix=home/", s3_constants.ACTION_READ, "", s3err.ErrAccessDenied},
|
||||
{"max-keys within s3:max-keys is allowed", "MaxKeysTen", "/" + bucket + "?list-type=2&prefix=home/&delimiter=/&max-keys=5", s3_constants.ACTION_LIST, "home/", s3err.ErrNone},
|
||||
{"max-keys above s3:max-keys is denied", "MaxKeysTen", "/" + bucket + "?list-type=2&prefix=home/&delimiter=/&max-keys=20", s3_constants.ACTION_LIST, "home/", s3err.ErrAccessDenied},
|
||||
{"versions max-keys within limit is allowed", "VersionsMaxKeysTen", "/" + bucket + "?versions&max-keys=5", s3_constants.ACTION_LIST, "", s3err.ErrNone},
|
||||
{"versions max-keys above limit is denied", "VersionsMaxKeysTen", "/" + bucket + "?versions&max-keys=20", s3_constants.ACTION_LIST, "", s3err.ErrAccessDenied},
|
||||
{"uploads max-uploads does not satisfy max-keys", "UploadsMaxKeysTen", "/" + bucket + "?uploads&max-uploads=5", s3_constants.ACTION_READ, "", s3err.ErrAccessDenied},
|
||||
{"uploads spoofed max-keys does not satisfy policy", "UploadsMaxKeysTen", "/" + bucket + "?uploads&max-uploads=1000&max-keys=5", s3_constants.ACTION_READ, "", s3err.ErrAccessDenied},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
identity := &IAMIdentity{
|
||||
Name: "reader",
|
||||
Principal: "arn:aws:iam::000000000000:user/reader",
|
||||
PolicyNames: []string{tt.policy},
|
||||
}
|
||||
req := httptest.NewRequest("GET", tt.url, http.NoBody)
|
||||
require.Equal(t, tt.expected,
|
||||
s3iam.AuthorizeAction(ctx, identity, tt.action, bucket, tt.object, req))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustPolicy(t *testing.T, doc map[string]any) string {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(doc)
|
||||
|
||||
@@ -3,8 +3,10 @@ package policy_engine
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -12,6 +14,148 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
)
|
||||
|
||||
// LRUNode represents a node in the doubly-linked list for efficient LRU operations
|
||||
type LRUNode struct {
|
||||
key string
|
||||
value []string
|
||||
prev *LRUNode
|
||||
next *LRUNode
|
||||
}
|
||||
|
||||
// NormalizedValueCache provides size-limited caching for normalized values with efficient LRU eviction
|
||||
type NormalizedValueCache struct {
|
||||
mu sync.RWMutex
|
||||
cache map[string]*LRUNode
|
||||
maxSize int
|
||||
head *LRUNode // Most recently used
|
||||
tail *LRUNode // Least recently used
|
||||
}
|
||||
|
||||
// NewNormalizedValueCache creates a new normalized value cache with configurable size
|
||||
func NewNormalizedValueCache(maxSize int) *NormalizedValueCache {
|
||||
if maxSize <= 0 {
|
||||
maxSize = 1000 // Default size
|
||||
}
|
||||
|
||||
// Create dummy head and tail nodes for easier list manipulation
|
||||
head := &LRUNode{}
|
||||
tail := &LRUNode{}
|
||||
head.next = tail
|
||||
tail.prev = head
|
||||
|
||||
return &NormalizedValueCache{
|
||||
cache: make(map[string]*LRUNode),
|
||||
maxSize: maxSize,
|
||||
head: head,
|
||||
tail: tail,
|
||||
}
|
||||
}
|
||||
|
||||
// Get retrieves a cached value and updates access order in O(1) time
|
||||
func (c *NormalizedValueCache) Get(key string) ([]string, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if node, exists := c.cache[key]; exists {
|
||||
// Move to head (most recently used) - O(1) operation
|
||||
c.moveToHead(node)
|
||||
return node.value, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Set stores a value in the cache with size limit enforcement in O(1) time
|
||||
func (c *NormalizedValueCache) Set(key string, value []string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if node, exists := c.cache[key]; exists {
|
||||
// Update existing node and move to head
|
||||
node.value = value
|
||||
c.moveToHead(node)
|
||||
return
|
||||
}
|
||||
|
||||
// Create new node
|
||||
newNode := &LRUNode{
|
||||
key: key,
|
||||
value: value,
|
||||
}
|
||||
|
||||
// If at max size, evict least recently used
|
||||
if len(c.cache) >= c.maxSize {
|
||||
c.evictLeastRecentlyUsed()
|
||||
}
|
||||
|
||||
// Add to cache and move to head
|
||||
c.cache[key] = newNode
|
||||
c.addToHead(newNode)
|
||||
}
|
||||
|
||||
// moveToHead moves a node to the head of the list (most recently used) - O(1)
|
||||
func (c *NormalizedValueCache) moveToHead(node *LRUNode) {
|
||||
c.removeNode(node)
|
||||
c.addToHead(node)
|
||||
}
|
||||
|
||||
// addToHead adds a node right after the head - O(1)
|
||||
func (c *NormalizedValueCache) addToHead(node *LRUNode) {
|
||||
node.prev = c.head
|
||||
node.next = c.head.next
|
||||
c.head.next.prev = node
|
||||
c.head.next = node
|
||||
}
|
||||
|
||||
// removeNode removes a node from the list - O(1)
|
||||
func (c *NormalizedValueCache) removeNode(node *LRUNode) {
|
||||
node.prev.next = node.next
|
||||
node.next.prev = node.prev
|
||||
}
|
||||
|
||||
// removeTail removes the last node before tail (least recently used) - O(1)
|
||||
func (c *NormalizedValueCache) removeTail() *LRUNode {
|
||||
lastNode := c.tail.prev
|
||||
c.removeNode(lastNode)
|
||||
return lastNode
|
||||
}
|
||||
|
||||
// evictLeastRecentlyUsed removes the least recently used item in O(1) time
|
||||
func (c *NormalizedValueCache) evictLeastRecentlyUsed() {
|
||||
tail := c.removeTail()
|
||||
delete(c.cache, tail.key)
|
||||
}
|
||||
|
||||
// Global cache instance with size limit
|
||||
var normalizedValueCache = NewNormalizedValueCache(1000)
|
||||
|
||||
// getCachedNormalizedValues returns cached normalized values or caches new ones
|
||||
func getCachedNormalizedValues(value interface{}) []string {
|
||||
if value == nil {
|
||||
return nil // no expected values; never reach reflect.TypeOf(nil)
|
||||
}
|
||||
// Create a string key for caching - more efficient than fmt.Sprintf
|
||||
typeStr := reflect.TypeOf(value).String()
|
||||
cacheKey := typeStr + ":" + fmt.Sprint(value)
|
||||
|
||||
// Try to get from cache
|
||||
if cached, exists := normalizedValueCache.Get(cacheKey); exists {
|
||||
return cached
|
||||
}
|
||||
|
||||
// Not in cache, normalize and store
|
||||
// Use the error-handling version for better error reporting
|
||||
normalized, err := normalizeToStringSliceWithError(value)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to normalize policy value %v: %v", value, err)
|
||||
// Fallback to string conversion for backward compatibility
|
||||
normalized = []string{fmt.Sprintf("%v", value)}
|
||||
}
|
||||
|
||||
normalizedValueCache.Set(cacheKey, normalized)
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
// ConditionEvaluator evaluates policy conditions
|
||||
type ConditionEvaluator interface {
|
||||
Evaluate(conditionValue interface{}, contextValues []string) bool
|
||||
@@ -21,7 +165,7 @@ type ConditionEvaluator interface {
|
||||
type StringEqualsEvaluator struct{}
|
||||
|
||||
func (e *StringEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if expected == contextValue {
|
||||
@@ -36,7 +180,7 @@ func (e *StringEqualsEvaluator) Evaluate(conditionValue interface{}, contextValu
|
||||
type StringNotEqualsEvaluator struct{}
|
||||
|
||||
func (e *StringNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if expected == contextValue {
|
||||
@@ -51,7 +195,7 @@ func (e *StringNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextV
|
||||
type StringLikeEvaluator struct{}
|
||||
|
||||
func (e *StringLikeEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
patterns := normalizeToStringSlice(conditionValue)
|
||||
patterns := getCachedNormalizedValues(conditionValue)
|
||||
for _, pattern := range patterns {
|
||||
for _, contextValue := range contextValues {
|
||||
if wildcard.MatchesWildcard(pattern, contextValue) {
|
||||
@@ -66,7 +210,7 @@ func (e *StringLikeEvaluator) Evaluate(conditionValue interface{}, contextValues
|
||||
type StringNotLikeEvaluator struct{}
|
||||
|
||||
func (e *StringNotLikeEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
patterns := normalizeToStringSlice(conditionValue)
|
||||
patterns := getCachedNormalizedValues(conditionValue)
|
||||
for _, pattern := range patterns {
|
||||
for _, contextValue := range contextValues {
|
||||
if wildcard.MatchesWildcard(pattern, contextValue) {
|
||||
@@ -81,7 +225,7 @@ func (e *StringNotLikeEvaluator) Evaluate(conditionValue interface{}, contextVal
|
||||
type StringEqualsIgnoreCaseEvaluator struct{}
|
||||
|
||||
func (e *StringEqualsIgnoreCaseEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if strings.EqualFold(expected, contextValue) {
|
||||
@@ -96,7 +240,7 @@ func (e *StringEqualsIgnoreCaseEvaluator) Evaluate(conditionValue interface{}, c
|
||||
type StringNotEqualsIgnoreCaseEvaluator struct{}
|
||||
|
||||
func (e *StringNotEqualsIgnoreCaseEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if strings.EqualFold(expected, contextValue) {
|
||||
@@ -111,7 +255,7 @@ func (e *StringNotEqualsIgnoreCaseEvaluator) Evaluate(conditionValue interface{}
|
||||
type NumericEqualsEvaluator struct{}
|
||||
|
||||
func (e *NumericEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -134,7 +278,7 @@ func (e *NumericEqualsEvaluator) Evaluate(conditionValue interface{}, contextVal
|
||||
type NumericNotEqualsEvaluator struct{}
|
||||
|
||||
func (e *NumericNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -157,7 +301,7 @@ func (e *NumericNotEqualsEvaluator) Evaluate(conditionValue interface{}, context
|
||||
type NumericLessThanEvaluator struct{}
|
||||
|
||||
func (e *NumericLessThanEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -180,7 +324,7 @@ func (e *NumericLessThanEvaluator) Evaluate(conditionValue interface{}, contextV
|
||||
type NumericLessThanEqualsEvaluator struct{}
|
||||
|
||||
func (e *NumericLessThanEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -203,7 +347,7 @@ func (e *NumericLessThanEqualsEvaluator) Evaluate(conditionValue interface{}, co
|
||||
type NumericGreaterThanEvaluator struct{}
|
||||
|
||||
func (e *NumericGreaterThanEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -226,7 +370,7 @@ func (e *NumericGreaterThanEvaluator) Evaluate(conditionValue interface{}, conte
|
||||
type NumericGreaterThanEqualsEvaluator struct{}
|
||||
|
||||
func (e *NumericGreaterThanEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -249,7 +393,7 @@ func (e *NumericGreaterThanEqualsEvaluator) Evaluate(conditionValue interface{},
|
||||
type DateEqualsEvaluator struct{}
|
||||
|
||||
func (e *DateEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -272,7 +416,7 @@ func (e *DateEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues
|
||||
type DateNotEqualsEvaluator struct{}
|
||||
|
||||
func (e *DateNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -295,7 +439,7 @@ func (e *DateNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextVal
|
||||
type DateLessThanEvaluator struct{}
|
||||
|
||||
func (e *DateLessThanEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -318,7 +462,7 @@ func (e *DateLessThanEvaluator) Evaluate(conditionValue interface{}, contextValu
|
||||
type DateLessThanEqualsEvaluator struct{}
|
||||
|
||||
func (e *DateLessThanEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -341,7 +485,7 @@ func (e *DateLessThanEqualsEvaluator) Evaluate(conditionValue interface{}, conte
|
||||
type DateGreaterThanEvaluator struct{}
|
||||
|
||||
func (e *DateGreaterThanEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -364,7 +508,7 @@ func (e *DateGreaterThanEvaluator) Evaluate(conditionValue interface{}, contextV
|
||||
type DateGreaterThanEqualsEvaluator struct{}
|
||||
|
||||
func (e *DateGreaterThanEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -387,7 +531,7 @@ func (e *DateGreaterThanEqualsEvaluator) Evaluate(conditionValue interface{}, co
|
||||
type BoolEvaluator struct{}
|
||||
|
||||
func (e *BoolEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if strings.ToLower(expected) == strings.ToLower(contextValue) {
|
||||
@@ -402,7 +546,7 @@ func (e *BoolEvaluator) Evaluate(conditionValue interface{}, contextValues []str
|
||||
type IpAddressEvaluator struct{}
|
||||
|
||||
func (e *IpAddressEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
_, expectedNet, err := net.ParseCIDR(expected)
|
||||
if err != nil {
|
||||
@@ -443,7 +587,7 @@ func (e *IpAddressEvaluator) Evaluate(conditionValue interface{}, contextValues
|
||||
type NotIpAddressEvaluator struct{}
|
||||
|
||||
func (e *NotIpAddressEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
_, expectedNet, err := net.ParseCIDR(expected)
|
||||
if err != nil {
|
||||
@@ -484,7 +628,7 @@ func (e *NotIpAddressEvaluator) Evaluate(conditionValue interface{}, contextValu
|
||||
type ArnEqualsEvaluator struct{}
|
||||
|
||||
func (e *ArnEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if expected == contextValue {
|
||||
@@ -499,7 +643,7 @@ func (e *ArnEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues
|
||||
type ArnLikeEvaluator struct{}
|
||||
|
||||
func (e *ArnLikeEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
patterns := normalizeToStringSlice(conditionValue)
|
||||
patterns := getCachedNormalizedValues(conditionValue)
|
||||
for _, pattern := range patterns {
|
||||
for _, contextValue := range contextValues {
|
||||
if wildcard.MatchesWildcard(pattern, contextValue) {
|
||||
@@ -514,7 +658,7 @@ func (e *ArnLikeEvaluator) Evaluate(conditionValue interface{}, contextValues []
|
||||
type NullEvaluator struct{}
|
||||
|
||||
func (e *NullEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedBool := strings.ToLower(expected) == "true"
|
||||
contextExists := len(contextValues) > 0
|
||||
@@ -707,6 +851,8 @@ func evaluateConditions(conditions PolicyConditions, contextValues map[string][]
|
||||
substitutedValues[i] = SubstituteVariables(v, contextValues, claims)
|
||||
}
|
||||
|
||||
// Pass substituted values (casted to interface{} to match signature if needed, or update evaluators to accept []string)
|
||||
// The evaluators take interface{}, but getCachedNormalizedValues handles []string.
|
||||
if !conditionEvaluator.Evaluate(substitutedValues, contextVals) {
|
||||
return false // If any condition fails, the whole condition block fails
|
||||
}
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
package policy_engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConditionValuesLookingAlikeStayDistinct(t *testing.T) {
|
||||
key := "aws:username"
|
||||
list := PolicyConditions{"StringEquals": {key: NewStringOrStringSlice("distinct-a", "distinct-b")}}
|
||||
spaced := PolicyConditions{"StringEquals": {key: NewStringOrStringSlice("distinct-a distinct-b")}}
|
||||
|
||||
if !EvaluateConditions(list, map[string][]string{key: {"distinct-a"}}, nil, nil) {
|
||||
t.Fatal("list condition must match one of its values")
|
||||
}
|
||||
if EvaluateConditions(spaced, map[string][]string{key: {"distinct-a"}}, nil, nil) {
|
||||
t.Error(`"distinct-a distinct-b" matched "distinct-a"`)
|
||||
}
|
||||
if !EvaluateConditions(spaced, map[string][]string{key: {"distinct-a distinct-b"}}, nil, nil) {
|
||||
t.Error(`"distinct-a distinct-b" did not match itself`)
|
||||
}
|
||||
|
||||
empty := PolicyConditions{"StringEquals": {key: NewStringOrStringSlice()}}
|
||||
blank := PolicyConditions{"StringEquals": {key: NewStringOrStringSlice("")}}
|
||||
if EvaluateConditions(empty, map[string][]string{key: {""}}, nil, nil) {
|
||||
t.Error("empty value list matched")
|
||||
}
|
||||
if !EvaluateConditions(blank, map[string][]string{key: {""}}, nil, nil) {
|
||||
t.Error(`"" did not match itself after an empty value list`)
|
||||
}
|
||||
|
||||
eval := &StringEqualsEvaluator{}
|
||||
if !eval.Evaluate([]interface{}{"json-a", "json-b"}, []string{"json-a"}) {
|
||||
t.Fatal("JSON list must match one of its values")
|
||||
}
|
||||
if eval.Evaluate([]interface{}{"json-a json-b"}, []string{"json-a"}) {
|
||||
t.Error(`JSON "json-a json-b" matched "json-a"`)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPolicyConditionValuesDoNotLeakAcrossBuckets(t *testing.T) {
|
||||
engine := NewPolicyEngine()
|
||||
policy := func(bucket, effect, key, value string) string {
|
||||
return fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":%q,"Principal":"*",
|
||||
"Action":"s3:PutObject","Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"StringEquals":{%q:%s}}}]}`, effect, bucket, key, value)
|
||||
}
|
||||
if err := engine.SetBucketPolicy("victim", policy("victim", "Deny", "s3:x-amz-acl", `["public-read","public-read-write"]`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := engine.SetBucketPolicy("other", policy("other", "Allow", "aws:UserAgent", `"public-read public-read-write"`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(bucket string, headers map[string]string) PolicyEvaluationResult {
|
||||
r := httptest.NewRequest(http.MethodPut, "/"+bucket+"/key", nil)
|
||||
for k, v := range headers {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
return engine.EvaluatePolicyForRequest(bucket, "key", "PutObject", "*", r)
|
||||
}
|
||||
|
||||
if got := put("other", map[string]string{"User-Agent": "public-read public-read-write"}); got != PolicyResultAllow {
|
||||
t.Fatalf("other bucket: got %v, want Allow", got)
|
||||
}
|
||||
if got := put("victim", map[string]string{"x-amz-acl": "public-read"}); got != PolicyResultDeny {
|
||||
t.Errorf("victim bucket: got %v, want Deny", got)
|
||||
}
|
||||
}
|
||||
@@ -454,8 +454,7 @@ func (engine *PolicyEngine) ExtractConditionValuesFromRequest(r *http.Request) m
|
||||
values["s3:prefix"] = []string{prefix}
|
||||
}
|
||||
|
||||
// Multipart listings use max-uploads, so max-keys cannot constrain them.
|
||||
if maxKeys := r.URL.Query().Get("max-keys"); maxKeys != "" && !r.URL.Query().Has("uploads") {
|
||||
if maxKeys := r.URL.Query().Get("max-keys"); maxKeys != "" {
|
||||
values["s3:max-keys"] = []string{maxKeys}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -420,22 +419,6 @@ func TestExtractConditionValuesFromRequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractConditionValuesFromRequestMaxKeys(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
query string
|
||||
want []string
|
||||
}{
|
||||
{"list-type=2&max-keys=5", []string{"5"}},
|
||||
{"versions&max-keys=5", []string{"5"}},
|
||||
{"uploads&max-uploads=1000&max-keys=5", nil},
|
||||
} {
|
||||
req := &http.Request{Method: "GET", URL: &url.URL{Path: "/test-bucket", RawQuery: tt.query}, RemoteAddr: "10.0.0.5:54321"}
|
||||
if got := NewPolicyEngine().ExtractConditionValuesFromRequest(req)["s3:max-keys"]; !slices.Equal(got, tt.want) {
|
||||
t.Errorf("%s: s3:max-keys = %v, want %v", tt.query, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractConditionValuesFromRequestSourceIPPrecedence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Loaded 100 of 202 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user