mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* test(s3): s3-listing-study integration suite (WIP) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): run the s3-listing-study tools against SeaweedFS (WIP) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): s3-listing-study deleted-prefix probe, fresh data dir per run Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): listing study ports keep clear of every gRPC companion free_ports.py compared only base ports, so a base port could land on another server's base+10000 gRPC listener. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study ports stay below the ephemeral range Base ports of 20000-55000 put both them and their gRPC companions inside 32768-60999, so an outgoing connection made while weed server starts could take a port before the server that was given it binds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study checks the order inside each listing page parse_page sorted Contents and CommonPrefixes together before the probes compared them, so keys returned out of order within one page passed. Check each list's byte order first; the versions pager does the same. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study applies --only before running a probe The filter was applied to the probe results, so a run limited to one tool still sent every reference listing request and the deleted-prefix check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study requires CommonPrefixes from shallow modes The CommonPrefixes comparison ran only when a tool returned at least one prefix, so a delimiter listing of the canary root that came back empty passed. Compare them for every shallow mode except the two whose normalizer keeps keys only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
214 lines
7.9 KiB
Python
214 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Install the listing tools pinned by the s3-listing-study checkout.
|
|
|
|
Every artifact locator and sha256 comes from the study's own
|
|
tools/<tool>/build/image.json, so bumping STUDY_REV bumps the tools with it.
|
|
Tools land under TOOLS_DIR at the paths listed in run_study.PATH_MAP.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import urllib.request
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
ALL_TOOLS = (
|
|
"aws-cli",
|
|
"minio-mc",
|
|
"ps3",
|
|
"rclone",
|
|
"s3-fast-list",
|
|
"s3kor",
|
|
"s3p",
|
|
"s4cmd",
|
|
"s5cmd",
|
|
"s7cmd",
|
|
"swath",
|
|
)
|
|
|
|
MC_COMMIT = "d6541ea280b73a834b64d4097e21f2be77676104"
|
|
"""github.com/minio/mc tag RELEASE.2025-08-13T08-35-41Z."""
|
|
|
|
|
|
def fetch(url: str, sha256: str, dest: Path) -> Path:
|
|
if dest.exists() and hashlib.sha256(dest.read_bytes()).hexdigest() == sha256:
|
|
return dest
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = dest.with_suffix(dest.suffix + ".part")
|
|
request = urllib.request.Request(url, headers={"User-Agent": "curl/8"})
|
|
with urllib.request.urlopen(request, timeout=300) as response, tmp.open("wb") as out:
|
|
shutil.copyfileobj(response, out)
|
|
digest = hashlib.sha256(tmp.read_bytes()).hexdigest()
|
|
if digest != sha256:
|
|
tmp.unlink()
|
|
raise SystemExit(f"{url}: sha256 {digest} != pinned {sha256}")
|
|
tmp.rename(dest)
|
|
return dest
|
|
|
|
|
|
def executable(path: Path) -> None:
|
|
path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
|
|
|
|
|
def wrapper(path: Path, body: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(f"#!/bin/sh\n{body}\n")
|
|
executable(path)
|
|
|
|
|
|
def tar_member(archive: Path, member: str, dest: Path) -> None:
|
|
with tarfile.open(archive) as tar:
|
|
source = tar.extractfile(member)
|
|
if source is None:
|
|
raise SystemExit(f"{archive}: no member {member}")
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes(source.read())
|
|
executable(dest)
|
|
|
|
|
|
def install(tool: str, study: Path, tools_dir: Path) -> None:
|
|
image = json.loads((study / "tools" / tool / "build" / "image.json").read_text())
|
|
artifact = image["tool_artifact"]
|
|
url, sha = artifact["locator"], artifact["sha256"]
|
|
dl = tools_dir / "downloads"
|
|
bin_dir = tools_dir / "bin"
|
|
bin_dir.mkdir(parents=True, exist_ok=True)
|
|
name = url.rsplit("/", 1)[-1]
|
|
stamp = tools_dir / "installed" / tool
|
|
if stamp.exists() and stamp.read_text() == sha:
|
|
print(f"{tool} {image.get('tool_version', '')} already installed")
|
|
return
|
|
|
|
if tool == "aws-cli":
|
|
archive = fetch(url, sha, dl / name)
|
|
work = tools_dir / "aws-cli-src"
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
with zipfile.ZipFile(archive) as zf:
|
|
zf.extractall(work)
|
|
for rel in ("aws/install", "aws/dist/aws", "aws/dist/aws_completer"):
|
|
executable(work / rel)
|
|
subprocess.run(
|
|
[
|
|
str(work / "aws" / "install"),
|
|
"--install-dir",
|
|
str(tools_dir / "aws-cli"),
|
|
"--bin-dir",
|
|
str(tools_dir / "aws-cli-bin"),
|
|
"--update",
|
|
],
|
|
check=True,
|
|
)
|
|
shutil.rmtree(work)
|
|
elif tool == "minio-mc":
|
|
# dl.min.io answers 410 Gone for every archived mc release, so build the
|
|
# same release tag from source instead.
|
|
if image["tool_version"] != "RELEASE.2025-08-13T08-35-41Z":
|
|
raise SystemExit(f"minio-mc: no source commit pinned for {image['tool_version']}")
|
|
subprocess.run(
|
|
["go", "install", f"github.com/minio/mc@{MC_COMMIT}"],
|
|
env={**os.environ, "GOBIN": str(bin_dir)},
|
|
check=True,
|
|
)
|
|
elif tool == "ps3":
|
|
shutil.copyfile(fetch(url, sha, dl / name), bin_dir / "pS3")
|
|
executable(bin_dir / "pS3")
|
|
elif tool == "rclone":
|
|
archive = fetch(url, sha, dl / name)
|
|
with zipfile.ZipFile(archive) as zf:
|
|
member = f"{name.removesuffix('.zip')}/rclone"
|
|
(bin_dir / "rclone").write_bytes(zf.read(member))
|
|
executable(bin_dir / "rclone")
|
|
elif tool == "s3kor":
|
|
tar_member(fetch(url, sha, dl / name), "s3kor", bin_dir / "s3kor")
|
|
elif tool == "s5cmd":
|
|
tar_member(fetch(url, sha, dl / name), "s5cmd", bin_dir / "s5cmd")
|
|
elif tool == "s7cmd":
|
|
tar_member(fetch(url, sha, dl / name), "s7cmd", bin_dir / "s7cmd")
|
|
elif tool == "s3p":
|
|
work = tools_dir / "s3p"
|
|
work.mkdir(parents=True, exist_ok=True)
|
|
for f in ("package.json", "package-lock.json"):
|
|
shutil.copyfile(study / "tools/s3p/build" / f, work / f)
|
|
subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], cwd=work, check=True)
|
|
wrapper(bin_dir / "s3p", f'exec node "{work}/node_modules/s3p/bin/s3p" "$@"')
|
|
elif tool == "s4cmd":
|
|
archive = fetch(url, sha, dl / name)
|
|
venv = tools_dir / "s4cmd-venv"
|
|
subprocess.run([sys.executable, "-m", "venv", "--clear", str(venv)], check=True)
|
|
subprocess.run(
|
|
[
|
|
str(venv / "bin" / "python"),
|
|
"-m",
|
|
"pip",
|
|
"install",
|
|
"--quiet",
|
|
"--require-hashes",
|
|
"--no-deps",
|
|
"-r",
|
|
str(study / "tools/s4cmd/build/requirements.txt"),
|
|
],
|
|
check=True,
|
|
)
|
|
script = tools_dir / "s4cmd" / "s4cmd.py"
|
|
with tarfile.open(archive) as tar:
|
|
source = tar.extractfile("s4cmd-2.1.0/s4cmd.py")
|
|
assert source is not None
|
|
script.parent.mkdir(parents=True, exist_ok=True)
|
|
script.write_bytes(source.read())
|
|
wrapper(bin_dir / "s4cmd", f'exec "{venv}/bin/python" "{script}" "$@"')
|
|
elif tool == "s3-fast-list":
|
|
archive = fetch(url, sha, dl / name)
|
|
work = tools_dir / "s3-fast-list-src"
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
work.mkdir(parents=True)
|
|
subprocess.run(["tar", "xzf", str(archive), "--strip-components=1", "-C", str(work)], check=True)
|
|
shutil.copyfile(study / "tools/s3-fast-list/build/Cargo.lock", work / "Cargo.lock")
|
|
subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=work, check=True)
|
|
for exe in ("s3-fast-list", "ks-tool"):
|
|
shutil.copyfile(work / "target/release" / exe, bin_dir / exe)
|
|
executable(bin_dir / exe)
|
|
elif tool == "swath":
|
|
ref = f"{url}@sha256:{sha}"
|
|
subprocess.run(["docker", "pull", "--quiet", ref], check=True, stdout=subprocess.DEVNULL)
|
|
cid = subprocess.run(
|
|
["docker", "create", ref], check=True, capture_output=True, text=True
|
|
).stdout.strip()
|
|
try:
|
|
dest = tools_dir / "swath"
|
|
shutil.rmtree(dest, ignore_errors=True)
|
|
dest.mkdir(parents=True)
|
|
subprocess.run(["docker", "cp", f"{cid}:/opt/java/openjdk", str(dest / "openjdk")], check=True)
|
|
subprocess.run(["docker", "cp", f"{cid}:/opt/swath/swath.jar", str(dest / "swath.jar")], check=True)
|
|
finally:
|
|
subprocess.run(["docker", "rm", cid], check=True, stdout=subprocess.DEVNULL)
|
|
else:
|
|
raise SystemExit(f"unknown tool {tool}")
|
|
stamp.parent.mkdir(parents=True, exist_ok=True)
|
|
stamp.write_text(sha)
|
|
print(f"installed {tool} {image.get('tool_version', '')}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--study", required=True, type=Path)
|
|
parser.add_argument("--tools-dir", required=True, type=Path)
|
|
parser.add_argument("--tools", default=",".join(ALL_TOOLS))
|
|
args = parser.parse_args()
|
|
for tool in filter(None, args.tools.split(",")):
|
|
install(tool, args.study.resolve(), args.tools_dir.resolve())
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
os.umask(0o022)
|
|
raise SystemExit(main())
|