test/s3: run the s3-listing-study listing tools and strategies in CI (#12750)

* test(s3): s3-listing-study integration suite (WIP)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* test(s3): run the s3-listing-study tools against SeaweedFS (WIP)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* test(s3): s3-listing-study deleted-prefix probe, fresh data dir per run

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* test(s3): listing study ports keep clear of every gRPC companion

free_ports.py compared only base ports, so a base port could land on
another server's base+10000 gRPC listener.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(s3): listing study ports stay below the ephemeral range

Base ports of 20000-55000 put both them and their gRPC companions inside
32768-60999, so an outgoing connection made while weed server starts could
take a port before the server that was given it binds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(s3): listing study checks the order inside each listing page

parse_page sorted Contents and CommonPrefixes together before the probes
compared them, so keys returned out of order within one page passed. Check
each list's byte order first; the versions pager does the same.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(s3): listing study applies --only before running a probe

The filter was applied to the probe results, so a run limited to one tool
still sent every reference listing request and the deleted-prefix check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(s3): listing study requires CommonPrefixes from shallow modes

The CommonPrefixes comparison ran only when a tool returned at least one
prefix, so a delimiter listing of the canary root that came back empty
passed. Compare them for every shallow mode except the two whose
normalizer keeps keys only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-18 15:17:25 -08:00
1 parent 04c7b12238
commit 23eef59417
9 files changed
+1377

No files matched your search

+1
View File
@@ -0,0 +1 @@
.work
+87
View File
@@ -0,0 +1,87 @@
# Runs the listing tools and listing strategies of
# https://github.com/varveio/s3-listing-study against a local SeaweedFS.
#
# make test build weed, fetch the study and its pinned tools, start
# weed server on free ports, run, stop
# make run ENDPOINT=... run against an already running endpoint
#
# Needs: go, python3, node/npm, cargo, docker (swath image), curl.
#
# The server runs with -s3.allowBackslashInKey, the AWS-compatible key mode: by
# default SeaweedFS rewrites '\' in object keys to '/', and the edge key set
# holds a key with a backslash.
STUDY_REPO ?= https://github.com/varveio/s3-listing-study.git
STUDY_REV ?= 666ec620d245e16baaaa0cf53461382f42c35917
WORK_DIR ?= $(CURDIR)/.work
STUDY_DIR ?= $(WORK_DIR)/s3-listing-study
TOOLS_DIR ?= $(WORK_DIR)/tools
VENV ?= $(WORK_DIR)/venv
WEED_BINARY ?= $(WORK_DIR)/weed
TOOLS ?= aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath
ACCESS_KEY ?= listing_study_access
SECRET_KEY ?= listing_study_secret
JOBS ?= 4
ONLY ?=
.PHONY: test run build-weed study venv tools start-server stop-server clean
test: build-weed tools
@$(MAKE) --no-print-directory start-server; status=$$?; \
if [ $$status -eq 0 ]; then \
$(MAKE) --no-print-directory run ENDPOINT=http://127.0.0.1:$$(cat $(WORK_DIR)/s3.port); status=$$?; \
fi; \
$(MAKE) --no-print-directory stop-server; exit $$status
build-weed:
mkdir -p $(WORK_DIR)
cd ../../../weed && go build -buildvcs=false -o $(WEED_BINARY) .
study:
@if [ "$$(git -C $(STUDY_DIR) rev-parse HEAD 2>/dev/null)" != "$(STUDY_REV)" ]; then \
git init -q $(STUDY_DIR) && \
git -C $(STUDY_DIR) fetch -q --depth 1 $(STUDY_REPO) $(STUDY_REV) && \
git -C $(STUDY_DIR) checkout -q --force FETCH_HEAD; \
fi
venv:
@test -x $(VENV)/bin/python || python3 -m venv $(VENV)
$(VENV)/bin/pip install -q -r requirements.txt
tools: study venv
$(VENV)/bin/python install_tools.py --study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --tools $(TOOLS)
start-server:
@mkdir -p $(WORK_DIR)
@set -e; data=$$(mktemp -d $(WORK_DIR)/data.XXXXXX); ports=$$(python3 free_ports.py 5); \
set -- $$ports; \
$(WEED_BINARY) server -filer -s3 -ip 127.0.0.1 -ip.bind 127.0.0.1 -dir=$$data \
-master.peers=none -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=50 -volume.preStopSeconds=1 \
-master.port=$$1 -volume.port=$$2 -filer.port=$$3 -s3.port=$$4 -metricsPort=$$5 \
-s3.port.iceberg=0 -s3.port.lance=0 -s3.allowBackslashInKey -s3.config=$(CURDIR)/s3.json \
> $(WORK_DIR)/weed.log 2>&1 & echo $$! > $(WORK_DIR)/weed.pid; \
echo $$4 > $(WORK_DIR)/s3.port; \
for i in $$(seq 1 90); do \
if curl -sf -o /dev/null http://127.0.0.1:$$3/ && curl -s -o /dev/null http://127.0.0.1:$$4/; then \
echo "SeaweedFS S3 ready on port $$4"; exit 0; fi; \
kill -0 $$(cat $(WORK_DIR)/weed.pid) || { tail -50 $(WORK_DIR)/weed.log; exit 1; }; \
sleep 1; \
done; tail -50 $(WORK_DIR)/weed.log; exit 1
stop-server:
@if [ -f $(WORK_DIR)/weed.pid ]; then \
pid=$$(cat $(WORK_DIR)/weed.pid); kill $$pid 2>/dev/null || true; \
for i in $$(seq 1 20); do kill -0 $$pid 2>/dev/null || break; sleep 0.5; done; \
rm -f $(WORK_DIR)/weed.pid; \
fi
run: study
$(VENV)/bin/python run_study.py --endpoint $(ENDPOINT) \
--access-key $(ACCESS_KEY) --secret-key $(SECRET_KEY) \
--study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --work-dir $(WORK_DIR)/run \
--tools $(TOOLS) --jobs $(JOBS) --expected-failures expected_failures.txt \
$(if $(ONLY),--only '$(ONLY)')
clean: stop-server
rm -rf .work
@@ -0,0 +1,29 @@
# Cases that fail for reasons outside SeaweedFS. A listed case that passes is
# reported as XPASS and fails the run, so this list has to shrink with fixes.
# Format: <case name> # reason
# aws-cli prints "None" for a --query over a page with no Contents (the canary
# has no root-level keys); the capsule normalizer reads it as a malformed row.
aws-cli:s3api-v2-remainder:canary:/ # capsule normalizer cannot parse aws-cli's "None"
# The edge bucket holds zero-byte directory markers (markers/dir/, markers/empty/).
# These modes list them as directories, not objects: rclone's --files-only and
# lsf --files-only drop them, s5cmd types any key ending in "/" as DIR and the
# capsule normalizer drops DIR rows from its recursive modes. The reference
# probes check that SeaweedFS returns both markers as Contents.
rclone:recursive-fastlist:edge:/ # tool hides directory marker objects
rclone:recursive-hierarchical:edge:/ # tool hides directory marker objects
rclone:recursive-walk:edge:/ # tool hides directory marker objects
rclone:listv1:edge:/ # tool hides directory marker objects
rclone:lsf:edge:/ # tool hides directory marker objects
s5cmd:recursive:edge:/ # tool hides directory marker objects
s5cmd:listv1:edge:/ # tool hides directory marker objects
s5cmd:allversions:edge:/ # tool hides directory marker objects
# Deleting the last key under a prefix leaves its filer directory behind, and an
# empty directory is still listed as a CommonPrefix until the filer's empty-folder
# cleanup removes it (2 minutes by default). This is deliberate: directories made
# through mount or the filer stay visible to S3 clients (dirHoldsOnlyHiddenEntries
# in weed/s3api/s3api_object_handlers_list.go). AWS derives CommonPrefixes from
# keys only, so prefix-discovery tools briefly see the deleted prefix.
reference:study-churn:deleted-prefixes # empty directories are listed until cleanup
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Print N free TCP ports whose +10000 gRPC companions are free too, all below
the Linux ephemeral range (32768+) that outgoing connections draw from."""
import random
import socket
import sys
def free(port: int) -> bool:
with socket.socket() as s:
try:
s.bind(("127.0.0.1", port))
return True
except OSError:
return False
picked: list[int] = []
taken: list[int] = []
while len(picked) < int(sys.argv[1]):
port = random.randrange(10000, 22000)
pair = (port, port + 10000)
if all(abs(p - t) > 20 for p in pair for t in taken) and all(free(p) for p in pair):
picked.append(port)
taken += pair
print(*picked)
+213
View File
@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""Install the listing tools pinned by the s3-listing-study checkout.
Every artifact locator and sha256 comes from the study's own
tools/<tool>/build/image.json, so bumping STUDY_REV bumps the tools with it.
Tools land under TOOLS_DIR at the paths listed in run_study.PATH_MAP.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
import stat
import subprocess
import sys
import tarfile
import urllib.request
import zipfile
from pathlib import Path
ALL_TOOLS = (
"aws-cli",
"minio-mc",
"ps3",
"rclone",
"s3-fast-list",
"s3kor",
"s3p",
"s4cmd",
"s5cmd",
"s7cmd",
"swath",
)
MC_COMMIT = "d6541ea280b73a834b64d4097e21f2be77676104"
"""github.com/minio/mc tag RELEASE.2025-08-13T08-35-41Z."""
def fetch(url: str, sha256: str, dest: Path) -> Path:
if dest.exists() and hashlib.sha256(dest.read_bytes()).hexdigest() == sha256:
return dest
dest.parent.mkdir(parents=True, exist_ok=True)
tmp = dest.with_suffix(dest.suffix + ".part")
request = urllib.request.Request(url, headers={"User-Agent": "curl/8"})
with urllib.request.urlopen(request, timeout=300) as response, tmp.open("wb") as out:
shutil.copyfileobj(response, out)
digest = hashlib.sha256(tmp.read_bytes()).hexdigest()
if digest != sha256:
tmp.unlink()
raise SystemExit(f"{url}: sha256 {digest} != pinned {sha256}")
tmp.rename(dest)
return dest
def executable(path: Path) -> None:
path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
def wrapper(path: Path, body: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f"#!/bin/sh\n{body}\n")
executable(path)
def tar_member(archive: Path, member: str, dest: Path) -> None:
with tarfile.open(archive) as tar:
source = tar.extractfile(member)
if source is None:
raise SystemExit(f"{archive}: no member {member}")
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(source.read())
executable(dest)
def install(tool: str, study: Path, tools_dir: Path) -> None:
image = json.loads((study / "tools" / tool / "build" / "image.json").read_text())
artifact = image["tool_artifact"]
url, sha = artifact["locator"], artifact["sha256"]
dl = tools_dir / "downloads"
bin_dir = tools_dir / "bin"
bin_dir.mkdir(parents=True, exist_ok=True)
name = url.rsplit("/", 1)[-1]
stamp = tools_dir / "installed" / tool
if stamp.exists() and stamp.read_text() == sha:
print(f"{tool} {image.get('tool_version', '')} already installed")
return
if tool == "aws-cli":
archive = fetch(url, sha, dl / name)
work = tools_dir / "aws-cli-src"
shutil.rmtree(work, ignore_errors=True)
with zipfile.ZipFile(archive) as zf:
zf.extractall(work)
for rel in ("aws/install", "aws/dist/aws", "aws/dist/aws_completer"):
executable(work / rel)
subprocess.run(
[
str(work / "aws" / "install"),
"--install-dir",
str(tools_dir / "aws-cli"),
"--bin-dir",
str(tools_dir / "aws-cli-bin"),
"--update",
],
check=True,
)
shutil.rmtree(work)
elif tool == "minio-mc":
# dl.min.io answers 410 Gone for every archived mc release, so build the
# same release tag from source instead.
if image["tool_version"] != "RELEASE.2025-08-13T08-35-41Z":
raise SystemExit(f"minio-mc: no source commit pinned for {image['tool_version']}")
subprocess.run(
["go", "install", f"github.com/minio/mc@{MC_COMMIT}"],
env={**os.environ, "GOBIN": str(bin_dir)},
check=True,
)
elif tool == "ps3":
shutil.copyfile(fetch(url, sha, dl / name), bin_dir / "pS3")
executable(bin_dir / "pS3")
elif tool == "rclone":
archive = fetch(url, sha, dl / name)
with zipfile.ZipFile(archive) as zf:
member = f"{name.removesuffix('.zip')}/rclone"
(bin_dir / "rclone").write_bytes(zf.read(member))
executable(bin_dir / "rclone")
elif tool == "s3kor":
tar_member(fetch(url, sha, dl / name), "s3kor", bin_dir / "s3kor")
elif tool == "s5cmd":
tar_member(fetch(url, sha, dl / name), "s5cmd", bin_dir / "s5cmd")
elif tool == "s7cmd":
tar_member(fetch(url, sha, dl / name), "s7cmd", bin_dir / "s7cmd")
elif tool == "s3p":
work = tools_dir / "s3p"
work.mkdir(parents=True, exist_ok=True)
for f in ("package.json", "package-lock.json"):
shutil.copyfile(study / "tools/s3p/build" / f, work / f)
subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], cwd=work, check=True)
wrapper(bin_dir / "s3p", f'exec node "{work}/node_modules/s3p/bin/s3p" "$@"')
elif tool == "s4cmd":
archive = fetch(url, sha, dl / name)
venv = tools_dir / "s4cmd-venv"
subprocess.run([sys.executable, "-m", "venv", "--clear", str(venv)], check=True)
subprocess.run(
[
str(venv / "bin" / "python"),
"-m",
"pip",
"install",
"--quiet",
"--require-hashes",
"--no-deps",
"-r",
str(study / "tools/s4cmd/build/requirements.txt"),
],
check=True,
)
script = tools_dir / "s4cmd" / "s4cmd.py"
with tarfile.open(archive) as tar:
source = tar.extractfile("s4cmd-2.1.0/s4cmd.py")
assert source is not None
script.parent.mkdir(parents=True, exist_ok=True)
script.write_bytes(source.read())
wrapper(bin_dir / "s4cmd", f'exec "{venv}/bin/python" "{script}" "$@"')
elif tool == "s3-fast-list":
archive = fetch(url, sha, dl / name)
work = tools_dir / "s3-fast-list-src"
shutil.rmtree(work, ignore_errors=True)
work.mkdir(parents=True)
subprocess.run(["tar", "xzf", str(archive), "--strip-components=1", "-C", str(work)], check=True)
shutil.copyfile(study / "tools/s3-fast-list/build/Cargo.lock", work / "Cargo.lock")
subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=work, check=True)
for exe in ("s3-fast-list", "ks-tool"):
shutil.copyfile(work / "target/release" / exe, bin_dir / exe)
executable(bin_dir / exe)
elif tool == "swath":
ref = f"{url}@sha256:{sha}"
subprocess.run(["docker", "pull", "--quiet", ref], check=True, stdout=subprocess.DEVNULL)
cid = subprocess.run(
["docker", "create", ref], check=True, capture_output=True, text=True
).stdout.strip()
try:
dest = tools_dir / "swath"
shutil.rmtree(dest, ignore_errors=True)
dest.mkdir(parents=True)
subprocess.run(["docker", "cp", f"{cid}:/opt/java/openjdk", str(dest / "openjdk")], check=True)
subprocess.run(["docker", "cp", f"{cid}:/opt/swath/swath.jar", str(dest / "swath.jar")], check=True)
finally:
subprocess.run(["docker", "rm", cid], check=True, stdout=subprocess.DEVNULL)
else:
raise SystemExit(f"unknown tool {tool}")
stamp.parent.mkdir(parents=True, exist_ok=True)
stamp.write_text(sha)
print(f"installed {tool} {image.get('tool_version', '')}")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--study", required=True, type=Path)
parser.add_argument("--tools-dir", required=True, type=Path)
parser.add_argument("--tools", default=",".join(ALL_TOOLS))
args = parser.parse_args()
for tool in filter(None, args.tools.split(",")):
install(tool, args.study.resolve(), args.tools_dir.resolve())
return 0
if __name__ == "__main__":
os.umask(0o022)
raise SystemExit(main())
+6
View File
@@ -0,0 +1,6 @@
# Versions the pinned s3-listing-study uv.lock resolves for the capsule normalizers.
duckdb==1.5.5
ijson==3.5.1
pyyaml==6.0.3
zstandard==0.25.0
botocore==1.44.5
+922
View File
@@ -0,0 +1,922 @@
#!/usr/bin/env python3
"""Run the s3-listing-study tool capsules and ListObjects reference probes against SeaweedFS.
Tool cases: every mode of every capsule under the study's tools/ is compiled by
the capsule's own command.py (benchmark.adapters.compile_command, with the
custom-endpoint wiring the study's replay-endpoint pilot established), run
against SeaweedFS with the pinned tool build, normalized by the capsule's own
normalize.py, and the normalized listing is compared exactly against the keys
that were written.
Reference probes: raw, signed ListObjectsV2/ListObjects requests implementing
the strategies the study classifies tools by (serial pagination, delimiter
prefix discovery, start-after range splitting) plus the request semantics in
docs/s3-reference.md, checked against the byte-ordered key set.
"""
from __future__ import annotations
import argparse
import concurrent.futures
import hashlib
import json
import os
import re
import shutil
import subprocess
import sys
import time
import traceback
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from pathlib import Path
REGION = "us-east-1"
S3NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
def canary_keys() -> dict[str, int]:
"""benchmark/fixtures/replay-canary/generate.sql: 2,048 keys in 16 prefixes."""
return {f"group-{i // 128:02d}/object-{i:06d}.dat": 1024 + i for i in range(2048)}
EDGE_KEYS = [
# docs/smoke-bucket.md: a root key beside the prefixes, for delimiter checks.
"index.html",
# docs/s3-reference.md examples.
"test_file(3).png",
"photos/2006/February/sample.jpg",
"photos/2006/January/sample.jpg",
"photos/2007/sample.jpg",
"notes/summer/july",
"notes/summer/august",
"notes/winter",
# Byte order around the '/' (0x2f) delimiter.
"order/a",
"order/a b",
"order/a!b",
"order/a-b",
"order/a.b",
"order/a/b",
"order/a/c/d",
"order/a0",
"order/aa",
"order/a~",
"order/A",
"order/b",
# Zero-byte directory markers.
"markers/dir/",
"markers/dir/child",
"markers/empty/",
# Characters that need XML or URL escaping.
"chars/space key",
"chars/plus+key",
"chars/percent%20key",
"chars/amp&key",
"chars/lt<gt>key",
'chars/quote"key',
"chars/apos'key",
"chars/hash#key",
"chars/question?key",
"chars/equals=key",
"chars/semicolon;key",
"chars/colon:key",
"chars/at@key",
"chars/comma,key",
"chars/star*key",
"chars/bang!key",
"chars/dollar$key",
"chars/caret^key",
"chars/backslash\\key",
"chars/brace{key}",
"chars/pipe|key",
"chars/backtick`key",
"chars/tilde~key",
# UTF-8 byte order, which differs from UTF-16 order above U+FFFF.
"unicode/café-nfc",
"unicode/café-nfd",
"unicode/日本語.txt",
"unicode/。-halfwidth",
"unicode/\U0001f600-emoji",
"unicode/z-ascii",
"deep/a/b/c/d/e/f/g/h/i/j/k.txt",
"long/" + "/".join(["y" * 200] * 4),
]
def edge_keys() -> dict[str, int]:
return {k: (0 if k.endswith("/") else 100 + i) for i, k in enumerate(EDGE_KEYS)}
def body_for(key: str, size: int) -> bytes:
seed = hashlib.sha256(key.encode()).digest()
return (seed * (size // len(seed) + 1))[:size]
def utf8_sorted(keys) -> list[str]:
return sorted(keys, key=lambda k: k.encode("utf-8"))
# ---------------------------------------------------------------- signed HTTP
class S3:
def __init__(self, endpoint: str, access: str, secret: str) -> None:
from botocore.auth import S3SigV4Auth
from botocore.credentials import Credentials
self.endpoint = endpoint.rstrip("/")
self.auth = S3SigV4Auth(Credentials(access, secret), "s3", REGION)
def request(
self, method: str, path: str, params: list[tuple[str, str]] | None = None, body: bytes = b""
) -> tuple[int, bytes]:
from botocore.awsrequest import AWSRequest
query = "&".join(
f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}"
for k, v in (params or [])
)
url = self.endpoint + urllib.parse.quote(path, safe="/-_.~") + (f"?{query}" if query else "")
req = AWSRequest(method=method, url=url, data=body)
req.headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest()
self.auth.add_auth(req)
prepared = req.prepare()
http = urllib.request.Request(prepared.url, data=body or None, method=method)
for k, v in prepared.headers.items():
http.add_header(k, v)
try:
with urllib.request.urlopen(http, timeout=60) as resp:
return resp.status, resp.read()
except urllib.error.HTTPError as err:
return err.code, err.read()
def put_object(self, bucket: str, key: str, body: bytes) -> None:
# The first writes to a new bucket can get a retryable 500 while its
# volumes grow; SDK clients retry those the same way.
for attempt in range(10):
status, out = self.request("PUT", f"/{bucket}/{key}", body=body)
if status < 500:
break
time.sleep(0.5 * (attempt + 1))
if status != 200:
raise RuntimeError(f"PUT {bucket}/{key!r}: {status} {out[:300]!r}")
def delete_object(self, bucket: str, key: str) -> None:
status, out = self.request("DELETE", f"/{bucket}/{key}")
if status != 204:
raise RuntimeError(f"DELETE {bucket}/{key!r}: {status} {out[:300]!r}")
def create_bucket(self, bucket: str) -> None:
status, out = self.request("PUT", f"/{bucket}")
if status not in (200, 409):
raise RuntimeError(f"create bucket {bucket}: {status} {out[:300]!r}")
def list_page(self, bucket: str, params: list[tuple[str, str]]) -> ET.Element:
status, out = self.request("GET", f"/{bucket}", params)
if status != 200:
raise AssertionError(f"list {params}: HTTP {status} {out[:300]!r}")
return ET.fromstring(out)
def text(el: ET.Element, name: str) -> str | None:
child = el.find(S3NS + name)
return None if child is None else (child.text or "")
@dataclass
class Page:
entries: list[tuple[str, str]] # ("key"|"prefix", value) in response order
truncated: bool
token: str | None
next_marker: str | None
key_count: str | None
raw: ET.Element
def check_byte_order(values: list[str], what: str) -> None:
for a, b in zip(values, values[1:]):
assert a.encode() <= b.encode(), f"{what} out of byte order: {a!r} before {b!r}"
def parse_page(root: ET.Element, url_encoded: bool) -> Page:
dec = urllib.parse.unquote_plus if url_encoded else (lambda s: s)
entries: list[tuple[str, str]] = []
for child in root:
if child.tag == S3NS + "Contents":
entries.append(("key", dec(text(child, "Key") or "")))
elif child.tag == S3NS + "CommonPrefixes":
entries.append(("prefix", dec(text(child, "Prefix") or "")))
nm = text(root, "NextMarker")
# XML carries Contents and CommonPrefixes as two lists, each in byte order;
# their merged byte order is what pagination has to respect.
for kind in ("key", "prefix"):
check_byte_order([v for k, v in entries if k == kind], f"{kind} list")
entries.sort(key=lambda e: e[1].encode())
return Page(
entries,
text(root, "IsTruncated") == "true",
text(root, "NextContinuationToken"),
dec(nm) if nm is not None else None,
text(root, "KeyCount"),
root,
)
# ---------------------------------------------------------------- expectations
def direct_view(keys: list[str], prefix: str, delimiter: str) -> list[tuple[str, str]]:
"""What a delimiter listing of prefix returns, in byte order."""
out: dict[str, str] = {}
for k in keys:
if not k.startswith(prefix):
continue
rest = k[len(prefix) :]
idx = rest.find(delimiter) if delimiter else -1
if idx >= 0:
out[prefix + rest[: idx + len(delimiter)]] = "prefix"
else:
out[k] = "key"
return [(out[k], k) for k in utf8_sorted(out)]
# ---------------------------------------------------------------- reference probes
class Probe:
def __init__(self, s3: S3, bucket: str, keys: list[str]) -> None:
self.s3, self.bucket, self.keys = s3, bucket, utf8_sorted(keys)
def v2_all(self, base: list[tuple[str, str]], max_keys: int, url: bool = False) -> list[Page]:
pages: list[Page] = []
token = None
while True:
params = [("list-type", "2"), ("max-keys", str(max_keys)), *base]
if url:
params.append(("encoding-type", "url"))
if token:
params.append(("continuation-token", token))
page = parse_page(self.s3.list_page(self.bucket, params), url)
pages.append(page)
n = len(page.entries)
assert n <= max_keys, f"page holds {n} > max-keys {max_keys}"
assert page.key_count == str(n), f"KeyCount {page.key_count} != {n} entries"
if not page.truncated:
assert not page.token, "NextContinuationToken on an untruncated page"
return pages
assert page.token, "IsTruncated without NextContinuationToken"
assert len(pages) < 10000, "pagination does not terminate"
token = page.token
def v1_all(self, base: list[tuple[str, str]], max_keys: int, delimiter: str) -> list[Page]:
pages: list[Page] = []
marker = None
while True:
params = [("max-keys", str(max_keys)), *base]
if delimiter:
params.append(("delimiter", delimiter))
if marker is not None:
params.append(("marker", marker))
page = parse_page(self.s3.list_page(self.bucket, params), False)
pages.append(page)
assert len(page.entries) <= max_keys
if not page.truncated:
return pages
if delimiter:
assert page.next_marker, "truncated delimiter page without NextMarker"
marker = page.next_marker
else:
marker = page.next_marker or page.entries[-1][1]
assert len(pages) < 10000, "pagination does not terminate"
@staticmethod
def flat(pages: list[Page]) -> list[tuple[str, str]]:
return [e for p in pages for e in p.entries]
def expect_eq(self, got, want, what: str) -> None:
if got != want:
gs, ws = set(got), set(want)
missing = [w for w in want if w not in gs][:5]
extra = [g for g in got if g not in ws][:5]
first = next((i for i, (a, b) in enumerate(zip(got, want)) if a != b), min(len(got), len(want)))
raise AssertionError(
f"{what}: got {len(got)} want {len(want)}; missing {missing} extra {extra}; "
f"first difference at {first}: got {got[first:first + 3]} want {want[first:first + 3]}"
)
# Serial pagination (aws-cli, mc, s3kor, s5cmd ls).
def serial(self, prefix: str, max_keys: int) -> None:
want = [("key", k) for k in self.keys if k.startswith(prefix)]
got = self.flat(self.v2_all([("prefix", prefix)], max_keys))
self.expect_eq(got, want, f"serial prefix={prefix!r} max-keys={max_keys}")
# Delimiter view and page boundaries inside it.
def delimiter(self, prefix: str, delimiter: str, max_keys: int, v1: bool = False) -> None:
want = direct_view(self.keys, prefix, delimiter)
if v1:
got = self.flat(self.v1_all([("prefix", prefix)], max_keys, delimiter))
else:
got = self.flat(self.v2_all([("prefix", prefix), ("delimiter", delimiter)], max_keys))
self.expect_eq(got, want, f"{'v1' if v1 else 'v2'} delimiter={delimiter!r} prefix={prefix!r} max-keys={max_keys}")
# Prefix discovery (rclone walk, s7cmd, s4cmd): recurse through CommonPrefixes.
def walk(self, max_keys: int, v1: bool = False) -> None:
found: list[str] = []
queue = [""]
while queue:
prefix = queue.pop()
if v1:
entries = self.flat(self.v1_all([("prefix", prefix)], max_keys, "/"))
else:
entries = self.flat(self.v2_all([("prefix", prefix), ("delimiter", "/")], max_keys))
for kind, value in entries:
(queue.append if kind == "prefix" else found.append)(value)
self.expect_eq(utf8_sorted(found), self.keys, f"{'v1' if v1 else 'v2'} delimiter walk max-keys={max_keys}")
assert len(found) == len(set(found)), "walk returned a key twice"
# Range splitting (S3P, s3-fast-list, Swath): start-after cut points.
def ranges(self, cuts: list[str], max_keys: int) -> None:
cuts = sorted(set(cuts), key=lambda c: c.encode())
bounds = [None, *cuts, None]
found: list[str] = []
for lo, hi in zip(bounds, bounds[1:]):
token = None
done = False
while not done:
params = [("list-type", "2"), ("max-keys", str(max_keys))]
if token:
params.append(("continuation-token", token))
elif lo is not None:
params.append(("start-after", lo))
page = parse_page(self.s3.list_page(self.bucket, params), False)
for _, key in page.entries:
if lo is not None:
assert key.encode() > lo.encode(), f"start-after {lo!r} returned {key!r}"
if hi is not None and key.encode() > hi.encode():
done = True
break
found.append(key)
if not page.truncated:
break
token = page.token
self.expect_eq(found, self.keys, f"start-after ranges ({len(cuts)} cuts, max-keys={max_keys})")
def start_after_with_delimiter(self, prefix: str, start_after: str) -> None:
want = [
e
for e in direct_view(self.keys, prefix, "/")
if e[1].encode() > start_after.encode()
]
got = self.flat(
self.v2_all([("prefix", prefix), ("delimiter", "/"), ("start-after", start_after)], 1000)
)
self.expect_eq(got, want, f"start-after={start_after!r} delimiter=/ prefix={prefix!r}")
def token_overrides_start_after(self) -> None:
first = parse_page(self.s3.list_page(self.bucket, [("list-type", "2"), ("max-keys", "5")]), False)
assert first.truncated and first.token
params = [("list-type", "2"), ("max-keys", "5"), ("continuation-token", first.token)]
plain = parse_page(self.s3.list_page(self.bucket, params), False)
mixed = parse_page(
self.s3.list_page(self.bucket, [*params, ("start-after", self.keys[len(self.keys) // 2])]),
False,
)
self.expect_eq(mixed.entries, plain.entries, "continuation-token with a later start-after")
def versions(self, prefix: str, delimiter: str, max_keys: int, url: bool) -> None:
"""ListObjectVersions paging (key-marker/version-id-marker) over unversioned keys."""
want = direct_view(self.keys, prefix, delimiter) if delimiter else [("key", k) for k in self.keys if k.startswith(prefix)]
got: list[tuple[str, str]] = []
key_marker = version_marker = None
dec = urllib.parse.unquote_plus if url else (lambda v: v)
for _ in range(10000):
params = [("versions", ""), ("max-keys", str(max_keys)), ("prefix", prefix)]
if delimiter:
params.append(("delimiter", delimiter))
if url:
params.append(("encoding-type", "url"))
if key_marker is not None:
params += [("key-marker", key_marker), ("version-id-marker", version_marker or "")]
root = self.s3.list_page(self.bucket, params)
page: list[tuple[str, str]] = []
for child in root:
if child.tag in (S3NS + "Version", S3NS + "DeleteMarker"):
page.append(("key", dec(text(child, "Key") or "")))
elif child.tag == S3NS + "CommonPrefixes":
page.append(("prefix", dec(text(child, "Prefix") or "")))
assert len(page) <= max_keys, f"{len(page)} entries > max-keys {max_keys}"
for kind in ("key", "prefix"):
check_byte_order([v for k, v in page if k == kind], f"version {kind} list")
got += sorted(page, key=lambda e: e[1].encode())
if text(root, "IsTruncated") != "true":
break
nk = text(root, "NextKeyMarker")
assert nk, "truncated versions page without NextKeyMarker"
key_marker, version_marker = dec(nk), text(root, "NextVersionIdMarker")
self.expect_eq(got, want, f"versions prefix={prefix!r} delimiter={delimiter!r} max-keys={max_keys} url={url}")
def url_encoding(self, prefix: str) -> None:
want = [("key", k) for k in self.keys if k.startswith(prefix)]
got = self.flat(self.v2_all([("prefix", prefix)], 7, url=True))
self.expect_eq(got, want, f"encoding-type=url prefix={prefix!r}")
root = self.s3.list_page(
self.bucket,
[("list-type", "2"), ("encoding-type", "url"), ("prefix", prefix), ("delimiter", "/"), ("start-after", prefix + "a b")],
)
for name, value in (("Prefix", prefix), ("Delimiter", "/"), ("StartAfter", prefix + "a b")):
raw = text(root, name)
assert raw is not None and urllib.parse.unquote_plus(raw) == value, f"encoded {name} {raw!r} != {value!r}"
assert text(root, "EncodingType") == "url", "EncodingType not echoed"
def deleted_prefixes(s3: S3, bucket: str) -> None:
"""A prefix whose last key was deleted is no longer a CommonPrefix."""
s3.create_bucket(bucket)
keys = ["keep", "gone/a/b", "gone/c", "half/x", "half/y/z"]
for k in keys:
s3.put_object(bucket, k, b"x")
for k in ("gone/a/b", "gone/c", "half/y/z"):
s3.delete_object(bucket, k)
p = Probe(s3, bucket, ["keep", "half/x"])
p.delimiter("", "/", 1000)
p.delimiter("half/", "/", 1000)
p.delimiter("gone/", "/", 1000)
def run_probes(
s3: S3, buckets: dict[str, dict[str, int]], churn_bucket: str, only: re.Pattern | None
) -> list[tuple[str, str | None]]:
results: list[tuple[str, str | None]] = []
def check(name: str, fn, *a) -> None:
if only and not only.search(name):
return
try:
fn(*a)
results.append((name, None))
except Exception as exc: # noqa: BLE001
results.append((name, f"{type(exc).__name__}: {exc}"))
check(f"reference:{churn_bucket}:deleted-prefixes", deleted_prefixes, s3, churn_bucket)
for bucket, keymap in buckets.items():
p = Probe(s3, bucket, list(keymap))
keys = p.keys
tag = f"reference:{bucket}"
for mk in (1, 7, 100, 1000):
check(f"{tag}:serial:max-keys={mk}", p.serial, "", mk)
check(f"{tag}:url-encoding", p.url_encoding, "")
check(f"{tag}:token-overrides-start-after", p.token_overrides_start_after)
for mk in (2, 1000):
check(f"{tag}:walk-v2:max-keys={mk}", p.walk, mk)
check(f"{tag}:walk-v1:max-keys={mk}", p.walk, mk, True)
prefixes = sorted({k[: k.index("/") + 1] for k in keys if "/" in k})
scopes = ["", *prefixes[:: max(1, len(prefixes) // 4)]]
if bucket.endswith("edge"):
scopes += ["order/", "order/a", "chars/", "unicode/", "markers/", "markers/dir/", "photos/2006/", "o"]
for prefix in scopes:
for mk in (1, 3, 1000):
check(f"{tag}:delimiter-v2:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk)
check(f"{tag}:delimiter-v1:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk, True)
for prefix in ("", scopes[-1]):
for delim in ("", "/"):
for mk in (2, 1000):
check(f"{tag}:versions:prefix={prefix}:delimiter={delim}:max-keys={mk}", p.versions, prefix, delim, mk, False)
check(f"{tag}:versions-url:prefix={prefix}", p.versions, prefix, "/", 3, True)
for d in ("-", "a", "ou"):
check(f"{tag}:delimiter={d}", p.delimiter, "", d, 1000)
check(f"{tag}:serial:prefix-no-slash", p.serial, keys[len(keys) // 3][:4], 5)
# Cut points that are existing keys, absent keys between keys, prefixes,
# and a common prefix with and without its trailing delimiter.
step = max(1, len(keys) // 9)
existing = keys[step::step]
absent = [k + "\x7f" for k in existing[::2]] + [k[:-1] for k in existing[1::2] if len(k) > 1]
dirs = [k[: k.rfind("/") + 1] for k in existing if "/" in k]
dirs += [d.rstrip("/") for d in dirs]
for name, cuts in (("existing", existing), ("absent", absent), ("dirs", dirs), ("mixed", existing + absent + dirs)):
for mk in (3, 1000):
check(f"{tag}:ranges-{name}:max-keys={mk}", p.ranges, cuts, mk)
for sa in (keys[len(keys) // 2], prefixes[len(prefixes) // 2], prefixes[len(prefixes) // 2].rstrip("/")):
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "", sa)
if bucket.endswith("edge"):
for sa in ("order/a", "order/a/", "order/a/b", "order/a.b", "order/a0"):
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "order/", sa)
check(f"{tag}:url-encoding:chars/", p.url_encoding, "chars/")
return results
# ---------------------------------------------------------------- tool capsules
@dataclass
class Case:
tool: str
mode: str
bucket: str
prefix: str
status: str = ""
detail: str = ""
argv: list[str] = field(default_factory=list)
class ToolRunner:
def __init__(self, args, study: Path, work: Path, endpoint: str) -> None:
sys.path.insert(0, str(study / "benchmark" / "src"))
from benchmark import adapters # noqa: PLC0415
from benchmark.runtime.command_adapter import CommandAdapterError, Stated # noqa: PLC0415
self.adapters, self.refusal, self.stated = adapters, CommandAdapterError, Stated
self.study, self.work, self.endpoint = study, work, endpoint
tools = Path(args.tools_dir).resolve()
self.fixture_dir = work / "fixtures"
self.path_map = {
"/usr/local/bin/aws": str(tools / "aws-cli-bin" / "aws"),
"/usr/bin/mc": str(tools / "bin" / "mc"),
"/usr/local/bin/pS3": str(tools / "bin" / "pS3"),
"/usr/local/bin/rclone": str(tools / "bin" / "rclone"),
"/usr/bin/s3-fast-list": str(tools / "bin" / "s3-fast-list"),
"/usr/bin/ks-tool": str(tools / "bin" / "ks-tool"),
"/usr/local/bin/s3kor": str(tools / "bin" / "s3kor"),
"/usr/local/bin/s3p": str(tools / "bin" / "s3p"),
"/usr/local/bin/s4cmd": str(tools / "bin" / "s4cmd"),
"/s5cmd": str(tools / "bin" / "s5cmd"),
"/usr/local/bin/s7cmd": str(tools / "bin" / "s7cmd"),
"/opt/java/openjdk/bin/java": str(tools / "swath" / "openjdk" / "bin" / "java"),
"/opt/swath/swath.jar": str(tools / "swath" / "swath.jar"),
"/usr/bin/python3": sys.executable,
"/opt/benchmark/tools/": str(study / "tools") + "/",
}
os.environ["PYTHONPATH"] = str(study / "benchmark" / "src")
self.env = {
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
"AWS_ACCESS_KEY_ID": args.access_key,
"AWS_SECRET_ACCESS_KEY": args.secret_key,
"AWS_REGION": REGION,
"AWS_DEFAULT_REGION": REGION,
"AWS_EC2_METADATA_DISABLED": "true",
"TZ": "UTC",
"LC_ALL": "C.UTF-8",
"PYTHONPATH": str(study / "benchmark" / "src"),
}
def localize(self, argv, bucket: str) -> list[str]:
out = []
path_map = {**self.path_map, "/fixtures/source/": f"{self.fixture_dir / bucket}/"}
for token in argv:
for src, dst in path_map.items():
if token == src or (src.endswith("/") and token.startswith(src)):
token = dst + token[len(src) :]
break
out.append(token)
return out
def adapter_dir(self, tool: str) -> Path:
return self.study / "tools" / tool / "adapter"
def write_fixture_companions(self, bucket: str, keys: list[str]) -> None:
"""The staged-fixture companions, built the way fixture_bundle.py builds them."""
import duckdb # noqa: PLC0415
out = self.fixture_dir / bucket
shutil.rmtree(out, ignore_errors=True)
out.mkdir(parents=True)
shards = sorted({k.split("/", 1)[0] for k in keys})
if all(re.fullmatch(r"[A-Za-z0-9._/-]+", s) for s in shards):
(out / "s5cmd-shards.input").write_text("".join(f"{s}\n" for s in shards))
parts = out / "parquet"
parts.mkdir()
with duckdb.connect() as con:
con.execute("CREATE TABLE f (key BLOB, row_type VARCHAR)")
con.executemany("INSERT INTO f VALUES (?, 'OBJECT')", [(k.encode(),) for k in keys])
con.execute(f"COPY f TO '{parts / 'part-00000.parquet'}' (FORMAT parquet)")
sys.path.insert(0, str(self.study / "tools" / "s3-fast-list" / "adapter"))
from fixture_hints import FixtureHintsError, generate_hints # noqa: PLC0415
try:
generate_hints(parts, out / "s3-fast-list-hints.input", segments=4)
except FixtureHintsError as exc:
print(f"{bucket}: no s3-fast-list fixture hints: {exc}", flush=True)
def inject_endpoint(self, tool: str, argv: list[str], env: dict) -> tuple[list[str], dict]:
"""Add the endpoint the way the replay-endpoint pilot (run.sh) did, for modes
the capsule refuses only because the replay server lacks the API."""
ep = self.endpoint
if tool == "rclone":
return [a.replace(f"region={REGION}", f'region={REGION},endpoint="{ep}",force_path_style=true') for a in argv], env
if tool == "minio-mc":
return argv, {**env, "MC_HOST_s3": ep, "MC_REGION": REGION}
if tool == "s3kor":
return [argv[0], argv[1], "--custom-endpoint-url", ep, *argv[2:]], env
if tool == "s7cmd":
return [*argv[:-1], "--target-endpoint-url", ep, "--target-force-path-style", argv[-1]], env
if tool == "s5cmd":
return [argv[0], "--endpoint-url", ep, *argv[1:]], env
return [*argv, "--endpoint-url", ep], env
def config_for(self, adapter, case: Case, mode: str, keys: list[str]) -> dict[str, object]:
config: dict[str, object] = {}
for name, axis in adapter.modes[mode].axes.items():
if isinstance(axis, self.stated):
config[name] = 4
if adapter.tool == "s5cmd" and mode == "fanout-with-dirs":
initials = sorted({k[len(case.prefix)] for k in keys if k.startswith(case.prefix) and len(k) > len(case.prefix)})
if not all(re.fullmatch(r"[A-Za-z0-9._-]", c) for c in initials):
raise NotApplicable(f"key initials {initials} are outside the capsule's safe shard set")
config["shard_initials"] = "".join(initials)
return config
def execute(self, adapter, case: Case, mode: str, sink: Path, keys: list[str], artifact_path: str = "") -> tuple[Path | None, dict]:
signed = adapter.supports_signed
inject = case.tool in INJECT_ENDPOINT and mode in INJECT_ENDPOINT[case.tool]
config = self.config_for(adapter, case, mode, keys)
argv, env = self.adapters.compile_command(
self.adapter_dir(case.tool),
case.tool,
mode=mode,
bucket=case.bucket,
region=REGION,
prefix=case.prefix,
signed=signed,
config=config,
sink_dir=str(sink),
artifact_path=artifact_path,
visible_memory_gb=2.0,
endpoint_url="" if inject else self.endpoint,
)
if inject:
argv, env = self.inject_endpoint(case.tool, list(argv), env)
case.detail = "endpoint injected; "
argv = self.localize(argv, case.bucket)
case.argv = argv
sink.mkdir(parents=True, exist_ok=True)
home = sink / "home"
home.mkdir(exist_ok=True)
manifest = adapter.modes[mode]
stdout_path = sink / (manifest.product_file if manifest.product_channel == "stdout" and manifest.product_artifact else "stdout")
with stdout_path.open("wb") as out, (sink / "stderr").open("wb") as err:
proc = subprocess.run(
argv,
stdout=out,
stderr=err,
env={**self.env, **env, "HOME": str(home)},
cwd=sink,
timeout=300,
check=False,
)
if proc.returncode != 0:
tail = (sink / "stderr").read_bytes()[-600:].decode(errors="replace")
raise RuntimeError(f"exit {proc.returncode}: {tail}")
product = sink / manifest.product_file if manifest.product_artifact else None
return product, config
def run_case(self, case: Case, expected: dict[str, tuple[int, str]]) -> None:
adapter = self.adapters.load_adapter(self.adapter_dir(case.tool), case.tool)
manifest = adapter.modes[case.mode]
if manifest.purpose_ceiling == "preparation" or not manifest.product_artifact or (case.tool, case.mode) in NOT_A_LISTING:
case.status = "skip"
case.detail = f"{manifest.purpose_ceiling} mode, not a compared listing"
return
if (case.tool, case.mode) in ROOT_ONLY and case.prefix:
case.status, case.detail = "skip", "mode lists the bucket root whatever the prefix"
return
fixture = self.fixture_dir / case.bucket
needs = {"fanout-fixture-with-dirs": "s5cmd-shards.input", "list-hinted-fixture": "s3-fast-list-hints.input"}.get(case.mode)
if needs and not (fixture / needs).exists():
case.status, case.detail = "skip", f"fixture_bundle.py would not build {needs} for this key set"
return
keys = list(expected)
sink = self.work / "cases" / case.tool / case.mode / (case.bucket + "~" + (case.prefix.replace("/", "_") or "root"))
shutil.rmtree(sink, ignore_errors=True)
try:
artifact_path = ""
for step in adapter.requires.get(case.mode, ()):
producer_sink = sink / f"requires-{step.mode}"
self.execute(adapter, case, step.mode, producer_sink, keys)
artifact_path = str(producer_sink / adapter.modes[step.mode].artifacts[step.artifact])
if manifest.inline:
inline_sink = sink / f"inline-{manifest.inline}"
self.execute(adapter, case, manifest.inline, inline_sink, keys, artifact_path)
inline_manifest = adapter.modes[manifest.inline]
artifact_path = str(inline_sink / next(iter(inline_manifest.artifacts.values())))
product, config = self.execute(adapter, case, case.mode, sink, keys, artifact_path)
except NotApplicable as exc:
case.status, case.detail = "skip", str(exc)
return
except self.adapters.AdapterError as exc:
if "custom endpoint" in str(exc) or "replay endpoint" in str(exc) or "no --prefix" in str(exc) or "requires HeadBucket" in str(exc):
case.status, case.detail = "skip", f"capsule refuses: {str(exc).split(': ', 2)[-1][:160]}"
return
raise
normalized = sink / "normalized.tsv"
self.adapters.normalize_to_path(
self.adapter_dir(case.tool),
case.tool,
case.mode,
case.prefix,
normalized,
input_path=None if manifest.product_channel == "dataset" else product,
dataset=product if manifest.product_channel == "dataset" else None,
config=config,
)
compare(case, normalized.read_bytes(), expected, manifest.fields)
INJECT_ENDPOINT = {
"aws-cli": {"s3api-v1-text", "s3api-versions-text"},
"minio-mc": {"versions-json", "find", "find-json"},
"rclone": {"listv1"},
"s3kor": {"list-versions"},
"s4cmd": {"recursive", "shallow", "show-directory", "du"},
"s5cmd": {"listv1", "allversions"},
"s7cmd": {"all-versions"},
}
"""Capsule modes refused only because the study's replay server serves
ListObjectsV2 alone; SeaweedFS serves ListObjects, ListObjectVersions and
HeadBucket too."""
NOT_A_LISTING = {
("rclone", "debug"),
("rclone", "walk-debug"),
("s3p", "summarize"),
("s4cmd", "du"),
("s4cmd", "show-directory"),
}
"""Modes with no per-key listing: rclone's normalizer has no debug modes, summarize
and du print one aggregate, and `ls -d` prints the directory itself."""
class NotApplicable(Exception):
pass
ROOT_ONLY = {
("aws-cli", "s3api-v2-remainder"),
("s5cmd", "fanout-fixture-with-dirs"),
("s3-fast-list", "list-hinted-fixture"),
}
"""Modes whose capsule lists the root (remainder) or a staged root-scope fixture."""
SHALLOW = re.compile(r"shallow|delimiter|rootkeys|remainder|max-depth")
KEYS_ONLY = re.compile(r"rootkeys|remainder")
"""Shallow modes whose normalizer keeps only the keys, dropping CommonPrefixes."""
def compare(case: Case, normalized: bytes, expected: dict[str, tuple[int, str]], fields) -> None:
rows = []
for line in normalized.split(b"\n"):
if not line:
continue
f = line.split(b"\t")
rows.append((f[0].decode("utf-8", "surrogateescape"), f[1].decode(), f[2].decode()))
scope = {k: v for k, v in expected.items() if k.startswith(case.prefix)}
keys = utf8_sorted(scope)
shallow = bool(SHALLOW.search(case.mode))
if shallow:
view = direct_view(keys, case.prefix, "/")
want_objects = [k for kind, k in view if kind == "key"]
allowed_dirs = {k for kind, k in view if kind == "prefix"}
else:
want_objects = keys
allowed_dirs = {k[: i + 1] for k in keys for i, c in enumerate(k) if c == "/" and i >= len(case.prefix)}
problems: list[str] = []
objects, dirs = [], []
for key, size, etag in rows:
is_dir = key.endswith("/") and size == "-" and etag == "-" and key not in scope
(dirs if is_dir else objects).append((key, size, etag))
got_keys = [k for k, _, _ in objects]
dup = {k for k in got_keys if got_keys.count(k) > 1} if len(got_keys) != len(set(got_keys)) else set()
if dup:
problems.append(f"duplicate keys {sorted(dup)[:5]}")
missing = [k for k in want_objects if k not in set(got_keys)]
extra = [k for k in set(got_keys) if k not in set(want_objects)]
if missing:
problems.append(f"missing {len(missing)}: {missing[:5]}")
if extra:
problems.append(f"unexpected {len(extra)}: {sorted(extra)[:5]}")
bad_dirs = [d for d, _, _ in dirs if d not in allowed_dirs]
if bad_dirs:
problems.append(f"unexpected directory rows {bad_dirs[:5]}")
if shallow and not KEYS_ONLY.search(case.mode) and {d for d, _, _ in dirs} != allowed_dirs:
problems.append(f"common prefixes {sorted(d for d, _, _ in dirs)[:8]} != {sorted(allowed_dirs)[:8]}")
for key, size, etag in objects:
if key not in scope:
continue
want_size, want_etag = scope[key]
if size != "-" and size != str(want_size):
problems.append(f"{key!r} size {size} != {want_size}")
if etag != "-" and etag != want_etag:
problems.append(f"{key!r} etag {etag} != {want_etag}")
case.status = "fail" if problems else "pass"
case.detail += "; ".join(problems)[:1500] if problems else f"{len(objects)} objects, {len(dirs)} prefixes"
# ---------------------------------------------------------------- main
def load_expected_failures(path: Path | None) -> dict[str, str]:
out: dict[str, str] = {}
if path and path.exists():
for line in path.read_text().splitlines():
line = line.strip()
if line and not line.startswith("#"):
name, _, reason = line.partition("#")
out[name.strip()] = reason.strip()
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--endpoint", required=True)
ap.add_argument("--access-key", required=True)
ap.add_argument("--secret-key", required=True)
ap.add_argument("--study", required=True)
ap.add_argument("--tools-dir", required=True)
ap.add_argument("--work-dir", required=True)
ap.add_argument("--tools", default="aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath")
ap.add_argument("--bucket-prefix", default="study")
ap.add_argument("--expected-failures", type=Path)
ap.add_argument("--skip-upload", action="store_true")
ap.add_argument("--only", default="", help="regex over case names")
ap.add_argument("--jobs", type=int, default=4)
args = ap.parse_args()
study = Path(args.study).resolve()
work = Path(args.work_dir).resolve()
work.mkdir(parents=True, exist_ok=True)
s3 = S3(args.endpoint, args.access_key, args.secret_key)
buckets = {f"{args.bucket_prefix}-canary": canary_keys(), f"{args.bucket_prefix}-edge": edge_keys()}
expected: dict[str, dict[str, tuple[int, str]]] = {}
for bucket, keymap in buckets.items():
expected[bucket] = {k: (size, hashlib.md5(body_for(k, size)).hexdigest()) for k, size in keymap.items()}
if args.skip_upload:
continue
s3.create_bucket(bucket)
start = time.time()
with concurrent.futures.ThreadPoolExecutor(16) as pool:
list(pool.map(lambda k: s3.put_object(bucket, k, body_for(k, keymap[k])), keymap))
print(f"uploaded {len(keymap)} objects to {bucket} in {time.time() - start:.1f}s", flush=True)
only = re.compile(args.only) if args.only else None
results: list[tuple[str, str, str]] = []
for name, err in run_probes(s3, buckets, f"{args.bucket_prefix}-churn", only):
results.append((name, "fail" if err else "pass", err or ""))
runner = ToolRunner(args, study, work, args.endpoint)
cases: list[Case] = []
for bucket in buckets:
runner.write_fixture_companions(bucket, list(buckets[bucket]))
scopes = ["", "group-07/"] if bucket.endswith("canary") else ["", "order/", "chars/"]
for tool in filter(None, args.tools.split(",")):
adapter = runner.adapters.load_adapter(runner.adapter_dir(tool), tool)
for mode in adapter.modes:
for prefix in scopes:
case = Case(tool, mode, bucket, prefix)
if only and not only.search(case_name(case)):
continue
cases.append(case)
def run(case: Case) -> Case:
start = time.time()
try:
runner.run_case(case, expected[case.bucket])
except Exception as exc: # noqa: BLE001
case.status = "fail"
case.detail = f"{type(exc).__name__}: {exc}"[:1500]
if os.environ.get("STUDY_DEBUG"):
traceback.print_exc()
print(f"{case.status:5} {case_name(case)} ({time.time() - start:.1f}s) {case.detail[:300]}", flush=True)
return case
with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
for case in pool.map(run, cases):
results.append((case_name(case), case.status, case.detail))
xfail = load_expected_failures(args.expected_failures)
regressions, fixed = [], []
counts: dict[str, int] = {}
for name, status, detail in results:
if status == "fail" and name in xfail:
status = "xfail"
elif status == "pass" and name in xfail:
fixed.append(name)
elif status == "fail":
regressions.append((name, detail))
counts[status] = counts.get(status, 0) + 1
(work / "results.json").write_text(json.dumps(results, indent=1, ensure_ascii=False))
print("\n== summary:", ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
for name in fixed:
print(f"XPASS (remove from expected failures): {name}")
for name, detail in regressions:
print(f"FAIL {name}: {detail}")
return 1 if regressions or fixed else 0
def case_name(case: Case) -> str:
return f"{case.tool}:{case.mode}:{case.bucket.rsplit('-', 1)[-1]}:{case.prefix or '/'}"
if __name__ == "__main__":
raise SystemExit(main())
+18
View File
@@ -0,0 +1,18 @@
{
"identities": [
{
"name": "listing_study_admin",
"credentials": [
{
"accessKey": "listing_study_access",
"secretKey": "listing_study_secret"
}
],
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
},
{
"name": "anonymous",
"actions": ["Read", "List"]
}
]
}