mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* test(s3): s3-listing-study integration suite (WIP) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): run the s3-listing-study tools against SeaweedFS (WIP) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): s3-listing-study deleted-prefix probe, fresh data dir per run Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): listing study ports keep clear of every gRPC companion free_ports.py compared only base ports, so a base port could land on another server's base+10000 gRPC listener. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study ports stay below the ephemeral range Base ports of 20000-55000 put both them and their gRPC companions inside 32768-60999, so an outgoing connection made while weed server starts could take a port before the server that was given it binds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study checks the order inside each listing page parse_page sorted Contents and CommonPrefixes together before the probes compared them, so keys returned out of order within one page passed. Check each list's byte order first; the versions pager does the same. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study applies --only before running a probe The filter was applied to the probe results, so a run limited to one tool still sent every reference listing request and the deleted-prefix check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study requires CommonPrefixes from shallow modes The CommonPrefixes comparison ran only when a tool returned at least one prefix, so a delimiter listing of the canary root that came back empty passed. Compare them for every shallow mode except the two whose normalizer keeps keys only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
923 lines
40 KiB
Python
923 lines
40 KiB
Python
#!/usr/bin/env python3
|
|
"""Run the s3-listing-study tool capsules and ListObjects reference probes against SeaweedFS.
|
|
|
|
Tool cases: every mode of every capsule under the study's tools/ is compiled by
|
|
the capsule's own command.py (benchmark.adapters.compile_command, with the
|
|
custom-endpoint wiring the study's replay-endpoint pilot established), run
|
|
against SeaweedFS with the pinned tool build, normalized by the capsule's own
|
|
normalize.py, and the normalized listing is compared exactly against the keys
|
|
that were written.
|
|
|
|
Reference probes: raw, signed ListObjectsV2/ListObjects requests implementing
|
|
the strategies the study classifies tools by (serial pagination, delimiter
|
|
prefix discovery, start-after range splitting) plus the request semantics in
|
|
docs/s3-reference.md, checked against the byte-ordered key set.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import xml.etree.ElementTree as ET
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
REGION = "us-east-1"
|
|
S3NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
|
|
|
|
|
def canary_keys() -> dict[str, int]:
|
|
"""benchmark/fixtures/replay-canary/generate.sql: 2,048 keys in 16 prefixes."""
|
|
return {f"group-{i // 128:02d}/object-{i:06d}.dat": 1024 + i for i in range(2048)}
|
|
|
|
|
|
EDGE_KEYS = [
|
|
# docs/smoke-bucket.md: a root key beside the prefixes, for delimiter checks.
|
|
"index.html",
|
|
# docs/s3-reference.md examples.
|
|
"test_file(3).png",
|
|
"photos/2006/February/sample.jpg",
|
|
"photos/2006/January/sample.jpg",
|
|
"photos/2007/sample.jpg",
|
|
"notes/summer/july",
|
|
"notes/summer/august",
|
|
"notes/winter",
|
|
# Byte order around the '/' (0x2f) delimiter.
|
|
"order/a",
|
|
"order/a b",
|
|
"order/a!b",
|
|
"order/a-b",
|
|
"order/a.b",
|
|
"order/a/b",
|
|
"order/a/c/d",
|
|
"order/a0",
|
|
"order/aa",
|
|
"order/a~",
|
|
"order/A",
|
|
"order/b",
|
|
# Zero-byte directory markers.
|
|
"markers/dir/",
|
|
"markers/dir/child",
|
|
"markers/empty/",
|
|
# Characters that need XML or URL escaping.
|
|
"chars/space key",
|
|
"chars/plus+key",
|
|
"chars/percent%20key",
|
|
"chars/amp&key",
|
|
"chars/lt<gt>key",
|
|
'chars/quote"key',
|
|
"chars/apos'key",
|
|
"chars/hash#key",
|
|
"chars/question?key",
|
|
"chars/equals=key",
|
|
"chars/semicolon;key",
|
|
"chars/colon:key",
|
|
"chars/at@key",
|
|
"chars/comma,key",
|
|
"chars/star*key",
|
|
"chars/bang!key",
|
|
"chars/dollar$key",
|
|
"chars/caret^key",
|
|
"chars/backslash\\key",
|
|
"chars/brace{key}",
|
|
"chars/pipe|key",
|
|
"chars/backtick`key",
|
|
"chars/tilde~key",
|
|
# UTF-8 byte order, which differs from UTF-16 order above U+FFFF.
|
|
"unicode/café-nfc",
|
|
"unicode/café-nfd",
|
|
"unicode/日本語.txt",
|
|
"unicode/。-halfwidth",
|
|
"unicode/\U0001f600-emoji",
|
|
"unicode/z-ascii",
|
|
"deep/a/b/c/d/e/f/g/h/i/j/k.txt",
|
|
"long/" + "/".join(["y" * 200] * 4),
|
|
]
|
|
|
|
|
|
def edge_keys() -> dict[str, int]:
|
|
return {k: (0 if k.endswith("/") else 100 + i) for i, k in enumerate(EDGE_KEYS)}
|
|
|
|
|
|
def body_for(key: str, size: int) -> bytes:
|
|
seed = hashlib.sha256(key.encode()).digest()
|
|
return (seed * (size // len(seed) + 1))[:size]
|
|
|
|
|
|
def utf8_sorted(keys) -> list[str]:
|
|
return sorted(keys, key=lambda k: k.encode("utf-8"))
|
|
|
|
|
|
# ---------------------------------------------------------------- signed HTTP
|
|
|
|
|
|
class S3:
|
|
def __init__(self, endpoint: str, access: str, secret: str) -> None:
|
|
from botocore.auth import S3SigV4Auth
|
|
from botocore.credentials import Credentials
|
|
|
|
self.endpoint = endpoint.rstrip("/")
|
|
self.auth = S3SigV4Auth(Credentials(access, secret), "s3", REGION)
|
|
|
|
def request(
|
|
self, method: str, path: str, params: list[tuple[str, str]] | None = None, body: bytes = b""
|
|
) -> tuple[int, bytes]:
|
|
from botocore.awsrequest import AWSRequest
|
|
|
|
query = "&".join(
|
|
f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}"
|
|
for k, v in (params or [])
|
|
)
|
|
url = self.endpoint + urllib.parse.quote(path, safe="/-_.~") + (f"?{query}" if query else "")
|
|
req = AWSRequest(method=method, url=url, data=body)
|
|
req.headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest()
|
|
self.auth.add_auth(req)
|
|
prepared = req.prepare()
|
|
http = urllib.request.Request(prepared.url, data=body or None, method=method)
|
|
for k, v in prepared.headers.items():
|
|
http.add_header(k, v)
|
|
try:
|
|
with urllib.request.urlopen(http, timeout=60) as resp:
|
|
return resp.status, resp.read()
|
|
except urllib.error.HTTPError as err:
|
|
return err.code, err.read()
|
|
|
|
def put_object(self, bucket: str, key: str, body: bytes) -> None:
|
|
# The first writes to a new bucket can get a retryable 500 while its
|
|
# volumes grow; SDK clients retry those the same way.
|
|
for attempt in range(10):
|
|
status, out = self.request("PUT", f"/{bucket}/{key}", body=body)
|
|
if status < 500:
|
|
break
|
|
time.sleep(0.5 * (attempt + 1))
|
|
if status != 200:
|
|
raise RuntimeError(f"PUT {bucket}/{key!r}: {status} {out[:300]!r}")
|
|
|
|
def delete_object(self, bucket: str, key: str) -> None:
|
|
status, out = self.request("DELETE", f"/{bucket}/{key}")
|
|
if status != 204:
|
|
raise RuntimeError(f"DELETE {bucket}/{key!r}: {status} {out[:300]!r}")
|
|
|
|
def create_bucket(self, bucket: str) -> None:
|
|
status, out = self.request("PUT", f"/{bucket}")
|
|
if status not in (200, 409):
|
|
raise RuntimeError(f"create bucket {bucket}: {status} {out[:300]!r}")
|
|
|
|
def list_page(self, bucket: str, params: list[tuple[str, str]]) -> ET.Element:
|
|
status, out = self.request("GET", f"/{bucket}", params)
|
|
if status != 200:
|
|
raise AssertionError(f"list {params}: HTTP {status} {out[:300]!r}")
|
|
return ET.fromstring(out)
|
|
|
|
|
|
def text(el: ET.Element, name: str) -> str | None:
|
|
child = el.find(S3NS + name)
|
|
return None if child is None else (child.text or "")
|
|
|
|
|
|
@dataclass
|
|
class Page:
|
|
entries: list[tuple[str, str]] # ("key"|"prefix", value) in response order
|
|
truncated: bool
|
|
token: str | None
|
|
next_marker: str | None
|
|
key_count: str | None
|
|
raw: ET.Element
|
|
|
|
|
|
def check_byte_order(values: list[str], what: str) -> None:
|
|
for a, b in zip(values, values[1:]):
|
|
assert a.encode() <= b.encode(), f"{what} out of byte order: {a!r} before {b!r}"
|
|
|
|
|
|
def parse_page(root: ET.Element, url_encoded: bool) -> Page:
|
|
dec = urllib.parse.unquote_plus if url_encoded else (lambda s: s)
|
|
entries: list[tuple[str, str]] = []
|
|
for child in root:
|
|
if child.tag == S3NS + "Contents":
|
|
entries.append(("key", dec(text(child, "Key") or "")))
|
|
elif child.tag == S3NS + "CommonPrefixes":
|
|
entries.append(("prefix", dec(text(child, "Prefix") or "")))
|
|
nm = text(root, "NextMarker")
|
|
# XML carries Contents and CommonPrefixes as two lists, each in byte order;
|
|
# their merged byte order is what pagination has to respect.
|
|
for kind in ("key", "prefix"):
|
|
check_byte_order([v for k, v in entries if k == kind], f"{kind} list")
|
|
entries.sort(key=lambda e: e[1].encode())
|
|
return Page(
|
|
entries,
|
|
text(root, "IsTruncated") == "true",
|
|
text(root, "NextContinuationToken"),
|
|
dec(nm) if nm is not None else None,
|
|
text(root, "KeyCount"),
|
|
root,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------- expectations
|
|
|
|
|
|
def direct_view(keys: list[str], prefix: str, delimiter: str) -> list[tuple[str, str]]:
|
|
"""What a delimiter listing of prefix returns, in byte order."""
|
|
out: dict[str, str] = {}
|
|
for k in keys:
|
|
if not k.startswith(prefix):
|
|
continue
|
|
rest = k[len(prefix) :]
|
|
idx = rest.find(delimiter) if delimiter else -1
|
|
if idx >= 0:
|
|
out[prefix + rest[: idx + len(delimiter)]] = "prefix"
|
|
else:
|
|
out[k] = "key"
|
|
return [(out[k], k) for k in utf8_sorted(out)]
|
|
|
|
|
|
# ---------------------------------------------------------------- reference probes
|
|
|
|
|
|
class Probe:
|
|
def __init__(self, s3: S3, bucket: str, keys: list[str]) -> None:
|
|
self.s3, self.bucket, self.keys = s3, bucket, utf8_sorted(keys)
|
|
|
|
def v2_all(self, base: list[tuple[str, str]], max_keys: int, url: bool = False) -> list[Page]:
|
|
pages: list[Page] = []
|
|
token = None
|
|
while True:
|
|
params = [("list-type", "2"), ("max-keys", str(max_keys)), *base]
|
|
if url:
|
|
params.append(("encoding-type", "url"))
|
|
if token:
|
|
params.append(("continuation-token", token))
|
|
page = parse_page(self.s3.list_page(self.bucket, params), url)
|
|
pages.append(page)
|
|
n = len(page.entries)
|
|
assert n <= max_keys, f"page holds {n} > max-keys {max_keys}"
|
|
assert page.key_count == str(n), f"KeyCount {page.key_count} != {n} entries"
|
|
if not page.truncated:
|
|
assert not page.token, "NextContinuationToken on an untruncated page"
|
|
return pages
|
|
assert page.token, "IsTruncated without NextContinuationToken"
|
|
assert len(pages) < 10000, "pagination does not terminate"
|
|
token = page.token
|
|
|
|
def v1_all(self, base: list[tuple[str, str]], max_keys: int, delimiter: str) -> list[Page]:
|
|
pages: list[Page] = []
|
|
marker = None
|
|
while True:
|
|
params = [("max-keys", str(max_keys)), *base]
|
|
if delimiter:
|
|
params.append(("delimiter", delimiter))
|
|
if marker is not None:
|
|
params.append(("marker", marker))
|
|
page = parse_page(self.s3.list_page(self.bucket, params), False)
|
|
pages.append(page)
|
|
assert len(page.entries) <= max_keys
|
|
if not page.truncated:
|
|
return pages
|
|
if delimiter:
|
|
assert page.next_marker, "truncated delimiter page without NextMarker"
|
|
marker = page.next_marker
|
|
else:
|
|
marker = page.next_marker or page.entries[-1][1]
|
|
assert len(pages) < 10000, "pagination does not terminate"
|
|
|
|
@staticmethod
|
|
def flat(pages: list[Page]) -> list[tuple[str, str]]:
|
|
return [e for p in pages for e in p.entries]
|
|
|
|
def expect_eq(self, got, want, what: str) -> None:
|
|
if got != want:
|
|
gs, ws = set(got), set(want)
|
|
missing = [w for w in want if w not in gs][:5]
|
|
extra = [g for g in got if g not in ws][:5]
|
|
first = next((i for i, (a, b) in enumerate(zip(got, want)) if a != b), min(len(got), len(want)))
|
|
raise AssertionError(
|
|
f"{what}: got {len(got)} want {len(want)}; missing {missing} extra {extra}; "
|
|
f"first difference at {first}: got {got[first:first + 3]} want {want[first:first + 3]}"
|
|
)
|
|
|
|
# Serial pagination (aws-cli, mc, s3kor, s5cmd ls).
|
|
def serial(self, prefix: str, max_keys: int) -> None:
|
|
want = [("key", k) for k in self.keys if k.startswith(prefix)]
|
|
got = self.flat(self.v2_all([("prefix", prefix)], max_keys))
|
|
self.expect_eq(got, want, f"serial prefix={prefix!r} max-keys={max_keys}")
|
|
|
|
# Delimiter view and page boundaries inside it.
|
|
def delimiter(self, prefix: str, delimiter: str, max_keys: int, v1: bool = False) -> None:
|
|
want = direct_view(self.keys, prefix, delimiter)
|
|
if v1:
|
|
got = self.flat(self.v1_all([("prefix", prefix)], max_keys, delimiter))
|
|
else:
|
|
got = self.flat(self.v2_all([("prefix", prefix), ("delimiter", delimiter)], max_keys))
|
|
self.expect_eq(got, want, f"{'v1' if v1 else 'v2'} delimiter={delimiter!r} prefix={prefix!r} max-keys={max_keys}")
|
|
|
|
# Prefix discovery (rclone walk, s7cmd, s4cmd): recurse through CommonPrefixes.
|
|
def walk(self, max_keys: int, v1: bool = False) -> None:
|
|
found: list[str] = []
|
|
queue = [""]
|
|
while queue:
|
|
prefix = queue.pop()
|
|
if v1:
|
|
entries = self.flat(self.v1_all([("prefix", prefix)], max_keys, "/"))
|
|
else:
|
|
entries = self.flat(self.v2_all([("prefix", prefix), ("delimiter", "/")], max_keys))
|
|
for kind, value in entries:
|
|
(queue.append if kind == "prefix" else found.append)(value)
|
|
self.expect_eq(utf8_sorted(found), self.keys, f"{'v1' if v1 else 'v2'} delimiter walk max-keys={max_keys}")
|
|
assert len(found) == len(set(found)), "walk returned a key twice"
|
|
|
|
# Range splitting (S3P, s3-fast-list, Swath): start-after cut points.
|
|
def ranges(self, cuts: list[str], max_keys: int) -> None:
|
|
cuts = sorted(set(cuts), key=lambda c: c.encode())
|
|
bounds = [None, *cuts, None]
|
|
found: list[str] = []
|
|
for lo, hi in zip(bounds, bounds[1:]):
|
|
token = None
|
|
done = False
|
|
while not done:
|
|
params = [("list-type", "2"), ("max-keys", str(max_keys))]
|
|
if token:
|
|
params.append(("continuation-token", token))
|
|
elif lo is not None:
|
|
params.append(("start-after", lo))
|
|
page = parse_page(self.s3.list_page(self.bucket, params), False)
|
|
for _, key in page.entries:
|
|
if lo is not None:
|
|
assert key.encode() > lo.encode(), f"start-after {lo!r} returned {key!r}"
|
|
if hi is not None and key.encode() > hi.encode():
|
|
done = True
|
|
break
|
|
found.append(key)
|
|
if not page.truncated:
|
|
break
|
|
token = page.token
|
|
self.expect_eq(found, self.keys, f"start-after ranges ({len(cuts)} cuts, max-keys={max_keys})")
|
|
|
|
def start_after_with_delimiter(self, prefix: str, start_after: str) -> None:
|
|
want = [
|
|
e
|
|
for e in direct_view(self.keys, prefix, "/")
|
|
if e[1].encode() > start_after.encode()
|
|
]
|
|
got = self.flat(
|
|
self.v2_all([("prefix", prefix), ("delimiter", "/"), ("start-after", start_after)], 1000)
|
|
)
|
|
self.expect_eq(got, want, f"start-after={start_after!r} delimiter=/ prefix={prefix!r}")
|
|
|
|
def token_overrides_start_after(self) -> None:
|
|
first = parse_page(self.s3.list_page(self.bucket, [("list-type", "2"), ("max-keys", "5")]), False)
|
|
assert first.truncated and first.token
|
|
params = [("list-type", "2"), ("max-keys", "5"), ("continuation-token", first.token)]
|
|
plain = parse_page(self.s3.list_page(self.bucket, params), False)
|
|
mixed = parse_page(
|
|
self.s3.list_page(self.bucket, [*params, ("start-after", self.keys[len(self.keys) // 2])]),
|
|
False,
|
|
)
|
|
self.expect_eq(mixed.entries, plain.entries, "continuation-token with a later start-after")
|
|
|
|
def versions(self, prefix: str, delimiter: str, max_keys: int, url: bool) -> None:
|
|
"""ListObjectVersions paging (key-marker/version-id-marker) over unversioned keys."""
|
|
want = direct_view(self.keys, prefix, delimiter) if delimiter else [("key", k) for k in self.keys if k.startswith(prefix)]
|
|
got: list[tuple[str, str]] = []
|
|
key_marker = version_marker = None
|
|
dec = urllib.parse.unquote_plus if url else (lambda v: v)
|
|
for _ in range(10000):
|
|
params = [("versions", ""), ("max-keys", str(max_keys)), ("prefix", prefix)]
|
|
if delimiter:
|
|
params.append(("delimiter", delimiter))
|
|
if url:
|
|
params.append(("encoding-type", "url"))
|
|
if key_marker is not None:
|
|
params += [("key-marker", key_marker), ("version-id-marker", version_marker or "")]
|
|
root = self.s3.list_page(self.bucket, params)
|
|
page: list[tuple[str, str]] = []
|
|
for child in root:
|
|
if child.tag in (S3NS + "Version", S3NS + "DeleteMarker"):
|
|
page.append(("key", dec(text(child, "Key") or "")))
|
|
elif child.tag == S3NS + "CommonPrefixes":
|
|
page.append(("prefix", dec(text(child, "Prefix") or "")))
|
|
assert len(page) <= max_keys, f"{len(page)} entries > max-keys {max_keys}"
|
|
for kind in ("key", "prefix"):
|
|
check_byte_order([v for k, v in page if k == kind], f"version {kind} list")
|
|
got += sorted(page, key=lambda e: e[1].encode())
|
|
if text(root, "IsTruncated") != "true":
|
|
break
|
|
nk = text(root, "NextKeyMarker")
|
|
assert nk, "truncated versions page without NextKeyMarker"
|
|
key_marker, version_marker = dec(nk), text(root, "NextVersionIdMarker")
|
|
self.expect_eq(got, want, f"versions prefix={prefix!r} delimiter={delimiter!r} max-keys={max_keys} url={url}")
|
|
|
|
def url_encoding(self, prefix: str) -> None:
|
|
want = [("key", k) for k in self.keys if k.startswith(prefix)]
|
|
got = self.flat(self.v2_all([("prefix", prefix)], 7, url=True))
|
|
self.expect_eq(got, want, f"encoding-type=url prefix={prefix!r}")
|
|
root = self.s3.list_page(
|
|
self.bucket,
|
|
[("list-type", "2"), ("encoding-type", "url"), ("prefix", prefix), ("delimiter", "/"), ("start-after", prefix + "a b")],
|
|
)
|
|
for name, value in (("Prefix", prefix), ("Delimiter", "/"), ("StartAfter", prefix + "a b")):
|
|
raw = text(root, name)
|
|
assert raw is not None and urllib.parse.unquote_plus(raw) == value, f"encoded {name} {raw!r} != {value!r}"
|
|
assert text(root, "EncodingType") == "url", "EncodingType not echoed"
|
|
|
|
|
|
def deleted_prefixes(s3: S3, bucket: str) -> None:
|
|
"""A prefix whose last key was deleted is no longer a CommonPrefix."""
|
|
s3.create_bucket(bucket)
|
|
keys = ["keep", "gone/a/b", "gone/c", "half/x", "half/y/z"]
|
|
for k in keys:
|
|
s3.put_object(bucket, k, b"x")
|
|
for k in ("gone/a/b", "gone/c", "half/y/z"):
|
|
s3.delete_object(bucket, k)
|
|
p = Probe(s3, bucket, ["keep", "half/x"])
|
|
p.delimiter("", "/", 1000)
|
|
p.delimiter("half/", "/", 1000)
|
|
p.delimiter("gone/", "/", 1000)
|
|
|
|
|
|
def run_probes(
|
|
s3: S3, buckets: dict[str, dict[str, int]], churn_bucket: str, only: re.Pattern | None
|
|
) -> list[tuple[str, str | None]]:
|
|
results: list[tuple[str, str | None]] = []
|
|
|
|
def check(name: str, fn, *a) -> None:
|
|
if only and not only.search(name):
|
|
return
|
|
try:
|
|
fn(*a)
|
|
results.append((name, None))
|
|
except Exception as exc: # noqa: BLE001
|
|
results.append((name, f"{type(exc).__name__}: {exc}"))
|
|
|
|
check(f"reference:{churn_bucket}:deleted-prefixes", deleted_prefixes, s3, churn_bucket)
|
|
for bucket, keymap in buckets.items():
|
|
p = Probe(s3, bucket, list(keymap))
|
|
keys = p.keys
|
|
tag = f"reference:{bucket}"
|
|
for mk in (1, 7, 100, 1000):
|
|
check(f"{tag}:serial:max-keys={mk}", p.serial, "", mk)
|
|
check(f"{tag}:url-encoding", p.url_encoding, "")
|
|
check(f"{tag}:token-overrides-start-after", p.token_overrides_start_after)
|
|
for mk in (2, 1000):
|
|
check(f"{tag}:walk-v2:max-keys={mk}", p.walk, mk)
|
|
check(f"{tag}:walk-v1:max-keys={mk}", p.walk, mk, True)
|
|
prefixes = sorted({k[: k.index("/") + 1] for k in keys if "/" in k})
|
|
scopes = ["", *prefixes[:: max(1, len(prefixes) // 4)]]
|
|
if bucket.endswith("edge"):
|
|
scopes += ["order/", "order/a", "chars/", "unicode/", "markers/", "markers/dir/", "photos/2006/", "o"]
|
|
for prefix in scopes:
|
|
for mk in (1, 3, 1000):
|
|
check(f"{tag}:delimiter-v2:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk)
|
|
check(f"{tag}:delimiter-v1:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk, True)
|
|
for prefix in ("", scopes[-1]):
|
|
for delim in ("", "/"):
|
|
for mk in (2, 1000):
|
|
check(f"{tag}:versions:prefix={prefix}:delimiter={delim}:max-keys={mk}", p.versions, prefix, delim, mk, False)
|
|
check(f"{tag}:versions-url:prefix={prefix}", p.versions, prefix, "/", 3, True)
|
|
for d in ("-", "a", "ou"):
|
|
check(f"{tag}:delimiter={d}", p.delimiter, "", d, 1000)
|
|
check(f"{tag}:serial:prefix-no-slash", p.serial, keys[len(keys) // 3][:4], 5)
|
|
# Cut points that are existing keys, absent keys between keys, prefixes,
|
|
# and a common prefix with and without its trailing delimiter.
|
|
step = max(1, len(keys) // 9)
|
|
existing = keys[step::step]
|
|
absent = [k + "\x7f" for k in existing[::2]] + [k[:-1] for k in existing[1::2] if len(k) > 1]
|
|
dirs = [k[: k.rfind("/") + 1] for k in existing if "/" in k]
|
|
dirs += [d.rstrip("/") for d in dirs]
|
|
for name, cuts in (("existing", existing), ("absent", absent), ("dirs", dirs), ("mixed", existing + absent + dirs)):
|
|
for mk in (3, 1000):
|
|
check(f"{tag}:ranges-{name}:max-keys={mk}", p.ranges, cuts, mk)
|
|
for sa in (keys[len(keys) // 2], prefixes[len(prefixes) // 2], prefixes[len(prefixes) // 2].rstrip("/")):
|
|
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "", sa)
|
|
if bucket.endswith("edge"):
|
|
for sa in ("order/a", "order/a/", "order/a/b", "order/a.b", "order/a0"):
|
|
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "order/", sa)
|
|
check(f"{tag}:url-encoding:chars/", p.url_encoding, "chars/")
|
|
return results
|
|
|
|
|
|
# ---------------------------------------------------------------- tool capsules
|
|
|
|
|
|
@dataclass
|
|
class Case:
|
|
tool: str
|
|
mode: str
|
|
bucket: str
|
|
prefix: str
|
|
status: str = ""
|
|
detail: str = ""
|
|
argv: list[str] = field(default_factory=list)
|
|
|
|
|
|
class ToolRunner:
|
|
def __init__(self, args, study: Path, work: Path, endpoint: str) -> None:
|
|
sys.path.insert(0, str(study / "benchmark" / "src"))
|
|
from benchmark import adapters # noqa: PLC0415
|
|
from benchmark.runtime.command_adapter import CommandAdapterError, Stated # noqa: PLC0415
|
|
|
|
self.adapters, self.refusal, self.stated = adapters, CommandAdapterError, Stated
|
|
self.study, self.work, self.endpoint = study, work, endpoint
|
|
tools = Path(args.tools_dir).resolve()
|
|
self.fixture_dir = work / "fixtures"
|
|
self.path_map = {
|
|
"/usr/local/bin/aws": str(tools / "aws-cli-bin" / "aws"),
|
|
"/usr/bin/mc": str(tools / "bin" / "mc"),
|
|
"/usr/local/bin/pS3": str(tools / "bin" / "pS3"),
|
|
"/usr/local/bin/rclone": str(tools / "bin" / "rclone"),
|
|
"/usr/bin/s3-fast-list": str(tools / "bin" / "s3-fast-list"),
|
|
"/usr/bin/ks-tool": str(tools / "bin" / "ks-tool"),
|
|
"/usr/local/bin/s3kor": str(tools / "bin" / "s3kor"),
|
|
"/usr/local/bin/s3p": str(tools / "bin" / "s3p"),
|
|
"/usr/local/bin/s4cmd": str(tools / "bin" / "s4cmd"),
|
|
"/s5cmd": str(tools / "bin" / "s5cmd"),
|
|
"/usr/local/bin/s7cmd": str(tools / "bin" / "s7cmd"),
|
|
"/opt/java/openjdk/bin/java": str(tools / "swath" / "openjdk" / "bin" / "java"),
|
|
"/opt/swath/swath.jar": str(tools / "swath" / "swath.jar"),
|
|
"/usr/bin/python3": sys.executable,
|
|
"/opt/benchmark/tools/": str(study / "tools") + "/",
|
|
}
|
|
os.environ["PYTHONPATH"] = str(study / "benchmark" / "src")
|
|
self.env = {
|
|
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
|
"AWS_ACCESS_KEY_ID": args.access_key,
|
|
"AWS_SECRET_ACCESS_KEY": args.secret_key,
|
|
"AWS_REGION": REGION,
|
|
"AWS_DEFAULT_REGION": REGION,
|
|
"AWS_EC2_METADATA_DISABLED": "true",
|
|
"TZ": "UTC",
|
|
"LC_ALL": "C.UTF-8",
|
|
"PYTHONPATH": str(study / "benchmark" / "src"),
|
|
}
|
|
|
|
def localize(self, argv, bucket: str) -> list[str]:
|
|
out = []
|
|
path_map = {**self.path_map, "/fixtures/source/": f"{self.fixture_dir / bucket}/"}
|
|
for token in argv:
|
|
for src, dst in path_map.items():
|
|
if token == src or (src.endswith("/") and token.startswith(src)):
|
|
token = dst + token[len(src) :]
|
|
break
|
|
out.append(token)
|
|
return out
|
|
|
|
def adapter_dir(self, tool: str) -> Path:
|
|
return self.study / "tools" / tool / "adapter"
|
|
|
|
def write_fixture_companions(self, bucket: str, keys: list[str]) -> None:
|
|
"""The staged-fixture companions, built the way fixture_bundle.py builds them."""
|
|
import duckdb # noqa: PLC0415
|
|
|
|
out = self.fixture_dir / bucket
|
|
shutil.rmtree(out, ignore_errors=True)
|
|
out.mkdir(parents=True)
|
|
shards = sorted({k.split("/", 1)[0] for k in keys})
|
|
if all(re.fullmatch(r"[A-Za-z0-9._/-]+", s) for s in shards):
|
|
(out / "s5cmd-shards.input").write_text("".join(f"{s}\n" for s in shards))
|
|
parts = out / "parquet"
|
|
parts.mkdir()
|
|
with duckdb.connect() as con:
|
|
con.execute("CREATE TABLE f (key BLOB, row_type VARCHAR)")
|
|
con.executemany("INSERT INTO f VALUES (?, 'OBJECT')", [(k.encode(),) for k in keys])
|
|
con.execute(f"COPY f TO '{parts / 'part-00000.parquet'}' (FORMAT parquet)")
|
|
sys.path.insert(0, str(self.study / "tools" / "s3-fast-list" / "adapter"))
|
|
from fixture_hints import FixtureHintsError, generate_hints # noqa: PLC0415
|
|
|
|
try:
|
|
generate_hints(parts, out / "s3-fast-list-hints.input", segments=4)
|
|
except FixtureHintsError as exc:
|
|
print(f"{bucket}: no s3-fast-list fixture hints: {exc}", flush=True)
|
|
|
|
def inject_endpoint(self, tool: str, argv: list[str], env: dict) -> tuple[list[str], dict]:
|
|
"""Add the endpoint the way the replay-endpoint pilot (run.sh) did, for modes
|
|
the capsule refuses only because the replay server lacks the API."""
|
|
ep = self.endpoint
|
|
if tool == "rclone":
|
|
return [a.replace(f"region={REGION}", f'region={REGION},endpoint="{ep}",force_path_style=true') for a in argv], env
|
|
if tool == "minio-mc":
|
|
return argv, {**env, "MC_HOST_s3": ep, "MC_REGION": REGION}
|
|
if tool == "s3kor":
|
|
return [argv[0], argv[1], "--custom-endpoint-url", ep, *argv[2:]], env
|
|
if tool == "s7cmd":
|
|
return [*argv[:-1], "--target-endpoint-url", ep, "--target-force-path-style", argv[-1]], env
|
|
if tool == "s5cmd":
|
|
return [argv[0], "--endpoint-url", ep, *argv[1:]], env
|
|
return [*argv, "--endpoint-url", ep], env
|
|
|
|
def config_for(self, adapter, case: Case, mode: str, keys: list[str]) -> dict[str, object]:
|
|
config: dict[str, object] = {}
|
|
for name, axis in adapter.modes[mode].axes.items():
|
|
if isinstance(axis, self.stated):
|
|
config[name] = 4
|
|
if adapter.tool == "s5cmd" and mode == "fanout-with-dirs":
|
|
initials = sorted({k[len(case.prefix)] for k in keys if k.startswith(case.prefix) and len(k) > len(case.prefix)})
|
|
if not all(re.fullmatch(r"[A-Za-z0-9._-]", c) for c in initials):
|
|
raise NotApplicable(f"key initials {initials} are outside the capsule's safe shard set")
|
|
config["shard_initials"] = "".join(initials)
|
|
return config
|
|
|
|
def execute(self, adapter, case: Case, mode: str, sink: Path, keys: list[str], artifact_path: str = "") -> tuple[Path | None, dict]:
|
|
signed = adapter.supports_signed
|
|
inject = case.tool in INJECT_ENDPOINT and mode in INJECT_ENDPOINT[case.tool]
|
|
config = self.config_for(adapter, case, mode, keys)
|
|
argv, env = self.adapters.compile_command(
|
|
self.adapter_dir(case.tool),
|
|
case.tool,
|
|
mode=mode,
|
|
bucket=case.bucket,
|
|
region=REGION,
|
|
prefix=case.prefix,
|
|
signed=signed,
|
|
config=config,
|
|
sink_dir=str(sink),
|
|
artifact_path=artifact_path,
|
|
visible_memory_gb=2.0,
|
|
endpoint_url="" if inject else self.endpoint,
|
|
)
|
|
if inject:
|
|
argv, env = self.inject_endpoint(case.tool, list(argv), env)
|
|
case.detail = "endpoint injected; "
|
|
argv = self.localize(argv, case.bucket)
|
|
case.argv = argv
|
|
sink.mkdir(parents=True, exist_ok=True)
|
|
home = sink / "home"
|
|
home.mkdir(exist_ok=True)
|
|
manifest = adapter.modes[mode]
|
|
stdout_path = sink / (manifest.product_file if manifest.product_channel == "stdout" and manifest.product_artifact else "stdout")
|
|
with stdout_path.open("wb") as out, (sink / "stderr").open("wb") as err:
|
|
proc = subprocess.run(
|
|
argv,
|
|
stdout=out,
|
|
stderr=err,
|
|
env={**self.env, **env, "HOME": str(home)},
|
|
cwd=sink,
|
|
timeout=300,
|
|
check=False,
|
|
)
|
|
if proc.returncode != 0:
|
|
tail = (sink / "stderr").read_bytes()[-600:].decode(errors="replace")
|
|
raise RuntimeError(f"exit {proc.returncode}: {tail}")
|
|
product = sink / manifest.product_file if manifest.product_artifact else None
|
|
return product, config
|
|
|
|
def run_case(self, case: Case, expected: dict[str, tuple[int, str]]) -> None:
|
|
adapter = self.adapters.load_adapter(self.adapter_dir(case.tool), case.tool)
|
|
manifest = adapter.modes[case.mode]
|
|
if manifest.purpose_ceiling == "preparation" or not manifest.product_artifact or (case.tool, case.mode) in NOT_A_LISTING:
|
|
case.status = "skip"
|
|
case.detail = f"{manifest.purpose_ceiling} mode, not a compared listing"
|
|
return
|
|
if (case.tool, case.mode) in ROOT_ONLY and case.prefix:
|
|
case.status, case.detail = "skip", "mode lists the bucket root whatever the prefix"
|
|
return
|
|
fixture = self.fixture_dir / case.bucket
|
|
needs = {"fanout-fixture-with-dirs": "s5cmd-shards.input", "list-hinted-fixture": "s3-fast-list-hints.input"}.get(case.mode)
|
|
if needs and not (fixture / needs).exists():
|
|
case.status, case.detail = "skip", f"fixture_bundle.py would not build {needs} for this key set"
|
|
return
|
|
keys = list(expected)
|
|
sink = self.work / "cases" / case.tool / case.mode / (case.bucket + "~" + (case.prefix.replace("/", "_") or "root"))
|
|
shutil.rmtree(sink, ignore_errors=True)
|
|
try:
|
|
artifact_path = ""
|
|
for step in adapter.requires.get(case.mode, ()):
|
|
producer_sink = sink / f"requires-{step.mode}"
|
|
self.execute(adapter, case, step.mode, producer_sink, keys)
|
|
artifact_path = str(producer_sink / adapter.modes[step.mode].artifacts[step.artifact])
|
|
if manifest.inline:
|
|
inline_sink = sink / f"inline-{manifest.inline}"
|
|
self.execute(adapter, case, manifest.inline, inline_sink, keys, artifact_path)
|
|
inline_manifest = adapter.modes[manifest.inline]
|
|
artifact_path = str(inline_sink / next(iter(inline_manifest.artifacts.values())))
|
|
product, config = self.execute(adapter, case, case.mode, sink, keys, artifact_path)
|
|
except NotApplicable as exc:
|
|
case.status, case.detail = "skip", str(exc)
|
|
return
|
|
except self.adapters.AdapterError as exc:
|
|
if "custom endpoint" in str(exc) or "replay endpoint" in str(exc) or "no --prefix" in str(exc) or "requires HeadBucket" in str(exc):
|
|
case.status, case.detail = "skip", f"capsule refuses: {str(exc).split(': ', 2)[-1][:160]}"
|
|
return
|
|
raise
|
|
normalized = sink / "normalized.tsv"
|
|
self.adapters.normalize_to_path(
|
|
self.adapter_dir(case.tool),
|
|
case.tool,
|
|
case.mode,
|
|
case.prefix,
|
|
normalized,
|
|
input_path=None if manifest.product_channel == "dataset" else product,
|
|
dataset=product if manifest.product_channel == "dataset" else None,
|
|
config=config,
|
|
)
|
|
compare(case, normalized.read_bytes(), expected, manifest.fields)
|
|
|
|
|
|
INJECT_ENDPOINT = {
|
|
"aws-cli": {"s3api-v1-text", "s3api-versions-text"},
|
|
"minio-mc": {"versions-json", "find", "find-json"},
|
|
"rclone": {"listv1"},
|
|
"s3kor": {"list-versions"},
|
|
"s4cmd": {"recursive", "shallow", "show-directory", "du"},
|
|
"s5cmd": {"listv1", "allversions"},
|
|
"s7cmd": {"all-versions"},
|
|
}
|
|
"""Capsule modes refused only because the study's replay server serves
|
|
ListObjectsV2 alone; SeaweedFS serves ListObjects, ListObjectVersions and
|
|
HeadBucket too."""
|
|
|
|
NOT_A_LISTING = {
|
|
("rclone", "debug"),
|
|
("rclone", "walk-debug"),
|
|
("s3p", "summarize"),
|
|
("s4cmd", "du"),
|
|
("s4cmd", "show-directory"),
|
|
}
|
|
"""Modes with no per-key listing: rclone's normalizer has no debug modes, summarize
|
|
and du print one aggregate, and `ls -d` prints the directory itself."""
|
|
|
|
|
|
class NotApplicable(Exception):
|
|
pass
|
|
|
|
|
|
ROOT_ONLY = {
|
|
("aws-cli", "s3api-v2-remainder"),
|
|
("s5cmd", "fanout-fixture-with-dirs"),
|
|
("s3-fast-list", "list-hinted-fixture"),
|
|
}
|
|
"""Modes whose capsule lists the root (remainder) or a staged root-scope fixture."""
|
|
|
|
SHALLOW = re.compile(r"shallow|delimiter|rootkeys|remainder|max-depth")
|
|
KEYS_ONLY = re.compile(r"rootkeys|remainder")
|
|
"""Shallow modes whose normalizer keeps only the keys, dropping CommonPrefixes."""
|
|
|
|
|
|
def compare(case: Case, normalized: bytes, expected: dict[str, tuple[int, str]], fields) -> None:
|
|
rows = []
|
|
for line in normalized.split(b"\n"):
|
|
if not line:
|
|
continue
|
|
f = line.split(b"\t")
|
|
rows.append((f[0].decode("utf-8", "surrogateescape"), f[1].decode(), f[2].decode()))
|
|
scope = {k: v for k, v in expected.items() if k.startswith(case.prefix)}
|
|
keys = utf8_sorted(scope)
|
|
shallow = bool(SHALLOW.search(case.mode))
|
|
if shallow:
|
|
view = direct_view(keys, case.prefix, "/")
|
|
want_objects = [k for kind, k in view if kind == "key"]
|
|
allowed_dirs = {k for kind, k in view if kind == "prefix"}
|
|
else:
|
|
want_objects = keys
|
|
allowed_dirs = {k[: i + 1] for k in keys for i, c in enumerate(k) if c == "/" and i >= len(case.prefix)}
|
|
problems: list[str] = []
|
|
objects, dirs = [], []
|
|
for key, size, etag in rows:
|
|
is_dir = key.endswith("/") and size == "-" and etag == "-" and key not in scope
|
|
(dirs if is_dir else objects).append((key, size, etag))
|
|
got_keys = [k for k, _, _ in objects]
|
|
dup = {k for k in got_keys if got_keys.count(k) > 1} if len(got_keys) != len(set(got_keys)) else set()
|
|
if dup:
|
|
problems.append(f"duplicate keys {sorted(dup)[:5]}")
|
|
missing = [k for k in want_objects if k not in set(got_keys)]
|
|
extra = [k for k in set(got_keys) if k not in set(want_objects)]
|
|
if missing:
|
|
problems.append(f"missing {len(missing)}: {missing[:5]}")
|
|
if extra:
|
|
problems.append(f"unexpected {len(extra)}: {sorted(extra)[:5]}")
|
|
bad_dirs = [d for d, _, _ in dirs if d not in allowed_dirs]
|
|
if bad_dirs:
|
|
problems.append(f"unexpected directory rows {bad_dirs[:5]}")
|
|
if shallow and not KEYS_ONLY.search(case.mode) and {d for d, _, _ in dirs} != allowed_dirs:
|
|
problems.append(f"common prefixes {sorted(d for d, _, _ in dirs)[:8]} != {sorted(allowed_dirs)[:8]}")
|
|
for key, size, etag in objects:
|
|
if key not in scope:
|
|
continue
|
|
want_size, want_etag = scope[key]
|
|
if size != "-" and size != str(want_size):
|
|
problems.append(f"{key!r} size {size} != {want_size}")
|
|
if etag != "-" and etag != want_etag:
|
|
problems.append(f"{key!r} etag {etag} != {want_etag}")
|
|
case.status = "fail" if problems else "pass"
|
|
case.detail += "; ".join(problems)[:1500] if problems else f"{len(objects)} objects, {len(dirs)} prefixes"
|
|
|
|
|
|
# ---------------------------------------------------------------- main
|
|
|
|
|
|
def load_expected_failures(path: Path | None) -> dict[str, str]:
|
|
out: dict[str, str] = {}
|
|
if path and path.exists():
|
|
for line in path.read_text().splitlines():
|
|
line = line.strip()
|
|
if line and not line.startswith("#"):
|
|
name, _, reason = line.partition("#")
|
|
out[name.strip()] = reason.strip()
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--endpoint", required=True)
|
|
ap.add_argument("--access-key", required=True)
|
|
ap.add_argument("--secret-key", required=True)
|
|
ap.add_argument("--study", required=True)
|
|
ap.add_argument("--tools-dir", required=True)
|
|
ap.add_argument("--work-dir", required=True)
|
|
ap.add_argument("--tools", default="aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath")
|
|
ap.add_argument("--bucket-prefix", default="study")
|
|
ap.add_argument("--expected-failures", type=Path)
|
|
ap.add_argument("--skip-upload", action="store_true")
|
|
ap.add_argument("--only", default="", help="regex over case names")
|
|
ap.add_argument("--jobs", type=int, default=4)
|
|
args = ap.parse_args()
|
|
|
|
study = Path(args.study).resolve()
|
|
work = Path(args.work_dir).resolve()
|
|
work.mkdir(parents=True, exist_ok=True)
|
|
s3 = S3(args.endpoint, args.access_key, args.secret_key)
|
|
buckets = {f"{args.bucket_prefix}-canary": canary_keys(), f"{args.bucket_prefix}-edge": edge_keys()}
|
|
expected: dict[str, dict[str, tuple[int, str]]] = {}
|
|
for bucket, keymap in buckets.items():
|
|
expected[bucket] = {k: (size, hashlib.md5(body_for(k, size)).hexdigest()) for k, size in keymap.items()}
|
|
if args.skip_upload:
|
|
continue
|
|
s3.create_bucket(bucket)
|
|
start = time.time()
|
|
with concurrent.futures.ThreadPoolExecutor(16) as pool:
|
|
list(pool.map(lambda k: s3.put_object(bucket, k, body_for(k, keymap[k])), keymap))
|
|
print(f"uploaded {len(keymap)} objects to {bucket} in {time.time() - start:.1f}s", flush=True)
|
|
|
|
only = re.compile(args.only) if args.only else None
|
|
results: list[tuple[str, str, str]] = []
|
|
for name, err in run_probes(s3, buckets, f"{args.bucket_prefix}-churn", only):
|
|
results.append((name, "fail" if err else "pass", err or ""))
|
|
|
|
runner = ToolRunner(args, study, work, args.endpoint)
|
|
cases: list[Case] = []
|
|
for bucket in buckets:
|
|
runner.write_fixture_companions(bucket, list(buckets[bucket]))
|
|
scopes = ["", "group-07/"] if bucket.endswith("canary") else ["", "order/", "chars/"]
|
|
for tool in filter(None, args.tools.split(",")):
|
|
adapter = runner.adapters.load_adapter(runner.adapter_dir(tool), tool)
|
|
for mode in adapter.modes:
|
|
for prefix in scopes:
|
|
case = Case(tool, mode, bucket, prefix)
|
|
if only and not only.search(case_name(case)):
|
|
continue
|
|
cases.append(case)
|
|
|
|
def run(case: Case) -> Case:
|
|
start = time.time()
|
|
try:
|
|
runner.run_case(case, expected[case.bucket])
|
|
except Exception as exc: # noqa: BLE001
|
|
case.status = "fail"
|
|
case.detail = f"{type(exc).__name__}: {exc}"[:1500]
|
|
if os.environ.get("STUDY_DEBUG"):
|
|
traceback.print_exc()
|
|
print(f"{case.status:5} {case_name(case)} ({time.time() - start:.1f}s) {case.detail[:300]}", flush=True)
|
|
return case
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
|
|
for case in pool.map(run, cases):
|
|
results.append((case_name(case), case.status, case.detail))
|
|
|
|
xfail = load_expected_failures(args.expected_failures)
|
|
regressions, fixed = [], []
|
|
counts: dict[str, int] = {}
|
|
for name, status, detail in results:
|
|
if status == "fail" and name in xfail:
|
|
status = "xfail"
|
|
elif status == "pass" and name in xfail:
|
|
fixed.append(name)
|
|
elif status == "fail":
|
|
regressions.append((name, detail))
|
|
counts[status] = counts.get(status, 0) + 1
|
|
(work / "results.json").write_text(json.dumps(results, indent=1, ensure_ascii=False))
|
|
print("\n== summary:", ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
|
|
for name in fixed:
|
|
print(f"XPASS (remove from expected failures): {name}")
|
|
for name, detail in regressions:
|
|
print(f"FAIL {name}: {detail}")
|
|
return 1 if regressions or fixed else 0
|
|
|
|
|
|
def case_name(case: Case) -> str:
|
|
return f"{case.tool}:{case.mode}:{case.bucket.rsplit('-', 1)[-1]}:{case.prefix or '/'}"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|