mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
test/s3: run the s3-listing-study listing tools and strategies in CI (#12750)
* test(s3): s3-listing-study integration suite (WIP) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): run the s3-listing-study tools against SeaweedFS (WIP) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): s3-listing-study deleted-prefix probe, fresh data dir per run Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): listing study ports keep clear of every gRPC companion free_ports.py compared only base ports, so a base port could land on another server's base+10000 gRPC listener. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study ports stay below the ephemeral range Base ports of 20000-55000 put both them and their gRPC companions inside 32768-60999, so an outgoing connection made while weed server starts could take a port before the server that was given it binds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study checks the order inside each listing page parse_page sorted Contents and CommonPrefixes together before the probes compared them, so keys returned out of order within one page passed. Check each list's byte order first; the versions pager does the same. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study applies --only before running a probe The filter was applied to the probe results, so a run limited to one tool still sent every reference listing request and the deleted-prefix check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study requires CommonPrefixes from shallow modes The CommonPrefixes comparison ran only when a tool returned at least one prefix, so a delimiter listing of the canary root that came back empty passed. Compare them for every shallow mode except the two whose normalizer keeps keys only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
04c7b12238
commit
23eef59417
9 files changed
+1377
No files matched your search
@@ -0,0 +1,74 @@
|
||||
name: "S3 listing study tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/listing_study/**'
|
||||
- '.github/workflows/s3-listing-study-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/listing_study/**'
|
||||
- '.github/workflows/s3-listing-study-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number || github.ref }}/s3-listing-study-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
listing-study:
|
||||
name: s3-listing-study tools and strategies
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
defaults:
|
||||
run:
|
||||
working-directory: test/s3/listing_study
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Cache pinned listing tools
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: test/s3/listing_study/.work/tools
|
||||
key: listing-study-tools-${{ runner.os }}-${{ hashFiles('test/s3/listing_study/Makefile', 'test/s3/listing_study/install_tools.py') }}
|
||||
|
||||
- name: Build weed and install the study's pinned tools
|
||||
run: make build-weed tools
|
||||
|
||||
- name: Run the listing study against SeaweedFS
|
||||
timeout-minutes: 15
|
||||
run: make test
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
tail -200 .work/weed.log || true
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-listing-study-results
|
||||
path: |
|
||||
test/s3/listing_study/.work/run/results.json
|
||||
test/s3/listing_study/.work/weed.log
|
||||
retention-days: 14
|
||||
@@ -0,0 +1 @@
|
||||
.work
|
||||
@@ -0,0 +1,87 @@
|
||||
# Runs the listing tools and listing strategies of
|
||||
# https://github.com/varveio/s3-listing-study against a local SeaweedFS.
|
||||
#
|
||||
# make test build weed, fetch the study and its pinned tools, start
|
||||
# weed server on free ports, run, stop
|
||||
# make run ENDPOINT=... run against an already running endpoint
|
||||
#
|
||||
# Needs: go, python3, node/npm, cargo, docker (swath image), curl.
|
||||
#
|
||||
# The server runs with -s3.allowBackslashInKey, the AWS-compatible key mode: by
|
||||
# default SeaweedFS rewrites '\' in object keys to '/', and the edge key set
|
||||
# holds a key with a backslash.
|
||||
|
||||
STUDY_REPO ?= https://github.com/varveio/s3-listing-study.git
|
||||
STUDY_REV ?= 666ec620d245e16baaaa0cf53461382f42c35917
|
||||
WORK_DIR ?= $(CURDIR)/.work
|
||||
STUDY_DIR ?= $(WORK_DIR)/s3-listing-study
|
||||
TOOLS_DIR ?= $(WORK_DIR)/tools
|
||||
VENV ?= $(WORK_DIR)/venv
|
||||
WEED_BINARY ?= $(WORK_DIR)/weed
|
||||
TOOLS ?= aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath
|
||||
ACCESS_KEY ?= listing_study_access
|
||||
SECRET_KEY ?= listing_study_secret
|
||||
JOBS ?= 4
|
||||
ONLY ?=
|
||||
|
||||
.PHONY: test run build-weed study venv tools start-server stop-server clean
|
||||
|
||||
test: build-weed tools
|
||||
@$(MAKE) --no-print-directory start-server; status=$$?; \
|
||||
if [ $$status -eq 0 ]; then \
|
||||
$(MAKE) --no-print-directory run ENDPOINT=http://127.0.0.1:$$(cat $(WORK_DIR)/s3.port); status=$$?; \
|
||||
fi; \
|
||||
$(MAKE) --no-print-directory stop-server; exit $$status
|
||||
|
||||
build-weed:
|
||||
mkdir -p $(WORK_DIR)
|
||||
cd ../../../weed && go build -buildvcs=false -o $(WEED_BINARY) .
|
||||
|
||||
study:
|
||||
@if [ "$$(git -C $(STUDY_DIR) rev-parse HEAD 2>/dev/null)" != "$(STUDY_REV)" ]; then \
|
||||
git init -q $(STUDY_DIR) && \
|
||||
git -C $(STUDY_DIR) fetch -q --depth 1 $(STUDY_REPO) $(STUDY_REV) && \
|
||||
git -C $(STUDY_DIR) checkout -q --force FETCH_HEAD; \
|
||||
fi
|
||||
|
||||
venv:
|
||||
@test -x $(VENV)/bin/python || python3 -m venv $(VENV)
|
||||
$(VENV)/bin/pip install -q -r requirements.txt
|
||||
|
||||
tools: study venv
|
||||
$(VENV)/bin/python install_tools.py --study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --tools $(TOOLS)
|
||||
|
||||
start-server:
|
||||
@mkdir -p $(WORK_DIR)
|
||||
@set -e; data=$$(mktemp -d $(WORK_DIR)/data.XXXXXX); ports=$$(python3 free_ports.py 5); \
|
||||
set -- $$ports; \
|
||||
$(WEED_BINARY) server -filer -s3 -ip 127.0.0.1 -ip.bind 127.0.0.1 -dir=$$data \
|
||||
-master.peers=none -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
|
||||
-volume.max=50 -volume.preStopSeconds=1 \
|
||||
-master.port=$$1 -volume.port=$$2 -filer.port=$$3 -s3.port=$$4 -metricsPort=$$5 \
|
||||
-s3.port.iceberg=0 -s3.port.lance=0 -s3.allowBackslashInKey -s3.config=$(CURDIR)/s3.json \
|
||||
> $(WORK_DIR)/weed.log 2>&1 & echo $$! > $(WORK_DIR)/weed.pid; \
|
||||
echo $$4 > $(WORK_DIR)/s3.port; \
|
||||
for i in $$(seq 1 90); do \
|
||||
if curl -sf -o /dev/null http://127.0.0.1:$$3/ && curl -s -o /dev/null http://127.0.0.1:$$4/; then \
|
||||
echo "SeaweedFS S3 ready on port $$4"; exit 0; fi; \
|
||||
kill -0 $$(cat $(WORK_DIR)/weed.pid) || { tail -50 $(WORK_DIR)/weed.log; exit 1; }; \
|
||||
sleep 1; \
|
||||
done; tail -50 $(WORK_DIR)/weed.log; exit 1
|
||||
|
||||
stop-server:
|
||||
@if [ -f $(WORK_DIR)/weed.pid ]; then \
|
||||
pid=$$(cat $(WORK_DIR)/weed.pid); kill $$pid 2>/dev/null || true; \
|
||||
for i in $$(seq 1 20); do kill -0 $$pid 2>/dev/null || break; sleep 0.5; done; \
|
||||
rm -f $(WORK_DIR)/weed.pid; \
|
||||
fi
|
||||
|
||||
run: study
|
||||
$(VENV)/bin/python run_study.py --endpoint $(ENDPOINT) \
|
||||
--access-key $(ACCESS_KEY) --secret-key $(SECRET_KEY) \
|
||||
--study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --work-dir $(WORK_DIR)/run \
|
||||
--tools $(TOOLS) --jobs $(JOBS) --expected-failures expected_failures.txt \
|
||||
$(if $(ONLY),--only '$(ONLY)')
|
||||
|
||||
clean: stop-server
|
||||
rm -rf .work
|
||||
@@ -0,0 +1,29 @@
|
||||
# Cases that fail for reasons outside SeaweedFS. A listed case that passes is
|
||||
# reported as XPASS and fails the run, so this list has to shrink with fixes.
|
||||
# Format: <case name> # reason
|
||||
|
||||
# aws-cli prints "None" for a --query over a page with no Contents (the canary
|
||||
# has no root-level keys); the capsule normalizer reads it as a malformed row.
|
||||
aws-cli:s3api-v2-remainder:canary:/ # capsule normalizer cannot parse aws-cli's "None"
|
||||
|
||||
# The edge bucket holds zero-byte directory markers (markers/dir/, markers/empty/).
|
||||
# These modes list them as directories, not objects: rclone's --files-only and
|
||||
# lsf --files-only drop them, s5cmd types any key ending in "/" as DIR and the
|
||||
# capsule normalizer drops DIR rows from its recursive modes. The reference
|
||||
# probes check that SeaweedFS returns both markers as Contents.
|
||||
rclone:recursive-fastlist:edge:/ # tool hides directory marker objects
|
||||
rclone:recursive-hierarchical:edge:/ # tool hides directory marker objects
|
||||
rclone:recursive-walk:edge:/ # tool hides directory marker objects
|
||||
rclone:listv1:edge:/ # tool hides directory marker objects
|
||||
rclone:lsf:edge:/ # tool hides directory marker objects
|
||||
s5cmd:recursive:edge:/ # tool hides directory marker objects
|
||||
s5cmd:listv1:edge:/ # tool hides directory marker objects
|
||||
s5cmd:allversions:edge:/ # tool hides directory marker objects
|
||||
|
||||
# Deleting the last key under a prefix leaves its filer directory behind, and an
|
||||
# empty directory is still listed as a CommonPrefix until the filer's empty-folder
|
||||
# cleanup removes it (2 minutes by default). This is deliberate: directories made
|
||||
# through mount or the filer stay visible to S3 clients (dirHoldsOnlyHiddenEntries
|
||||
# in weed/s3api/s3api_object_handlers_list.go). AWS derives CommonPrefixes from
|
||||
# keys only, so prefix-discovery tools briefly see the deleted prefix.
|
||||
reference:study-churn:deleted-prefixes # empty directories are listed until cleanup
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Print N free TCP ports whose +10000 gRPC companions are free too, all below
|
||||
the Linux ephemeral range (32768+) that outgoing connections draw from."""
|
||||
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
|
||||
|
||||
def free(port: int) -> bool:
|
||||
with socket.socket() as s:
|
||||
try:
|
||||
s.bind(("127.0.0.1", port))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
picked: list[int] = []
|
||||
taken: list[int] = []
|
||||
while len(picked) < int(sys.argv[1]):
|
||||
port = random.randrange(10000, 22000)
|
||||
pair = (port, port + 10000)
|
||||
if all(abs(p - t) > 20 for p in pair for t in taken) and all(free(p) for p in pair):
|
||||
picked.append(port)
|
||||
taken += pair
|
||||
print(*picked)
|
||||
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install the listing tools pinned by the s3-listing-study checkout.
|
||||
|
||||
Every artifact locator and sha256 comes from the study's own
|
||||
tools/<tool>/build/image.json, so bumping STUDY_REV bumps the tools with it.
|
||||
Tools land under TOOLS_DIR at the paths listed in run_study.PATH_MAP.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ALL_TOOLS = (
|
||||
"aws-cli",
|
||||
"minio-mc",
|
||||
"ps3",
|
||||
"rclone",
|
||||
"s3-fast-list",
|
||||
"s3kor",
|
||||
"s3p",
|
||||
"s4cmd",
|
||||
"s5cmd",
|
||||
"s7cmd",
|
||||
"swath",
|
||||
)
|
||||
|
||||
MC_COMMIT = "d6541ea280b73a834b64d4097e21f2be77676104"
|
||||
"""github.com/minio/mc tag RELEASE.2025-08-13T08-35-41Z."""
|
||||
|
||||
|
||||
def fetch(url: str, sha256: str, dest: Path) -> Path:
|
||||
if dest.exists() and hashlib.sha256(dest.read_bytes()).hexdigest() == sha256:
|
||||
return dest
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "curl/8"})
|
||||
with urllib.request.urlopen(request, timeout=300) as response, tmp.open("wb") as out:
|
||||
shutil.copyfileobj(response, out)
|
||||
digest = hashlib.sha256(tmp.read_bytes()).hexdigest()
|
||||
if digest != sha256:
|
||||
tmp.unlink()
|
||||
raise SystemExit(f"{url}: sha256 {digest} != pinned {sha256}")
|
||||
tmp.rename(dest)
|
||||
return dest
|
||||
|
||||
|
||||
def executable(path: Path) -> None:
|
||||
path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
def wrapper(path: Path, body: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(f"#!/bin/sh\n{body}\n")
|
||||
executable(path)
|
||||
|
||||
|
||||
def tar_member(archive: Path, member: str, dest: Path) -> None:
|
||||
with tarfile.open(archive) as tar:
|
||||
source = tar.extractfile(member)
|
||||
if source is None:
|
||||
raise SystemExit(f"{archive}: no member {member}")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(source.read())
|
||||
executable(dest)
|
||||
|
||||
|
||||
def install(tool: str, study: Path, tools_dir: Path) -> None:
|
||||
image = json.loads((study / "tools" / tool / "build" / "image.json").read_text())
|
||||
artifact = image["tool_artifact"]
|
||||
url, sha = artifact["locator"], artifact["sha256"]
|
||||
dl = tools_dir / "downloads"
|
||||
bin_dir = tools_dir / "bin"
|
||||
bin_dir.mkdir(parents=True, exist_ok=True)
|
||||
name = url.rsplit("/", 1)[-1]
|
||||
stamp = tools_dir / "installed" / tool
|
||||
if stamp.exists() and stamp.read_text() == sha:
|
||||
print(f"{tool} {image.get('tool_version', '')} already installed")
|
||||
return
|
||||
|
||||
if tool == "aws-cli":
|
||||
archive = fetch(url, sha, dl / name)
|
||||
work = tools_dir / "aws-cli-src"
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
zf.extractall(work)
|
||||
for rel in ("aws/install", "aws/dist/aws", "aws/dist/aws_completer"):
|
||||
executable(work / rel)
|
||||
subprocess.run(
|
||||
[
|
||||
str(work / "aws" / "install"),
|
||||
"--install-dir",
|
||||
str(tools_dir / "aws-cli"),
|
||||
"--bin-dir",
|
||||
str(tools_dir / "aws-cli-bin"),
|
||||
"--update",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
shutil.rmtree(work)
|
||||
elif tool == "minio-mc":
|
||||
# dl.min.io answers 410 Gone for every archived mc release, so build the
|
||||
# same release tag from source instead.
|
||||
if image["tool_version"] != "RELEASE.2025-08-13T08-35-41Z":
|
||||
raise SystemExit(f"minio-mc: no source commit pinned for {image['tool_version']}")
|
||||
subprocess.run(
|
||||
["go", "install", f"github.com/minio/mc@{MC_COMMIT}"],
|
||||
env={**os.environ, "GOBIN": str(bin_dir)},
|
||||
check=True,
|
||||
)
|
||||
elif tool == "ps3":
|
||||
shutil.copyfile(fetch(url, sha, dl / name), bin_dir / "pS3")
|
||||
executable(bin_dir / "pS3")
|
||||
elif tool == "rclone":
|
||||
archive = fetch(url, sha, dl / name)
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
member = f"{name.removesuffix('.zip')}/rclone"
|
||||
(bin_dir / "rclone").write_bytes(zf.read(member))
|
||||
executable(bin_dir / "rclone")
|
||||
elif tool == "s3kor":
|
||||
tar_member(fetch(url, sha, dl / name), "s3kor", bin_dir / "s3kor")
|
||||
elif tool == "s5cmd":
|
||||
tar_member(fetch(url, sha, dl / name), "s5cmd", bin_dir / "s5cmd")
|
||||
elif tool == "s7cmd":
|
||||
tar_member(fetch(url, sha, dl / name), "s7cmd", bin_dir / "s7cmd")
|
||||
elif tool == "s3p":
|
||||
work = tools_dir / "s3p"
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
for f in ("package.json", "package-lock.json"):
|
||||
shutil.copyfile(study / "tools/s3p/build" / f, work / f)
|
||||
subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], cwd=work, check=True)
|
||||
wrapper(bin_dir / "s3p", f'exec node "{work}/node_modules/s3p/bin/s3p" "$@"')
|
||||
elif tool == "s4cmd":
|
||||
archive = fetch(url, sha, dl / name)
|
||||
venv = tools_dir / "s4cmd-venv"
|
||||
subprocess.run([sys.executable, "-m", "venv", "--clear", str(venv)], check=True)
|
||||
subprocess.run(
|
||||
[
|
||||
str(venv / "bin" / "python"),
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--quiet",
|
||||
"--require-hashes",
|
||||
"--no-deps",
|
||||
"-r",
|
||||
str(study / "tools/s4cmd/build/requirements.txt"),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
script = tools_dir / "s4cmd" / "s4cmd.py"
|
||||
with tarfile.open(archive) as tar:
|
||||
source = tar.extractfile("s4cmd-2.1.0/s4cmd.py")
|
||||
assert source is not None
|
||||
script.parent.mkdir(parents=True, exist_ok=True)
|
||||
script.write_bytes(source.read())
|
||||
wrapper(bin_dir / "s4cmd", f'exec "{venv}/bin/python" "{script}" "$@"')
|
||||
elif tool == "s3-fast-list":
|
||||
archive = fetch(url, sha, dl / name)
|
||||
work = tools_dir / "s3-fast-list-src"
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
work.mkdir(parents=True)
|
||||
subprocess.run(["tar", "xzf", str(archive), "--strip-components=1", "-C", str(work)], check=True)
|
||||
shutil.copyfile(study / "tools/s3-fast-list/build/Cargo.lock", work / "Cargo.lock")
|
||||
subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=work, check=True)
|
||||
for exe in ("s3-fast-list", "ks-tool"):
|
||||
shutil.copyfile(work / "target/release" / exe, bin_dir / exe)
|
||||
executable(bin_dir / exe)
|
||||
elif tool == "swath":
|
||||
ref = f"{url}@sha256:{sha}"
|
||||
subprocess.run(["docker", "pull", "--quiet", ref], check=True, stdout=subprocess.DEVNULL)
|
||||
cid = subprocess.run(
|
||||
["docker", "create", ref], check=True, capture_output=True, text=True
|
||||
).stdout.strip()
|
||||
try:
|
||||
dest = tools_dir / "swath"
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
dest.mkdir(parents=True)
|
||||
subprocess.run(["docker", "cp", f"{cid}:/opt/java/openjdk", str(dest / "openjdk")], check=True)
|
||||
subprocess.run(["docker", "cp", f"{cid}:/opt/swath/swath.jar", str(dest / "swath.jar")], check=True)
|
||||
finally:
|
||||
subprocess.run(["docker", "rm", cid], check=True, stdout=subprocess.DEVNULL)
|
||||
else:
|
||||
raise SystemExit(f"unknown tool {tool}")
|
||||
stamp.parent.mkdir(parents=True, exist_ok=True)
|
||||
stamp.write_text(sha)
|
||||
print(f"installed {tool} {image.get('tool_version', '')}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--study", required=True, type=Path)
|
||||
parser.add_argument("--tools-dir", required=True, type=Path)
|
||||
parser.add_argument("--tools", default=",".join(ALL_TOOLS))
|
||||
args = parser.parse_args()
|
||||
for tool in filter(None, args.tools.split(",")):
|
||||
install(tool, args.study.resolve(), args.tools_dir.resolve())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.umask(0o022)
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
# Versions the pinned s3-listing-study uv.lock resolves for the capsule normalizers.
|
||||
duckdb==1.5.5
|
||||
ijson==3.5.1
|
||||
pyyaml==6.0.3
|
||||
zstandard==0.25.0
|
||||
botocore==1.44.5
|
||||
@@ -0,0 +1,922 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the s3-listing-study tool capsules and ListObjects reference probes against SeaweedFS.
|
||||
|
||||
Tool cases: every mode of every capsule under the study's tools/ is compiled by
|
||||
the capsule's own command.py (benchmark.adapters.compile_command, with the
|
||||
custom-endpoint wiring the study's replay-endpoint pilot established), run
|
||||
against SeaweedFS with the pinned tool build, normalized by the capsule's own
|
||||
normalize.py, and the normalized listing is compared exactly against the keys
|
||||
that were written.
|
||||
|
||||
Reference probes: raw, signed ListObjectsV2/ListObjects requests implementing
|
||||
the strategies the study classifies tools by (serial pagination, delimiter
|
||||
prefix discovery, start-after range splitting) plus the request semantics in
|
||||
docs/s3-reference.md, checked against the byte-ordered key set.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
REGION = "us-east-1"
|
||||
S3NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
||||
|
||||
|
||||
def canary_keys() -> dict[str, int]:
|
||||
"""benchmark/fixtures/replay-canary/generate.sql: 2,048 keys in 16 prefixes."""
|
||||
return {f"group-{i // 128:02d}/object-{i:06d}.dat": 1024 + i for i in range(2048)}
|
||||
|
||||
|
||||
EDGE_KEYS = [
|
||||
# docs/smoke-bucket.md: a root key beside the prefixes, for delimiter checks.
|
||||
"index.html",
|
||||
# docs/s3-reference.md examples.
|
||||
"test_file(3).png",
|
||||
"photos/2006/February/sample.jpg",
|
||||
"photos/2006/January/sample.jpg",
|
||||
"photos/2007/sample.jpg",
|
||||
"notes/summer/july",
|
||||
"notes/summer/august",
|
||||
"notes/winter",
|
||||
# Byte order around the '/' (0x2f) delimiter.
|
||||
"order/a",
|
||||
"order/a b",
|
||||
"order/a!b",
|
||||
"order/a-b",
|
||||
"order/a.b",
|
||||
"order/a/b",
|
||||
"order/a/c/d",
|
||||
"order/a0",
|
||||
"order/aa",
|
||||
"order/a~",
|
||||
"order/A",
|
||||
"order/b",
|
||||
# Zero-byte directory markers.
|
||||
"markers/dir/",
|
||||
"markers/dir/child",
|
||||
"markers/empty/",
|
||||
# Characters that need XML or URL escaping.
|
||||
"chars/space key",
|
||||
"chars/plus+key",
|
||||
"chars/percent%20key",
|
||||
"chars/amp&key",
|
||||
"chars/lt<gt>key",
|
||||
'chars/quote"key',
|
||||
"chars/apos'key",
|
||||
"chars/hash#key",
|
||||
"chars/question?key",
|
||||
"chars/equals=key",
|
||||
"chars/semicolon;key",
|
||||
"chars/colon:key",
|
||||
"chars/at@key",
|
||||
"chars/comma,key",
|
||||
"chars/star*key",
|
||||
"chars/bang!key",
|
||||
"chars/dollar$key",
|
||||
"chars/caret^key",
|
||||
"chars/backslash\\key",
|
||||
"chars/brace{key}",
|
||||
"chars/pipe|key",
|
||||
"chars/backtick`key",
|
||||
"chars/tilde~key",
|
||||
# UTF-8 byte order, which differs from UTF-16 order above U+FFFF.
|
||||
"unicode/café-nfc",
|
||||
"unicode/café-nfd",
|
||||
"unicode/日本語.txt",
|
||||
"unicode/。-halfwidth",
|
||||
"unicode/\U0001f600-emoji",
|
||||
"unicode/z-ascii",
|
||||
"deep/a/b/c/d/e/f/g/h/i/j/k.txt",
|
||||
"long/" + "/".join(["y" * 200] * 4),
|
||||
]
|
||||
|
||||
|
||||
def edge_keys() -> dict[str, int]:
|
||||
return {k: (0 if k.endswith("/") else 100 + i) for i, k in enumerate(EDGE_KEYS)}
|
||||
|
||||
|
||||
def body_for(key: str, size: int) -> bytes:
|
||||
seed = hashlib.sha256(key.encode()).digest()
|
||||
return (seed * (size // len(seed) + 1))[:size]
|
||||
|
||||
|
||||
def utf8_sorted(keys) -> list[str]:
|
||||
return sorted(keys, key=lambda k: k.encode("utf-8"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- signed HTTP
|
||||
|
||||
|
||||
class S3:
|
||||
def __init__(self, endpoint: str, access: str, secret: str) -> None:
|
||||
from botocore.auth import S3SigV4Auth
|
||||
from botocore.credentials import Credentials
|
||||
|
||||
self.endpoint = endpoint.rstrip("/")
|
||||
self.auth = S3SigV4Auth(Credentials(access, secret), "s3", REGION)
|
||||
|
||||
def request(
|
||||
self, method: str, path: str, params: list[tuple[str, str]] | None = None, body: bytes = b""
|
||||
) -> tuple[int, bytes]:
|
||||
from botocore.awsrequest import AWSRequest
|
||||
|
||||
query = "&".join(
|
||||
f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}"
|
||||
for k, v in (params or [])
|
||||
)
|
||||
url = self.endpoint + urllib.parse.quote(path, safe="/-_.~") + (f"?{query}" if query else "")
|
||||
req = AWSRequest(method=method, url=url, data=body)
|
||||
req.headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest()
|
||||
self.auth.add_auth(req)
|
||||
prepared = req.prepare()
|
||||
http = urllib.request.Request(prepared.url, data=body or None, method=method)
|
||||
for k, v in prepared.headers.items():
|
||||
http.add_header(k, v)
|
||||
try:
|
||||
with urllib.request.urlopen(http, timeout=60) as resp:
|
||||
return resp.status, resp.read()
|
||||
except urllib.error.HTTPError as err:
|
||||
return err.code, err.read()
|
||||
|
||||
def put_object(self, bucket: str, key: str, body: bytes) -> None:
|
||||
# The first writes to a new bucket can get a retryable 500 while its
|
||||
# volumes grow; SDK clients retry those the same way.
|
||||
for attempt in range(10):
|
||||
status, out = self.request("PUT", f"/{bucket}/{key}", body=body)
|
||||
if status < 500:
|
||||
break
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
if status != 200:
|
||||
raise RuntimeError(f"PUT {bucket}/{key!r}: {status} {out[:300]!r}")
|
||||
|
||||
def delete_object(self, bucket: str, key: str) -> None:
|
||||
status, out = self.request("DELETE", f"/{bucket}/{key}")
|
||||
if status != 204:
|
||||
raise RuntimeError(f"DELETE {bucket}/{key!r}: {status} {out[:300]!r}")
|
||||
|
||||
def create_bucket(self, bucket: str) -> None:
|
||||
status, out = self.request("PUT", f"/{bucket}")
|
||||
if status not in (200, 409):
|
||||
raise RuntimeError(f"create bucket {bucket}: {status} {out[:300]!r}")
|
||||
|
||||
def list_page(self, bucket: str, params: list[tuple[str, str]]) -> ET.Element:
|
||||
status, out = self.request("GET", f"/{bucket}", params)
|
||||
if status != 200:
|
||||
raise AssertionError(f"list {params}: HTTP {status} {out[:300]!r}")
|
||||
return ET.fromstring(out)
|
||||
|
||||
|
||||
def text(el: ET.Element, name: str) -> str | None:
|
||||
child = el.find(S3NS + name)
|
||||
return None if child is None else (child.text or "")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Page:
|
||||
entries: list[tuple[str, str]] # ("key"|"prefix", value) in response order
|
||||
truncated: bool
|
||||
token: str | None
|
||||
next_marker: str | None
|
||||
key_count: str | None
|
||||
raw: ET.Element
|
||||
|
||||
|
||||
def check_byte_order(values: list[str], what: str) -> None:
|
||||
for a, b in zip(values, values[1:]):
|
||||
assert a.encode() <= b.encode(), f"{what} out of byte order: {a!r} before {b!r}"
|
||||
|
||||
|
||||
def parse_page(root: ET.Element, url_encoded: bool) -> Page:
|
||||
dec = urllib.parse.unquote_plus if url_encoded else (lambda s: s)
|
||||
entries: list[tuple[str, str]] = []
|
||||
for child in root:
|
||||
if child.tag == S3NS + "Contents":
|
||||
entries.append(("key", dec(text(child, "Key") or "")))
|
||||
elif child.tag == S3NS + "CommonPrefixes":
|
||||
entries.append(("prefix", dec(text(child, "Prefix") or "")))
|
||||
nm = text(root, "NextMarker")
|
||||
# XML carries Contents and CommonPrefixes as two lists, each in byte order;
|
||||
# their merged byte order is what pagination has to respect.
|
||||
for kind in ("key", "prefix"):
|
||||
check_byte_order([v for k, v in entries if k == kind], f"{kind} list")
|
||||
entries.sort(key=lambda e: e[1].encode())
|
||||
return Page(
|
||||
entries,
|
||||
text(root, "IsTruncated") == "true",
|
||||
text(root, "NextContinuationToken"),
|
||||
dec(nm) if nm is not None else None,
|
||||
text(root, "KeyCount"),
|
||||
root,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- expectations
|
||||
|
||||
|
||||
def direct_view(keys: list[str], prefix: str, delimiter: str) -> list[tuple[str, str]]:
|
||||
"""What a delimiter listing of prefix returns, in byte order."""
|
||||
out: dict[str, str] = {}
|
||||
for k in keys:
|
||||
if not k.startswith(prefix):
|
||||
continue
|
||||
rest = k[len(prefix) :]
|
||||
idx = rest.find(delimiter) if delimiter else -1
|
||||
if idx >= 0:
|
||||
out[prefix + rest[: idx + len(delimiter)]] = "prefix"
|
||||
else:
|
||||
out[k] = "key"
|
||||
return [(out[k], k) for k in utf8_sorted(out)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- reference probes
|
||||
|
||||
|
||||
class Probe:
|
||||
def __init__(self, s3: S3, bucket: str, keys: list[str]) -> None:
|
||||
self.s3, self.bucket, self.keys = s3, bucket, utf8_sorted(keys)
|
||||
|
||||
def v2_all(self, base: list[tuple[str, str]], max_keys: int, url: bool = False) -> list[Page]:
|
||||
pages: list[Page] = []
|
||||
token = None
|
||||
while True:
|
||||
params = [("list-type", "2"), ("max-keys", str(max_keys)), *base]
|
||||
if url:
|
||||
params.append(("encoding-type", "url"))
|
||||
if token:
|
||||
params.append(("continuation-token", token))
|
||||
page = parse_page(self.s3.list_page(self.bucket, params), url)
|
||||
pages.append(page)
|
||||
n = len(page.entries)
|
||||
assert n <= max_keys, f"page holds {n} > max-keys {max_keys}"
|
||||
assert page.key_count == str(n), f"KeyCount {page.key_count} != {n} entries"
|
||||
if not page.truncated:
|
||||
assert not page.token, "NextContinuationToken on an untruncated page"
|
||||
return pages
|
||||
assert page.token, "IsTruncated without NextContinuationToken"
|
||||
assert len(pages) < 10000, "pagination does not terminate"
|
||||
token = page.token
|
||||
|
||||
def v1_all(self, base: list[tuple[str, str]], max_keys: int, delimiter: str) -> list[Page]:
|
||||
pages: list[Page] = []
|
||||
marker = None
|
||||
while True:
|
||||
params = [("max-keys", str(max_keys)), *base]
|
||||
if delimiter:
|
||||
params.append(("delimiter", delimiter))
|
||||
if marker is not None:
|
||||
params.append(("marker", marker))
|
||||
page = parse_page(self.s3.list_page(self.bucket, params), False)
|
||||
pages.append(page)
|
||||
assert len(page.entries) <= max_keys
|
||||
if not page.truncated:
|
||||
return pages
|
||||
if delimiter:
|
||||
assert page.next_marker, "truncated delimiter page without NextMarker"
|
||||
marker = page.next_marker
|
||||
else:
|
||||
marker = page.next_marker or page.entries[-1][1]
|
||||
assert len(pages) < 10000, "pagination does not terminate"
|
||||
|
||||
@staticmethod
|
||||
def flat(pages: list[Page]) -> list[tuple[str, str]]:
|
||||
return [e for p in pages for e in p.entries]
|
||||
|
||||
def expect_eq(self, got, want, what: str) -> None:
|
||||
if got != want:
|
||||
gs, ws = set(got), set(want)
|
||||
missing = [w for w in want if w not in gs][:5]
|
||||
extra = [g for g in got if g not in ws][:5]
|
||||
first = next((i for i, (a, b) in enumerate(zip(got, want)) if a != b), min(len(got), len(want)))
|
||||
raise AssertionError(
|
||||
f"{what}: got {len(got)} want {len(want)}; missing {missing} extra {extra}; "
|
||||
f"first difference at {first}: got {got[first:first + 3]} want {want[first:first + 3]}"
|
||||
)
|
||||
|
||||
# Serial pagination (aws-cli, mc, s3kor, s5cmd ls).
|
||||
def serial(self, prefix: str, max_keys: int) -> None:
|
||||
want = [("key", k) for k in self.keys if k.startswith(prefix)]
|
||||
got = self.flat(self.v2_all([("prefix", prefix)], max_keys))
|
||||
self.expect_eq(got, want, f"serial prefix={prefix!r} max-keys={max_keys}")
|
||||
|
||||
# Delimiter view and page boundaries inside it.
|
||||
def delimiter(self, prefix: str, delimiter: str, max_keys: int, v1: bool = False) -> None:
|
||||
want = direct_view(self.keys, prefix, delimiter)
|
||||
if v1:
|
||||
got = self.flat(self.v1_all([("prefix", prefix)], max_keys, delimiter))
|
||||
else:
|
||||
got = self.flat(self.v2_all([("prefix", prefix), ("delimiter", delimiter)], max_keys))
|
||||
self.expect_eq(got, want, f"{'v1' if v1 else 'v2'} delimiter={delimiter!r} prefix={prefix!r} max-keys={max_keys}")
|
||||
|
||||
# Prefix discovery (rclone walk, s7cmd, s4cmd): recurse through CommonPrefixes.
|
||||
def walk(self, max_keys: int, v1: bool = False) -> None:
|
||||
found: list[str] = []
|
||||
queue = [""]
|
||||
while queue:
|
||||
prefix = queue.pop()
|
||||
if v1:
|
||||
entries = self.flat(self.v1_all([("prefix", prefix)], max_keys, "/"))
|
||||
else:
|
||||
entries = self.flat(self.v2_all([("prefix", prefix), ("delimiter", "/")], max_keys))
|
||||
for kind, value in entries:
|
||||
(queue.append if kind == "prefix" else found.append)(value)
|
||||
self.expect_eq(utf8_sorted(found), self.keys, f"{'v1' if v1 else 'v2'} delimiter walk max-keys={max_keys}")
|
||||
assert len(found) == len(set(found)), "walk returned a key twice"
|
||||
|
||||
# Range splitting (S3P, s3-fast-list, Swath): start-after cut points.
|
||||
def ranges(self, cuts: list[str], max_keys: int) -> None:
|
||||
cuts = sorted(set(cuts), key=lambda c: c.encode())
|
||||
bounds = [None, *cuts, None]
|
||||
found: list[str] = []
|
||||
for lo, hi in zip(bounds, bounds[1:]):
|
||||
token = None
|
||||
done = False
|
||||
while not done:
|
||||
params = [("list-type", "2"), ("max-keys", str(max_keys))]
|
||||
if token:
|
||||
params.append(("continuation-token", token))
|
||||
elif lo is not None:
|
||||
params.append(("start-after", lo))
|
||||
page = parse_page(self.s3.list_page(self.bucket, params), False)
|
||||
for _, key in page.entries:
|
||||
if lo is not None:
|
||||
assert key.encode() > lo.encode(), f"start-after {lo!r} returned {key!r}"
|
||||
if hi is not None and key.encode() > hi.encode():
|
||||
done = True
|
||||
break
|
||||
found.append(key)
|
||||
if not page.truncated:
|
||||
break
|
||||
token = page.token
|
||||
self.expect_eq(found, self.keys, f"start-after ranges ({len(cuts)} cuts, max-keys={max_keys})")
|
||||
|
||||
def start_after_with_delimiter(self, prefix: str, start_after: str) -> None:
|
||||
want = [
|
||||
e
|
||||
for e in direct_view(self.keys, prefix, "/")
|
||||
if e[1].encode() > start_after.encode()
|
||||
]
|
||||
got = self.flat(
|
||||
self.v2_all([("prefix", prefix), ("delimiter", "/"), ("start-after", start_after)], 1000)
|
||||
)
|
||||
self.expect_eq(got, want, f"start-after={start_after!r} delimiter=/ prefix={prefix!r}")
|
||||
|
||||
def token_overrides_start_after(self) -> None:
|
||||
first = parse_page(self.s3.list_page(self.bucket, [("list-type", "2"), ("max-keys", "5")]), False)
|
||||
assert first.truncated and first.token
|
||||
params = [("list-type", "2"), ("max-keys", "5"), ("continuation-token", first.token)]
|
||||
plain = parse_page(self.s3.list_page(self.bucket, params), False)
|
||||
mixed = parse_page(
|
||||
self.s3.list_page(self.bucket, [*params, ("start-after", self.keys[len(self.keys) // 2])]),
|
||||
False,
|
||||
)
|
||||
self.expect_eq(mixed.entries, plain.entries, "continuation-token with a later start-after")
|
||||
|
||||
def versions(self, prefix: str, delimiter: str, max_keys: int, url: bool) -> None:
|
||||
"""ListObjectVersions paging (key-marker/version-id-marker) over unversioned keys."""
|
||||
want = direct_view(self.keys, prefix, delimiter) if delimiter else [("key", k) for k in self.keys if k.startswith(prefix)]
|
||||
got: list[tuple[str, str]] = []
|
||||
key_marker = version_marker = None
|
||||
dec = urllib.parse.unquote_plus if url else (lambda v: v)
|
||||
for _ in range(10000):
|
||||
params = [("versions", ""), ("max-keys", str(max_keys)), ("prefix", prefix)]
|
||||
if delimiter:
|
||||
params.append(("delimiter", delimiter))
|
||||
if url:
|
||||
params.append(("encoding-type", "url"))
|
||||
if key_marker is not None:
|
||||
params += [("key-marker", key_marker), ("version-id-marker", version_marker or "")]
|
||||
root = self.s3.list_page(self.bucket, params)
|
||||
page: list[tuple[str, str]] = []
|
||||
for child in root:
|
||||
if child.tag in (S3NS + "Version", S3NS + "DeleteMarker"):
|
||||
page.append(("key", dec(text(child, "Key") or "")))
|
||||
elif child.tag == S3NS + "CommonPrefixes":
|
||||
page.append(("prefix", dec(text(child, "Prefix") or "")))
|
||||
assert len(page) <= max_keys, f"{len(page)} entries > max-keys {max_keys}"
|
||||
for kind in ("key", "prefix"):
|
||||
check_byte_order([v for k, v in page if k == kind], f"version {kind} list")
|
||||
got += sorted(page, key=lambda e: e[1].encode())
|
||||
if text(root, "IsTruncated") != "true":
|
||||
break
|
||||
nk = text(root, "NextKeyMarker")
|
||||
assert nk, "truncated versions page without NextKeyMarker"
|
||||
key_marker, version_marker = dec(nk), text(root, "NextVersionIdMarker")
|
||||
self.expect_eq(got, want, f"versions prefix={prefix!r} delimiter={delimiter!r} max-keys={max_keys} url={url}")
|
||||
|
||||
def url_encoding(self, prefix: str) -> None:
|
||||
want = [("key", k) for k in self.keys if k.startswith(prefix)]
|
||||
got = self.flat(self.v2_all([("prefix", prefix)], 7, url=True))
|
||||
self.expect_eq(got, want, f"encoding-type=url prefix={prefix!r}")
|
||||
root = self.s3.list_page(
|
||||
self.bucket,
|
||||
[("list-type", "2"), ("encoding-type", "url"), ("prefix", prefix), ("delimiter", "/"), ("start-after", prefix + "a b")],
|
||||
)
|
||||
for name, value in (("Prefix", prefix), ("Delimiter", "/"), ("StartAfter", prefix + "a b")):
|
||||
raw = text(root, name)
|
||||
assert raw is not None and urllib.parse.unquote_plus(raw) == value, f"encoded {name} {raw!r} != {value!r}"
|
||||
assert text(root, "EncodingType") == "url", "EncodingType not echoed"
|
||||
|
||||
|
||||
def deleted_prefixes(s3: S3, bucket: str) -> None:
|
||||
"""A prefix whose last key was deleted is no longer a CommonPrefix."""
|
||||
s3.create_bucket(bucket)
|
||||
keys = ["keep", "gone/a/b", "gone/c", "half/x", "half/y/z"]
|
||||
for k in keys:
|
||||
s3.put_object(bucket, k, b"x")
|
||||
for k in ("gone/a/b", "gone/c", "half/y/z"):
|
||||
s3.delete_object(bucket, k)
|
||||
p = Probe(s3, bucket, ["keep", "half/x"])
|
||||
p.delimiter("", "/", 1000)
|
||||
p.delimiter("half/", "/", 1000)
|
||||
p.delimiter("gone/", "/", 1000)
|
||||
|
||||
|
||||
def run_probes(
|
||||
s3: S3, buckets: dict[str, dict[str, int]], churn_bucket: str, only: re.Pattern | None
|
||||
) -> list[tuple[str, str | None]]:
|
||||
results: list[tuple[str, str | None]] = []
|
||||
|
||||
def check(name: str, fn, *a) -> None:
|
||||
if only and not only.search(name):
|
||||
return
|
||||
try:
|
||||
fn(*a)
|
||||
results.append((name, None))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
results.append((name, f"{type(exc).__name__}: {exc}"))
|
||||
|
||||
check(f"reference:{churn_bucket}:deleted-prefixes", deleted_prefixes, s3, churn_bucket)
|
||||
for bucket, keymap in buckets.items():
|
||||
p = Probe(s3, bucket, list(keymap))
|
||||
keys = p.keys
|
||||
tag = f"reference:{bucket}"
|
||||
for mk in (1, 7, 100, 1000):
|
||||
check(f"{tag}:serial:max-keys={mk}", p.serial, "", mk)
|
||||
check(f"{tag}:url-encoding", p.url_encoding, "")
|
||||
check(f"{tag}:token-overrides-start-after", p.token_overrides_start_after)
|
||||
for mk in (2, 1000):
|
||||
check(f"{tag}:walk-v2:max-keys={mk}", p.walk, mk)
|
||||
check(f"{tag}:walk-v1:max-keys={mk}", p.walk, mk, True)
|
||||
prefixes = sorted({k[: k.index("/") + 1] for k in keys if "/" in k})
|
||||
scopes = ["", *prefixes[:: max(1, len(prefixes) // 4)]]
|
||||
if bucket.endswith("edge"):
|
||||
scopes += ["order/", "order/a", "chars/", "unicode/", "markers/", "markers/dir/", "photos/2006/", "o"]
|
||||
for prefix in scopes:
|
||||
for mk in (1, 3, 1000):
|
||||
check(f"{tag}:delimiter-v2:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk)
|
||||
check(f"{tag}:delimiter-v1:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk, True)
|
||||
for prefix in ("", scopes[-1]):
|
||||
for delim in ("", "/"):
|
||||
for mk in (2, 1000):
|
||||
check(f"{tag}:versions:prefix={prefix}:delimiter={delim}:max-keys={mk}", p.versions, prefix, delim, mk, False)
|
||||
check(f"{tag}:versions-url:prefix={prefix}", p.versions, prefix, "/", 3, True)
|
||||
for d in ("-", "a", "ou"):
|
||||
check(f"{tag}:delimiter={d}", p.delimiter, "", d, 1000)
|
||||
check(f"{tag}:serial:prefix-no-slash", p.serial, keys[len(keys) // 3][:4], 5)
|
||||
# Cut points that are existing keys, absent keys between keys, prefixes,
|
||||
# and a common prefix with and without its trailing delimiter.
|
||||
step = max(1, len(keys) // 9)
|
||||
existing = keys[step::step]
|
||||
absent = [k + "\x7f" for k in existing[::2]] + [k[:-1] for k in existing[1::2] if len(k) > 1]
|
||||
dirs = [k[: k.rfind("/") + 1] for k in existing if "/" in k]
|
||||
dirs += [d.rstrip("/") for d in dirs]
|
||||
for name, cuts in (("existing", existing), ("absent", absent), ("dirs", dirs), ("mixed", existing + absent + dirs)):
|
||||
for mk in (3, 1000):
|
||||
check(f"{tag}:ranges-{name}:max-keys={mk}", p.ranges, cuts, mk)
|
||||
for sa in (keys[len(keys) // 2], prefixes[len(prefixes) // 2], prefixes[len(prefixes) // 2].rstrip("/")):
|
||||
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "", sa)
|
||||
if bucket.endswith("edge"):
|
||||
for sa in ("order/a", "order/a/", "order/a/b", "order/a.b", "order/a0"):
|
||||
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "order/", sa)
|
||||
check(f"{tag}:url-encoding:chars/", p.url_encoding, "chars/")
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- tool capsules
|
||||
|
||||
|
||||
@dataclass
|
||||
class Case:
|
||||
tool: str
|
||||
mode: str
|
||||
bucket: str
|
||||
prefix: str
|
||||
status: str = ""
|
||||
detail: str = ""
|
||||
argv: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class ToolRunner:
|
||||
def __init__(self, args, study: Path, work: Path, endpoint: str) -> None:
|
||||
sys.path.insert(0, str(study / "benchmark" / "src"))
|
||||
from benchmark import adapters # noqa: PLC0415
|
||||
from benchmark.runtime.command_adapter import CommandAdapterError, Stated # noqa: PLC0415
|
||||
|
||||
self.adapters, self.refusal, self.stated = adapters, CommandAdapterError, Stated
|
||||
self.study, self.work, self.endpoint = study, work, endpoint
|
||||
tools = Path(args.tools_dir).resolve()
|
||||
self.fixture_dir = work / "fixtures"
|
||||
self.path_map = {
|
||||
"/usr/local/bin/aws": str(tools / "aws-cli-bin" / "aws"),
|
||||
"/usr/bin/mc": str(tools / "bin" / "mc"),
|
||||
"/usr/local/bin/pS3": str(tools / "bin" / "pS3"),
|
||||
"/usr/local/bin/rclone": str(tools / "bin" / "rclone"),
|
||||
"/usr/bin/s3-fast-list": str(tools / "bin" / "s3-fast-list"),
|
||||
"/usr/bin/ks-tool": str(tools / "bin" / "ks-tool"),
|
||||
"/usr/local/bin/s3kor": str(tools / "bin" / "s3kor"),
|
||||
"/usr/local/bin/s3p": str(tools / "bin" / "s3p"),
|
||||
"/usr/local/bin/s4cmd": str(tools / "bin" / "s4cmd"),
|
||||
"/s5cmd": str(tools / "bin" / "s5cmd"),
|
||||
"/usr/local/bin/s7cmd": str(tools / "bin" / "s7cmd"),
|
||||
"/opt/java/openjdk/bin/java": str(tools / "swath" / "openjdk" / "bin" / "java"),
|
||||
"/opt/swath/swath.jar": str(tools / "swath" / "swath.jar"),
|
||||
"/usr/bin/python3": sys.executable,
|
||||
"/opt/benchmark/tools/": str(study / "tools") + "/",
|
||||
}
|
||||
os.environ["PYTHONPATH"] = str(study / "benchmark" / "src")
|
||||
self.env = {
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"AWS_ACCESS_KEY_ID": args.access_key,
|
||||
"AWS_SECRET_ACCESS_KEY": args.secret_key,
|
||||
"AWS_REGION": REGION,
|
||||
"AWS_DEFAULT_REGION": REGION,
|
||||
"AWS_EC2_METADATA_DISABLED": "true",
|
||||
"TZ": "UTC",
|
||||
"LC_ALL": "C.UTF-8",
|
||||
"PYTHONPATH": str(study / "benchmark" / "src"),
|
||||
}
|
||||
|
||||
def localize(self, argv, bucket: str) -> list[str]:
|
||||
out = []
|
||||
path_map = {**self.path_map, "/fixtures/source/": f"{self.fixture_dir / bucket}/"}
|
||||
for token in argv:
|
||||
for src, dst in path_map.items():
|
||||
if token == src or (src.endswith("/") and token.startswith(src)):
|
||||
token = dst + token[len(src) :]
|
||||
break
|
||||
out.append(token)
|
||||
return out
|
||||
|
||||
def adapter_dir(self, tool: str) -> Path:
|
||||
return self.study / "tools" / tool / "adapter"
|
||||
|
||||
def write_fixture_companions(self, bucket: str, keys: list[str]) -> None:
|
||||
"""The staged-fixture companions, built the way fixture_bundle.py builds them."""
|
||||
import duckdb # noqa: PLC0415
|
||||
|
||||
out = self.fixture_dir / bucket
|
||||
shutil.rmtree(out, ignore_errors=True)
|
||||
out.mkdir(parents=True)
|
||||
shards = sorted({k.split("/", 1)[0] for k in keys})
|
||||
if all(re.fullmatch(r"[A-Za-z0-9._/-]+", s) for s in shards):
|
||||
(out / "s5cmd-shards.input").write_text("".join(f"{s}\n" for s in shards))
|
||||
parts = out / "parquet"
|
||||
parts.mkdir()
|
||||
with duckdb.connect() as con:
|
||||
con.execute("CREATE TABLE f (key BLOB, row_type VARCHAR)")
|
||||
con.executemany("INSERT INTO f VALUES (?, 'OBJECT')", [(k.encode(),) for k in keys])
|
||||
con.execute(f"COPY f TO '{parts / 'part-00000.parquet'}' (FORMAT parquet)")
|
||||
sys.path.insert(0, str(self.study / "tools" / "s3-fast-list" / "adapter"))
|
||||
from fixture_hints import FixtureHintsError, generate_hints # noqa: PLC0415
|
||||
|
||||
try:
|
||||
generate_hints(parts, out / "s3-fast-list-hints.input", segments=4)
|
||||
except FixtureHintsError as exc:
|
||||
print(f"{bucket}: no s3-fast-list fixture hints: {exc}", flush=True)
|
||||
|
||||
def inject_endpoint(self, tool: str, argv: list[str], env: dict) -> tuple[list[str], dict]:
|
||||
"""Add the endpoint the way the replay-endpoint pilot (run.sh) did, for modes
|
||||
the capsule refuses only because the replay server lacks the API."""
|
||||
ep = self.endpoint
|
||||
if tool == "rclone":
|
||||
return [a.replace(f"region={REGION}", f'region={REGION},endpoint="{ep}",force_path_style=true') for a in argv], env
|
||||
if tool == "minio-mc":
|
||||
return argv, {**env, "MC_HOST_s3": ep, "MC_REGION": REGION}
|
||||
if tool == "s3kor":
|
||||
return [argv[0], argv[1], "--custom-endpoint-url", ep, *argv[2:]], env
|
||||
if tool == "s7cmd":
|
||||
return [*argv[:-1], "--target-endpoint-url", ep, "--target-force-path-style", argv[-1]], env
|
||||
if tool == "s5cmd":
|
||||
return [argv[0], "--endpoint-url", ep, *argv[1:]], env
|
||||
return [*argv, "--endpoint-url", ep], env
|
||||
|
||||
def config_for(self, adapter, case: Case, mode: str, keys: list[str]) -> dict[str, object]:
|
||||
config: dict[str, object] = {}
|
||||
for name, axis in adapter.modes[mode].axes.items():
|
||||
if isinstance(axis, self.stated):
|
||||
config[name] = 4
|
||||
if adapter.tool == "s5cmd" and mode == "fanout-with-dirs":
|
||||
initials = sorted({k[len(case.prefix)] for k in keys if k.startswith(case.prefix) and len(k) > len(case.prefix)})
|
||||
if not all(re.fullmatch(r"[A-Za-z0-9._-]", c) for c in initials):
|
||||
raise NotApplicable(f"key initials {initials} are outside the capsule's safe shard set")
|
||||
config["shard_initials"] = "".join(initials)
|
||||
return config
|
||||
|
||||
def execute(self, adapter, case: Case, mode: str, sink: Path, keys: list[str], artifact_path: str = "") -> tuple[Path | None, dict]:
|
||||
signed = adapter.supports_signed
|
||||
inject = case.tool in INJECT_ENDPOINT and mode in INJECT_ENDPOINT[case.tool]
|
||||
config = self.config_for(adapter, case, mode, keys)
|
||||
argv, env = self.adapters.compile_command(
|
||||
self.adapter_dir(case.tool),
|
||||
case.tool,
|
||||
mode=mode,
|
||||
bucket=case.bucket,
|
||||
region=REGION,
|
||||
prefix=case.prefix,
|
||||
signed=signed,
|
||||
config=config,
|
||||
sink_dir=str(sink),
|
||||
artifact_path=artifact_path,
|
||||
visible_memory_gb=2.0,
|
||||
endpoint_url="" if inject else self.endpoint,
|
||||
)
|
||||
if inject:
|
||||
argv, env = self.inject_endpoint(case.tool, list(argv), env)
|
||||
case.detail = "endpoint injected; "
|
||||
argv = self.localize(argv, case.bucket)
|
||||
case.argv = argv
|
||||
sink.mkdir(parents=True, exist_ok=True)
|
||||
home = sink / "home"
|
||||
home.mkdir(exist_ok=True)
|
||||
manifest = adapter.modes[mode]
|
||||
stdout_path = sink / (manifest.product_file if manifest.product_channel == "stdout" and manifest.product_artifact else "stdout")
|
||||
with stdout_path.open("wb") as out, (sink / "stderr").open("wb") as err:
|
||||
proc = subprocess.run(
|
||||
argv,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
env={**self.env, **env, "HOME": str(home)},
|
||||
cwd=sink,
|
||||
timeout=300,
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
tail = (sink / "stderr").read_bytes()[-600:].decode(errors="replace")
|
||||
raise RuntimeError(f"exit {proc.returncode}: {tail}")
|
||||
product = sink / manifest.product_file if manifest.product_artifact else None
|
||||
return product, config
|
||||
|
||||
def run_case(self, case: Case, expected: dict[str, tuple[int, str]]) -> None:
|
||||
adapter = self.adapters.load_adapter(self.adapter_dir(case.tool), case.tool)
|
||||
manifest = adapter.modes[case.mode]
|
||||
if manifest.purpose_ceiling == "preparation" or not manifest.product_artifact or (case.tool, case.mode) in NOT_A_LISTING:
|
||||
case.status = "skip"
|
||||
case.detail = f"{manifest.purpose_ceiling} mode, not a compared listing"
|
||||
return
|
||||
if (case.tool, case.mode) in ROOT_ONLY and case.prefix:
|
||||
case.status, case.detail = "skip", "mode lists the bucket root whatever the prefix"
|
||||
return
|
||||
fixture = self.fixture_dir / case.bucket
|
||||
needs = {"fanout-fixture-with-dirs": "s5cmd-shards.input", "list-hinted-fixture": "s3-fast-list-hints.input"}.get(case.mode)
|
||||
if needs and not (fixture / needs).exists():
|
||||
case.status, case.detail = "skip", f"fixture_bundle.py would not build {needs} for this key set"
|
||||
return
|
||||
keys = list(expected)
|
||||
sink = self.work / "cases" / case.tool / case.mode / (case.bucket + "~" + (case.prefix.replace("/", "_") or "root"))
|
||||
shutil.rmtree(sink, ignore_errors=True)
|
||||
try:
|
||||
artifact_path = ""
|
||||
for step in adapter.requires.get(case.mode, ()):
|
||||
producer_sink = sink / f"requires-{step.mode}"
|
||||
self.execute(adapter, case, step.mode, producer_sink, keys)
|
||||
artifact_path = str(producer_sink / adapter.modes[step.mode].artifacts[step.artifact])
|
||||
if manifest.inline:
|
||||
inline_sink = sink / f"inline-{manifest.inline}"
|
||||
self.execute(adapter, case, manifest.inline, inline_sink, keys, artifact_path)
|
||||
inline_manifest = adapter.modes[manifest.inline]
|
||||
artifact_path = str(inline_sink / next(iter(inline_manifest.artifacts.values())))
|
||||
product, config = self.execute(adapter, case, case.mode, sink, keys, artifact_path)
|
||||
except NotApplicable as exc:
|
||||
case.status, case.detail = "skip", str(exc)
|
||||
return
|
||||
except self.adapters.AdapterError as exc:
|
||||
if "custom endpoint" in str(exc) or "replay endpoint" in str(exc) or "no --prefix" in str(exc) or "requires HeadBucket" in str(exc):
|
||||
case.status, case.detail = "skip", f"capsule refuses: {str(exc).split(': ', 2)[-1][:160]}"
|
||||
return
|
||||
raise
|
||||
normalized = sink / "normalized.tsv"
|
||||
self.adapters.normalize_to_path(
|
||||
self.adapter_dir(case.tool),
|
||||
case.tool,
|
||||
case.mode,
|
||||
case.prefix,
|
||||
normalized,
|
||||
input_path=None if manifest.product_channel == "dataset" else product,
|
||||
dataset=product if manifest.product_channel == "dataset" else None,
|
||||
config=config,
|
||||
)
|
||||
compare(case, normalized.read_bytes(), expected, manifest.fields)
|
||||
|
||||
|
||||
INJECT_ENDPOINT = {
|
||||
"aws-cli": {"s3api-v1-text", "s3api-versions-text"},
|
||||
"minio-mc": {"versions-json", "find", "find-json"},
|
||||
"rclone": {"listv1"},
|
||||
"s3kor": {"list-versions"},
|
||||
"s4cmd": {"recursive", "shallow", "show-directory", "du"},
|
||||
"s5cmd": {"listv1", "allversions"},
|
||||
"s7cmd": {"all-versions"},
|
||||
}
|
||||
"""Capsule modes refused only because the study's replay server serves
|
||||
ListObjectsV2 alone; SeaweedFS serves ListObjects, ListObjectVersions and
|
||||
HeadBucket too."""
|
||||
|
||||
NOT_A_LISTING = {
|
||||
("rclone", "debug"),
|
||||
("rclone", "walk-debug"),
|
||||
("s3p", "summarize"),
|
||||
("s4cmd", "du"),
|
||||
("s4cmd", "show-directory"),
|
||||
}
|
||||
"""Modes with no per-key listing: rclone's normalizer has no debug modes, summarize
|
||||
and du print one aggregate, and `ls -d` prints the directory itself."""
|
||||
|
||||
|
||||
class NotApplicable(Exception):
|
||||
pass
|
||||
|
||||
|
||||
ROOT_ONLY = {
|
||||
("aws-cli", "s3api-v2-remainder"),
|
||||
("s5cmd", "fanout-fixture-with-dirs"),
|
||||
("s3-fast-list", "list-hinted-fixture"),
|
||||
}
|
||||
"""Modes whose capsule lists the root (remainder) or a staged root-scope fixture."""
|
||||
|
||||
SHALLOW = re.compile(r"shallow|delimiter|rootkeys|remainder|max-depth")
|
||||
KEYS_ONLY = re.compile(r"rootkeys|remainder")
|
||||
"""Shallow modes whose normalizer keeps only the keys, dropping CommonPrefixes."""
|
||||
|
||||
|
||||
def compare(case: Case, normalized: bytes, expected: dict[str, tuple[int, str]], fields) -> None:
|
||||
rows = []
|
||||
for line in normalized.split(b"\n"):
|
||||
if not line:
|
||||
continue
|
||||
f = line.split(b"\t")
|
||||
rows.append((f[0].decode("utf-8", "surrogateescape"), f[1].decode(), f[2].decode()))
|
||||
scope = {k: v for k, v in expected.items() if k.startswith(case.prefix)}
|
||||
keys = utf8_sorted(scope)
|
||||
shallow = bool(SHALLOW.search(case.mode))
|
||||
if shallow:
|
||||
view = direct_view(keys, case.prefix, "/")
|
||||
want_objects = [k for kind, k in view if kind == "key"]
|
||||
allowed_dirs = {k for kind, k in view if kind == "prefix"}
|
||||
else:
|
||||
want_objects = keys
|
||||
allowed_dirs = {k[: i + 1] for k in keys for i, c in enumerate(k) if c == "/" and i >= len(case.prefix)}
|
||||
problems: list[str] = []
|
||||
objects, dirs = [], []
|
||||
for key, size, etag in rows:
|
||||
is_dir = key.endswith("/") and size == "-" and etag == "-" and key not in scope
|
||||
(dirs if is_dir else objects).append((key, size, etag))
|
||||
got_keys = [k for k, _, _ in objects]
|
||||
dup = {k for k in got_keys if got_keys.count(k) > 1} if len(got_keys) != len(set(got_keys)) else set()
|
||||
if dup:
|
||||
problems.append(f"duplicate keys {sorted(dup)[:5]}")
|
||||
missing = [k for k in want_objects if k not in set(got_keys)]
|
||||
extra = [k for k in set(got_keys) if k not in set(want_objects)]
|
||||
if missing:
|
||||
problems.append(f"missing {len(missing)}: {missing[:5]}")
|
||||
if extra:
|
||||
problems.append(f"unexpected {len(extra)}: {sorted(extra)[:5]}")
|
||||
bad_dirs = [d for d, _, _ in dirs if d not in allowed_dirs]
|
||||
if bad_dirs:
|
||||
problems.append(f"unexpected directory rows {bad_dirs[:5]}")
|
||||
if shallow and not KEYS_ONLY.search(case.mode) and {d for d, _, _ in dirs} != allowed_dirs:
|
||||
problems.append(f"common prefixes {sorted(d for d, _, _ in dirs)[:8]} != {sorted(allowed_dirs)[:8]}")
|
||||
for key, size, etag in objects:
|
||||
if key not in scope:
|
||||
continue
|
||||
want_size, want_etag = scope[key]
|
||||
if size != "-" and size != str(want_size):
|
||||
problems.append(f"{key!r} size {size} != {want_size}")
|
||||
if etag != "-" and etag != want_etag:
|
||||
problems.append(f"{key!r} etag {etag} != {want_etag}")
|
||||
case.status = "fail" if problems else "pass"
|
||||
case.detail += "; ".join(problems)[:1500] if problems else f"{len(objects)} objects, {len(dirs)} prefixes"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- main
|
||||
|
||||
|
||||
def load_expected_failures(path: Path | None) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
if path and path.exists():
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
name, _, reason = line.partition("#")
|
||||
out[name.strip()] = reason.strip()
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--endpoint", required=True)
|
||||
ap.add_argument("--access-key", required=True)
|
||||
ap.add_argument("--secret-key", required=True)
|
||||
ap.add_argument("--study", required=True)
|
||||
ap.add_argument("--tools-dir", required=True)
|
||||
ap.add_argument("--work-dir", required=True)
|
||||
ap.add_argument("--tools", default="aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath")
|
||||
ap.add_argument("--bucket-prefix", default="study")
|
||||
ap.add_argument("--expected-failures", type=Path)
|
||||
ap.add_argument("--skip-upload", action="store_true")
|
||||
ap.add_argument("--only", default="", help="regex over case names")
|
||||
ap.add_argument("--jobs", type=int, default=4)
|
||||
args = ap.parse_args()
|
||||
|
||||
study = Path(args.study).resolve()
|
||||
work = Path(args.work_dir).resolve()
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
s3 = S3(args.endpoint, args.access_key, args.secret_key)
|
||||
buckets = {f"{args.bucket_prefix}-canary": canary_keys(), f"{args.bucket_prefix}-edge": edge_keys()}
|
||||
expected: dict[str, dict[str, tuple[int, str]]] = {}
|
||||
for bucket, keymap in buckets.items():
|
||||
expected[bucket] = {k: (size, hashlib.md5(body_for(k, size)).hexdigest()) for k, size in keymap.items()}
|
||||
if args.skip_upload:
|
||||
continue
|
||||
s3.create_bucket(bucket)
|
||||
start = time.time()
|
||||
with concurrent.futures.ThreadPoolExecutor(16) as pool:
|
||||
list(pool.map(lambda k: s3.put_object(bucket, k, body_for(k, keymap[k])), keymap))
|
||||
print(f"uploaded {len(keymap)} objects to {bucket} in {time.time() - start:.1f}s", flush=True)
|
||||
|
||||
only = re.compile(args.only) if args.only else None
|
||||
results: list[tuple[str, str, str]] = []
|
||||
for name, err in run_probes(s3, buckets, f"{args.bucket_prefix}-churn", only):
|
||||
results.append((name, "fail" if err else "pass", err or ""))
|
||||
|
||||
runner = ToolRunner(args, study, work, args.endpoint)
|
||||
cases: list[Case] = []
|
||||
for bucket in buckets:
|
||||
runner.write_fixture_companions(bucket, list(buckets[bucket]))
|
||||
scopes = ["", "group-07/"] if bucket.endswith("canary") else ["", "order/", "chars/"]
|
||||
for tool in filter(None, args.tools.split(",")):
|
||||
adapter = runner.adapters.load_adapter(runner.adapter_dir(tool), tool)
|
||||
for mode in adapter.modes:
|
||||
for prefix in scopes:
|
||||
case = Case(tool, mode, bucket, prefix)
|
||||
if only and not only.search(case_name(case)):
|
||||
continue
|
||||
cases.append(case)
|
||||
|
||||
def run(case: Case) -> Case:
|
||||
start = time.time()
|
||||
try:
|
||||
runner.run_case(case, expected[case.bucket])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
case.status = "fail"
|
||||
case.detail = f"{type(exc).__name__}: {exc}"[:1500]
|
||||
if os.environ.get("STUDY_DEBUG"):
|
||||
traceback.print_exc()
|
||||
print(f"{case.status:5} {case_name(case)} ({time.time() - start:.1f}s) {case.detail[:300]}", flush=True)
|
||||
return case
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
|
||||
for case in pool.map(run, cases):
|
||||
results.append((case_name(case), case.status, case.detail))
|
||||
|
||||
xfail = load_expected_failures(args.expected_failures)
|
||||
regressions, fixed = [], []
|
||||
counts: dict[str, int] = {}
|
||||
for name, status, detail in results:
|
||||
if status == "fail" and name in xfail:
|
||||
status = "xfail"
|
||||
elif status == "pass" and name in xfail:
|
||||
fixed.append(name)
|
||||
elif status == "fail":
|
||||
regressions.append((name, detail))
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
(work / "results.json").write_text(json.dumps(results, indent=1, ensure_ascii=False))
|
||||
print("\n== summary:", ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
|
||||
for name in fixed:
|
||||
print(f"XPASS (remove from expected failures): {name}")
|
||||
for name, detail in regressions:
|
||||
print(f"FAIL {name}: {detail}")
|
||||
return 1 if regressions or fixed else 0
|
||||
|
||||
|
||||
def case_name(case: Case) -> str:
|
||||
return f"{case.tool}:{case.mode}:{case.bucket.rsplit('-', 1)[-1]}:{case.prefix or '/'}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "listing_study_admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "listing_study_access",
|
||||
"secretKey": "listing_study_secret"
|
||||
}
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
},
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": ["Read", "List"]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user