diff --git a/.github/workflows/s3-listing-study-tests.yml b/.github/workflows/s3-listing-study-tests.yml new file mode 100644 index 0000000000..ee204928f7 --- /dev/null +++ b/.github/workflows/s3-listing-study-tests.yml @@ -0,0 +1,74 @@ +name: "S3 listing study tests" + +on: + push: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'weed/server/**' + - 'test/s3/listing_study/**' + - '.github/workflows/s3-listing-study-tests.yml' + pull_request: + branches: [ master ] + paths: + - 'weed/s3api/**' + - 'weed/filer/**' + - 'weed/server/**' + - 'test/s3/listing_study/**' + - '.github/workflows/s3-listing-study-tests.yml' + +concurrency: + group: ${{ github.event.pull_request.number || github.ref }}/s3-listing-study-tests + cancel-in-progress: true + +permissions: + contents: read + +jobs: + listing-study: + name: s3-listing-study tools and strategies + runs-on: ubuntu-24.04 + timeout-minutes: 40 + defaults: + run: + working-directory: test/s3/listing_study + steps: + - name: Check out code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} + + - name: Cache pinned listing tools + uses: actions/cache@v6 + with: + path: test/s3/listing_study/.work/tools + key: listing-study-tools-${{ runner.os }}-${{ hashFiles('test/s3/listing_study/Makefile', 'test/s3/listing_study/install_tools.py') }} + + - name: Build weed and install the study's pinned tools + run: make build-weed tools + + - name: Run the listing study against SeaweedFS + timeout-minutes: 15 + run: make test + + - name: Show logs on failure + if: failure() + run: | + tail -200 .work/weed.log || true + + - name: Upload results + if: always() + uses: actions/upload-artifact@v7 + with: + name: s3-listing-study-results + path: | + test/s3/listing_study/.work/run/results.json + test/s3/listing_study/.work/weed.log + retention-days: 14 diff --git a/test/s3/listing_study/.gitignore b/test/s3/listing_study/.gitignore new file mode 100644 index 0000000000..925841af6d --- /dev/null +++ b/test/s3/listing_study/.gitignore @@ -0,0 +1 @@ +.work diff --git a/test/s3/listing_study/Makefile b/test/s3/listing_study/Makefile new file mode 100644 index 0000000000..1508eaa239 --- /dev/null +++ b/test/s3/listing_study/Makefile @@ -0,0 +1,87 @@ +# Runs the listing tools and listing strategies of +# https://github.com/varveio/s3-listing-study against a local SeaweedFS. +# +# make test build weed, fetch the study and its pinned tools, start +# weed server on free ports, run, stop +# make run ENDPOINT=... run against an already running endpoint +# +# Needs: go, python3, node/npm, cargo, docker (swath image), curl. +# +# The server runs with -s3.allowBackslashInKey, the AWS-compatible key mode: by +# default SeaweedFS rewrites '\' in object keys to '/', and the edge key set +# holds a key with a backslash. + +STUDY_REPO ?= https://github.com/varveio/s3-listing-study.git +STUDY_REV ?= 666ec620d245e16baaaa0cf53461382f42c35917 +WORK_DIR ?= $(CURDIR)/.work +STUDY_DIR ?= $(WORK_DIR)/s3-listing-study +TOOLS_DIR ?= $(WORK_DIR)/tools +VENV ?= $(WORK_DIR)/venv +WEED_BINARY ?= $(WORK_DIR)/weed +TOOLS ?= aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath +ACCESS_KEY ?= listing_study_access +SECRET_KEY ?= listing_study_secret +JOBS ?= 4 +ONLY ?= + +.PHONY: test run build-weed study venv tools start-server stop-server clean + +test: build-weed tools + @$(MAKE) --no-print-directory start-server; status=$$?; \ + if [ $$status -eq 0 ]; then \ + $(MAKE) --no-print-directory run ENDPOINT=http://127.0.0.1:$$(cat $(WORK_DIR)/s3.port); status=$$?; \ + fi; \ + $(MAKE) --no-print-directory stop-server; exit $$status + +build-weed: + mkdir -p $(WORK_DIR) + cd ../../../weed && go build -buildvcs=false -o $(WEED_BINARY) . + +study: + @if [ "$$(git -C $(STUDY_DIR) rev-parse HEAD 2>/dev/null)" != "$(STUDY_REV)" ]; then \ + git init -q $(STUDY_DIR) && \ + git -C $(STUDY_DIR) fetch -q --depth 1 $(STUDY_REPO) $(STUDY_REV) && \ + git -C $(STUDY_DIR) checkout -q --force FETCH_HEAD; \ + fi + +venv: + @test -x $(VENV)/bin/python || python3 -m venv $(VENV) + $(VENV)/bin/pip install -q -r requirements.txt + +tools: study venv + $(VENV)/bin/python install_tools.py --study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --tools $(TOOLS) + +start-server: + @mkdir -p $(WORK_DIR) + @set -e; data=$$(mktemp -d $(WORK_DIR)/data.XXXXXX); ports=$$(python3 free_ports.py 5); \ + set -- $$ports; \ + $(WEED_BINARY) server -filer -s3 -ip 127.0.0.1 -ip.bind 127.0.0.1 -dir=$$data \ + -master.peers=none -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ + -volume.max=50 -volume.preStopSeconds=1 \ + -master.port=$$1 -volume.port=$$2 -filer.port=$$3 -s3.port=$$4 -metricsPort=$$5 \ + -s3.port.iceberg=0 -s3.port.lance=0 -s3.allowBackslashInKey -s3.config=$(CURDIR)/s3.json \ + > $(WORK_DIR)/weed.log 2>&1 & echo $$! > $(WORK_DIR)/weed.pid; \ + echo $$4 > $(WORK_DIR)/s3.port; \ + for i in $$(seq 1 90); do \ + if curl -sf -o /dev/null http://127.0.0.1:$$3/ && curl -s -o /dev/null http://127.0.0.1:$$4/; then \ + echo "SeaweedFS S3 ready on port $$4"; exit 0; fi; \ + kill -0 $$(cat $(WORK_DIR)/weed.pid) || { tail -50 $(WORK_DIR)/weed.log; exit 1; }; \ + sleep 1; \ + done; tail -50 $(WORK_DIR)/weed.log; exit 1 + +stop-server: + @if [ -f $(WORK_DIR)/weed.pid ]; then \ + pid=$$(cat $(WORK_DIR)/weed.pid); kill $$pid 2>/dev/null || true; \ + for i in $$(seq 1 20); do kill -0 $$pid 2>/dev/null || break; sleep 0.5; done; \ + rm -f $(WORK_DIR)/weed.pid; \ + fi + +run: study + $(VENV)/bin/python run_study.py --endpoint $(ENDPOINT) \ + --access-key $(ACCESS_KEY) --secret-key $(SECRET_KEY) \ + --study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --work-dir $(WORK_DIR)/run \ + --tools $(TOOLS) --jobs $(JOBS) --expected-failures expected_failures.txt \ + $(if $(ONLY),--only '$(ONLY)') + +clean: stop-server + rm -rf .work diff --git a/test/s3/listing_study/expected_failures.txt b/test/s3/listing_study/expected_failures.txt new file mode 100644 index 0000000000..d28371f82b --- /dev/null +++ b/test/s3/listing_study/expected_failures.txt @@ -0,0 +1,29 @@ +# Cases that fail for reasons outside SeaweedFS. A listed case that passes is +# reported as XPASS and fails the run, so this list has to shrink with fixes. +# Format: # reason + +# aws-cli prints "None" for a --query over a page with no Contents (the canary +# has no root-level keys); the capsule normalizer reads it as a malformed row. +aws-cli:s3api-v2-remainder:canary:/ # capsule normalizer cannot parse aws-cli's "None" + +# The edge bucket holds zero-byte directory markers (markers/dir/, markers/empty/). +# These modes list them as directories, not objects: rclone's --files-only and +# lsf --files-only drop them, s5cmd types any key ending in "/" as DIR and the +# capsule normalizer drops DIR rows from its recursive modes. The reference +# probes check that SeaweedFS returns both markers as Contents. +rclone:recursive-fastlist:edge:/ # tool hides directory marker objects +rclone:recursive-hierarchical:edge:/ # tool hides directory marker objects +rclone:recursive-walk:edge:/ # tool hides directory marker objects +rclone:listv1:edge:/ # tool hides directory marker objects +rclone:lsf:edge:/ # tool hides directory marker objects +s5cmd:recursive:edge:/ # tool hides directory marker objects +s5cmd:listv1:edge:/ # tool hides directory marker objects +s5cmd:allversions:edge:/ # tool hides directory marker objects + +# Deleting the last key under a prefix leaves its filer directory behind, and an +# empty directory is still listed as a CommonPrefix until the filer's empty-folder +# cleanup removes it (2 minutes by default). This is deliberate: directories made +# through mount or the filer stay visible to S3 clients (dirHoldsOnlyHiddenEntries +# in weed/s3api/s3api_object_handlers_list.go). AWS derives CommonPrefixes from +# keys only, so prefix-discovery tools briefly see the deleted prefix. +reference:study-churn:deleted-prefixes # empty directories are listed until cleanup diff --git a/test/s3/listing_study/free_ports.py b/test/s3/listing_study/free_ports.py new file mode 100644 index 0000000000..871cc751b8 --- /dev/null +++ b/test/s3/listing_study/free_ports.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Print N free TCP ports whose +10000 gRPC companions are free too, all below +the Linux ephemeral range (32768+) that outgoing connections draw from.""" + +import random +import socket +import sys + + +def free(port: int) -> bool: + with socket.socket() as s: + try: + s.bind(("127.0.0.1", port)) + return True + except OSError: + return False + + +picked: list[int] = [] +taken: list[int] = [] +while len(picked) < int(sys.argv[1]): + port = random.randrange(10000, 22000) + pair = (port, port + 10000) + if all(abs(p - t) > 20 for p in pair for t in taken) and all(free(p) for p in pair): + picked.append(port) + taken += pair +print(*picked) diff --git a/test/s3/listing_study/install_tools.py b/test/s3/listing_study/install_tools.py new file mode 100644 index 0000000000..dbce02aa51 --- /dev/null +++ b/test/s3/listing_study/install_tools.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Install the listing tools pinned by the s3-listing-study checkout. + +Every artifact locator and sha256 comes from the study's own +tools//build/image.json, so bumping STUDY_REV bumps the tools with it. +Tools land under TOOLS_DIR at the paths listed in run_study.PATH_MAP. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import stat +import subprocess +import sys +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +ALL_TOOLS = ( + "aws-cli", + "minio-mc", + "ps3", + "rclone", + "s3-fast-list", + "s3kor", + "s3p", + "s4cmd", + "s5cmd", + "s7cmd", + "swath", +) + +MC_COMMIT = "d6541ea280b73a834b64d4097e21f2be77676104" +"""github.com/minio/mc tag RELEASE.2025-08-13T08-35-41Z.""" + + +def fetch(url: str, sha256: str, dest: Path) -> Path: + if dest.exists() and hashlib.sha256(dest.read_bytes()).hexdigest() == sha256: + return dest + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_suffix(dest.suffix + ".part") + request = urllib.request.Request(url, headers={"User-Agent": "curl/8"}) + with urllib.request.urlopen(request, timeout=300) as response, tmp.open("wb") as out: + shutil.copyfileobj(response, out) + digest = hashlib.sha256(tmp.read_bytes()).hexdigest() + if digest != sha256: + tmp.unlink() + raise SystemExit(f"{url}: sha256 {digest} != pinned {sha256}") + tmp.rename(dest) + return dest + + +def executable(path: Path) -> None: + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def wrapper(path: Path, body: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"#!/bin/sh\n{body}\n") + executable(path) + + +def tar_member(archive: Path, member: str, dest: Path) -> None: + with tarfile.open(archive) as tar: + source = tar.extractfile(member) + if source is None: + raise SystemExit(f"{archive}: no member {member}") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(source.read()) + executable(dest) + + +def install(tool: str, study: Path, tools_dir: Path) -> None: + image = json.loads((study / "tools" / tool / "build" / "image.json").read_text()) + artifact = image["tool_artifact"] + url, sha = artifact["locator"], artifact["sha256"] + dl = tools_dir / "downloads" + bin_dir = tools_dir / "bin" + bin_dir.mkdir(parents=True, exist_ok=True) + name = url.rsplit("/", 1)[-1] + stamp = tools_dir / "installed" / tool + if stamp.exists() and stamp.read_text() == sha: + print(f"{tool} {image.get('tool_version', '')} already installed") + return + + if tool == "aws-cli": + archive = fetch(url, sha, dl / name) + work = tools_dir / "aws-cli-src" + shutil.rmtree(work, ignore_errors=True) + with zipfile.ZipFile(archive) as zf: + zf.extractall(work) + for rel in ("aws/install", "aws/dist/aws", "aws/dist/aws_completer"): + executable(work / rel) + subprocess.run( + [ + str(work / "aws" / "install"), + "--install-dir", + str(tools_dir / "aws-cli"), + "--bin-dir", + str(tools_dir / "aws-cli-bin"), + "--update", + ], + check=True, + ) + shutil.rmtree(work) + elif tool == "minio-mc": + # dl.min.io answers 410 Gone for every archived mc release, so build the + # same release tag from source instead. + if image["tool_version"] != "RELEASE.2025-08-13T08-35-41Z": + raise SystemExit(f"minio-mc: no source commit pinned for {image['tool_version']}") + subprocess.run( + ["go", "install", f"github.com/minio/mc@{MC_COMMIT}"], + env={**os.environ, "GOBIN": str(bin_dir)}, + check=True, + ) + elif tool == "ps3": + shutil.copyfile(fetch(url, sha, dl / name), bin_dir / "pS3") + executable(bin_dir / "pS3") + elif tool == "rclone": + archive = fetch(url, sha, dl / name) + with zipfile.ZipFile(archive) as zf: + member = f"{name.removesuffix('.zip')}/rclone" + (bin_dir / "rclone").write_bytes(zf.read(member)) + executable(bin_dir / "rclone") + elif tool == "s3kor": + tar_member(fetch(url, sha, dl / name), "s3kor", bin_dir / "s3kor") + elif tool == "s5cmd": + tar_member(fetch(url, sha, dl / name), "s5cmd", bin_dir / "s5cmd") + elif tool == "s7cmd": + tar_member(fetch(url, sha, dl / name), "s7cmd", bin_dir / "s7cmd") + elif tool == "s3p": + work = tools_dir / "s3p" + work.mkdir(parents=True, exist_ok=True) + for f in ("package.json", "package-lock.json"): + shutil.copyfile(study / "tools/s3p/build" / f, work / f) + subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], cwd=work, check=True) + wrapper(bin_dir / "s3p", f'exec node "{work}/node_modules/s3p/bin/s3p" "$@"') + elif tool == "s4cmd": + archive = fetch(url, sha, dl / name) + venv = tools_dir / "s4cmd-venv" + subprocess.run([sys.executable, "-m", "venv", "--clear", str(venv)], check=True) + subprocess.run( + [ + str(venv / "bin" / "python"), + "-m", + "pip", + "install", + "--quiet", + "--require-hashes", + "--no-deps", + "-r", + str(study / "tools/s4cmd/build/requirements.txt"), + ], + check=True, + ) + script = tools_dir / "s4cmd" / "s4cmd.py" + with tarfile.open(archive) as tar: + source = tar.extractfile("s4cmd-2.1.0/s4cmd.py") + assert source is not None + script.parent.mkdir(parents=True, exist_ok=True) + script.write_bytes(source.read()) + wrapper(bin_dir / "s4cmd", f'exec "{venv}/bin/python" "{script}" "$@"') + elif tool == "s3-fast-list": + archive = fetch(url, sha, dl / name) + work = tools_dir / "s3-fast-list-src" + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + subprocess.run(["tar", "xzf", str(archive), "--strip-components=1", "-C", str(work)], check=True) + shutil.copyfile(study / "tools/s3-fast-list/build/Cargo.lock", work / "Cargo.lock") + subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=work, check=True) + for exe in ("s3-fast-list", "ks-tool"): + shutil.copyfile(work / "target/release" / exe, bin_dir / exe) + executable(bin_dir / exe) + elif tool == "swath": + ref = f"{url}@sha256:{sha}" + subprocess.run(["docker", "pull", "--quiet", ref], check=True, stdout=subprocess.DEVNULL) + cid = subprocess.run( + ["docker", "create", ref], check=True, capture_output=True, text=True + ).stdout.strip() + try: + dest = tools_dir / "swath" + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir(parents=True) + subprocess.run(["docker", "cp", f"{cid}:/opt/java/openjdk", str(dest / "openjdk")], check=True) + subprocess.run(["docker", "cp", f"{cid}:/opt/swath/swath.jar", str(dest / "swath.jar")], check=True) + finally: + subprocess.run(["docker", "rm", cid], check=True, stdout=subprocess.DEVNULL) + else: + raise SystemExit(f"unknown tool {tool}") + stamp.parent.mkdir(parents=True, exist_ok=True) + stamp.write_text(sha) + print(f"installed {tool} {image.get('tool_version', '')}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--study", required=True, type=Path) + parser.add_argument("--tools-dir", required=True, type=Path) + parser.add_argument("--tools", default=",".join(ALL_TOOLS)) + args = parser.parse_args() + for tool in filter(None, args.tools.split(",")): + install(tool, args.study.resolve(), args.tools_dir.resolve()) + return 0 + + +if __name__ == "__main__": + os.umask(0o022) + raise SystemExit(main()) diff --git a/test/s3/listing_study/requirements.txt b/test/s3/listing_study/requirements.txt new file mode 100644 index 0000000000..4bd306c82a --- /dev/null +++ b/test/s3/listing_study/requirements.txt @@ -0,0 +1,6 @@ +# Versions the pinned s3-listing-study uv.lock resolves for the capsule normalizers. +duckdb==1.5.5 +ijson==3.5.1 +pyyaml==6.0.3 +zstandard==0.25.0 +botocore==1.44.5 diff --git a/test/s3/listing_study/run_study.py b/test/s3/listing_study/run_study.py new file mode 100644 index 0000000000..4cc6e28f44 --- /dev/null +++ b/test/s3/listing_study/run_study.py @@ -0,0 +1,922 @@ +#!/usr/bin/env python3 +"""Run the s3-listing-study tool capsules and ListObjects reference probes against SeaweedFS. + +Tool cases: every mode of every capsule under the study's tools/ is compiled by +the capsule's own command.py (benchmark.adapters.compile_command, with the +custom-endpoint wiring the study's replay-endpoint pilot established), run +against SeaweedFS with the pinned tool build, normalized by the capsule's own +normalize.py, and the normalized listing is compared exactly against the keys +that were written. + +Reference probes: raw, signed ListObjectsV2/ListObjects requests implementing +the strategies the study classifies tools by (serial pagination, delimiter +prefix discovery, start-after range splitting) plus the request semantics in +docs/s3-reference.md, checked against the byte-ordered key set. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import time +import traceback +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path + +REGION = "us-east-1" +S3NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def canary_keys() -> dict[str, int]: + """benchmark/fixtures/replay-canary/generate.sql: 2,048 keys in 16 prefixes.""" + return {f"group-{i // 128:02d}/object-{i:06d}.dat": 1024 + i for i in range(2048)} + + +EDGE_KEYS = [ + # docs/smoke-bucket.md: a root key beside the prefixes, for delimiter checks. + "index.html", + # docs/s3-reference.md examples. + "test_file(3).png", + "photos/2006/February/sample.jpg", + "photos/2006/January/sample.jpg", + "photos/2007/sample.jpg", + "notes/summer/july", + "notes/summer/august", + "notes/winter", + # Byte order around the '/' (0x2f) delimiter. + "order/a", + "order/a b", + "order/a!b", + "order/a-b", + "order/a.b", + "order/a/b", + "order/a/c/d", + "order/a0", + "order/aa", + "order/a~", + "order/A", + "order/b", + # Zero-byte directory markers. + "markers/dir/", + "markers/dir/child", + "markers/empty/", + # Characters that need XML or URL escaping. + "chars/space key", + "chars/plus+key", + "chars/percent%20key", + "chars/amp&key", + "chars/ltkey", + 'chars/quote"key', + "chars/apos'key", + "chars/hash#key", + "chars/question?key", + "chars/equals=key", + "chars/semicolon;key", + "chars/colon:key", + "chars/at@key", + "chars/comma,key", + "chars/star*key", + "chars/bang!key", + "chars/dollar$key", + "chars/caret^key", + "chars/backslash\\key", + "chars/brace{key}", + "chars/pipe|key", + "chars/backtick`key", + "chars/tilde~key", + # UTF-8 byte order, which differs from UTF-16 order above U+FFFF. + "unicode/café-nfc", + "unicode/café-nfd", + "unicode/日本語.txt", + "unicode/。-halfwidth", + "unicode/\U0001f600-emoji", + "unicode/z-ascii", + "deep/a/b/c/d/e/f/g/h/i/j/k.txt", + "long/" + "/".join(["y" * 200] * 4), +] + + +def edge_keys() -> dict[str, int]: + return {k: (0 if k.endswith("/") else 100 + i) for i, k in enumerate(EDGE_KEYS)} + + +def body_for(key: str, size: int) -> bytes: + seed = hashlib.sha256(key.encode()).digest() + return (seed * (size // len(seed) + 1))[:size] + + +def utf8_sorted(keys) -> list[str]: + return sorted(keys, key=lambda k: k.encode("utf-8")) + + +# ---------------------------------------------------------------- signed HTTP + + +class S3: + def __init__(self, endpoint: str, access: str, secret: str) -> None: + from botocore.auth import S3SigV4Auth + from botocore.credentials import Credentials + + self.endpoint = endpoint.rstrip("/") + self.auth = S3SigV4Auth(Credentials(access, secret), "s3", REGION) + + def request( + self, method: str, path: str, params: list[tuple[str, str]] | None = None, body: bytes = b"" + ) -> tuple[int, bytes]: + from botocore.awsrequest import AWSRequest + + query = "&".join( + f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}" + for k, v in (params or []) + ) + url = self.endpoint + urllib.parse.quote(path, safe="/-_.~") + (f"?{query}" if query else "") + req = AWSRequest(method=method, url=url, data=body) + req.headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest() + self.auth.add_auth(req) + prepared = req.prepare() + http = urllib.request.Request(prepared.url, data=body or None, method=method) + for k, v in prepared.headers.items(): + http.add_header(k, v) + try: + with urllib.request.urlopen(http, timeout=60) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as err: + return err.code, err.read() + + def put_object(self, bucket: str, key: str, body: bytes) -> None: + # The first writes to a new bucket can get a retryable 500 while its + # volumes grow; SDK clients retry those the same way. + for attempt in range(10): + status, out = self.request("PUT", f"/{bucket}/{key}", body=body) + if status < 500: + break + time.sleep(0.5 * (attempt + 1)) + if status != 200: + raise RuntimeError(f"PUT {bucket}/{key!r}: {status} {out[:300]!r}") + + def delete_object(self, bucket: str, key: str) -> None: + status, out = self.request("DELETE", f"/{bucket}/{key}") + if status != 204: + raise RuntimeError(f"DELETE {bucket}/{key!r}: {status} {out[:300]!r}") + + def create_bucket(self, bucket: str) -> None: + status, out = self.request("PUT", f"/{bucket}") + if status not in (200, 409): + raise RuntimeError(f"create bucket {bucket}: {status} {out[:300]!r}") + + def list_page(self, bucket: str, params: list[tuple[str, str]]) -> ET.Element: + status, out = self.request("GET", f"/{bucket}", params) + if status != 200: + raise AssertionError(f"list {params}: HTTP {status} {out[:300]!r}") + return ET.fromstring(out) + + +def text(el: ET.Element, name: str) -> str | None: + child = el.find(S3NS + name) + return None if child is None else (child.text or "") + + +@dataclass +class Page: + entries: list[tuple[str, str]] # ("key"|"prefix", value) in response order + truncated: bool + token: str | None + next_marker: str | None + key_count: str | None + raw: ET.Element + + +def check_byte_order(values: list[str], what: str) -> None: + for a, b in zip(values, values[1:]): + assert a.encode() <= b.encode(), f"{what} out of byte order: {a!r} before {b!r}" + + +def parse_page(root: ET.Element, url_encoded: bool) -> Page: + dec = urllib.parse.unquote_plus if url_encoded else (lambda s: s) + entries: list[tuple[str, str]] = [] + for child in root: + if child.tag == S3NS + "Contents": + entries.append(("key", dec(text(child, "Key") or ""))) + elif child.tag == S3NS + "CommonPrefixes": + entries.append(("prefix", dec(text(child, "Prefix") or ""))) + nm = text(root, "NextMarker") + # XML carries Contents and CommonPrefixes as two lists, each in byte order; + # their merged byte order is what pagination has to respect. + for kind in ("key", "prefix"): + check_byte_order([v for k, v in entries if k == kind], f"{kind} list") + entries.sort(key=lambda e: e[1].encode()) + return Page( + entries, + text(root, "IsTruncated") == "true", + text(root, "NextContinuationToken"), + dec(nm) if nm is not None else None, + text(root, "KeyCount"), + root, + ) + + +# ---------------------------------------------------------------- expectations + + +def direct_view(keys: list[str], prefix: str, delimiter: str) -> list[tuple[str, str]]: + """What a delimiter listing of prefix returns, in byte order.""" + out: dict[str, str] = {} + for k in keys: + if not k.startswith(prefix): + continue + rest = k[len(prefix) :] + idx = rest.find(delimiter) if delimiter else -1 + if idx >= 0: + out[prefix + rest[: idx + len(delimiter)]] = "prefix" + else: + out[k] = "key" + return [(out[k], k) for k in utf8_sorted(out)] + + +# ---------------------------------------------------------------- reference probes + + +class Probe: + def __init__(self, s3: S3, bucket: str, keys: list[str]) -> None: + self.s3, self.bucket, self.keys = s3, bucket, utf8_sorted(keys) + + def v2_all(self, base: list[tuple[str, str]], max_keys: int, url: bool = False) -> list[Page]: + pages: list[Page] = [] + token = None + while True: + params = [("list-type", "2"), ("max-keys", str(max_keys)), *base] + if url: + params.append(("encoding-type", "url")) + if token: + params.append(("continuation-token", token)) + page = parse_page(self.s3.list_page(self.bucket, params), url) + pages.append(page) + n = len(page.entries) + assert n <= max_keys, f"page holds {n} > max-keys {max_keys}" + assert page.key_count == str(n), f"KeyCount {page.key_count} != {n} entries" + if not page.truncated: + assert not page.token, "NextContinuationToken on an untruncated page" + return pages + assert page.token, "IsTruncated without NextContinuationToken" + assert len(pages) < 10000, "pagination does not terminate" + token = page.token + + def v1_all(self, base: list[tuple[str, str]], max_keys: int, delimiter: str) -> list[Page]: + pages: list[Page] = [] + marker = None + while True: + params = [("max-keys", str(max_keys)), *base] + if delimiter: + params.append(("delimiter", delimiter)) + if marker is not None: + params.append(("marker", marker)) + page = parse_page(self.s3.list_page(self.bucket, params), False) + pages.append(page) + assert len(page.entries) <= max_keys + if not page.truncated: + return pages + if delimiter: + assert page.next_marker, "truncated delimiter page without NextMarker" + marker = page.next_marker + else: + marker = page.next_marker or page.entries[-1][1] + assert len(pages) < 10000, "pagination does not terminate" + + @staticmethod + def flat(pages: list[Page]) -> list[tuple[str, str]]: + return [e for p in pages for e in p.entries] + + def expect_eq(self, got, want, what: str) -> None: + if got != want: + gs, ws = set(got), set(want) + missing = [w for w in want if w not in gs][:5] + extra = [g for g in got if g not in ws][:5] + first = next((i for i, (a, b) in enumerate(zip(got, want)) if a != b), min(len(got), len(want))) + raise AssertionError( + f"{what}: got {len(got)} want {len(want)}; missing {missing} extra {extra}; " + f"first difference at {first}: got {got[first:first + 3]} want {want[first:first + 3]}" + ) + + # Serial pagination (aws-cli, mc, s3kor, s5cmd ls). + def serial(self, prefix: str, max_keys: int) -> None: + want = [("key", k) for k in self.keys if k.startswith(prefix)] + got = self.flat(self.v2_all([("prefix", prefix)], max_keys)) + self.expect_eq(got, want, f"serial prefix={prefix!r} max-keys={max_keys}") + + # Delimiter view and page boundaries inside it. + def delimiter(self, prefix: str, delimiter: str, max_keys: int, v1: bool = False) -> None: + want = direct_view(self.keys, prefix, delimiter) + if v1: + got = self.flat(self.v1_all([("prefix", prefix)], max_keys, delimiter)) + else: + got = self.flat(self.v2_all([("prefix", prefix), ("delimiter", delimiter)], max_keys)) + self.expect_eq(got, want, f"{'v1' if v1 else 'v2'} delimiter={delimiter!r} prefix={prefix!r} max-keys={max_keys}") + + # Prefix discovery (rclone walk, s7cmd, s4cmd): recurse through CommonPrefixes. + def walk(self, max_keys: int, v1: bool = False) -> None: + found: list[str] = [] + queue = [""] + while queue: + prefix = queue.pop() + if v1: + entries = self.flat(self.v1_all([("prefix", prefix)], max_keys, "/")) + else: + entries = self.flat(self.v2_all([("prefix", prefix), ("delimiter", "/")], max_keys)) + for kind, value in entries: + (queue.append if kind == "prefix" else found.append)(value) + self.expect_eq(utf8_sorted(found), self.keys, f"{'v1' if v1 else 'v2'} delimiter walk max-keys={max_keys}") + assert len(found) == len(set(found)), "walk returned a key twice" + + # Range splitting (S3P, s3-fast-list, Swath): start-after cut points. + def ranges(self, cuts: list[str], max_keys: int) -> None: + cuts = sorted(set(cuts), key=lambda c: c.encode()) + bounds = [None, *cuts, None] + found: list[str] = [] + for lo, hi in zip(bounds, bounds[1:]): + token = None + done = False + while not done: + params = [("list-type", "2"), ("max-keys", str(max_keys))] + if token: + params.append(("continuation-token", token)) + elif lo is not None: + params.append(("start-after", lo)) + page = parse_page(self.s3.list_page(self.bucket, params), False) + for _, key in page.entries: + if lo is not None: + assert key.encode() > lo.encode(), f"start-after {lo!r} returned {key!r}" + if hi is not None and key.encode() > hi.encode(): + done = True + break + found.append(key) + if not page.truncated: + break + token = page.token + self.expect_eq(found, self.keys, f"start-after ranges ({len(cuts)} cuts, max-keys={max_keys})") + + def start_after_with_delimiter(self, prefix: str, start_after: str) -> None: + want = [ + e + for e in direct_view(self.keys, prefix, "/") + if e[1].encode() > start_after.encode() + ] + got = self.flat( + self.v2_all([("prefix", prefix), ("delimiter", "/"), ("start-after", start_after)], 1000) + ) + self.expect_eq(got, want, f"start-after={start_after!r} delimiter=/ prefix={prefix!r}") + + def token_overrides_start_after(self) -> None: + first = parse_page(self.s3.list_page(self.bucket, [("list-type", "2"), ("max-keys", "5")]), False) + assert first.truncated and first.token + params = [("list-type", "2"), ("max-keys", "5"), ("continuation-token", first.token)] + plain = parse_page(self.s3.list_page(self.bucket, params), False) + mixed = parse_page( + self.s3.list_page(self.bucket, [*params, ("start-after", self.keys[len(self.keys) // 2])]), + False, + ) + self.expect_eq(mixed.entries, plain.entries, "continuation-token with a later start-after") + + def versions(self, prefix: str, delimiter: str, max_keys: int, url: bool) -> None: + """ListObjectVersions paging (key-marker/version-id-marker) over unversioned keys.""" + want = direct_view(self.keys, prefix, delimiter) if delimiter else [("key", k) for k in self.keys if k.startswith(prefix)] + got: list[tuple[str, str]] = [] + key_marker = version_marker = None + dec = urllib.parse.unquote_plus if url else (lambda v: v) + for _ in range(10000): + params = [("versions", ""), ("max-keys", str(max_keys)), ("prefix", prefix)] + if delimiter: + params.append(("delimiter", delimiter)) + if url: + params.append(("encoding-type", "url")) + if key_marker is not None: + params += [("key-marker", key_marker), ("version-id-marker", version_marker or "")] + root = self.s3.list_page(self.bucket, params) + page: list[tuple[str, str]] = [] + for child in root: + if child.tag in (S3NS + "Version", S3NS + "DeleteMarker"): + page.append(("key", dec(text(child, "Key") or ""))) + elif child.tag == S3NS + "CommonPrefixes": + page.append(("prefix", dec(text(child, "Prefix") or ""))) + assert len(page) <= max_keys, f"{len(page)} entries > max-keys {max_keys}" + for kind in ("key", "prefix"): + check_byte_order([v for k, v in page if k == kind], f"version {kind} list") + got += sorted(page, key=lambda e: e[1].encode()) + if text(root, "IsTruncated") != "true": + break + nk = text(root, "NextKeyMarker") + assert nk, "truncated versions page without NextKeyMarker" + key_marker, version_marker = dec(nk), text(root, "NextVersionIdMarker") + self.expect_eq(got, want, f"versions prefix={prefix!r} delimiter={delimiter!r} max-keys={max_keys} url={url}") + + def url_encoding(self, prefix: str) -> None: + want = [("key", k) for k in self.keys if k.startswith(prefix)] + got = self.flat(self.v2_all([("prefix", prefix)], 7, url=True)) + self.expect_eq(got, want, f"encoding-type=url prefix={prefix!r}") + root = self.s3.list_page( + self.bucket, + [("list-type", "2"), ("encoding-type", "url"), ("prefix", prefix), ("delimiter", "/"), ("start-after", prefix + "a b")], + ) + for name, value in (("Prefix", prefix), ("Delimiter", "/"), ("StartAfter", prefix + "a b")): + raw = text(root, name) + assert raw is not None and urllib.parse.unquote_plus(raw) == value, f"encoded {name} {raw!r} != {value!r}" + assert text(root, "EncodingType") == "url", "EncodingType not echoed" + + +def deleted_prefixes(s3: S3, bucket: str) -> None: + """A prefix whose last key was deleted is no longer a CommonPrefix.""" + s3.create_bucket(bucket) + keys = ["keep", "gone/a/b", "gone/c", "half/x", "half/y/z"] + for k in keys: + s3.put_object(bucket, k, b"x") + for k in ("gone/a/b", "gone/c", "half/y/z"): + s3.delete_object(bucket, k) + p = Probe(s3, bucket, ["keep", "half/x"]) + p.delimiter("", "/", 1000) + p.delimiter("half/", "/", 1000) + p.delimiter("gone/", "/", 1000) + + +def run_probes( + s3: S3, buckets: dict[str, dict[str, int]], churn_bucket: str, only: re.Pattern | None +) -> list[tuple[str, str | None]]: + results: list[tuple[str, str | None]] = [] + + def check(name: str, fn, *a) -> None: + if only and not only.search(name): + return + try: + fn(*a) + results.append((name, None)) + except Exception as exc: # noqa: BLE001 + results.append((name, f"{type(exc).__name__}: {exc}")) + + check(f"reference:{churn_bucket}:deleted-prefixes", deleted_prefixes, s3, churn_bucket) + for bucket, keymap in buckets.items(): + p = Probe(s3, bucket, list(keymap)) + keys = p.keys + tag = f"reference:{bucket}" + for mk in (1, 7, 100, 1000): + check(f"{tag}:serial:max-keys={mk}", p.serial, "", mk) + check(f"{tag}:url-encoding", p.url_encoding, "") + check(f"{tag}:token-overrides-start-after", p.token_overrides_start_after) + for mk in (2, 1000): + check(f"{tag}:walk-v2:max-keys={mk}", p.walk, mk) + check(f"{tag}:walk-v1:max-keys={mk}", p.walk, mk, True) + prefixes = sorted({k[: k.index("/") + 1] for k in keys if "/" in k}) + scopes = ["", *prefixes[:: max(1, len(prefixes) // 4)]] + if bucket.endswith("edge"): + scopes += ["order/", "order/a", "chars/", "unicode/", "markers/", "markers/dir/", "photos/2006/", "o"] + for prefix in scopes: + for mk in (1, 3, 1000): + check(f"{tag}:delimiter-v2:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk) + check(f"{tag}:delimiter-v1:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk, True) + for prefix in ("", scopes[-1]): + for delim in ("", "/"): + for mk in (2, 1000): + check(f"{tag}:versions:prefix={prefix}:delimiter={delim}:max-keys={mk}", p.versions, prefix, delim, mk, False) + check(f"{tag}:versions-url:prefix={prefix}", p.versions, prefix, "/", 3, True) + for d in ("-", "a", "ou"): + check(f"{tag}:delimiter={d}", p.delimiter, "", d, 1000) + check(f"{tag}:serial:prefix-no-slash", p.serial, keys[len(keys) // 3][:4], 5) + # Cut points that are existing keys, absent keys between keys, prefixes, + # and a common prefix with and without its trailing delimiter. + step = max(1, len(keys) // 9) + existing = keys[step::step] + absent = [k + "\x7f" for k in existing[::2]] + [k[:-1] for k in existing[1::2] if len(k) > 1] + dirs = [k[: k.rfind("/") + 1] for k in existing if "/" in k] + dirs += [d.rstrip("/") for d in dirs] + for name, cuts in (("existing", existing), ("absent", absent), ("dirs", dirs), ("mixed", existing + absent + dirs)): + for mk in (3, 1000): + check(f"{tag}:ranges-{name}:max-keys={mk}", p.ranges, cuts, mk) + for sa in (keys[len(keys) // 2], prefixes[len(prefixes) // 2], prefixes[len(prefixes) // 2].rstrip("/")): + check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "", sa) + if bucket.endswith("edge"): + for sa in ("order/a", "order/a/", "order/a/b", "order/a.b", "order/a0"): + check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "order/", sa) + check(f"{tag}:url-encoding:chars/", p.url_encoding, "chars/") + return results + + +# ---------------------------------------------------------------- tool capsules + + +@dataclass +class Case: + tool: str + mode: str + bucket: str + prefix: str + status: str = "" + detail: str = "" + argv: list[str] = field(default_factory=list) + + +class ToolRunner: + def __init__(self, args, study: Path, work: Path, endpoint: str) -> None: + sys.path.insert(0, str(study / "benchmark" / "src")) + from benchmark import adapters # noqa: PLC0415 + from benchmark.runtime.command_adapter import CommandAdapterError, Stated # noqa: PLC0415 + + self.adapters, self.refusal, self.stated = adapters, CommandAdapterError, Stated + self.study, self.work, self.endpoint = study, work, endpoint + tools = Path(args.tools_dir).resolve() + self.fixture_dir = work / "fixtures" + self.path_map = { + "/usr/local/bin/aws": str(tools / "aws-cli-bin" / "aws"), + "/usr/bin/mc": str(tools / "bin" / "mc"), + "/usr/local/bin/pS3": str(tools / "bin" / "pS3"), + "/usr/local/bin/rclone": str(tools / "bin" / "rclone"), + "/usr/bin/s3-fast-list": str(tools / "bin" / "s3-fast-list"), + "/usr/bin/ks-tool": str(tools / "bin" / "ks-tool"), + "/usr/local/bin/s3kor": str(tools / "bin" / "s3kor"), + "/usr/local/bin/s3p": str(tools / "bin" / "s3p"), + "/usr/local/bin/s4cmd": str(tools / "bin" / "s4cmd"), + "/s5cmd": str(tools / "bin" / "s5cmd"), + "/usr/local/bin/s7cmd": str(tools / "bin" / "s7cmd"), + "/opt/java/openjdk/bin/java": str(tools / "swath" / "openjdk" / "bin" / "java"), + "/opt/swath/swath.jar": str(tools / "swath" / "swath.jar"), + "/usr/bin/python3": sys.executable, + "/opt/benchmark/tools/": str(study / "tools") + "/", + } + os.environ["PYTHONPATH"] = str(study / "benchmark" / "src") + self.env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "AWS_ACCESS_KEY_ID": args.access_key, + "AWS_SECRET_ACCESS_KEY": args.secret_key, + "AWS_REGION": REGION, + "AWS_DEFAULT_REGION": REGION, + "AWS_EC2_METADATA_DISABLED": "true", + "TZ": "UTC", + "LC_ALL": "C.UTF-8", + "PYTHONPATH": str(study / "benchmark" / "src"), + } + + def localize(self, argv, bucket: str) -> list[str]: + out = [] + path_map = {**self.path_map, "/fixtures/source/": f"{self.fixture_dir / bucket}/"} + for token in argv: + for src, dst in path_map.items(): + if token == src or (src.endswith("/") and token.startswith(src)): + token = dst + token[len(src) :] + break + out.append(token) + return out + + def adapter_dir(self, tool: str) -> Path: + return self.study / "tools" / tool / "adapter" + + def write_fixture_companions(self, bucket: str, keys: list[str]) -> None: + """The staged-fixture companions, built the way fixture_bundle.py builds them.""" + import duckdb # noqa: PLC0415 + + out = self.fixture_dir / bucket + shutil.rmtree(out, ignore_errors=True) + out.mkdir(parents=True) + shards = sorted({k.split("/", 1)[0] for k in keys}) + if all(re.fullmatch(r"[A-Za-z0-9._/-]+", s) for s in shards): + (out / "s5cmd-shards.input").write_text("".join(f"{s}\n" for s in shards)) + parts = out / "parquet" + parts.mkdir() + with duckdb.connect() as con: + con.execute("CREATE TABLE f (key BLOB, row_type VARCHAR)") + con.executemany("INSERT INTO f VALUES (?, 'OBJECT')", [(k.encode(),) for k in keys]) + con.execute(f"COPY f TO '{parts / 'part-00000.parquet'}' (FORMAT parquet)") + sys.path.insert(0, str(self.study / "tools" / "s3-fast-list" / "adapter")) + from fixture_hints import FixtureHintsError, generate_hints # noqa: PLC0415 + + try: + generate_hints(parts, out / "s3-fast-list-hints.input", segments=4) + except FixtureHintsError as exc: + print(f"{bucket}: no s3-fast-list fixture hints: {exc}", flush=True) + + def inject_endpoint(self, tool: str, argv: list[str], env: dict) -> tuple[list[str], dict]: + """Add the endpoint the way the replay-endpoint pilot (run.sh) did, for modes + the capsule refuses only because the replay server lacks the API.""" + ep = self.endpoint + if tool == "rclone": + return [a.replace(f"region={REGION}", f'region={REGION},endpoint="{ep}",force_path_style=true') for a in argv], env + if tool == "minio-mc": + return argv, {**env, "MC_HOST_s3": ep, "MC_REGION": REGION} + if tool == "s3kor": + return [argv[0], argv[1], "--custom-endpoint-url", ep, *argv[2:]], env + if tool == "s7cmd": + return [*argv[:-1], "--target-endpoint-url", ep, "--target-force-path-style", argv[-1]], env + if tool == "s5cmd": + return [argv[0], "--endpoint-url", ep, *argv[1:]], env + return [*argv, "--endpoint-url", ep], env + + def config_for(self, adapter, case: Case, mode: str, keys: list[str]) -> dict[str, object]: + config: dict[str, object] = {} + for name, axis in adapter.modes[mode].axes.items(): + if isinstance(axis, self.stated): + config[name] = 4 + if adapter.tool == "s5cmd" and mode == "fanout-with-dirs": + initials = sorted({k[len(case.prefix)] for k in keys if k.startswith(case.prefix) and len(k) > len(case.prefix)}) + if not all(re.fullmatch(r"[A-Za-z0-9._-]", c) for c in initials): + raise NotApplicable(f"key initials {initials} are outside the capsule's safe shard set") + config["shard_initials"] = "".join(initials) + return config + + def execute(self, adapter, case: Case, mode: str, sink: Path, keys: list[str], artifact_path: str = "") -> tuple[Path | None, dict]: + signed = adapter.supports_signed + inject = case.tool in INJECT_ENDPOINT and mode in INJECT_ENDPOINT[case.tool] + config = self.config_for(adapter, case, mode, keys) + argv, env = self.adapters.compile_command( + self.adapter_dir(case.tool), + case.tool, + mode=mode, + bucket=case.bucket, + region=REGION, + prefix=case.prefix, + signed=signed, + config=config, + sink_dir=str(sink), + artifact_path=artifact_path, + visible_memory_gb=2.0, + endpoint_url="" if inject else self.endpoint, + ) + if inject: + argv, env = self.inject_endpoint(case.tool, list(argv), env) + case.detail = "endpoint injected; " + argv = self.localize(argv, case.bucket) + case.argv = argv + sink.mkdir(parents=True, exist_ok=True) + home = sink / "home" + home.mkdir(exist_ok=True) + manifest = adapter.modes[mode] + stdout_path = sink / (manifest.product_file if manifest.product_channel == "stdout" and manifest.product_artifact else "stdout") + with stdout_path.open("wb") as out, (sink / "stderr").open("wb") as err: + proc = subprocess.run( + argv, + stdout=out, + stderr=err, + env={**self.env, **env, "HOME": str(home)}, + cwd=sink, + timeout=300, + check=False, + ) + if proc.returncode != 0: + tail = (sink / "stderr").read_bytes()[-600:].decode(errors="replace") + raise RuntimeError(f"exit {proc.returncode}: {tail}") + product = sink / manifest.product_file if manifest.product_artifact else None + return product, config + + def run_case(self, case: Case, expected: dict[str, tuple[int, str]]) -> None: + adapter = self.adapters.load_adapter(self.adapter_dir(case.tool), case.tool) + manifest = adapter.modes[case.mode] + if manifest.purpose_ceiling == "preparation" or not manifest.product_artifact or (case.tool, case.mode) in NOT_A_LISTING: + case.status = "skip" + case.detail = f"{manifest.purpose_ceiling} mode, not a compared listing" + return + if (case.tool, case.mode) in ROOT_ONLY and case.prefix: + case.status, case.detail = "skip", "mode lists the bucket root whatever the prefix" + return + fixture = self.fixture_dir / case.bucket + needs = {"fanout-fixture-with-dirs": "s5cmd-shards.input", "list-hinted-fixture": "s3-fast-list-hints.input"}.get(case.mode) + if needs and not (fixture / needs).exists(): + case.status, case.detail = "skip", f"fixture_bundle.py would not build {needs} for this key set" + return + keys = list(expected) + sink = self.work / "cases" / case.tool / case.mode / (case.bucket + "~" + (case.prefix.replace("/", "_") or "root")) + shutil.rmtree(sink, ignore_errors=True) + try: + artifact_path = "" + for step in adapter.requires.get(case.mode, ()): + producer_sink = sink / f"requires-{step.mode}" + self.execute(adapter, case, step.mode, producer_sink, keys) + artifact_path = str(producer_sink / adapter.modes[step.mode].artifacts[step.artifact]) + if manifest.inline: + inline_sink = sink / f"inline-{manifest.inline}" + self.execute(adapter, case, manifest.inline, inline_sink, keys, artifact_path) + inline_manifest = adapter.modes[manifest.inline] + artifact_path = str(inline_sink / next(iter(inline_manifest.artifacts.values()))) + product, config = self.execute(adapter, case, case.mode, sink, keys, artifact_path) + except NotApplicable as exc: + case.status, case.detail = "skip", str(exc) + return + except self.adapters.AdapterError as exc: + if "custom endpoint" in str(exc) or "replay endpoint" in str(exc) or "no --prefix" in str(exc) or "requires HeadBucket" in str(exc): + case.status, case.detail = "skip", f"capsule refuses: {str(exc).split(': ', 2)[-1][:160]}" + return + raise + normalized = sink / "normalized.tsv" + self.adapters.normalize_to_path( + self.adapter_dir(case.tool), + case.tool, + case.mode, + case.prefix, + normalized, + input_path=None if manifest.product_channel == "dataset" else product, + dataset=product if manifest.product_channel == "dataset" else None, + config=config, + ) + compare(case, normalized.read_bytes(), expected, manifest.fields) + + +INJECT_ENDPOINT = { + "aws-cli": {"s3api-v1-text", "s3api-versions-text"}, + "minio-mc": {"versions-json", "find", "find-json"}, + "rclone": {"listv1"}, + "s3kor": {"list-versions"}, + "s4cmd": {"recursive", "shallow", "show-directory", "du"}, + "s5cmd": {"listv1", "allversions"}, + "s7cmd": {"all-versions"}, +} +"""Capsule modes refused only because the study's replay server serves +ListObjectsV2 alone; SeaweedFS serves ListObjects, ListObjectVersions and +HeadBucket too.""" + +NOT_A_LISTING = { + ("rclone", "debug"), + ("rclone", "walk-debug"), + ("s3p", "summarize"), + ("s4cmd", "du"), + ("s4cmd", "show-directory"), +} +"""Modes with no per-key listing: rclone's normalizer has no debug modes, summarize +and du print one aggregate, and `ls -d` prints the directory itself.""" + + +class NotApplicable(Exception): + pass + + +ROOT_ONLY = { + ("aws-cli", "s3api-v2-remainder"), + ("s5cmd", "fanout-fixture-with-dirs"), + ("s3-fast-list", "list-hinted-fixture"), +} +"""Modes whose capsule lists the root (remainder) or a staged root-scope fixture.""" + +SHALLOW = re.compile(r"shallow|delimiter|rootkeys|remainder|max-depth") +KEYS_ONLY = re.compile(r"rootkeys|remainder") +"""Shallow modes whose normalizer keeps only the keys, dropping CommonPrefixes.""" + + +def compare(case: Case, normalized: bytes, expected: dict[str, tuple[int, str]], fields) -> None: + rows = [] + for line in normalized.split(b"\n"): + if not line: + continue + f = line.split(b"\t") + rows.append((f[0].decode("utf-8", "surrogateescape"), f[1].decode(), f[2].decode())) + scope = {k: v for k, v in expected.items() if k.startswith(case.prefix)} + keys = utf8_sorted(scope) + shallow = bool(SHALLOW.search(case.mode)) + if shallow: + view = direct_view(keys, case.prefix, "/") + want_objects = [k for kind, k in view if kind == "key"] + allowed_dirs = {k for kind, k in view if kind == "prefix"} + else: + want_objects = keys + allowed_dirs = {k[: i + 1] for k in keys for i, c in enumerate(k) if c == "/" and i >= len(case.prefix)} + problems: list[str] = [] + objects, dirs = [], [] + for key, size, etag in rows: + is_dir = key.endswith("/") and size == "-" and etag == "-" and key not in scope + (dirs if is_dir else objects).append((key, size, etag)) + got_keys = [k for k, _, _ in objects] + dup = {k for k in got_keys if got_keys.count(k) > 1} if len(got_keys) != len(set(got_keys)) else set() + if dup: + problems.append(f"duplicate keys {sorted(dup)[:5]}") + missing = [k for k in want_objects if k not in set(got_keys)] + extra = [k for k in set(got_keys) if k not in set(want_objects)] + if missing: + problems.append(f"missing {len(missing)}: {missing[:5]}") + if extra: + problems.append(f"unexpected {len(extra)}: {sorted(extra)[:5]}") + bad_dirs = [d for d, _, _ in dirs if d not in allowed_dirs] + if bad_dirs: + problems.append(f"unexpected directory rows {bad_dirs[:5]}") + if shallow and not KEYS_ONLY.search(case.mode) and {d for d, _, _ in dirs} != allowed_dirs: + problems.append(f"common prefixes {sorted(d for d, _, _ in dirs)[:8]} != {sorted(allowed_dirs)[:8]}") + for key, size, etag in objects: + if key not in scope: + continue + want_size, want_etag = scope[key] + if size != "-" and size != str(want_size): + problems.append(f"{key!r} size {size} != {want_size}") + if etag != "-" and etag != want_etag: + problems.append(f"{key!r} etag {etag} != {want_etag}") + case.status = "fail" if problems else "pass" + case.detail += "; ".join(problems)[:1500] if problems else f"{len(objects)} objects, {len(dirs)} prefixes" + + +# ---------------------------------------------------------------- main + + +def load_expected_failures(path: Path | None) -> dict[str, str]: + out: dict[str, str] = {} + if path and path.exists(): + for line in path.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + name, _, reason = line.partition("#") + out[name.strip()] = reason.strip() + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--endpoint", required=True) + ap.add_argument("--access-key", required=True) + ap.add_argument("--secret-key", required=True) + ap.add_argument("--study", required=True) + ap.add_argument("--tools-dir", required=True) + ap.add_argument("--work-dir", required=True) + ap.add_argument("--tools", default="aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath") + ap.add_argument("--bucket-prefix", default="study") + ap.add_argument("--expected-failures", type=Path) + ap.add_argument("--skip-upload", action="store_true") + ap.add_argument("--only", default="", help="regex over case names") + ap.add_argument("--jobs", type=int, default=4) + args = ap.parse_args() + + study = Path(args.study).resolve() + work = Path(args.work_dir).resolve() + work.mkdir(parents=True, exist_ok=True) + s3 = S3(args.endpoint, args.access_key, args.secret_key) + buckets = {f"{args.bucket_prefix}-canary": canary_keys(), f"{args.bucket_prefix}-edge": edge_keys()} + expected: dict[str, dict[str, tuple[int, str]]] = {} + for bucket, keymap in buckets.items(): + expected[bucket] = {k: (size, hashlib.md5(body_for(k, size)).hexdigest()) for k, size in keymap.items()} + if args.skip_upload: + continue + s3.create_bucket(bucket) + start = time.time() + with concurrent.futures.ThreadPoolExecutor(16) as pool: + list(pool.map(lambda k: s3.put_object(bucket, k, body_for(k, keymap[k])), keymap)) + print(f"uploaded {len(keymap)} objects to {bucket} in {time.time() - start:.1f}s", flush=True) + + only = re.compile(args.only) if args.only else None + results: list[tuple[str, str, str]] = [] + for name, err in run_probes(s3, buckets, f"{args.bucket_prefix}-churn", only): + results.append((name, "fail" if err else "pass", err or "")) + + runner = ToolRunner(args, study, work, args.endpoint) + cases: list[Case] = [] + for bucket in buckets: + runner.write_fixture_companions(bucket, list(buckets[bucket])) + scopes = ["", "group-07/"] if bucket.endswith("canary") else ["", "order/", "chars/"] + for tool in filter(None, args.tools.split(",")): + adapter = runner.adapters.load_adapter(runner.adapter_dir(tool), tool) + for mode in adapter.modes: + for prefix in scopes: + case = Case(tool, mode, bucket, prefix) + if only and not only.search(case_name(case)): + continue + cases.append(case) + + def run(case: Case) -> Case: + start = time.time() + try: + runner.run_case(case, expected[case.bucket]) + except Exception as exc: # noqa: BLE001 + case.status = "fail" + case.detail = f"{type(exc).__name__}: {exc}"[:1500] + if os.environ.get("STUDY_DEBUG"): + traceback.print_exc() + print(f"{case.status:5} {case_name(case)} ({time.time() - start:.1f}s) {case.detail[:300]}", flush=True) + return case + + with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool: + for case in pool.map(run, cases): + results.append((case_name(case), case.status, case.detail)) + + xfail = load_expected_failures(args.expected_failures) + regressions, fixed = [], [] + counts: dict[str, int] = {} + for name, status, detail in results: + if status == "fail" and name in xfail: + status = "xfail" + elif status == "pass" and name in xfail: + fixed.append(name) + elif status == "fail": + regressions.append((name, detail)) + counts[status] = counts.get(status, 0) + 1 + (work / "results.json").write_text(json.dumps(results, indent=1, ensure_ascii=False)) + print("\n== summary:", ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))) + for name in fixed: + print(f"XPASS (remove from expected failures): {name}") + for name, detail in regressions: + print(f"FAIL {name}: {detail}") + return 1 if regressions or fixed else 0 + + +def case_name(case: Case) -> str: + return f"{case.tool}:{case.mode}:{case.bucket.rsplit('-', 1)[-1]}:{case.prefix or '/'}" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/s3/listing_study/s3.json b/test/s3/listing_study/s3.json new file mode 100644 index 0000000000..addd184326 --- /dev/null +++ b/test/s3/listing_study/s3.json @@ -0,0 +1,18 @@ +{ + "identities": [ + { + "name": "listing_study_admin", + "credentials": [ + { + "accessKey": "listing_study_access", + "secretKey": "listing_study_secret" + } + ], + "actions": ["Admin", "Read", "List", "Tagging", "Write"] + }, + { + "name": "anonymous", + "actions": ["Read", "List"] + } + ] +}