From dcbea0626fce7ef1409a7236c57430e8b6e4f42d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 19 Nov 2026 12:21:32 -0800 Subject: [PATCH] test/s3: patch s3-fast-list's data map race in the listing study build (#12766) test/s3: build s3-fast-list with its data map draining the last page The data map task reads the channel and then the list-task state. When the list task sends its last page and completes between the two reads, the map is dumped without that page and the tool exits 0, so a one-page prefix comes back empty. Read the state before the channel. Co-authored-by: Claude Opus 5.5 --- test/s3/listing_study/install_tools.py | 38 ++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/test/s3/listing_study/install_tools.py b/test/s3/listing_study/install_tools.py index dbce02aa51..0f003555cf 100644 --- a/test/s3/listing_study/install_tools.py +++ b/test/s3/listing_study/install_tools.py @@ -38,6 +38,38 @@ ALL_TOOLS = ( MC_COMMIT = "d6541ea280b73a834b64d4097e21f2be77676104" """github.com/minio/mc tag RELEASE.2025-08-13T08-35-41Z.""" +SOURCE_PATCHES = { + "s3-fast-list": ( + "s3-fast-list/src/data_map.rs", + ( + ( + " match ctx.data_map_channel.try_recv() {", + " let list_tasks_running = ctx.all_list_tasks_is_running();\n" + " match ctx.data_map_channel.try_recv() {", + ), + ( + "!ctx.all_list_tasks_is_running() && !has_more_in_queue", + "!list_tasks_running && !has_more_in_queue", + ), + ), + ), +} +"""Source fixes applied before a build. s3-fast-list's data map reads the channel +and then the list-task state, so a last page sent and completed between the two +reads is dropped and the run exits 0 with a short or empty listing. Reading the +state first means a finished task's pages are already in the channel.""" + + +def patch_source(tool: str, work: Path) -> None: + rel, edits = SOURCE_PATCHES[tool] + path = work / rel + text = path.read_text() + for old, new in edits: + if text.count(old) != 1: + raise SystemExit(f"{tool}: {rel} no longer matches its source patch: {old!r}") + text = text.replace(old, new) + path.write_text(text) + def fetch(url: str, sha256: str, dest: Path) -> Path: if dest.exists() and hashlib.sha256(dest.read_bytes()).hexdigest() == sha256: @@ -84,7 +116,8 @@ def install(tool: str, study: Path, tools_dir: Path) -> None: bin_dir.mkdir(parents=True, exist_ok=True) name = url.rsplit("/", 1)[-1] stamp = tools_dir / "installed" / tool - if stamp.exists() and stamp.read_text() == sha: + installed = sha + ("+patched" if tool in SOURCE_PATCHES else "") + if stamp.exists() and stamp.read_text() == installed: print(f"{tool} {image.get('tool_version', '')} already installed") return @@ -172,6 +205,7 @@ def install(tool: str, study: Path, tools_dir: Path) -> None: work.mkdir(parents=True) subprocess.run(["tar", "xzf", str(archive), "--strip-components=1", "-C", str(work)], check=True) shutil.copyfile(study / "tools/s3-fast-list/build/Cargo.lock", work / "Cargo.lock") + patch_source(tool, work) subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=work, check=True) for exe in ("s3-fast-list", "ks-tool"): shutil.copyfile(work / "target/release" / exe, bin_dir / exe) @@ -193,7 +227,7 @@ def install(tool: str, study: Path, tools_dir: Path) -> None: else: raise SystemExit(f"unknown tool {tool}") stamp.parent.mkdir(parents=True, exist_ok=True) - stamp.write_text(sha) + stamp.write_text(installed) print(f"installed {tool} {image.get('tool_version', '')}")