mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* test(s3): s3-listing-study integration suite (WIP) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): run the s3-listing-study tools against SeaweedFS (WIP) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): s3-listing-study deleted-prefix probe, fresh data dir per run Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3): listing study ports keep clear of every gRPC companion free_ports.py compared only base ports, so a base port could land on another server's base+10000 gRPC listener. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study ports stay below the ephemeral range Base ports of 20000-55000 put both them and their gRPC companions inside 32768-60999, so an outgoing connection made while weed server starts could take a port before the server that was given it binds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study checks the order inside each listing page parse_page sorted Contents and CommonPrefixes together before the probes compared them, so keys returned out of order within one page passed. Check each list's byte order first; the versions pager does the same. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study applies --only before running a probe The filter was applied to the probe results, so a run limited to one tool still sent every reference listing request and the deleted-prefix check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(s3): listing study requires CommonPrefixes from shallow modes The CommonPrefixes comparison ran only when a tool returned at least one prefix, so a delimiter listing of the canary root that came back empty passed. Compare them for every shallow mode except the two whose normalizer keeps keys only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
88 lines
3.5 KiB
Makefile
88 lines
3.5 KiB
Makefile
# Runs the listing tools and listing strategies of
|
|
# https://github.com/varveio/s3-listing-study against a local SeaweedFS.
|
|
#
|
|
# make test build weed, fetch the study and its pinned tools, start
|
|
# weed server on free ports, run, stop
|
|
# make run ENDPOINT=... run against an already running endpoint
|
|
#
|
|
# Needs: go, python3, node/npm, cargo, docker (swath image), curl.
|
|
#
|
|
# The server runs with -s3.allowBackslashInKey, the AWS-compatible key mode: by
|
|
# default SeaweedFS rewrites '\' in object keys to '/', and the edge key set
|
|
# holds a key with a backslash.
|
|
|
|
STUDY_REPO ?= https://github.com/varveio/s3-listing-study.git
|
|
STUDY_REV ?= 666ec620d245e16baaaa0cf53461382f42c35917
|
|
WORK_DIR ?= $(CURDIR)/.work
|
|
STUDY_DIR ?= $(WORK_DIR)/s3-listing-study
|
|
TOOLS_DIR ?= $(WORK_DIR)/tools
|
|
VENV ?= $(WORK_DIR)/venv
|
|
WEED_BINARY ?= $(WORK_DIR)/weed
|
|
TOOLS ?= aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath
|
|
ACCESS_KEY ?= listing_study_access
|
|
SECRET_KEY ?= listing_study_secret
|
|
JOBS ?= 4
|
|
ONLY ?=
|
|
|
|
.PHONY: test run build-weed study venv tools start-server stop-server clean
|
|
|
|
test: build-weed tools
|
|
@$(MAKE) --no-print-directory start-server; status=$$?; \
|
|
if [ $$status -eq 0 ]; then \
|
|
$(MAKE) --no-print-directory run ENDPOINT=http://127.0.0.1:$$(cat $(WORK_DIR)/s3.port); status=$$?; \
|
|
fi; \
|
|
$(MAKE) --no-print-directory stop-server; exit $$status
|
|
|
|
build-weed:
|
|
mkdir -p $(WORK_DIR)
|
|
cd ../../../weed && go build -buildvcs=false -o $(WEED_BINARY) .
|
|
|
|
study:
|
|
@if [ "$$(git -C $(STUDY_DIR) rev-parse HEAD 2>/dev/null)" != "$(STUDY_REV)" ]; then \
|
|
git init -q $(STUDY_DIR) && \
|
|
git -C $(STUDY_DIR) fetch -q --depth 1 $(STUDY_REPO) $(STUDY_REV) && \
|
|
git -C $(STUDY_DIR) checkout -q --force FETCH_HEAD; \
|
|
fi
|
|
|
|
venv:
|
|
@test -x $(VENV)/bin/python || python3 -m venv $(VENV)
|
|
$(VENV)/bin/pip install -q -r requirements.txt
|
|
|
|
tools: study venv
|
|
$(VENV)/bin/python install_tools.py --study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --tools $(TOOLS)
|
|
|
|
start-server:
|
|
@mkdir -p $(WORK_DIR)
|
|
@set -e; data=$$(mktemp -d $(WORK_DIR)/data.XXXXXX); ports=$$(python3 free_ports.py 5); \
|
|
set -- $$ports; \
|
|
$(WEED_BINARY) server -filer -s3 -ip 127.0.0.1 -ip.bind 127.0.0.1 -dir=$$data \
|
|
-master.peers=none -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
|
|
-volume.max=50 -volume.preStopSeconds=1 \
|
|
-master.port=$$1 -volume.port=$$2 -filer.port=$$3 -s3.port=$$4 -metricsPort=$$5 \
|
|
-s3.port.iceberg=0 -s3.port.lance=0 -s3.allowBackslashInKey -s3.config=$(CURDIR)/s3.json \
|
|
> $(WORK_DIR)/weed.log 2>&1 & echo $$! > $(WORK_DIR)/weed.pid; \
|
|
echo $$4 > $(WORK_DIR)/s3.port; \
|
|
for i in $$(seq 1 90); do \
|
|
if curl -sf -o /dev/null http://127.0.0.1:$$3/ && curl -s -o /dev/null http://127.0.0.1:$$4/; then \
|
|
echo "SeaweedFS S3 ready on port $$4"; exit 0; fi; \
|
|
kill -0 $$(cat $(WORK_DIR)/weed.pid) || { tail -50 $(WORK_DIR)/weed.log; exit 1; }; \
|
|
sleep 1; \
|
|
done; tail -50 $(WORK_DIR)/weed.log; exit 1
|
|
|
|
stop-server:
|
|
@if [ -f $(WORK_DIR)/weed.pid ]; then \
|
|
pid=$$(cat $(WORK_DIR)/weed.pid); kill $$pid 2>/dev/null || true; \
|
|
for i in $$(seq 1 20); do kill -0 $$pid 2>/dev/null || break; sleep 0.5; done; \
|
|
rm -f $(WORK_DIR)/weed.pid; \
|
|
fi
|
|
|
|
run: study
|
|
$(VENV)/bin/python run_study.py --endpoint $(ENDPOINT) \
|
|
--access-key $(ACCESS_KEY) --secret-key $(SECRET_KEY) \
|
|
--study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --work-dir $(WORK_DIR)/run \
|
|
--tools $(TOOLS) --jobs $(JOBS) --expected-failures expected_failures.txt \
|
|
$(if $(ONLY),--only '$(ONLY)')
|
|
|
|
clean: stop-server
|
|
rm -rf .work
|