From 8ac5133abeda188a5b63d98c7a4eb00656fc39a3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 18 Oct 2026 21:21:05 -0700 Subject: [PATCH] admin: default to port 12535, keep 23646 as a best-effort compat listener (#11816) * admin: default to port 12535, keep 23646 as a best-effort compat listener The default admin HTTP port 23646 derives a worker gRPC port of 33646 (port + GrpcPortOffset), inside the Linux default ephemeral range (32768+). One of our own startup dials can claim 33646 before the admin binds it, killing startup. This is the same failure #10209 fixed for the test port allocator, still reachable via the mini default (#10838) and via standalone `weed admin` defaults. Move the default admin port to 12535 (gRPC 22535) everywhere, keeping the http+10000 convention so worker -admin derivation, the Rust worker and host:http.grpc addressing keep working: - weed admin -port, weed mini -admin.port - weed worker -admin, the plugin worker fallback, Rust worker and weed-lance-worker defaults - help text, README, helm values, docker compose, tests So old workers, bookmarks and scripts keep working, admin also serves the same handler and the worker gRPC server on the previous default (-port.legacy / -admin.port.legacy, default 23646, 0 disables). The legacy gRPC port binds best-effort: it lives in the ephemeral range, so a failed bind only warns instead of reintroducing the startup failure. weed mini reserves it early like the primary port and keeps it out of its own port relocation. grpc.Server tracks every listener handed to Serve, so the compat listener stops with the primary. Also warn when an explicitly configured admin port puts the derived gRPC port at or above the ephemeral floor, the other fix suggested in the issue. Supersedes the pinned-grpc approach in #11814: pinning -admin.port.grpc off the +10000 convention breaks external workers that derive the admin gRPC port from the HTTP address. ClickHouse/ClickHouse#115512 pins -admin.port on their side; this makes that pin unnecessary. Fixes #10838 * admin: let services win the legacy ports, close compat listener on failure Review findings on the compat listeners: - Do not put the legacy ports in mini's reservedPorts: an explicit -s3.port=23646 or -master.port.grpc=33646 must be honored, not rejected or stolen by the early reservation. Mini now checks the finalized port set and skips only the conflicting compat listener, so HTTP and gRPC compat degrade independently and the service keeps its port. - Also skip the legacy HTTP listener when a mini service owns the port: a wildcard-bound service can still leave loopback bindable, which would let the compat listener shadow it for loopback clients. - Close legacyServer's raw listener in the deferred cleanup: Serve may not have registered it yet when startAdminServer returns early. - Helm: add admin.legacyPort (default 23646) and expose the legacy http/grpc ports through the admin Service, container ports and NetworkPolicy so older workers resolved via the Service keep working. * helm: only pass -port.legacy when it differs from the binary default ct install deploys the released image with the new chart, and 4.48 does not know -port.legacy, so the admin pod crash-loops. Omitting the flag at the default value is behavior-identical on new binaries (their default is also 23646) and keeps older releases installable; legacyPort: 0 or a custom port still renders the flag. --- design-lance-catalog.md | 2 +- docker/admin_integration/EC-TESTING-README.md | 6 +- docker/admin_integration/Makefile | 30 +++--- .../create_vacuum_test_data.go | 4 +- .../admin_integration/demo_vacuum_testing.sh | 10 +- .../docker-compose-ec-test.yml | 16 +-- docker/admin_integration/test-integration.sh | 4 +- docker/seaweedfs-dev-compose.yml | 2 +- install.sh | 2 +- k8s/charts/seaweedfs/README.md | 19 ++-- .../templates/admin/admin-service.yaml | 10 ++ .../templates/admin/admin-statefulset.yaml | 9 ++ .../templates/shared/networkpolicy.yaml | 6 +- k8s/charts/seaweedfs/values.yaml | 10 +- seaweed-worker/README.md | 8 +- seaweed-worker/crates/core/src/config.rs | 4 +- seaweed-worker/crates/core/src/lib.rs | 2 +- seaweed-worker/crates/lance/src/main.rs | 2 +- terraform/modules/aws/security_group.tf | 2 +- .../admin_dockertest/ec_integration_test.go | 6 +- test/s3/lifecycle/Makefile | 2 +- test/s3/lifecycle/s3_lifecycle_test.go | 2 +- test/s3/policy/policy_test.go | 4 +- test/testutil/ports.go | 2 +- test/testutil/ports_test.go | 2 +- weed/admin/dash/admin_server.go | 8 ++ weed/admin/dash/worker_grpc_server.go | 11 ++ weed/admin/plugin/DESIGN.md | 6 +- weed/command/admin.go | 102 ++++++++++++++++-- weed/command/mini.go | 69 +++++++++--- weed/command/mini_admin_auth_test.go | 30 ++++++ weed/command/worker.go | 23 ++-- weed/command/worker_runtime.go | 2 +- 33 files changed, 316 insertions(+), 101 deletions(-) diff --git a/design-lance-catalog.md b/design-lance-catalog.md index ce657b98a3..a942ed6ad2 100644 --- a/design-lance-catalog.md +++ b/design-lance-catalog.md @@ -543,7 +543,7 @@ a language-agnostic gRPC stream that external maintenance workers connect on: th opens `WorkerStream`, sends `WorkerHello` with the job types it can `detect` and `execute`, answers `RequestConfigSchema` with a `JobTypeDescriptor`, replies to `RunDetectionRequest` with `JobProposal`s and to `ExecuteJobRequest` with `JobProgressUpdate`s and `JobCompleted`. -`weed worker -admin=host:23646` is the Go reference implementation of exactly that contract, +`weed worker -admin=host:12535` is the Go reference implementation of exactly that contract, from outside the admin process. Nothing in it is Go-specific, and the Rust toolchain is already in the tree. diff --git a/docker/admin_integration/EC-TESTING-README.md b/docker/admin_integration/EC-TESTING-README.md index 57e0a59854..e9e13d8530 100644 --- a/docker/admin_integration/EC-TESTING-README.md +++ b/docker/admin_integration/EC-TESTING-README.md @@ -21,7 +21,7 @@ The testing environment uses **official SeaweedFS commands** and includes: - **1 Master Server** (port 9333) - Coordinates the cluster with 50MB volume size limit - **6 Volume Servers** (ports 8080-8085) - Distributed across 2 data centers and 3 racks for diversity - **1 Filer** (port 8888) - Provides file system interface -- **1 Admin Server** (port 23646) - Detects volumes needing EC and manages workers using official `admin` command +- **1 Admin Server** (port 12535) - Detects volumes needing EC and manages workers using official `admin` command - **3 EC Workers** - Execute erasure coding tasks using official `worker` command with task-specific working directories - **1 Load Generator** - Continuously writes and deletes files using SeaweedFS shell commands - **1 Monitor** - Tracks cluster health and EC progress using shell scripts @@ -47,7 +47,7 @@ This provides: - Docker and Docker Compose installed - GNU Make installed - At least 4GB RAM available for containers -- Ports 8080-8085, 8888, 9333, 23646 available +- Ports 8080-8085, 8888, 9333, 12535 available ### Start the Environment @@ -127,7 +127,7 @@ Run `make help` to see all available targets: |---------|-----|-------------| | Master UI | http://localhost:9333 | Cluster status and topology | | Filer | http://localhost:8888 | File operations | -| Admin Server | http://localhost:23646/ | Task management | +| Admin Server | http://localhost:12535/ | Task management | | Monitor | http://localhost:9999/status | Complete cluster monitoring | | Volume Servers | http://localhost:8080-8085/status | Individual volume server stats | diff --git a/docker/admin_integration/Makefile b/docker/admin_integration/Makefile index 68fb0cec6f..2997143aa6 100644 --- a/docker/admin_integration/Makefile +++ b/docker/admin_integration/Makefile @@ -26,7 +26,7 @@ build-and-restart: build ## Build with latest changes and restart services @echo "2๏ธโƒฃ Recreating workers to reconnect..." @docker-compose -f $(COMPOSE_FILE) up -d worker1 worker2 worker3 @echo "โœ… All services recreated with latest changes!" - @echo "๐ŸŒ Admin UI: http://localhost:23646/" + @echo "๐ŸŒ Admin UI: http://localhost:12535/" @echo "๐Ÿ’ก Workers will reconnect to the new admin server" restart-workers: ## Restart all workers to reconnect to admin server @@ -64,7 +64,7 @@ start: ## Start the complete SeaweedFS cluster with admin and workers @echo "โœ… Cluster started!" @echo "" @echo "๐Ÿ“Š Access points:" - @echo " โ€ข Admin UI: http://localhost:23646/" + @echo " โ€ข Admin UI: http://localhost:12535/" @echo " โ€ข Master UI: http://localhost:9333/" @echo " โ€ข Filer: http://localhost:8888/" @echo "" @@ -107,7 +107,7 @@ start-staged: ## Start services in proper order with delays @echo "โœ… All services started!" @echo "" @echo "๐Ÿ“Š Access points:" - @echo " โ€ข Admin UI: http://localhost:23646/" + @echo " โ€ข Admin UI: http://localhost:12535/" @echo " โ€ข Master UI: http://localhost:9333/" @echo " โ€ข Filer: http://localhost:8888/" @echo "" @@ -136,7 +136,7 @@ status: ## Check the status of all services @echo "Master:" @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' 2>/dev/null || echo " โŒ Master not ready" @echo "Admin:" - @curl -s http://localhost:23646/ | grep -q "Admin" && echo " โœ… Admin ready" || echo " โŒ Admin not ready" + @curl -s http://localhost:12535/ | grep -q "Admin" && echo " โœ… Admin ready" || echo " โŒ Admin not ready" logs: ## Show logs from all services @echo "๐Ÿ“œ Following logs from all services..." @@ -157,7 +157,7 @@ master-logs: ## Show logs from master server admin-ui: ## Open admin UI in browser (macOS) @echo "๐ŸŒ Opening admin UI in browser..." - @open http://localhost:23646/ || echo "๐Ÿ’ก Manually open: http://localhost:23646/" + @open http://localhost:12535/ || echo "๐Ÿ’ก Manually open: http://localhost:12535/" test: ## Run integration test to verify task assignment and completion @echo "๐Ÿงช Running Admin-Worker Integration Test" @@ -166,11 +166,11 @@ test: ## Run integration test to verify task assignment and completion @echo "1๏ธโƒฃ Checking cluster health..." @sleep 5 @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "โœ… Master healthy" || echo "โŒ Master not ready" - @curl -s http://localhost:23646/ | grep -q "Admin" && echo "โœ… Admin healthy" || echo "โŒ Admin not ready" + @curl -s http://localhost:12535/ | grep -q "Admin" && echo "โœ… Admin healthy" || echo "โŒ Admin not ready" @echo "" @echo "2๏ธโƒฃ Checking worker registration..." @sleep 10 - @echo "๐Ÿ’ก Check admin UI for connected workers: http://localhost:23646/" + @echo "๐Ÿ’ก Check admin UI for connected workers: http://localhost:12535/" @echo "" @echo "3๏ธโƒฃ Generating load to trigger EC tasks..." @echo "๐Ÿ“ Creating test files to fill volumes..." @@ -193,13 +193,13 @@ test: ## Run integration test to verify task assignment and completion @echo " โ€ข EC shards being distributed" @echo "" @echo "โœ… Integration test setup complete!" - @echo "๐Ÿ“Š Monitor progress at: http://localhost:23646/" + @echo "๐Ÿ“Š Monitor progress at: http://localhost:12535/" quick-test: ## Quick verification that core services are running @echo "โšก Quick Health Check" @echo "====================" @echo "Master: $$(curl -s http://localhost:9333/cluster/status | jq -r '.IsLeader // "not ready"')" - @echo "Admin: $$(curl -s http://localhost:23646/ | grep -q "Admin" && echo "ready" || echo "not ready")" + @echo "Admin: $$(curl -s http://localhost:12535/ | grep -q "Admin" && echo "ready" || echo "not ready")" @echo "Workers: $$(docker-compose -f $(COMPOSE_FILE) ps worker1 worker2 worker3 | grep -c Up) running" validate: ## Validate integration test configuration @@ -223,7 +223,7 @@ demo: start ## Start cluster and run demonstration @echo " โ€ข Load generator (creates files continuously)" @echo "" @echo "๐Ÿ“Š Watch the process:" - @echo " 1. Visit: http://localhost:23646/" + @echo " 1. Visit: http://localhost:12535/" @echo " 2. Observe workers connecting" @echo " 3. Watch tasks being created and assigned" @echo " 4. See tasks progress from pending โ†’ completed" @@ -244,13 +244,13 @@ vacuum-test: ## Create test data with garbage and verify vacuum detection @echo "" @echo "1๏ธโƒฃ Checking cluster health..." @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "โœ… Master ready" || (echo "โŒ Master not ready. Run 'make start' first." && exit 1) - @curl -s http://localhost:23646/ | grep -q "Admin" && echo "โœ… Admin ready" || (echo "โŒ Admin not ready. Run 'make start' first." && exit 1) + @curl -s http://localhost:12535/ | grep -q "Admin" && echo "โœ… Admin ready" || (echo "โŒ Admin not ready. Run 'make start' first." && exit 1) @echo "" @echo "2๏ธโƒฃ Creating test data with garbage..." @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=25 -delete=0.5 -size=200 @echo "" @echo "3๏ธโƒฃ Configuration Instructions:" - @echo " Visit: http://localhost:23646/maintenance/config/vacuum" + @echo " Visit: http://localhost:12535/maintenance/config/vacuum" @echo " Set for testing:" @echo " โ€ข Enable Vacuum Tasks: โœ… Checked" @echo " โ€ข Garbage Threshold: 0.20 (20%)" @@ -258,7 +258,7 @@ vacuum-test: ## Create test data with garbage and verify vacuum detection @echo " โ€ข Min Volume Age: [0] [Minutes]" @echo " โ€ข Max Concurrent: 2" @echo "" - @echo "4๏ธโƒฃ Monitor vacuum tasks at: http://localhost:23646/maintenance" + @echo "4๏ธโƒฃ Monitor vacuum tasks at: http://localhost:12535/maintenance" @echo "" @echo "๐Ÿ’ก Use 'make vacuum-status' to check volume garbage ratios" @@ -335,8 +335,8 @@ vacuum-help: ## Show vacuum testing help and examples @echo " make vacuum-demo # Full guided demonstration" @echo "" @echo "Configuration:" - @echo " Visit: http://localhost:23646/maintenance/config/vacuum" - @echo " Monitor: http://localhost:23646/maintenance" + @echo " Visit: http://localhost:12535/maintenance/config/vacuum" + @echo " Monitor: http://localhost:12535/maintenance" @echo "" @echo "Custom Parameters:" @echo " make vacuum-data FILES=50 DELETE=0.8 SIZE=200" diff --git a/docker/admin_integration/create_vacuum_test_data.go b/docker/admin_integration/create_vacuum_test_data.go index 46acdd4cd3..1679d98b4d 100644 --- a/docker/admin_integration/create_vacuum_test_data.go +++ b/docker/admin_integration/create_vacuum_test_data.go @@ -243,7 +243,7 @@ func printTestingInstructions() { fmt.Println("๐Ÿงช Testing Instructions:") fmt.Println() fmt.Println("1. Configure Vacuum for Testing:") - fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum") + fmt.Println(" Visit: http://localhost:12535/maintenance/config/vacuum") fmt.Println(" Set:") fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n") fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n") @@ -253,7 +253,7 @@ func printTestingInstructions() { fmt.Println() fmt.Println("2. Monitor Vacuum Tasks:") - fmt.Println(" Visit: http://localhost:23646/maintenance") + fmt.Println(" Visit: http://localhost:12535/maintenance") fmt.Println(" Watch for vacuum tasks to appear in the queue") fmt.Println() diff --git a/docker/admin_integration/demo_vacuum_testing.sh b/docker/admin_integration/demo_vacuum_testing.sh index 6835e14cc1..a0c8678778 100755 --- a/docker/admin_integration/demo_vacuum_testing.sh +++ b/docker/admin_integration/demo_vacuum_testing.sh @@ -7,7 +7,7 @@ echo "" # Check if SeaweedFS is running echo "๐Ÿ“‹ Checking SeaweedFS status..." MASTER_URL="${MASTER_HOST:-master:9333}" -ADMIN_URL="${ADMIN_HOST:-admin:23646}" +ADMIN_URL="${ADMIN_HOST:-admin:12535}" if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then echo "โŒ SeaweedFS master not running at $MASTER_URL" @@ -42,7 +42,7 @@ echo "" # Phase 3: Configure vacuum echo "โš™๏ธ Phase 3: Vacuum configuration instructions..." -echo " 1. Visit: http://localhost:23646/maintenance/config/vacuum" +echo " 1. Visit: http://localhost:12535/maintenance/config/vacuum" echo " 2. Set these values for testing:" echo " - Enable Vacuum Tasks: โœ… Checked" echo " - Garbage Threshold: 0.30" @@ -57,7 +57,7 @@ echo "" # Phase 4: Monitor tasks echo "๐ŸŽฏ Phase 4: Monitoring vacuum tasks..." -echo " Visit: http://localhost:23646/maintenance" +echo " Visit: http://localhost:12535/maintenance" echo " You should see vacuum tasks appear within 30 seconds" echo "" @@ -93,9 +93,9 @@ echo "" echo "๐ŸŽ‰ Demo Complete!" echo "" echo "๐Ÿ” Things to check:" -echo " 1. Maintenance Queue: http://localhost:23646/maintenance" +echo " 1. Maintenance Queue: http://localhost:12535/maintenance" echo " 2. Volume Status: http://localhost:9333/vol/status" -echo " 3. Admin Dashboard: http://localhost:23646" +echo " 3. Admin Dashboard: http://localhost:12535" echo "" echo "๐Ÿ’ก Next Steps:" echo " - Try different garbage thresholds (0.10, 0.50, 0.80)" diff --git a/docker/admin_integration/docker-compose-ec-test.yml b/docker/admin_integration/docker-compose-ec-test.yml index 1726e61fa5..619b45e8c5 100644 --- a/docker/admin_integration/docker-compose-ec-test.yml +++ b/docker/admin_integration/docker-compose-ec-test.yml @@ -114,9 +114,9 @@ services: admin: image: chrislusf/seaweedfs:local ports: - - "23646:23646" # HTTP admin interface (default port) - - "33646:33646" # gRPC worker communication (23646 + 10000) - command: "-v=2 admin -port=23646 -masters=master:9333 -dataDir=/data" + - "12535:12535" # HTTP admin interface (default port) + - "22535:22535" # gRPC worker communication (12535 + 10000) + command: "-v=2 admin -port=12535 -masters=master:9333 -dataDir=/data" depends_on: - master - filer @@ -127,7 +127,7 @@ services: worker1: image: chrislusf/seaweedfs:local - command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2" depends_on: - admin volumes: @@ -139,7 +139,7 @@ services: worker2: image: chrislusf/seaweedfs:local - command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2" depends_on: - admin volumes: @@ -151,7 +151,7 @@ services: worker3: image: chrislusf/seaweedfs:local - command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2" depends_on: - admin volumes: @@ -199,7 +199,7 @@ services: curl -s http://master:9333/cluster/status | jq '.IsLeader, .Peers' || echo 'Master not ready'; echo; echo 'Admin status:'; - curl -s http://admin:23646/ | grep -o 'Admin.*Interface' || echo 'Admin not ready'; + curl -s http://admin:12535/ | grep -o 'Admin.*Interface' || echo 'Admin not ready'; echo; echo 'Volume count by server:'; curl -s http://master:9333/vol/status | jq '.Volumes | length' || echo 'Volumes not ready'; @@ -237,4 +237,4 @@ services: - seaweed_net environment: - MASTER_HOST=master:9333 - - ADMIN_HOST=admin:23646 \ No newline at end of file + - ADMIN_HOST=admin:12535 \ No newline at end of file diff --git a/docker/admin_integration/test-integration.sh b/docker/admin_integration/test-integration.sh index b355b1dfd4..0b811ca3b2 100755 --- a/docker/admin_integration/test-integration.sh +++ b/docker/admin_integration/test-integration.sh @@ -23,7 +23,7 @@ else fi echo -e "${BLUE}2. Checking if required ports are available...${NC}" -for port in 9333 8080 8081 8082 8083 8084 8085 8888 23646; do +for port in 9333 8080 8081 8082 8083 8084 8085 8888 12535; do if lsof -i :$port > /dev/null 2>&1; then echo -e "${YELLOW}โš ๏ธ Port $port is in use${NC}" else @@ -41,7 +41,7 @@ else fi echo -e "${BLUE}4. Verifying admin server configuration...${NC}" -if docker-compose -f docker-compose-ec-test.yml config | grep -q "admin:23646"; then +if docker-compose -f docker-compose-ec-test.yml config | grep -q "admin:12535"; then echo -e "${GREEN}โœ… Admin server port configuration is correct${NC}" else echo -e "${RED}โŒ Admin server port configuration is incorrect${NC}" diff --git a/docker/seaweedfs-dev-compose.yml b/docker/seaweedfs-dev-compose.yml index 47c3598acb..5fafecfa7d 100644 --- a/docker/seaweedfs-dev-compose.yml +++ b/docker/seaweedfs-dev-compose.yml @@ -45,7 +45,7 @@ services: admin: image: chrislusf/seaweedfs:dev # use a remote dev image ports: - - 23646:23646 + - 12535:12535 command: 'admin -master=master:9333' depends_on: - master diff --git a/install.sh b/install.sh index b22acf083f..8bede247af 100755 --- a/install.sh +++ b/install.sh @@ -350,7 +350,7 @@ main() { info " weed volume -mserver=localhost:9333 # Start Go volume server" info " weed-volume -mserver localhost:9333 # Start Rust volume server" if [ "$WORKER_INSTALLED" = true ]; then - info " weed-worker --admin localhost:23646 # Start the Rust maintenance worker" + info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker" fi } diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index 43f82499d8..d6a23cb431 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -345,8 +345,9 @@ To enable the admin interface, add the following to your values.yaml: ```yaml admin: enabled: true - port: 23646 - grpcPort: 33646 # For worker connections + port: 12535 + grpcPort: 22535 # For worker connections + legacyPort: 23646 # Compat: also serve http + gRPC (+10000) for pre-12535 clients secret: adminUser: "admin" adminPassword: "your-secure-password" # Leave empty to disable auth @@ -364,7 +365,9 @@ admin: className: "nginx" ``` -The admin interface will be available at `http://:23646` (or via ingress). Workers connect to the admin server via gRPC on port `33646`. +The admin interface will be available at `http://:12535` (or via ingress). Workers connect to the admin server via gRPC on port `22535`. + +For backward compatibility with older clients and workers, admin also serves the previous default ports: HTTP on `23646` and gRPC on `33646` (legacyPort + 10000). The Service and NetworkPolicy expose both. Set `admin.legacyPort: 0` to disable. ### Admin Authentication @@ -436,7 +439,7 @@ worker: # Auto-discovery only works if the admin is in the same namespace and same Helm release. # For cross-namespace or separate release scenarios, explicitly set this value. # Example: If main SeaweedFS is deployed in "production" namespace: - # adminServer: "seaweedfs-admin.production.svc:33646" + # adminServer: "seaweedfs-admin.production.svc:22535" adminServer: "" # Workers need storage for task execution @@ -502,8 +505,8 @@ worker: # REQUIRED: Point to the admin service of your main SeaweedFS release # Replace with the namespace where your main seaweedfs is deployed # Example: If deploying in namespace "production": - # adminServer: "seaweedfs-admin.production.svc:33646" - adminServer: "seaweedfs-admin..svc:33646" + # adminServer: "seaweedfs-admin.production.svc:22535" + adminServer: "seaweedfs-admin..svc:22535" ``` **values-worker-balance.yaml** (for balance operations): @@ -528,8 +531,8 @@ worker: # REQUIRED: Point to the admin service of your main SeaweedFS release # Replace with the namespace where your main seaweedfs is deployed # Example: If deploying in namespace "production": - # adminServer: "seaweedfs-admin.production.svc:33646" - adminServer: "seaweedfs-admin..svc:33646" + # adminServer: "seaweedfs-admin.production.svc:22535" + adminServer: "seaweedfs-admin..svc:22535" ``` Deploy the specialized workers as separate releases: diff --git a/k8s/charts/seaweedfs/templates/admin/admin-service.yaml b/k8s/charts/seaweedfs/templates/admin/admin-service.yaml index baf79610c3..c10960f683 100644 --- a/k8s/charts/seaweedfs/templates/admin/admin-service.yaml +++ b/k8s/charts/seaweedfs/templates/admin/admin-service.yaml @@ -37,6 +37,16 @@ spec: nodePort: {{ $nodePorts.grpc }} {{- end }} protocol: TCP + {{- if gt (int .Values.admin.legacyPort) 0 }} + - name: "legacy-http" + port: {{ .Values.admin.legacyPort }} + targetPort: {{ .Values.admin.legacyPort }} + protocol: TCP + - name: "legacy-grpc" + port: {{ add (int .Values.admin.legacyPort) 10000 }} + targetPort: {{ add (int .Values.admin.legacyPort) 10000 }} + protocol: TCP + {{- end }} selector: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml b/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml index 82bf7b7c7d..35d2d4be2c 100644 --- a/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml @@ -170,6 +170,9 @@ spec: -ip={{ .Values.admin.ip | default "0.0.0.0" }} \ -port={{ .Values.admin.port }} \ -port.grpc={{ .Values.admin.grpcPort }} \ + {{- if ne (int .Values.admin.legacyPort) 23646 }} + -port.legacy={{ .Values.admin.legacyPort }} \ + {{- end }} {{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }} -dataDir=/data \ {{- else if .Values.admin.dataDir }} @@ -233,6 +236,12 @@ spec: name: http - containerPort: {{ .Values.admin.grpcPort }} name: grpc + {{- if gt (int .Values.admin.legacyPort) 0 }} + - containerPort: {{ .Values.admin.legacyPort }} + name: legacy-http + - containerPort: {{ add (int .Values.admin.legacyPort) 10000 }} + name: legacy-grpc + {{- end }} {{- if .Values.admin.readinessProbe.enabled }} readinessProbe: httpGet: diff --git a/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml b/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml index 92404a7547..a9b4e316a6 100644 --- a/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml +++ b/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml @@ -93,7 +93,11 @@ ClusterRole under global.seaweedfs.createClusterRole is a leftover from a migration and is not exercised by any component. */}} {{- if .Values.admin.enabled }} -{{- $targets = append $targets (dict "component" "admin" "ports" (list .Values.admin.port .Values.admin.grpcPort)) }} +{{- $ports := list .Values.admin.port .Values.admin.grpcPort }} +{{- if gt (int .Values.admin.legacyPort) 0 }} +{{- $ports = concat $ports (list .Values.admin.legacyPort (add (int .Values.admin.legacyPort) 10000)) }} +{{- end }} +{{- $targets = append $targets (dict "component" "admin" "ports" $ports) }} {{- end }} {{- if .Values.worker.enabled }} {{- $ports := list }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index eff19c5dbc..6c6b8673db 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -1511,8 +1511,14 @@ admin: imageOverride: null restartPolicy: null replicas: 1 - port: 23646 # Default admin port - grpcPort: 33646 # Default gRPC port for worker connections + port: 12535 # Default admin port + grpcPort: 22535 # Default gRPC port for worker connections + # Legacy compat port: admin also serves HTTP and worker gRPC (legacyPort + + # 10000) on this port so workers and clients from before the 23646 -> 12535 + # default change keep connecting during migration. 0 disables. The flag is + # only passed to the binary when the value differs from its 23646 default, + # so the chart keeps installing on releases that predate -port.legacy. + legacyPort: 23646 # IP address the admin server listens on. Since `weed admin` 4.46 defaults to # loopback (127.0.0.1), the chart must bind a non-loopback address for the # kubelet's httpGet readiness/liveness probes (which dial the pod IP) to ever diff --git a/seaweed-worker/README.md b/seaweed-worker/README.md index 4548348cbe..9dde317711 100644 --- a/seaweed-worker/README.md +++ b/seaweed-worker/README.md @@ -2,7 +2,7 @@ `weed/pb/plugin.proto` is a language-agnostic contract: a maintenance worker connects out to admin, announces the job types it can detect and execute, and -answers requests on that one stream. `weed worker -admin=host:23646` is the Go +answers requests on that one stream. `weed worker -admin=host:12535` is the Go implementation of it from outside the admin process. This workspace is the Rust one. @@ -35,7 +35,7 @@ binary for the runner's platform, resolved from the version in `Cargo.lock`. ## Running - cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646 + cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 The admin's *HTTP* address is what an operator has; the gRPC port is derived from it the way the Go side does. Dialling the HTTP port fails as "frame with @@ -49,7 +49,7 @@ Released builds do not need a toolchain. The worker ships inside the SeaweedFS image, beside the Rust volume server, under the verb that mirrors `volume-rust`: - docker run chrislusf/seaweedfs worker-rust --admin admin:23646 + docker run chrislusf/seaweedfs worker-rust --admin admin:12535 and as `weed-worker_linux_{amd64,arm64}.tar.gz` on each GitHub release. Both are linux amd64/arm64 only โ€” lance, arrow and datafusion make every extra target an @@ -59,7 +59,7 @@ entrypoint says so rather than failing as "not found". ## Metrics - cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646 --metrics-port 9328 + cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 --metrics-port 9328 Serves `/health`, `/ready` and `/metrics` on that port, the same three the Go worker serves under `weed worker -metricsPort`, so one scrape config covers diff --git a/seaweed-worker/crates/core/src/config.rs b/seaweed-worker/crates/core/src/config.rs index 2941c5d457..af3742c15e 100644 --- a/seaweed-worker/crates/core/src/config.rs +++ b/seaweed-worker/crates/core/src/config.rs @@ -3,7 +3,7 @@ use std::time::Duration; /// How one worker process connects and how much work it will take on. #[derive(Clone, Debug)] pub struct WorkerOptions { - /// Admin gRPC address, e.g. "localhost:23646". + /// Admin gRPC address, e.g. "localhost:12535". pub admin_address: String, pub worker_id: String, pub worker_version: String, @@ -44,7 +44,7 @@ pub struct TlsOptions { impl Default for WorkerOptions { fn default() -> Self { Self { - admin_address: "localhost:23646".to_string(), + admin_address: "localhost:12535".to_string(), worker_id: String::new(), worker_version: env!("CARGO_PKG_VERSION").to_string(), worker_address: String::new(), diff --git a/seaweed-worker/crates/core/src/lib.rs b/seaweed-worker/crates/core/src/lib.rs index 8a9a548dc4..d297a7b2f1 100644 --- a/seaweed-worker/crates/core/src/lib.rs +++ b/seaweed-worker/crates/core/src/lib.rs @@ -2,7 +2,7 @@ //! //! `weed/pb/plugin.proto` is a language-agnostic gRPC stream: a worker connects //! out to admin, announces the job types it can detect and execute, and then -//! answers requests on that one stream. `weed worker -admin=host:23646` is the +//! answers requests on that one stream. `weed worker -admin=host:12535` is the //! Go implementation of the same contract from outside the admin process; this //! is the Rust one, and it knows nothing about any particular job. diff --git a/seaweed-worker/crates/lance/src/main.rs b/seaweed-worker/crates/lance/src/main.rs index 4c268c7c1a..8f688cf357 100644 --- a/seaweed-worker/crates/lance/src/main.rs +++ b/seaweed-worker/crates/lance/src/main.rs @@ -15,7 +15,7 @@ use weed_lance_worker::metrics::LanceMetrics; #[command(name = "weed-worker", about = "SeaweedFS maintenance worker")] struct Args { /// Admin server gRPC address. - #[arg(long, default_value = "localhost:23646", env = "WEED_ADMIN")] + #[arg(long, default_value = "localhost:12535", env = "WEED_ADMIN")] admin: String, /// Worker identity reported to admin. diff --git a/terraform/modules/aws/security_group.tf b/terraform/modules/aws/security_group.tf index 93fa32af7d..cc9a5d5c87 100644 --- a/terraform/modules/aws/security_group.tf +++ b/terraform/modules/aws/security_group.tf @@ -1,6 +1,6 @@ # Cluster security group + the SeaweedFS port matrix. # Intra-cluster: all TCP allowed within the SG (master 9333/19333, volume -# 8080/18080, filer 8888/18888, admin 33646, ...). Client-facing: only S3 (8333) +# 8080/18080, filer 8888/18888, admin 22535, ...). Client-facing: only S3 (8333) # and filer (8888). Metrics ports (9327) are never opened to clients. resource "aws_security_group" "cluster" { diff --git a/test/erasure_coding/admin_dockertest/ec_integration_test.go b/test/erasure_coding/admin_dockertest/ec_integration_test.go index 7840aca1d9..d8266cef1a 100644 --- a/test/erasure_coding/admin_dockertest/ec_integration_test.go +++ b/test/erasure_coding/admin_dockertest/ec_integration_test.go @@ -18,7 +18,7 @@ import ( ) const ( - AdminUrl = "http://localhost:23646" + AdminUrl = "http://localhost:12535" MasterUrl = "http://localhost:9333" FilerUrl = "http://localhost:8888" ) @@ -147,13 +147,13 @@ func ensureEnvironment(t *testing.T) { debugPort := 6060 + i dir, _ := filepath.Abs(filepath.Join("tmp", workerName)) os.MkdirAll(dir, 0755) - startWeed(t, workerName, "worker", "-admin=localhost:23646", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort)) + startWeed(t, workerName, "worker", "-admin=localhost:12535", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort)) } // 6. Start Admin os.RemoveAll(filepath.Join("tmp", "admin")) os.MkdirAll(filepath.Join("tmp", "admin"), 0755) - startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin") + startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=12535", "-dataDir=./tmp/admin") // Admin is started after master, 14 volume servers, filer and 2 workers, // so under cold CI conditions the wait here has to absorb the tail of // every earlier subprocess coming up. 60s is too tight and has flaked; diff --git a/test/s3/lifecycle/Makefile b/test/s3/lifecycle/Makefile index 5fafcb10e2..98aec8eceb 100644 --- a/test/s3/lifecycle/Makefile +++ b/test/s3/lifecycle/Makefile @@ -12,7 +12,7 @@ FILER_PORT := 8888 # Pin the filer gRPC port off the FILER_PORT+10000 convention so any # code path that assumes the offset breaks here, not in production. FILER_GRPC_PORT := 18890 -ADMIN_PORT := 23646 +ADMIN_PORT := 12535 ACCESS_KEY ?= some_access_key1 SECRET_KEY ?= some_secret_key1 TEST_TIMEOUT := 10m diff --git a/test/s3/lifecycle/s3_lifecycle_test.go b/test/s3/lifecycle/s3_lifecycle_test.go index dd8bf117a1..a275f29863 100644 --- a/test/s3/lifecycle/s3_lifecycle_test.go +++ b/test/s3/lifecycle/s3_lifecycle_test.go @@ -41,7 +41,7 @@ const ( defaultS3Endpoint = "http://localhost:8333" defaultS3GrpcEndpoint = "localhost:18333" defaultMasterEndpt = "http://localhost:9333" - defaultAdminEndpoint = "http://localhost:23646" + defaultAdminEndpoint = "http://localhost:12535" // Pinned off the FILER_PORT+10000 convention; see Makefile. defaultFilerGRPC = "localhost:18890" bucketLifecycleXMLKey = "s3-bucket-lifecycle-configuration-xml" diff --git a/test/s3/policy/policy_test.go b/test/s3/policy/policy_test.go index 488f528841..2e3e1364e3 100644 --- a/test/s3/policy/policy_test.go +++ b/test/s3/policy/policy_test.go @@ -705,10 +705,10 @@ func uniqueName(prefix string) string { func startMiniCluster(t *testing.T) (*TestCluster, error) { // Allocate two extra ports for admin HTTP + gRPC. Without explicit - // values mini falls back to 23646/33646, which collide across the + // values mini falls back to 12535/22535, which collide across the // sequential subtests in this package: each subtest spins up a fresh // mini in a goroutine, but the previous mini's admin goroutine is - // still binding 23646 โ€” so the next test's admin can never become + // still binding 12535 โ€” so the next test's admin can never become // ready and mini.go fatals on its readiness check. ports := testutil.MustAllocatePorts(t, 10) masterPort, masterGrpcPort := ports[0], ports[1] diff --git a/test/testutil/ports.go b/test/testutil/ports.go index cf14c47f43..0950cf5209 100644 --- a/test/testutil/ports.go +++ b/test/testutil/ports.go @@ -40,7 +40,7 @@ var miniDefaultPorts = []int{ 8181, // s3.port.iceberg 9101, // s3.port.lance 7333, // webdav.port - 23646, // admin.port + 12535, // admin.port } func reservedMiniPorts() map[int]bool { diff --git a/test/testutil/ports_test.go b/test/testutil/ports_test.go index 8d22c51e76..e65f1d99e0 100644 --- a/test/testutil/ports_test.go +++ b/test/testutil/ports_test.go @@ -4,7 +4,7 @@ import "testing" // AllocateMiniPorts must never hand out a port that weed mini will reserve // for one of its default services (or that default's gRPC offset). A real -// failure: Filer was given 33646 (Admin default 23646 + GrpcPortOffset), +// failure: Filer was given 22535 (Admin default 12535 + GrpcPortOffset), // which mini then refused as "reserved for gRPC calculation". func TestAllocateMiniPortsAvoidsMiniDefaults(t *testing.T) { reserved := reservedMiniPorts() diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index c77ab3c391..4f66c9fc46 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -1642,6 +1642,14 @@ func (s *AdminServer) WorkerGrpcMTLSEnabled() bool { return s.workerGrpcServer != nil && s.workerGrpcServer.mtlsEnabled } +// ServeWorkerGrpcOn serves the worker gRPC API on an additional already-bound +// listener, e.g. a legacy compat port kept for backward compatibility. +func (s *AdminServer) ServeWorkerGrpcOn(listener net.Listener) { + if s.workerGrpcServer != nil { + s.workerGrpcServer.ServeOn(listener) + } +} + // StopWorkerGrpcServer stops the worker gRPC server func (s *AdminServer) StopWorkerGrpcServer() error { if s.workerGrpcServer != nil { diff --git a/weed/admin/dash/worker_grpc_server.go b/weed/admin/dash/worker_grpc_server.go index a55e6d7da2..a6e9294d87 100644 --- a/weed/admin/dash/worker_grpc_server.go +++ b/weed/admin/dash/worker_grpc_server.go @@ -133,6 +133,17 @@ func (s *WorkerGrpcServer) StartWithTLS(bindIp string, port int, listener net.Li return nil } +// ServeOn serves the worker gRPC API on an additional already-bound listener, +// e.g. a legacy compat port kept alongside the primary one. grpc.Server tracks +// every listener passed to Serve, so Stop closes it together with the primary. +func (s *WorkerGrpcServer) ServeOn(listener net.Listener) { + go func() { + if err := s.grpcServer.Serve(listener); err != nil && s.running { + glog.Warningf("Worker gRPC compat listener on %s stopped: %v", listener.Addr(), err) + } + }() +} + // ListenPort returns the currently bound worker gRPC listen port. func (s *WorkerGrpcServer) ListenPort() int { if s == nil || s.listener == nil { diff --git a/weed/admin/plugin/DESIGN.md b/weed/admin/plugin/DESIGN.md index 04688cbae1..fb7067d4ee 100644 --- a/weed/admin/plugin/DESIGN.md +++ b/weed/admin/plugin/DESIGN.md @@ -180,9 +180,9 @@ Phase 4 status (starter) Run example: - Start admin: `weed admin -master=localhost:9333` -- Start worker: `weed worker -admin=localhost:23646` -- Optional explicit job type: `weed worker -admin=localhost:23646 -jobType=vacuum` -- Optional stable worker ID persistence: `weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin` +- Start worker: `weed worker -admin=localhost:12535` +- Optional explicit job type: `weed worker -admin=localhost:12535 -jobType=vacuum` +- Optional stable worker ID persistence: `weed worker -admin=localhost:12535 -workingDir=/var/lib/seaweedfs-plugin` Phase 5 - Migrate remaining job types and deprecate old mechanism. diff --git a/weed/command/admin.go b/weed/command/admin.go index acba8f3e9a..a4844f1b8f 100644 --- a/weed/command/admin.go +++ b/weed/command/admin.go @@ -5,6 +5,7 @@ import ( "context" "crypto/rand" "crypto/tls" + "errors" "fmt" "log" "net" @@ -42,6 +43,7 @@ var ( type AdminOptions struct { port *int grpcPort *int + legacyPort *int ip *string master *string masters *string // deprecated, for backward compatibility @@ -68,6 +70,23 @@ type AdminOptions struct { // binds it only after every other service is up. workerGrpcListener net.Listener + // workerGrpcLegacyListener is the same reservation for the legacy + // compat gRPC port. Its bind failure is tolerated: the legacy port lives + // in the ephemeral range, so best-effort compat must not take startup down. + workerGrpcLegacyListener net.Listener + + // workerGrpcLegacySkip disables just the legacy gRPC listener. `weed + // mini` sets it when another mini service already owns the legacy gRPC + // port, so the compat bind can neither steal the port nor warn about a + // failure that is expected. + workerGrpcLegacySkip bool + + // adminHttpLegacySkip is the same for the legacy HTTP port. Skipping + // matters even when the bind would succeed: a service bound on a + // different address family or wildcard can leave loopback free, and the + // compat listener would then shadow it for loopback clients. + adminHttpLegacySkip bool + // workerGrpcBindIp, when non-empty, is the address the worker gRPC // listener binds to. It is separate from ip because the worker gRPC has // no password auth (its mTLS comes from grpc.admin, not https.admin), so @@ -83,8 +102,9 @@ type AdminOptions struct { func init() { cmdAdmin.Run = runAdmin // break init cycle - a.port = cmdAdmin.Flag.Int("port", 23646, "admin server port") + a.port = cmdAdmin.Flag.Int("port", defaultAdminHTTPPort, "admin server port") a.grpcPort = cmdAdmin.Flag.Int("port.grpc", 0, "gRPC server port for worker connections (default: http port + 10000)") + a.legacyPort = cmdAdmin.Flag.Int("port.legacy", defaultAdminHTTPLegacyPort, "also serve admin http and gRPC (port + 10000) on this port for backward compatibility (0 to disable)") a.ip = cmdAdmin.Flag.String("ip", "127.0.0.1", "ip address to listen on. Defaults to loopback when auth is disabled, or 0.0.0.0 when -adminPassword or [https.admin] mTLS is configured. Set explicitly to override.") a.master = cmdAdmin.Flag.String("master", "localhost:9333", "comma-separated master servers") a.masters = cmdAdmin.Flag.String("masters", "", "comma-separated master servers (deprecated, use -master instead)") @@ -108,7 +128,7 @@ func init() { } var cmdAdmin = &Command{ - UsageLine: "admin -port=23646 -master=localhost:9333 [-filerGroup=group] [-port.grpc=33646] [-dataDir=/path/to/data]", + UsageLine: "admin -port=12535 -master=localhost:9333 [-filerGroup=group] [-port.grpc=22535] [-dataDir=/path/to/data]", Short: "start SeaweedFS web admin interface", Long: `Start a web admin interface for SeaweedFS cluster management. @@ -124,12 +144,12 @@ var cmdAdmin = &Command{ A gRPC server for worker connections runs on the configured gRPC port (default: HTTP port + 10000). Example Usage: - weed admin -port=23646 -master="master1:9333,master2:9333" - weed admin -port=23646 -master="localhost:9333" -filerGroup="tenant-a" - weed admin -port=23646 -master="localhost:9333" -dataDir="/var/lib/seaweedfs-admin" - weed admin -port=23646 -port.grpc=33646 -master="localhost:9333" -dataDir="~/seaweedfs-admin" + weed admin -port=12535 -master="master1:9333,master2:9333" + weed admin -port=12535 -master="localhost:9333" -filerGroup="tenant-a" + weed admin -port=12535 -master="localhost:9333" -dataDir="/var/lib/seaweedfs-admin" + weed admin -port=12535 -port.grpc=22535 -master="localhost:9333" -dataDir="~/seaweedfs-admin" weed admin -port=9900 -port.grpc=19900 -master="localhost:9333" - weed admin -port=23646 -master="localhost:9333" -urlPrefix="/seaweedfs" + weed admin -port=12535 -master="localhost:9333" -urlPrefix="/seaweedfs" Data Directory: - If dataDir is specified, admin configuration and maintenance data is persisted @@ -295,6 +315,9 @@ func runAdmin(cmd *Command, args []string) bool { if *a.grpcPort == 0 { *a.grpcPort = *a.port + 10000 } + if *a.grpcPort >= linuxEphemeralPortFloor { + glog.Warningf("Admin gRPC port %d is at or above the Linux ephemeral floor %d; an outbound connection may claim it before Admin binds it. Pick a lower -port.", *a.grpcPort, linuxEphemeralPortFloor) + } hasMTLS := viper.GetString("https.admin.key") != "" && viper.GetString("https.admin.ca") != "" @@ -504,6 +527,31 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, } warnInsecureWorkerGrpcBind(workerGrpcIp, *options.grpcPort, adminServer.WorkerGrpcMTLSEnabled()) + // Legacy compat listeners keep the previous default admin port reachable + // for old workers, bookmarks and scripts while the deployment moves to the + // new default. Best-effort only: the legacy gRPC port sits inside the + // Linux ephemeral range, so it may already be taken and must never fail + // startup. + legacyPort := 0 + if options.legacyPort != nil && *options.legacyPort > 0 && *options.legacyPort != *options.port { + legacyPort = *options.legacyPort + } + if legacyPort > 0 && !options.workerGrpcLegacySkip { + legacyGrpcPort := legacyPort + GrpcPortOffset + legacyGrpcListener := options.workerGrpcLegacyListener + if legacyGrpcListener == nil { + if l, listenErr := net.Listen("tcp", util.JoinHostPort(workerGrpcIp, legacyGrpcPort)); listenErr == nil { + legacyGrpcListener = l + } else { + glog.Warningf("Legacy admin gRPC port %d unavailable, skipping compat listener: %v", legacyGrpcPort, listenErr) + } + } + if legacyGrpcListener != nil { + adminServer.ServeWorkerGrpcOn(legacyGrpcListener) + glog.Infof("Worker gRPC compat listener on %s", legacyGrpcListener.Addr()) + } + } + // Set up cleanup for gRPC server defer func() { if stopErr := adminServer.StopWorkerGrpcServer(); stopErr != nil { @@ -578,6 +626,43 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, server.TLSConfig.GetCertificate = getCert } + // Serve the same handler on the legacy port so old clients keep working. + // Best-effort like the gRPC compat listener above. + var legacyServer *http.Server + if legacyPort > 0 && !options.adminHttpLegacySkip { + legacyAddr := util.JoinHostPort(*options.ip, legacyPort) + if legacyListener, listenErr := net.Listen("tcp", legacyAddr); listenErr != nil { + glog.Warningf("Legacy admin HTTP port %d unavailable, skipping compat listener: %v", legacyPort, listenErr) + } else { + legacyServer = &http.Server{ + Addr: legacyAddr, + Handler: server.Handler, + TLSConfig: server.TLSConfig, + } + // Close on every exit path so a failed primary startup cannot + // leave the compat port held by a zombie admin. On the normal + // path the graceful Shutdown below runs first; this Close then + // no-ops. Close the raw listener too: Serve may not have + // registered it with legacyServer yet when this runs. + defer func() { + _ = legacyListener.Close() + _ = legacyServer.Close() + }() + go func() { + glog.Infof("Admin compat listener on %s", legacyAddr) + var serveErr error + if useTLS { + serveErr = legacyServer.ServeTLS(legacyListener, "", "") + } else { + serveErr = legacyServer.Serve(legacyListener) + } + if serveErr != nil && serveErr != http.ErrServerClosed && !errors.Is(serveErr, net.ErrClosed) { + glog.Warningf("Admin compat listener on %s stopped: %v", legacyAddr, serveErr) + } + }() + } + } + // Start server. Surface immediate failures (e.g. bind error) via // serveErrCh so the caller doesn't block on ctx.Done() while no server // is actually listening. http.ErrServerClosed after Shutdown is normal @@ -612,6 +697,9 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, if err := server.Shutdown(shutdownCtx); err != nil { return fmt.Errorf("admin server forced to shutdown: %w", err) } + if legacyServer != nil { + _ = legacyServer.Shutdown(shutdownCtx) + } adminServer.Shutdown() diff --git a/weed/command/mini.go b/weed/command/mini.go index 853c8e0077..1ded2f936c 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -49,13 +49,16 @@ type MiniOptions struct { } const ( - bytesPerMB = 1024 * 1024 // Bytes per MB - miniVolumeMinFreeSpace = "1" // 1% minimum free space - minVolumeSizeMB = 64 // Minimum volume size in MB - defaultMiniVolumeSizeMB = 128 // Default volume size for mini mode - maxVolumeSizeMB = 1024 // Maximum volume size in MB (1GB) - GrpcPortOffset = 10000 // Offset used to calculate gRPC port from HTTP port - defaultMiniPluginJobTypes = "all" + bytesPerMB = 1024 * 1024 // Bytes per MB + miniVolumeMinFreeSpace = "1" // 1% minimum free space + minVolumeSizeMB = 64 // Minimum volume size in MB + defaultMiniVolumeSizeMB = 128 // Default volume size for mini mode + maxVolumeSizeMB = 1024 // Maximum volume size in MB (1GB) + GrpcPortOffset = 10000 // Offset used to calculate gRPC port from HTTP port + linuxEphemeralPortFloor = 32768 // Default net.ipv4.ip_local_port_range floor on Linux + defaultAdminHTTPPort = 12535 // Keeps the derived admin gRPC port (port+GrpcPortOffset) below linuxEphemeralPortFloor + defaultAdminHTTPLegacyPort = 23646 // Pre-12535 default; served alongside the new port for backward compatibility + defaultMiniPluginJobTypes = "all" ) var ( @@ -371,12 +374,12 @@ After starting, you can access: - S3 Endpoint: http://localhost:8333 - Iceberg Catalog: http://localhost:8181 - WebDAV: http://localhost:7333 -- Admin UI: http://localhost:23646 +- Admin UI: http://localhost:12535 S3 Access: The S3 endpoint is available at http://localhost:8333. For client configuration and IAM setup, see the project documentation or use the -Admin UI (http://localhost:23646) to manage users and policies. +Admin UI (http://localhost:12535) to manage users and policies. Notes: - Values mini computes itself (currently master.volumeSizeLimitMB) are @@ -574,8 +577,9 @@ func initMiniWebDAVFlags() { // initMiniAdminFlags initializes Admin server flag options func initMiniAdminFlags() { - miniAdminOptions.port = cmdMini.Flag.Int("admin.port", 23646, "admin server http listen port") + miniAdminOptions.port = cmdMini.Flag.Int("admin.port", defaultAdminHTTPPort, "admin server http listen port") miniAdminOptions.grpcPort = cmdMini.Flag.Int("admin.port.grpc", 0, "admin server grpc listen port (default: admin http port + GrpcPortOffset)") + miniAdminOptions.legacyPort = cmdMini.Flag.Int("admin.port.legacy", defaultAdminHTTPLegacyPort, "also serve admin http and grpc (port + GrpcPortOffset) on this port for backward compatibility (0 to disable)") miniAdminOptions.master = cmdMini.Flag.String("admin.master", "", "master server address (automatically set)") // admin discovery must query the same group the co-located filer registers under miniAdminOptions.filerGroup = miniFilerOptions.filerGroup @@ -966,7 +970,7 @@ func ensureAllPortsAvailableOnIP(bindIp string) error { // Initialize all gRPC ports before services start // This ensures they won't be recalculated and cause conflicts // All gRPC port handling (calculation, validation, and assignment) is performed exclusively in initializeGrpcPortsOnIP - initializeGrpcPortsOnIP(bindIp) + allocatedPorts := initializeGrpcPortsOnIP(bindIp) // Every other service binds within a moment of this check, but the admin // waits for all of them first. The admin gRPC port sits inside the Linux @@ -986,6 +990,33 @@ func ensureAllPortsAvailableOnIP(bindIp string) error { miniAdminOptions.workerGrpcListener = listener } + // Same early reservation for the legacy compat gRPC port โ€” but never take + // a port a mini service already owns: compat is a convenience, not a + // startup requirement. Failure is only a warning โ€” the legacy port sits + // inside the ephemeral range. Without a reservation the admin's late + // best-effort bind simply fails harmlessly if the port is taken by then. + miniAdminOptions.workerGrpcLegacyListener = nil + miniAdminOptions.workerGrpcLegacySkip = false + miniAdminOptions.adminHttpLegacySkip = false + if *miniAdminOptions.legacyPort > 0 && *miniAdminOptions.legacyPort != *miniAdminOptions.port { + if allocatedPorts[*miniAdminOptions.legacyPort] { + glog.Warningf("Legacy Admin HTTP port %d is used by another mini service, skipping compat listener", *miniAdminOptions.legacyPort) + miniAdminOptions.adminHttpLegacySkip = true + } + legacyGrpcPort := *miniAdminOptions.legacyPort + GrpcPortOffset + switch { + case allocatedPorts[legacyGrpcPort]: + glog.Warningf("Legacy Admin gRPC port %d is used by another mini service, skipping compat listener", legacyGrpcPort) + miniAdminOptions.workerGrpcLegacySkip = true + default: + if listener, err := listenMiniAdminWorker(workerGrpcBindIP, legacyGrpcPort); err != nil { + glog.Warningf("Could not reserve legacy Admin gRPC port %d: %v", legacyGrpcPort, err) + } else { + miniAdminOptions.workerGrpcLegacyListener = listener + } + } + } + // Log the final port configuration icebergPortStr := "disabled" if miniS3Options.portIceberg != nil && *miniS3Options.portIceberg > 0 { @@ -1016,7 +1047,9 @@ func listenMiniAdminWorker(bindIP string, port int) (net.Listener, error) { // initializeGrpcPortsOnIP initializes all gRPC ports based on their HTTP ports on a specific IP // If a gRPC port is 0, it will be set to httpPort + GrpcPortOffset // This must be called after HTTP ports are finalized and before services start -func initializeGrpcPortsOnIP(bindIp string) { +// Returns the set of ports allocated to mini services, for callers that must +// keep optional listeners from stealing them. +func initializeGrpcPortsOnIP(bindIp string) map[int]bool { // Track all ports allocated (both HTTP and gRPC) to prevent collisions. // We must reserve HTTP ports so that gRPC fallback allocation never picks // a port already assigned to an HTTP service (which hasn't bound yet). @@ -1086,6 +1119,15 @@ func initializeGrpcPortsOnIP(bindIp string) { allocatedPorts[*config.grpcPort] = true glog.V(1).Infof("%s gRPC port set to %d", config.name, *config.grpcPort) } + + // The Linux kernel draws outbound source ports from the ephemeral range. + // A gRPC port inside it can be claimed by one of mini's own startup dials + // before the service binds it. + if *miniAdminOptions.grpcPort >= linuxEphemeralPortFloor { + glog.Warningf("Admin gRPC port %d is at or above the Linux ephemeral floor %d; an outbound connection may claim it before Admin binds it. Pick a lower -admin.port.", *miniAdminOptions.grpcPort, linuxEphemeralPortFloor) + } + + return allocatedPorts } // miniOptionsFilePath returns the path of the mini.options file under the @@ -1880,6 +1922,9 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) { if adminOptions.workerGrpcListener != nil { adminOptions.workerGrpcListener.Close() } + if adminOptions.workerGrpcLegacyListener != nil { + adminOptions.workerGrpcLegacyListener.Close() + } }() // Wait for admin server's HTTP port to be ready before launching worker diff --git a/weed/command/mini_admin_auth_test.go b/weed/command/mini_admin_auth_test.go index 85fe8f2ad1..7eec8a3914 100644 --- a/weed/command/mini_admin_auth_test.go +++ b/weed/command/mini_admin_auth_test.go @@ -2,6 +2,7 @@ package command import ( "net" + "strconv" "testing" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -254,3 +255,32 @@ func TestMiniAdminAdvertisedIP(t *testing.T) { }) } } + +func TestDefaultAdminGrpcPortBelowEphemeralFloor(t *testing.T) { + if grpcPort := defaultAdminHTTPPort + GrpcPortOffset; grpcPort >= linuxEphemeralPortFloor { + t.Fatalf("default admin HTTP port %d derives gRPC port %d, which must stay below ephemeral floor %d", + defaultAdminHTTPPort, grpcPort, linuxEphemeralPortFloor) + } + for _, f := range []struct { + name string + flagName string + def string + }{ + {"mini", "admin.port", strconv.Itoa(defaultAdminHTTPPort)}, + {"mini", "admin.port.legacy", strconv.Itoa(defaultAdminHTTPLegacyPort)}, + {"admin", "port", strconv.Itoa(defaultAdminHTTPPort)}, + {"admin", "port.legacy", strconv.Itoa(defaultAdminHTTPLegacyPort)}, + } { + cmd := cmdMini + if f.name == "admin" { + cmd = cmdAdmin + } + flag := cmd.Flag.Lookup(f.flagName) + if flag == nil { + t.Fatalf("%s: flag %s not registered", f.name, f.flagName) + } + if flag.DefValue != f.def { + t.Fatalf("%s -%s default = %s, want %s so its gRPC port stays below the ephemeral floor", f.name, f.flagName, flag.DefValue, f.def) + } + } +} diff --git a/weed/command/worker.go b/weed/command/worker.go index bddb1c4241..1579c42fe9 100644 --- a/weed/command/worker.go +++ b/weed/command/worker.go @@ -1,6 +1,7 @@ package command import ( + "fmt" "time" "github.com/seaweedfs/seaweedfs/weed/util" @@ -25,21 +26,21 @@ Behavior: - Use -metricsPort/-metricsIp to expose /health, /ready, and /metrics Examples: - weed worker -admin=localhost:23646 - weed worker -admin=localhost:23646 -jobType=all - weed worker -admin=localhost:23646 -jobType=default - weed worker -admin=localhost:23646 -jobType=heavy - weed worker -admin=localhost:23646 -jobType=default,iceberg - weed worker -admin=localhost:23646 -jobType=vacuum,volume_balance - weed worker -admin=localhost:23646 -jobType=erasure_coding - weed worker -admin=admin.example.com:23646 -id=plugin-vacuum-a -heartbeat=10s - weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin - weed worker -admin=localhost:23646 -metricsPort=9327 -metricsIp=0.0.0.0 + weed worker -admin=localhost:12535 + weed worker -admin=localhost:12535 -jobType=all + weed worker -admin=localhost:12535 -jobType=default + weed worker -admin=localhost:12535 -jobType=heavy + weed worker -admin=localhost:12535 -jobType=default,iceberg + weed worker -admin=localhost:12535 -jobType=vacuum,volume_balance + weed worker -admin=localhost:12535 -jobType=erasure_coding + weed worker -admin=admin.example.com:12535 -id=plugin-vacuum-a -heartbeat=10s + weed worker -admin=localhost:12535 -workingDir=/var/lib/seaweedfs-plugin + weed worker -admin=localhost:12535 -metricsPort=9327 -metricsIp=0.0.0.0 `, } var ( - workerAdminServer = cmdWorker.Flag.String("admin", "localhost:23646", "admin server address") + workerAdminServer = cmdWorker.Flag.String("admin", fmt.Sprintf("localhost:%d", defaultAdminHTTPPort), "admin server address") workerID = cmdWorker.Flag.String("id", "", "worker ID (auto-generated when empty)") workerWorkingDir = cmdWorker.Flag.String("workingDir", "", "working directory for persistent worker state") workerJobType = cmdWorker.Flag.String("jobType", defaultPluginWorkerJobTypes, "job types or categories to serve: all, default, heavy, or explicit names/aliases such as ec, balance, iceberg (comma-separated)") diff --git a/weed/command/worker_runtime.go b/weed/command/worker_runtime.go index a60aa269c0..ff8d00d94d 100644 --- a/weed/command/worker_runtime.go +++ b/weed/command/worker_runtime.go @@ -45,7 +45,7 @@ func runPluginWorkerWithOptions(options pluginWorkerRunOptions) bool { options.AdminServer = strings.TrimSpace(options.AdminServer) if options.AdminServer == "" { - options.AdminServer = "localhost:23646" + options.AdminServer = fmt.Sprintf("localhost:%d", defaultAdminHTTPPort) } options.JobTypes = strings.TrimSpace(options.JobTypes)