diff --git a/design-lance-catalog.md b/design-lance-catalog.md index ce657b98a3..a942ed6ad2 100644 --- a/design-lance-catalog.md +++ b/design-lance-catalog.md @@ -543,7 +543,7 @@ a language-agnostic gRPC stream that external maintenance workers connect on: th opens `WorkerStream`, sends `WorkerHello` with the job types it can `detect` and `execute`, answers `RequestConfigSchema` with a `JobTypeDescriptor`, replies to `RunDetectionRequest` with `JobProposal`s and to `ExecuteJobRequest` with `JobProgressUpdate`s and `JobCompleted`. -`weed worker -admin=host:23646` is the Go reference implementation of exactly that contract, +`weed worker -admin=host:12535` is the Go reference implementation of exactly that contract, from outside the admin process. Nothing in it is Go-specific, and the Rust toolchain is already in the tree. diff --git a/docker/admin_integration/EC-TESTING-README.md b/docker/admin_integration/EC-TESTING-README.md index 57e0a59854..e9e13d8530 100644 --- a/docker/admin_integration/EC-TESTING-README.md +++ b/docker/admin_integration/EC-TESTING-README.md @@ -21,7 +21,7 @@ The testing environment uses **official SeaweedFS commands** and includes: - **1 Master Server** (port 9333) - Coordinates the cluster with 50MB volume size limit - **6 Volume Servers** (ports 8080-8085) - Distributed across 2 data centers and 3 racks for diversity - **1 Filer** (port 8888) - Provides file system interface -- **1 Admin Server** (port 23646) - Detects volumes needing EC and manages workers using official `admin` command +- **1 Admin Server** (port 12535) - Detects volumes needing EC and manages workers using official `admin` command - **3 EC Workers** - Execute erasure coding tasks using official `worker` command with task-specific working directories - **1 Load Generator** - Continuously writes and deletes files using SeaweedFS shell commands - **1 Monitor** - Tracks cluster health and EC progress using shell scripts @@ -47,7 +47,7 @@ This provides: - Docker and Docker Compose installed - GNU Make installed - At least 4GB RAM available for containers -- Ports 8080-8085, 8888, 9333, 23646 available +- Ports 8080-8085, 8888, 9333, 12535 available ### Start the Environment @@ -127,7 +127,7 @@ Run `make help` to see all available targets: |---------|-----|-------------| | Master UI | http://localhost:9333 | Cluster status and topology | | Filer | http://localhost:8888 | File operations | -| Admin Server | http://localhost:23646/ | Task management | +| Admin Server | http://localhost:12535/ | Task management | | Monitor | http://localhost:9999/status | Complete cluster monitoring | | Volume Servers | http://localhost:8080-8085/status | Individual volume server stats | diff --git a/docker/admin_integration/Makefile b/docker/admin_integration/Makefile index 68fb0cec6f..2997143aa6 100644 --- a/docker/admin_integration/Makefile +++ b/docker/admin_integration/Makefile @@ -26,7 +26,7 @@ build-and-restart: build ## Build with latest changes and restart services @echo "2๏ธโƒฃ Recreating workers to reconnect..." @docker-compose -f $(COMPOSE_FILE) up -d worker1 worker2 worker3 @echo "โœ… All services recreated with latest changes!" - @echo "๐ŸŒ Admin UI: http://localhost:23646/" + @echo "๐ŸŒ Admin UI: http://localhost:12535/" @echo "๐Ÿ’ก Workers will reconnect to the new admin server" restart-workers: ## Restart all workers to reconnect to admin server @@ -64,7 +64,7 @@ start: ## Start the complete SeaweedFS cluster with admin and workers @echo "โœ… Cluster started!" @echo "" @echo "๐Ÿ“Š Access points:" - @echo " โ€ข Admin UI: http://localhost:23646/" + @echo " โ€ข Admin UI: http://localhost:12535/" @echo " โ€ข Master UI: http://localhost:9333/" @echo " โ€ข Filer: http://localhost:8888/" @echo "" @@ -107,7 +107,7 @@ start-staged: ## Start services in proper order with delays @echo "โœ… All services started!" @echo "" @echo "๐Ÿ“Š Access points:" - @echo " โ€ข Admin UI: http://localhost:23646/" + @echo " โ€ข Admin UI: http://localhost:12535/" @echo " โ€ข Master UI: http://localhost:9333/" @echo " โ€ข Filer: http://localhost:8888/" @echo "" @@ -136,7 +136,7 @@ status: ## Check the status of all services @echo "Master:" @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' 2>/dev/null || echo " โŒ Master not ready" @echo "Admin:" - @curl -s http://localhost:23646/ | grep -q "Admin" && echo " โœ… Admin ready" || echo " โŒ Admin not ready" + @curl -s http://localhost:12535/ | grep -q "Admin" && echo " โœ… Admin ready" || echo " โŒ Admin not ready" logs: ## Show logs from all services @echo "๐Ÿ“œ Following logs from all services..." @@ -157,7 +157,7 @@ master-logs: ## Show logs from master server admin-ui: ## Open admin UI in browser (macOS) @echo "๐ŸŒ Opening admin UI in browser..." - @open http://localhost:23646/ || echo "๐Ÿ’ก Manually open: http://localhost:23646/" + @open http://localhost:12535/ || echo "๐Ÿ’ก Manually open: http://localhost:12535/" test: ## Run integration test to verify task assignment and completion @echo "๐Ÿงช Running Admin-Worker Integration Test" @@ -166,11 +166,11 @@ test: ## Run integration test to verify task assignment and completion @echo "1๏ธโƒฃ Checking cluster health..." @sleep 5 @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "โœ… Master healthy" || echo "โŒ Master not ready" - @curl -s http://localhost:23646/ | grep -q "Admin" && echo "โœ… Admin healthy" || echo "โŒ Admin not ready" + @curl -s http://localhost:12535/ | grep -q "Admin" && echo "โœ… Admin healthy" || echo "โŒ Admin not ready" @echo "" @echo "2๏ธโƒฃ Checking worker registration..." @sleep 10 - @echo "๐Ÿ’ก Check admin UI for connected workers: http://localhost:23646/" + @echo "๐Ÿ’ก Check admin UI for connected workers: http://localhost:12535/" @echo "" @echo "3๏ธโƒฃ Generating load to trigger EC tasks..." @echo "๐Ÿ“ Creating test files to fill volumes..." @@ -193,13 +193,13 @@ test: ## Run integration test to verify task assignment and completion @echo " โ€ข EC shards being distributed" @echo "" @echo "โœ… Integration test setup complete!" - @echo "๐Ÿ“Š Monitor progress at: http://localhost:23646/" + @echo "๐Ÿ“Š Monitor progress at: http://localhost:12535/" quick-test: ## Quick verification that core services are running @echo "โšก Quick Health Check" @echo "====================" @echo "Master: $$(curl -s http://localhost:9333/cluster/status | jq -r '.IsLeader // "not ready"')" - @echo "Admin: $$(curl -s http://localhost:23646/ | grep -q "Admin" && echo "ready" || echo "not ready")" + @echo "Admin: $$(curl -s http://localhost:12535/ | grep -q "Admin" && echo "ready" || echo "not ready")" @echo "Workers: $$(docker-compose -f $(COMPOSE_FILE) ps worker1 worker2 worker3 | grep -c Up) running" validate: ## Validate integration test configuration @@ -223,7 +223,7 @@ demo: start ## Start cluster and run demonstration @echo " โ€ข Load generator (creates files continuously)" @echo "" @echo "๐Ÿ“Š Watch the process:" - @echo " 1. Visit: http://localhost:23646/" + @echo " 1. Visit: http://localhost:12535/" @echo " 2. Observe workers connecting" @echo " 3. Watch tasks being created and assigned" @echo " 4. See tasks progress from pending โ†’ completed" @@ -244,13 +244,13 @@ vacuum-test: ## Create test data with garbage and verify vacuum detection @echo "" @echo "1๏ธโƒฃ Checking cluster health..." @curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "โœ… Master ready" || (echo "โŒ Master not ready. Run 'make start' first." && exit 1) - @curl -s http://localhost:23646/ | grep -q "Admin" && echo "โœ… Admin ready" || (echo "โŒ Admin not ready. Run 'make start' first." && exit 1) + @curl -s http://localhost:12535/ | grep -q "Admin" && echo "โœ… Admin ready" || (echo "โŒ Admin not ready. Run 'make start' first." && exit 1) @echo "" @echo "2๏ธโƒฃ Creating test data with garbage..." @docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=25 -delete=0.5 -size=200 @echo "" @echo "3๏ธโƒฃ Configuration Instructions:" - @echo " Visit: http://localhost:23646/maintenance/config/vacuum" + @echo " Visit: http://localhost:12535/maintenance/config/vacuum" @echo " Set for testing:" @echo " โ€ข Enable Vacuum Tasks: โœ… Checked" @echo " โ€ข Garbage Threshold: 0.20 (20%)" @@ -258,7 +258,7 @@ vacuum-test: ## Create test data with garbage and verify vacuum detection @echo " โ€ข Min Volume Age: [0] [Minutes]" @echo " โ€ข Max Concurrent: 2" @echo "" - @echo "4๏ธโƒฃ Monitor vacuum tasks at: http://localhost:23646/maintenance" + @echo "4๏ธโƒฃ Monitor vacuum tasks at: http://localhost:12535/maintenance" @echo "" @echo "๐Ÿ’ก Use 'make vacuum-status' to check volume garbage ratios" @@ -335,8 +335,8 @@ vacuum-help: ## Show vacuum testing help and examples @echo " make vacuum-demo # Full guided demonstration" @echo "" @echo "Configuration:" - @echo " Visit: http://localhost:23646/maintenance/config/vacuum" - @echo " Monitor: http://localhost:23646/maintenance" + @echo " Visit: http://localhost:12535/maintenance/config/vacuum" + @echo " Monitor: http://localhost:12535/maintenance" @echo "" @echo "Custom Parameters:" @echo " make vacuum-data FILES=50 DELETE=0.8 SIZE=200" diff --git a/docker/admin_integration/create_vacuum_test_data.go b/docker/admin_integration/create_vacuum_test_data.go index 46acdd4cd3..1679d98b4d 100644 --- a/docker/admin_integration/create_vacuum_test_data.go +++ b/docker/admin_integration/create_vacuum_test_data.go @@ -243,7 +243,7 @@ func printTestingInstructions() { fmt.Println("๐Ÿงช Testing Instructions:") fmt.Println() fmt.Println("1. Configure Vacuum for Testing:") - fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum") + fmt.Println(" Visit: http://localhost:12535/maintenance/config/vacuum") fmt.Println(" Set:") fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n") fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n") @@ -253,7 +253,7 @@ func printTestingInstructions() { fmt.Println() fmt.Println("2. Monitor Vacuum Tasks:") - fmt.Println(" Visit: http://localhost:23646/maintenance") + fmt.Println(" Visit: http://localhost:12535/maintenance") fmt.Println(" Watch for vacuum tasks to appear in the queue") fmt.Println() diff --git a/docker/admin_integration/demo_vacuum_testing.sh b/docker/admin_integration/demo_vacuum_testing.sh index 6835e14cc1..a0c8678778 100755 --- a/docker/admin_integration/demo_vacuum_testing.sh +++ b/docker/admin_integration/demo_vacuum_testing.sh @@ -7,7 +7,7 @@ echo "" # Check if SeaweedFS is running echo "๐Ÿ“‹ Checking SeaweedFS status..." MASTER_URL="${MASTER_HOST:-master:9333}" -ADMIN_URL="${ADMIN_HOST:-admin:23646}" +ADMIN_URL="${ADMIN_HOST:-admin:12535}" if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then echo "โŒ SeaweedFS master not running at $MASTER_URL" @@ -42,7 +42,7 @@ echo "" # Phase 3: Configure vacuum echo "โš™๏ธ Phase 3: Vacuum configuration instructions..." -echo " 1. Visit: http://localhost:23646/maintenance/config/vacuum" +echo " 1. Visit: http://localhost:12535/maintenance/config/vacuum" echo " 2. Set these values for testing:" echo " - Enable Vacuum Tasks: โœ… Checked" echo " - Garbage Threshold: 0.30" @@ -57,7 +57,7 @@ echo "" # Phase 4: Monitor tasks echo "๐ŸŽฏ Phase 4: Monitoring vacuum tasks..." -echo " Visit: http://localhost:23646/maintenance" +echo " Visit: http://localhost:12535/maintenance" echo " You should see vacuum tasks appear within 30 seconds" echo "" @@ -93,9 +93,9 @@ echo "" echo "๐ŸŽ‰ Demo Complete!" echo "" echo "๐Ÿ” Things to check:" -echo " 1. Maintenance Queue: http://localhost:23646/maintenance" +echo " 1. Maintenance Queue: http://localhost:12535/maintenance" echo " 2. Volume Status: http://localhost:9333/vol/status" -echo " 3. Admin Dashboard: http://localhost:23646" +echo " 3. Admin Dashboard: http://localhost:12535" echo "" echo "๐Ÿ’ก Next Steps:" echo " - Try different garbage thresholds (0.10, 0.50, 0.80)" diff --git a/docker/admin_integration/docker-compose-ec-test.yml b/docker/admin_integration/docker-compose-ec-test.yml index 1726e61fa5..619b45e8c5 100644 --- a/docker/admin_integration/docker-compose-ec-test.yml +++ b/docker/admin_integration/docker-compose-ec-test.yml @@ -114,9 +114,9 @@ services: admin: image: chrislusf/seaweedfs:local ports: - - "23646:23646" # HTTP admin interface (default port) - - "33646:33646" # gRPC worker communication (23646 + 10000) - command: "-v=2 admin -port=23646 -masters=master:9333 -dataDir=/data" + - "12535:12535" # HTTP admin interface (default port) + - "22535:22535" # gRPC worker communication (12535 + 10000) + command: "-v=2 admin -port=12535 -masters=master:9333 -dataDir=/data" depends_on: - master - filer @@ -127,7 +127,7 @@ services: worker1: image: chrislusf/seaweedfs:local - command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2" depends_on: - admin volumes: @@ -139,7 +139,7 @@ services: worker2: image: chrislusf/seaweedfs:local - command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2" depends_on: - admin volumes: @@ -151,7 +151,7 @@ services: worker3: image: chrislusf/seaweedfs:local - command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2" + command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2" depends_on: - admin volumes: @@ -199,7 +199,7 @@ services: curl -s http://master:9333/cluster/status | jq '.IsLeader, .Peers' || echo 'Master not ready'; echo; echo 'Admin status:'; - curl -s http://admin:23646/ | grep -o 'Admin.*Interface' || echo 'Admin not ready'; + curl -s http://admin:12535/ | grep -o 'Admin.*Interface' || echo 'Admin not ready'; echo; echo 'Volume count by server:'; curl -s http://master:9333/vol/status | jq '.Volumes | length' || echo 'Volumes not ready'; @@ -237,4 +237,4 @@ services: - seaweed_net environment: - MASTER_HOST=master:9333 - - ADMIN_HOST=admin:23646 \ No newline at end of file + - ADMIN_HOST=admin:12535 \ No newline at end of file diff --git a/docker/admin_integration/test-integration.sh b/docker/admin_integration/test-integration.sh index b355b1dfd4..0b811ca3b2 100755 --- a/docker/admin_integration/test-integration.sh +++ b/docker/admin_integration/test-integration.sh @@ -23,7 +23,7 @@ else fi echo -e "${BLUE}2. Checking if required ports are available...${NC}" -for port in 9333 8080 8081 8082 8083 8084 8085 8888 23646; do +for port in 9333 8080 8081 8082 8083 8084 8085 8888 12535; do if lsof -i :$port > /dev/null 2>&1; then echo -e "${YELLOW}โš ๏ธ Port $port is in use${NC}" else @@ -41,7 +41,7 @@ else fi echo -e "${BLUE}4. Verifying admin server configuration...${NC}" -if docker-compose -f docker-compose-ec-test.yml config | grep -q "admin:23646"; then +if docker-compose -f docker-compose-ec-test.yml config | grep -q "admin:12535"; then echo -e "${GREEN}โœ… Admin server port configuration is correct${NC}" else echo -e "${RED}โŒ Admin server port configuration is incorrect${NC}" diff --git a/docker/seaweedfs-dev-compose.yml b/docker/seaweedfs-dev-compose.yml index 47c3598acb..5fafecfa7d 100644 --- a/docker/seaweedfs-dev-compose.yml +++ b/docker/seaweedfs-dev-compose.yml @@ -45,7 +45,7 @@ services: admin: image: chrislusf/seaweedfs:dev # use a remote dev image ports: - - 23646:23646 + - 12535:12535 command: 'admin -master=master:9333' depends_on: - master diff --git a/install.sh b/install.sh index b22acf083f..8bede247af 100755 --- a/install.sh +++ b/install.sh @@ -350,7 +350,7 @@ main() { info " weed volume -mserver=localhost:9333 # Start Go volume server" info " weed-volume -mserver localhost:9333 # Start Rust volume server" if [ "$WORKER_INSTALLED" = true ]; then - info " weed-worker --admin localhost:23646 # Start the Rust maintenance worker" + info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker" fi } diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index 43f82499d8..d6a23cb431 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -345,8 +345,9 @@ To enable the admin interface, add the following to your values.yaml: ```yaml admin: enabled: true - port: 23646 - grpcPort: 33646 # For worker connections + port: 12535 + grpcPort: 22535 # For worker connections + legacyPort: 23646 # Compat: also serve http + gRPC (+10000) for pre-12535 clients secret: adminUser: "admin" adminPassword: "your-secure-password" # Leave empty to disable auth @@ -364,7 +365,9 @@ admin: className: "nginx" ``` -The admin interface will be available at `http://:23646` (or via ingress). Workers connect to the admin server via gRPC on port `33646`. +The admin interface will be available at `http://:12535` (or via ingress). Workers connect to the admin server via gRPC on port `22535`. + +For backward compatibility with older clients and workers, admin also serves the previous default ports: HTTP on `23646` and gRPC on `33646` (legacyPort + 10000). The Service and NetworkPolicy expose both. Set `admin.legacyPort: 0` to disable. ### Admin Authentication @@ -436,7 +439,7 @@ worker: # Auto-discovery only works if the admin is in the same namespace and same Helm release. # For cross-namespace or separate release scenarios, explicitly set this value. # Example: If main SeaweedFS is deployed in "production" namespace: - # adminServer: "seaweedfs-admin.production.svc:33646" + # adminServer: "seaweedfs-admin.production.svc:22535" adminServer: "" # Workers need storage for task execution @@ -502,8 +505,8 @@ worker: # REQUIRED: Point to the admin service of your main SeaweedFS release # Replace with the namespace where your main seaweedfs is deployed # Example: If deploying in namespace "production": - # adminServer: "seaweedfs-admin.production.svc:33646" - adminServer: "seaweedfs-admin..svc:33646" + # adminServer: "seaweedfs-admin.production.svc:22535" + adminServer: "seaweedfs-admin..svc:22535" ``` **values-worker-balance.yaml** (for balance operations): @@ -528,8 +531,8 @@ worker: # REQUIRED: Point to the admin service of your main SeaweedFS release # Replace with the namespace where your main seaweedfs is deployed # Example: If deploying in namespace "production": - # adminServer: "seaweedfs-admin.production.svc:33646" - adminServer: "seaweedfs-admin..svc:33646" + # adminServer: "seaweedfs-admin.production.svc:22535" + adminServer: "seaweedfs-admin..svc:22535" ``` Deploy the specialized workers as separate releases: diff --git a/k8s/charts/seaweedfs/templates/admin/admin-service.yaml b/k8s/charts/seaweedfs/templates/admin/admin-service.yaml index baf79610c3..c10960f683 100644 --- a/k8s/charts/seaweedfs/templates/admin/admin-service.yaml +++ b/k8s/charts/seaweedfs/templates/admin/admin-service.yaml @@ -37,6 +37,16 @@ spec: nodePort: {{ $nodePorts.grpc }} {{- end }} protocol: TCP + {{- if gt (int .Values.admin.legacyPort) 0 }} + - name: "legacy-http" + port: {{ .Values.admin.legacyPort }} + targetPort: {{ .Values.admin.legacyPort }} + protocol: TCP + - name: "legacy-grpc" + port: {{ add (int .Values.admin.legacyPort) 10000 }} + targetPort: {{ add (int .Values.admin.legacyPort) 10000 }} + protocol: TCP + {{- end }} selector: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml b/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml index 82bf7b7c7d..35d2d4be2c 100644 --- a/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml @@ -170,6 +170,9 @@ spec: -ip={{ .Values.admin.ip | default "0.0.0.0" }} \ -port={{ .Values.admin.port }} \ -port.grpc={{ .Values.admin.grpcPort }} \ + {{- if ne (int .Values.admin.legacyPort) 23646 }} + -port.legacy={{ .Values.admin.legacyPort }} \ + {{- end }} {{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }} -dataDir=/data \ {{- else if .Values.admin.dataDir }} @@ -233,6 +236,12 @@ spec: name: http - containerPort: {{ .Values.admin.grpcPort }} name: grpc + {{- if gt (int .Values.admin.legacyPort) 0 }} + - containerPort: {{ .Values.admin.legacyPort }} + name: legacy-http + - containerPort: {{ add (int .Values.admin.legacyPort) 10000 }} + name: legacy-grpc + {{- end }} {{- if .Values.admin.readinessProbe.enabled }} readinessProbe: httpGet: diff --git a/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml b/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml index 92404a7547..a9b4e316a6 100644 --- a/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml +++ b/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml @@ -93,7 +93,11 @@ ClusterRole under global.seaweedfs.createClusterRole is a leftover from a migration and is not exercised by any component. */}} {{- if .Values.admin.enabled }} -{{- $targets = append $targets (dict "component" "admin" "ports" (list .Values.admin.port .Values.admin.grpcPort)) }} +{{- $ports := list .Values.admin.port .Values.admin.grpcPort }} +{{- if gt (int .Values.admin.legacyPort) 0 }} +{{- $ports = concat $ports (list .Values.admin.legacyPort (add (int .Values.admin.legacyPort) 10000)) }} +{{- end }} +{{- $targets = append $targets (dict "component" "admin" "ports" $ports) }} {{- end }} {{- if .Values.worker.enabled }} {{- $ports := list }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index eff19c5dbc..6c6b8673db 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -1511,8 +1511,14 @@ admin: imageOverride: null restartPolicy: null replicas: 1 - port: 23646 # Default admin port - grpcPort: 33646 # Default gRPC port for worker connections + port: 12535 # Default admin port + grpcPort: 22535 # Default gRPC port for worker connections + # Legacy compat port: admin also serves HTTP and worker gRPC (legacyPort + + # 10000) on this port so workers and clients from before the 23646 -> 12535 + # default change keep connecting during migration. 0 disables. The flag is + # only passed to the binary when the value differs from its 23646 default, + # so the chart keeps installing on releases that predate -port.legacy. + legacyPort: 23646 # IP address the admin server listens on. Since `weed admin` 4.46 defaults to # loopback (127.0.0.1), the chart must bind a non-loopback address for the # kubelet's httpGet readiness/liveness probes (which dial the pod IP) to ever diff --git a/seaweed-worker/README.md b/seaweed-worker/README.md index 4548348cbe..9dde317711 100644 --- a/seaweed-worker/README.md +++ b/seaweed-worker/README.md @@ -2,7 +2,7 @@ `weed/pb/plugin.proto` is a language-agnostic contract: a maintenance worker connects out to admin, announces the job types it can detect and execute, and -answers requests on that one stream. `weed worker -admin=host:23646` is the Go +answers requests on that one stream. `weed worker -admin=host:12535` is the Go implementation of it from outside the admin process. This workspace is the Rust one. @@ -35,7 +35,7 @@ binary for the runner's platform, resolved from the version in `Cargo.lock`. ## Running - cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646 + cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 The admin's *HTTP* address is what an operator has; the gRPC port is derived from it the way the Go side does. Dialling the HTTP port fails as "frame with @@ -49,7 +49,7 @@ Released builds do not need a toolchain. The worker ships inside the SeaweedFS image, beside the Rust volume server, under the verb that mirrors `volume-rust`: - docker run chrislusf/seaweedfs worker-rust --admin admin:23646 + docker run chrislusf/seaweedfs worker-rust --admin admin:12535 and as `weed-worker_linux_{amd64,arm64}.tar.gz` on each GitHub release. Both are linux amd64/arm64 only โ€” lance, arrow and datafusion make every extra target an @@ -59,7 +59,7 @@ entrypoint says so rather than failing as "not found". ## Metrics - cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646 --metrics-port 9328 + cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 --metrics-port 9328 Serves `/health`, `/ready` and `/metrics` on that port, the same three the Go worker serves under `weed worker -metricsPort`, so one scrape config covers diff --git a/seaweed-worker/crates/core/src/config.rs b/seaweed-worker/crates/core/src/config.rs index 2941c5d457..af3742c15e 100644 --- a/seaweed-worker/crates/core/src/config.rs +++ b/seaweed-worker/crates/core/src/config.rs @@ -3,7 +3,7 @@ use std::time::Duration; /// How one worker process connects and how much work it will take on. #[derive(Clone, Debug)] pub struct WorkerOptions { - /// Admin gRPC address, e.g. "localhost:23646". + /// Admin gRPC address, e.g. "localhost:12535". pub admin_address: String, pub worker_id: String, pub worker_version: String, @@ -44,7 +44,7 @@ pub struct TlsOptions { impl Default for WorkerOptions { fn default() -> Self { Self { - admin_address: "localhost:23646".to_string(), + admin_address: "localhost:12535".to_string(), worker_id: String::new(), worker_version: env!("CARGO_PKG_VERSION").to_string(), worker_address: String::new(), diff --git a/seaweed-worker/crates/core/src/lib.rs b/seaweed-worker/crates/core/src/lib.rs index 8a9a548dc4..d297a7b2f1 100644 --- a/seaweed-worker/crates/core/src/lib.rs +++ b/seaweed-worker/crates/core/src/lib.rs @@ -2,7 +2,7 @@ //! //! `weed/pb/plugin.proto` is a language-agnostic gRPC stream: a worker connects //! out to admin, announces the job types it can detect and execute, and then -//! answers requests on that one stream. `weed worker -admin=host:23646` is the +//! answers requests on that one stream. `weed worker -admin=host:12535` is the //! Go implementation of the same contract from outside the admin process; this //! is the Rust one, and it knows nothing about any particular job. diff --git a/seaweed-worker/crates/lance/src/main.rs b/seaweed-worker/crates/lance/src/main.rs index 4c268c7c1a..8f688cf357 100644 --- a/seaweed-worker/crates/lance/src/main.rs +++ b/seaweed-worker/crates/lance/src/main.rs @@ -15,7 +15,7 @@ use weed_lance_worker::metrics::LanceMetrics; #[command(name = "weed-worker", about = "SeaweedFS maintenance worker")] struct Args { /// Admin server gRPC address. - #[arg(long, default_value = "localhost:23646", env = "WEED_ADMIN")] + #[arg(long, default_value = "localhost:12535", env = "WEED_ADMIN")] admin: String, /// Worker identity reported to admin. diff --git a/terraform/modules/aws/security_group.tf b/terraform/modules/aws/security_group.tf index 93fa32af7d..cc9a5d5c87 100644 --- a/terraform/modules/aws/security_group.tf +++ b/terraform/modules/aws/security_group.tf @@ -1,6 +1,6 @@ # Cluster security group + the SeaweedFS port matrix. # Intra-cluster: all TCP allowed within the SG (master 9333/19333, volume -# 8080/18080, filer 8888/18888, admin 33646, ...). Client-facing: only S3 (8333) +# 8080/18080, filer 8888/18888, admin 22535, ...). Client-facing: only S3 (8333) # and filer (8888). Metrics ports (9327) are never opened to clients. resource "aws_security_group" "cluster" { diff --git a/test/erasure_coding/admin_dockertest/ec_integration_test.go b/test/erasure_coding/admin_dockertest/ec_integration_test.go index 7840aca1d9..d8266cef1a 100644 --- a/test/erasure_coding/admin_dockertest/ec_integration_test.go +++ b/test/erasure_coding/admin_dockertest/ec_integration_test.go @@ -18,7 +18,7 @@ import ( ) const ( - AdminUrl = "http://localhost:23646" + AdminUrl = "http://localhost:12535" MasterUrl = "http://localhost:9333" FilerUrl = "http://localhost:8888" ) @@ -147,13 +147,13 @@ func ensureEnvironment(t *testing.T) { debugPort := 6060 + i dir, _ := filepath.Abs(filepath.Join("tmp", workerName)) os.MkdirAll(dir, 0755) - startWeed(t, workerName, "worker", "-admin=localhost:23646", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort)) + startWeed(t, workerName, "worker", "-admin=localhost:12535", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort)) } // 6. Start Admin os.RemoveAll(filepath.Join("tmp", "admin")) os.MkdirAll(filepath.Join("tmp", "admin"), 0755) - startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin") + startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=12535", "-dataDir=./tmp/admin") // Admin is started after master, 14 volume servers, filer and 2 workers, // so under cold CI conditions the wait here has to absorb the tail of // every earlier subprocess coming up. 60s is too tight and has flaked; diff --git a/test/s3/lifecycle/Makefile b/test/s3/lifecycle/Makefile index 5fafcb10e2..98aec8eceb 100644 --- a/test/s3/lifecycle/Makefile +++ b/test/s3/lifecycle/Makefile @@ -12,7 +12,7 @@ FILER_PORT := 8888 # Pin the filer gRPC port off the FILER_PORT+10000 convention so any # code path that assumes the offset breaks here, not in production. FILER_GRPC_PORT := 18890 -ADMIN_PORT := 23646 +ADMIN_PORT := 12535 ACCESS_KEY ?= some_access_key1 SECRET_KEY ?= some_secret_key1 TEST_TIMEOUT := 10m diff --git a/test/s3/lifecycle/s3_lifecycle_test.go b/test/s3/lifecycle/s3_lifecycle_test.go index dd8bf117a1..a275f29863 100644 --- a/test/s3/lifecycle/s3_lifecycle_test.go +++ b/test/s3/lifecycle/s3_lifecycle_test.go @@ -41,7 +41,7 @@ const ( defaultS3Endpoint = "http://localhost:8333" defaultS3GrpcEndpoint = "localhost:18333" defaultMasterEndpt = "http://localhost:9333" - defaultAdminEndpoint = "http://localhost:23646" + defaultAdminEndpoint = "http://localhost:12535" // Pinned off the FILER_PORT+10000 convention; see Makefile. defaultFilerGRPC = "localhost:18890" bucketLifecycleXMLKey = "s3-bucket-lifecycle-configuration-xml" diff --git a/test/s3/policy/policy_test.go b/test/s3/policy/policy_test.go index 488f528841..2e3e1364e3 100644 --- a/test/s3/policy/policy_test.go +++ b/test/s3/policy/policy_test.go @@ -705,10 +705,10 @@ func uniqueName(prefix string) string { func startMiniCluster(t *testing.T) (*TestCluster, error) { // Allocate two extra ports for admin HTTP + gRPC. Without explicit - // values mini falls back to 23646/33646, which collide across the + // values mini falls back to 12535/22535, which collide across the // sequential subtests in this package: each subtest spins up a fresh // mini in a goroutine, but the previous mini's admin goroutine is - // still binding 23646 โ€” so the next test's admin can never become + // still binding 12535 โ€” so the next test's admin can never become // ready and mini.go fatals on its readiness check. ports := testutil.MustAllocatePorts(t, 10) masterPort, masterGrpcPort := ports[0], ports[1] diff --git a/test/testutil/ports.go b/test/testutil/ports.go index cf14c47f43..0950cf5209 100644 --- a/test/testutil/ports.go +++ b/test/testutil/ports.go @@ -40,7 +40,7 @@ var miniDefaultPorts = []int{ 8181, // s3.port.iceberg 9101, // s3.port.lance 7333, // webdav.port - 23646, // admin.port + 12535, // admin.port } func reservedMiniPorts() map[int]bool { diff --git a/test/testutil/ports_test.go b/test/testutil/ports_test.go index 8d22c51e76..e65f1d99e0 100644 --- a/test/testutil/ports_test.go +++ b/test/testutil/ports_test.go @@ -4,7 +4,7 @@ import "testing" // AllocateMiniPorts must never hand out a port that weed mini will reserve // for one of its default services (or that default's gRPC offset). A real -// failure: Filer was given 33646 (Admin default 23646 + GrpcPortOffset), +// failure: Filer was given 22535 (Admin default 12535 + GrpcPortOffset), // which mini then refused as "reserved for gRPC calculation". func TestAllocateMiniPortsAvoidsMiniDefaults(t *testing.T) { reserved := reservedMiniPorts() diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index c77ab3c391..4f66c9fc46 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -1642,6 +1642,14 @@ func (s *AdminServer) WorkerGrpcMTLSEnabled() bool { return s.workerGrpcServer != nil && s.workerGrpcServer.mtlsEnabled } +// ServeWorkerGrpcOn serves the worker gRPC API on an additional already-bound +// listener, e.g. a legacy compat port kept for backward compatibility. +func (s *AdminServer) ServeWorkerGrpcOn(listener net.Listener) { + if s.workerGrpcServer != nil { + s.workerGrpcServer.ServeOn(listener) + } +} + // StopWorkerGrpcServer stops the worker gRPC server func (s *AdminServer) StopWorkerGrpcServer() error { if s.workerGrpcServer != nil { diff --git a/weed/admin/dash/worker_grpc_server.go b/weed/admin/dash/worker_grpc_server.go index a55e6d7da2..a6e9294d87 100644 --- a/weed/admin/dash/worker_grpc_server.go +++ b/weed/admin/dash/worker_grpc_server.go @@ -133,6 +133,17 @@ func (s *WorkerGrpcServer) StartWithTLS(bindIp string, port int, listener net.Li return nil } +// ServeOn serves the worker gRPC API on an additional already-bound listener, +// e.g. a legacy compat port kept alongside the primary one. grpc.Server tracks +// every listener passed to Serve, so Stop closes it together with the primary. +func (s *WorkerGrpcServer) ServeOn(listener net.Listener) { + go func() { + if err := s.grpcServer.Serve(listener); err != nil && s.running { + glog.Warningf("Worker gRPC compat listener on %s stopped: %v", listener.Addr(), err) + } + }() +} + // ListenPort returns the currently bound worker gRPC listen port. func (s *WorkerGrpcServer) ListenPort() int { if s == nil || s.listener == nil { diff --git a/weed/admin/plugin/DESIGN.md b/weed/admin/plugin/DESIGN.md index 04688cbae1..fb7067d4ee 100644 --- a/weed/admin/plugin/DESIGN.md +++ b/weed/admin/plugin/DESIGN.md @@ -180,9 +180,9 @@ Phase 4 status (starter) Run example: - Start admin: `weed admin -master=localhost:9333` -- Start worker: `weed worker -admin=localhost:23646` -- Optional explicit job type: `weed worker -admin=localhost:23646 -jobType=vacuum` -- Optional stable worker ID persistence: `weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin` +- Start worker: `weed worker -admin=localhost:12535` +- Optional explicit job type: `weed worker -admin=localhost:12535 -jobType=vacuum` +- Optional stable worker ID persistence: `weed worker -admin=localhost:12535 -workingDir=/var/lib/seaweedfs-plugin` Phase 5 - Migrate remaining job types and deprecate old mechanism. diff --git a/weed/command/admin.go b/weed/command/admin.go index acba8f3e9a..a4844f1b8f 100644 --- a/weed/command/admin.go +++ b/weed/command/admin.go @@ -5,6 +5,7 @@ import ( "context" "crypto/rand" "crypto/tls" + "errors" "fmt" "log" "net" @@ -42,6 +43,7 @@ var ( type AdminOptions struct { port *int grpcPort *int + legacyPort *int ip *string master *string masters *string // deprecated, for backward compatibility @@ -68,6 +70,23 @@ type AdminOptions struct { // binds it only after every other service is up. workerGrpcListener net.Listener + // workerGrpcLegacyListener is the same reservation for the legacy + // compat gRPC port. Its bind failure is tolerated: the legacy port lives + // in the ephemeral range, so best-effort compat must not take startup down. + workerGrpcLegacyListener net.Listener + + // workerGrpcLegacySkip disables just the legacy gRPC listener. `weed + // mini` sets it when another mini service already owns the legacy gRPC + // port, so the compat bind can neither steal the port nor warn about a + // failure that is expected. + workerGrpcLegacySkip bool + + // adminHttpLegacySkip is the same for the legacy HTTP port. Skipping + // matters even when the bind would succeed: a service bound on a + // different address family or wildcard can leave loopback free, and the + // compat listener would then shadow it for loopback clients. + adminHttpLegacySkip bool + // workerGrpcBindIp, when non-empty, is the address the worker gRPC // listener binds to. It is separate from ip because the worker gRPC has // no password auth (its mTLS comes from grpc.admin, not https.admin), so @@ -83,8 +102,9 @@ type AdminOptions struct { func init() { cmdAdmin.Run = runAdmin // break init cycle - a.port = cmdAdmin.Flag.Int("port", 23646, "admin server port") + a.port = cmdAdmin.Flag.Int("port", defaultAdminHTTPPort, "admin server port") a.grpcPort = cmdAdmin.Flag.Int("port.grpc", 0, "gRPC server port for worker connections (default: http port + 10000)") + a.legacyPort = cmdAdmin.Flag.Int("port.legacy", defaultAdminHTTPLegacyPort, "also serve admin http and gRPC (port + 10000) on this port for backward compatibility (0 to disable)") a.ip = cmdAdmin.Flag.String("ip", "127.0.0.1", "ip address to listen on. Defaults to loopback when auth is disabled, or 0.0.0.0 when -adminPassword or [https.admin] mTLS is configured. Set explicitly to override.") a.master = cmdAdmin.Flag.String("master", "localhost:9333", "comma-separated master servers") a.masters = cmdAdmin.Flag.String("masters", "", "comma-separated master servers (deprecated, use -master instead)") @@ -108,7 +128,7 @@ func init() { } var cmdAdmin = &Command{ - UsageLine: "admin -port=23646 -master=localhost:9333 [-filerGroup=group] [-port.grpc=33646] [-dataDir=/path/to/data]", + UsageLine: "admin -port=12535 -master=localhost:9333 [-filerGroup=group] [-port.grpc=22535] [-dataDir=/path/to/data]", Short: "start SeaweedFS web admin interface", Long: `Start a web admin interface for SeaweedFS cluster management. @@ -124,12 +144,12 @@ var cmdAdmin = &Command{ A gRPC server for worker connections runs on the configured gRPC port (default: HTTP port + 10000). Example Usage: - weed admin -port=23646 -master="master1:9333,master2:9333" - weed admin -port=23646 -master="localhost:9333" -filerGroup="tenant-a" - weed admin -port=23646 -master="localhost:9333" -dataDir="/var/lib/seaweedfs-admin" - weed admin -port=23646 -port.grpc=33646 -master="localhost:9333" -dataDir="~/seaweedfs-admin" + weed admin -port=12535 -master="master1:9333,master2:9333" + weed admin -port=12535 -master="localhost:9333" -filerGroup="tenant-a" + weed admin -port=12535 -master="localhost:9333" -dataDir="/var/lib/seaweedfs-admin" + weed admin -port=12535 -port.grpc=22535 -master="localhost:9333" -dataDir="~/seaweedfs-admin" weed admin -port=9900 -port.grpc=19900 -master="localhost:9333" - weed admin -port=23646 -master="localhost:9333" -urlPrefix="/seaweedfs" + weed admin -port=12535 -master="localhost:9333" -urlPrefix="/seaweedfs" Data Directory: - If dataDir is specified, admin configuration and maintenance data is persisted @@ -295,6 +315,9 @@ func runAdmin(cmd *Command, args []string) bool { if *a.grpcPort == 0 { *a.grpcPort = *a.port + 10000 } + if *a.grpcPort >= linuxEphemeralPortFloor { + glog.Warningf("Admin gRPC port %d is at or above the Linux ephemeral floor %d; an outbound connection may claim it before Admin binds it. Pick a lower -port.", *a.grpcPort, linuxEphemeralPortFloor) + } hasMTLS := viper.GetString("https.admin.key") != "" && viper.GetString("https.admin.ca") != "" @@ -504,6 +527,31 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, } warnInsecureWorkerGrpcBind(workerGrpcIp, *options.grpcPort, adminServer.WorkerGrpcMTLSEnabled()) + // Legacy compat listeners keep the previous default admin port reachable + // for old workers, bookmarks and scripts while the deployment moves to the + // new default. Best-effort only: the legacy gRPC port sits inside the + // Linux ephemeral range, so it may already be taken and must never fail + // startup. + legacyPort := 0 + if options.legacyPort != nil && *options.legacyPort > 0 && *options.legacyPort != *options.port { + legacyPort = *options.legacyPort + } + if legacyPort > 0 && !options.workerGrpcLegacySkip { + legacyGrpcPort := legacyPort + GrpcPortOffset + legacyGrpcListener := options.workerGrpcLegacyListener + if legacyGrpcListener == nil { + if l, listenErr := net.Listen("tcp", util.JoinHostPort(workerGrpcIp, legacyGrpcPort)); listenErr == nil { + legacyGrpcListener = l + } else { + glog.Warningf("Legacy admin gRPC port %d unavailable, skipping compat listener: %v", legacyGrpcPort, listenErr) + } + } + if legacyGrpcListener != nil { + adminServer.ServeWorkerGrpcOn(legacyGrpcListener) + glog.Infof("Worker gRPC compat listener on %s", legacyGrpcListener.Addr()) + } + } + // Set up cleanup for gRPC server defer func() { if stopErr := adminServer.StopWorkerGrpcServer(); stopErr != nil { @@ -578,6 +626,43 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, server.TLSConfig.GetCertificate = getCert } + // Serve the same handler on the legacy port so old clients keep working. + // Best-effort like the gRPC compat listener above. + var legacyServer *http.Server + if legacyPort > 0 && !options.adminHttpLegacySkip { + legacyAddr := util.JoinHostPort(*options.ip, legacyPort) + if legacyListener, listenErr := net.Listen("tcp", legacyAddr); listenErr != nil { + glog.Warningf("Legacy admin HTTP port %d unavailable, skipping compat listener: %v", legacyPort, listenErr) + } else { + legacyServer = &http.Server{ + Addr: legacyAddr, + Handler: server.Handler, + TLSConfig: server.TLSConfig, + } + // Close on every exit path so a failed primary startup cannot + // leave the compat port held by a zombie admin. On the normal + // path the graceful Shutdown below runs first; this Close then + // no-ops. Close the raw listener too: Serve may not have + // registered it with legacyServer yet when this runs. + defer func() { + _ = legacyListener.Close() + _ = legacyServer.Close() + }() + go func() { + glog.Infof("Admin compat listener on %s", legacyAddr) + var serveErr error + if useTLS { + serveErr = legacyServer.ServeTLS(legacyListener, "", "") + } else { + serveErr = legacyServer.Serve(legacyListener) + } + if serveErr != nil && serveErr != http.ErrServerClosed && !errors.Is(serveErr, net.ErrClosed) { + glog.Warningf("Admin compat listener on %s stopped: %v", legacyAddr, serveErr) + } + }() + } + } + // Start server. Surface immediate failures (e.g. bind error) via // serveErrCh so the caller doesn't block on ctx.Done() while no server // is actually listening. http.ErrServerClosed after Shutdown is normal @@ -612,6 +697,9 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, if err := server.Shutdown(shutdownCtx); err != nil { return fmt.Errorf("admin server forced to shutdown: %w", err) } + if legacyServer != nil { + _ = legacyServer.Shutdown(shutdownCtx) + } adminServer.Shutdown() diff --git a/weed/command/mini.go b/weed/command/mini.go index 853c8e0077..1ded2f936c 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -49,13 +49,16 @@ type MiniOptions struct { } const ( - bytesPerMB = 1024 * 1024 // Bytes per MB - miniVolumeMinFreeSpace = "1" // 1% minimum free space - minVolumeSizeMB = 64 // Minimum volume size in MB - defaultMiniVolumeSizeMB = 128 // Default volume size for mini mode - maxVolumeSizeMB = 1024 // Maximum volume size in MB (1GB) - GrpcPortOffset = 10000 // Offset used to calculate gRPC port from HTTP port - defaultMiniPluginJobTypes = "all" + bytesPerMB = 1024 * 1024 // Bytes per MB + miniVolumeMinFreeSpace = "1" // 1% minimum free space + minVolumeSizeMB = 64 // Minimum volume size in MB + defaultMiniVolumeSizeMB = 128 // Default volume size for mini mode + maxVolumeSizeMB = 1024 // Maximum volume size in MB (1GB) + GrpcPortOffset = 10000 // Offset used to calculate gRPC port from HTTP port + linuxEphemeralPortFloor = 32768 // Default net.ipv4.ip_local_port_range floor on Linux + defaultAdminHTTPPort = 12535 // Keeps the derived admin gRPC port (port+GrpcPortOffset) below linuxEphemeralPortFloor + defaultAdminHTTPLegacyPort = 23646 // Pre-12535 default; served alongside the new port for backward compatibility + defaultMiniPluginJobTypes = "all" ) var ( @@ -371,12 +374,12 @@ After starting, you can access: - S3 Endpoint: http://localhost:8333 - Iceberg Catalog: http://localhost:8181 - WebDAV: http://localhost:7333 -- Admin UI: http://localhost:23646 +- Admin UI: http://localhost:12535 S3 Access: The S3 endpoint is available at http://localhost:8333. For client configuration and IAM setup, see the project documentation or use the -Admin UI (http://localhost:23646) to manage users and policies. +Admin UI (http://localhost:12535) to manage users and policies. Notes: - Values mini computes itself (currently master.volumeSizeLimitMB) are @@ -574,8 +577,9 @@ func initMiniWebDAVFlags() { // initMiniAdminFlags initializes Admin server flag options func initMiniAdminFlags() { - miniAdminOptions.port = cmdMini.Flag.Int("admin.port", 23646, "admin server http listen port") + miniAdminOptions.port = cmdMini.Flag.Int("admin.port", defaultAdminHTTPPort, "admin server http listen port") miniAdminOptions.grpcPort = cmdMini.Flag.Int("admin.port.grpc", 0, "admin server grpc listen port (default: admin http port + GrpcPortOffset)") + miniAdminOptions.legacyPort = cmdMini.Flag.Int("admin.port.legacy", defaultAdminHTTPLegacyPort, "also serve admin http and grpc (port + GrpcPortOffset) on this port for backward compatibility (0 to disable)") miniAdminOptions.master = cmdMini.Flag.String("admin.master", "", "master server address (automatically set)") // admin discovery must query the same group the co-located filer registers under miniAdminOptions.filerGroup = miniFilerOptions.filerGroup @@ -966,7 +970,7 @@ func ensureAllPortsAvailableOnIP(bindIp string) error { // Initialize all gRPC ports before services start // This ensures they won't be recalculated and cause conflicts // All gRPC port handling (calculation, validation, and assignment) is performed exclusively in initializeGrpcPortsOnIP - initializeGrpcPortsOnIP(bindIp) + allocatedPorts := initializeGrpcPortsOnIP(bindIp) // Every other service binds within a moment of this check, but the admin // waits for all of them first. The admin gRPC port sits inside the Linux @@ -986,6 +990,33 @@ func ensureAllPortsAvailableOnIP(bindIp string) error { miniAdminOptions.workerGrpcListener = listener } + // Same early reservation for the legacy compat gRPC port โ€” but never take + // a port a mini service already owns: compat is a convenience, not a + // startup requirement. Failure is only a warning โ€” the legacy port sits + // inside the ephemeral range. Without a reservation the admin's late + // best-effort bind simply fails harmlessly if the port is taken by then. + miniAdminOptions.workerGrpcLegacyListener = nil + miniAdminOptions.workerGrpcLegacySkip = false + miniAdminOptions.adminHttpLegacySkip = false + if *miniAdminOptions.legacyPort > 0 && *miniAdminOptions.legacyPort != *miniAdminOptions.port { + if allocatedPorts[*miniAdminOptions.legacyPort] { + glog.Warningf("Legacy Admin HTTP port %d is used by another mini service, skipping compat listener", *miniAdminOptions.legacyPort) + miniAdminOptions.adminHttpLegacySkip = true + } + legacyGrpcPort := *miniAdminOptions.legacyPort + GrpcPortOffset + switch { + case allocatedPorts[legacyGrpcPort]: + glog.Warningf("Legacy Admin gRPC port %d is used by another mini service, skipping compat listener", legacyGrpcPort) + miniAdminOptions.workerGrpcLegacySkip = true + default: + if listener, err := listenMiniAdminWorker(workerGrpcBindIP, legacyGrpcPort); err != nil { + glog.Warningf("Could not reserve legacy Admin gRPC port %d: %v", legacyGrpcPort, err) + } else { + miniAdminOptions.workerGrpcLegacyListener = listener + } + } + } + // Log the final port configuration icebergPortStr := "disabled" if miniS3Options.portIceberg != nil && *miniS3Options.portIceberg > 0 { @@ -1016,7 +1047,9 @@ func listenMiniAdminWorker(bindIP string, port int) (net.Listener, error) { // initializeGrpcPortsOnIP initializes all gRPC ports based on their HTTP ports on a specific IP // If a gRPC port is 0, it will be set to httpPort + GrpcPortOffset // This must be called after HTTP ports are finalized and before services start -func initializeGrpcPortsOnIP(bindIp string) { +// Returns the set of ports allocated to mini services, for callers that must +// keep optional listeners from stealing them. +func initializeGrpcPortsOnIP(bindIp string) map[int]bool { // Track all ports allocated (both HTTP and gRPC) to prevent collisions. // We must reserve HTTP ports so that gRPC fallback allocation never picks // a port already assigned to an HTTP service (which hasn't bound yet). @@ -1086,6 +1119,15 @@ func initializeGrpcPortsOnIP(bindIp string) { allocatedPorts[*config.grpcPort] = true glog.V(1).Infof("%s gRPC port set to %d", config.name, *config.grpcPort) } + + // The Linux kernel draws outbound source ports from the ephemeral range. + // A gRPC port inside it can be claimed by one of mini's own startup dials + // before the service binds it. + if *miniAdminOptions.grpcPort >= linuxEphemeralPortFloor { + glog.Warningf("Admin gRPC port %d is at or above the Linux ephemeral floor %d; an outbound connection may claim it before Admin binds it. Pick a lower -admin.port.", *miniAdminOptions.grpcPort, linuxEphemeralPortFloor) + } + + return allocatedPorts } // miniOptionsFilePath returns the path of the mini.options file under the @@ -1880,6 +1922,9 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) { if adminOptions.workerGrpcListener != nil { adminOptions.workerGrpcListener.Close() } + if adminOptions.workerGrpcLegacyListener != nil { + adminOptions.workerGrpcLegacyListener.Close() + } }() // Wait for admin server's HTTP port to be ready before launching worker diff --git a/weed/command/mini_admin_auth_test.go b/weed/command/mini_admin_auth_test.go index 85fe8f2ad1..7eec8a3914 100644 --- a/weed/command/mini_admin_auth_test.go +++ b/weed/command/mini_admin_auth_test.go @@ -2,6 +2,7 @@ package command import ( "net" + "strconv" "testing" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -254,3 +255,32 @@ func TestMiniAdminAdvertisedIP(t *testing.T) { }) } } + +func TestDefaultAdminGrpcPortBelowEphemeralFloor(t *testing.T) { + if grpcPort := defaultAdminHTTPPort + GrpcPortOffset; grpcPort >= linuxEphemeralPortFloor { + t.Fatalf("default admin HTTP port %d derives gRPC port %d, which must stay below ephemeral floor %d", + defaultAdminHTTPPort, grpcPort, linuxEphemeralPortFloor) + } + for _, f := range []struct { + name string + flagName string + def string + }{ + {"mini", "admin.port", strconv.Itoa(defaultAdminHTTPPort)}, + {"mini", "admin.port.legacy", strconv.Itoa(defaultAdminHTTPLegacyPort)}, + {"admin", "port", strconv.Itoa(defaultAdminHTTPPort)}, + {"admin", "port.legacy", strconv.Itoa(defaultAdminHTTPLegacyPort)}, + } { + cmd := cmdMini + if f.name == "admin" { + cmd = cmdAdmin + } + flag := cmd.Flag.Lookup(f.flagName) + if flag == nil { + t.Fatalf("%s: flag %s not registered", f.name, f.flagName) + } + if flag.DefValue != f.def { + t.Fatalf("%s -%s default = %s, want %s so its gRPC port stays below the ephemeral floor", f.name, f.flagName, flag.DefValue, f.def) + } + } +} diff --git a/weed/command/worker.go b/weed/command/worker.go index bddb1c4241..1579c42fe9 100644 --- a/weed/command/worker.go +++ b/weed/command/worker.go @@ -1,6 +1,7 @@ package command import ( + "fmt" "time" "github.com/seaweedfs/seaweedfs/weed/util" @@ -25,21 +26,21 @@ Behavior: - Use -metricsPort/-metricsIp to expose /health, /ready, and /metrics Examples: - weed worker -admin=localhost:23646 - weed worker -admin=localhost:23646 -jobType=all - weed worker -admin=localhost:23646 -jobType=default - weed worker -admin=localhost:23646 -jobType=heavy - weed worker -admin=localhost:23646 -jobType=default,iceberg - weed worker -admin=localhost:23646 -jobType=vacuum,volume_balance - weed worker -admin=localhost:23646 -jobType=erasure_coding - weed worker -admin=admin.example.com:23646 -id=plugin-vacuum-a -heartbeat=10s - weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin - weed worker -admin=localhost:23646 -metricsPort=9327 -metricsIp=0.0.0.0 + weed worker -admin=localhost:12535 + weed worker -admin=localhost:12535 -jobType=all + weed worker -admin=localhost:12535 -jobType=default + weed worker -admin=localhost:12535 -jobType=heavy + weed worker -admin=localhost:12535 -jobType=default,iceberg + weed worker -admin=localhost:12535 -jobType=vacuum,volume_balance + weed worker -admin=localhost:12535 -jobType=erasure_coding + weed worker -admin=admin.example.com:12535 -id=plugin-vacuum-a -heartbeat=10s + weed worker -admin=localhost:12535 -workingDir=/var/lib/seaweedfs-plugin + weed worker -admin=localhost:12535 -metricsPort=9327 -metricsIp=0.0.0.0 `, } var ( - workerAdminServer = cmdWorker.Flag.String("admin", "localhost:23646", "admin server address") + workerAdminServer = cmdWorker.Flag.String("admin", fmt.Sprintf("localhost:%d", defaultAdminHTTPPort), "admin server address") workerID = cmdWorker.Flag.String("id", "", "worker ID (auto-generated when empty)") workerWorkingDir = cmdWorker.Flag.String("workingDir", "", "working directory for persistent worker state") workerJobType = cmdWorker.Flag.String("jobType", defaultPluginWorkerJobTypes, "job types or categories to serve: all, default, heavy, or explicit names/aliases such as ec, balance, iceberg (comma-separated)") diff --git a/weed/command/worker_runtime.go b/weed/command/worker_runtime.go index a60aa269c0..ff8d00d94d 100644 --- a/weed/command/worker_runtime.go +++ b/weed/command/worker_runtime.go @@ -45,7 +45,7 @@ func runPluginWorkerWithOptions(options pluginWorkerRunOptions) bool { options.AdminServer = strings.TrimSpace(options.AdminServer) if options.AdminServer == "" { - options.AdminServer = "localhost:23646" + options.AdminServer = fmt.Sprintf("localhost:%d", defaultAdminHTTPPort) } options.JobTypes = strings.TrimSpace(options.JobTypes)