package testrunner import ( "archive/tar" "compress/gzip" "context" "encoding/json" "fmt" "io" "log" "net" "net/http" "os" "path/filepath" "strings" "sync" "time" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra" ) // Agent runs on each test node, executing actions locally on behalf of the coordinator. type Agent struct { ID string Hostname string Port int CoordinatorURL string Token string AllowExec bool Persistent bool // stay running across coordinator restarts Nodes []string // node names this agent handles registry *Registry localNode *LocalNode startTime time.Time log *log.Logger mu sync.Mutex server *http.Server listener net.Listener } // AgentConfig holds configuration for creating an Agent. type AgentConfig struct { Hostname string Port int CoordinatorURL string Token string AllowExec bool Persistent bool // stay running, re-register with coordinator Nodes []string Registry *Registry Logger *log.Logger } // NewAgent creates a new Agent with the given configuration. func NewAgent(cfg AgentConfig) *Agent { hostname := cfg.Hostname if hostname == "" { hostname, _ = os.Hostname() } agentID := fmt.Sprintf("agent-%s-%d", hostname, os.Getpid()) logger := cfg.Logger if logger == nil { logger = log.New(os.Stderr, "[agent] ", log.LstdFlags) } return &Agent{ ID: agentID, Hostname: hostname, Port: cfg.Port, CoordinatorURL: cfg.CoordinatorURL, Token: cfg.Token, AllowExec: cfg.AllowExec, Persistent: cfg.Persistent, Nodes: cfg.Nodes, registry: cfg.Registry, localNode: NewLocalNode(hostname), startTime: time.Now(), log: logger, } } // Start begins listening for coordinator requests. Blocks until the server is shut down. func (a *Agent) Start(ctx context.Context) error { mux := http.NewServeMux() mux.HandleFunc("/health", a.handleHealth) mux.HandleFunc("/phase", a.authMiddleware(a.handlePhase)) mux.HandleFunc("/upload", a.authMiddleware(a.handleUpload)) mux.HandleFunc("/artifacts", a.authMiddleware(a.handleArtifacts)) mux.HandleFunc("/exec", a.authMiddleware(a.handleExec)) addr := fmt.Sprintf(":%d", a.Port) ln, err := net.Listen("tcp", addr) if err != nil { return fmt.Errorf("listen %s: %w", addr, err) } a.mu.Lock() a.listener = ln a.server = &http.Server{Handler: mux} a.mu.Unlock() a.log.Printf("agent %s listening on %s (root=%v, exec=%v)", a.ID, addr, a.localNode.IsRoot(), a.AllowExec) // Register with coordinator if URL is set. if a.CoordinatorURL != "" { if a.Persistent { go a.registrationLoop(ctx) } else { go a.registerWithCoordinator() } } // Shutdown on context cancellation. go func() { <-ctx.Done() a.Stop() }() if err := a.server.Serve(ln); err != nil && err != http.ErrServerClosed { return err } return nil } // Stop gracefully shuts down the agent HTTP server. func (a *Agent) Stop() { a.mu.Lock() srv := a.server a.mu.Unlock() if srv != nil { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() srv.Shutdown(ctx) } } // ListenAddr returns the address the agent is listening on (useful in tests). func (a *Agent) ListenAddr() string { a.mu.Lock() defer a.mu.Unlock() if a.listener != nil { return a.listener.Addr().String() } return "" } func (a *Agent) registerWithCoordinator() { req := RegisterRequest{ AgentID: a.ID, Hostname: a.Hostname, IP: a.detectIP(), Port: a.Port, Nodes: a.Nodes, } // Detect capabilities. var caps []string if a.localNode.IsRoot() { caps = append(caps, "root") } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if _, _, code, err := a.localNode.Run(ctx, "which iscsiadm"); err == nil && code == 0 { caps = append(caps, "iscsi") } if _, _, code, err := a.localNode.Run(ctx, "which fio"); err == nil && code == 0 { caps = append(caps, "fio") } req.Capabilities = caps body, _ := json.Marshal(req) url := strings.TrimRight(a.CoordinatorURL, "/") + "/register" // Retry registration up to 60s with 2s intervals. deadline := time.Now().Add(60 * time.Second) for attempt := 1; ; attempt++ { httpReq, _ := http.NewRequest("POST", url, strings.NewReader(string(body))) httpReq.Header.Set("Content-Type", "application/json") if a.Token != "" { httpReq.Header.Set(AuthTokenHeader, a.Token) } resp, err := http.DefaultClient.Do(httpReq) if err != nil { if time.Now().After(deadline) { a.log.Printf("register failed after retries: %v", err) return } a.log.Printf("register attempt %d failed: %v (retrying...)", attempt, err) time.Sleep(2 * time.Second) continue } var rr RegisterResponse json.NewDecoder(resp.Body).Decode(&rr) resp.Body.Close() if rr.OK { a.log.Printf("registered with coordinator (index=%d, total=%d)", rr.AgentIndex, rr.TotalAgents) } else { a.log.Printf("registration rejected: %s", rr.Error) } return } } // registrationLoop continuously re-registers with the coordinator. // Used in persistent mode so the agent survives coordinator restarts. func (a *Agent) registrationLoop(ctx context.Context) { req := a.buildRegisterRequest() body, _ := json.Marshal(req) url := strings.TrimRight(a.CoordinatorURL, "/") + "/register" registered := false for { select { case <-ctx.Done(): return default: } httpReq, _ := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(body))) httpReq.Header.Set("Content-Type", "application/json") if a.Token != "" { httpReq.Header.Set(AuthTokenHeader, a.Token) } resp, err := http.DefaultClient.Do(httpReq) if err != nil { if registered { a.log.Printf("coordinator unreachable, will retry: %v", err) registered = false } sleepCtx(ctx, 5*time.Second) continue } var rr RegisterResponse json.NewDecoder(resp.Body).Decode(&rr) resp.Body.Close() if rr.OK { if !registered { a.log.Printf("registered with coordinator (index=%d, total=%d)", rr.AgentIndex, rr.TotalAgents) registered = true } sleepCtx(ctx, 10*time.Second) } else { a.log.Printf("registration rejected: %s", rr.Error) sleepCtx(ctx, 5*time.Second) } } } // buildRegisterRequest creates the registration payload with capability detection. func (a *Agent) buildRegisterRequest() RegisterRequest { req := RegisterRequest{ AgentID: a.ID, Hostname: a.Hostname, IP: a.detectIP(), Port: a.Port, Nodes: a.Nodes, } var caps []string if a.localNode.IsRoot() { caps = append(caps, "root") } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if _, _, code, err := a.localNode.Run(ctx, "which iscsiadm"); err == nil && code == 0 { caps = append(caps, "iscsi") } if _, _, code, err := a.localNode.Run(ctx, "which fio"); err == nil && code == 0 { caps = append(caps, "fio") } req.Capabilities = caps return req } // sleepCtx sleeps for d or until ctx is cancelled. func sleepCtx(ctx context.Context, d time.Duration) { select { case <-time.After(d): case <-ctx.Done(): } } func (a *Agent) detectIP() string { addrs, err := net.InterfaceAddrs() if err != nil { return "" } for _, addr := range addrs { if ipNet, ok := addr.(*net.IPNet); ok && !ipNet.IP.IsLoopback() && ipNet.IP.To4() != nil { return ipNet.IP.String() } } return "" } // authMiddleware wraps a handler with token authentication. func (a *Agent) authMiddleware(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if a.Token != "" { token := r.Header.Get(AuthTokenHeader) if token != a.Token { writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "invalid auth token"}) return } } next(w, r) } } // GET /health func (a *Agent) handleHealth(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } resp := HealthResponse{ OK: true, AgentID: a.ID, UptimeS: int64(time.Since(a.startTime).Seconds()), HasRoot: a.localNode.IsRoot(), Hostname: a.Hostname, } writeJSON(w, http.StatusOK, resp) } // POST /phase func (a *Agent) handlePhase(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } var req PhaseRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeJSON(w, http.StatusBadRequest, PhaseResponse{Error: fmt.Sprintf("decode: %v", err)}) return } a.log.Printf("phase %d (%s): %d actions", req.PhaseIndex, req.PhaseName, len(req.Actions)) resp := a.executePhase(r.Context(), &req) writeJSON(w, http.StatusOK, resp) } // executePhase runs actions sequentially using the local engine. func (a *Agent) executePhase(ctx context.Context, req *PhaseRequest) PhaseResponse { resp := PhaseResponse{ PhaseIndex: req.PhaseIndex, NewVars: make(map[string]string), } // Build action context. actx := &ActionContext{ Scenario: req.Scenario, Nodes: make(map[string]NodeRunner), Targets: make(map[string]TargetRunner), Vars: make(map[string]string), Log: func(format string, args ...interface{}) { a.log.Printf(format, args...) }, } // Copy global vars. for k, v := range req.GlobalVars { actx.Vars[k] = v } // Create infra.Node in native mode for local command execution. nativeNode := &infra.Node{IsNative: true} // Map agent's nodes to native node runner. for _, nodeName := range a.Nodes { actx.Nodes[nodeName] = nativeNode } // Set up targets for nodes this agent handles. myNodes := make(map[string]bool) for _, n := range a.Nodes { myNodes[n] = true } if req.Scenario != nil { // Map node names to agents via topology. nodeToAgentName := make(map[string]string) for nodeName, nodeSpec := range req.Scenario.Topology.Nodes { nodeToAgentName[nodeName] = nodeSpec.Agent } for tgtName, tgtSpec := range req.Scenario.Targets { agentName := nodeToAgentName[tgtSpec.Node] if !myNodes[agentName] { continue } htSpec := infra.HATargetSpec{ VolSize: tgtSpec.VolSize, WALSize: tgtSpec.WALSize, IQN: tgtSpec.IQN(), ISCSIPort: tgtSpec.ISCSIPort, AdminPort: tgtSpec.AdminPort, ReplicaDataPort: tgtSpec.ReplicaDataPort, ReplicaCtrlPort: tgtSpec.ReplicaCtrlPort, RebuildPort: tgtSpec.RebuildPort, TPGID: tgtSpec.TPGID, NvmePort: tgtSpec.NvmePort, NQN: tgtSpec.NQN(), MaxConcurrentWrites: tgtSpec.MaxConcurrentWrites, NvmeIOQueues: tgtSpec.NvmeIOQueues, } actx.Targets[tgtName] = infra.NewHATargetFromSpec(nativeNode, tgtName, htSpec) } // Also map topology node names to native node if this agent handles them. for nodeName, nodeSpec := range req.Scenario.Topology.Nodes { if myNodes[nodeSpec.Agent] { actx.Nodes[nodeName] = nativeNode } } } engine := NewEngine(a.registry, actx.Log) for _, act := range req.Actions { start := time.Now() resolved := resolveAction(act, actx.Vars) yamlDef := marshalActionYAML(resolved) handler, err := engine.registry.Get(resolved.Action) if err != nil { resp.Results = append(resp.Results, ActionResult{ Action: resolved.Action, Status: StatusFail, Duration: time.Since(start), Error: err.Error(), }) resp.Error = err.Error() return resp } // Handle delay param. if d, ok := resolved.Params["delay"]; ok { if dur, err := time.ParseDuration(d); err == nil { select { case <-time.After(dur): case <-ctx.Done(): resp.Error = ctx.Err().Error() return resp } } } output, execErr := handler.Execute(ctx, actx, resolved) ar := ActionResult{ Action: resolved.Action, Duration: time.Since(start), YAML: yamlDef, } if execErr != nil { ar.Status = StatusFail ar.Error = execErr.Error() if !act.IgnoreError { resp.Results = append(resp.Results, ar) resp.Error = execErr.Error() return resp } ar.Status = StatusPass } else { ar.Status = StatusPass } // Store save_as and __ vars. if resolved.SaveAs != "" && output != nil { if v, ok := output["value"]; ok { actx.Vars[resolved.SaveAs] = v resp.NewVars[resolved.SaveAs] = v } } if output != nil { for k, v := range output { if strings.HasPrefix(k, "__") { actx.Vars[k] = v resp.NewVars[k] = v } } if v, ok := output["value"]; ok { ar.Output = truncate(v, 4096) } } resp.Results = append(resp.Results, ar) } return resp } // POST /upload — streaming binary upload with path safety. func (a *Agent) handleUpload(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } path := r.URL.Query().Get("path") if path == "" { writeJSON(w, http.StatusBadRequest, UploadResponse{Error: "path query parameter required"}) return } // Path safety: must start with UploadBasePath and no traversal. if !isPathSafe(path) { writeJSON(w, http.StatusForbidden, UploadResponse{Error: fmt.Sprintf("path must start with %s and contain no '..'", UploadBasePath)}) return } // Ensure directory exists. dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0755); err != nil { writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("mkdir: %v", err)}) return } f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) if err != nil { writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("create file: %v", err)}) return } n, err := io.Copy(f, r.Body) f.Close() if err != nil { os.Remove(path) writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("write: %v", err)}) return } a.log.Printf("uploaded %d bytes to %s", n, path) writeJSON(w, http.StatusOK, UploadResponse{OK: true, Size: n, Path: path}) } // GET /artifacts — stream a directory as tar.gz. func (a *Agent) handleArtifacts(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } dir := r.URL.Query().Get("dir") if dir == "" { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "dir query parameter required"}) return } // Path safety: must be under UploadBasePath, no traversal. if !isPathSafe(dir) { writeJSON(w, http.StatusForbidden, map[string]string{"error": fmt.Sprintf("dir must be under %s with no '..'", UploadBasePath)}) return } info, err := os.Stat(dir) if err != nil || !info.IsDir() { writeJSON(w, http.StatusNotFound, map[string]string{"error": "directory not found"}) return } // Check if directory has any files. entries, err := os.ReadDir(dir) if err != nil || len(entries) == 0 { writeJSON(w, http.StatusNotFound, map[string]string{"error": "directory empty"}) return } w.Header().Set("Content-Type", "application/gzip") w.WriteHeader(http.StatusOK) gw := gzip.NewWriter(w) defer gw.Close() tw := tar.NewWriter(gw) defer tw.Close() filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { if err != nil { return nil // skip unreadable files } if fi.IsDir() { return nil } // Relative path inside the tar. rel, err := filepath.Rel(dir, path) if err != nil { return nil } // Normalize to forward slashes for tar. rel = filepath.ToSlash(rel) hdr := &tar.Header{ Name: rel, Size: fi.Size(), Mode: int64(fi.Mode()), ModTime: fi.ModTime(), } if err := tw.WriteHeader(hdr); err != nil { return err } f, err := os.Open(path) if err != nil { return nil } defer f.Close() io.Copy(tw, f) return nil }) } // POST /exec — ad-hoc command execution (disabled by default). func (a *Agent) handleExec(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } if !a.AllowExec { writeJSON(w, http.StatusForbidden, ExecResponse{Error: "exec endpoint disabled; start agent with --allow-exec"}) return } var req ExecRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeJSON(w, http.StatusBadRequest, ExecResponse{Error: fmt.Sprintf("decode: %v", err)}) return } ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() var stdout, stderr string var exitCode int var err error if req.Root { stdout, stderr, exitCode, err = a.localNode.RunRoot(ctx, req.Cmd) } else { stdout, stderr, exitCode, err = a.localNode.Run(ctx, req.Cmd) } if err != nil { writeJSON(w, http.StatusOK, ExecResponse{ Stdout: stdout, Stderr: stderr, ExitCode: -1, Error: err.Error(), }) return } writeJSON(w, http.StatusOK, ExecResponse{ Stdout: stdout, Stderr: stderr, ExitCode: exitCode, }) } // isPathSafe checks that a path is under UploadBasePath and has no traversal. func isPathSafe(p string) bool { if strings.Contains(p, "..") { return false } // Use forward-slash path cleaning (Unix paths, even on Windows coordinator). cleaned := posixClean(p) return strings.HasPrefix(cleaned, UploadBasePath) } // posixClean normalizes a path using forward slashes (Unix convention). func posixClean(p string) string { // Replace backslashes, then clean. p = strings.ReplaceAll(p, "\\", "/") // Remove double slashes. for strings.Contains(p, "//") { p = strings.ReplaceAll(p, "//", "/") } // Remove trailing slash (unless root). if len(p) > 1 && strings.HasSuffix(p, "/") { p = p[:len(p)-1] } return p } func writeJSON(w http.ResponseWriter, status int, v interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) json.NewEncoder(w).Encode(v) }