shell: retry filer discovery until master knows a filer (#11989)

* shell: discover the filer on first use instead of at startup

The startup retry loop made every weed shell invocation wait out the
connect timeout on clusters that have no filer, even for commands that
only talk to the master. Discovery now runs once, on the first command
that actually needs a filer, keeping the retry-until-registered fix for
the fresh-cluster race without stalling master-only commands.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* shell: stop filer discovery retries on non-transient errors

A master that keeps answering ListClusterNodes with a permanent error
(permission denied, unimplemented) cannot gain a filer by retrying, so
discovery returns instead of polling until the deadline.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* shell: explain an empty filer list when discovery times out

* shell: retry filer discovery while the address is still empty

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Chris LuandDevin authored and GitHub committed 2026-10-23 22:52:51 -07:00
1 parent b86b2f3cee
commit 6a34d82f05
18 files changed
+246 -325

No files matched your search

-66
View File
@@ -3,19 +3,14 @@ package shell
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"math/rand/v2"
"os"
"path"
"slices"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
@@ -26,47 +21,6 @@ import (
var historyPath = path.Join(os.TempDir(), "weed-shell")
// errNoFilersDiscovered marks a successful ListClusterNodes call that came
// back empty, which usually means the filer has not registered with the
// master yet. Unlike an RPC failure it is expected at startup, so filer
// discovery retries on it until the connect timeout expires.
var errNoFilersDiscovered = errors.New("no filer registered with master yet")
// listFilers polls ListClusterNodes until at least one filer is registered
// with the master or ctx expires. An empty list is expected while a filer is
// still starting, so it retries like a transient failure. On return, a nil
// error with an empty list means the master answered but no filer registered
// in time; a non-nil error is the last attempt's failure.
func listFilers(ctx context.Context, commandEnv *CommandEnv, filerGroup string, maxWait time.Duration) (filers []pb.ServerAddress, listErr error) {
util.RetryWithBackoff(ctx, "filer discovery", maxWait,
func(err error) bool {
return errors.Is(err, errNoFilersDiscovered) || util.IsTransientError(err)
}, func() error {
filers = filers[:0]
listErr = commandEnv.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
ClientType: cluster.FilerType,
FilerGroup: filerGroup,
})
if err != nil {
return err
}
for _, clusterNode := range resp.ClusterNodes {
filers = append(filers, pb.ServerAddress(clusterNode.Address))
}
return nil
})
if listErr != nil {
return listErr
}
if len(filers) == 0 {
return errNoFilersDiscovered
}
return nil
})
return filers, listErr
}
// Piped stdin returns the last command failure; an interactive session shows
// the error to the operator and keeps going.
func RunShell(options ShellOptions) error {
@@ -116,26 +70,6 @@ func RunShell(options ShellOptions) error {
return err
}
if commandEnv.option.FilerAddress == "" {
discoverCtx, cancelDiscover := context.WithTimeout(ctx, connectTimeout)
defer cancelDiscover()
filers, listErr := listFilers(discoverCtx, commandEnv, *options.FilerGroup, connectTimeout)
if len(filers) > 0 {
commandEnv.option.FilerAddress = filers[rand.IntN(len(filers))]
} else if listErr != nil {
fmt.Fprintf(os.Stderr, "cannot list filers from master %s: %v\n", *options.Masters, listErr)
} else {
fmt.Fprintf(os.Stderr, "no filer registered with master %s within %s\n", *options.Masters, connectTimeout)
}
if options.Debug {
if len(filers) > 0 {
fmt.Fprintf(os.Stderr, "master: %s filers: %v\n", *options.Masters, filers)
} else {
fmt.Fprintf(os.Stderr, "master: %s\n", *options.Masters)
}
}
}
if interactive {
for {
cmd, err := line.Prompt("> ")