mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
* filer: fail health check when absent from lock ring If the master broadcasts a lock ring that does not include this filer (e.g. the late-cleanup race in seaweedfs#11442), every lock acquisition fails with "no lock server found" and locked writes, including S3 PUTs, hang indefinitely while /healthz and /readyz still report OK. Make the filer health handler return 503 once this filer has been missing from its own lock ring for longer than a 30s grace period, so Kubernetes liveness probes restart it and it re-registers with the master. The grace period covers startup before the first ring update and the master's ring-change batching delay. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: hold the lock-ring health clock until the filer announces AnnounceOnceCaughtUp can hold the master registration for up to a minute while HTTP already serves, so the filer cannot be in the ring yet. Report healthy until the announce gate opens and start the grace period after it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep a probe racing a ring rejoin from reading a reset clock A probe that loses the CompareAndSwap to a concurrent healthy probe could load the reset zero and report the filer absent since the epoch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: trim the lock-ring health comments Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: restart the lock-ring health grace when a ring update brings the filer back The clock only reset when a probe saw the filer in the ring, so a rejoin and a second removal between two probes reused the first absence and failed the next probe early. Reset it from the ring snapshot callback. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test the announce gate leaves the lock-ring health clock unset Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: re-read lock-ring membership after starting the health clock A probe whose first read raced a rejoin could start the clock after the snapshot callback reset it, leaving a stale start for the next removal. The ring is installed before the callback runs, so a second read sees the rejoin whenever the reset came first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: treat a health clock changed during the probe as a rejoin A rejoin and a second removal can both land between the probe's two membership reads. The second read then shows the filer absent again, and the probe judged the new absence by the expired start it had read before the rejoin's reset. The snapshot callback resets the clock before the next ring update is installed, so a clock that no longer matches the probe's start means a rejoin happened; report healthy and let the next probe use the fresh clock. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test ring updates that land during a lock-ring health probe Move the absence clock into lockRingAbsence, which takes the membership read as a function, so the test can install ring updates between the probe's reads: a rejoin after the first read, and a rejoin plus removal between the two reads. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: give every lock-ring health reset a new clock value A probe that saw the filer absent started the clock with a CAS from 0, so a rejoin that reset the clock to 0 again before that CAS was invisible, and the probe could install a start time from before the rejoin. Each reset now stores a new non-positive value and the probe swaps from the value it loaded before the membership read, so any rejoin during the probe makes the CAS fail. This drops the second membership read and never waits on the LockRing mutex. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test lock-ring rejoins before and after the probe's membership read Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep the lock-ring version lookup out of the unhealthy health path LockRing.Version takes the ring mutex, which SetSnapshot holds while the snapshot callback transfers locks, so an expired-absence probe could wait on those transfers and time out instead of answering 503. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test an expired-absence health probe while the ring mutex is held Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
459 lines
15 KiB
Go
459 lines
15 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"path"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
|
"github.com/seaweedfs/seaweedfs/weed/util/version"
|
|
)
|
|
|
|
func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
|
start := time.Now()
|
|
|
|
inFlightGauge := stats.FilerInFlightRequestsGauge.WithLabelValues(r.Method)
|
|
inFlightGauge.Inc()
|
|
defer inFlightGauge.Dec()
|
|
|
|
statusRecorder := stats.NewStatusResponseWriter(w)
|
|
w = statusRecorder
|
|
origin := r.Header.Get("Origin")
|
|
if origin != "" {
|
|
if fs.option.AllowedOrigins == nil || len(fs.option.AllowedOrigins) == 0 || fs.option.AllowedOrigins[0] == "*" {
|
|
origin = "*"
|
|
} else {
|
|
originFound := false
|
|
for _, allowedOrigin := range fs.option.AllowedOrigins {
|
|
if origin == allowedOrigin {
|
|
originFound = true
|
|
}
|
|
}
|
|
if !originFound {
|
|
writeJsonError(w, r, http.StatusForbidden, errors.New("origin not allowed"))
|
|
return
|
|
}
|
|
}
|
|
|
|
w.Header().Set("Access-Control-Allow-Origin", origin)
|
|
w.Header().Set("Access-Control-Expose-Headers", "*")
|
|
w.Header().Set("Access-Control-Allow-Headers", "*")
|
|
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
|
w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
|
|
}
|
|
|
|
if r.Method == http.MethodOptions {
|
|
OptionsHandler(w, r, false)
|
|
return
|
|
}
|
|
|
|
requestMethod := r.Method
|
|
defer func(method *string) {
|
|
stats.FilerRequestCounter.WithLabelValues(*method, strconv.Itoa(statusRecorder.Status)).Inc()
|
|
stats.FilerRequestHistogram.WithLabelValues(*method).Observe(time.Since(start).Seconds())
|
|
}(&requestMethod)
|
|
|
|
isReadHttpCall := r.Method == http.MethodGet || r.Method == http.MethodHead
|
|
if !fs.maybeCheckJwtAuthorization(r, !isReadHttpCall) {
|
|
writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
|
|
return
|
|
}
|
|
|
|
// proxy to volume servers, after the gate: this is the one port operators
|
|
// expose, and the branch reaches any needle in the cluster by file id.
|
|
if r.URL.Path == "/" {
|
|
if fileId := r.URL.Query().Get(util_http.ProxyChunkIdParam); fileId != "" {
|
|
fs.proxyToVolumeServer(w, r, fileId)
|
|
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkProxy).Inc()
|
|
// Name the deferred observation after the proxy rather than
|
|
// observing a second time, which would count the request twice.
|
|
requestMethod = stats.ChunkProxy
|
|
return
|
|
}
|
|
}
|
|
|
|
w.Header().Set("Server", "SeaweedFS "+version.VERSION)
|
|
|
|
switch r.Method {
|
|
case http.MethodGet, http.MethodHead:
|
|
fs.GetOrHeadHandler(w, r)
|
|
case http.MethodDelete:
|
|
if _, ok := r.URL.Query()["tagging"]; ok {
|
|
fs.DeleteTaggingHandler(w, r)
|
|
} else {
|
|
fs.DeleteHandler(w, r)
|
|
}
|
|
case http.MethodPost, http.MethodPut:
|
|
// wait until in flight data is less than the limit
|
|
contentLength := getContentLength(r)
|
|
if !fs.waitForUploadSlot(w, r, contentLength) {
|
|
return
|
|
}
|
|
defer func() {
|
|
fs.inFlightDataLimitCond.L.Lock()
|
|
defer fs.inFlightDataLimitCond.L.Unlock()
|
|
// Decrement counters
|
|
newUploads := atomic.AddInt64(&fs.inFlightUploads, -1)
|
|
newSize := atomic.AddInt64(&fs.inFlightDataSize, -contentLength)
|
|
// Update metrics
|
|
stats.FilerInFlightUploadCountGauge.Set(float64(newUploads))
|
|
stats.FilerInFlightUploadBytesGauge.Set(float64(newSize))
|
|
fs.inFlightDataLimitCond.Broadcast()
|
|
}()
|
|
|
|
if r.Method == http.MethodPut {
|
|
if _, ok := r.URL.Query()["tagging"]; ok {
|
|
fs.PutTaggingHandler(w, r)
|
|
} else {
|
|
fs.PostHandler(w, r, contentLength)
|
|
}
|
|
} else { // method == "POST"
|
|
fs.PostHandler(w, r, contentLength)
|
|
}
|
|
default:
|
|
requestMethod = "INVALID"
|
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
|
}
|
|
}
|
|
|
|
func (fs *FilerServer) waitForUploadSlot(w http.ResponseWriter, r *http.Request, contentLength int64) bool {
|
|
fs.inFlightDataLimitCond.L.Lock()
|
|
err := util.WaitUntil(r.Context(), fs.inFlightDataLimitCond, fs.option.InflightUploadDataTimeout, func() bool {
|
|
inFlightDataSize := atomic.LoadInt64(&fs.inFlightDataSize)
|
|
if fs.option.ConcurrentUploadLimit != 0 && inFlightDataSize > fs.option.ConcurrentUploadLimit {
|
|
glog.V(4).Infof("wait because inflight data %d > %d", inFlightDataSize, fs.option.ConcurrentUploadLimit)
|
|
return false
|
|
}
|
|
inFlightUploads := atomic.LoadInt64(&fs.inFlightUploads)
|
|
if fs.option.ConcurrentFileUploadLimit != 0 && inFlightUploads >= fs.option.ConcurrentFileUploadLimit {
|
|
glog.V(4).Infof("wait because inflight uploads %d >= %d", inFlightUploads, fs.option.ConcurrentFileUploadLimit)
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
if err == nil {
|
|
// Increment counters
|
|
newUploads := atomic.AddInt64(&fs.inFlightUploads, 1)
|
|
newSize := atomic.AddInt64(&fs.inFlightDataSize, contentLength)
|
|
// Update metrics
|
|
stats.FilerInFlightUploadCountGauge.Set(float64(newUploads))
|
|
stats.FilerInFlightUploadBytesGauge.Set(float64(newSize))
|
|
}
|
|
fs.inFlightDataLimitCond.L.Unlock()
|
|
|
|
if err == nil {
|
|
return true
|
|
}
|
|
// Closing an HTTP/1 connection stops net/http from reading the unread
|
|
// body before the reply.
|
|
if r.ProtoMajor == 1 {
|
|
w.Header().Set("Connection", "close")
|
|
}
|
|
if r.Context().Err() != nil {
|
|
glog.V(1).Infof("upload from %s cancelled while waiting for an upload slot: %v", r.RemoteAddr, err)
|
|
writeJsonError(w, r, util.HttpStatusCancelled, r.Context().Err())
|
|
return false
|
|
}
|
|
err = fmt.Errorf("reject because no upload slot freed within %v", fs.option.InflightUploadDataTimeout)
|
|
glog.V(1).Infof("too many requests from %s: %v", r.RemoteAddr, err)
|
|
writeJsonError(w, r, http.StatusTooManyRequests, err)
|
|
return false
|
|
}
|
|
|
|
func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {
|
|
|
|
start := time.Now()
|
|
statusRecorder := stats.NewStatusResponseWriter(w)
|
|
w = statusRecorder
|
|
|
|
glog.V(4).Infof("Request: %s %s", r.Method, r.URL.Path)
|
|
|
|
origin := r.Header.Get("Origin")
|
|
if origin != "" {
|
|
if fs.option.AllowedOrigins == nil || len(fs.option.AllowedOrigins) == 0 || fs.option.AllowedOrigins[0] == "*" {
|
|
origin = "*"
|
|
} else {
|
|
originFound := false
|
|
for _, allowedOrigin := range fs.option.AllowedOrigins {
|
|
if origin == allowedOrigin {
|
|
originFound = true
|
|
}
|
|
}
|
|
if !originFound {
|
|
writeJsonError(w, r, http.StatusForbidden, errors.New("origin not allowed"))
|
|
return
|
|
}
|
|
}
|
|
|
|
w.Header().Set("Access-Control-Allow-Origin", origin)
|
|
w.Header().Set("Access-Control-Allow-Headers", "OPTIONS, GET, HEAD")
|
|
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
|
}
|
|
requestMethod := r.Method
|
|
defer func(method *string) {
|
|
stats.FilerRequestCounter.WithLabelValues(*method, strconv.Itoa(statusRecorder.Status)).Inc()
|
|
stats.FilerRequestHistogram.WithLabelValues(*method).Observe(time.Since(start).Seconds())
|
|
}(&requestMethod)
|
|
// We handle OPTIONS first because it never should be authenticated
|
|
if r.Method == http.MethodOptions {
|
|
OptionsHandler(w, r, true)
|
|
return
|
|
}
|
|
|
|
if !fs.maybeCheckJwtAuthorization(r, false) {
|
|
writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Server", "SeaweedFS "+version.VERSION)
|
|
|
|
switch r.Method {
|
|
case http.MethodGet, http.MethodHead:
|
|
fs.GetOrHeadHandler(w, r.WithContext(context.WithValue(r.Context(), readOnlyListingKey{}, true)))
|
|
default:
|
|
requestMethod = "INVALID"
|
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
|
}
|
|
}
|
|
|
|
func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool) {
|
|
if isReadOnly {
|
|
w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
|
|
} else {
|
|
w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
|
|
w.Header().Set("Access-Control-Expose-Headers", "*")
|
|
}
|
|
w.Header().Set("Access-Control-Allow-Headers", "*")
|
|
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
|
}
|
|
|
|
// maybeCheckJwtAuthorization returns true if access should be granted, false if it should be denied
|
|
func (fs *FilerServer) maybeCheckJwtAuthorization(r *http.Request, isWrite bool) bool {
|
|
return fs.checkJwtAuthorization(r, isWrite, jwtScopedRequestPaths(r))
|
|
}
|
|
|
|
// checkJwtAuthorization verifies the request carries a valid filer JWT for the
|
|
// requested access level and, for prefix-restricted tokens, that every path in
|
|
// scopedPaths falls within AllowedPrefixes.
|
|
func (fs *FilerServer) checkJwtAuthorization(r *http.Request, isWrite bool, scopedPaths []string) bool {
|
|
claims, ok := fs.authenticateFilerJwt(r, isWrite)
|
|
if !ok {
|
|
return false
|
|
}
|
|
return authorizeFilerJwtPaths(r, claims, scopedPaths)
|
|
}
|
|
|
|
// authenticateFilerJwt verifies the JWT signature and method claims. A nil claims
|
|
// with ok true means authentication is disabled for this access level. Splitting
|
|
// authentication from path authorization lets a handler load an indirect resource
|
|
// (the TUS session target) only after the caller's credential is verified.
|
|
func (fs *FilerServer) authenticateFilerJwt(r *http.Request, isWrite bool) (*security.SeaweedFilerClaims, bool) {
|
|
|
|
var signingKey security.SigningKey
|
|
if isWrite {
|
|
signingKey = fs.filerGuard.SigningKey()
|
|
} else {
|
|
signingKey = fs.filerGuard.ReadSigningKey()
|
|
}
|
|
if len(signingKey) == 0 {
|
|
return nil, true
|
|
}
|
|
|
|
tokenStr := security.GetJwt(r)
|
|
if tokenStr == "" {
|
|
glog.V(1).Infof("missing jwt from %s", r.RemoteAddr)
|
|
return nil, false
|
|
}
|
|
|
|
token, err := security.DecodeJwt(signingKey, tokenStr, &security.SeaweedFilerClaims{})
|
|
if err != nil {
|
|
glog.V(1).Infof("jwt verification error from %s: %v", r.RemoteAddr, err)
|
|
return nil, false
|
|
}
|
|
if !token.Valid {
|
|
glog.V(1).Infof("jwt invalid from %s: %v", r.RemoteAddr, tokenStr)
|
|
return nil, false
|
|
}
|
|
|
|
claims, ok := token.Claims.(*security.SeaweedFilerClaims)
|
|
if !ok {
|
|
glog.V(1).Infof("jwt claims not of type *SeaweedFilerClaims from %s", r.RemoteAddr)
|
|
return nil, false
|
|
}
|
|
if claims.SessionId != "" {
|
|
glog.V(1).Infof("jwt is an STS session token, not a filer credential, from %s", r.RemoteAddr)
|
|
return nil, false
|
|
}
|
|
|
|
if len(claims.AllowedMethods) > 0 {
|
|
hasMethod := false
|
|
for _, method := range claims.AllowedMethods {
|
|
if method == r.Method {
|
|
hasMethod = true
|
|
break
|
|
}
|
|
}
|
|
if !hasMethod {
|
|
glog.V(1).Infof("jwt method not allowed from %s: %v", r.RemoteAddr, r.Method)
|
|
return nil, false
|
|
}
|
|
}
|
|
|
|
return claims, true
|
|
}
|
|
|
|
// authorizeFilerJwtPaths checks the resource scope after authentication. A
|
|
// prefix-restricted token must name at least one resource path; an empty list
|
|
// fails closed, so a caller that cannot resolve its target never falls open.
|
|
func authorizeFilerJwtPaths(r *http.Request, claims *security.SeaweedFilerClaims, scopedPaths []string) bool {
|
|
if claims == nil || len(claims.AllowedPrefixes) == 0 {
|
|
return true
|
|
}
|
|
if len(scopedPaths) == 0 {
|
|
glog.V(1).Infof("jwt resource path missing from %s", r.RemoteAddr)
|
|
return false
|
|
}
|
|
for _, p := range scopedPaths {
|
|
if !anyComponentPrefixMatches(claims.AllowedPrefixes, p) {
|
|
glog.V(1).Infof("jwt path not allowed from %s: %v", r.RemoteAddr, p)
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// jwtScopedRequestPaths returns every filer path a request touches that must be
|
|
// covered by the JWT AllowedPrefixes: the write target (r.URL.Path), the
|
|
// directory a zip extract writes under, and any copy/move source named by the
|
|
// cp.from / mv.from query parameters.
|
|
func jwtScopedRequestPaths(r *http.Request) []string {
|
|
paths := []string{r.URL.Path}
|
|
if (r.Method == http.MethodPost || r.Method == http.MethodPut) && isExtractZip(r) {
|
|
paths = append(paths, extractZipRoot(r)+"/")
|
|
}
|
|
if query := r.URL.Query(); query.Has("cp.from") || query.Has("mv.from") {
|
|
if from := query.Get("cp.from"); from != "" {
|
|
paths = append(paths, from)
|
|
}
|
|
if from := query.Get("mv.from"); from != "" {
|
|
paths = append(paths, from)
|
|
}
|
|
}
|
|
return paths
|
|
}
|
|
|
|
// anyComponentPrefixMatches reports whether p is within any of the prefixes.
|
|
func anyComponentPrefixMatches(prefixes []string, p string) bool {
|
|
for _, prefix := range prefixes {
|
|
if pathHasComponentPrefix(p, prefix) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// pathHasComponentPrefix reports whether reqPath is contained within the
|
|
// directory subtree denoted by prefix, treating both as "/"-separated
|
|
// path components. Both inputs are normalised with path.Clean to neutralise
|
|
// "." and ".." segments and collapse duplicate slashes. A prefix of "/"
|
|
// matches any path.
|
|
func pathHasComponentPrefix(reqPath, prefix string) bool {
|
|
if prefix == "" {
|
|
return false
|
|
}
|
|
cleanedPath := path.Clean(reqPath)
|
|
if cleanedPath == "." {
|
|
cleanedPath = "/"
|
|
}
|
|
cleanedPrefix := path.Clean(prefix)
|
|
if cleanedPrefix == "." {
|
|
cleanedPrefix = "/"
|
|
}
|
|
if cleanedPrefix == "/" {
|
|
return true
|
|
}
|
|
if cleanedPath == cleanedPrefix {
|
|
return true
|
|
}
|
|
return strings.HasPrefix(cleanedPath, cleanedPrefix+"/")
|
|
}
|
|
|
|
// lockRingUnhealthyGracePeriod covers the first ring update after announcing
|
|
// and the master's ring-change batching delay.
|
|
const lockRingUnhealthyGracePeriod = 30 * time.Second
|
|
|
|
// lockRingHealthy reports false once this filer has been missing from its own
|
|
// lock ring past the grace period: locked writes cannot be routed to it, and
|
|
// a restart re-registers it with the master.
|
|
func (fs *FilerServer) lockRingHealthy() bool {
|
|
dlm := fs.filer.Dlm
|
|
if dlm == nil || dlm.LockRing == nil || dlm.LockRing.Ring == nil {
|
|
return true
|
|
}
|
|
if announce := fs.option.AnnounceCh; announce != nil {
|
|
select {
|
|
case <-announce:
|
|
default:
|
|
return true
|
|
}
|
|
}
|
|
absent := fs.lockRingAbsence(func() bool { return slices.Contains(dlm.LockRing.Ring.GetServers(), dlm.Host) })
|
|
if absent < lockRingUnhealthyGracePeriod {
|
|
return true
|
|
}
|
|
glog.Warningf("filerHealthz: %s absent from lock ring for %v (ring: %v)",
|
|
dlm.Host, absent.Round(time.Second), dlm.LockRing.Ring.GetServers())
|
|
return false
|
|
}
|
|
|
|
// lockRingAbsence returns how long inRing has reported this filer missing,
|
|
// or 0 while it is in the ring.
|
|
func (fs *FilerServer) lockRingAbsence(inRing func() bool) time.Duration {
|
|
since := fs.lockRingUnhealthySince.Load()
|
|
if inRing() {
|
|
fs.resetLockRingClock()
|
|
return 0
|
|
}
|
|
// A rejoin after the load has stored a new reset value, so the clock
|
|
// neither starts nor is read across it.
|
|
if since <= 0 {
|
|
fs.lockRingUnhealthySince.CompareAndSwap(since, time.Now().UnixNano())
|
|
return 0
|
|
}
|
|
if fs.lockRingUnhealthySince.Load() != since {
|
|
return 0
|
|
}
|
|
return time.Since(time.Unix(0, since))
|
|
}
|
|
|
|
func (fs *FilerServer) resetLockRingClock() {
|
|
fs.lockRingUnhealthySince.Store(-fs.lockRingResets.Add(1))
|
|
}
|
|
|
|
func (fs *FilerServer) filerHealthzHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Server", "SeaweedFS "+version.VERSION)
|
|
if _, err := fs.filer.Store.FindEntry(context.Background(), filer.TopicsDir); err != nil && err != filer_pb.ErrNotFound {
|
|
glog.Warningf("filerHealthzHandler FindEntry: %+v", err)
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if !fs.lockRingHealthy() {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
}
|