The server notices its own token has run out

Signing in worked and everything after it did not: the login screen took the
password, PocketBase handed back the record and the token, and then /api/me
answered "The requested resource wasn't found." The three symptoms underneath
looked unrelated — a 404 on the profile, an organization whose name came back
empty, a user list with nobody in it — and they are one thing. The API Server's
superuser token had expired.

PocketBase does not say so. A record call carrying a token it no longer accepts
is not refused; the header is ignored and the request is served as a guest, and
the collection rules answer in the transport's place. The organizations
collection is superuser-only, so it 403s. A user record is guarded by a view
rule, so it 404s — hidden rather than denied. The users list is rule-filtered,
so it comes back 200 with an empty array. Not one of those is a 401, and a 401
was the only thing that made this client sign in again.

So the token was acquired once at startup and then kept for the life of the
container, and the retry meant to renew it could never fire. Uptime longer than
the token's lifetime was all it took. Nothing had to change for it to break,
which is why it broke on a stack nobody had touched.

The client now reads the exp claim PocketBase stamps into the token and renews
before spending it, a minute early so a call cannot land just after it lapses.
A token with no readable expiry is still taken at face value, and the 401 retry
stays where it is as the backstop for the case this does not cover.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

API Server/internal/pb/client.go

@@ -6,12 +6,14 @@ package pb
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
 	"mime/multipart"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
 )
@@ -29,6 +31,7 @@ type Client struct {
 	email    string
 	password string
 	token    string
+	tokenExp time.Time
 }
 
 func New(baseURL, email, password string) *Client {
@@ -69,6 +72,7 @@ func (c *Client) Reconfigure(baseURL, email, password string) {
 	c.email = email
 	c.password = password
 	c.token = ""
+	c.tokenExp = time.Time{}
 	c.mu.Unlock()
 }
 
@@ -120,6 +124,7 @@ func (c *Client) Authenticate(ctx context.Context) error {
 			}
 			c.mu.Lock()
 			c.token = out.Token
+			c.tokenExp = jwtExpiry(out.Token)
 			c.mu.Unlock()
 			return nil
 		}
@@ -134,8 +139,46 @@ func (c *Client) currentToken() string {
 	return c.token
 }
 
-// ensureToken acquires a superuser token when none is cached yet, so that a
-// superuser call never goes out unauthenticated.
+// tokenSkew is how long before its stated expiry a cached token is treated as
+// spent. It covers clock drift between this server and PocketBase, and the
+// flight time of a request that passes the check and then arrives just late.
+const tokenSkew = 60 * time.Second
+
+// tokenLive reports whether the cached token can still be used. A token with no
+// readable expiry is taken at face value — the 401 retry remains the backstop.
+func (c *Client) tokenLive() bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.token == "" {
+		return false
+	}
+	return c.tokenExp.IsZero() || time.Now().Add(tokenSkew).Before(c.tokenExp)
+}
+
+// jwtExpiry reads the exp claim out of a PocketBase auth token. The signature is
+// PocketBase's business — this only needs the expiry the server itself stamped,
+// so the payload is decoded without verification. Anything unreadable comes back
+// as the zero time, which tokenLive treats as "no expiry known".
+func jwtExpiry(token string) time.Time {
+	parts := strings.Split(token, ".")
+	if len(parts) != 3 {
+		return time.Time{}
+	}
+	raw, err := base64.RawURLEncoding.DecodeString(parts[1])
+	if err != nil {
+		return time.Time{}
+	}
+	var claims struct {
+		Exp int64 `json:"exp"`
+	}
+	if err := json.Unmarshal(raw, &claims); err != nil || claims.Exp == 0 {
+		return time.Time{}
+	}
+	return time.Unix(claims.Exp, 0)
+}
+
+// ensureToken acquires a superuser token when the cached one is missing or
+// spent, so that a superuser call never goes out unauthenticated.
 //
 // It matters because PocketBase answers a record read it will not allow with
 // 404, not 401 — it hides the record rather than refusing the credentials. So a
@@ -149,8 +192,18 @@ func (c *Client) currentToken() string {
 // With no service account configured there is nothing to acquire, so the call
 // proceeds as before — the endpoints that need superuser access answer 503 on
 // their own.
+//
+// Expiry is checked here rather than left to the 401 retry below, because that
+// retry never fires for this client: PocketBase does not reject a stale token on
+// a record call, it ignores the header and serves the request as a guest. The
+// collection rules then answer instead of the transport — a superuser-only
+// collection 403s, a rule-guarded record 404s, and a rule-filtered list comes
+// back 200 with nothing in it. None of those are a 401, so a server whose token
+// has lapsed keeps sending it and keeps being treated as a stranger to its own
+// database until it is restarted. Superuser tokens are long-lived, which only
+// means the failure waits weeks and then arrives as "the app forgot my account".
 func (c *Client) ensureToken(ctx context.Context) error {
-	if c.currentToken() != "" || !c.Configured() {
+	if c.tokenLive() || !c.Configured() {
 		return nil
 	}
 	return c.Authenticate(ctx)

API Server/internal/pb/client_test.go

@@ -2,11 +2,13 @@ package pb
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"sync"
 	"testing"
+	"time"
 )
 
 // PocketBase hides a record it will not let you read behind a 404 rather than a
@@ -31,9 +33,14 @@ type fakePB struct {
 	email    string
 	password string
 
+	// When set, tokens are minted as JWTs carrying this lifetime in their exp
+	// claim, the way PocketBase issues them. Zero keeps the opaque test token.
+	jwtTTL time.Duration
+
 	authCalls     int
 	lastIdentity  string
-	guestAttempts int // record reads that arrived without the superuser token
+	issued        string // the token most recently handed out
+	guestAttempts int    // record reads that arrived without the superuser token
 }
 
 func (f *fakePB) handler() http.Handler {
@@ -51,11 +58,21 @@ func (f *fakePB) handler() http.Handler {
 			writeTestJSON(w, 400, map[string]any{"message": "Failed to authenticate."})
 			return
 		}
-		writeTestJSON(w, 200, map[string]any{"token": fakeSvcToken})
+		token := fakeSvcToken
+		if f.jwtTTL > 0 {
+			token = testJWT(time.Now().Add(f.jwtTTL))
+		}
+		f.mu.Lock()
+		f.issued = token
+		f.mu.Unlock()
+		writeTestJSON(w, 200, map[string]any{"token": token})
 	})
 
 	mux.HandleFunc("GET /api/collections/users/records/{id}", func(w http.ResponseWriter, r *http.Request) {
-		if r.Header.Get("Authorization") != fakeSvcToken {
+		f.mu.Lock()
+		live := f.issued
+		f.mu.Unlock()
+		if got := r.Header.Get("Authorization"); got == "" || got != live {
 			f.mu.Lock()
 			f.guestAttempts++
 			f.mu.Unlock()
@@ -190,3 +207,101 @@ func TestNoServiceAccountSkipsAuthentication(t *testing.T) {
 		t.Errorf("attempted %d sign-in(s) with no credentials, want 0", auth)
 	}
 }
+
+// testJWT builds a token shaped like PocketBase's: three base64url segments,
+// the middle one carrying the exp claim. Only that claim is ever read, so the
+// header and signature are filler.
+func testJWT(exp time.Time) string {
+	enc := func(v any) string {
+		b, _ := json.Marshal(v)
+		return base64.RawURLEncoding.EncodeToString(b)
+	}
+	return enc(map[string]string{"alg": "HS256", "typ": "JWT"}) + "." +
+		enc(map[string]int64{"exp": exp.Unix()}) + ".sig"
+}
+
+func TestJWTExpiry(t *testing.T) {
+	want := time.Now().Add(time.Hour).Truncate(time.Second)
+	if got := jwtExpiry(testJWT(want)); !got.Equal(want) {
+		t.Errorf("jwtExpiry = %v, want %v", got, want)
+	}
+	// An opaque (non-JWT) token has no readable expiry, and must not be mistaken
+	// for one that expired at the zero time — tokenLive takes it at face value.
+	for _, tok := range []string{"", "opaque", "a.b", "a.!!.c", "a." + base64.RawURLEncoding.EncodeToString([]byte("{}")) + ".c"} {
+		if got := jwtExpiry(tok); !got.IsZero() {
+			t.Errorf("jwtExpiry(%q) = %v, want zero", tok, got)
+		}
+	}
+}
+
+// The failure this whole mechanism exists for: a superuser token that has run
+// out. PocketBase does not answer 401 for one — it ignores the header and serves
+// the request as a guest, so the 401 retry never fires and the client would go
+// on presenting a dead token forever. The expiry check has to catch it first.
+func TestExpiredTokenIsRenewedBeforeUse(t *testing.T) {
+	f := &fakePB{email: "admin@test.local", password: "pw", jwtTTL: time.Hour}
+	c := New(newFakePB(t, f), f.email, f.password)
+
+	if err := c.GetOne(context.Background(), "users", fakeRecordID, new(struct{})); err != nil {
+		t.Fatalf("GetOne: %v", err)
+	}
+
+	// Age the cached token past its expiry, as an uptime longer than the token's
+	// lifetime would.
+	c.mu.Lock()
+	c.tokenExp = time.Now().Add(-time.Minute)
+	c.mu.Unlock()
+
+	var rec struct{ ID string }
+	if err := c.GetOne(context.Background(), "users", fakeRecordID, &rec); err != nil {
+		t.Fatalf("GetOne with an expired token: %v", err)
+	}
+	if rec.ID != fakeRecordID {
+		t.Fatalf("record id = %q, want %q", rec.ID, fakeRecordID)
+	}
+
+	auth, guest := f.counts()
+	if guest != 0 {
+		t.Errorf("%d record read(s) went out on a dead token, want 0", guest)
+	}
+	if auth != 2 {
+		t.Errorf("authenticated %d time(s), want 2 (startup + renewal)", auth)
+	}
+}
+
+// A token close enough to its expiry that it could lapse mid-flight is renewed
+// rather than spent, so a call cannot land just after the token dies.
+func TestTokenNearingExpiryIsRenewed(t *testing.T) {
+	f := &fakePB{email: "admin@test.local", password: "pw", jwtTTL: time.Hour}
+	c := New(newFakePB(t, f), f.email, f.password)
+
+	if err := c.GetOne(context.Background(), "users", fakeRecordID, new(struct{})); err != nil {
+		t.Fatalf("GetOne: %v", err)
+	}
+	c.mu.Lock()
+	c.tokenExp = time.Now().Add(tokenSkew / 2)
+	c.mu.Unlock()
+
+	if err := c.GetOne(context.Background(), "users", fakeRecordID, new(struct{})); err != nil {
+		t.Fatalf("GetOne near expiry: %v", err)
+	}
+	if auth, _ := f.counts(); auth != 2 {
+		t.Errorf("authenticated %d time(s), want 2 (startup + renewal)", auth)
+	}
+}
+
+// A token with a real lifetime still ahead of it is reused — the expiry check
+// must not turn every call into a fresh sign-in.
+func TestLiveJWTIsReused(t *testing.T) {
+	f := &fakePB{email: "admin@test.local", password: "pw", jwtTTL: time.Hour}
+	c := New(newFakePB(t, f), f.email, f.password)
+
+	for i := 0; i < 3; i++ {
+		if err := c.GetOne(context.Background(), "users", fakeRecordID, new(struct{})); err != nil {
+			t.Fatalf("GetOne #%d: %v", i, err)
+		}
+	}
+	if auth, _ := f.counts(); auth != 1 {
+		t.Errorf("authenticated %d time(s), want 1", auth)
+	}
+}