PocketBase as a standalone database server
CI / Test (ubuntu-latest) (push) Canceled after 0s
CI / Test (windows-latest) (push) Canceled after 0s
CI / Storage tests on PostgreSQL (push) Canceled after 0s
CI / Complement (federation off) (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
CI / Smoke test the binary (push) Canceled after 0s
CI / Admin panel (push) Canceled after 0s
CI / Web client (push) Canceled after 0s
CI / Desktop client (ubuntu-24.04) (push) Canceled after 0s
CI / Desktop client (windows-latest) (push) Canceled after 0s

The pocketbase driver no longer embeds PocketBase: Warp keeps its data as
PocketBase collections on a server of its own and reaches it only through
the REST API, the way the postgres driver reaches PostgreSQL.
database.connection_string is the server's URL, and Warp signs in as the
superuser in WARP_PB_SUPERUSER_EMAIL / WARP_PB_SUPERUSER_PASSWORD (or a
WARP_PB_SUPERUSER_FILE of the two).

Every SQL table is a superuser-only collection of the same name and
columns, created by warp migrate. Reads go through PocketBase's SQL
endpoint with the SQL drivers' queries, via a read-only database/sql
driver that inlines arguments, moves long ones into warp_params and pages
past 1000 rows. Writes are record operations in batch requests under one
write lock in the Warp process. Media is a protected file field in
warp_media, local or in the S3 bucket Warp writes into PocketBase's
settings. Search matches word prefixes with LIKE. The storage tests run
against an in-process PocketBase and pass as on SQLite.

Docker: the PocketBase stacks get a pocketbase service running
tajniak81/pocketbase-docker-root (WARP_PB_IMAGE), with a pb-data volume
and its admin UI on loopback; the AIO images run that PocketBase binary
as a process beside the homeserver, making up a superuser when none is
set. The .env examples, READMEs, docs and warp.yaml.example follow.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
tajniak81andClaude Opus 5.5 committed 2026-10-26 16:31:08 +01:00
1 parent 0994156705
commit a47e15cc1c
69 files changed
+8209 -374

No files matched your search

+342
View File
@@ -0,0 +1,342 @@
package pocketbase
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Admin is the PocketBase implementation of storage.AdminStore.
type Admin struct{ *base }
// registrationTokenColumns reads uses_allowed as NULL for a token without a
// limit, as the SQL drivers store it.
const registrationTokenColumns = `token, CASE WHEN has_uses_allowed THEN uses_allowed END,
completed, expires_at_ms, created_at_ms`
// CreateRegistrationToken implements storage.AdminStore.
func (s *Admin) CreateRegistrationToken(ctx context.Context, rt storage.RegistrationToken) error {
err := s.write(ctx, func(t *tx) error {
taken, err := t.exists("admin_registration_tokens", `token = ?`, rt.Token)
if err != nil {
return err
}
if taken {
return storage.ErrRegistrationTokenExists
}
row := rec{
"token": rt.Token, "completed": rt.Completed,
"expires_at_ms": rt.ExpiresAtMS, "created_at_ms": rt.CreatedAtMS,
}
setUsesAllowed(row, rt.UsesAllowed)
t.put("admin_registration_tokens", row)
return nil
})
switch {
case errors.Is(err, storage.ErrRegistrationTokenExists):
return err
case err != nil:
return fmt.Errorf("create a registration token: %w", err)
}
return nil
}
// setUsesAllowed writes an optional use limit into a row.
func setUsesAllowed(row rec, n *int) {
if n == nil {
row["uses_allowed"], row["has_uses_allowed"] = int64(0), false
return
}
row["uses_allowed"], row["has_uses_allowed"] = int64(*n), true
}
// RegistrationToken implements storage.AdminStore.
func (s *Admin) RegistrationToken(ctx context.Context, token string) (storage.RegistrationToken, error) {
rt, err := scanRegistrationToken(s.db.QueryRowContext(ctx, `SELECT `+registrationTokenColumns+`
FROM admin_registration_tokens WHERE token = ?`, token).Scan)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.RegistrationToken{}, storage.ErrNoSuchRegistrationToken
case err != nil:
return storage.RegistrationToken{}, fmt.Errorf("read a registration token: %w", err)
}
return rt, nil
}
// RegistrationTokens implements storage.AdminStore.
func (s *Admin) RegistrationTokens(ctx context.Context) ([]storage.RegistrationToken, error) {
rows, err := s.db.QueryContext(ctx, `SELECT `+registrationTokenColumns+`
FROM admin_registration_tokens ORDER BY token`)
if err != nil {
return nil, fmt.Errorf("list registration tokens: %w", err)
}
defer func() { _ = rows.Close() }()
out := []storage.RegistrationToken{}
for rows.Next() {
rt, err := scanRegistrationToken(rows.Scan)
if err != nil {
return nil, fmt.Errorf("list registration tokens: %w", err)
}
out = append(out, rt)
}
return out, rows.Err()
}
// UpdateRegistrationToken implements storage.AdminStore.
func (s *Admin) UpdateRegistrationToken(ctx context.Context, token string, usesAllowed *int, expiresAtMS int64) error {
err := s.write(ctx, func(t *tx) error {
row := rec{"expires_at_ms": expiresAtMS}
setUsesAllowed(row, usesAllowed)
n, err := t.setWhere("admin_registration_tokens", row, `token = ?`, token)
if err != nil {
return fmt.Errorf("update a registration token: %w", err)
}
if n == 0 {
return storage.ErrNoSuchRegistrationToken
}
return nil
})
return err
}
// DeleteRegistrationToken implements storage.AdminStore.
func (s *Admin) DeleteRegistrationToken(ctx context.Context, token string) error {
return s.write(ctx, func(t *tx) error {
found, err := t.delKey("admin_registration_tokens", token)
if err != nil {
return fmt.Errorf("delete a registration token: %w", err)
}
if !found {
return storage.ErrNoSuchRegistrationToken
}
return nil
})
}
// UseRegistrationToken implements storage.AdminStore. The check and the
// spend happen under the write lock, so two registrations cannot both take
// the last use.
func (s *Admin) UseRegistrationToken(ctx context.Context, token string, nowMS int64) error {
return s.write(ctx, func(t *tx) error {
var completed int64
err := t.queryRow(`SELECT completed FROM admin_registration_tokens
WHERE token = ?
AND (NOT has_uses_allowed OR completed < uses_allowed)
AND (expires_at_ms = 0 OR expires_at_ms > ?)`, token, nowMS).Scan(&completed)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.ErrNoSuchRegistrationToken
case err != nil:
return fmt.Errorf("spend a registration token: %w", err)
}
t.put("admin_registration_tokens", rec{"token": token, "completed": completed + 1})
return nil
})
}
// ReleaseRegistrationToken implements storage.AdminStore. A token deleted in
// the meantime has nothing to give back to, which is not an error.
func (s *Admin) ReleaseRegistrationToken(ctx context.Context, token string) error {
err := s.write(ctx, func(t *tx) error {
var completed int64
err := t.queryRow(`SELECT completed FROM admin_registration_tokens WHERE token = ? AND completed > 0`, token).
Scan(&completed)
switch {
case errors.Is(err, sql.ErrNoRows):
return nil
case err != nil:
return err
}
t.put("admin_registration_tokens", rec{"token": token, "completed": completed - 1})
return nil
})
if err != nil {
return fmt.Errorf("release a registration token: %w", err)
}
return nil
}
func scanRegistrationToken(scan func(...any) error) (storage.RegistrationToken, error) {
var (
rt storage.RegistrationToken
uses sql.NullInt64
)
if err := scan(&rt.Token, &uses, &rt.Completed, &rt.ExpiresAtMS, &rt.CreatedAtMS); err != nil {
return storage.RegistrationToken{}, err
}
if uses.Valid {
n := int(uses.Int64)
rt.UsesAllowed = &n
}
return rt, nil
}
const (
auditColumns = `audit_id, ts_ms, actor, action, target, details_json`
defaultAuditPage = 50
maxAuditPage = 1000
)
// AppendAudit implements storage.AdminStore.
func (s *Admin) AppendAudit(ctx context.Context, e storage.AuditEntry) (int64, error) {
details := string(e.Details)
if details == "" {
details = "{}"
}
var id int64
err := s.write(ctx, func(t *tx) error {
var err error
if id, err = t.next(adminSequences, seqAudit); err != nil {
return err
}
t.put("admin_audit_log", rec{
"audit_id": id, "ts_ms": e.TSMS, "actor": e.Actor,
"action": e.Action, "target": e.Target, "details_json": details,
})
return nil
})
if err != nil {
return 0, fmt.Errorf("append to the audit log: %w", err)
}
return id, nil
}
// AuditLog implements storage.AdminStore.
func (s *Admin) AuditLog(ctx context.Context, f storage.AuditFilter) ([]storage.AuditEntry, error) {
var (
conditions []string
args []any
)
if f.Before > 0 {
conditions = append(conditions, `audit_id < ?`)
args = append(args, f.Before)
}
if f.Actor != "" {
conditions = append(conditions, `actor = ?`)
args = append(args, f.Actor)
}
if f.Action != "" {
conditions = append(conditions, `action = ?`)
args = append(args, f.Action)
}
limit := f.Limit
switch {
case limit <= 0:
limit = defaultAuditPage
case limit > maxAuditPage:
limit = maxAuditPage
}
args = append(args, limit)
query := `SELECT ` + auditColumns + ` FROM admin_audit_log`
if len(conditions) > 0 {
query += ` WHERE ` + strings.Join(conditions, " AND ")
}
query += ` ORDER BY audit_id DESC LIMIT ?`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read the audit log: %w", err)
}
defer func() { _ = rows.Close() }()
out := []storage.AuditEntry{}
for rows.Next() {
var (
e storage.AuditEntry
details string
)
if err := rows.Scan(&e.ID, &e.TSMS, &e.Actor, &e.Action, &e.Target, &details); err != nil {
return nil, fmt.Errorf("read the audit log: %w", err)
}
e.Details = []byte(details)
out = append(out, e)
}
return out, rows.Err()
}
// ServerNoticeRoom implements storage.AdminStore.
func (s *Admin) ServerNoticeRoom(ctx context.Context, localpart string) (string, error) {
var roomID string
err := s.db.QueryRowContext(ctx, `SELECT room_id FROM admin_server_notice_rooms WHERE localpart = ?`, localpart).
Scan(&roomID)
switch {
case errors.Is(err, sql.ErrNoRows):
return "", storage.ErrNoServerNoticeRoom
case err != nil:
return "", fmt.Errorf("read the server notice room of %q: %w", localpart, err)
}
return roomID, nil
}
// SetServerNoticeRoom implements storage.AdminStore.
func (s *Admin) SetServerNoticeRoom(ctx context.Context, localpart, roomID string) error {
err := s.write(ctx, func(t *tx) error {
t.put("admin_server_notice_rooms", rec{"localpart": localpart, "room_id": roomID})
return nil
})
if err != nil {
return fmt.Errorf("record the server notice room of %q: %w", localpart, err)
}
return nil
}
// AppServices is the PocketBase implementation of storage.AppServiceStore.
type AppServices struct{ *base }
// AppServiceState implements storage.AppServiceStore.
func (s *AppServices) AppServiceState(ctx context.Context, id string, startPos int64) (storage.AppServiceState, error) {
var st storage.AppServiceState
err := s.write(ctx, func(t *tx) error {
err := t.queryRow(`SELECT as_id, stream_pos, txn_id, txn_to FROM appservice_state WHERE as_id = ?`, id).
Scan(&st.ID, &st.StreamPos, &st.TxnID, &st.TxnTo)
switch {
case errors.Is(err, sql.ErrNoRows):
st = storage.AppServiceState{ID: id, StreamPos: startPos}
t.put("appservice_state", rec{"as_id": id, "stream_pos": startPos, "txn_id": int64(0), "txn_to": int64(0)})
return nil
case err != nil:
return err
}
return nil
})
if err != nil {
return storage.AppServiceState{}, fmt.Errorf("read the cursor of application service %q: %w", id, err)
}
return st, nil
}
// BeginTransaction implements storage.AppServiceStore.
func (s *AppServices) BeginTransaction(ctx context.Context, id string, txnID, to int64) error {
err := s.write(ctx, func(t *tx) error {
_, err := t.setWhere("appservice_state", rec{"txn_id": txnID, "txn_to": to}, `as_id = ?`, id)
return err
})
if err != nil {
return fmt.Errorf("begin transaction %d of application service %q: %w", txnID, id, err)
}
return nil
}
// CompleteTransaction implements storage.AppServiceStore.
func (s *AppServices) CompleteTransaction(ctx context.Context, id string, to int64) error {
err := s.write(ctx, func(t *tx) error {
_, err := t.setWhere("appservice_state", rec{"stream_pos": to, "txn_to": int64(0)}, `as_id = ?`, id)
return err
})
if err != nil {
return fmt.Errorf("advance the cursor of application service %q: %w", id, err)
}
return nil
}
var (
_ storage.AdminStore = (*Admin)(nil)
_ storage.AppServiceStore = (*AppServices)(nil)
)
+282
View File
@@ -0,0 +1,282 @@
package pocketbase
import (
"context"
"database/sql"
"fmt"
"strings"
"sync"
)
// base is what every store shares: the client, the read-only SQL pool and
// the write lock.
//
// Writes are serialised by the lock, in this process, the way the sqlite
// driver serialises them onto one connection: PocketBase is SQLite and takes
// one writer at a time anyway, and holding the lock across a store call's
// reads and its batch is what makes "read, decide, write" safe without a
// transaction that spans requests. It does mean one Warp server per
// PocketBase.
type base struct {
c *client
db *sql.DB
lock *sync.RWMutex
}
// rec is one row's columns, by name.
type rec map[string]any
// maxBatch is how many operations go into one batch request. PocketBase is
// told to allow this many (see configure); a store call that writes more is
// split, and each part is atomic on its own.
const maxBatch = 1000
// tx collects the writes of one store call.
type tx struct {
ctx context.Context
b *base
// seqs are the counters this call has read or advanced, by table and
// name, and seqOrder the order they were first touched in.
seqs map[string]int64
seqOrder []string
ops []batchRequest
deleted map[string]bool
}
// write runs fn with the write lock held and sends what it collected.
//
// The counters it advanced are written first, in the first batch, so that a
// call split over several batches that fails part-way leaves a gap in a
// stream — which every reader already copes with — and never a position
// handed out twice.
func (b *base) write(ctx context.Context, fn func(*tx) error) error {
b.lock.Lock()
defer b.lock.Unlock()
t := &tx{ctx: withWriteLock(ctx), b: b, seqs: map[string]int64{}, deleted: map[string]bool{}}
if err := fn(t); err != nil {
return err
}
return t.commit()
}
// read runs fn with the read lock held, so that its queries see one state:
// no write lands between them. The SQL drivers get that from a
// transaction; here it is the lock, since every write takes it.
func (b *base) read(ctx context.Context, fn func(ctx context.Context) error) error {
b.lock.RLock()
defer b.lock.RUnlock()
return fn(withWriteLock(ctx))
}
func (t *tx) commit() error {
ops := make([]batchRequest, 0, len(t.seqOrder)+len(t.ops))
for _, key := range t.seqOrder {
table, name, _ := strings.Cut(key, "\x00")
ops = append(ops, putRequest(table, rec{"name": name, "value": t.seqs[key]}))
}
ops = append(ops, t.ops...)
for len(ops) > 0 {
n := min(len(ops), maxBatch)
if err := t.b.c.batch(t.ctx, ops[:n]); err != nil {
return err
}
ops = ops[n:]
}
return nil
}
// putRequest is an upsert of one row: created if its key is new, otherwise
// updated in the columns given and left alone in the rest.
func putRequest(collection string, row rec) batchRequest {
tb, ok := tables[collection]
if !ok {
panic("pocketbase: no collection " + collection)
}
key := make([]any, len(tb.key))
for i, col := range tb.key {
v, ok := row[col]
if !ok {
panic(fmt.Sprintf("pocketbase: a %s row without its key column %s", collection, col))
}
key[i] = v
}
return patchRequest(collection, rowID(collection, key...), row)
}
// patchRequest is an upsert of a row whose id is already known.
func patchRequest(collection, id string, row rec) batchRequest {
body := make(map[string]any, len(row)+1)
for col, v := range row {
body[col] = jsonValue(v)
}
body["id"] = id
return batchRequest{Method: "PUT", URL: "/api/collections/" + collection + "/records", Body: body}
}
// jsonValue turns a column value into what the record API takes.
func jsonValue(v any) any {
switch v := v.(type) {
case []byte:
return string(v)
case int:
return int64(v)
case int32:
return int64(v)
default:
return v
}
}
// put upserts a row; see putRequest.
func (t *tx) put(collection string, row rec) {
req := putRequest(collection, row)
delete(t.deleted, collection+"/"+req.Body["id"].(string))
t.ops = append(t.ops, req)
}
// patch upserts columns of the row with this id.
func (t *tx) patch(collection, id string, row rec) {
delete(t.deleted, collection+"/"+id)
t.ops = append(t.ops, patchRequest(collection, id, row))
}
// del deletes a row that exists. Deleting one that does not fails the whole
// batch, so callers only delete what they have just read.
func (t *tx) del(collection, id string) bool {
key := collection + "/" + id
if t.deleted[key] {
return false
}
t.deleted[key] = true
t.ops = append(t.ops, batchRequest{Method: "DELETE", URL: "/api/collections/" + collection + "/records/" + id})
return true
}
// delKey deletes the row with this primary key, if it exists.
func (t *tx) delKey(collection string, key ...any) (bool, error) {
id := rowID(collection, key...)
found, err := t.exists(collection, `id = ?`, id)
if err != nil || !found {
return false, err
}
return t.del(collection, id), nil
}
// ids reads the ids of the rows matching a condition.
func (t *tx) ids(collection, where string, args ...any) ([]string, error) {
rows, err := t.b.db.QueryContext(t.ctx, `SELECT id FROM `+collection+` WHERE `+where, args...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
out = append(out, id)
}
return out, rows.Err()
}
// delWhere deletes the rows matching a condition and reports how many there
// were.
func (t *tx) delWhere(collection, where string, args ...any) (int, error) {
ids, err := t.ids(collection, where, args...)
if err != nil {
return 0, fmt.Errorf("find the %s rows to delete: %w", collection, err)
}
n := 0
for _, id := range ids {
if t.del(collection, id) {
n++
}
}
return n, nil
}
// setWhere updates columns of the rows matching a condition and reports how
// many there were.
func (t *tx) setWhere(collection string, row rec, where string, args ...any) (int, error) {
ids, err := t.ids(collection, where, args...)
if err != nil {
return 0, fmt.Errorf("find the %s rows to update: %w", collection, err)
}
n := 0
for _, id := range ids {
if t.deleted[collection+"/"+id] {
continue
}
t.patch(collection, id, row)
n++
}
return n, nil
}
// exists reports whether a row matches a condition.
func (t *tx) exists(collection, where string, args ...any) (bool, error) {
var one int
err := t.b.db.QueryRowContext(t.ctx, `SELECT 1 FROM `+collection+` WHERE `+where+` LIMIT 1`, args...).Scan(&one)
switch {
case err == sql.ErrNoRows:
return false, nil
case err != nil:
return false, err
}
return true, nil
}
// queryRow reads one row inside the call.
func (t *tx) queryRow(query string, args ...any) *sql.Row {
return t.b.db.QueryRowContext(t.ctx, query, args...)
}
// query reads rows inside the call.
func (t *tx) query(query string, args ...any) (*sql.Rows, error) {
return t.b.db.QueryContext(t.ctx, query, args...)
}
// next hands out the next value of a counter.
func (t *tx) next(table, name string) (int64, error) {
return t.advance(table, name, 1)
}
// advance moves a counter by n and returns the last value of the block, as
// the SQL drivers' advanceSequence does. n may be negative.
func (t *tx) advance(table, name string, n int64) (int64, error) {
cur, err := t.current(table, name)
if err != nil {
return 0, err
}
cur += n
t.seqs[table+"\x00"+name] = cur
return cur, nil
}
// current reads a counter as this call has left it.
func (t *tx) current(table, name string) (int64, error) {
key := table + "\x00" + name
if v, ok := t.seqs[key]; ok {
return v, nil
}
var v int64
if err := t.queryRow(`SELECT value FROM `+table+` WHERE name = ?`, name).Scan(&v); err != nil {
return 0, fmt.Errorf("read the %s counter: %w", name, err)
}
t.seqs[key] = v
t.seqOrder = append(t.seqOrder, key)
return v, nil
}
// currentSequence reads a counter outside any write.
func (b *base) currentSequence(ctx context.Context, table, name string) (int64, error) {
var v int64
if err := b.db.QueryRowContext(ctx, `SELECT value FROM `+table+` WHERE name = ?`, name).Scan(&v); err != nil {
return 0, fmt.Errorf("read the %s counter: %w", name, err)
}
return v, nil
}
+210
View File
@@ -0,0 +1,210 @@
package pocketbase
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"os"
"strconv"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// mediaCollection holds media content: one record per distinct content,
// keyed by its SHA-256, with the bytes in a protected file field. Where
// PocketBase keeps the file — its pb_data/storage or an S3 bucket — is its
// own storage setting.
const mediaCollection = "warp_media"
// blobStore is storage.BlobStore over warp_media.
type blobStore struct{ *base }
// validSum reports whether s is a lower-case hex SHA-256: the only names
// content is stored under, and so the only ones worth asking about.
func validSum(s string) bool {
if len(s) != sha256.Size*2 {
return false
}
for _, c := range s {
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return false
}
}
return true
}
// Put implements storage.BlobStore.
//
// The content is spooled to a temporary file first: its name is its hash,
// which is not known until the last byte has been read, and the upload has
// to say both. The spool is also what keeps an oversize upload from reaching
// PocketBase at all.
func (b *blobStore) Put(ctx context.Context, r io.Reader, limit int64) (storage.BlobInfo, error) {
tmp, err := os.CreateTemp("", "warp-media-*")
if err != nil {
return storage.BlobInfo{}, fmt.Errorf("create a temporary file: %w", err)
}
defer func() {
_ = tmp.Close()
_ = os.Remove(tmp.Name())
}()
hash := sha256.New()
n, err := io.Copy(io.MultiWriter(tmp, hash), &ctxReader{ctx: ctx, r: io.LimitReader(r, limit+1)})
if err != nil {
return storage.BlobInfo{}, fmt.Errorf("store content: %w", err)
}
if n > limit {
return storage.BlobInfo{}, storage.ErrBlobTooLarge
}
info := storage.BlobInfo{SHA256: hex.EncodeToString(hash.Sum(nil)), Size: n}
id := rowID(mediaCollection, info.SHA256)
if _, err := b.c.getRecord(ctx, mediaCollection, id); err == nil {
// The same content is already stored.
return info, nil
} else if !isNotFound(err) {
return storage.BlobInfo{}, fmt.Errorf("look up stored content: %w", err)
}
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
return storage.BlobInfo{}, fmt.Errorf("store content: %w", err)
}
err = b.c.createRecordWithFile(ctx, mediaCollection, map[string]string{
"id": id,
"sha256": info.SHA256,
"size": strconv.FormatInt(n, 10),
}, "content", info.SHA256, tmp)
if err != nil {
// Two uploads of the same content can race to create the record;
// the loser finds the winner's.
if _, getErr := b.c.getRecord(ctx, mediaCollection, id); getErr == nil {
return info, nil
}
return storage.BlobInfo{}, fmt.Errorf("store content: %w", err)
}
return info, nil
}
// Open implements storage.BlobStore.
func (b *blobStore) Open(ctx context.Context, sum string) (io.ReadSeekCloser, error) {
if !validSum(sum) {
return nil, storage.ErrNoSuchBlob
}
id := rowID(mediaCollection, sum)
record, err := b.c.getRecord(ctx, mediaCollection, id)
switch {
case isNotFound(err):
return nil, storage.ErrNoSuchBlob
case err != nil:
return nil, fmt.Errorf("open media content: %w", err)
}
name, _ := record["content"].(string)
size, _ := record["size"].(float64)
if name == "" {
return nil, storage.ErrNoSuchBlob
}
token, err := b.c.fileToken(ctx)
if err != nil {
return nil, fmt.Errorf("open media content: %w", err)
}
return &blobReader{ctx: ctx, c: b.c, id: id, name: name, token: token, size: int64(size)}, nil
}
// Delete implements storage.BlobStore.
func (b *blobStore) Delete(ctx context.Context, sum string) error {
if !validSum(sum) {
return nil
}
if err := b.c.deleteRecord(ctx, mediaCollection, rowID(mediaCollection, sum)); err != nil && !isNotFound(err) {
return fmt.Errorf("delete media content: %w", err)
}
return nil
}
// blobReader reads a stored file over HTTP. A seek only moves the offset;
// the next read starts a ranged download from there, so reading a file
// through costs one request however it is read.
type blobReader struct {
ctx context.Context
c *client
id string
name string
token string
size int64
offset int64
body io.ReadCloser
}
func (r *blobReader) Read(p []byte) (int, error) {
if r.offset >= r.size {
return 0, io.EOF
}
if r.body == nil {
resp, err := r.c.openFile(r.ctx, mediaCollection, r.id, r.name, r.token, r.offset)
if err != nil {
return 0, err
}
if r.offset > 0 && resp.StatusCode != http.StatusPartialContent {
_ = resp.Body.Close()
return 0, fmt.Errorf("pocketbase: the file server ignored a range request (HTTP %d)", resp.StatusCode)
}
r.body = resp.Body
}
n, err := r.body.Read(p)
r.offset += int64(n)
if errors.Is(err, io.EOF) && r.offset < r.size {
err = io.ErrUnexpectedEOF
}
return n, err
}
func (r *blobReader) Seek(offset int64, whence int) (int64, error) {
var next int64
switch whence {
case io.SeekStart:
next = offset
case io.SeekCurrent:
next = r.offset + offset
case io.SeekEnd:
next = r.size + offset
default:
return 0, errors.New("pocketbase: invalid whence")
}
if next < 0 {
return 0, errors.New("pocketbase: negative position")
}
if next != r.offset && r.body != nil {
_ = r.body.Close()
r.body = nil
}
r.offset = next
return next, nil
}
func (r *blobReader) Close() error {
if r.body != nil {
return r.body.Close()
}
return nil
}
// ctxReader stops a copy when its context ends.
type ctxReader struct {
ctx context.Context
r io.Reader
}
func (c *ctxReader) Read(p []byte) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
}
return c.r.Read(p)
}
var _ storage.BlobStore = (*blobStore)(nil)
@@ -0,0 +1,331 @@
package pocketbase
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"strings"
"sync"
)
// client speaks PocketBase's REST API as a superuser. Everything the driver
// does goes through it: the SQL endpoint for reads, the batch endpoint for
// writes, and the record and file endpoints for media.
type client struct {
base *url.URL
email string
password string
http *http.Client
mu sync.Mutex
token string
}
// newClient parses the server address. Credentials are checked on first use,
// not here: a PocketBase that is still starting is not a configuration
// error.
func newClient(rawURL, email, password string) (*client, error) {
if rawURL == "" {
return nil, errors.New("pocketbase: connection_string must be the server's URL, e.g. http://pocketbase:8070")
}
base, err := url.Parse(strings.TrimSuffix(rawURL, "/"))
if err != nil || (base.Scheme != "http" && base.Scheme != "https") || base.Host == "" {
return nil, fmt.Errorf("pocketbase: connection_string %q is not an http(s) URL", rawURL)
}
if email == "" || password == "" {
return nil, errors.New("pocketbase: a superuser email and password are required")
}
return &client{base: base, email: email, password: password, http: &http.Client{}}, nil
}
// apiError is a non-2xx answer from PocketBase.
type apiError struct {
Status int
Message string
Data json.RawMessage
}
func (e *apiError) Error() string {
msg := fmt.Sprintf("pocketbase: HTTP %d: %s", e.Status, e.Message)
if len(e.Data) > 0 && string(e.Data) != "{}" && string(e.Data) != "null" {
msg += " " + string(e.Data)
}
return msg
}
// isNotFound reports a 404 from PocketBase.
func isNotFound(err error) bool {
var apiErr *apiError
return errors.As(err, &apiErr) && apiErr.Status == http.StatusNotFound
}
// url resolves an API path against the server address.
func (c *client) url(path string) string { return c.base.String() + path }
// authToken returns the current superuser token, signing in if there is
// none yet.
func (c *client) authToken(ctx context.Context) (string, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.token != "" {
return c.token, nil
}
body, err := json.Marshal(map[string]string{"identity": c.email, "password": c.password})
if err != nil {
return "", err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
c.url("/api/collections/_superusers/auth-with-password"), bytes.NewReader(body))
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/json")
var out struct {
Token string `json:"token"`
}
if err := c.send(req, &out); err != nil {
return "", fmt.Errorf("pocketbase: sign in as superuser %q: %w", c.email, err)
}
if out.Token == "" {
return "", errors.New("pocketbase: sign in returned no token")
}
c.token = out.Token
return c.token, nil
}
// forgetToken drops a token PocketBase refused, so the next call signs in
// again. A superuser token outlives most deployments' restarts, but not all.
func (c *client) forgetToken(token string) {
c.mu.Lock()
if c.token == token {
c.token = ""
}
c.mu.Unlock()
}
// do sends one authenticated request, signing in again once if the token has
// expired. body is JSON-encoded unless it is an io.Reader, which is sent as
// is with contentType.
func (c *client) do(ctx context.Context, method, path string, body any, contentType string, out any) error {
var payload []byte
if body != nil {
if r, ok := body.(io.Reader); ok {
// A streamed body cannot be replayed, so it is sent once and a
// stale token is the caller's error to retry.
return c.doOnce(ctx, method, path, r, contentType, out)
}
var err error
if payload, err = json.Marshal(body); err != nil {
return err
}
contentType = "application/json"
}
for attempt := 0; ; attempt++ {
var r io.Reader
if payload != nil {
r = bytes.NewReader(payload)
}
err := c.doOnce(ctx, method, path, r, contentType, out)
var apiErr *apiError
if attempt == 0 && errors.As(err, &apiErr) && apiErr.Status == http.StatusUnauthorized {
continue
}
return err
}
}
func (c *client) doOnce(ctx context.Context, method, path string, body io.Reader, contentType string, out any) error {
token, err := c.authToken(ctx)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, method, c.url(path), body)
if err != nil {
return err
}
req.Header.Set("Authorization", token)
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
err = c.send(req, out)
var apiErr *apiError
if errors.As(err, &apiErr) && apiErr.Status == http.StatusUnauthorized {
c.forgetToken(token)
}
return err
}
// send runs a request and decodes a JSON answer into out, if out is set.
func (c *client) send(req *http.Request, out any) error {
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("pocketbase: %s %s: %w", req.Method, req.URL.Path, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode >= 300 {
return readAPIError(resp)
}
if out == nil || resp.StatusCode == http.StatusNoContent {
_, _ = io.Copy(io.Discard, resp.Body)
return nil
}
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
return fmt.Errorf("pocketbase: decode the answer to %s %s: %w", req.Method, req.URL.Path, err)
}
return nil
}
func readAPIError(resp *http.Response) error {
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
apiErr := &apiError{Status: resp.StatusCode}
var body struct {
Message string `json:"message"`
Data json.RawMessage `json:"data"`
}
if json.Unmarshal(raw, &body) == nil && body.Message != "" {
apiErr.Message = body.Message
apiErr.Data = body.Data
} else {
apiErr.Message = strings.TrimSpace(string(raw))
}
return apiErr
}
// health checks the server answers at all; it needs no token.
func (c *client) health(ctx context.Context) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url("/api/health"), nil)
if err != nil {
return err
}
return c.send(req, nil)
}
// sqlResult is what PocketBase's SQL endpoint answers with. Every value comes
// back as text, or null.
type sqlResult struct {
Columns []struct {
Name string `json:"name"`
} `json:"columns"`
Rows [][]*string `json:"rows"`
}
// query runs one SELECT through the SQL endpoint.
func (c *client) query(ctx context.Context, query string) (*sqlResult, error) {
var out sqlResult
if err := c.do(ctx, http.MethodPost, "/api/sql", map[string]string{"query": query}, "", &out); err != nil {
return nil, err
}
return &out, nil
}
// batchRequest is one record operation inside a batch.
type batchRequest struct {
Method string `json:"method"`
URL string `json:"url"`
Body map[string]any `json:"body,omitempty"`
}
// batch runs record operations in one PocketBase transaction: all of them
// happen, or none do.
func (c *client) batch(ctx context.Context, reqs []batchRequest) error {
if len(reqs) == 0 {
return nil
}
return c.do(ctx, http.MethodPost, "/api/batch", map[string]any{"requests": reqs}, "", nil)
}
// patchSettings changes part of PocketBase's application settings.
func (c *client) patchSettings(ctx context.Context, settings map[string]any) error {
return c.do(ctx, http.MethodPatch, "/api/settings", settings, "", nil)
}
// importCollections creates or updates collections, leaving any not named
// alone.
func (c *client) importCollections(ctx context.Context, collections []map[string]any) error {
return c.do(ctx, http.MethodPut, "/api/collections/import",
map[string]any{"collections": collections, "deleteMissing": false}, "", nil)
}
// getRecord reads one record as JSON fields.
func (c *client) getRecord(ctx context.Context, collection, id string) (map[string]any, error) {
var out map[string]any
if err := c.do(ctx, http.MethodGet,
"/api/collections/"+collection+"/records/"+url.PathEscape(id), nil, "", &out); err != nil {
return nil, err
}
return out, nil
}
// deleteRecord removes one record.
func (c *client) deleteRecord(ctx context.Context, collection, id string) error {
return c.do(ctx, http.MethodDelete,
"/api/collections/"+collection+"/records/"+url.PathEscape(id), nil, "", nil)
}
// createRecordWithFile creates a record with one file field, streaming the
// file as multipart so that it is never held in memory whole.
func (c *client) createRecordWithFile(ctx context.Context, collection string, fields map[string]string,
fileField, fileName string, file io.Reader) error {
pr, pw := io.Pipe()
mw := multipart.NewWriter(pw)
go func() {
err := func() error {
for k, v := range fields {
if err := mw.WriteField(k, v); err != nil {
return err
}
}
part, err := mw.CreateFormFile(fileField, fileName)
if err != nil {
return err
}
if _, err := io.Copy(part, file); err != nil {
return err
}
return mw.Close()
}()
_ = pw.CloseWithError(err)
}()
err := c.do(ctx, http.MethodPost, "/api/collections/"+collection+"/records", pr, mw.FormDataContentType(), nil)
_ = pr.Close()
return err
}
// fileToken mints the short-lived token protected files are read with.
func (c *client) fileToken(ctx context.Context) (string, error) {
var out struct {
Token string `json:"token"`
}
if err := c.do(ctx, http.MethodPost, "/api/files/token", nil, "", &out); err != nil {
return "", err
}
return out.Token, nil
}
// openFile starts a download of a stored file from offset on.
func (c *client) openFile(ctx context.Context, collection, id, name, token string, offset int64) (*http.Response, error) {
path := "/api/files/" + collection + "/" + url.PathEscape(id) + "/" + url.PathEscape(name) +
"?token=" + url.QueryEscape(token)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url(path), nil)
if err != nil {
return nil, err
}
if offset > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
}
resp, err := c.http.Do(req)
if err != nil {
return nil, fmt.Errorf("pocketbase: download %s: %w", name, err)
}
if resp.StatusCode >= 300 {
defer func() { _ = resp.Body.Close() }()
return nil, readAPIError(resp)
}
return resp, nil
}
@@ -0,0 +1,232 @@
package pocketbase
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// DelayedEvents is the PocketBase implementation of
// storage.DelayedEventStore.
type DelayedEvents struct{ *base }
// delayedEventColumns reads state_key as NULL for a message event, as the
// SQL drivers store it.
const delayedEventColumns = `
delay_id, user_id, device_id, room_id, event_type, CASE WHEN has_state_key THEN state_key END, content,
delay_ms, running_since_ms, send_at_ms,
finalised_at_ms, outcome, reason, event_id, error`
// InsertDelayedEvent implements storage.DelayedEventStore.
func (s *DelayedEvents) InsertDelayedEvent(ctx context.Context, e storage.DelayedEvent) error {
stateKey, hasStateKey := "", false
if e.StateKey != nil {
stateKey, hasStateKey = *e.StateKey, true
}
err := s.write(ctx, func(t *tx) error {
t.put("delayed_events", rec{
"delay_id": e.DelayID, "user_id": e.UserID, "device_id": e.DeviceID, "room_id": e.RoomID,
"event_type": e.Type, "state_key": stateKey, "has_state_key": hasStateKey,
"content": string(e.Content), "delay_ms": e.DelayMS, "running_since_ms": e.RunningSinceMS,
"send_at_ms": e.SendAtMS, "finalised_at_ms": int64(0),
"outcome": "", "reason": "", "event_id": "", "error": "",
})
return nil
})
if err != nil {
return fmt.Errorf("insert delayed event: %w", err)
}
return nil
}
// DelayedEvent implements storage.DelayedEventStore.
func (s *DelayedEvents) DelayedEvent(ctx context.Context, userID, delayID string) (storage.DelayedEvent, error) {
return s.one(ctx, `SELECT`+delayedEventColumns+`
FROM delayed_events WHERE user_id = ? AND delay_id = ?`, userID, delayID)
}
// ScheduledDelayedEvent implements storage.DelayedEventStore.
func (s *DelayedEvents) ScheduledDelayedEvent(ctx context.Context, delayID string) (storage.DelayedEvent, error) {
return s.one(ctx, `SELECT`+delayedEventColumns+`
FROM delayed_events WHERE delay_id = ? AND finalised_at_ms = 0`, delayID)
}
func (s *DelayedEvents) one(ctx context.Context, query string, args ...any) (storage.DelayedEvent, error) {
events, err := s.list(ctx, query, args...)
if err != nil {
return storage.DelayedEvent{}, err
}
if len(events) == 0 {
return storage.DelayedEvent{}, storage.ErrNoSuchDelayedEvent
}
return events[0], nil
}
// DelayedEvents implements storage.DelayedEventStore.
func (s *DelayedEvents) DelayedEvents(ctx context.Context, userID string) ([]storage.DelayedEvent, error) {
return s.list(ctx, `SELECT`+delayedEventColumns+`
FROM delayed_events WHERE user_id = ?
ORDER BY finalised_at_ms = 0 DESC, send_at_ms ASC, finalised_at_ms DESC`, userID)
}
// CountScheduledDelayedEvents implements storage.DelayedEventStore.
func (s *DelayedEvents) CountScheduledDelayedEvents(ctx context.Context, userID string) (int, error) {
var n int
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM delayed_events
WHERE user_id = ? AND finalised_at_ms = 0`, userID).Scan(&n); err != nil {
return 0, fmt.Errorf("count delayed events: %w", err)
}
return n, nil
}
// RestartDelayedEvent implements storage.DelayedEventStore.
func (s *DelayedEvents) RestartDelayedEvent(ctx context.Context, delayID string, nowMS int64) error {
return s.write(ctx, func(t *tx) error {
var delay int64
err := t.queryRow(`SELECT delay_ms FROM delayed_events WHERE delay_id = ? AND finalised_at_ms = 0`, delayID).
Scan(&delay)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.ErrNoSuchDelayedEvent
case err != nil:
return fmt.Errorf("restart delayed event: %w", err)
}
t.put("delayed_events", rec{"delay_id": delayID, "running_since_ms": nowMS, "send_at_ms": nowMS + delay})
return nil
})
}
// FinaliseDelayedEvent implements storage.DelayedEventStore.
func (s *DelayedEvents) FinaliseDelayedEvent(ctx context.Context, e storage.DelayedEvent) error {
return s.write(ctx, func(t *tx) error {
n, err := t.setWhere("delayed_events", rec{
"finalised_at_ms": e.FinalisedAtMS, "outcome": e.Outcome, "reason": e.Reason,
"event_id": e.EventID, "error": string(e.Error),
}, `delay_id = ? AND finalised_at_ms = 0`, e.DelayID)
if err != nil {
return fmt.Errorf("finalise delayed event: %w", err)
}
if n == 0 {
return storage.ErrNoSuchDelayedEvent
}
return nil
})
}
// DueDelayedEvents implements storage.DelayedEventStore.
func (s *DelayedEvents) DueDelayedEvents(ctx context.Context, nowMS int64, limit int) ([]storage.DelayedEvent, error) {
return s.list(ctx, `SELECT`+delayedEventColumns+`
FROM delayed_events WHERE finalised_at_ms = 0 AND send_at_ms <= ?
ORDER BY send_at_ms LIMIT ?`, nowMS, limit)
}
// NextDelayedEventDue implements storage.DelayedEventStore.
func (s *DelayedEvents) NextDelayedEventDue(ctx context.Context) (int64, bool, error) {
var next sql.NullInt64
if err := s.db.QueryRowContext(ctx, `SELECT MIN(send_at_ms) FROM delayed_events
WHERE finalised_at_ms = 0`).Scan(&next); err != nil {
return 0, false, fmt.Errorf("next delayed event: %w", err)
}
return next.Int64, next.Valid, nil
}
// ScheduledDelayedStateEvents implements storage.DelayedEventStore.
func (s *DelayedEvents) ScheduledDelayedStateEvents(ctx context.Context, roomID, eventType, stateKey string) ([]storage.DelayedEvent, error) {
return s.list(ctx, `SELECT`+delayedEventColumns+`
FROM delayed_events
WHERE room_id = ? AND event_type = ? AND has_state_key AND state_key = ? AND finalised_at_ms = 0`,
roomID, eventType, stateKey)
}
// PurgeFinalisedDelayedEvents implements storage.DelayedEventStore.
func (s *DelayedEvents) PurgeFinalisedDelayedEvents(ctx context.Context, beforeMS int64) error {
err := s.write(ctx, func(t *tx) error {
_, err := t.delWhere("delayed_events", `finalised_at_ms <> 0 AND finalised_at_ms < ?`, beforeMS)
return err
})
if err != nil {
return fmt.Errorf("purge delayed events: %w", err)
}
return nil
}
func (s *DelayedEvents) list(ctx context.Context, query string, args ...any) ([]storage.DelayedEvent, error) {
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read delayed events: %w", err)
}
defer func() { _ = rows.Close() }()
var out []storage.DelayedEvent
for rows.Next() {
var (
e storage.DelayedEvent
stateKey sql.NullString
content, errorContent string
)
if err := rows.Scan(&e.DelayID, &e.UserID, &e.DeviceID, &e.RoomID, &e.Type, &stateKey, &content,
&e.DelayMS, &e.RunningSinceMS, &e.SendAtMS,
&e.FinalisedAtMS, &e.Outcome, &e.Reason, &e.EventID, &errorContent,
); err != nil {
return nil, fmt.Errorf("read delayed event: %w", err)
}
if stateKey.Valid {
e.StateKey = &stateKey.String
}
e.Content = []byte(content)
if errorContent != "" {
e.Error = []byte(errorContent)
}
out = append(out, e)
}
return out, rows.Err()
}
// OpenIDTokens is the PocketBase implementation of storage.OpenIDTokenStore.
type OpenIDTokens struct{ *base }
// InsertOpenIDToken implements storage.OpenIDTokenStore.
func (s *OpenIDTokens) InsertOpenIDToken(ctx context.Context, tokenHash, userID string, expiresAtMS int64) error {
err := s.write(ctx, func(t *tx) error {
t.put("openid_tokens", rec{"token_hash": tokenHash, "user_id": userID, "expires_at_ms": expiresAtMS})
return nil
})
if err != nil {
return fmt.Errorf("insert OpenID token: %w", err)
}
return nil
}
// OpenIDTokenUser implements storage.OpenIDTokenStore.
func (s *OpenIDTokens) OpenIDTokenUser(ctx context.Context, tokenHash string, nowMS int64) (string, error) {
var userID string
err := s.db.QueryRowContext(ctx, `SELECT user_id FROM openid_tokens
WHERE token_hash = ? AND expires_at_ms > ?`, tokenHash, nowMS).Scan(&userID)
if errors.Is(err, sql.ErrNoRows) {
return "", storage.ErrNoSuchOpenIDToken
}
if err != nil {
return "", fmt.Errorf("read OpenID token: %w", err)
}
return userID, nil
}
// PurgeOpenIDTokens implements storage.OpenIDTokenStore.
func (s *OpenIDTokens) PurgeOpenIDTokens(ctx context.Context, beforeMS int64) error {
err := s.write(ctx, func(t *tx) error {
_, err := t.delWhere("openid_tokens", `expires_at_ms < ?`, beforeMS)
return err
})
if err != nil {
return fmt.Errorf("purge OpenID tokens: %w", err)
}
return nil
}
var (
_ storage.DelayedEventStore = (*DelayedEvents)(nil)
_ storage.OpenIDTokenStore = (*OpenIDTokens)(nil)
)
@@ -0,0 +1,247 @@
package pocketbase
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Federation is the PocketBase implementation of storage.FederationStore.
type Federation struct{ *base }
// StoreServerKeys implements storage.FederationStore.
func (s *Federation) StoreServerKeys(ctx context.Context, keys []storage.ServerKey) error {
if len(keys) == 0 {
return nil
}
return s.write(ctx, func(t *tx) error {
for _, k := range keys {
t.put("federation_server_keys", rec{
"server_name": k.ServerName, "key_id": k.KeyID, "public_key": k.PublicKey,
"valid_until_ts": k.ValidUntilTS, "expired_ts": k.ExpiredTS, "fetched_at_ms": k.FetchedAtMS,
})
}
return nil
})
}
// ServerKeys implements storage.FederationStore.
func (s *Federation) ServerKeys(ctx context.Context, serverName string, keyIDs []string) ([]storage.ServerKey, error) {
query := `
SELECT server_name, key_id, public_key, valid_until_ts, expired_ts, fetched_at_ms
FROM federation_server_keys WHERE server_name = ?`
args := []any{serverName}
if len(keyIDs) > 0 {
query += ` AND key_id IN ` + inList
args = append(args, jsonList(keyIDs))
}
query += ` ORDER BY key_id`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read the keys of %q: %w", serverName, err)
}
defer func() { _ = rows.Close() }()
var out []storage.ServerKey
for rows.Next() {
var k storage.ServerKey
if err := rows.Scan(&k.ServerName, &k.KeyID, &k.PublicKey,
&k.ValidUntilTS, &k.ExpiredTS, &k.FetchedAtMS); err != nil {
return nil, fmt.Errorf("read the keys of %q: %w", serverName, err)
}
out = append(out, k)
}
return out, rows.Err()
}
// Enqueue implements storage.FederationStore.
func (s *Federation) Enqueue(ctx context.Context, items []storage.QueuedItem) ([]storage.QueuedItem, error) {
if len(items) == 0 {
return nil, nil
}
out := make([]storage.QueuedItem, len(items))
copy(out, items)
err := s.write(ctx, func(t *tx) error {
top, err := t.advance(federationSequences, seqQueue, int64(len(out)))
if err != nil {
return err
}
first := top - int64(len(out)) + 1
for i := range out {
out[i].ID = first + int64(i)
item := out[i]
t.put("federation_queue", rec{
"queue_id": item.ID, "destination": item.Destination, "kind": item.Kind,
"room_id": item.RoomID, "event_id": item.EventID,
"payload": string(item.Payload), "queued_at_ms": item.QueuedAtMS,
})
}
return nil
})
if err != nil {
return nil, fmt.Errorf("enqueue %d items: %w", len(items), err)
}
return out, nil
}
// Queued implements storage.FederationStore.
func (s *Federation) Queued(ctx context.Context, destination, kind string, limit int) ([]storage.QueuedItem, error) {
if limit <= 0 {
return nil, nil
}
rows, err := s.db.QueryContext(ctx, `
SELECT queue_id, destination, kind, room_id, event_id, payload, queued_at_ms
FROM federation_queue
WHERE destination = ? AND kind = ?
ORDER BY queue_id
LIMIT ?`, destination, kind, limit)
if err != nil {
return nil, fmt.Errorf("read the queue for %q: %w", destination, err)
}
defer func() { _ = rows.Close() }()
var out []storage.QueuedItem
for rows.Next() {
var (
item storage.QueuedItem
payload string
)
if err := rows.Scan(&item.ID, &item.Destination, &item.Kind, &item.RoomID,
&item.EventID, &payload, &item.QueuedAtMS); err != nil {
return nil, fmt.Errorf("read the queue for %q: %w", destination, err)
}
item.Payload = []byte(payload)
out = append(out, item)
}
return out, rows.Err()
}
// Dequeue implements storage.FederationStore.
func (s *Federation) Dequeue(ctx context.Context, ids []int64) error {
if len(ids) == 0 {
return nil
}
err := s.write(ctx, func(t *tx) error {
_, err := t.delWhere("federation_queue", `queue_id IN `+inList, jsonInts(ids))
return err
})
if err != nil {
return fmt.Errorf("dequeue %d items: %w", len(ids), err)
}
return nil
}
// QueuedDestinations implements storage.FederationStore.
func (s *Federation) QueuedDestinations(ctx context.Context) ([]string, error) {
out, err := s.queryStrings(ctx, `SELECT DISTINCT destination FROM federation_queue ORDER BY destination`)
if err != nil {
return nil, fmt.Errorf("read the queued destinations: %w", err)
}
return out, nil
}
const destinationColumns = `
server_name, failures, retry_at_ms, last_success_ms, last_failure_ms, last_error`
// Destination implements storage.FederationStore.
func (s *Federation) Destination(ctx context.Context, serverName string) (storage.Destination, error) {
var d storage.Destination
err := s.db.QueryRowContext(ctx, `SELECT `+destinationColumns+`
FROM federation_destinations WHERE server_name = ?`, serverName).
Scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS, &d.LastFailureMS, &d.LastError)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.Destination{ServerName: serverName}, nil
case err != nil:
return storage.Destination{}, fmt.Errorf("read destination %q: %w", serverName, err)
}
return d, nil
}
// SetDestination implements storage.FederationStore.
func (s *Federation) SetDestination(ctx context.Context, d storage.Destination) error {
err := s.write(ctx, func(t *tx) error {
t.put("federation_destinations", rec{
"server_name": d.ServerName, "failures": d.Failures, "retry_at_ms": d.RetryAtMS,
"last_success_ms": d.LastSuccessMS, "last_failure_ms": d.LastFailureMS, "last_error": d.LastError,
})
return nil
})
if err != nil {
return fmt.Errorf("record destination %q: %w", d.ServerName, err)
}
return nil
}
// Destinations implements storage.FederationStore: every server with a
// health row or something queued.
func (s *Federation) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT n.server_name,
COALESCE(d.failures, 0), COALESCE(d.retry_at_ms, 0),
COALESCE(d.last_success_ms, 0), COALESCE(d.last_failure_ms, 0),
COALESCE(d.last_error, ''),
(SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'pdu'),
(SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'edu')
FROM (
SELECT server_name FROM federation_destinations
UNION
SELECT DISTINCT destination FROM federation_queue
) n
LEFT JOIN federation_destinations d ON d.server_name = n.server_name
ORDER BY n.server_name`)
if err != nil {
return nil, fmt.Errorf("read the destinations: %w", err)
}
defer func() { _ = rows.Close() }()
var out []storage.DestinationSummary
for rows.Next() {
var d storage.DestinationSummary
if err := rows.Scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS,
&d.LastFailureMS, &d.LastError, &d.QueuedPDUs, &d.QueuedEDUs); err != nil {
return nil, fmt.Errorf("read the destinations: %w", err)
}
out = append(out, d)
}
return out, rows.Err()
}
// InboundTransaction implements storage.FederationStore.
func (s *Federation) InboundTransaction(ctx context.Context, origin, txnID string) ([]byte, bool, error) {
var response string
err := s.db.QueryRowContext(ctx, `SELECT response_json FROM federation_inbound_txns
WHERE origin = ? AND txn_id = ?`, origin, txnID).Scan(&response)
switch {
case errors.Is(err, sql.ErrNoRows):
return nil, false, nil
case err != nil:
return nil, false, fmt.Errorf("read transaction %q from %q: %w", txnID, origin, err)
}
return []byte(response), true, nil
}
// RecordInboundTransaction implements storage.FederationStore. A
// transaction already recorded keeps its first response.
func (s *Federation) RecordInboundTransaction(ctx context.Context, origin, txnID string, response []byte, atMS int64) error {
err := s.write(ctx, func(t *tx) error {
found, err := t.exists("federation_inbound_txns", `origin = ? AND txn_id = ?`, origin, txnID)
if err != nil || found {
return err
}
t.put("federation_inbound_txns", rec{
"origin": origin, "txn_id": txnID, "response_json": string(response), "received_at_ms": atMS,
})
return nil
})
if err != nil {
return fmt.Errorf("record transaction %q from %q: %w", txnID, origin, err)
}
return nil
}
var _ storage.FederationStore = (*Federation)(nil)
@@ -0,0 +1,83 @@
package pocketbase
import (
"encoding/json"
"strings"
)
// The counter tables and the names in them, as the SQL drivers have them.
const (
roomSequences = "roomserver_sequences"
syncSequences = "sync_sequences"
federationSequences = "federation_sequences"
keySequences = "keyserver_sequences"
pushSequences = "push_sequences"
adminSequences = "admin_sequences"
seqStreamOrdering = "stream_ordering"
seqStateSnapshot = "state_snapshot"
seqBackfillOrdering = "backfill_ordering"
seqReceipt = "receipt"
seqToDevice = "to_device"
seqAccountData = "account_data"
seqPresence = "presence"
seqQueue = "queue"
seqDeviceList = "device_list"
seqNotification = "notification"
seqAudit = "audit"
)
// inList is the IN-list form every store uses: one JSON array argument,
// expanded by json_each, instead of a placeholder per element. The SQL
// endpoint takes no parameters and at most 5000 characters, and a single
// argument is what the driver can move out of a query that would be too
// long; a few hundred separate ones would each have to be moved.
const inList = `(SELECT value FROM json_each(?))`
// jsonList renders strings as the JSON array inList takes.
func jsonList(values []string) string {
if values == nil {
values = []string{}
}
raw, _ := json.Marshal(values)
return string(raw)
}
// jsonInts renders integers as the JSON array inList takes.
func jsonInts(values []int64) string {
if values == nil {
values = []int64{}
}
raw, _ := json.Marshal(values)
return string(raw)
}
// pageBounds clamps a caller's limit and offset.
func pageBounds(limit, offset, defaultLimit, maxLimit int) (int, int) {
if limit <= 0 {
limit = defaultLimit
}
if limit > maxLimit {
limit = maxLimit
}
if offset < 0 {
offset = 0
}
return limit, offset
}
// escapeLike makes a literal string safe to embed in a LIKE pattern that
// declares '\' as its escape character.
func escapeLike(s string) string {
var b strings.Builder
b.Grow(len(s))
for _, r := range s {
if r == '\\' || r == '%' || r == '_' {
b.WriteRune('\\')
}
b.WriteRune(r)
}
return b.String()
}
+766
View File
@@ -0,0 +1,766 @@
package pocketbase
import (
"cmp"
"context"
"database/sql"
"errors"
"fmt"
"slices"
"strconv"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Keys is the PocketBase implementation of storage.KeyStore.
//
// Callers hand it key JSON already in canonical form, so "the same key
// uploaded again" is a byte comparison. Where the SQL store reads back what
// it has written inside its transaction — a device list position moved
// earlier in the call, a backup's key count and etag after a change — this
// one carries the value in the call instead.
type Keys struct{ *base }
// maxInList caps an IN list, as the SQL store does.
const maxInList = 500
// keysWrite is one write call's device list positions, as it has left them.
type keysWrite struct {
*tx
lists map[string]int64
}
func newKeysWrite(t *tx) *keysWrite { return &keysWrite{tx: t, lists: map[string]int64{}} }
// --- the device list stream ---------------------------------------------------
// bumpDeviceList moves a user's device list once per device named, or once
// for the user when none is, and returns an update per move. Each device
// gets a position of its own, because a remote server treats a stream ID it
// has already seen as a duplicate.
func (w *keysWrite) bumpDeviceList(userID string, deviceIDs []string) ([]storage.DeviceListUpdate, error) {
if len(deviceIDs) == 0 {
deviceIDs = []string{""}
}
n := int64(len(deviceIDs))
last, err := w.advance(keySequences, seqDeviceList, n)
if err != nil {
return nil, err
}
prev, ok := w.lists[userID]
if !ok {
err = w.queryRow(`SELECT stream_pos FROM keyserver_device_lists WHERE user_id = ?`, userID).Scan(&prev)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, fmt.Errorf("read the device list position of %q: %w", userID, err)
}
}
out := make([]storage.DeviceListUpdate, 0, n)
for i, deviceID := range deviceIDs {
pos := last - n + 1 + int64(i)
out = append(out, storage.DeviceListUpdate{
UserID: userID, DeviceID: deviceID, StreamPos: pos, PrevPos: prev,
})
prev = pos
}
w.put("keyserver_device_lists", rec{"user_id": userID, "stream_pos": last})
w.lists[userID] = last
return out, nil
}
// MarkDeviceListsChanged implements storage.KeyStore.
func (s *Keys) MarkDeviceListsChanged(ctx context.Context, userIDs []string) (int64, error) {
var top int64
err := s.write(ctx, func(t *tx) error {
w := newKeysWrite(t)
for _, userID := range dedupe(userIDs) {
updates, err := w.bumpDeviceList(userID, nil)
if err != nil {
return err
}
top = updates[len(updates)-1].StreamPos
}
return nil
})
return top, err
}
// DeviceListChanges implements storage.KeyStore.
func (s *Keys) DeviceListChanges(ctx context.Context, userIDs []string, from, to int64) ([]storage.DeviceListChange, error) {
var out []storage.DeviceListChange
for chunk := range slices.Chunk(dedupe(userIDs), maxInList) {
query := `
SELECT user_id, stream_pos FROM keyserver_device_lists
WHERE user_id IN ` + inList + ` AND stream_pos > ?`
args := []any{jsonList(chunk), from}
if to > 0 {
query += ` AND stream_pos <= ?`
args = append(args, to)
}
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read device list changes: %w", err)
}
for rows.Next() {
var c storage.DeviceListChange
if err := rows.Scan(&c.UserID, &c.StreamPos); err != nil {
_ = rows.Close()
return nil, fmt.Errorf("read device list changes: %w", err)
}
out = append(out, c)
}
_ = rows.Close()
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read device list changes: %w", err)
}
}
slices.SortFunc(out, func(a, b storage.DeviceListChange) int { return cmp.Compare(a.StreamPos, b.StreamPos) })
return out, nil
}
// DeviceListPosition implements storage.KeyStore.
func (s *Keys) DeviceListPosition(ctx context.Context, userID string) (int64, error) {
var pos int64
err := s.db.QueryRowContext(ctx, `SELECT stream_pos FROM keyserver_device_lists WHERE user_id = ?`, userID).Scan(&pos)
switch {
case errors.Is(err, sql.ErrNoRows):
return 0, nil
case err != nil:
return 0, fmt.Errorf("read the device list position of %q: %w", userID, err)
}
return pos, nil
}
// --- device keys --------------------------------------------------------------
// SetDeviceKeys implements storage.KeyStore.
func (s *Keys) SetDeviceKeys(ctx context.Context, keys storage.DeviceKeys) (storage.DeviceListUpdate, error) {
var update storage.DeviceListUpdate
err := s.write(ctx, func(t *tx) error {
var existing string
err := t.queryRow(`SELECT key_json FROM keyserver_device_keys WHERE user_id = ? AND device_id = ?`,
keys.UserID, keys.DeviceID).Scan(&existing)
switch {
case err == nil && existing == string(keys.JSON):
return nil
case err != nil && !errors.Is(err, sql.ErrNoRows):
return fmt.Errorf("read the keys of %s/%s: %w", keys.UserID, keys.DeviceID, err)
}
t.put("keyserver_device_keys", rec{"user_id": keys.UserID, "device_id": keys.DeviceID, "key_json": string(keys.JSON)})
updates, err := newKeysWrite(t).bumpDeviceList(keys.UserID, []string{keys.DeviceID})
if err != nil {
return err
}
update = updates[0]
return nil
})
return update, err
}
// DeviceKeys implements storage.KeyStore.
func (s *Keys) DeviceKeys(ctx context.Context, userID string, deviceIDs []string) ([]storage.DeviceKeys, error) {
query := `SELECT device_id, key_json FROM keyserver_device_keys WHERE user_id = ?`
args := []any{userID}
if len(deviceIDs) > 0 {
deviceIDs = dedupe(deviceIDs)
if len(deviceIDs) > maxInList {
deviceIDs = deviceIDs[:maxInList]
}
query += ` AND device_id IN ` + inList
args = append(args, jsonList(deviceIDs))
}
query += ` ORDER BY device_id`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read the device keys of %q: %w", userID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.DeviceKeys
for rows.Next() {
k := storage.DeviceKeys{UserID: userID}
var keyJSON string
if err := rows.Scan(&k.DeviceID, &keyJSON); err != nil {
return nil, fmt.Errorf("read the device keys of %q: %w", userID, err)
}
k.JSON = []byte(keyJSON)
out = append(out, k)
}
return out, rows.Err()
}
// DeleteDeviceKeys implements storage.KeyStore.
func (s *Keys) DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []string) ([]storage.DeviceListUpdate, error) {
var updates []storage.DeviceListUpdate
err := s.write(ctx, func(t *tx) error {
where := `user_id = ?`
args := []any{userID}
if len(deviceIDs) > 0 {
where += ` AND device_id IN ` + inList
args = append(args, jsonList(dedupe(deviceIDs)))
}
// The devices that lose identity keys are the ones whose deletion
// moves the stream.
had, err := s.queryStringsTx(t, `SELECT device_id FROM keyserver_device_keys WHERE `+where+` ORDER BY device_id`, args...)
if err != nil {
return fmt.Errorf("read the devices of %q: %w", userID, err)
}
for _, table := range []string{"keyserver_device_keys", "keyserver_one_time_keys", "keyserver_fallback_keys"} {
if _, err := t.delWhere(table, where, args...); err != nil {
return fmt.Errorf("delete from %s for %q: %w", table, userID, err)
}
}
if len(had) == 0 {
return nil
}
updates, err = newKeysWrite(t).bumpDeviceList(userID, had)
return err
})
return updates, err
}
// queryStringsTx runs a one-column query inside a write.
func (s *Keys) queryStringsTx(t *tx, query string, args ...any) ([]string, error) {
rows, err := t.query(query, args...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []string
for rows.Next() {
var v string
if err := rows.Scan(&v); err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
// --- one-time and fallback keys -----------------------------------------------
// AddOneTimeKeys implements storage.KeyStore. The same key again is not an
// error; another key under the same ID is a conflict, and nothing is
// stored.
func (s *Keys) AddOneTimeKeys(ctx context.Context, userID, deviceID string, keys []storage.OneTimeKey) error {
if len(keys) == 0 {
return nil
}
return s.write(ctx, func(t *tx) error {
ids := make([]string, len(keys))
for i, k := range keys {
ids[i] = k.KeyID
}
held := map[string]string{}
rows, err := t.query(`SELECT key_id, key_json FROM keyserver_one_time_keys
WHERE user_id = ? AND device_id = ? AND key_id IN `+inList, userID, deviceID, jsonList(ids))
if err != nil {
return fmt.Errorf("read the one-time keys of %s/%s: %w", userID, deviceID, err)
}
for rows.Next() {
var id, keyJSON string
if err := rows.Scan(&id, &keyJSON); err != nil {
_ = rows.Close()
return fmt.Errorf("read the one-time keys of %s/%s: %w", userID, deviceID, err)
}
held[id] = keyJSON
}
_ = rows.Close()
if err := rows.Err(); err != nil {
return fmt.Errorf("read the one-time keys of %s/%s: %w", userID, deviceID, err)
}
for _, k := range keys {
if existing, ok := held[k.KeyID]; ok {
if existing == string(k.JSON) {
continue
}
return fmt.Errorf("%w: %s", storage.ErrKeyConflict, k.KeyID)
}
t.put("keyserver_one_time_keys", rec{
"user_id": userID, "device_id": deviceID, "key_id": k.KeyID,
"algorithm": k.Algorithm(), "key_json": string(k.JSON),
})
held[k.KeyID] = string(k.JSON)
}
return nil
})
}
// OneTimeKeyCounts implements storage.KeyStore.
func (s *Keys) OneTimeKeyCounts(ctx context.Context, userID, deviceID string) (map[string]int, error) {
rows, err := s.db.QueryContext(ctx, `SELECT algorithm, COUNT(*) FROM keyserver_one_time_keys
WHERE user_id = ? AND device_id = ? GROUP BY algorithm`, userID, deviceID)
if err != nil {
return nil, fmt.Errorf("count the one-time keys of %s/%s: %w", userID, deviceID, err)
}
defer func() { _ = rows.Close() }()
out := map[string]int{}
for rows.Next() {
var (
algorithm string
n int
)
if err := rows.Scan(&algorithm, &n); err != nil {
return nil, fmt.Errorf("count the one-time keys of %s/%s: %w", userID, deviceID, err)
}
out[algorithm] = n
}
return out, rows.Err()
}
const selectFallbackKeySQL = `
SELECT key_id, key_json FROM keyserver_fallback_keys
WHERE user_id = ? AND device_id = ? AND algorithm = ?`
// SetFallbackKeys implements storage.KeyStore. The same key again keeps its
// used flag; a new one starts unused.
func (s *Keys) SetFallbackKeys(ctx context.Context, userID, deviceID string, keys []storage.OneTimeKey) error {
if len(keys) == 0 {
return nil
}
return s.write(ctx, func(t *tx) error {
for _, k := range keys {
var existingID, existingJSON string
err := t.queryRow(selectFallbackKeySQL, userID, deviceID, k.Algorithm()).Scan(&existingID, &existingJSON)
switch {
case err == nil && existingID == k.KeyID && existingJSON == string(k.JSON):
continue
case err != nil && !errors.Is(err, sql.ErrNoRows):
return fmt.Errorf("read the fallback key of %s/%s: %w", userID, deviceID, err)
}
t.put("keyserver_fallback_keys", rec{
"user_id": userID, "device_id": deviceID, "algorithm": k.Algorithm(),
"key_id": k.KeyID, "key_json": string(k.JSON), "used": false,
})
}
return nil
})
}
// UnusedFallbackKeyTypes implements storage.KeyStore.
func (s *Keys) UnusedFallbackKeyTypes(ctx context.Context, userID, deviceID string) ([]string, error) {
out, err := s.queryStrings(ctx, `SELECT algorithm FROM keyserver_fallback_keys
WHERE user_id = ? AND device_id = ? AND used = ? ORDER BY algorithm`, userID, deviceID, false)
if err != nil {
return nil, fmt.Errorf("read the fallback keys of %s/%s: %w", userID, deviceID, err)
}
return out, nil
}
// ClaimKey implements storage.KeyStore. The read and the delete happen under
// the write lock, so a claim cannot lose the key it read to another one.
func (s *Keys) ClaimKey(ctx context.Context, userID, deviceID, algorithm string) (storage.ClaimedKey, bool, error) {
var (
claimed storage.ClaimedKey
found bool
)
err := s.write(ctx, func(t *tx) error {
var keyID, keyJSON string
err := t.queryRow(`SELECT key_id, key_json FROM keyserver_one_time_keys
WHERE user_id = ? AND device_id = ? AND algorithm = ?
ORDER BY key_id LIMIT 1`, userID, deviceID, algorithm).Scan(&keyID, &keyJSON)
switch {
case err == nil:
t.del("keyserver_one_time_keys", rowID("keyserver_one_time_keys", userID, deviceID, keyID))
claimed = storage.ClaimedKey{OneTimeKey: storage.OneTimeKey{KeyID: keyID, JSON: []byte(keyJSON)}}
found = true
return nil
case !errors.Is(err, sql.ErrNoRows):
return fmt.Errorf("claim a one-time key of %s/%s: %w", userID, deviceID, err)
}
err = t.queryRow(selectFallbackKeySQL, userID, deviceID, algorithm).Scan(&keyID, &keyJSON)
switch {
case errors.Is(err, sql.ErrNoRows):
return nil
case err != nil:
return fmt.Errorf("claim the fallback key of %s/%s: %w", userID, deviceID, err)
}
t.put("keyserver_fallback_keys", rec{"user_id": userID, "device_id": deviceID, "algorithm": algorithm, "used": true})
claimed = storage.ClaimedKey{OneTimeKey: storage.OneTimeKey{KeyID: keyID, JSON: []byte(keyJSON)}, Fallback: true}
found = true
return nil
})
if err != nil {
return storage.ClaimedKey{}, false, err
}
return claimed, found, nil
}
// --- cross-signing ------------------------------------------------------------
// SetCrossSigningKeys implements storage.KeyStore. A nil key deletes that
// purpose.
func (s *Keys) SetCrossSigningKeys(ctx context.Context, userID string, keys map[string][]byte) (storage.DeviceListUpdate, error) {
var update storage.DeviceListUpdate
err := s.write(ctx, func(t *tx) error {
for _, purpose := range sortedKeys(keys) {
keyJSON := keys[purpose]
if keyJSON == nil {
if _, err := t.delKey("keyserver_cross_signing_keys", userID, purpose); err != nil {
return fmt.Errorf("store the %s key of %q: %w", purpose, userID, err)
}
continue
}
t.put("keyserver_cross_signing_keys", rec{"user_id": userID, "purpose": purpose, "key_json": string(keyJSON)})
}
updates, err := newKeysWrite(t).bumpDeviceList(userID, nil)
if err != nil {
return err
}
update = updates[0]
return nil
})
return update, err
}
// CrossSigningKeys implements storage.KeyStore.
func (s *Keys) CrossSigningKeys(ctx context.Context, userID string) (map[string][]byte, error) {
rows, err := s.db.QueryContext(ctx, `SELECT purpose, key_json FROM keyserver_cross_signing_keys WHERE user_id = ?`, userID)
if err != nil {
return nil, fmt.Errorf("read the cross-signing keys of %q: %w", userID, err)
}
defer func() { _ = rows.Close() }()
out := map[string][]byte{}
for rows.Next() {
var purpose, keyJSON string
if err := rows.Scan(&purpose, &keyJSON); err != nil {
return nil, fmt.Errorf("read the cross-signing keys of %q: %w", userID, err)
}
out[purpose] = []byte(keyJSON)
}
return out, rows.Err()
}
// --- signatures ---------------------------------------------------------------
// AddSignatures implements storage.KeyStore.
func (s *Keys) AddSignatures(ctx context.Context, sigs []storage.KeySignature) ([]storage.DeviceListUpdate, error) {
if len(sigs) == 0 {
return nil, nil
}
var updates []storage.DeviceListUpdate
err := s.write(ctx, func(t *tx) error {
w := newKeysWrite(t)
targets := map[string][]string{}
for _, sig := range sigs {
t.put("keyserver_signatures", rec{
"target_user_id": sig.TargetUserID, "target_key_id": sig.TargetKeyID,
"signer_user_id": sig.SignerUserID, "signer_key_id": sig.SignerKeyID,
"signature": sig.Signature,
})
if !slices.Contains(targets[sig.TargetUserID], sig.TargetKeyID) {
targets[sig.TargetUserID] = append(targets[sig.TargetUserID], sig.TargetKeyID)
}
}
for _, userID := range sortedKeys(targets) {
moved, err := w.bumpDeviceList(userID, targets[userID])
if err != nil {
return err
}
updates = append(updates, moved...)
}
return nil
})
return updates, err
}
// Signatures implements storage.KeyStore.
func (s *Keys) Signatures(ctx context.Context, targetUserID string) ([]storage.KeySignature, error) {
rows, err := s.db.QueryContext(ctx, `SELECT target_key_id, signer_user_id, signer_key_id, signature
FROM keyserver_signatures WHERE target_user_id = ?
ORDER BY target_key_id, signer_user_id, signer_key_id`, targetUserID)
if err != nil {
return nil, fmt.Errorf("read the signatures of %q's keys: %w", targetUserID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.KeySignature
for rows.Next() {
sig := storage.KeySignature{TargetUserID: targetUserID}
if err := rows.Scan(&sig.TargetKeyID, &sig.SignerUserID, &sig.SignerKeyID, &sig.Signature); err != nil {
return nil, fmt.Errorf("read the signatures of %q's keys: %w", targetUserID, err)
}
out = append(out, sig)
}
return out, rows.Err()
}
// --- key backup ---------------------------------------------------------------
// parseBackupVersion reads a version number. A version that is not one is a
// version that does not exist.
func parseBackupVersion(version string) (int64, bool) {
n, err := strconv.ParseInt(version, 10, 64)
return n, err == nil && n > 0
}
// rowQuerier is what readBackup reads through: the pool, or a write.
type rowQuerier func(query string, args ...any) *sql.Row
func (s *Keys) poolRow(ctx context.Context) rowQuerier {
return func(query string, args ...any) *sql.Row { return s.db.QueryRowContext(ctx, query, args...) }
}
// readBackup reads one live version, with its key count.
func readBackup(row rowQuerier, userID, version string) (storage.BackupVersion, int64, int64, error) {
var n int64
if version == "" {
if err := row(`SELECT COALESCE(MAX(version), 0) FROM keyserver_backup_versions
WHERE user_id = ? AND deleted = ?`, userID, false).Scan(&n); err != nil {
return storage.BackupVersion{}, 0, 0, fmt.Errorf("read the backup version of %q: %w", userID, err)
}
if n == 0 {
return storage.BackupVersion{}, 0, 0, storage.ErrNoSuchBackup
}
} else {
var ok bool
if n, ok = parseBackupVersion(version); !ok {
return storage.BackupVersion{}, 0, 0, storage.ErrNoSuchBackup
}
}
out := storage.BackupVersion{UserID: userID, Version: strconv.FormatInt(n, 10)}
var (
authData string
etag int64
)
err := row(`SELECT algorithm, auth_data, etag FROM keyserver_backup_versions
WHERE user_id = ? AND version = ? AND deleted = ?`, userID, n, false).Scan(&out.Algorithm, &authData, &etag)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.BackupVersion{}, 0, 0, storage.ErrNoSuchBackup
case err != nil:
return storage.BackupVersion{}, 0, 0, fmt.Errorf("read backup version %d of %q: %w", n, userID, err)
}
out.AuthData = []byte(authData)
out.ETag = strconv.FormatInt(etag, 10)
if err := row(`SELECT COUNT(*) FROM keyserver_backup_keys WHERE user_id = ? AND version = ?`, userID, n).
Scan(&out.Count); err != nil {
return storage.BackupVersion{}, 0, 0, fmt.Errorf("count the keys of backup version %d of %q: %w", n, userID, err)
}
return out, n, etag, nil
}
// CreateBackup implements storage.KeyStore.
func (s *Keys) CreateBackup(ctx context.Context, userID, algorithm string, authData []byte) (string, error) {
var version int64
err := s.write(ctx, func(t *tx) error {
if err := t.queryRow(`SELECT COALESCE(MAX(version), 0) FROM keyserver_backup_versions WHERE user_id = ?`,
userID).Scan(&version); err != nil {
return fmt.Errorf("read the backup versions of %q: %w", userID, err)
}
version++
t.put("keyserver_backup_versions", rec{
"user_id": userID, "version": version, "algorithm": algorithm,
"auth_data": string(authData), "etag": int64(0), "deleted": false,
})
return nil
})
if err != nil {
return "", err
}
return strconv.FormatInt(version, 10), nil
}
// Backup implements storage.KeyStore.
func (s *Keys) Backup(ctx context.Context, userID, version string) (storage.BackupVersion, error) {
out, _, _, err := readBackup(s.poolRow(ctx), userID, version)
return out, err
}
// UpdateBackup implements storage.KeyStore.
func (s *Keys) UpdateBackup(ctx context.Context, userID, version string, authData []byte) error {
n, ok := parseBackupVersion(version)
if !ok {
return storage.ErrNoSuchBackup
}
return s.write(ctx, func(t *tx) error {
affected, err := t.setWhere("keyserver_backup_versions", rec{"auth_data": string(authData)},
`user_id = ? AND version = ? AND deleted = ?`, userID, n, false)
if err != nil {
return fmt.Errorf("update backup version %d of %q: %w", n, userID, err)
}
if affected == 0 {
return storage.ErrNoSuchBackup
}
return nil
})
}
// DeleteBackup implements storage.KeyStore.
func (s *Keys) DeleteBackup(ctx context.Context, userID, version string) error {
n, ok := parseBackupVersion(version)
if !ok {
return storage.ErrNoSuchBackup
}
return s.write(ctx, func(t *tx) error {
affected, err := t.setWhere("keyserver_backup_versions", rec{"deleted": true},
`user_id = ? AND version = ? AND deleted = ?`, userID, n, false)
if err != nil {
return fmt.Errorf("delete backup version %d of %q: %w", n, userID, err)
}
if affected == 0 {
return storage.ErrNoSuchBackup
}
if _, err := t.delWhere("keyserver_backup_keys", `user_id = ? AND version = ?`, userID, n); err != nil {
return fmt.Errorf("delete the keys of backup version %d of %q: %w", n, userID, err)
}
return nil
})
}
// PutBackupKeys implements storage.KeyStore.
func (s *Keys) PutBackupKeys(ctx context.Context, userID, version string, keys []storage.BackupKey) (storage.BackupVersion, error) {
var out storage.BackupVersion
err := s.write(ctx, func(t *tx) error {
current, n, etag, err := readBackup(t.queryRow, userID, version)
if err != nil {
return err
}
changed := false
written := map[string]storage.BackupKey{}
for _, k := range keys {
slot := k.RoomID + "\x00" + k.SessionID
existing, have := written[slot]
if !have {
var sessionData string
err := t.queryRow(`SELECT first_message_index, forwarded_count, is_verified, session_data
FROM keyserver_backup_keys
WHERE user_id = ? AND version = ? AND room_id = ? AND session_id = ?`, userID, n, k.RoomID, k.SessionID).
Scan(&existing.FirstMessageIndex, &existing.ForwardedCount, &existing.IsVerified, &sessionData)
switch {
case err == nil:
have = true
case !errors.Is(err, sql.ErrNoRows):
return fmt.Errorf("read backed-up key %s/%s: %w", k.RoomID, k.SessionID, err)
}
if !have {
current.Count++
}
}
if have && !k.Replaces(existing) {
written[slot] = existing
continue
}
t.put("keyserver_backup_keys", rec{
"user_id": userID, "version": n, "room_id": k.RoomID, "session_id": k.SessionID,
"first_message_index": k.FirstMessageIndex, "forwarded_count": k.ForwardedCount,
"is_verified": k.IsVerified, "session_data": string(k.SessionData),
})
written[slot] = k
changed = true
}
if changed {
etag++
t.put("keyserver_backup_versions", rec{"user_id": userID, "version": n, "etag": etag})
}
current.ETag = strconv.FormatInt(etag, 10)
out = current
return nil
})
return out, err
}
// backupKeyScope is the WHERE clause the key reads and deletes share.
func backupKeyScope(userID string, version int64, roomID, sessionID string) (string, []any) {
where := `user_id = ? AND version = ?`
args := []any{userID, version}
if roomID != "" {
where += ` AND room_id = ?`
args = append(args, roomID)
if sessionID != "" {
where += ` AND session_id = ?`
args = append(args, sessionID)
}
}
return where, args
}
// BackupKeys implements storage.KeyStore.
func (s *Keys) BackupKeys(ctx context.Context, userID, version, roomID, sessionID string) ([]storage.BackupKey, error) {
_, n, _, err := readBackup(s.poolRow(ctx), userID, version)
if err != nil {
return nil, err
}
where, args := backupKeyScope(userID, n, roomID, sessionID)
rows, err := s.db.QueryContext(ctx, `
SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data
FROM keyserver_backup_keys WHERE `+where+` ORDER BY room_id, session_id`, args...)
if err != nil {
return nil, fmt.Errorf("read the keys of backup version %d of %q: %w", n, userID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.BackupKey
for rows.Next() {
var (
k storage.BackupKey
sessionData string
)
if err := rows.Scan(&k.RoomID, &k.SessionID, &k.FirstMessageIndex, &k.ForwardedCount,
&k.IsVerified, &sessionData); err != nil {
return nil, fmt.Errorf("read the keys of backup version %d of %q: %w", n, userID, err)
}
k.SessionData = []byte(sessionData)
out = append(out, k)
}
return out, rows.Err()
}
// DeleteBackupKeys implements storage.KeyStore.
func (s *Keys) DeleteBackupKeys(ctx context.Context, userID, version, roomID, sessionID string) (storage.BackupVersion, error) {
var out storage.BackupVersion
err := s.write(ctx, func(t *tx) error {
current, n, etag, err := readBackup(t.queryRow, userID, version)
if err != nil {
return err
}
where, args := backupKeyScope(userID, n, roomID, sessionID)
deleted, err := t.delWhere("keyserver_backup_keys", where, args...)
if err != nil {
return fmt.Errorf("delete keys from backup version %d of %q: %w", n, userID, err)
}
if deleted > 0 {
etag++
t.put("keyserver_backup_versions", rec{"user_id": userID, "version": n, "etag": etag})
}
current.Count -= deleted
current.ETag = strconv.FormatInt(etag, 10)
out = current
return nil
})
return out, err
}
// --- helpers ------------------------------------------------------------------
// dedupe returns the distinct values of a list, sorted.
func dedupe(values []string) []string {
out := slices.Clone(values)
slices.Sort(out)
return slices.Compact(out)
}
// sortedKeys is the keys of a map in order, which makes the order of writes
// — and so of the positions handed out — deterministic.
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
slices.Sort(out)
return out
}
var _ storage.KeyStore = (*Keys)(nil)
+195
View File
@@ -0,0 +1,195 @@
package pocketbase
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Media is the PocketBase implementation of storage.MediaStore: the
// metadata of media. The content is the blob store's.
type Media struct{ *base }
// CreateMedia implements storage.MediaStore.
func (s *Media) CreateMedia(ctx context.Context, m storage.Media) error {
return s.write(ctx, func(t *tx) error {
taken, err := t.exists("media_repository", `media_origin = ? AND media_id = ?`, m.Origin, m.MediaID)
if err != nil {
return fmt.Errorf("store media %s/%s: %w", m.Origin, m.MediaID, err)
}
if taken {
return storage.ErrMediaExists
}
t.put("media_repository", mediaRow(m))
return nil
})
}
func mediaRow(m storage.Media) rec {
return rec{
"media_origin": m.Origin, "media_id": m.MediaID, "content_type": m.ContentType,
"upload_name": m.UploadName, "file_size": m.Size, "sha256": m.SHA256,
"user_id": m.UserID, "created_at_ms": m.CreatedAtMS, "pending": m.Pending,
"expires_at_ms": m.ExpiresAtMS,
}
}
// Media implements storage.MediaStore.
func (s *Media) Media(ctx context.Context, origin, mediaID string) (storage.Media, error) {
var m storage.Media
err := s.db.QueryRowContext(ctx, `
SELECT media_origin, media_id, content_type, upload_name, file_size, sha256,
user_id, created_at_ms, pending, expires_at_ms
FROM media_repository WHERE media_origin = ? AND media_id = ?`, origin, mediaID).Scan(
&m.Origin, &m.MediaID, &m.ContentType, &m.UploadName, &m.Size, &m.SHA256,
&m.UserID, &m.CreatedAtMS, &m.Pending, &m.ExpiresAtMS)
switch {
case err == nil:
return m, nil
case errors.Is(err, sql.ErrNoRows):
return storage.Media{}, storage.ErrNoSuchMedia
default:
return storage.Media{}, fmt.Errorf("read media %s/%s: %w", origin, mediaID, err)
}
}
// CompleteUpload implements storage.MediaStore. Only pending media is
// completed, so that two uploads for the same ID cannot both win.
func (s *Media) CompleteUpload(ctx context.Context, origin, mediaID string, c storage.MediaContent) error {
var completed bool
err := s.write(ctx, func(t *tx) error {
n, err := t.setWhere("media_repository", rec{
"content_type": c.ContentType, "upload_name": c.UploadName, "file_size": c.Size,
"sha256": c.SHA256, "pending": false, "expires_at_ms": int64(0),
}, `media_origin = ? AND media_id = ? AND pending = ?`, origin, mediaID, true)
completed = n == 1
return err
})
if err != nil {
return fmt.Errorf("complete upload %s/%s: %w", origin, mediaID, err)
}
if completed {
return nil
}
// Nothing pending matched: either there is no such media, or its
// content is already there.
if _, err := s.Media(ctx, origin, mediaID); err != nil {
return err
}
return storage.ErrMediaUploaded
}
// PendingUploads implements storage.MediaStore.
func (s *Media) PendingUploads(ctx context.Context, userID string, nowMS int64) (int, error) {
var n int
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM media_repository
WHERE pending = ? AND user_id = ? AND expires_at_ms > ?`, true, userID, nowMS).Scan(&n); err != nil {
return 0, fmt.Errorf("count the pending uploads of %s: %w", userID, err)
}
return n, nil
}
// DeleteExpiredUploads implements storage.MediaStore.
func (s *Media) DeleteExpiredUploads(ctx context.Context, nowMS int64) (int, error) {
var n int
err := s.write(ctx, func(t *tx) error {
var err error
n, err = t.delWhere("media_repository", `pending = ? AND expires_at_ms <= ?`, true, nowMS)
return err
})
if err != nil {
return 0, fmt.Errorf("delete expired uploads: %w", err)
}
return n, nil
}
// Thumbnail implements storage.MediaStore.
func (s *Media) Thumbnail(ctx context.Context, origin, mediaID string, width, height int, method string) (storage.Thumbnail, error) {
var t storage.Thumbnail
err := s.db.QueryRowContext(ctx, `
SELECT media_origin, media_id, width, height, method, content_type,
file_size, sha256, created_at_ms
FROM media_thumbnails
WHERE media_origin = ? AND media_id = ? AND width = ? AND height = ? AND method = ?`,
origin, mediaID, width, height, method).Scan(
&t.Origin, &t.MediaID, &t.Width, &t.Height, &t.Method, &t.ContentType,
&t.Size, &t.SHA256, &t.CreatedAtMS)
switch {
case err == nil:
return t, nil
case errors.Is(err, sql.ErrNoRows):
return storage.Thumbnail{}, storage.ErrNoSuchMedia
default:
return storage.Thumbnail{}, fmt.Errorf("read a thumbnail of %s/%s: %w", origin, mediaID, err)
}
}
// StoreThumbnail implements storage.MediaStore.
func (s *Media) StoreThumbnail(ctx context.Context, t storage.Thumbnail) error {
err := s.write(ctx, func(w *tx) error {
w.put("media_thumbnails", rec{
"media_origin": t.Origin, "media_id": t.MediaID, "width": t.Width, "height": t.Height,
"method": t.Method, "content_type": t.ContentType, "file_size": t.Size,
"sha256": t.SHA256, "created_at_ms": t.CreatedAtMS,
})
return nil
})
if err != nil {
return fmt.Errorf("store a thumbnail of %s/%s: %w", t.Origin, t.MediaID, err)
}
return nil
}
// MediaUsage implements storage.MediaStore. It counts every upload at its
// own size, even when two of them are the same bytes.
func (s *Media) MediaUsage(ctx context.Context, origin, userID string) (storage.MediaUsage, error) {
var u storage.MediaUsage
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*), COALESCE(SUM(file_size), 0) FROM media_repository
WHERE user_id = ? AND media_origin = ? AND pending = ?`, userID, origin, false).
Scan(&u.Count, &u.Bytes); err != nil {
return storage.MediaUsage{}, fmt.Errorf("total the uploads of %s: %w", userID, err)
}
return u, nil
}
// MediaQuota implements storage.MediaStore.
func (s *Media) MediaQuota(ctx context.Context, userID string) (int64, bool, error) {
var maxBytes int64
err := s.db.QueryRowContext(ctx, `SELECT max_bytes FROM media_quotas WHERE user_id = ?`, userID).Scan(&maxBytes)
switch {
case errors.Is(err, sql.ErrNoRows):
return 0, false, nil
case err != nil:
return 0, false, fmt.Errorf("read the media quota of %s: %w", userID, err)
}
return maxBytes, true, nil
}
// SetMediaQuota implements storage.MediaStore.
func (s *Media) SetMediaQuota(ctx context.Context, userID string, maxBytes int64) error {
err := s.write(ctx, func(t *tx) error {
t.put("media_quotas", rec{"user_id": userID, "max_bytes": maxBytes})
return nil
})
if err != nil {
return fmt.Errorf("set the media quota of %s: %w", userID, err)
}
return nil
}
// DeleteMediaQuota implements storage.MediaStore.
func (s *Media) DeleteMediaQuota(ctx context.Context, userID string) error {
err := s.write(ctx, func(t *tx) error {
_, err := t.delKey("media_quotas", userID)
return err
})
if err != nil {
return fmt.Errorf("delete the media quota of %s: %w", userID, err)
}
return nil
}
var _ storage.MediaStore = (*Media)(nil)
@@ -0,0 +1,64 @@
// Package pbtest runs a real PocketBase server in-process for tests, so the
// pocketbase driver is tested against the same REST API it uses in
// production, with no container to start.
package pbtest
import (
"net/http/httptest"
"testing"
"github.com/pocketbase/pocketbase/apis"
"github.com/pocketbase/pocketbase/core"
_ "github.com/pocketbase/pocketbase/migrations" // PocketBase's system migrations; without them its bootstrap has no tables
)
// Superuser is the account the server is started with.
const (
SuperuserEmail = "warp-tests@example.com"
SuperuserPassword = "warp-tests-password"
)
// Server is a running PocketBase.
type Server struct {
// URL is the server's address, for database.connection_string.
URL string
// DataDir is its pb_data, where a test can look for what it stored.
DataDir string
}
// Start runs a fresh PocketBase with an empty data directory. It is stopped
// when the test ends.
func Start(tb testing.TB) Server {
tb.Helper()
dataDir := tb.TempDir()
app := core.NewBaseApp(core.BaseAppConfig{DataDir: dataDir})
if err := app.Bootstrap(); err != nil {
tb.Fatalf("pocketbase: bootstrap: %v", err)
}
tb.Cleanup(func() { _ = app.ResetBootstrapState() })
superusers, err := app.FindCollectionByNameOrId(core.CollectionNameSuperusers)
if err != nil {
tb.Fatalf("pocketbase: %v", err)
}
su := core.NewRecord(superusers)
su.SetEmail(SuperuserEmail)
su.SetPassword(SuperuserPassword)
if err := app.Save(su); err != nil {
tb.Fatalf("pocketbase: create the superuser: %v", err)
}
router, err := apis.NewRouter(app)
if err != nil {
tb.Fatalf("pocketbase: router: %v", err)
}
mux, err := router.BuildMux()
if err != nil {
tb.Fatalf("pocketbase: router: %v", err)
}
server := httptest.NewServer(mux)
tb.Cleanup(server.Close)
return Server{URL: server.URL, DataDir: dataDir}
}
+297 -237
View File
@@ -1,278 +1,338 @@
// Package pocketbase is the PocketBase storage driver: Warp on an embedded
// PocketBase app, for "appliance" deployments that want one binary and one
// data directory.
// Package pocketbase is the PocketBase storage driver: Warp's data in a
// standalone PocketBase server, as collections, reached only through
// PocketBase's REST API — the way the postgres driver reaches PostgreSQL
// over its wire protocol.
//
// PLAN.md section 4 set out the shape, and open question 4 asked how far
// to take it. The answer here is the cheaper of the two: the event DAG and
// everything else Warp stores are a poor fit for PocketBase's record
// model, so the Warp tables are the sqlite driver's own, migrated by Warp
// and written through the same shared store, on PocketBase's underlying
// SQLite connection rather than through its collection API. What PocketBase
// adds is everything around that: its data directory and its system
// tables, which a PocketBase admin or a later Warp feature can build on, and
// its filesystem abstraction, which puts media content on local disk or in
// an S3 bucket.
//
// The result is at parity with the sqlite driver by construction — it is
// the same SQL on the same engine — and the storage tests run against it
// to keep it that way.
// Each of the SQL drivers' tables is a collection of the same name and
// columns (see schema.go), so a PocketBase admin sees Warp's rooms, events
// and accounts as records. The stores read them through the superuser SQL
// endpoint with the same queries the SQL drivers use (sqldriver.go), and
// write them as record operations in batch requests (batch.go), which
// PocketBase runs as one transaction each. Media content goes into a file
// field, which is PocketBase's own storage: its pb_data/storage, or an S3
// bucket.
package pocketbase
import (
"context"
"crypto/sha256"
"encoding/hex"
"database/sql"
"errors"
"fmt"
"io"
"log/slog"
"os"
"path/filepath"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
_ "github.com/pocketbase/pocketbase/migrations" // PocketBase's system migrations; without them its bootstrap has no tables
"github.com/pocketbase/pocketbase/tools/filesystem"
"sync"
"time"
"github.com/dariuszjaniak/warp/server/internal/storage"
"github.com/dariuszjaniak/warp/server/internal/storage/sqlite"
"github.com/dariuszjaniak/warp/server/internal/storage/sqlstore"
)
func init() {
storage.Register(storage.DriverPocketBase, Open)
}
// mediaPrefix is where media content lives in PocketBase's filesystem,
// apart from the files of PocketBase's own collections.
const mediaPrefix = "warp_media"
// Database is the PocketBase implementation of storage.Database.
type Database struct {
*sqlstore.Store
app *core.BaseApp
blobs *blobStore
*base
log *slog.Logger
s3 storage.S3Config
settings *Settings
users *Users
rooms *Rooms
sync *Sync
fed *Federation
media *Media
keys *Keys
push *Push
admin *Admin
apps *AppServices
delayed *DelayedEvents
openID *OpenIDTokens
blobs *blobStore
}
// Open bootstraps a PocketBase app in the directory named by
// cfg.ConnectionString — PocketBase's pb_data — and opens the Warp store on
// its database.
func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Database, error) {
dir := cfg.ConnectionString
if dir == "" {
return nil, errors.New("pocketbase: connection_string must name the data directory")
}
if err := ctx.Err(); err != nil {
return nil, err
}
app := core.NewBaseApp(core.BaseAppConfig{
DataDir: dir,
DataMaxOpenConns: cfg.MaxOpenConns,
DataMaxIdleConns: cfg.MaxIdleConns,
})
if err := app.Bootstrap(); err != nil {
_ = app.ResetBootstrapState()
return nil, fmt.Errorf("pocketbase: bootstrap %s: %w", dir, err)
}
if cfg.S3.Enabled {
// Applied in memory on every start rather than saved: the Warp
// config is the authority, and PocketBase's own settings would
// otherwise be a second place for the bucket to be set, and to
// disagree.
app.Settings().S3 = core.S3Config{
Enabled: true,
Bucket: cfg.S3.Bucket,
Region: cfg.S3.Region,
Endpoint: cfg.S3.Endpoint,
AccessKey: cfg.S3.AccessKey,
Secret: cfg.S3.Secret,
ForcePathStyle: cfg.S3.ForcePathStyle,
}
}
// The writer pool: one connection, the way the sqlite driver runs, and
// the way PocketBase itself writes. Warp's store reads in the same
// transactions it writes in, so it is given the one pool for both.
writer, ok := app.NonconcurrentDB().(*dbx.DB)
if !ok {
_ = app.ResetBootstrapState()
return nil, errors.New("pocketbase: the app's database is not a *dbx.DB")
}
opts := sqlite.StoreOptions(writer.DB(), log)
opts.DriverName = storage.DriverPocketBase
store, err := sqlstore.New(opts)
if err != nil {
_ = app.ResetBootstrapState()
return nil, err
}
return &Database{
Store: store,
app: app,
blobs: &blobStore{app: app, tmpDir: filepath.Join(app.DataDir(), ".warp_tmp")},
}, nil
}
// App is the PocketBase app, for code that wants PocketBase's own
// features: its collections, its auth, its realtime.
func (d *Database) App() core.App { return d.app }
// Blobs implements storage.BlobProvider: media content goes into
// PocketBase's filesystem.
func (d *Database) Blobs() storage.BlobStore { return d.blobs }
// Close implements storage.Database. The app owns the connection pools.
func (d *Database) Close() error { return d.app.ResetBootstrapState() }
// blobStore is storage.BlobStore over PocketBase's filesystem: the local
// pb_data/storage directory, or the S3 bucket the settings name.
// Open connects to the PocketBase server at cfg.ConnectionString and signs
// in as the configured superuser.
//
// The filesystem can only be written a whole file at a time, and a blob's
// name is its hash, which is not known until the last byte has been read.
// So an upload is spooled to a temporary file first, hashed on the way, and
// handed over once it is complete — which is also what keeps a half-written
// object from ever appearing under a hash.
type blobStore struct {
app *core.BaseApp
tmpDir string
}
func blobKey(sum string) (string, bool) {
if len(sum) != sha256.Size*2 {
return "", false
}
for _, c := range sum {
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return "", false
}
}
return mediaPrefix + "/" + sum[0:2] + "/" + sum[2:4] + "/" + sum, true
}
// fs opens the filesystem for one operation. It is cheap for both kinds,
// and an instance per operation is what lets each carry its own context.
func (b *blobStore) fs(ctx context.Context) (*filesystem.System, error) {
fsys, err := b.app.NewFilesystem()
if err != nil {
return nil, fmt.Errorf("open the media filesystem: %w", err)
}
fsys.SetContext(ctx)
return fsys, nil
}
// Put implements storage.BlobStore.
func (b *blobStore) Put(ctx context.Context, r io.Reader, limit int64) (storage.BlobInfo, error) {
if err := os.MkdirAll(b.tmpDir, 0o750); err != nil {
return storage.BlobInfo{}, fmt.Errorf("create the media spool directory: %w", err)
}
tmp, err := os.CreateTemp(b.tmpDir, "upload-*")
if err != nil {
return storage.BlobInfo{}, fmt.Errorf("create a temporary file: %w", err)
}
defer func() {
_ = tmp.Close()
_ = os.Remove(tmp.Name())
}()
hash := sha256.New()
n, err := io.Copy(io.MultiWriter(tmp, hash), &ctxReader{ctx: ctx, r: io.LimitReader(r, limit+1)})
if err != nil {
return storage.BlobInfo{}, fmt.Errorf("store content: %w", err)
}
if n > limit {
return storage.BlobInfo{}, storage.ErrBlobTooLarge
}
if err := tmp.Close(); err != nil {
return storage.BlobInfo{}, fmt.Errorf("store content: %w", err)
}
info := storage.BlobInfo{SHA256: hex.EncodeToString(hash.Sum(nil)), Size: n}
key, _ := blobKey(info.SHA256)
fsys, err := b.fs(ctx)
if err != nil {
return storage.BlobInfo{}, err
}
defer func() { _ = fsys.Close() }()
if exists, err := fsys.Exists(key); err == nil && exists {
// The same content is already stored.
return info, nil
}
file, err := filesystem.NewFileFromPath(tmp.Name())
if err != nil {
return storage.BlobInfo{}, fmt.Errorf("store content: %w", err)
}
if err := fsys.UploadFile(file, key); err != nil {
return storage.BlobInfo{}, fmt.Errorf("store content: %w", err)
}
return info, nil
}
// Open implements storage.BlobStore.
func (b *blobStore) Open(ctx context.Context, sum string) (io.ReadSeekCloser, error) {
key, ok := blobKey(sum)
if !ok {
return nil, storage.ErrNoSuchBlob
}
fsys, err := b.fs(ctx)
// It configures the server for Warp before anything else — batch requests
// on, and the S3 bucket when one is configured — so a server that cannot
// be reached or refuses the account fails here, at startup.
func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Database, error) {
c, err := newClient(cfg.ConnectionString, cfg.SuperuserEmail, cfg.SuperuserPassword)
if err != nil {
return nil, err
}
reader, err := fsys.GetReader(key)
if err != nil {
_ = fsys.Close()
if errors.Is(err, filesystem.ErrNotFound) {
return nil, storage.ErrNoSuchBlob
}
return nil, fmt.Errorf("open media content: %w", err)
lock := &sync.RWMutex{}
db := sql.OpenDB(&sqlConnector{c: c, lock: lock})
// The connections hold nothing, so the pool only bounds how many reads
// are in flight at once.
maxConns := cfg.MaxOpenConns
if maxConns <= 0 {
maxConns = 16
}
return &blobReader{ReadSeekCloser: reader, fsys: fsys}, nil
db.SetMaxOpenConns(maxConns)
b := &base{c: c, db: db, lock: lock}
d := &Database{
base: b,
log: log,
s3: cfg.S3,
settings: &Settings{base: b},
users: &Users{base: b},
rooms: &Rooms{base: b},
sync: &Sync{base: b},
fed: &Federation{base: b},
media: &Media{base: b},
keys: &Keys{base: b},
push: &Push{base: b},
admin: &Admin{base: b},
apps: &AppServices{base: b},
delayed: &DelayedEvents{base: b},
openID: &OpenIDTokens{base: b},
blobs: &blobStore{base: b},
}
if err := d.configure(ctx); err != nil {
_ = db.Close()
return nil, err
}
return d, nil
}
// Delete implements storage.BlobStore.
func (b *blobStore) Delete(ctx context.Context, sum string) error {
key, ok := blobKey(sum)
if !ok {
return nil
// configure applies the PocketBase settings Warp depends on.
//
// The S3 settings are written from Warp's configuration on every start
// rather than left to the PocketBase admin UI: Warp's config is the
// authority, and a second place for the bucket to be set would be a second
// place for it to disagree. A deployment without S3 in Warp's config leaves
// PocketBase's own setting as it is.
func (d *Database) configure(ctx context.Context) error {
settings := map[string]any{
"batch": map[string]any{
"enabled": true,
"maxRequests": maxBatch,
"timeout": 300,
"maxBodySize": 0,
},
}
fsys, err := b.fs(ctx)
if err != nil {
return err
if d.s3.Enabled {
settings["s3"] = map[string]any{
"enabled": true,
"bucket": d.s3.Bucket,
"region": d.s3.Region,
"endpoint": d.s3.Endpoint,
"accessKey": d.s3.AccessKey,
"secret": d.s3.Secret,
"forcePathStyle": d.s3.ForcePathStyle,
}
}
defer func() { _ = fsys.Close() }()
if err := fsys.Delete(key); err != nil && !errors.Is(err, filesystem.ErrNotFound) {
return fmt.Errorf("delete media content: %w", err)
if err := d.c.patchSettings(ctx, settings); err != nil {
return fmt.Errorf("pocketbase: configure the server: %w", err)
}
return nil
}
// blobReader closes the filesystem with the reader it opened.
type blobReader struct {
io.ReadSeekCloser
fsys *filesystem.System
}
// DB exposes the read-only SQL pool, for tests and diagnostics: queries run
// on the PocketBase server through its SQL endpoint, and anything else is
// refused.
func (d *Database) DB() *sql.DB { return d.db }
func (r *blobReader) Close() error {
return errors.Join(r.ReadSeekCloser.Close(), r.fsys.Close())
}
// Driver implements storage.Database.
func (d *Database) Driver() string { return storage.DriverPocketBase }
// ctxReader stops a copy when its context ends.
type ctxReader struct {
ctx context.Context
r io.Reader
}
// Ping implements storage.Database.
func (d *Database) Ping(ctx context.Context) error { return d.db.PingContext(ctx) }
func (c *ctxReader) Read(p []byte) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
// Close implements storage.Database.
func (d *Database) Close() error { return d.db.Close() }
// Settings implements storage.Database.
func (d *Database) Settings() storage.SettingsStore { return d.settings }
// Users implements storage.Database.
func (d *Database) Users() storage.UserStore { return d.users }
// Rooms implements storage.Database.
func (d *Database) Rooms() storage.RoomStore { return d.rooms }
// Sync implements storage.Database.
func (d *Database) Sync() storage.SyncStore { return d.sync }
// Federation implements storage.Database.
func (d *Database) Federation() storage.FederationStore { return d.fed }
// Media implements storage.Database.
func (d *Database) Media() storage.MediaStore { return d.media }
// Keys implements storage.Database.
func (d *Database) Keys() storage.KeyStore { return d.keys }
// Push implements storage.Database.
func (d *Database) Push() storage.PushStore { return d.push }
// Admin implements storage.Database.
func (d *Database) Admin() storage.AdminStore { return d.admin }
// AppServices implements storage.Database.
func (d *Database) AppServices() storage.AppServiceStore { return d.apps }
// DelayedEvents implements storage.Database.
func (d *Database) DelayedEvents() storage.DelayedEventStore { return d.delayed }
// OpenIDTokens implements storage.Database.
func (d *Database) OpenIDTokens() storage.OpenIDTokenStore { return d.openID }
// Blobs implements storage.BlobProvider: media content goes into
// PocketBase's storage.
func (d *Database) Blobs() storage.BlobStore { return d.blobs }
// schemaMetaKey is the warp_meta row holding the schema version applied.
const schemaMetaKey = "schema"
// Migrate implements storage.Database.
//
// There are no numbered migrations: the schema is declared in schema.go
// and imported as it is, which PocketBase applies as a diff — a new
// collection is created, a new field added, an unchanged one left alone.
// The version stamped afterwards is a hash of the declaration, so any
// change to it is a pending migration.
func (d *Database) Migrate(ctx context.Context) error {
pending, err := d.PendingMigrations(ctx)
if err != nil {
return err
}
return c.r.Read(p)
if len(pending) == 0 {
d.log.Debug("database schema is up to date", "version", schemaVersion)
return nil
}
collections := make([]map[string]any, len(schema))
for i := range schema {
collections[i] = schema[i].collectionJSON()
}
if err := d.c.importCollections(ctx, collections); err != nil {
return fmt.Errorf("pocketbase: import the schema: %w", err)
}
err = d.write(ctx, func(t *tx) error {
for table, names := range sequenceRows {
for _, name := range names {
found, err := t.exists(table, `name = ?`, name)
if err != nil {
return err
}
if !found {
t.put(table, rec{"name": name, "value": int64(0)})
}
}
}
// Arguments a crashed query left behind.
if _, err := t.delWhere(paramsCollection, `1 = 1`); err != nil {
return err
}
t.put("warp_meta", rec{"meta_key": schemaMetaKey, "meta_value": schemaVersion})
return nil
})
if err != nil {
return fmt.Errorf("pocketbase: initialise the schema: %w", err)
}
d.log.Info("applied the database schema", "version", schemaVersion)
return nil
}
// PendingMigrations implements storage.Database.
func (d *Database) PendingMigrations(ctx context.Context) ([]string, error) {
pending := []string{"pocketbase_schema_" + schemaVersion}
var n int
if err := d.db.QueryRowContext(ctx,
`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'warp_meta'`).Scan(&n); err != nil {
return nil, fmt.Errorf("pocketbase: read the schema version: %w", err)
}
if n == 0 {
return pending, nil
}
var version string
err := d.db.QueryRowContext(ctx,
`SELECT meta_value FROM warp_meta WHERE meta_key = ?`, schemaMetaKey).Scan(&version)
switch {
case errors.Is(err, sql.ErrNoRows):
return pending, nil
case err != nil:
return nil, fmt.Errorf("pocketbase: read the schema version: %w", err)
case version != schemaVersion:
return pending, nil
}
return nil, nil
}
// Settings is the server_settings store.
type Settings struct{ *base }
// Get implements storage.SettingsStore.
func (s *Settings) Get(ctx context.Context, key string) (string, bool, error) {
var value string
err := s.db.QueryRowContext(ctx,
`SELECT setting_value FROM server_settings WHERE setting_key = ?`, key).Scan(&value)
switch {
case errors.Is(err, sql.ErrNoRows):
return "", false, nil
case err != nil:
return "", false, fmt.Errorf("get setting %q: %w", key, err)
}
return value, true, nil
}
// GetAll implements storage.SettingsStore.
func (s *Settings) GetAll(ctx context.Context) (map[string]string, error) {
rows, err := s.db.QueryContext(ctx, `SELECT setting_key, setting_value FROM server_settings`)
if err != nil {
return nil, fmt.Errorf("list settings: %w", err)
}
defer func() { _ = rows.Close() }()
out := map[string]string{}
for rows.Next() {
var k, v string
if err := rows.Scan(&k, &v); err != nil {
return nil, fmt.Errorf("scan setting: %w", err)
}
out[k] = v
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list settings: %w", err)
}
return out, nil
}
// Set implements storage.SettingsStore.
func (s *Settings) Set(ctx context.Context, key, value string) error {
err := s.write(ctx, func(t *tx) error {
t.put("server_settings", rec{
"setting_key": key,
"setting_value": value,
"updated_at": time.Now().UTC().Format(time.RFC3339Nano),
})
return nil
})
if err != nil {
return fmt.Errorf("set setting %q: %w", key, err)
}
return nil
}
// Delete implements storage.SettingsStore.
func (s *Settings) Delete(ctx context.Context, key string) error {
err := s.write(ctx, func(t *tx) error {
_, err := t.delKey("server_settings", key)
return err
})
if err != nil {
return fmt.Errorf("delete setting %q: %w", key, err)
}
return nil
}
var (
_ storage.Database = (*Database)(nil)
_ storage.BlobProvider = (*Database)(nil)
_ storage.BlobStore = (*blobStore)(nil)
_ storage.Database = (*Database)(nil)
_ storage.BlobProvider = (*Database)(nil)
_ storage.SettingsStore = (*Settings)(nil)
)
+328
View File
@@ -0,0 +1,328 @@
package pocketbase
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Push is the PocketBase implementation of storage.PushStore.
type Push struct{ *base }
const pusherColumns = `localpart, app_id, pushkey, device_id, kind, app_display_name,
device_display_name, profile_tag, lang, data_json, pushkey_ts_ms`
// Pushers implements storage.PushStore.
func (s *Push) Pushers(ctx context.Context, localpart string) ([]storage.Pusher, error) {
rows, err := s.db.QueryContext(ctx, `SELECT `+pusherColumns+` FROM push_pushers
WHERE localpart = ? ORDER BY app_id, pushkey`, localpart)
if err != nil {
return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err)
}
defer func() { _ = rows.Close() }()
var out []storage.Pusher
for rows.Next() {
var p storage.Pusher
var data string
if err := rows.Scan(&p.Localpart, &p.AppID, &p.PushKey, &p.DeviceID, &p.Kind, &p.AppDisplayName,
&p.DeviceDisplayName, &p.ProfileTag, &p.Lang, &data, &p.PushKeyTSMS); err != nil {
return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err)
}
p.Data = []byte(data)
out = append(out, p)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err)
}
return out, nil
}
// SetPusher implements storage.PushStore.
func (s *Push) SetPusher(ctx context.Context, p storage.Pusher, appendOthers bool) error {
return s.write(ctx, func(t *tx) error {
if !appendOthers {
if _, err := t.delWhere("push_pushers", `app_id = ? AND pushkey = ? AND localpart <> ?`,
p.AppID, p.PushKey, p.Localpart); err != nil {
return fmt.Errorf("remove other users' pushers for the key: %w", err)
}
}
t.put("push_pushers", rec{
"localpart": p.Localpart, "app_id": p.AppID, "pushkey": p.PushKey, "device_id": p.DeviceID,
"kind": p.Kind, "app_display_name": p.AppDisplayName, "device_display_name": p.DeviceDisplayName,
"profile_tag": p.ProfileTag, "lang": p.Lang, "data_json": string(p.Data), "pushkey_ts_ms": p.PushKeyTSMS,
})
return nil
})
}
// DeletePusher implements storage.PushStore.
func (s *Push) DeletePusher(ctx context.Context, localpart, appID, pushKey string) error {
err := s.write(ctx, func(t *tx) error {
_, err := t.delKey("push_pushers", localpart, appID, pushKey)
return err
})
if err != nil {
return fmt.Errorf("delete a pusher of %q: %w", localpart, err)
}
return nil
}
// DeletePushersForDevices implements storage.PushStore.
func (s *Push) DeletePushersForDevices(ctx context.Context, localpart string, deviceIDs []string) error {
where, args := `localpart = ?`, []any{localpart}
if len(deviceIDs) > 0 {
where += ` AND device_id IN ` + inList
args = append(args, jsonList(deviceIDs))
}
err := s.write(ctx, func(t *tx) error {
_, err := t.delWhere("push_pushers", where, args...)
return err
})
if err != nil {
return fmt.Errorf("delete the pushers of %q's devices: %w", localpart, err)
}
return nil
}
const notificationColumns = `notification_id, user_id, room_id, event_id, event_pos, thread_id, ts_ms, highlight, actions, is_read`
// roomCounts is one room's counts for one user, as a write leaves them.
type roomCounts struct{ notifications, highlights int64 }
// AddNotifications implements storage.PushStore. Every notification takes a
// position of its own, which is both its ID and the stamp on its room's
// counts.
func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotification) (int64, error) {
if len(ns) == 0 {
return s.currentSequence(ctx, pushSequences, seqNotification)
}
var last int64
err := s.write(ctx, func(t *tx) error {
counts := map[[2]string]roomCounts{}
for _, n := range ns {
pos, err := t.next(pushSequences, seqNotification)
if err != nil {
return err
}
t.put("push_notifications", rec{
"notification_id": pos, "user_id": n.UserID, "room_id": n.RoomID, "event_id": n.EventID,
"event_pos": n.EventPos, "thread_id": n.ThreadID, "ts_ms": n.TimestampMS,
"highlight": n.Highlight, "actions": string(n.Actions), "is_read": false,
})
key := [2]string{n.UserID, n.RoomID}
c, ok := counts[key]
if !ok {
err := t.queryRow(`SELECT notification_count, highlight_count FROM push_room_counts
WHERE user_id = ? AND room_id = ?`, n.UserID, n.RoomID).Scan(&c.notifications, &c.highlights)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("count a notification for %q: %w", n.UserID, err)
}
}
c.notifications++
if n.Highlight {
c.highlights++
}
counts[key] = c
t.put("push_room_counts", rec{
"user_id": n.UserID, "room_id": n.RoomID,
"notification_count": c.notifications, "highlight_count": c.highlights, "stream_pos": pos,
})
last = pos
}
return nil
})
return last, err
}
// MarkNotificationsRead implements storage.PushStore.
//
// The counts are recounted from the rows that stay unread rather than
// decremented, so whatever got them out of step is put right by the next
// receipt. The room's counts stay the room's total across every thread.
func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID, thread string, upTo int64) (int64, bool, error) {
var (
pos int64
moved bool
)
err := s.write(ctx, func(t *tx) error {
marked := `event_pos <= ?`
markedArgs := []any{upTo}
if thread != storage.AllThreads {
marked += ` AND thread_id = ?`
markedArgs = append(markedArgs, thread)
}
n, err := t.setWhere("push_notifications", rec{"is_read": true},
`user_id = ? AND room_id = ? AND is_read = ? AND `+marked,
append([]any{userID, roomID, false}, markedArgs...)...)
if err != nil {
return fmt.Errorf("mark %q's notifications read: %w", userID, err)
}
if n == 0 {
pos, err = t.current(pushSequences, seqNotification)
return err
}
var notifications, highlights int64
if err := t.queryRow(`SELECT COUNT(*), CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT)
FROM push_notifications WHERE user_id = ? AND room_id = ? AND is_read = ? AND NOT (`+marked+`)`,
append([]any{userID, roomID, false}, markedArgs...)...).
Scan(&notifications, &highlights); err != nil {
return fmt.Errorf("count %q's unread notifications: %w", userID, err)
}
if pos, err = t.next(pushSequences, seqNotification); err != nil {
return err
}
t.put("push_room_counts", rec{
"user_id": userID, "room_id": roomID,
"notification_count": notifications, "highlight_count": highlights, "stream_pos": pos,
})
moved = true
return nil
})
return pos, moved, err
}
// NotificationCounts implements storage.PushStore.
func (s *Push) NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]storage.NotificationCounts, error) {
out := make(map[string]storage.NotificationCounts, len(roomIDs))
if len(roomIDs) == 0 {
return out, nil
}
for _, id := range roomIDs {
out[id] = storage.NotificationCounts{}
}
rows, err := s.db.QueryContext(ctx, `SELECT room_id, notification_count, highlight_count FROM push_room_counts
WHERE user_id = ? AND room_id IN `+inList, userID, jsonList(roomIDs))
if err != nil {
return nil, fmt.Errorf("read %q's notification counts: %w", userID, err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var roomID string
var c storage.NotificationCounts
if err := rows.Scan(&roomID, &c.Notifications, &c.Highlights); err != nil {
return nil, fmt.Errorf("read %q's notification counts: %w", userID, err)
}
out[roomID] = c
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read %q's notification counts: %w", userID, err)
}
return out, nil
}
// ThreadNotificationCounts implements storage.PushStore. These are counted
// from the unread rows rather than kept like the room totals.
func (s *Push) ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error) {
out := make(map[string]map[string]storage.NotificationCounts, len(roomIDs))
if len(roomIDs) == 0 {
return out, nil
}
rows, err := s.db.QueryContext(ctx, `SELECT room_id, thread_id, COUNT(*),
CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT)
FROM push_notifications
WHERE user_id = ? AND is_read = ? AND room_id IN `+inList+`
GROUP BY room_id, thread_id`, userID, false, jsonList(roomIDs))
if err != nil {
return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var roomID, thread string
var notifications, highlights int64
if err := rows.Scan(&roomID, &thread, &notifications, &highlights); err != nil {
return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
}
if out[roomID] == nil {
out[roomID] = map[string]storage.NotificationCounts{}
}
out[roomID][thread] = storage.NotificationCounts{
Notifications: int(notifications),
Highlights: int(highlights),
}
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
}
return out, nil
}
// NotificationCountChanges implements storage.PushStore.
func (s *Push) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) {
out, err := s.queryStrings(ctx, `SELECT room_id FROM push_room_counts
WHERE user_id = ? AND stream_pos > ? AND stream_pos <= ? ORDER BY room_id`, userID, from, to)
if err != nil {
return nil, fmt.Errorf("read %q's notification count changes: %w", userID, err)
}
return out, nil
}
// UnreadNotifications implements storage.PushStore.
func (s *Push) UnreadNotifications(ctx context.Context, userID string) (int, error) {
var n int64
if err := s.db.QueryRowContext(ctx, `SELECT CAST(COALESCE(SUM(notification_count), 0) AS BIGINT)
FROM push_room_counts WHERE user_id = ?`, userID).Scan(&n); err != nil {
return 0, fmt.Errorf("count %q's unread notifications: %w", userID, err)
}
return int(n), nil
}
// Notifications implements storage.PushStore.
func (s *Push) Notifications(ctx context.Context, userID string, before int64, limit int, onlyHighlight bool) ([]storage.Notification, error) {
query := `SELECT ` + notificationColumns + ` FROM push_notifications WHERE user_id = ?`
args := []any{userID}
if before > 0 {
query += ` AND notification_id < ?`
args = append(args, before)
}
if onlyHighlight {
query += ` AND highlight = ?`
args = append(args, true)
}
query += ` ORDER BY notification_id DESC LIMIT ?`
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read %q's notifications: %w", userID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.Notification
for rows.Next() {
var n storage.Notification
var actions string
if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.ThreadID, &n.TimestampMS,
&n.Highlight, &actions, &n.Read); err != nil {
return nil, fmt.Errorf("read %q's notifications: %w", userID, err)
}
n.Actions = []byte(actions)
out = append(out, n)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read %q's notifications: %w", userID, err)
}
return out, nil
}
// PruneNotifications implements storage.PushStore.
func (s *Push) PruneNotifications(ctx context.Context, beforeMS int64) (int64, error) {
var n int
err := s.write(ctx, func(t *tx) error {
var err error
n, err = t.delWhere("push_notifications", `is_read = ? AND ts_ms < ?`, true, beforeMS)
return err
})
if err != nil {
return 0, fmt.Errorf("prune read notifications: %w", err)
}
return int64(n), nil
}
var _ storage.PushStore = (*Push)(nil)
@@ -0,0 +1,369 @@
package pocketbase
import (
"context"
"fmt"
"strings"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// The relation and search half of the room store.
//
// Relations are the SQL store's queries. Search has no full-text engine to
// lean on — PocketBase does not expose FTS5 — so roomserver_event_search
// holds each event's searchable text as its words, lower-cased and padded
// with spaces, and a search matches every word of the term as the start of
// a word: what the SQLite driver's prefix query matches, short of folding
// diacritics. Every match ranks the same, so a ranked search comes back
// newest first.
// searchText is how an event's text is stored for searching: its words,
// lower-cased, each with a space before it and the whole with one after.
func searchText(text string) string {
words := storage.SearchWords(text)
if len(words) == 0 {
return ""
}
return " " + strings.Join(words, " ") + " "
}
// indexEvent adds an event that has just joined a timeline to the search
// index, if it has anything to find it by.
func (w *roomWrite) indexEvent(e storage.StoredEvent, ordering int64) {
body := searchText(e.SearchText)
if e.SearchKey == "" || body == "" {
return
}
w.put("roomserver_event_search", rec{
"stream_ordering": ordering,
"room_id": e.RoomID,
"event_id": e.EventID,
"search_key": e.SearchKey,
"sender": e.Sender,
"event_type": e.Type,
"body": body,
})
}
// Relations implements storage.RoomStore.
func (s *Rooms) Relations(ctx context.Context, req storage.RelationsRequest) ([]storage.StoredEvent, error) {
if req.Limit <= 0 || len(req.ParentIDs) == 0 {
return nil, nil
}
conditions := []string{
`e.room_id = ?`,
`e.relates_to_id IN ` + inList,
`NOT e.outlier`,
}
args := []any{req.RoomID, jsonList(req.ParentIDs)}
if req.RelType != "" {
conditions = append(conditions, `e.rel_type = ?`)
args = append(args, req.RelType)
} else {
conditions = append(conditions, `e.rel_type <> ''`)
}
if req.EventType != "" {
conditions = append(conditions, `e.event_type = ?`)
args = append(args, req.EventType)
}
if req.Low != 0 {
conditions = append(conditions, `e.stream_ordering > ?`)
args = append(args, req.Low)
}
if req.High != 0 {
conditions = append(conditions, `e.stream_ordering <= ?`)
args = append(args, req.High)
}
order := `ASC`
if req.Direction == storage.DirectionBackward {
order = `DESC`
}
args = append(args, req.Limit)
rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+`
FROM roomserver_events e
WHERE `+strings.Join(conditions, " AND ")+`
ORDER BY e.stream_ordering `+order+`
LIMIT ?`, args...)
if err != nil {
return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err)
}
events, err := scanEvents(rows)
if err != nil {
return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err)
}
return events, nil
}
// ChildEventIDs implements storage.RoomStore.
func (s *Rooms) ChildEventIDs(ctx context.Context, roomID string, parentIDs []string, relType, eventType string) ([]string, error) {
if len(parentIDs) == 0 {
return nil, nil
}
query := `SELECT event_id FROM roomserver_events
WHERE room_id = ? AND relates_to_id IN ` + inList + `
AND rel_type <> '' AND NOT outlier`
args := []any{roomID, jsonList(parentIDs)}
if relType != "" {
query += ` AND rel_type = ?`
args = append(args, relType)
}
if eventType != "" {
query += ` AND event_type = ?`
args = append(args, eventType)
}
query += ` ORDER BY stream_ordering`
ids, err := s.queryStrings(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read the children of %v in %q: %w", parentIDs, roomID, err)
}
return ids, nil
}
// RelationAggregates implements storage.RoomStore.
func (s *Rooms) RelationAggregates(ctx context.Context, roomID string, parentIDs []string, high int64) ([]storage.RelationAggregate, error) {
if len(parentIDs) == 0 {
return nil, nil
}
inner := `SELECT relates_to_id, rel_type, sender, COUNT(*) AS n, MAX(stream_ordering) AS latest
FROM roomserver_events
WHERE room_id = ? AND relates_to_id IN ` + inList + `
AND rel_type IN (?, ?, ?) AND NOT outlier`
args := []any{roomID, jsonList(parentIDs), storage.RelThread, storage.RelReplace, storage.RelReference}
if high != 0 {
inner += ` AND stream_ordering <= ?`
args = append(args, high)
}
inner += ` GROUP BY relates_to_id, rel_type, sender`
rows, err := s.db.QueryContext(ctx, `SELECT g.relates_to_id, g.rel_type, g.sender, g.n, g.latest, e.event_id
FROM (`+inner+`) g
JOIN roomserver_events e ON e.stream_ordering = g.latest`, args...)
if err != nil {
return nil, fmt.Errorf("summarise the relations of %d events in %q: %w", len(parentIDs), roomID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.RelationAggregate
for rows.Next() {
var a storage.RelationAggregate
if err := rows.Scan(&a.ParentID, &a.RelType, &a.Sender, &a.Count, &a.LatestOrdering, &a.LatestEventID); err != nil {
return nil, fmt.Errorf("summarise the relations in %q: %w", roomID, err)
}
out = append(out, a)
}
return out, rows.Err()
}
// RelationEventIDs implements storage.RoomStore.
func (s *Rooms) RelationEventIDs(ctx context.Context, roomID string, parentIDs []string, relType string, high int64, limit int) (map[string][]string, error) {
if len(parentIDs) == 0 || limit <= 0 {
return nil, nil
}
query := `SELECT relates_to_id, event_id FROM roomserver_events
WHERE room_id = ? AND relates_to_id IN ` + inList + `
AND rel_type = ? AND NOT outlier`
args := []any{roomID, jsonList(parentIDs), relType}
if high != 0 {
query += ` AND stream_ordering <= ?`
args = append(args, high)
}
query += ` ORDER BY stream_ordering LIMIT ?`
args = append(args, limit*len(parentIDs))
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err)
}
defer func() { _ = rows.Close() }()
out := map[string][]string{}
for rows.Next() {
var parent, id string
if err := rows.Scan(&parent, &id); err != nil {
return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err)
}
if len(out[parent]) < limit {
out[parent] = append(out[parent], id)
}
}
return out, rows.Err()
}
// Threads implements storage.RoomStore.
func (s *Rooms) Threads(ctx context.Context, req storage.ThreadsRequest) ([]storage.Thread, error) {
if req.Limit <= 0 {
return nil, nil
}
inner := `SELECT relates_to_id AS root, MAX(stream_ordering) AS latest
FROM roomserver_events
WHERE room_id = ? AND rel_type = ? AND NOT outlier`
args := []any{req.RoomID, storage.RelThread}
if req.High != 0 {
inner += ` AND stream_ordering <= ?`
args = append(args, req.High)
}
inner += ` GROUP BY relates_to_id`
conditions := []string{`r.room_id = ?`, `NOT r.outlier`}
args = append(args, req.RoomID)
if req.Before != 0 {
conditions = append(conditions, `g.latest < ?`)
args = append(args, req.Before)
}
if req.ParticipatedBy != "" {
conditions = append(conditions, `(r.sender = ? OR EXISTS (
SELECT 1 FROM roomserver_events p
WHERE p.room_id = r.room_id AND p.relates_to_id = g.root AND p.rel_type = ? AND p.sender = ?))`)
args = append(args, req.ParticipatedBy, storage.RelThread, req.ParticipatedBy)
}
args = append(args, req.Limit)
rows, err := s.db.QueryContext(ctx, `SELECT g.root, g.latest
FROM (`+inner+`) g
JOIN roomserver_events r ON r.event_id = g.root
WHERE `+strings.Join(conditions, " AND ")+`
ORDER BY g.latest DESC
LIMIT ?`, args...)
if err != nil {
return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.Thread
for rows.Next() {
var t storage.Thread
if err := rows.Scan(&t.RootID, &t.LatestOrdering); err != nil {
return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err)
}
out = append(out, t)
}
return out, rows.Err()
}
// searchKeys are the fields the index holds.
var searchKeys = []string{"content.body", "content.name", "content.topic"}
// Search implements storage.RoomStore.
func (s *Rooms) Search(ctx context.Context, req storage.SearchRequest) ([]storage.SearchResult, int, error) {
if req.Limit <= 0 || len(req.Rooms) == 0 {
return nil, 0, nil
}
words := storage.SearchWords(req.Term)
if len(words) == 0 {
return nil, 0, nil
}
var (
conditions []string
args []any
)
for _, w := range words {
conditions = append(conditions, `body LIKE ? ESCAPE '\'`)
args = append(args, `% `+escapeLike(w)+`%`)
}
// One bounded clause per room: what a user may search of a room is a
// window of its stream, and the windows differ from room to room.
rooms := make([]string, 0, len(req.Rooms))
for _, r := range req.Rooms {
clause := `room_id = ?`
args = append(args, r.RoomID)
if r.Low != 0 {
clause += ` AND stream_ordering > ?`
args = append(args, r.Low)
}
if r.High != 0 {
clause += ` AND stream_ordering <= ?`
args = append(args, r.High)
}
rooms = append(rooms, `(`+clause+`)`)
}
conditions = append(conditions, `(`+strings.Join(rooms, ` OR `)+`)`)
keys := req.Keys
if len(keys) == 0 {
keys = searchKeys
}
conditions = append(conditions, `search_key IN `+inList)
args = append(args, jsonList(keys))
if clause, filterArgs := filterConditionsOn(`event_type`, `sender`, req.Filter); clause != "" {
conditions = append(conditions, clause)
args = append(args, filterArgs...)
}
where := strings.Join(conditions, " AND ")
var total int
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM roomserver_event_search WHERE `+where, args...).
Scan(&total); err != nil {
return nil, 0, fmt.Errorf("count search results: %w", err)
}
pageWhere := where
pageArgs := append([]any{}, args...)
if req.Order == storage.SearchByRecent && req.Before != 0 {
pageWhere += ` AND stream_ordering < ?`
pageArgs = append(pageArgs, req.Before)
}
offset := req.Offset
if req.Order == storage.SearchByRecent {
offset = 0
}
pageArgs = append(pageArgs, req.Limit, offset)
positions, err := s.queryInts(ctx, `SELECT stream_ordering FROM roomserver_event_search
WHERE `+pageWhere+`
ORDER BY stream_ordering DESC
LIMIT ? OFFSET ?`, pageArgs...)
if err != nil {
return nil, 0, fmt.Errorf("search: %w", err)
}
if len(positions) == 0 {
return nil, total, nil
}
eventRows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+`
FROM roomserver_events e WHERE e.stream_ordering IN `+inList, jsonInts(positions))
if err != nil {
return nil, 0, fmt.Errorf("read search results: %w", err)
}
events, err := scanEvents(eventRows)
if err != nil {
return nil, 0, fmt.Errorf("read search results: %w", err)
}
byPos := make(map[int64]storage.StoredEvent, len(events))
for _, e := range events {
byPos[e.StreamOrdering] = e
}
out := make([]storage.SearchResult, 0, len(positions))
for _, pos := range positions {
if e, ok := byPos[pos]; ok {
out = append(out, storage.SearchResult{Event: e, Rank: 1})
}
}
return out, total, nil
}
// queryInts runs a query returning one integer column.
func (b *base) queryInts(ctx context.Context, query string, args ...any) ([]int64, error) {
rows, err := b.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []int64
for rows.Next() {
var v int64
if err := rows.Scan(&v); err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
@@ -0,0 +1,247 @@
package pocketbase
import (
"context"
"database/sql"
"errors"
"fmt"
"slices"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// The cache of other servers' users' keys, as the SQL store keeps it.
var remoteKeyTables = []string{
"keyserver_remote_users", "keyserver_remote_device_keys", "keyserver_remote_cross_signing_keys",
}
// remoteMarker reads a cached user's marker row.
func remoteMarker(row rowQuerier, userID string) (streamID int64, stale, ok bool, err error) {
err = row(`SELECT stream_id, stale FROM keyserver_remote_users WHERE user_id = ?`, userID).Scan(&streamID, &stale)
switch {
case errors.Is(err, sql.ErrNoRows):
return 0, false, false, nil
case err != nil:
return 0, false, false, fmt.Errorf("read the cached keys of %q: %w", userID, err)
}
return streamID, stale, true, nil
}
// RemoteKeys implements storage.KeyStore. The marker and the keys are read
// under one read lock, so that a copy halfway through its replacement is
// never served.
func (s *Keys) RemoteKeys(ctx context.Context, userID string) (storage.RemoteKeys, bool, error) {
out := storage.RemoteKeys{UserID: userID, CrossSigning: map[string][]byte{}}
var found bool
err := s.read(ctx, func(ctx context.Context) error {
var err error
out.StreamID, out.Stale, found, err = remoteMarker(s.poolRow(ctx), userID)
if err != nil || !found {
return err
}
if out.Devices, err = s.remoteDevices(ctx, userID); err != nil {
return err
}
out.CrossSigning, err = s.remoteCrossSigning(ctx, userID)
return err
})
if err != nil || !found {
return storage.RemoteKeys{}, false, err
}
return out, true, nil
}
func (s *Keys) remoteDevices(ctx context.Context, userID string) ([]storage.RemoteDevice, error) {
rows, err := s.db.QueryContext(ctx, `SELECT device_id, display_name, key_json FROM keyserver_remote_device_keys
WHERE user_id = ? ORDER BY device_id`, userID)
if err != nil {
return nil, fmt.Errorf("read the cached devices of %q: %w", userID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.RemoteDevice
for rows.Next() {
var d storage.RemoteDevice
var keyJSON string
if err := rows.Scan(&d.DeviceID, &d.DisplayName, &keyJSON); err != nil {
return nil, fmt.Errorf("read the cached devices of %q: %w", userID, err)
}
d.JSON = []byte(keyJSON)
out = append(out, d)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read the cached devices of %q: %w", userID, err)
}
return out, nil
}
func (s *Keys) remoteCrossSigning(ctx context.Context, userID string) (map[string][]byte, error) {
rows, err := s.db.QueryContext(ctx, `SELECT purpose, key_json FROM keyserver_remote_cross_signing_keys
WHERE user_id = ?`, userID)
if err != nil {
return nil, fmt.Errorf("read the cached cross-signing keys of %q: %w", userID, err)
}
defer func() { _ = rows.Close() }()
out := map[string][]byte{}
for rows.Next() {
var purpose, keyJSON string
if err := rows.Scan(&purpose, &keyJSON); err != nil {
return nil, fmt.Errorf("read the cached cross-signing keys of %q: %w", userID, err)
}
out[purpose] = []byte(keyJSON)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read the cached cross-signing keys of %q: %w", userID, err)
}
return out, nil
}
// SetRemoteKeys implements storage.KeyStore. A snapshot older than what is
// held is not stored: it was fetched while an update was on its way in.
func (s *Keys) SetRemoteKeys(ctx context.Context, keys storage.RemoteKeys) (bool, error) {
stored := false
err := s.write(ctx, func(t *tx) error {
held, _, ok, err := remoteMarker(t.queryRow, keys.UserID)
if err != nil {
return err
}
if ok && held > keys.StreamID {
return nil
}
if err := deleteRemote(t, keys.UserID); err != nil {
return err
}
t.put("keyserver_remote_users", rec{"user_id": keys.UserID, "stream_id": keys.StreamID, "stale": false})
for _, d := range keys.Devices {
t.put("keyserver_remote_device_keys", rec{
"user_id": keys.UserID, "device_id": d.DeviceID,
"display_name": d.DisplayName, "key_json": string(d.JSON),
})
}
putRemoteCrossSigning(t, keys.UserID, keys.CrossSigning)
stored = true
return nil
})
return stored, err
}
// UpdateRemoteDevice implements storage.KeyStore.
func (s *Keys) UpdateRemoteDevice(ctx context.Context, u storage.RemoteDeviceUpdate) (storage.RemoteUpdateResult, error) {
var result storage.RemoteUpdateResult
err := s.write(ctx, func(t *tx) error {
held, stale, ok, err := remoteMarker(t.queryRow, u.UserID)
if err != nil {
return err
}
// A stale copy keeps the highest position it has heard of, so that
// a snapshot fetched before this update cannot be stored over it.
markStale := func(r storage.RemoteUpdateResult) error {
result = r
t.put("keyserver_remote_users", rec{"user_id": u.UserID, "stream_id": max(held, u.StreamID), "stale": true})
return nil
}
switch {
case !ok:
return markStale(storage.RemoteUpdateNotCached)
case stale:
return markStale(storage.RemoteUpdateStale)
case u.StreamID <= held:
result = storage.RemoteUpdateDuplicate
return nil
case !followsOn(held, u.PrevIDs):
return markStale(storage.RemoteUpdateStale)
case !u.Deleted && u.JSON == nil:
return markStale(storage.RemoteUpdateStale)
}
if u.Deleted {
if _, err := t.delKey("keyserver_remote_device_keys", u.UserID, u.DeviceID); err != nil {
return fmt.Errorf("apply an update to the cached keys of %s/%s: %w", u.UserID, u.DeviceID, err)
}
} else {
t.put("keyserver_remote_device_keys", rec{
"user_id": u.UserID, "device_id": u.DeviceID,
"display_name": u.DisplayName, "key_json": string(u.JSON),
})
}
t.put("keyserver_remote_users", rec{"user_id": u.UserID, "stream_id": u.StreamID, "stale": false})
result = storage.RemoteUpdateApplied
return nil
})
return result, err
}
// followsOn reports whether an update with the given prev_id comes right
// after the position held.
func followsOn(held int64, prevIDs []int64) bool {
if len(prevIDs) == 0 {
return held == 0
}
return slices.Contains(prevIDs, held)
}
// SetRemoteCrossSigningKeys implements storage.KeyStore.
func (s *Keys) SetRemoteCrossSigningKeys(ctx context.Context, userID string, keys map[string][]byte) (bool, error) {
stored := false
err := s.write(ctx, func(t *tx) error {
_, stale, ok, err := remoteMarker(t.queryRow, userID)
if err != nil || !ok || stale {
return err
}
if _, err := t.delWhere("keyserver_remote_cross_signing_keys", `user_id = ?`, userID); err != nil {
return fmt.Errorf("replace the cached cross-signing keys of %q: %w", userID, err)
}
putRemoteCrossSigning(t, userID, keys)
stored = true
return nil
})
return stored, err
}
func putRemoteCrossSigning(t *tx, userID string, keys map[string][]byte) {
for _, purpose := range sortedKeys(keys) {
if keys[purpose] == nil {
continue
}
t.put("keyserver_remote_cross_signing_keys", rec{"user_id": userID, "purpose": purpose, "key_json": string(keys[purpose])})
}
}
// RemoteKeyUsers implements storage.KeyStore.
func (s *Keys) RemoteKeyUsers(ctx context.Context) ([]string, error) {
out, err := s.queryStrings(ctx, `SELECT user_id FROM keyserver_remote_users ORDER BY user_id`)
if err != nil {
return nil, fmt.Errorf("read the users with cached keys: %w", err)
}
return out, nil
}
// DeleteRemoteKeys implements storage.KeyStore.
func (s *Keys) DeleteRemoteKeys(ctx context.Context, userIDs []string) error {
if len(userIDs) == 0 {
return nil
}
return s.write(ctx, func(t *tx) error {
for _, userID := range dedupe(userIDs) {
if err := deleteRemote(t, userID); err != nil {
return err
}
}
return nil
})
}
// deleteRemote drops every row cached for a user.
func deleteRemote(t *tx, userID string) error {
for _, table := range remoteKeyTables {
if _, err := t.delWhere(table, `user_id = ?`, userID); err != nil {
return fmt.Errorf("delete from %s for %q: %w", table, userID, err)
}
}
return nil
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,449 @@
package pocketbase
import (
"crypto/sha256"
"encoding/base32"
"encoding/hex"
"encoding/json"
"fmt"
"strconv"
"strings"
)
// The schema is the SQL drivers' tables as PocketBase collections: the same
// names and columns, so that the reads can be the same SQL, with three
// differences PocketBase forces.
//
// - Every record has a text `id`, which is the primary key. Warp's own keys
// are kept as ordinary fields under a unique index, and a record's id is
// derived from them (see rowID): a row is always addressed by its key.
// - Nothing can be NULL. The two nullable state_key columns and
// admin_registration_tokens.uses_allowed get a has_* flag beside them.
// - `id` is taken, so push_notifications.id is notification_id and
// admin_audit_log.id is audit_id.
//
// Full-text search has no FTS5 here — it is a plain collection searched with
// LIKE — and media content lives in warp_media's file field, which is
// PocketBase's own storage: its pb_data/storage or an S3 bucket.
type fieldKind int
const (
text fieldKind = iota
number
boolean
file
)
type field struct {
name string
kind fieldKind
}
// table is one collection.
type table struct {
name string
// key is the primary key the SQL drivers declare; the record id is a
// hash of its values. Empty for a collection whose ids are chosen by
// the writer (warp_params).
key []string
fields []field
// indexes are column lists; a leading "unique:" makes one unique.
indexes []string
}
func tbl(name string, key []string, fields []field, indexes ...string) table {
return table{name: name, key: key, fields: fields, indexes: indexes}
}
func pk(cols ...string) []string { return cols }
func fld(name string, kind fieldKind) field { return field{name: name, kind: kind} }
func texts(names ...string) []field {
out := make([]field, len(names))
for i, n := range names {
out[i] = field{name: n, kind: text}
}
return out
}
func numbers(names ...string) []field {
out := make([]field, len(names))
for i, n := range names {
out[i] = field{name: n, kind: number}
}
return out
}
func fields(groups ...[]field) []field {
var out []field
for _, g := range groups {
out = append(out, g...)
}
return out
}
func sequenceTable(name string) table {
return tbl(name, pk("name"), fields(texts("name"), numbers("value")))
}
// The sequence rows each table starts with, as the SQL migrations insert
// them.
var sequenceRows = map[string][]string{
"roomserver_sequences": {"stream_ordering", "state_snapshot", "backfill_ordering"},
"sync_sequences": {"receipt", "to_device", "account_data", "presence"},
"federation_sequences": {"queue"},
"keyserver_sequences": {"device_list"},
"push_sequences": {"notification"},
"admin_sequences": {"audit"},
}
var schema = []table{
tbl("warp_meta", pk("meta_key"), texts("meta_key", "meta_value")),
{name: paramsCollection, fields: texts("value")},
tbl("warp_media", pk("sha256"), fields(texts("sha256"), numbers("size"), []field{fld("content", file)})),
tbl("server_settings", pk("setting_key"), texts("setting_key", "setting_value", "updated_at")),
// userapi
tbl("user_accounts", pk("localpart"), fields(
texts("localpart", "password_hash", "display_name", "avatar_url"),
[]field{fld("admin", boolean), fld("deactivated", boolean)},
numbers("created_at_ms"))),
tbl("user_devices", pk("localpart", "device_id"), fields(
texts("localpart", "device_id", "display_name"),
numbers("created_at_ms", "last_seen_ms"),
texts("last_seen_ip", "user_agent"))),
tbl("user_access_tokens", pk("token_hash"), fields(
texts("token_hash", "localpart", "device_id"), numbers("created_at_ms", "expires_at_ms")),
"localpart, device_id"),
tbl("user_refresh_tokens", pk("token_hash"), fields(
texts("token_hash", "localpart", "device_id"), numbers("created_at_ms", "expires_at_ms")),
"localpart, device_id"),
tbl("user_account_data", pk("localpart", "room_id", "data_type"), fields(
texts("localpart", "room_id", "data_type", "content"), numbers("stream_pos")),
"localpart, stream_pos"),
// roomserver
tbl("roomserver_rooms", pk("room_id"), fields(
texts("room_id", "room_version", "creator"),
[]field{fld("is_published", boolean)},
numbers("created_at_ms"),
texts("name", "topic", "avatar_url", "canonical_alias",
"join_rule", "history_visibility", "guest_access", "room_type"),
numbers("joined_members")),
"is_published, joined_members"),
sequenceTable("roomserver_sequences"),
tbl("roomserver_state_snapshots", pk("snapshot_id"), fields(
numbers("snapshot_id"), texts("room_id", "state_hash")),
"unique: room_id, state_hash"),
tbl("roomserver_state_entries", pk("snapshot_id", "event_type", "state_key"), fields(
numbers("snapshot_id"), texts("event_type", "state_key", "event_id"))),
tbl("roomserver_events", pk("event_id"), fields(
texts("event_id", "room_id"),
numbers("stream_ordering"),
texts("event_type", "state_key"),
[]field{fld("has_state_key", boolean)},
texts("sender"),
numbers("depth", "origin_server_ts"),
texts("event_json"),
numbers("state_before_id"),
texts("redacted_by"),
[]field{fld("outlier", boolean)},
texts("relates_to_id", "rel_type")),
"unique: stream_ordering",
"room_id, stream_ordering",
"room_id, relates_to_id, rel_type, stream_ordering",
"event_type, state_key"),
tbl("roomserver_extremities", pk("room_id", "event_id"), texts("room_id", "event_id")),
tbl("roomserver_current_state", pk("room_id", "event_type", "state_key"),
texts("room_id", "event_type", "state_key", "event_id", "membership"),
"event_type, state_key, membership"),
tbl("roomserver_transactions", pk("localpart", "device_id", "txn_id"), fields(
texts("localpart", "device_id", "txn_id", "event_id", "room_id"), numbers("sent_at_ms"))),
tbl("roomserver_aliases", pk("alias"), fields(
texts("alias", "room_id", "creator"), numbers("created_at_ms")),
"room_id"),
tbl("roomserver_invite_state", pk("room_id", "user_id"), texts("room_id", "user_id", "state_json")),
tbl("roomserver_event_search", pk("stream_ordering"), fields(
numbers("stream_ordering"),
texts("room_id", "event_id", "search_key", "sender", "event_type", "body")),
"room_id, stream_ordering"),
// syncapi
sequenceTable("sync_sequences"),
tbl("sync_receipts", pk("room_id", "user_id", "receipt_type", "thread_id"), fields(
texts("room_id", "user_id", "receipt_type", "thread_id", "event_id"), numbers("ts_ms", "stream_pos")),
"stream_pos", "room_id, stream_pos"),
tbl("sync_to_device", pk("stream_pos"), fields(
numbers("stream_pos"),
texts("localpart", "device_id", "sender", "event_type", "content"),
numbers("sent_at_ms")),
"localpart, device_id, stream_pos"),
tbl("sync_presence", pk("user_id"), fields(
texts("user_id", "presence", "status_msg"), numbers("last_active_ms", "stream_pos")),
"stream_pos"),
tbl("sync_filters", pk("localpart", "filter_id"), fields(
texts("localpart", "filter_id", "filter_json"), numbers("created_at_ms"))),
// federation
tbl("federation_server_keys", pk("server_name", "key_id"), fields(
texts("server_name", "key_id", "public_key"),
numbers("valid_until_ts", "expired_ts", "fetched_at_ms"))),
sequenceTable("federation_sequences"),
tbl("federation_queue", pk("queue_id"), fields(
numbers("queue_id"),
texts("destination", "kind", "room_id", "event_id", "payload"),
numbers("queued_at_ms")),
"destination, kind, queue_id"),
tbl("federation_destinations", pk("server_name"), fields(
texts("server_name"),
numbers("failures", "retry_at_ms", "last_success_ms", "last_failure_ms"),
texts("last_error"))),
tbl("federation_inbound_txns", pk("origin", "txn_id"), fields(
texts("origin", "txn_id", "response_json"), numbers("received_at_ms"))),
// mediaapi
tbl("media_repository", pk("media_origin", "media_id"), fields(
texts("media_origin", "media_id", "content_type", "upload_name"),
numbers("file_size"),
texts("sha256", "user_id"),
numbers("created_at_ms"),
[]field{fld("pending", boolean)},
numbers("expires_at_ms")),
"pending, user_id, expires_at_ms", "user_id, media_origin", "sha256"),
tbl("media_thumbnails", pk("media_origin", "media_id", "width", "height", "method"), fields(
texts("media_origin", "media_id"),
numbers("width", "height"),
texts("method", "content_type"),
numbers("file_size"),
texts("sha256"),
numbers("created_at_ms")),
"sha256"),
tbl("media_quotas", pk("user_id"), fields(texts("user_id"), numbers("max_bytes"))),
// keyserver
sequenceTable("keyserver_sequences"),
tbl("keyserver_device_keys", pk("user_id", "device_id"), texts("user_id", "device_id", "key_json")),
tbl("keyserver_one_time_keys", pk("user_id", "device_id", "key_id"),
texts("user_id", "device_id", "key_id", "algorithm", "key_json"),
"user_id, device_id, algorithm"),
tbl("keyserver_fallback_keys", pk("user_id", "device_id", "algorithm"), fields(
texts("user_id", "device_id", "algorithm", "key_id", "key_json"),
[]field{fld("used", boolean)})),
tbl("keyserver_cross_signing_keys", pk("user_id", "purpose"), texts("user_id", "purpose", "key_json")),
tbl("keyserver_signatures", pk("target_user_id", "target_key_id", "signer_user_id", "signer_key_id"),
texts("target_user_id", "target_key_id", "signer_user_id", "signer_key_id", "signature")),
tbl("keyserver_device_lists", pk("user_id"), fields(texts("user_id"), numbers("stream_pos")),
"stream_pos"),
tbl("keyserver_backup_versions", pk("user_id", "version"), fields(
texts("user_id"), numbers("version"), texts("algorithm", "auth_data"), numbers("etag"),
[]field{fld("deleted", boolean)})),
tbl("keyserver_backup_keys", pk("user_id", "version", "room_id", "session_id"), fields(
texts("user_id"), numbers("version"), texts("room_id", "session_id"),
numbers("first_message_index", "forwarded_count"),
[]field{fld("is_verified", boolean)},
texts("session_data"))),
tbl("keyserver_remote_users", pk("user_id"), fields(
texts("user_id"), numbers("stream_id"), []field{fld("stale", boolean)})),
tbl("keyserver_remote_device_keys", pk("user_id", "device_id"),
texts("user_id", "device_id", "display_name", "key_json")),
tbl("keyserver_remote_cross_signing_keys", pk("user_id", "purpose"), texts("user_id", "purpose", "key_json")),
// push
sequenceTable("push_sequences"),
tbl("push_pushers", pk("localpart", "app_id", "pushkey"), fields(
texts("localpart", "app_id", "pushkey", "device_id", "kind", "app_display_name",
"device_display_name", "profile_tag", "lang", "data_json"),
numbers("pushkey_ts_ms")),
"app_id, pushkey"),
tbl("push_notifications", pk("notification_id"), fields(
numbers("notification_id"),
texts("user_id", "room_id", "event_id"),
numbers("event_pos", "ts_ms"),
[]field{fld("highlight", boolean)},
texts("actions"),
[]field{fld("is_read", boolean)},
texts("thread_id")),
"user_id, notification_id",
"user_id, room_id, is_read, event_pos",
"is_read, ts_ms",
"user_id, room_id, thread_id, is_read, event_pos"),
tbl("push_room_counts", pk("user_id", "room_id"), fields(
texts("user_id", "room_id"), numbers("notification_count", "highlight_count", "stream_pos")),
"user_id, stream_pos"),
// admin and application services
tbl("admin_registration_tokens", pk("token"), fields(
texts("token"),
numbers("uses_allowed"),
[]field{fld("has_uses_allowed", boolean)},
numbers("completed", "expires_at_ms", "created_at_ms"))),
tbl("admin_server_notice_rooms", pk("localpart"), texts("localpart", "room_id")),
sequenceTable("admin_sequences"),
tbl("admin_audit_log", pk("audit_id"), fields(
numbers("audit_id", "ts_ms"), texts("actor", "action", "target", "details_json")),
"actor, audit_id", "action, audit_id"),
tbl("appservice_state", pk("as_id"), fields(texts("as_id"), numbers("stream_pos", "txn_id", "txn_to"))),
// delayed events and OpenID tokens
tbl("delayed_events", pk("delay_id"), fields(
texts("delay_id", "user_id", "device_id", "room_id", "event_type", "state_key"),
[]field{fld("has_state_key", boolean)},
texts("content"),
numbers("delay_ms", "running_since_ms", "send_at_ms", "finalised_at_ms"),
texts("outcome", "reason", "event_id", "error")),
"user_id, finalised_at_ms",
"finalised_at_ms, send_at_ms",
"room_id, event_type, state_key, finalised_at_ms"),
tbl("openid_tokens", pk("token_hash"), fields(texts("token_hash", "user_id"), numbers("expires_at_ms")),
"expires_at_ms"),
}
// tables indexes the schema by collection name.
var tables = func() map[string]*table {
out := make(map[string]*table, len(schema))
for i := range schema {
out[schema[i].name] = &schema[i]
}
return out
}()
// maxText is the longest a text field may be. PocketBase's default is 5000
// characters, which an event can pass; this is far above the 64 KiB a
// Matrix event may be, and above any account data a client sends.
const maxText = 1 << 24
// maxFileSize bounds one media file. The media API enforces the configured
// upload limit before content ever gets here.
const maxFileSize = 1 << 40
// shortHash is a lower-case base32 digest, which fits PocketBase's id
// alphabet of [a-z0-9].
func shortHash(n int, parts ...string) string {
h := sha256.New()
for _, p := range parts {
_, _ = fmt.Fprintf(h, "%d:%s", len(p), p)
}
enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h.Sum(nil))
return strings.ToLower(enc)[:n]
}
// rowID is the record id of the row whose primary key has these values.
func rowID(collection string, key ...any) string {
parts := make([]string, 0, len(key)+1)
parts = append(parts, collection)
for _, v := range key {
parts = append(parts, keyString(v))
}
return shortHash(15, parts...)
}
func keyString(v any) string {
switch v := v.(type) {
case string:
return v
case int64:
return strconv.FormatInt(v, 10)
case int:
return strconv.Itoa(v)
case bool:
return strconv.FormatBool(v)
default:
return fmt.Sprint(v)
}
}
// collectionJSON renders a table as PocketBase's collection import format.
// Collection and field ids are derived from their names, so that importing
// the same schema again changes nothing, and a later version that adds a
// field adds just that field.
func (tb *table) collectionJSON() map[string]any {
out := []map[string]any{{
"id": "text3208210256", "name": "id", "type": "text", "system": true,
"primaryKey": true, "required": true, "hidden": false, "presentable": false,
"min": 15, "max": 15, "pattern": "^[a-z0-9]+$", "autogeneratePattern": "[a-z0-9]{15}",
}}
for _, fd := range tb.fields {
m := map[string]any{
"id": "f" + shortHash(12, tb.name, fd.name),
"name": fd.name,
"system": false, "required": false, "hidden": false, "presentable": false,
}
switch fd.kind {
case text:
m["type"] = "text"
m["min"] = 0
m["max"] = maxText
m["pattern"] = ""
m["autogeneratePattern"] = ""
m["primaryKey"] = false
case number:
m["type"] = "number"
m["onlyInt"] = true
m["min"] = nil
m["max"] = nil
case boolean:
m["type"] = "bool"
case file:
m["type"] = "file"
m["maxSelect"] = 1
m["maxSize"] = maxFileSize
m["mimeTypes"] = []string{}
m["thumbs"] = []string{}
m["protected"] = true
}
out = append(out, m)
}
indexes := []string{}
if len(tb.key) > 0 {
indexes = append(indexes, indexSQL(tb.name, "unique: "+strings.Join(tb.key, ", ")))
}
for _, idx := range tb.indexes {
indexes = append(indexes, indexSQL(tb.name, idx))
}
return map[string]any{
"id": "warp" + shortHash(11, tb.name),
"name": tb.name,
"type": "base",
"system": false,
"listRule": nil,
"viewRule": nil,
"createRule": nil,
"updateRule": nil,
"deleteRule": nil,
"fields": out,
"indexes": indexes,
}
}
func indexSQL(tableName, spec string) string {
unique := ""
if cols, ok := strings.CutPrefix(spec, "unique:"); ok {
unique, spec = "UNIQUE ", cols
}
var cols []string
for _, c := range strings.Split(spec, ",") {
cols = append(cols, strings.TrimSpace(c))
}
name := "idx_" + shortHash(10, tableName, unique, strings.Join(cols, ","))
return fmt.Sprintf("CREATE %sINDEX `%s` ON `%s` (%s)", unique, name, tableName, "`"+strings.Join(cols, "`, `")+"`")
}
// schemaVersion identifies the schema: any change to a table changes it, and
// a database stamped with another one has a migration pending.
var schemaVersion = func() string {
cols := make([]map[string]any, len(schema))
for i := range schema {
cols[i] = schema[i].collectionJSON()
}
raw, _ := json.Marshal(cols)
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])[:12]
}()
@@ -0,0 +1,362 @@
package pocketbase
import (
"context"
"crypto/rand"
"database/sql/driver"
"encoding/hex"
"errors"
"fmt"
"io"
"slices"
"strconv"
"strings"
"sync"
"time"
"unicode/utf8"
)
// The read side of the driver is a database/sql driver over PocketBase's SQL
// endpoint, so that the stores can read the way the SQL drivers do: with
// `?` placeholders, joins and aggregates, and rows.Scan. It cannot write —
// every write goes through the batch endpoint, as record operations — and
// it cannot begin a transaction.
//
// The endpoint has three limits the driver hides. It takes no parameters, so
// every argument is inlined as a SQL literal. It takes at most 5000
// characters, so a query that would be longer has its longest arguments
// moved into temporary records of warp_params and read back by subquery. And
// it returns at most 1000 rows, so a result that fills a page is read on in
// further pages.
// errReadOnly is returned for anything but a query.
var errReadOnly = errors.New("pocketbase: the SQL connection is read-only; writes go through the batch API")
const (
// maxQueryLength is the SQL endpoint's limit, with room for the paging
// wrapper.
maxQueryLength = 5000 - len(pageWrapperPrefix) - len(pageWrapperSuffix) - 32
// pageSize is how many rows the SQL endpoint returns at most.
pageSize = 1000
pageWrapperPrefix = `SELECT * FROM (`
pageWrapperSuffix = `) LIMIT 1000 OFFSET `
// paramsCollection holds the arguments too long to inline.
paramsCollection = "warp_params"
)
// sqlConnector opens connections to one PocketBase.
type sqlConnector struct {
c *client
// lock is held for reading around every query that runs outside a
// write, so that a query read in several pages is not split by one.
lock *sync.RWMutex
}
func (k *sqlConnector) Connect(context.Context) (driver.Conn, error) { return &sqlConn{k: k}, nil }
func (k *sqlConnector) Driver() driver.Driver { return sqlDriver{k: k} }
// sqlDriver exists because database/sql wants one; connections come from the
// connector.
type sqlDriver struct{ k *sqlConnector }
func (d sqlDriver) Open(string) (driver.Conn, error) { return &sqlConn{k: d.k}, nil }
type sqlConn struct{ k *sqlConnector }
func (c *sqlConn) Prepare(query string) (driver.Stmt, error) {
return &sqlStmt{c: c, query: query}, nil
}
func (c *sqlConn) Close() error { return nil }
func (c *sqlConn) Begin() (driver.Tx, error) { return nil, errReadOnly }
// Ping implements driver.Pinger.
func (c *sqlConn) Ping(ctx context.Context) error { return c.k.c.health(ctx) }
// QueryContext implements driver.QueryerContext.
func (c *sqlConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
if !holdsWriteLock(ctx) {
c.k.lock.RLock()
defer c.k.lock.RUnlock()
}
return c.k.run(ctx, query, args)
}
type sqlStmt struct {
c *sqlConn
query string
}
func (s *sqlStmt) Close() error { return nil }
func (s *sqlStmt) NumInput() int { return -1 }
func (s *sqlStmt) Exec([]driver.Value) (driver.Result, error) { return nil, errReadOnly }
func (s *sqlStmt) Query(args []driver.Value) (driver.Rows, error) {
named := make([]driver.NamedValue, len(args))
for i, v := range args {
named[i] = driver.NamedValue{Ordinal: i + 1, Value: v}
}
return s.c.QueryContext(context.Background(), s.query, named)
}
// writeLockKey marks a context whose caller already holds the store's lock,
// for writing or for a multi-query read, so that its queries do not take
// the read lock again behind it.
type writeLockKey struct{}
func withWriteLock(ctx context.Context) context.Context {
return context.WithValue(ctx, writeLockKey{}, true)
}
func holdsWriteLock(ctx context.Context) bool {
held, _ := ctx.Value(writeLockKey{}).(bool)
return held
}
// run inlines the arguments, moves what does not fit into warp_params, and
// reads every page of the result.
func (k *sqlConnector) run(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
parts, literals, err := splitPlaceholders(query, args)
if err != nil {
return nil, err
}
params, err := k.fitLiterals(ctx, parts, literals)
if err != nil {
return nil, err
}
if len(params) > 0 {
defer k.dropParams(params)
}
full := joinLiterals(parts, literals)
res, err := k.c.query(ctx, full)
if err != nil {
return nil, fmt.Errorf("%w\n\tin query: %s", err, abbreviate(full))
}
rows := &sqlRows{columns: make([]string, len(res.Columns)), data: res.Rows}
for i, col := range res.Columns {
rows.columns[i] = col.Name
}
for page := 1; len(res.Rows) == pageSize; page++ {
res, err = k.c.query(ctx, pageWrapperPrefix+full+pageWrapperSuffix+strconv.Itoa(page*pageSize))
if err != nil {
return nil, fmt.Errorf("%w\n\tin page %d of query: %s", err, page, abbreviate(full))
}
rows.data = append(rows.data, res.Rows...)
}
return rows, nil
}
// splitPlaceholders cuts a query at its `?` placeholders, skipping those
// inside quoted strings and identifiers, and renders each argument as a SQL
// literal.
func splitPlaceholders(query string, args []driver.NamedValue) ([]string, []string, error) {
var (
parts []string
literals []string
start int
quote byte
)
for i := 0; i < len(query); i++ {
ch := query[i]
switch {
case quote != 0:
if ch == quote {
quote = 0
}
case ch == '\'' || ch == '"':
quote = ch
case ch == '?':
n := len(literals)
if n >= len(args) {
return nil, nil, fmt.Errorf("pocketbase: query has more placeholders than its %d arguments", len(args))
}
lit, err := literal(args[n].Value)
if err != nil {
return nil, nil, err
}
parts = append(parts, query[start:i])
literals = append(literals, lit)
start = i + 1
}
}
if len(literals) != len(args) {
return nil, nil, fmt.Errorf("pocketbase: query has %d placeholders for %d arguments", len(literals), len(args))
}
return append(parts, query[start:]), literals, nil
}
// literal renders one value as SQLite would read it back.
func literal(v any) (string, error) {
switch v := v.(type) {
case nil:
return "NULL", nil
case bool:
if v {
return "1", nil
}
return "0", nil
case int64:
return strconv.FormatInt(v, 10), nil
case float64:
return strconv.FormatFloat(v, 'g', -1, 64), nil
case string:
return textLiteral(v), nil
case []byte:
return textLiteral(string(v)), nil
case time.Time:
return textLiteral(v.UTC().Format(time.RFC3339Nano)), nil
default:
return "", fmt.Errorf("pocketbase: cannot inline a %T into SQL", v)
}
}
// textLiteral quotes a string. Text that is not valid UTF-8, or holds a NUL,
// goes in as hex and is cast back, so that nothing in it can end the literal
// early.
func textLiteral(s string) string {
if utf8.ValidString(s) && !strings.ContainsRune(s, 0) {
return "'" + strings.ReplaceAll(s, "'", "''") + "'"
}
return "CAST(X'" + hex.EncodeToString([]byte(s)) + "' AS TEXT)"
}
func joinLiterals(parts, literals []string) string {
var b strings.Builder
for i, p := range parts {
b.WriteString(p)
if i < len(literals) {
b.WriteString(literals[i])
}
}
return b.String()
}
// queryLength is what the SQL endpoint counts: characters, not bytes.
func queryLength(parts, literals []string) int {
n := 0
for _, p := range parts {
n += utf8.RuneCountInString(p)
}
for _, l := range literals {
n += utf8.RuneCountInString(l)
}
return n
}
// fitLiterals moves the longest text literals into warp_params until the
// query fits, rewriting each into a subquery reading it back. It returns the
// IDs of the records it created.
func (k *sqlConnector) fitLiterals(ctx context.Context, parts, literals []string) ([]string, error) {
if queryLength(parts, literals) <= maxQueryLength {
return nil, nil
}
order := make([]int, len(literals))
for i := range order {
order[i] = i
}
slices.SortFunc(order, func(a, b int) int { return len(literals[b]) - len(literals[a]) })
var (
ids []string
reqs []batchRequest
)
for _, i := range order {
if queryLength(parts, literals) <= maxQueryLength {
break
}
text, ok := unquote(literals[i])
if !ok {
continue
}
id, err := randomID()
if err != nil {
return nil, err
}
reqs = append(reqs, batchRequest{
Method: "POST",
URL: "/api/collections/" + paramsCollection + "/records",
Body: map[string]any{"id": id, "value": text},
})
ids = append(ids, id)
literals[i] = `(SELECT value FROM ` + paramsCollection + ` WHERE id = '` + id + `')`
}
if queryLength(parts, literals) > maxQueryLength {
return nil, fmt.Errorf("pocketbase: query is too long for the SQL endpoint even with its arguments moved out")
}
if err := k.c.batch(ctx, reqs); err != nil {
return nil, fmt.Errorf("pocketbase: store query arguments: %w", err)
}
return ids, nil
}
// dropParams deletes the records a query's arguments were moved into. It
// runs after the query whatever happened to it, on a context of its own, and
// a failure only leaves rows behind for the next start to clear.
func (k *sqlConnector) dropParams(ids []string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
reqs := make([]batchRequest, len(ids))
for i, id := range ids {
reqs[i] = batchRequest{Method: "DELETE", URL: "/api/collections/" + paramsCollection + "/records/" + id}
}
_ = k.c.batch(ctx, reqs)
}
// unquote recovers the text of a literal textLiteral made.
func unquote(lit string) (string, bool) {
if strings.HasPrefix(lit, "'") {
return strings.ReplaceAll(lit[1:len(lit)-1], "''", "'"), true
}
if hexText, ok := strings.CutPrefix(lit, "CAST(X'"); ok {
raw, err := hex.DecodeString(strings.TrimSuffix(hexText, "' AS TEXT)"))
return string(raw), err == nil
}
return "", false
}
// randomID makes a record ID in PocketBase's default alphabet.
func randomID() (string, error) {
var raw [8]byte
if _, err := rand.Read(raw[:]); err != nil {
return "", err
}
return "p" + hex.EncodeToString(raw[:])[:14], nil
}
func abbreviate(query string) string {
query = strings.Join(strings.Fields(query), " ")
if len(query) > 500 {
return query[:500] + "…"
}
return query
}
// sqlRows is a whole result, already read.
type sqlRows struct {
columns []string
data [][]*string
next int
}
func (r *sqlRows) Columns() []string { return r.columns }
func (r *sqlRows) Close() error { return nil }
func (r *sqlRows) Next(dest []driver.Value) error {
if r.next >= len(r.data) {
return io.EOF
}
row := r.data[r.next]
r.next++
for i := range dest {
if i < len(row) && row[i] != nil {
dest[i] = *row[i]
} else {
dest[i] = nil
}
}
return nil
}
+461
View File
@@ -0,0 +1,461 @@
package pocketbase
import (
"context"
"crypto/rand"
"database/sql"
"encoding/base64"
"errors"
"fmt"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Sync is the PocketBase implementation of storage.SyncStore. Every stream
// here is a counter row advanced in the same batch as the row it stamps.
type Sync struct{ *base }
// maxToDeviceBatch bounds how many to-device messages one sync carries.
const maxToDeviceBatch = 100
// StreamPositions implements storage.SyncStore. The positions are read under
// one read lock, so that a caller gets one consistent set.
func (s *Sync) StreamPositions(ctx context.Context) (storage.StreamPositions, error) {
var out storage.StreamPositions
err := s.read(ctx, func(ctx context.Context) error {
err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(stream_ordering), 0) FROM roomserver_events`).Scan(&out.PDU)
if err != nil {
return fmt.Errorf("read the top of the event stream: %w", err)
}
rows, err := s.db.QueryContext(ctx, `SELECT name, value FROM sync_sequences`)
if err != nil {
return fmt.Errorf("read the stream positions: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var (
name string
value int64
)
if err := rows.Scan(&name, &value); err != nil {
return fmt.Errorf("read the stream positions: %w", err)
}
switch name {
case seqReceipt:
out.Receipt = value
case seqToDevice:
out.ToDevice = value
case seqAccountData:
out.AccountData = value
case seqPresence:
out.Presence = value
}
}
if err := rows.Err(); err != nil {
return fmt.Errorf("read the stream positions: %w", err)
}
if out.DeviceList, err = s.currentSequence(ctx, keySequences, seqDeviceList); err != nil {
return err
}
out.Notification, err = s.currentSequence(ctx, pushSequences, seqNotification)
return err
})
if err != nil {
return storage.StreamPositions{}, err
}
return out, nil
}
// SetReceipt implements storage.SyncStore. A receipt that would move
// backwards is dropped: the comparison is on the position of the event
// being acknowledged, not on when the receipt arrived.
func (s *Sync) SetReceipt(ctx context.Context, r storage.Receipt) (int64, bool, error) {
var (
pos int64
moved bool
)
err := s.write(ctx, func(t *tx) error {
var target int64
err := t.queryRow(`SELECT e.stream_ordering FROM roomserver_events e
WHERE e.room_id = ? AND e.event_id = ?`, r.RoomID, r.EventID).Scan(&target)
switch {
case errors.Is(err, sql.ErrNoRows):
return fmt.Errorf("event %q: %w", r.EventID, storage.ErrNoSuchEvent)
case err != nil:
return fmt.Errorf("locate the event a receipt names: %w", err)
}
var (
haveEventID string
havePos sql.NullInt64
haveStream int64
)
err = t.queryRow(`SELECT r.event_id, e.stream_ordering, r.stream_pos
FROM sync_receipts r
LEFT JOIN roomserver_events e ON e.event_id = r.event_id
WHERE r.room_id = ? AND r.user_id = ? AND r.receipt_type = ? AND r.thread_id = ?`,
r.RoomID, r.UserID, r.ReceiptType, r.ThreadID).Scan(&haveEventID, &havePos, &haveStream)
switch {
case err == nil:
if havePos.Valid && havePos.Int64 >= target {
pos, moved = haveStream, false
return nil
}
case !errors.Is(err, sql.ErrNoRows):
return fmt.Errorf("read the current receipt: %w", err)
}
if pos, err = t.next(syncSequences, seqReceipt); err != nil {
return err
}
t.put("sync_receipts", rec{
"room_id": r.RoomID, "user_id": r.UserID, "receipt_type": r.ReceiptType, "thread_id": r.ThreadID,
"event_id": r.EventID, "ts_ms": r.TimestampMS, "stream_pos": pos,
})
moved = true
return nil
})
if err != nil {
return 0, false, fmt.Errorf("set receipt in %q: %w", r.RoomID, err)
}
return pos, moved, nil
}
const receiptColumns = `
room_id, user_id, receipt_type, thread_id, event_id, ts_ms, stream_pos`
func scanReceipts(rows *sql.Rows) ([]storage.Receipt, error) {
defer func() { _ = rows.Close() }()
var out []storage.Receipt
for rows.Next() {
var r storage.Receipt
if err := rows.Scan(&r.RoomID, &r.UserID, &r.ReceiptType, &r.ThreadID,
&r.EventID, &r.TimestampMS, &r.StreamPos); err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
// Receipts implements storage.SyncStore.
func (s *Sync) Receipts(ctx context.Context, roomIDs []string, from, to int64) ([]storage.Receipt, error) {
if len(roomIDs) == 0 {
return nil, nil
}
query := `SELECT ` + receiptColumns + ` FROM sync_receipts
WHERE room_id IN ` + inList + ` AND stream_pos > ?`
args := []any{jsonList(roomIDs), from}
if to > 0 {
query += ` AND stream_pos <= ?`
args = append(args, to)
}
query += ` ORDER BY stream_pos`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read receipts: %w", err)
}
receipts, err := scanReceipts(rows)
if err != nil {
return nil, fmt.Errorf("read receipts: %w", err)
}
return receipts, nil
}
// ReceiptsForUser implements storage.SyncStore.
func (s *Sync) ReceiptsForUser(ctx context.Context, userID string, roomIDs []string) ([]storage.Receipt, error) {
if len(roomIDs) == 0 {
return nil, nil
}
rows, err := s.db.QueryContext(ctx, `SELECT `+receiptColumns+` FROM sync_receipts
WHERE user_id = ? AND room_id IN `+inList+`
ORDER BY stream_pos`, userID, jsonList(roomIDs))
if err != nil {
return nil, fmt.Errorf("read the receipts of %q: %w", userID, err)
}
receipts, err := scanReceipts(rows)
if err != nil {
return nil, fmt.Errorf("read the receipts of %q: %w", userID, err)
}
return receipts, nil
}
// SendToDevice implements storage.SyncStore. Each message takes a position
// of its own, so that one device acknowledging its copy does not drop
// another's.
func (s *Sync) SendToDevice(ctx context.Context, msgs []storage.NewToDeviceMessage) (int64, error) {
if len(msgs) == 0 {
top, err := s.currentSequence(ctx, syncSequences, seqToDevice)
if err != nil {
return 0, fmt.Errorf("send to-device messages: %w", err)
}
return top, nil
}
var top int64
err := s.write(ctx, func(t *tx) error {
for _, m := range msgs {
pos, err := t.next(syncSequences, seqToDevice)
if err != nil {
return err
}
t.put("sync_to_device", rec{
"stream_pos": pos, "localpart": m.Localpart, "device_id": m.DeviceID,
"sender": m.Sender, "event_type": m.Type, "content": string(m.Content), "sent_at_ms": m.SentAtMS,
})
top = pos
}
return nil
})
if err != nil {
return 0, fmt.Errorf("send to-device messages: %w", err)
}
return top, nil
}
// ToDeviceMessages implements storage.SyncStore.
func (s *Sync) ToDeviceMessages(ctx context.Context, localpart, deviceID string, from, to int64, limit int) ([]storage.ToDeviceMessage, error) {
if limit <= 0 || limit > maxToDeviceBatch {
limit = maxToDeviceBatch
}
query := `SELECT stream_pos, localpart, device_id, sender, event_type, content, sent_at_ms
FROM sync_to_device
WHERE localpart = ? AND device_id = ? AND stream_pos > ?`
args := []any{localpart, deviceID, from}
if to > 0 {
query += ` AND stream_pos <= ?`
args = append(args, to)
}
query += ` ORDER BY stream_pos LIMIT ?`
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read to-device messages for %s/%s: %w", localpart, deviceID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.ToDeviceMessage
for rows.Next() {
var (
m storage.ToDeviceMessage
content string
)
if err := rows.Scan(&m.StreamPos, &m.Localpart, &m.DeviceID,
&m.Sender, &m.Type, &content, &m.SentAtMS); err != nil {
return nil, fmt.Errorf("read to-device messages for %s/%s: %w", localpart, deviceID, err)
}
m.Content = []byte(content)
out = append(out, m)
}
return out, rows.Err()
}
// DeleteToDeviceMessages implements storage.SyncStore.
func (s *Sync) DeleteToDeviceMessages(ctx context.Context, localpart, deviceID string, pos int64) error {
if pos <= 0 {
return nil
}
err := s.write(ctx, func(t *tx) error {
_, err := t.delWhere("sync_to_device", `localpart = ? AND device_id = ? AND stream_pos <= ?`,
localpart, deviceID, pos)
return err
})
if err != nil {
return fmt.Errorf("delete to-device messages for %s/%s: %w", localpart, deviceID, err)
}
return nil
}
const presenceColumns = `user_id, presence, status_msg, last_active_ms, stream_pos`
// SetPresence implements storage.SyncStore. A write that changes nothing a
// peer would notice refreshes the last-active timestamp and leaves the
// stream where it was.
func (s *Sync) SetPresence(ctx context.Context, p storage.PresenceState) (int64, bool, error) {
var (
pos int64
moved bool
)
err := s.write(ctx, func(t *tx) error {
var existing storage.PresenceState
err := t.queryRow(`SELECT `+presenceColumns+` FROM sync_presence WHERE user_id = ?`, p.UserID).Scan(
&existing.UserID, &existing.Presence, &existing.StatusMsg,
&existing.LastActiveMS, &existing.StreamPos)
switch {
case err == nil:
if existing.Presence == p.Presence && existing.StatusMsg == p.StatusMsg {
pos = existing.StreamPos
t.put("sync_presence", rec{"user_id": p.UserID, "last_active_ms": p.LastActiveMS})
return nil
}
case !errors.Is(err, sql.ErrNoRows):
return fmt.Errorf("read the current presence: %w", err)
}
if pos, err = t.next(syncSequences, seqPresence); err != nil {
return err
}
t.put("sync_presence", rec{
"user_id": p.UserID, "presence": p.Presence, "status_msg": p.StatusMsg,
"last_active_ms": p.LastActiveMS, "stream_pos": pos,
})
moved = true
return nil
})
if err != nil {
return 0, false, fmt.Errorf("set the presence of %q: %w", p.UserID, err)
}
return pos, moved, nil
}
// Presence implements storage.SyncStore.
func (s *Sync) Presence(ctx context.Context, userID string) (storage.PresenceState, error) {
var p storage.PresenceState
err := s.db.QueryRowContext(ctx, `SELECT `+presenceColumns+` FROM sync_presence WHERE user_id = ?`, userID).
Scan(&p.UserID, &p.Presence, &p.StatusMsg, &p.LastActiveMS, &p.StreamPos)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.PresenceState{}, fmt.Errorf("presence of %q: %w", userID, storage.ErrNoSuchUser)
case err != nil:
return storage.PresenceState{}, fmt.Errorf("read the presence of %q: %w", userID, err)
}
return p, nil
}
// PresenceForUsers implements storage.SyncStore.
func (s *Sync) PresenceForUsers(ctx context.Context, userIDs []string, from, to int64) ([]storage.PresenceState, error) {
if len(userIDs) == 0 {
return nil, nil
}
query := `SELECT ` + presenceColumns + ` FROM sync_presence
WHERE user_id IN ` + inList + ` AND stream_pos > ?`
args := []any{jsonList(userIDs), from}
if to > 0 {
query += ` AND stream_pos <= ?`
args = append(args, to)
}
query += ` ORDER BY stream_pos`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read presence: %w", err)
}
defer func() { _ = rows.Close() }()
var out []storage.PresenceState
for rows.Next() {
var p storage.PresenceState
if err := rows.Scan(&p.UserID, &p.Presence, &p.StatusMsg, &p.LastActiveMS, &p.StreamPos); err != nil {
return nil, fmt.Errorf("read presence: %w", err)
}
out = append(out, p)
}
return out, rows.Err()
}
// AccountDataChanges implements storage.SyncStore.
func (s *Sync) AccountDataChanges(ctx context.Context, localpart string, from, to int64) ([]storage.AccountDataEntry, error) {
query := `SELECT room_id, data_type, stream_pos FROM user_account_data
WHERE localpart = ? AND stream_pos > ?`
args := []any{localpart, from}
if to > 0 {
query += ` AND stream_pos <= ?`
args = append(args, to)
}
query += ` ORDER BY stream_pos`
return s.accountData(ctx, localpart, query, args)
}
// AccountDataAt implements storage.SyncStore.
func (s *Sync) AccountDataAt(ctx context.Context, localpart string, to int64) ([]storage.AccountDataEntry, error) {
query := `SELECT room_id, data_type, stream_pos FROM user_account_data WHERE localpart = ?`
args := []any{localpart}
if to > 0 {
query += ` AND stream_pos <= ?`
args = append(args, to)
}
query += ` ORDER BY room_id, data_type`
return s.accountData(ctx, localpart, query, args)
}
func (s *Sync) accountData(ctx context.Context, localpart, query string, args []any) ([]storage.AccountDataEntry, error) {
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read the account data of %q: %w", localpart, err)
}
defer func() { _ = rows.Close() }()
var out []storage.AccountDataEntry
for rows.Next() {
var e storage.AccountDataEntry
if err := rows.Scan(&e.RoomID, &e.DataType, &e.StreamPos); err != nil {
return nil, fmt.Errorf("read the account data of %q: %w", localpart, err)
}
out = append(out, e)
}
return out, rows.Err()
}
// SaveFilter implements storage.SyncStore. The ID is random rather than a
// counter, so that it says nothing about whose filter it is.
func (s *Sync) SaveFilter(ctx context.Context, localpart string, filter []byte, createdAtMS int64) (string, error) {
var filterID string
err := s.write(ctx, func(t *tx) error {
for range 3 {
id, err := newFilterID()
if err != nil {
return err
}
taken, err := t.exists("sync_filters", `localpart = ? AND filter_id = ?`, localpart, id)
if err != nil {
return err
}
if taken {
continue
}
t.put("sync_filters", rec{
"localpart": localpart, "filter_id": id,
"filter_json": string(filter), "created_at_ms": createdAtMS,
})
filterID = id
return nil
}
return errors.New("could not find an unused filter ID")
})
if err != nil {
return "", fmt.Errorf("save a filter for %q: %w", localpart, err)
}
return filterID, nil
}
// newFilterID mints a filter ID: 12 random bytes, URL-safe base64.
func newFilterID() (string, error) {
var raw [12]byte
if _, err := rand.Read(raw[:]); err != nil {
return "", fmt.Errorf("generate a filter ID: %w", err)
}
return base64.RawURLEncoding.EncodeToString(raw[:]), nil
}
// Filter implements storage.SyncStore.
func (s *Sync) Filter(ctx context.Context, localpart, filterID string) ([]byte, error) {
var body string
err := s.db.QueryRowContext(ctx, `SELECT filter_json FROM sync_filters
WHERE localpart = ? AND filter_id = ?`, localpart, filterID).Scan(&body)
switch {
case errors.Is(err, sql.ErrNoRows):
return nil, fmt.Errorf("filter %q: %w", filterID, storage.ErrNoSuchFilter)
case err != nil:
return nil, fmt.Errorf("read filter %q: %w", filterID, err)
}
return []byte(body), nil
}
var _ storage.SyncStore = (*Sync)(nil)
@@ -0,0 +1,334 @@
package pocketbase
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// The stream reads of the room store: the queries the sync API and
// /messages are built on. They are the SQL store's.
// MaxStreamOrdering implements storage.RoomStore. It reads the events, not
// the counter: a batch writes both at once, but an event skipped as a
// duplicate leaves the counter above every event there is.
func (s *Rooms) MaxStreamOrdering(ctx context.Context) (int64, error) {
var top int64
if err := s.db.QueryRowContext(ctx,
`SELECT COALESCE(MAX(stream_ordering), 0) FROM roomserver_events`).Scan(&top); err != nil {
return 0, fmt.Errorf("read the top of the event stream: %w", err)
}
return top, nil
}
// MembershipsForUser implements storage.RoomStore.
func (s *Rooms) MembershipsForUser(ctx context.Context, userID string, memberships []string) ([]storage.SyncMembership, error) {
query := `
SELECT cs.room_id, cs.membership, cs.event_id, e.stream_ordering
FROM roomserver_current_state cs
JOIN roomserver_events e ON e.event_id = cs.event_id
WHERE cs.event_type = 'm.room.member' AND cs.state_key = ?`
args := []any{userID}
if len(memberships) > 0 {
query += ` AND cs.membership IN ` + inList
args = append(args, jsonList(memberships))
}
query += ` ORDER BY cs.room_id`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read the memberships of %q: %w", userID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.SyncMembership
for rows.Next() {
var m storage.SyncMembership
if err := rows.Scan(&m.RoomID, &m.Membership, &m.EventID, &m.StreamOrdering); err != nil {
return nil, fmt.Errorf("read the memberships of %q: %w", userID, err)
}
out = append(out, m)
}
return out, rows.Err()
}
// UserMembershipEvents implements storage.RoomStore.
func (s *Rooms) UserMembershipEvents(ctx context.Context, roomID, userID string) ([]storage.StoredEvent, error) {
rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+`
FROM roomserver_events e
WHERE e.room_id = ? AND e.event_type = 'm.room.member' AND e.has_state_key AND e.state_key = ?
AND NOT e.outlier
ORDER BY e.stream_ordering`, roomID, userID)
if err != nil {
return nil, fmt.Errorf("read the membership events of %q in %q: %w", userID, roomID, err)
}
events, err := scanEvents(rows)
if err != nil {
return nil, fmt.Errorf("read the membership events of %q in %q: %w", userID, roomID, err)
}
return events, nil
}
// Timeline implements storage.RoomStore.
func (s *Rooms) Timeline(ctx context.Context, req storage.TimelineRequest) ([]storage.StoredEvent, error) {
if req.Limit <= 0 {
return nil, nil
}
conditions := []string{`e.room_id = ?`, `NOT e.outlier`}
args := []any{req.RoomID}
if req.Low != 0 {
conditions = append(conditions, `e.stream_ordering > ?`)
args = append(args, req.Low)
}
if req.High != 0 {
conditions = append(conditions, `e.stream_ordering <= ?`)
args = append(args, req.High)
}
order := `ASC`
if req.Direction == storage.DirectionBackward {
order = `DESC`
}
if clause, filterArgs := filterConditions(req.Filter); clause != "" {
conditions = append(conditions, clause)
args = append(args, filterArgs...)
}
args = append(args, req.Limit)
rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+`
FROM roomserver_events e
WHERE `+strings.Join(conditions, " AND ")+`
ORDER BY e.stream_ordering `+order+`
LIMIT ?`, args...)
if err != nil {
return nil, fmt.Errorf("read the timeline of %q: %w", req.RoomID, err)
}
events, err := scanEvents(rows)
if err != nil {
return nil, fmt.Errorf("read the timeline of %q: %w", req.RoomID, err)
}
return events, nil
}
// StreamEvents implements storage.RoomStore.
func (s *Rooms) StreamEvents(ctx context.Context, after, upTo int64, limit int) ([]storage.StoredEvent, error) {
if limit <= 0 {
return nil, nil
}
conditions := []string{`e.stream_ordering > ?`, `NOT e.outlier`}
args := []any{after}
if upTo != 0 {
conditions = append(conditions, `e.stream_ordering <= ?`)
args = append(args, upTo)
}
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+`
FROM roomserver_events e
WHERE `+strings.Join(conditions, " AND ")+`
ORDER BY e.stream_ordering
LIMIT ?`, args...)
if err != nil {
return nil, fmt.Errorf("read the event stream after %d: %w", after, err)
}
events, err := scanEvents(rows)
if err != nil {
return nil, fmt.Errorf("read the event stream after %d: %w", after, err)
}
return events, nil
}
// filterConditions renders the part of an event filter the store applies in
// SQL. The exclusions are ANDed in after the inclusions, which is what makes
// not_types win over types.
func filterConditions(f storage.TimelineFilter) (string, []any) {
return filterConditionsOn(`e.event_type`, `e.sender`, f)
}
// filterConditionsOn is filterConditions against named columns.
func filterConditionsOn(typeColumn, senderColumn string, f storage.TimelineFilter) (string, []any) {
var (
clauses []string
args []any
)
add := func(column string, patterns []string, negate bool) {
if len(patterns) == 0 {
return
}
clause, clauseArgs := patternMatch(column, patterns)
if negate {
clause = `NOT (` + clause + `)`
}
clauses = append(clauses, clause)
args = append(args, clauseArgs...)
}
add(typeColumn, f.Types, false)
add(typeColumn, f.NotTypes, true)
add(senderColumn, f.Senders, false)
add(senderColumn, f.NotSenders, true)
if len(clauses) == 0 {
return "", nil
}
return strings.Join(clauses, " AND "), args
}
// patternMatch renders "this column matches any of these patterns", where a
// trailing '*' matches any suffix.
func patternMatch(column string, patterns []string) (string, []any) {
var (
exact []string
wildcard []string
args []any
)
for _, p := range patterns {
if prefix, isWildcard := strings.CutSuffix(p, "*"); isWildcard {
wildcard = append(wildcard, prefix)
continue
}
exact = append(exact, p)
}
var parts []string
if len(exact) > 0 {
parts = append(parts, column+` IN `+inList)
args = append(args, jsonList(exact))
}
for _, prefix := range wildcard {
parts = append(parts, column+` LIKE ? ESCAPE '\'`)
args = append(args, escapeLike(prefix)+`%`)
}
return strings.Join(parts, " OR "), args
}
// RoomsWithEvents implements storage.RoomStore.
func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to int64) ([]string, error) {
if len(roomIDs) == 0 {
return nil, nil
}
query := `
SELECT DISTINCT room_id FROM roomserver_events
WHERE room_id IN ` + inList + ` AND NOT outlier`
args := []any{jsonList(roomIDs)}
if from != 0 {
query += ` AND stream_ordering > ?`
args = append(args, from)
}
if to != 0 {
query += ` AND stream_ordering <= ?`
args = append(args, to)
}
query += ` ORDER BY room_id`
out, err := s.queryStrings(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("find the rooms with events after %d: %w", from, err)
}
return out, nil
}
// LatestEventPositions implements storage.RoomStore.
func (s *Rooms) LatestEventPositions(ctx context.Context, roomIDs, types []string, upTo int64) (map[string]int64, error) {
if len(roomIDs) == 0 || len(types) == 0 {
return map[string]int64{}, nil
}
query := `
SELECT room_id, MAX(stream_ordering) FROM roomserver_events
WHERE room_id IN ` + inList + `
AND event_type IN ` + inList + ` AND NOT outlier`
args := []any{jsonList(roomIDs), jsonList(types)}
if upTo != 0 {
query += ` AND stream_ordering <= ?`
args = append(args, upTo)
}
query += ` GROUP BY room_id`
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err)
}
defer func() { _ = rows.Close() }()
out := make(map[string]int64, len(roomIDs))
for rows.Next() {
var (
roomID string
pos int64
)
if err := rows.Scan(&roomID, &pos); err != nil {
return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err)
}
out[roomID] = pos
}
return out, rows.Err()
}
// StateAtStream implements storage.RoomStore: the snapshot recorded against
// the last event at or before the position, plus that event itself when it
// is a state event.
func (s *Rooms) StateAtStream(ctx context.Context, roomID string, pos int64) ([]storage.StateEntry, error) {
if pos == 0 {
return nil, nil
}
var (
eventID string
eventType string
stateKey sql.NullString
snapshotID int64
)
err := s.db.QueryRowContext(ctx, `
SELECT e.event_id, e.event_type, CASE WHEN e.has_state_key THEN e.state_key END, e.state_before_id
FROM roomserver_events e
WHERE e.room_id = ? AND NOT e.outlier AND e.stream_ordering <= ?
ORDER BY e.stream_ordering DESC
LIMIT 1`, roomID, pos).
Scan(&eventID, &eventType, &stateKey, &snapshotID)
switch {
case errors.Is(err, sql.ErrNoRows):
return nil, nil
case err != nil:
return nil, fmt.Errorf("read the state of %q at %d: %w", roomID, pos, err)
}
var entries []storage.StateEntry
if snapshotID != 0 {
rows, qErr := s.db.QueryContext(ctx, `
SELECT se.event_type, se.state_key, se.event_id, ''
FROM roomserver_state_entries se
WHERE se.snapshot_id = ?
ORDER BY se.event_type, se.state_key`, snapshotID)
if qErr != nil {
return nil, fmt.Errorf("read the state of %q at %d: %w", roomID, pos, qErr)
}
if entries, err = scanStateEntries(rows); err != nil {
return nil, fmt.Errorf("read the state of %q at %d: %w", roomID, pos, err)
}
}
if stateKey.Valid {
entries = upsertStateEntry(entries, storage.StateEntry{
Type: eventType,
StateKey: stateKey.String,
EventID: eventID,
})
}
return entries, nil
}
// upsertStateEntry replaces the matching tuple in entries, or appends it.
func upsertStateEntry(entries []storage.StateEntry, entry storage.StateEntry) []storage.StateEntry {
for i, existing := range entries {
if existing.Type == entry.Type && existing.StateKey == entry.StateKey {
entries[i] = entry
return entries
}
}
return append(entries, entry)
}
+557
View File
@@ -0,0 +1,557 @@
package pocketbase
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Users is the PocketBase implementation of storage.UserStore.
//
// The reads are the SQL store's statements. The writes are record
// operations, and where the SQL store lets a primary key or an UPDATE's
// affected-row count answer a question — is this localpart taken, does
// this account exist — this one asks first: the write lock is held from
// the question to the batch, so the answer cannot change in between.
type Users struct{ *base }
const (
defaultAccountPageSize = 100
maxAccountPageSize = 1000
)
// CreateAccount implements storage.UserStore.
func (u *Users) CreateAccount(ctx context.Context, acc storage.NewAccount) (storage.Account, error) {
err := u.write(ctx, func(t *tx) error {
taken, err := t.exists("user_accounts", `localpart = ?`, acc.Localpart)
if err != nil {
return err
}
if taken {
return storage.ErrUserExists
}
t.put("user_accounts", rec{
"localpart": acc.Localpart,
"password_hash": acc.PasswordHash,
"display_name": acc.DisplayName,
"avatar_url": "",
"admin": acc.Admin,
"deactivated": false,
"created_at_ms": acc.CreatedAtMS,
})
return nil
})
if err != nil {
return storage.Account{}, fmt.Errorf("create account %q: %w", acc.Localpart, err)
}
return storage.Account{
Localpart: acc.Localpart,
DisplayName: acc.DisplayName,
Admin: acc.Admin,
CreatedAtMS: acc.CreatedAtMS,
}, nil
}
const selectAccountSQL = `
SELECT localpart, display_name, avatar_url, admin, deactivated, created_at_ms
FROM user_accounts WHERE localpart = ?`
// Account implements storage.UserStore.
func (u *Users) Account(ctx context.Context, localpart string) (storage.Account, error) {
var acc storage.Account
err := u.db.QueryRowContext(ctx, selectAccountSQL, localpart).Scan(
&acc.Localpart, &acc.DisplayName, &acc.AvatarURL,
&acc.Admin, &acc.Deactivated, &acc.CreatedAtMS)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.Account{}, fmt.Errorf("account %q: %w", localpart, storage.ErrNoSuchUser)
case err != nil:
return storage.Account{}, fmt.Errorf("account %q: %w", localpart, err)
}
return acc, nil
}
// PasswordHash implements storage.UserStore.
func (u *Users) PasswordHash(ctx context.Context, localpart string) (string, error) {
var hash string
err := u.db.QueryRowContext(ctx, `SELECT password_hash FROM user_accounts WHERE localpart = ?`, localpart).Scan(&hash)
switch {
case errors.Is(err, sql.ErrNoRows):
return "", fmt.Errorf("password for %q: %w", localpart, storage.ErrNoSuchUser)
case err != nil:
return "", fmt.Errorf("password for %q: %w", localpart, err)
}
return hash, nil
}
// SetPasswordHash implements storage.UserStore.
func (u *Users) SetPasswordHash(ctx context.Context, localpart, hash string) error {
return u.updateAccount(ctx, localpart, rec{"password_hash": hash})
}
// SetAdmin implements storage.UserStore.
func (u *Users) SetAdmin(ctx context.Context, localpart string, admin bool) error {
return u.updateAccount(ctx, localpart, rec{"admin": admin})
}
// SetDeactivated implements storage.UserStore.
func (u *Users) SetDeactivated(ctx context.Context, localpart string, deactivated bool) error {
return u.updateAccount(ctx, localpart, rec{"deactivated": deactivated})
}
// SetDisplayName implements storage.UserStore.
func (u *Users) SetDisplayName(ctx context.Context, localpart, displayName string) error {
return u.updateAccount(ctx, localpart, rec{"display_name": displayName})
}
// SetAvatarURL implements storage.UserStore.
func (u *Users) SetAvatarURL(ctx context.Context, localpart, avatarURL string) error {
return u.updateAccount(ctx, localpart, rec{"avatar_url": avatarURL})
}
// updateAccount changes columns of one account, returning ErrNoSuchUser for
// an account that is not there.
func (u *Users) updateAccount(ctx context.Context, localpart string, row rec) error {
err := u.write(ctx, func(t *tx) error {
n, err := t.setWhere("user_accounts", row, `localpart = ?`, localpart)
if err != nil {
return err
}
if n == 0 {
return storage.ErrNoSuchUser
}
return nil
})
if err != nil {
return fmt.Errorf("update account %q: %w", localpart, err)
}
return nil
}
// likeEscape is the escape character the search patterns use.
const likeEscape = '!'
const (
countAccountsSQL = `
SELECT COUNT(*) FROM user_accounts
WHERE LOWER(localpart) LIKE ? ESCAPE '!' OR LOWER(display_name) LIKE ? ESCAPE '!'`
listAccountsSQL = `
SELECT localpart, display_name, avatar_url, admin, deactivated, created_at_ms
FROM user_accounts
WHERE LOWER(localpart) LIKE ? ESCAPE '!' OR LOWER(display_name) LIKE ? ESCAPE '!'
ORDER BY localpart
LIMIT ? OFFSET ?`
)
// ListAccounts implements storage.UserStore.
func (u *Users) ListAccounts(ctx context.Context, filter storage.AccountFilter) ([]storage.Account, int, error) {
pattern := likeContains(filter.Search)
limit := filter.Limit
switch {
case limit <= 0:
limit = defaultAccountPageSize
case limit > maxAccountPageSize:
limit = maxAccountPageSize
}
offset := max(filter.Offset, 0)
var total int
if err := u.db.QueryRowContext(ctx, countAccountsSQL, pattern, pattern).Scan(&total); err != nil {
return nil, 0, fmt.Errorf("count accounts: %w", err)
}
rows, err := u.db.QueryContext(ctx, listAccountsSQL, pattern, pattern, limit, offset)
if err != nil {
return nil, 0, fmt.Errorf("list accounts: %w", err)
}
defer func() { _ = rows.Close() }()
accounts := []storage.Account{}
for rows.Next() {
var acc storage.Account
if err := rows.Scan(&acc.Localpart, &acc.DisplayName, &acc.AvatarURL,
&acc.Admin, &acc.Deactivated, &acc.CreatedAtMS); err != nil {
return nil, 0, fmt.Errorf("scan account: %w", err)
}
accounts = append(accounts, acc)
}
if err := rows.Err(); err != nil {
return nil, 0, fmt.Errorf("list accounts: %w", err)
}
return accounts, total, nil
}
// likeContains turns a search term into a LIKE pattern matching it
// anywhere, with the wildcards and the escape character escaped.
func likeContains(search string) string {
search = strings.ToLower(strings.TrimSpace(search))
var b strings.Builder
b.Grow(len(search) + 2)
b.WriteByte('%')
for _, r := range search {
if r == '%' || r == '_' || r == likeEscape {
b.WriteByte(likeEscape)
}
b.WriteRune(r)
}
b.WriteByte('%')
return b.String()
}
const statsSQL = `
SELECT
(SELECT COUNT(*) FROM user_accounts),
(SELECT COUNT(*) FROM user_accounts WHERE admin = ?),
(SELECT COUNT(*) FROM user_accounts WHERE deactivated = ?),
(SELECT COUNT(*) FROM user_devices)`
// Stats implements storage.UserStore.
func (u *Users) Stats(ctx context.Context) (storage.Stats, error) {
var s storage.Stats
err := u.db.QueryRowContext(ctx, statsSQL, true, true).
Scan(&s.Users, &s.Admins, &s.Deactivated, &s.Devices)
if err != nil {
return storage.Stats{}, fmt.Errorf("user stats: %w", err)
}
return s, nil
}
// CreateSession implements storage.UserStore.
//
// Logging in on a device ID that already exists keeps the device and its
// display name but revokes the tokens it held, which is what the spec
// requires.
func (u *Users) CreateSession(ctx context.Context, s storage.NewSession) (storage.Device, error) {
device := storage.Device{
Localpart: s.Localpart,
ID: s.DeviceID,
DisplayName: s.DisplayName,
CreatedAtMS: s.CreatedAtMS,
LastSeenMS: s.CreatedAtMS,
LastSeenIP: s.LastSeenIP,
UserAgent: s.UserAgent,
}
err := u.write(ctx, func(t *tx) error {
found, err := t.exists("user_accounts", `localpart = ?`, s.Localpart)
if err != nil {
return fmt.Errorf("create session for %q: %w", s.Localpart, err)
}
if !found {
return fmt.Errorf("create session for %q: %w", s.Localpart, storage.ErrNoSuchUser)
}
var existingName string
var existingCreated int64
err = t.queryRow(`SELECT display_name, created_at_ms FROM user_devices
WHERE localpart = ? AND device_id = ?`, s.Localpart, s.DeviceID).
Scan(&existingName, &existingCreated)
switch {
case errors.Is(err, sql.ErrNoRows):
t.put("user_devices", rec{
"localpart": s.Localpart,
"device_id": s.DeviceID,
"display_name": s.DisplayName,
"created_at_ms": s.CreatedAtMS,
"last_seen_ms": s.CreatedAtMS,
"last_seen_ip": s.LastSeenIP,
"user_agent": s.UserAgent,
})
case err != nil:
return fmt.Errorf("read device %q: %w", s.DeviceID, err)
default:
device.DisplayName = existingName
device.CreatedAtMS = existingCreated
t.put("user_devices", rec{
"localpart": s.Localpart,
"device_id": s.DeviceID,
"last_seen_ms": s.CreatedAtMS,
"last_seen_ip": s.LastSeenIP,
"user_agent": s.UserAgent,
})
if err := revokeDeviceTokens(t, `localpart = ? AND device_id = ?`, s.Localpart, s.DeviceID); err != nil {
return err
}
}
t.put("user_access_tokens", rec{
"token_hash": s.AccessTokenHash,
"localpart": s.Localpart,
"device_id": s.DeviceID,
"created_at_ms": s.CreatedAtMS,
"expires_at_ms": s.AccessTokenExpiresAtMS,
})
if s.RefreshTokenHash != "" {
t.put("user_refresh_tokens", rec{
"token_hash": s.RefreshTokenHash,
"localpart": s.Localpart,
"device_id": s.DeviceID,
"created_at_ms": s.CreatedAtMS,
"expires_at_ms": s.RefreshTokenExpiresAtMS,
})
}
return nil
})
if err != nil {
return storage.Device{}, err
}
return device, nil
}
// revokeDeviceTokens deletes the access and refresh tokens matching a
// condition on (localpart, device_id).
func revokeDeviceTokens(t *tx, where string, args ...any) error {
for _, table := range []string{"user_access_tokens", "user_refresh_tokens"} {
if _, err := t.delWhere(table, where, args...); err != nil {
return fmt.Errorf("revoke tokens: %w", err)
}
}
return nil
}
// RefreshSession implements storage.UserStore.
//
// The spent refresh token and the access token it had minted both go away
// immediately.
func (u *Users) RefreshSession(ctx context.Context, r storage.RefreshedSession) (storage.TokenInfo, error) {
var info storage.TokenInfo
var expired bool
err := u.write(ctx, func(t *tx) error {
var expiresAt int64
err := t.queryRow(`SELECT localpart, device_id, expires_at_ms FROM user_refresh_tokens WHERE token_hash = ?`,
r.RefreshTokenHash).Scan(&info.Localpart, &info.DeviceID, &expiresAt)
switch {
case errors.Is(err, sql.ErrNoRows):
return fmt.Errorf("refresh token: %w", storage.ErrNoSuchToken)
case err != nil:
return fmt.Errorf("refresh token: %w", err)
}
if _, err := t.delKey("user_refresh_tokens", r.RefreshTokenHash); err != nil {
return fmt.Errorf("spend refresh token: %w", err)
}
if expiresAt != 0 && expiresAt <= r.RefreshedAtMS {
// The dead token is cleaned up on the way past: the batch is
// still sent, and the error reported after it.
expired = true
return nil
}
if _, err := t.delWhere("user_access_tokens", `localpart = ? AND device_id = ?`,
info.Localpart, info.DeviceID); err != nil {
return fmt.Errorf("revoke previous access token: %w", err)
}
t.put("user_access_tokens", rec{
"token_hash": r.NewAccessTokenHash,
"localpart": info.Localpart,
"device_id": info.DeviceID,
"created_at_ms": r.RefreshedAtMS,
"expires_at_ms": r.AccessTokenExpiresAtMS,
})
if r.NewRefreshTokenHash != "" {
t.put("user_refresh_tokens", rec{
"token_hash": r.NewRefreshTokenHash,
"localpart": info.Localpart,
"device_id": info.DeviceID,
"created_at_ms": r.RefreshedAtMS,
"expires_at_ms": r.RefreshTokenExpiresAtMS,
})
}
if err := t.queryRow(`SELECT admin, deactivated FROM user_accounts WHERE localpart = ?`, info.Localpart).
Scan(&info.Admin, &info.Deactivated); err != nil {
return fmt.Errorf("read account %q: %w", info.Localpart, err)
}
info.ExpiresAtMS = r.AccessTokenExpiresAtMS
return nil
})
if err != nil {
return storage.TokenInfo{}, err
}
if expired {
return storage.TokenInfo{}, fmt.Errorf("refresh token has expired: %w", storage.ErrNoSuchToken)
}
return info, nil
}
const selectTokenInfoSQL = `
SELECT t.localpart, t.device_id, t.expires_at_ms, a.admin, a.deactivated
FROM user_access_tokens t
JOIN user_accounts a ON a.localpart = t.localpart
WHERE t.token_hash = ?`
// TokenInfo implements storage.UserStore.
func (u *Users) TokenInfo(ctx context.Context, tokenHash string) (storage.TokenInfo, error) {
var info storage.TokenInfo
err := u.db.QueryRowContext(ctx, selectTokenInfoSQL, tokenHash).Scan(
&info.Localpart, &info.DeviceID, &info.ExpiresAtMS, &info.Admin, &info.Deactivated)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.TokenInfo{}, storage.ErrNoSuchToken
case err != nil:
return storage.TokenInfo{}, fmt.Errorf("look up access token: %w", err)
}
return info, nil
}
// TouchDevice implements storage.UserStore. A device that has gone away
// between authenticating and recording the visit is not an error.
func (u *Users) TouchDevice(ctx context.Context, localpart, deviceID, ip, userAgent string, seenAtMS int64) error {
err := u.write(ctx, func(t *tx) error {
_, err := t.setWhere("user_devices", rec{
"last_seen_ms": seenAtMS,
"last_seen_ip": ip,
"user_agent": userAgent,
}, `localpart = ? AND device_id = ?`, localpart, deviceID)
return err
})
if err != nil {
return fmt.Errorf("touch device %q: %w", deviceID, err)
}
return nil
}
const deviceColumns = `localpart, device_id, display_name, created_at_ms, last_seen_ms, last_seen_ip, user_agent`
// Device implements storage.UserStore.
func (u *Users) Device(ctx context.Context, localpart, deviceID string) (storage.Device, error) {
var d storage.Device
err := u.db.QueryRowContext(ctx, `SELECT `+deviceColumns+`
FROM user_devices WHERE localpart = ? AND device_id = ?`, localpart, deviceID).Scan(
&d.Localpart, &d.ID, &d.DisplayName, &d.CreatedAtMS,
&d.LastSeenMS, &d.LastSeenIP, &d.UserAgent)
switch {
case errors.Is(err, sql.ErrNoRows):
return storage.Device{}, fmt.Errorf("device %q: %w", deviceID, storage.ErrNoSuchDevice)
case err != nil:
return storage.Device{}, fmt.Errorf("device %q: %w", deviceID, err)
}
return d, nil
}
// Devices implements storage.UserStore.
func (u *Users) Devices(ctx context.Context, localpart string) ([]storage.Device, error) {
rows, err := u.db.QueryContext(ctx, `SELECT `+deviceColumns+`
FROM user_devices WHERE localpart = ? ORDER BY device_id`, localpart)
if err != nil {
return nil, fmt.Errorf("list devices of %q: %w", localpart, err)
}
defer func() { _ = rows.Close() }()
devices := []storage.Device{}
for rows.Next() {
var d storage.Device
if err := rows.Scan(&d.Localpart, &d.ID, &d.DisplayName, &d.CreatedAtMS,
&d.LastSeenMS, &d.LastSeenIP, &d.UserAgent); err != nil {
return nil, fmt.Errorf("scan device: %w", err)
}
devices = append(devices, d)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list devices of %q: %w", localpart, err)
}
return devices, nil
}
// SetDeviceDisplayName implements storage.UserStore.
func (u *Users) SetDeviceDisplayName(ctx context.Context, localpart, deviceID, displayName string) error {
err := u.write(ctx, func(t *tx) error {
n, err := t.setWhere("user_devices", rec{"display_name": displayName},
`localpart = ? AND device_id = ?`, localpart, deviceID)
if err != nil {
return err
}
if n == 0 {
return storage.ErrNoSuchDevice
}
return nil
})
if err != nil {
return fmt.Errorf("rename device %q: %w", deviceID, err)
}
return nil
}
// DeleteDevices implements storage.UserStore.
func (u *Users) DeleteDevices(ctx context.Context, localpart string, deviceIDs []string) (int, error) {
if len(deviceIDs) == 0 {
return 0, nil
}
var deleted int
err := u.write(ctx, func(t *tx) error {
where := `localpart = ? AND device_id IN (SELECT value FROM json_each(?))`
list := jsonList(deviceIDs)
if err := revokeDeviceTokens(t, where, localpart, list); err != nil {
return err
}
n, err := t.delWhere("user_devices", where, localpart, list)
if err != nil {
return fmt.Errorf("delete devices: %w", err)
}
deleted = n
return nil
})
return deleted, err
}
// DeleteAllDevices implements storage.UserStore.
func (u *Users) DeleteAllDevices(ctx context.Context, localpart string) (int, error) {
var deleted int
err := u.write(ctx, func(t *tx) error {
if err := revokeDeviceTokens(t, `localpart = ?`, localpart); err != nil {
return err
}
n, err := t.delWhere("user_devices", `localpart = ?`, localpart)
if err != nil {
return fmt.Errorf("delete devices of %q: %w", localpart, err)
}
deleted = n
return nil
})
return deleted, err
}
// AccountData implements storage.UserStore.
func (u *Users) AccountData(ctx context.Context, localpart, roomID, dataType string) ([]byte, error) {
var content string
err := u.db.QueryRowContext(ctx, `SELECT content FROM user_account_data
WHERE localpart = ? AND room_id = ? AND data_type = ?`, localpart, roomID, dataType).Scan(&content)
switch {
case errors.Is(err, sql.ErrNoRows):
return nil, fmt.Errorf("account data %q: %w", dataType, storage.ErrNoSuchAccountData)
case err != nil:
return nil, fmt.Errorf("account data %q: %w", dataType, err)
}
return []byte(content), nil
}
// SetAccountData implements storage.UserStore. The write stamps the row with
// a position in the account data stream, in the same batch as the content.
func (u *Users) SetAccountData(ctx context.Context, localpart, roomID, dataType string, content []byte) error {
err := u.write(ctx, func(t *tx) error {
pos, err := t.next(syncSequences, seqAccountData)
if err != nil {
return err
}
t.put("user_account_data", rec{
"localpart": localpart,
"room_id": roomID,
"data_type": dataType,
"content": string(content),
"stream_pos": pos,
})
return nil
})
if err != nil {
return fmt.Errorf("set account data %q: %w", dataType, err)
}
return nil
}
var _ storage.UserStore = (*Users)(nil)
+7 -1
View File
@@ -27,7 +27,7 @@ const (
)
// ErrNotImplemented is returned by a driver that is compiled in but not
// finished — currently PocketBase.
// finished.
var ErrNotImplemented = errors.New("not implemented by this storage driver")
// ErrNoSuchSetting is returned by SettingsStore.Get for an unknown key.
@@ -41,6 +41,12 @@ type Config struct {
MaxIdleConns int
ConnMaxLifetime time.Duration
// SuperuserEmail and SuperuserPassword are the account the pocketbase
// driver signs in to its server with. Only the pocketbase driver reads
// them.
SuperuserEmail string
SuperuserPassword string
// S3 is the bucket a driver with its own file storage puts media in.
// Only the pocketbase driver reads it.
S3 S3Config
+22 -6
View File
@@ -11,11 +11,13 @@ import (
"github.com/dariuszjaniak/warp/server/internal/storage"
_ "github.com/dariuszjaniak/warp/server/internal/storage/drivers"
"github.com/dariuszjaniak/warp/server/internal/storage/pocketbase/pbtest"
)
// openTestDB opens each driver that can run without external services. The
// postgres driver joins in when WARP_TEST_POSTGRES_DSN is set, which is how
// CI runs the same tests against both engines.
// CI runs the same tests against both engines; the pocketbase driver gets a
// real PocketBase server of its own, run in-process.
func openTestDB(t *testing.T, driver string) storage.Database {
t.Helper()
@@ -31,7 +33,9 @@ func openTestDB(t *testing.T, driver string) storage.Database {
}
cfg.ConnectionString = dsn
case storage.DriverPocketBase:
cfg.ConnectionString = filepath.Join(t.TempDir(), "pb_data")
cfg.ConnectionString = pbtest.Start(t).URL
cfg.SuperuserEmail = pbtest.SuperuserEmail
cfg.SuperuserPassword = pbtest.SuperuserPassword
default:
t.Fatalf("no test setup for driver %q", driver)
}
@@ -45,7 +49,7 @@ func openTestDB(t *testing.T, driver string) storage.Database {
}
// testDrivers are run by every storage test. PocketBase is here because
// parity with sqlite is its whole contract.
// parity with the SQL drivers is its whole contract.
var testDrivers = []string{storage.DriverSQLite, storage.DriverPostgres, storage.DriverPocketBase}
func TestDriversRegistered(t *testing.T) {
@@ -64,14 +68,26 @@ func TestOpenUnknownDriver(t *testing.T) {
}
}
func TestPocketBaseNeedsADataDirectory(t *testing.T) {
func TestPocketBaseNeedsAServer(t *testing.T) {
if _, err := storage.Open(t.Context(), storage.Config{Driver: storage.DriverPocketBase}, nil); err == nil {
t.Fatal("expected an error for a pocketbase driver with no data directory")
t.Fatal("expected an error for a pocketbase driver with no server URL")
}
_, err := storage.Open(t.Context(), storage.Config{
Driver: storage.DriverPocketBase,
ConnectionString: pbtest.Start(t).URL,
SuperuserEmail: pbtest.SuperuserEmail,
SuperuserPassword: "not-the-password",
}, nil)
if err == nil {
t.Fatal("expected an error for a superuser password the server refuses")
}
}
func TestPocketBaseKeepsMediaInItsFilesystem(t *testing.T) {
func TestPocketBaseKeepsMediaInItsStorage(t *testing.T) {
db := openTestDB(t, storage.DriverPocketBase)
if err := db.Migrate(t.Context()); err != nil {
t.Fatalf("Migrate: %v", err)
}
provider, ok := db.(storage.BlobProvider)
if !ok {
t.Fatal("the pocketbase driver does not provide its own blob store")