tajniak81andClaude Opus 5.5
a47e15cc1c
CI / Test (ubuntu-latest) (push) Waiting to run
CI / Test (windows-latest) (push) Waiting to run
CI / Storage tests on PostgreSQL (push) Waiting to run
CI / Complement (federation off) (push) Waiting to run
CI / golangci-lint (push) Waiting to run
CI / Smoke test the binary (push) Waiting to run
CI / Admin panel (push) Waiting to run
CI / Web client (push) Waiting to run
CI / Desktop client (ubuntu-24.04) (push) Waiting to run
CI / Desktop client (windows-latest) (push) Waiting to run
The pocketbase driver no longer embeds PocketBase: Warp keeps its data as PocketBase collections on a server of its own and reaches it only through the REST API, the way the postgres driver reaches PostgreSQL. database.connection_string is the server's URL, and Warp signs in as the superuser in WARP_PB_SUPERUSER_EMAIL / WARP_PB_SUPERUSER_PASSWORD (or a WARP_PB_SUPERUSER_FILE of the two). Every SQL table is a superuser-only collection of the same name and columns, created by warp migrate. Reads go through PocketBase's SQL endpoint with the SQL drivers' queries, via a read-only database/sql driver that inlines arguments, moves long ones into warp_params and pages past 1000 rows. Writes are record operations in batch requests under one write lock in the Warp process. Media is a protected file field in warp_media, local or in the S3 bucket Warp writes into PocketBase's settings. Search matches word prefixes with LIKE. The storage tests run against an in-process PocketBase and pass as on SQLite. Docker: the PocketBase stacks get a pocketbase service running tajniak81/pocketbase-docker-root (WARP_PB_IMAGE), with a pb-data volume and its admin UI on loopback; the AIO images run that PocketBase binary as a process beside the homeserver, making up a superuser when none is set. The .env examples, READMEs, docs and warp.yaml.example follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
450 lines
16 KiB
Go
450 lines
16 KiB
Go
package pocketbase
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/base32"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// The schema is the SQL drivers' tables as PocketBase collections: the same
|
|
// names and columns, so that the reads can be the same SQL, with three
|
|
// differences PocketBase forces.
|
|
//
|
|
// - Every record has a text `id`, which is the primary key. Warp's own keys
|
|
// are kept as ordinary fields under a unique index, and a record's id is
|
|
// derived from them (see rowID): a row is always addressed by its key.
|
|
// - Nothing can be NULL. The two nullable state_key columns and
|
|
// admin_registration_tokens.uses_allowed get a has_* flag beside them.
|
|
// - `id` is taken, so push_notifications.id is notification_id and
|
|
// admin_audit_log.id is audit_id.
|
|
//
|
|
// Full-text search has no FTS5 here — it is a plain collection searched with
|
|
// LIKE — and media content lives in warp_media's file field, which is
|
|
// PocketBase's own storage: its pb_data/storage or an S3 bucket.
|
|
|
|
type fieldKind int
|
|
|
|
const (
|
|
text fieldKind = iota
|
|
number
|
|
boolean
|
|
file
|
|
)
|
|
|
|
type field struct {
|
|
name string
|
|
kind fieldKind
|
|
}
|
|
|
|
// table is one collection.
|
|
type table struct {
|
|
name string
|
|
|
|
// key is the primary key the SQL drivers declare; the record id is a
|
|
// hash of its values. Empty for a collection whose ids are chosen by
|
|
// the writer (warp_params).
|
|
key []string
|
|
|
|
fields []field
|
|
|
|
// indexes are column lists; a leading "unique:" makes one unique.
|
|
indexes []string
|
|
}
|
|
|
|
func tbl(name string, key []string, fields []field, indexes ...string) table {
|
|
return table{name: name, key: key, fields: fields, indexes: indexes}
|
|
}
|
|
|
|
func pk(cols ...string) []string { return cols }
|
|
|
|
func fld(name string, kind fieldKind) field { return field{name: name, kind: kind} }
|
|
|
|
func texts(names ...string) []field {
|
|
out := make([]field, len(names))
|
|
for i, n := range names {
|
|
out[i] = field{name: n, kind: text}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func numbers(names ...string) []field {
|
|
out := make([]field, len(names))
|
|
for i, n := range names {
|
|
out[i] = field{name: n, kind: number}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func fields(groups ...[]field) []field {
|
|
var out []field
|
|
for _, g := range groups {
|
|
out = append(out, g...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func sequenceTable(name string) table {
|
|
return tbl(name, pk("name"), fields(texts("name"), numbers("value")))
|
|
}
|
|
|
|
// The sequence rows each table starts with, as the SQL migrations insert
|
|
// them.
|
|
var sequenceRows = map[string][]string{
|
|
"roomserver_sequences": {"stream_ordering", "state_snapshot", "backfill_ordering"},
|
|
"sync_sequences": {"receipt", "to_device", "account_data", "presence"},
|
|
"federation_sequences": {"queue"},
|
|
"keyserver_sequences": {"device_list"},
|
|
"push_sequences": {"notification"},
|
|
"admin_sequences": {"audit"},
|
|
}
|
|
|
|
var schema = []table{
|
|
tbl("warp_meta", pk("meta_key"), texts("meta_key", "meta_value")),
|
|
{name: paramsCollection, fields: texts("value")},
|
|
tbl("warp_media", pk("sha256"), fields(texts("sha256"), numbers("size"), []field{fld("content", file)})),
|
|
|
|
tbl("server_settings", pk("setting_key"), texts("setting_key", "setting_value", "updated_at")),
|
|
|
|
// userapi
|
|
tbl("user_accounts", pk("localpart"), fields(
|
|
texts("localpart", "password_hash", "display_name", "avatar_url"),
|
|
[]field{fld("admin", boolean), fld("deactivated", boolean)},
|
|
numbers("created_at_ms"))),
|
|
tbl("user_devices", pk("localpart", "device_id"), fields(
|
|
texts("localpart", "device_id", "display_name"),
|
|
numbers("created_at_ms", "last_seen_ms"),
|
|
texts("last_seen_ip", "user_agent"))),
|
|
tbl("user_access_tokens", pk("token_hash"), fields(
|
|
texts("token_hash", "localpart", "device_id"), numbers("created_at_ms", "expires_at_ms")),
|
|
"localpart, device_id"),
|
|
tbl("user_refresh_tokens", pk("token_hash"), fields(
|
|
texts("token_hash", "localpart", "device_id"), numbers("created_at_ms", "expires_at_ms")),
|
|
"localpart, device_id"),
|
|
tbl("user_account_data", pk("localpart", "room_id", "data_type"), fields(
|
|
texts("localpart", "room_id", "data_type", "content"), numbers("stream_pos")),
|
|
"localpart, stream_pos"),
|
|
|
|
// roomserver
|
|
tbl("roomserver_rooms", pk("room_id"), fields(
|
|
texts("room_id", "room_version", "creator"),
|
|
[]field{fld("is_published", boolean)},
|
|
numbers("created_at_ms"),
|
|
texts("name", "topic", "avatar_url", "canonical_alias",
|
|
"join_rule", "history_visibility", "guest_access", "room_type"),
|
|
numbers("joined_members")),
|
|
"is_published, joined_members"),
|
|
sequenceTable("roomserver_sequences"),
|
|
tbl("roomserver_state_snapshots", pk("snapshot_id"), fields(
|
|
numbers("snapshot_id"), texts("room_id", "state_hash")),
|
|
"unique: room_id, state_hash"),
|
|
tbl("roomserver_state_entries", pk("snapshot_id", "event_type", "state_key"), fields(
|
|
numbers("snapshot_id"), texts("event_type", "state_key", "event_id"))),
|
|
tbl("roomserver_events", pk("event_id"), fields(
|
|
texts("event_id", "room_id"),
|
|
numbers("stream_ordering"),
|
|
texts("event_type", "state_key"),
|
|
[]field{fld("has_state_key", boolean)},
|
|
texts("sender"),
|
|
numbers("depth", "origin_server_ts"),
|
|
texts("event_json"),
|
|
numbers("state_before_id"),
|
|
texts("redacted_by"),
|
|
[]field{fld("outlier", boolean)},
|
|
texts("relates_to_id", "rel_type")),
|
|
"unique: stream_ordering",
|
|
"room_id, stream_ordering",
|
|
"room_id, relates_to_id, rel_type, stream_ordering",
|
|
"event_type, state_key"),
|
|
tbl("roomserver_extremities", pk("room_id", "event_id"), texts("room_id", "event_id")),
|
|
tbl("roomserver_current_state", pk("room_id", "event_type", "state_key"),
|
|
texts("room_id", "event_type", "state_key", "event_id", "membership"),
|
|
"event_type, state_key, membership"),
|
|
tbl("roomserver_transactions", pk("localpart", "device_id", "txn_id"), fields(
|
|
texts("localpart", "device_id", "txn_id", "event_id", "room_id"), numbers("sent_at_ms"))),
|
|
tbl("roomserver_aliases", pk("alias"), fields(
|
|
texts("alias", "room_id", "creator"), numbers("created_at_ms")),
|
|
"room_id"),
|
|
tbl("roomserver_invite_state", pk("room_id", "user_id"), texts("room_id", "user_id", "state_json")),
|
|
tbl("roomserver_event_search", pk("stream_ordering"), fields(
|
|
numbers("stream_ordering"),
|
|
texts("room_id", "event_id", "search_key", "sender", "event_type", "body")),
|
|
"room_id, stream_ordering"),
|
|
|
|
// syncapi
|
|
sequenceTable("sync_sequences"),
|
|
tbl("sync_receipts", pk("room_id", "user_id", "receipt_type", "thread_id"), fields(
|
|
texts("room_id", "user_id", "receipt_type", "thread_id", "event_id"), numbers("ts_ms", "stream_pos")),
|
|
"stream_pos", "room_id, stream_pos"),
|
|
tbl("sync_to_device", pk("stream_pos"), fields(
|
|
numbers("stream_pos"),
|
|
texts("localpart", "device_id", "sender", "event_type", "content"),
|
|
numbers("sent_at_ms")),
|
|
"localpart, device_id, stream_pos"),
|
|
tbl("sync_presence", pk("user_id"), fields(
|
|
texts("user_id", "presence", "status_msg"), numbers("last_active_ms", "stream_pos")),
|
|
"stream_pos"),
|
|
tbl("sync_filters", pk("localpart", "filter_id"), fields(
|
|
texts("localpart", "filter_id", "filter_json"), numbers("created_at_ms"))),
|
|
|
|
// federation
|
|
tbl("federation_server_keys", pk("server_name", "key_id"), fields(
|
|
texts("server_name", "key_id", "public_key"),
|
|
numbers("valid_until_ts", "expired_ts", "fetched_at_ms"))),
|
|
sequenceTable("federation_sequences"),
|
|
tbl("federation_queue", pk("queue_id"), fields(
|
|
numbers("queue_id"),
|
|
texts("destination", "kind", "room_id", "event_id", "payload"),
|
|
numbers("queued_at_ms")),
|
|
"destination, kind, queue_id"),
|
|
tbl("federation_destinations", pk("server_name"), fields(
|
|
texts("server_name"),
|
|
numbers("failures", "retry_at_ms", "last_success_ms", "last_failure_ms"),
|
|
texts("last_error"))),
|
|
tbl("federation_inbound_txns", pk("origin", "txn_id"), fields(
|
|
texts("origin", "txn_id", "response_json"), numbers("received_at_ms"))),
|
|
|
|
// mediaapi
|
|
tbl("media_repository", pk("media_origin", "media_id"), fields(
|
|
texts("media_origin", "media_id", "content_type", "upload_name"),
|
|
numbers("file_size"),
|
|
texts("sha256", "user_id"),
|
|
numbers("created_at_ms"),
|
|
[]field{fld("pending", boolean)},
|
|
numbers("expires_at_ms")),
|
|
"pending, user_id, expires_at_ms", "user_id, media_origin", "sha256"),
|
|
tbl("media_thumbnails", pk("media_origin", "media_id", "width", "height", "method"), fields(
|
|
texts("media_origin", "media_id"),
|
|
numbers("width", "height"),
|
|
texts("method", "content_type"),
|
|
numbers("file_size"),
|
|
texts("sha256"),
|
|
numbers("created_at_ms")),
|
|
"sha256"),
|
|
tbl("media_quotas", pk("user_id"), fields(texts("user_id"), numbers("max_bytes"))),
|
|
|
|
// keyserver
|
|
sequenceTable("keyserver_sequences"),
|
|
tbl("keyserver_device_keys", pk("user_id", "device_id"), texts("user_id", "device_id", "key_json")),
|
|
tbl("keyserver_one_time_keys", pk("user_id", "device_id", "key_id"),
|
|
texts("user_id", "device_id", "key_id", "algorithm", "key_json"),
|
|
"user_id, device_id, algorithm"),
|
|
tbl("keyserver_fallback_keys", pk("user_id", "device_id", "algorithm"), fields(
|
|
texts("user_id", "device_id", "algorithm", "key_id", "key_json"),
|
|
[]field{fld("used", boolean)})),
|
|
tbl("keyserver_cross_signing_keys", pk("user_id", "purpose"), texts("user_id", "purpose", "key_json")),
|
|
tbl("keyserver_signatures", pk("target_user_id", "target_key_id", "signer_user_id", "signer_key_id"),
|
|
texts("target_user_id", "target_key_id", "signer_user_id", "signer_key_id", "signature")),
|
|
tbl("keyserver_device_lists", pk("user_id"), fields(texts("user_id"), numbers("stream_pos")),
|
|
"stream_pos"),
|
|
tbl("keyserver_backup_versions", pk("user_id", "version"), fields(
|
|
texts("user_id"), numbers("version"), texts("algorithm", "auth_data"), numbers("etag"),
|
|
[]field{fld("deleted", boolean)})),
|
|
tbl("keyserver_backup_keys", pk("user_id", "version", "room_id", "session_id"), fields(
|
|
texts("user_id"), numbers("version"), texts("room_id", "session_id"),
|
|
numbers("first_message_index", "forwarded_count"),
|
|
[]field{fld("is_verified", boolean)},
|
|
texts("session_data"))),
|
|
tbl("keyserver_remote_users", pk("user_id"), fields(
|
|
texts("user_id"), numbers("stream_id"), []field{fld("stale", boolean)})),
|
|
tbl("keyserver_remote_device_keys", pk("user_id", "device_id"),
|
|
texts("user_id", "device_id", "display_name", "key_json")),
|
|
tbl("keyserver_remote_cross_signing_keys", pk("user_id", "purpose"), texts("user_id", "purpose", "key_json")),
|
|
|
|
// push
|
|
sequenceTable("push_sequences"),
|
|
tbl("push_pushers", pk("localpart", "app_id", "pushkey"), fields(
|
|
texts("localpart", "app_id", "pushkey", "device_id", "kind", "app_display_name",
|
|
"device_display_name", "profile_tag", "lang", "data_json"),
|
|
numbers("pushkey_ts_ms")),
|
|
"app_id, pushkey"),
|
|
tbl("push_notifications", pk("notification_id"), fields(
|
|
numbers("notification_id"),
|
|
texts("user_id", "room_id", "event_id"),
|
|
numbers("event_pos", "ts_ms"),
|
|
[]field{fld("highlight", boolean)},
|
|
texts("actions"),
|
|
[]field{fld("is_read", boolean)},
|
|
texts("thread_id")),
|
|
"user_id, notification_id",
|
|
"user_id, room_id, is_read, event_pos",
|
|
"is_read, ts_ms",
|
|
"user_id, room_id, thread_id, is_read, event_pos"),
|
|
tbl("push_room_counts", pk("user_id", "room_id"), fields(
|
|
texts("user_id", "room_id"), numbers("notification_count", "highlight_count", "stream_pos")),
|
|
"user_id, stream_pos"),
|
|
|
|
// admin and application services
|
|
tbl("admin_registration_tokens", pk("token"), fields(
|
|
texts("token"),
|
|
numbers("uses_allowed"),
|
|
[]field{fld("has_uses_allowed", boolean)},
|
|
numbers("completed", "expires_at_ms", "created_at_ms"))),
|
|
tbl("admin_server_notice_rooms", pk("localpart"), texts("localpart", "room_id")),
|
|
sequenceTable("admin_sequences"),
|
|
tbl("admin_audit_log", pk("audit_id"), fields(
|
|
numbers("audit_id", "ts_ms"), texts("actor", "action", "target", "details_json")),
|
|
"actor, audit_id", "action, audit_id"),
|
|
tbl("appservice_state", pk("as_id"), fields(texts("as_id"), numbers("stream_pos", "txn_id", "txn_to"))),
|
|
|
|
// delayed events and OpenID tokens
|
|
tbl("delayed_events", pk("delay_id"), fields(
|
|
texts("delay_id", "user_id", "device_id", "room_id", "event_type", "state_key"),
|
|
[]field{fld("has_state_key", boolean)},
|
|
texts("content"),
|
|
numbers("delay_ms", "running_since_ms", "send_at_ms", "finalised_at_ms"),
|
|
texts("outcome", "reason", "event_id", "error")),
|
|
"user_id, finalised_at_ms",
|
|
"finalised_at_ms, send_at_ms",
|
|
"room_id, event_type, state_key, finalised_at_ms"),
|
|
tbl("openid_tokens", pk("token_hash"), fields(texts("token_hash", "user_id"), numbers("expires_at_ms")),
|
|
"expires_at_ms"),
|
|
}
|
|
|
|
// tables indexes the schema by collection name.
|
|
var tables = func() map[string]*table {
|
|
out := make(map[string]*table, len(schema))
|
|
for i := range schema {
|
|
out[schema[i].name] = &schema[i]
|
|
}
|
|
return out
|
|
}()
|
|
|
|
// maxText is the longest a text field may be. PocketBase's default is 5000
|
|
// characters, which an event can pass; this is far above the 64 KiB a
|
|
// Matrix event may be, and above any account data a client sends.
|
|
const maxText = 1 << 24
|
|
|
|
// maxFileSize bounds one media file. The media API enforces the configured
|
|
// upload limit before content ever gets here.
|
|
const maxFileSize = 1 << 40
|
|
|
|
// shortHash is a lower-case base32 digest, which fits PocketBase's id
|
|
// alphabet of [a-z0-9].
|
|
func shortHash(n int, parts ...string) string {
|
|
h := sha256.New()
|
|
for _, p := range parts {
|
|
_, _ = fmt.Fprintf(h, "%d:%s", len(p), p)
|
|
}
|
|
enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h.Sum(nil))
|
|
return strings.ToLower(enc)[:n]
|
|
}
|
|
|
|
// rowID is the record id of the row whose primary key has these values.
|
|
func rowID(collection string, key ...any) string {
|
|
parts := make([]string, 0, len(key)+1)
|
|
parts = append(parts, collection)
|
|
for _, v := range key {
|
|
parts = append(parts, keyString(v))
|
|
}
|
|
return shortHash(15, parts...)
|
|
}
|
|
|
|
func keyString(v any) string {
|
|
switch v := v.(type) {
|
|
case string:
|
|
return v
|
|
case int64:
|
|
return strconv.FormatInt(v, 10)
|
|
case int:
|
|
return strconv.Itoa(v)
|
|
case bool:
|
|
return strconv.FormatBool(v)
|
|
default:
|
|
return fmt.Sprint(v)
|
|
}
|
|
}
|
|
|
|
// collectionJSON renders a table as PocketBase's collection import format.
|
|
// Collection and field ids are derived from their names, so that importing
|
|
// the same schema again changes nothing, and a later version that adds a
|
|
// field adds just that field.
|
|
func (tb *table) collectionJSON() map[string]any {
|
|
out := []map[string]any{{
|
|
"id": "text3208210256", "name": "id", "type": "text", "system": true,
|
|
"primaryKey": true, "required": true, "hidden": false, "presentable": false,
|
|
"min": 15, "max": 15, "pattern": "^[a-z0-9]+$", "autogeneratePattern": "[a-z0-9]{15}",
|
|
}}
|
|
for _, fd := range tb.fields {
|
|
m := map[string]any{
|
|
"id": "f" + shortHash(12, tb.name, fd.name),
|
|
"name": fd.name,
|
|
"system": false, "required": false, "hidden": false, "presentable": false,
|
|
}
|
|
switch fd.kind {
|
|
case text:
|
|
m["type"] = "text"
|
|
m["min"] = 0
|
|
m["max"] = maxText
|
|
m["pattern"] = ""
|
|
m["autogeneratePattern"] = ""
|
|
m["primaryKey"] = false
|
|
case number:
|
|
m["type"] = "number"
|
|
m["onlyInt"] = true
|
|
m["min"] = nil
|
|
m["max"] = nil
|
|
case boolean:
|
|
m["type"] = "bool"
|
|
case file:
|
|
m["type"] = "file"
|
|
m["maxSelect"] = 1
|
|
m["maxSize"] = maxFileSize
|
|
m["mimeTypes"] = []string{}
|
|
m["thumbs"] = []string{}
|
|
m["protected"] = true
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
|
|
indexes := []string{}
|
|
if len(tb.key) > 0 {
|
|
indexes = append(indexes, indexSQL(tb.name, "unique: "+strings.Join(tb.key, ", ")))
|
|
}
|
|
for _, idx := range tb.indexes {
|
|
indexes = append(indexes, indexSQL(tb.name, idx))
|
|
}
|
|
|
|
return map[string]any{
|
|
"id": "warp" + shortHash(11, tb.name),
|
|
"name": tb.name,
|
|
"type": "base",
|
|
"system": false,
|
|
"listRule": nil,
|
|
"viewRule": nil,
|
|
"createRule": nil,
|
|
"updateRule": nil,
|
|
"deleteRule": nil,
|
|
"fields": out,
|
|
"indexes": indexes,
|
|
}
|
|
}
|
|
|
|
func indexSQL(tableName, spec string) string {
|
|
unique := ""
|
|
if cols, ok := strings.CutPrefix(spec, "unique:"); ok {
|
|
unique, spec = "UNIQUE ", cols
|
|
}
|
|
var cols []string
|
|
for _, c := range strings.Split(spec, ",") {
|
|
cols = append(cols, strings.TrimSpace(c))
|
|
}
|
|
name := "idx_" + shortHash(10, tableName, unique, strings.Join(cols, ","))
|
|
return fmt.Sprintf("CREATE %sINDEX `%s` ON `%s` (%s)", unique, name, tableName, "`"+strings.Join(cols, "`, `")+"`")
|
|
}
|
|
|
|
// schemaVersion identifies the schema: any change to a table changes it, and
|
|
// a database stamped with another one has a migration pending.
|
|
var schemaVersion = func() string {
|
|
cols := make([]map[string]any, len(schema))
|
|
for i := range schema {
|
|
cols[i] = schema[i].collectionJSON()
|
|
}
|
|
raw, _ := json.Marshal(cols)
|
|
sum := sha256.Sum256(raw)
|
|
return hex.EncodeToString(sum[:])[:12]
|
|
}()
|