package pocketbase import ( "crypto/sha256" "encoding/base32" "encoding/hex" "encoding/json" "fmt" "strconv" "strings" ) // The schema is the SQL drivers' tables as PocketBase collections: the same // names and columns, so that the reads can be the same SQL, with three // differences PocketBase forces. // // - Every record has a text `id`, which is the primary key. Warp's own keys // are kept as ordinary fields under a unique index, and a record's id is // derived from them (see rowID): a row is always addressed by its key. // - Nothing can be NULL. The two nullable state_key columns and // admin_registration_tokens.uses_allowed get a has_* flag beside them. // - `id` is taken, so push_notifications.id is notification_id and // admin_audit_log.id is audit_id. // // Full-text search has no FTS5 here — it is a plain collection searched with // LIKE — and media content lives in warp_media's file field, which is // PocketBase's own storage: its pb_data/storage or an S3 bucket. type fieldKind int const ( text fieldKind = iota number boolean file ) type field struct { name string kind fieldKind } // table is one collection. type table struct { name string // key is the primary key the SQL drivers declare; the record id is a // hash of its values. Empty for a collection whose ids are chosen by // the writer (warp_params). key []string fields []field // indexes are column lists; a leading "unique:" makes one unique. indexes []string } func tbl(name string, key []string, fields []field, indexes ...string) table { return table{name: name, key: key, fields: fields, indexes: indexes} } func pk(cols ...string) []string { return cols } func fld(name string, kind fieldKind) field { return field{name: name, kind: kind} } func texts(names ...string) []field { out := make([]field, len(names)) for i, n := range names { out[i] = field{name: n, kind: text} } return out } func numbers(names ...string) []field { out := make([]field, len(names)) for i, n := range names { out[i] = field{name: n, kind: number} } return out } func fields(groups ...[]field) []field { var out []field for _, g := range groups { out = append(out, g...) } return out } func sequenceTable(name string) table { return tbl(name, pk("name"), fields(texts("name"), numbers("value"))) } // The sequence rows each table starts with, as the SQL migrations insert // them. var sequenceRows = map[string][]string{ "roomserver_sequences": {"stream_ordering", "state_snapshot", "backfill_ordering"}, "sync_sequences": {"receipt", "to_device", "account_data", "presence"}, "federation_sequences": {"queue"}, "keyserver_sequences": {"device_list"}, "push_sequences": {"notification"}, "admin_sequences": {"audit"}, } var schema = []table{ tbl("warp_meta", pk("meta_key"), texts("meta_key", "meta_value")), {name: paramsCollection, fields: texts("value")}, tbl("warp_media", pk("sha256"), fields(texts("sha256"), numbers("size"), []field{fld("content", file)})), tbl("server_settings", pk("setting_key"), texts("setting_key", "setting_value", "updated_at")), // userapi tbl("user_accounts", pk("localpart"), fields( texts("localpart", "password_hash", "display_name", "avatar_url"), []field{fld("admin", boolean), fld("deactivated", boolean)}, numbers("created_at_ms"))), tbl("user_devices", pk("localpart", "device_id"), fields( texts("localpart", "device_id", "display_name"), numbers("created_at_ms", "last_seen_ms"), texts("last_seen_ip", "user_agent"))), tbl("user_access_tokens", pk("token_hash"), fields( texts("token_hash", "localpart", "device_id"), numbers("created_at_ms", "expires_at_ms")), "localpart, device_id"), tbl("user_refresh_tokens", pk("token_hash"), fields( texts("token_hash", "localpart", "device_id"), numbers("created_at_ms", "expires_at_ms")), "localpart, device_id"), tbl("user_account_data", pk("localpart", "room_id", "data_type"), fields( texts("localpart", "room_id", "data_type", "content"), numbers("stream_pos")), "localpart, stream_pos"), // roomserver tbl("roomserver_rooms", pk("room_id"), fields( texts("room_id", "room_version", "creator"), []field{fld("is_published", boolean)}, numbers("created_at_ms"), texts("name", "topic", "avatar_url", "canonical_alias", "join_rule", "history_visibility", "guest_access", "room_type"), numbers("joined_members")), "is_published, joined_members"), sequenceTable("roomserver_sequences"), tbl("roomserver_state_snapshots", pk("snapshot_id"), fields( numbers("snapshot_id"), texts("room_id", "state_hash")), "unique: room_id, state_hash"), tbl("roomserver_state_entries", pk("snapshot_id", "event_type", "state_key"), fields( numbers("snapshot_id"), texts("event_type", "state_key", "event_id"))), tbl("roomserver_events", pk("event_id"), fields( texts("event_id", "room_id"), numbers("stream_ordering"), texts("event_type", "state_key"), []field{fld("has_state_key", boolean)}, texts("sender"), numbers("depth", "origin_server_ts"), texts("event_json"), numbers("state_before_id"), texts("redacted_by"), []field{fld("outlier", boolean)}, texts("relates_to_id", "rel_type")), "unique: stream_ordering", "room_id, stream_ordering", "room_id, relates_to_id, rel_type, stream_ordering", "event_type, state_key"), tbl("roomserver_extremities", pk("room_id", "event_id"), texts("room_id", "event_id")), tbl("roomserver_current_state", pk("room_id", "event_type", "state_key"), texts("room_id", "event_type", "state_key", "event_id", "membership"), "event_type, state_key, membership"), tbl("roomserver_transactions", pk("localpart", "device_id", "txn_id"), fields( texts("localpart", "device_id", "txn_id", "event_id", "room_id"), numbers("sent_at_ms"))), tbl("roomserver_aliases", pk("alias"), fields( texts("alias", "room_id", "creator"), numbers("created_at_ms")), "room_id"), tbl("roomserver_invite_state", pk("room_id", "user_id"), texts("room_id", "user_id", "state_json")), tbl("roomserver_event_search", pk("stream_ordering"), fields( numbers("stream_ordering"), texts("room_id", "event_id", "search_key", "sender", "event_type", "body")), "room_id, stream_ordering"), // syncapi sequenceTable("sync_sequences"), tbl("sync_receipts", pk("room_id", "user_id", "receipt_type", "thread_id"), fields( texts("room_id", "user_id", "receipt_type", "thread_id", "event_id"), numbers("ts_ms", "stream_pos")), "stream_pos", "room_id, stream_pos"), tbl("sync_to_device", pk("stream_pos"), fields( numbers("stream_pos"), texts("localpart", "device_id", "sender", "event_type", "content"), numbers("sent_at_ms")), "localpart, device_id, stream_pos"), tbl("sync_presence", pk("user_id"), fields( texts("user_id", "presence", "status_msg"), numbers("last_active_ms", "stream_pos")), "stream_pos"), tbl("sync_filters", pk("localpart", "filter_id"), fields( texts("localpart", "filter_id", "filter_json"), numbers("created_at_ms"))), // federation tbl("federation_server_keys", pk("server_name", "key_id"), fields( texts("server_name", "key_id", "public_key"), numbers("valid_until_ts", "expired_ts", "fetched_at_ms"))), sequenceTable("federation_sequences"), tbl("federation_queue", pk("queue_id"), fields( numbers("queue_id"), texts("destination", "kind", "room_id", "event_id", "payload"), numbers("queued_at_ms")), "destination, kind, queue_id"), tbl("federation_destinations", pk("server_name"), fields( texts("server_name"), numbers("failures", "retry_at_ms", "last_success_ms", "last_failure_ms"), texts("last_error"))), tbl("federation_inbound_txns", pk("origin", "txn_id"), fields( texts("origin", "txn_id", "response_json"), numbers("received_at_ms"))), // mediaapi tbl("media_repository", pk("media_origin", "media_id"), fields( texts("media_origin", "media_id", "content_type", "upload_name"), numbers("file_size"), texts("sha256", "user_id"), numbers("created_at_ms"), []field{fld("pending", boolean)}, numbers("expires_at_ms")), "pending, user_id, expires_at_ms", "user_id, media_origin", "sha256"), tbl("media_thumbnails", pk("media_origin", "media_id", "width", "height", "method"), fields( texts("media_origin", "media_id"), numbers("width", "height"), texts("method", "content_type"), numbers("file_size"), texts("sha256"), numbers("created_at_ms")), "sha256"), tbl("media_quotas", pk("user_id"), fields(texts("user_id"), numbers("max_bytes"))), // keyserver sequenceTable("keyserver_sequences"), tbl("keyserver_device_keys", pk("user_id", "device_id"), texts("user_id", "device_id", "key_json")), tbl("keyserver_one_time_keys", pk("user_id", "device_id", "key_id"), texts("user_id", "device_id", "key_id", "algorithm", "key_json"), "user_id, device_id, algorithm"), tbl("keyserver_fallback_keys", pk("user_id", "device_id", "algorithm"), fields( texts("user_id", "device_id", "algorithm", "key_id", "key_json"), []field{fld("used", boolean)})), tbl("keyserver_cross_signing_keys", pk("user_id", "purpose"), texts("user_id", "purpose", "key_json")), tbl("keyserver_signatures", pk("target_user_id", "target_key_id", "signer_user_id", "signer_key_id"), texts("target_user_id", "target_key_id", "signer_user_id", "signer_key_id", "signature")), tbl("keyserver_device_lists", pk("user_id"), fields(texts("user_id"), numbers("stream_pos")), "stream_pos"), tbl("keyserver_backup_versions", pk("user_id", "version"), fields( texts("user_id"), numbers("version"), texts("algorithm", "auth_data"), numbers("etag"), []field{fld("deleted", boolean)})), tbl("keyserver_backup_keys", pk("user_id", "version", "room_id", "session_id"), fields( texts("user_id"), numbers("version"), texts("room_id", "session_id"), numbers("first_message_index", "forwarded_count"), []field{fld("is_verified", boolean)}, texts("session_data"))), tbl("keyserver_remote_users", pk("user_id"), fields( texts("user_id"), numbers("stream_id"), []field{fld("stale", boolean)})), tbl("keyserver_remote_device_keys", pk("user_id", "device_id"), texts("user_id", "device_id", "display_name", "key_json")), tbl("keyserver_remote_cross_signing_keys", pk("user_id", "purpose"), texts("user_id", "purpose", "key_json")), // push sequenceTable("push_sequences"), tbl("push_pushers", pk("localpart", "app_id", "pushkey"), fields( texts("localpart", "app_id", "pushkey", "device_id", "kind", "app_display_name", "device_display_name", "profile_tag", "lang", "data_json"), numbers("pushkey_ts_ms")), "app_id, pushkey"), tbl("push_notifications", pk("notification_id"), fields( numbers("notification_id"), texts("user_id", "room_id", "event_id"), numbers("event_pos", "ts_ms"), []field{fld("highlight", boolean)}, texts("actions"), []field{fld("is_read", boolean)}, texts("thread_id")), "user_id, notification_id", "user_id, room_id, is_read, event_pos", "is_read, ts_ms", "user_id, room_id, thread_id, is_read, event_pos"), tbl("push_room_counts", pk("user_id", "room_id"), fields( texts("user_id", "room_id"), numbers("notification_count", "highlight_count", "stream_pos")), "user_id, stream_pos"), // admin and application services tbl("admin_registration_tokens", pk("token"), fields( texts("token"), numbers("uses_allowed"), []field{fld("has_uses_allowed", boolean)}, numbers("completed", "expires_at_ms", "created_at_ms"))), tbl("admin_server_notice_rooms", pk("localpart"), texts("localpart", "room_id")), sequenceTable("admin_sequences"), tbl("admin_audit_log", pk("audit_id"), fields( numbers("audit_id", "ts_ms"), texts("actor", "action", "target", "details_json")), "actor, audit_id", "action, audit_id"), tbl("appservice_state", pk("as_id"), fields(texts("as_id"), numbers("stream_pos", "txn_id", "txn_to"))), // delayed events and OpenID tokens tbl("delayed_events", pk("delay_id"), fields( texts("delay_id", "user_id", "device_id", "room_id", "event_type", "state_key"), []field{fld("has_state_key", boolean)}, texts("content"), numbers("delay_ms", "running_since_ms", "send_at_ms", "finalised_at_ms"), texts("outcome", "reason", "event_id", "error")), "user_id, finalised_at_ms", "finalised_at_ms, send_at_ms", "room_id, event_type, state_key, finalised_at_ms"), tbl("openid_tokens", pk("token_hash"), fields(texts("token_hash", "user_id"), numbers("expires_at_ms")), "expires_at_ms"), } // tables indexes the schema by collection name. var tables = func() map[string]*table { out := make(map[string]*table, len(schema)) for i := range schema { out[schema[i].name] = &schema[i] } return out }() // maxText is the longest a text field may be. PocketBase's default is 5000 // characters, which an event can pass; this is far above the 64 KiB a // Matrix event may be, and above any account data a client sends. const maxText = 1 << 24 // maxFileSize bounds one media file. The media API enforces the configured // upload limit before content ever gets here. const maxFileSize = 1 << 40 // shortHash is a lower-case base32 digest, which fits PocketBase's id // alphabet of [a-z0-9]. func shortHash(n int, parts ...string) string { h := sha256.New() for _, p := range parts { _, _ = fmt.Fprintf(h, "%d:%s", len(p), p) } enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h.Sum(nil)) return strings.ToLower(enc)[:n] } // rowID is the record id of the row whose primary key has these values. func rowID(collection string, key ...any) string { parts := make([]string, 0, len(key)+1) parts = append(parts, collection) for _, v := range key { parts = append(parts, keyString(v)) } return shortHash(15, parts...) } func keyString(v any) string { switch v := v.(type) { case string: return v case int64: return strconv.FormatInt(v, 10) case int: return strconv.Itoa(v) case bool: return strconv.FormatBool(v) default: return fmt.Sprint(v) } } // collectionJSON renders a table as PocketBase's collection import format. // Collection and field ids are derived from their names, so that importing // the same schema again changes nothing, and a later version that adds a // field adds just that field. func (tb *table) collectionJSON() map[string]any { out := []map[string]any{{ "id": "text3208210256", "name": "id", "type": "text", "system": true, "primaryKey": true, "required": true, "hidden": false, "presentable": false, "min": 15, "max": 15, "pattern": "^[a-z0-9]+$", "autogeneratePattern": "[a-z0-9]{15}", }} for _, fd := range tb.fields { m := map[string]any{ "id": "f" + shortHash(12, tb.name, fd.name), "name": fd.name, "system": false, "required": false, "hidden": false, "presentable": false, } switch fd.kind { case text: m["type"] = "text" m["min"] = 0 m["max"] = maxText m["pattern"] = "" m["autogeneratePattern"] = "" m["primaryKey"] = false case number: m["type"] = "number" m["onlyInt"] = true m["min"] = nil m["max"] = nil case boolean: m["type"] = "bool" case file: m["type"] = "file" m["maxSelect"] = 1 m["maxSize"] = maxFileSize m["mimeTypes"] = []string{} m["thumbs"] = []string{} m["protected"] = true } out = append(out, m) } indexes := []string{} if len(tb.key) > 0 { indexes = append(indexes, indexSQL(tb.name, "unique: "+strings.Join(tb.key, ", "))) } for _, idx := range tb.indexes { indexes = append(indexes, indexSQL(tb.name, idx)) } return map[string]any{ "id": "warp" + shortHash(11, tb.name), "name": tb.name, "type": "base", "system": false, "listRule": nil, "viewRule": nil, "createRule": nil, "updateRule": nil, "deleteRule": nil, "fields": out, "indexes": indexes, } } func indexSQL(tableName, spec string) string { unique := "" if cols, ok := strings.CutPrefix(spec, "unique:"); ok { unique, spec = "UNIQUE ", cols } var cols []string for _, c := range strings.Split(spec, ",") { cols = append(cols, strings.TrimSpace(c)) } name := "idx_" + shortHash(10, tableName, unique, strings.Join(cols, ",")) return fmt.Sprintf("CREATE %sINDEX `%s` ON `%s` (%s)", unique, name, tableName, "`"+strings.Join(cols, "`, `")+"`") } // schemaVersion identifies the schema: any change to a table changes it, and // a database stamped with another one has a migration pending. var schemaVersion = func() string { cols := make([]map[string]any, len(schema)) for i := range schema { cols[i] = schema[i].collectionJSON() } raw, _ := json.Marshal(cols) sum := sha256.Sum256(raw) return hex.EncodeToString(sum[:])[:12] }()