From a47e15cc1c82ee28283846fe4643a80e7c6e4e06 Mon Sep 17 00:00:00 2001 From: tajniak81 <13187254+tajniak81@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:31:08 +0100 Subject: [PATCH] PocketBase as a standalone database server The pocketbase driver no longer embeds PocketBase: Warp keeps its data as PocketBase collections on a server of its own and reaches it only through the REST API, the way the postgres driver reaches PostgreSQL. database.connection_string is the server's URL, and Warp signs in as the superuser in WARP_PB_SUPERUSER_EMAIL / WARP_PB_SUPERUSER_PASSWORD (or a WARP_PB_SUPERUSER_FILE of the two). Every SQL table is a superuser-only collection of the same name and columns, created by warp migrate. Reads go through PocketBase's SQL endpoint with the SQL drivers' queries, via a read-only database/sql driver that inlines arguments, moves long ones into warp_params and pages past 1000 rows. Writes are record operations in batch requests under one write lock in the Warp process. Media is a protected file field in warp_media, local or in the S3 bucket Warp writes into PocketBase's settings. Search matches word prefixes with LIKE. The storage tests run against an in-process PocketBase and pass as on SQLite. Docker: the PocketBase stacks get a pocketbase service running tajniak81/pocketbase-docker-root (WARP_PB_IMAGE), with a pb-data volume and its admin UI on loopback; the AIO images run that PocketBase binary as a process beside the homeserver, making up a superuser when none is set. The .env examples, READMEs, docs and warp.yaml.example follow. Co-Authored-By: Claude Opus 5.5 --- Docker-AIO/.env.pocketbase.example | 16 +- Docker-AIO/.env.pocketbase.prod.example | 16 +- Docker-AIO/.env.pocketbase.s3.example | 18 +- Docker-AIO/.env.pocketbase.s3.prod.example | 18 +- Docker-AIO/.env.pocketbase.seaweedfs.example | 18 +- .../.env.pocketbase.seaweedfs.prod.example | 18 +- .../.env.pocketbase.seaweedfs.split.example | 18 +- ...nv.pocketbase.seaweedfs.split.prod.example | 18 +- Docker-AIO/Dockerfile.PocketBase | 19 +- Docker-AIO/Dockerfile.PocketBase.SeaweedFS | 15 +- .../Dockerfile.PocketBase.SeaweedFS.split | 12 +- Docker-AIO/Dockerfile.PocketBase.s3 | 16 +- Docker-AIO/README.md | 13 +- Docker-AIO/docker-compose.pocketbase.prod.yml | 9 +- .../docker-compose.pocketbase.s3.prod.yml | 4 +- Docker-AIO/docker-compose.pocketbase.s3.yml | 4 +- ...cker-compose.pocketbase.seaweedfs.prod.yml | 4 +- ...ompose.pocketbase.seaweedfs.split.prod.yml | 4 +- ...ker-compose.pocketbase.seaweedfs.split.yml | 4 +- .../docker-compose.pocketbase.seaweedfs.yml | 4 +- Docker-AIO/docker-compose.pocketbase.yml | 9 +- Docker-AIO/entrypoint.sh | 34 + Docker/.env.pocketbase.example | 24 +- Docker/.env.pocketbase.prod.example | 27 +- Docker/.env.pocketbase.s3.example | 26 +- Docker/.env.pocketbase.s3.prod.example | 31 +- Docker/.env.pocketbase.seaweedfs.example | 26 +- Docker/.env.pocketbase.seaweedfs.prod.example | 29 +- .../.env.pocketbase.seaweedfs.split.example | 26 +- ...nv.pocketbase.seaweedfs.split.prod.example | 29 +- Docker/README.md | 16 +- Docker/docker-compose.pocketbase.prod.yml | 42 +- Docker/docker-compose.pocketbase.s3.prod.yml | 39 +- Docker/docker-compose.pocketbase.s3.yml | 38 +- ...cker-compose.pocketbase.seaweedfs.prod.yml | 34 +- ...ompose.pocketbase.seaweedfs.split.prod.yml | 34 +- ...ker-compose.pocketbase.seaweedfs.split.yml | 33 +- .../docker-compose.pocketbase.seaweedfs.yml | 33 +- Docker/docker-compose.pocketbase.yml | 41 +- README.md | 6 +- docs/configuration.md | 12 +- docs/deployment.md | 25 +- server/go.mod | 2 +- server/internal/config/config.go | 69 +- server/internal/config/pocketbase_test.go | 37 + server/internal/setup/phase10_test.go | 18 +- server/internal/storage/pocketbase/admin.go | 342 +++++ server/internal/storage/pocketbase/batch.go | 282 ++++ server/internal/storage/pocketbase/blobs.go | 210 +++ server/internal/storage/pocketbase/client.go | 331 +++++ server/internal/storage/pocketbase/delayed.go | 232 ++++ .../internal/storage/pocketbase/federation.go | 247 ++++ server/internal/storage/pocketbase/helpers.go | 83 ++ server/internal/storage/pocketbase/keys.go | 766 +++++++++++ server/internal/storage/pocketbase/media.go | 195 +++ .../storage/pocketbase/pbtest/pbtest.go | 64 + .../internal/storage/pocketbase/pocketbase.go | 534 ++++---- server/internal/storage/pocketbase/push.go | 328 +++++ .../internal/storage/pocketbase/relations.go | 369 ++++++ .../internal/storage/pocketbase/remotekeys.go | 247 ++++ server/internal/storage/pocketbase/rooms.go | 1149 +++++++++++++++++ server/internal/storage/pocketbase/schema.go | 449 +++++++ .../internal/storage/pocketbase/sqldriver.go | 362 ++++++ server/internal/storage/pocketbase/sync.go | 461 +++++++ .../internal/storage/pocketbase/timeline.go | 334 +++++ server/internal/storage/pocketbase/users.go | 557 ++++++++ server/internal/storage/storage.go | 8 +- server/internal/storage/storage_test.go | 28 +- warp.yaml.example | 17 +- 69 files changed, 8209 insertions(+), 374 deletions(-) create mode 100644 server/internal/config/pocketbase_test.go create mode 100644 server/internal/storage/pocketbase/admin.go create mode 100644 server/internal/storage/pocketbase/batch.go create mode 100644 server/internal/storage/pocketbase/blobs.go create mode 100644 server/internal/storage/pocketbase/client.go create mode 100644 server/internal/storage/pocketbase/delayed.go create mode 100644 server/internal/storage/pocketbase/federation.go create mode 100644 server/internal/storage/pocketbase/helpers.go create mode 100644 server/internal/storage/pocketbase/keys.go create mode 100644 server/internal/storage/pocketbase/media.go create mode 100644 server/internal/storage/pocketbase/pbtest/pbtest.go create mode 100644 server/internal/storage/pocketbase/push.go create mode 100644 server/internal/storage/pocketbase/relations.go create mode 100644 server/internal/storage/pocketbase/remotekeys.go create mode 100644 server/internal/storage/pocketbase/rooms.go create mode 100644 server/internal/storage/pocketbase/schema.go create mode 100644 server/internal/storage/pocketbase/sqldriver.go create mode 100644 server/internal/storage/pocketbase/sync.go create mode 100644 server/internal/storage/pocketbase/timeline.go create mode 100644 server/internal/storage/pocketbase/users.go diff --git a/Docker-AIO/.env.pocketbase.example b/Docker-AIO/.env.pocketbase.example index 9136e20..9b784f8 100644 --- a/Docker-AIO/.env.pocketbase.example +++ b/Docker-AIO/.env.pocketbase.example @@ -20,10 +20,24 @@ WARP_PORT=8008 WARP_ADMIN_PORT=8009 WARP_WEB_PORT=8080 +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. Its password is set to this one on every start, and needs 8 +# characters or more. Empty: the container makes one up on first start and +# keeps it in /data/pocketbase-superuser.env. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. # The container runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The container's /data (warp-data): the signing key, and PocketBase's +# pb_data, with the database and media unless it is in S3. WARP_DATA_VOLUME= # ------------------------------------------------------------------- aio diff --git a/Docker-AIO/.env.pocketbase.prod.example b/Docker-AIO/.env.pocketbase.prod.example index f52523c..61d435e 100644 --- a/Docker-AIO/.env.pocketbase.prod.example +++ b/Docker-AIO/.env.pocketbase.prod.example @@ -25,10 +25,24 @@ WARP_WEB_PORT=8080 WARP_REGISTRY=10.2.1.10:5500/admin WARP_VERSION=latest +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. Its password is set to this one on every start, and needs 8 +# characters or more. Empty: the container makes one up on first start and +# keeps it in /data/pocketbase-superuser.env. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. # The container runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The container's /data (warp-data): the signing key, and PocketBase's +# pb_data, with the database and media unless it is in S3. WARP_DATA_VOLUME= # ------------------------------------------------------------------- aio diff --git a/Docker-AIO/.env.pocketbase.s3.example b/Docker-AIO/.env.pocketbase.s3.example index 321dbc5..e539dfe 100644 --- a/Docker-AIO/.env.pocketbase.s3.example +++ b/Docker-AIO/.env.pocketbase.s3.example @@ -23,10 +23,24 @@ WARP_PORT=8008 WARP_ADMIN_PORT=8009 WARP_WEB_PORT=8080 +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. Its password is set to this one on every start, and needs 8 +# characters or more. Empty: the container makes one up on first start and +# keeps it in /data/pocketbase-superuser.env. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. # The container runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The container's /data (warp-data): the signing key, and PocketBase's +# pb_data, with the database and media unless it is in S3. WARP_DATA_VOLUME= # ------------------------------------------------------------------- aio @@ -37,7 +51,7 @@ WARP_AIO_WEB=true # ------------------------------------------------------------- media (S3) -# Media goes to this bucket; the database stays in /data/pb_data. The bucket +# Media goes to this bucket; the database stays in PocketBase. The bucket # must already exist: nothing here runs a gateway or creates it. For a # gateway on the Docker host, use http://host.docker.internal:8333. Media # stored before S3 was on is not moved into the bucket. diff --git a/Docker-AIO/.env.pocketbase.s3.prod.example b/Docker-AIO/.env.pocketbase.s3.prod.example index 0026c30..1365d88 100644 --- a/Docker-AIO/.env.pocketbase.s3.prod.example +++ b/Docker-AIO/.env.pocketbase.s3.prod.example @@ -28,10 +28,24 @@ WARP_WEB_PORT=8080 WARP_REGISTRY=10.2.1.10:5500/admin WARP_VERSION=latest +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. Its password is set to this one on every start, and needs 8 +# characters or more. Empty: the container makes one up on first start and +# keeps it in /data/pocketbase-superuser.env. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. # The container runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The container's /data (warp-data): the signing key, and PocketBase's +# pb_data, with the database and media unless it is in S3. WARP_DATA_VOLUME= # ------------------------------------------------------------------- aio @@ -42,7 +56,7 @@ WARP_AIO_WEB=true # ------------------------------------------------------------- media (S3) -# Media goes to this bucket; the database stays in /data/pb_data. The bucket +# Media goes to this bucket; the database stays in PocketBase. The bucket # must already exist: nothing here runs a gateway or creates it. For a # gateway on the Docker host, use http://host.docker.internal:8333. Media # stored before S3 was on is not moved into the bucket. diff --git a/Docker-AIO/.env.pocketbase.seaweedfs.example b/Docker-AIO/.env.pocketbase.seaweedfs.example index d15375e..61c3971 100644 --- a/Docker-AIO/.env.pocketbase.seaweedfs.example +++ b/Docker-AIO/.env.pocketbase.seaweedfs.example @@ -22,10 +22,24 @@ WARP_PORT=8008 WARP_ADMIN_PORT=8009 WARP_WEB_PORT=8080 +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. Its password is set to this one on every start, and needs 8 +# characters or more. Empty: the container makes one up on first start and +# keeps it in /data/pocketbase-superuser.env. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. # The container runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The container's /data (warp-data): the signing key, and PocketBase's +# pb_data, with the database and media unless it is in S3. WARP_DATA_VOLUME= # SeaweedFS's data (seaweed-data). SEAWEED_DATA_VOLUME= @@ -39,7 +53,7 @@ WARP_AIO_WEB=true # -------------------------------------------------------------- seaweedfs # Media goes to this bucket in SeaweedFS, created on start; the database -# stays in /data/pb_data. The key pair is the homeserver's S3 identity: +# stays in PocketBase. The key pair is the homeserver's S3 identity: # the gateway's only one, and what the homeserver signs with. # Media stored before S3 was on is not moved into the bucket. WARP_PB_S3_BUCKET=warp diff --git a/Docker-AIO/.env.pocketbase.seaweedfs.prod.example b/Docker-AIO/.env.pocketbase.seaweedfs.prod.example index 8b620be..41aa230 100644 --- a/Docker-AIO/.env.pocketbase.seaweedfs.prod.example +++ b/Docker-AIO/.env.pocketbase.seaweedfs.prod.example @@ -27,10 +27,24 @@ WARP_WEB_PORT=8080 WARP_REGISTRY=10.2.1.10:5500/admin WARP_VERSION=latest +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. Its password is set to this one on every start, and needs 8 +# characters or more. Empty: the container makes one up on first start and +# keeps it in /data/pocketbase-superuser.env. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. # The container runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The container's /data (warp-data): the signing key, and PocketBase's +# pb_data, with the database and media unless it is in S3. WARP_DATA_VOLUME= # SeaweedFS's data (seaweed-data). SEAWEED_DATA_VOLUME= @@ -44,7 +58,7 @@ WARP_AIO_WEB=true # -------------------------------------------------------------- seaweedfs # Media goes to this bucket in SeaweedFS, created on start; the database -# stays in /data/pb_data. The key pair is the homeserver's S3 identity: +# stays in PocketBase. The key pair is the homeserver's S3 identity: # the gateway's only one, and what the homeserver signs with. # Media stored before S3 was on is not moved into the bucket. WARP_PB_S3_BUCKET=warp diff --git a/Docker-AIO/.env.pocketbase.seaweedfs.split.example b/Docker-AIO/.env.pocketbase.seaweedfs.split.example index c5f7f34..2f02091 100644 --- a/Docker-AIO/.env.pocketbase.seaweedfs.split.example +++ b/Docker-AIO/.env.pocketbase.seaweedfs.split.example @@ -23,10 +23,24 @@ WARP_PORT=8008 WARP_ADMIN_PORT=8009 WARP_WEB_PORT=8080 +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. Its password is set to this one on every start, and needs 8 +# characters or more. Empty: the container makes one up on first start and +# keeps it in /data/pocketbase-superuser.env. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. # The container runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The container's /data (warp-data): the signing key, and PocketBase's +# pb_data, with the database and media unless it is in S3. WARP_DATA_VOLUME= # SeaweedFS's data (seaweed-data). SEAWEED_DATA_VOLUME= @@ -42,7 +56,7 @@ WARP_AIO_WEB=true # -------------------------------------------------------------- seaweedfs # Media goes to this bucket in SeaweedFS, created on start; the database -# stays in /data/pb_data. The key pair is the homeserver's S3 identity: +# stays in PocketBase. The key pair is the homeserver's S3 identity: # written into the filer's IAM store on every start, beside any identity # added in the admin UI, so a new secret here lands on the next start. # Media stored before S3 was on is not moved into the bucket. diff --git a/Docker-AIO/.env.pocketbase.seaweedfs.split.prod.example b/Docker-AIO/.env.pocketbase.seaweedfs.split.prod.example index 2298b37..7a0265c 100644 --- a/Docker-AIO/.env.pocketbase.seaweedfs.split.prod.example +++ b/Docker-AIO/.env.pocketbase.seaweedfs.split.prod.example @@ -28,10 +28,24 @@ WARP_WEB_PORT=8080 WARP_REGISTRY=10.2.1.10:5500/admin WARP_VERSION=latest +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. Its password is set to this one on every start, and needs 8 +# characters or more. Empty: the container makes one up on first start and +# keeps it in /data/pocketbase-superuser.env. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. # The container runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The container's /data (warp-data): the signing key, and PocketBase's +# pb_data, with the database and media unless it is in S3. WARP_DATA_VOLUME= # SeaweedFS's data (seaweed-data). SEAWEED_DATA_VOLUME= @@ -47,7 +61,7 @@ WARP_AIO_WEB=true # -------------------------------------------------------------- seaweedfs # Media goes to this bucket in SeaweedFS, created on start; the database -# stays in /data/pb_data. The key pair is the homeserver's S3 identity: +# stays in PocketBase. The key pair is the homeserver's S3 identity: # written into the filer's IAM store on every start, beside any identity # added in the admin UI, so a new secret here lands on the next start. # Media stored before S3 was on is not moved into the bucket. diff --git a/Docker-AIO/Dockerfile.PocketBase b/Docker-AIO/Dockerfile.PocketBase index 8bea6ab..bf4dca5 100644 --- a/Docker-AIO/Dockerfile.PocketBase +++ b/Docker-AIO/Dockerfile.PocketBase @@ -1,7 +1,8 @@ # warp-aio-pocketbase — the homeserver, the admin panel and the web client -# in one image, on PocketBase. PocketBase is the homeserver's embedded -# database driver: its data directory, with media in its storage, is -# /data/pb_data. Set WARP_PB_S3_* to keep media in an S3 bucket instead. +# in one image, on PocketBase. PocketBase runs as a process of its own in +# the container, on port 8070, with its pb_data — the database, and media +# in its storage — in /data/pb_data. Set WARP_PB_S3_* to keep media in an +# S3 bucket instead. # # Build from the repository root: # @@ -11,6 +12,10 @@ # # The database, signing key and media live in /data; mount a volume on it. +# PocketBase itself: the binary of this image, PocketBase 0.40 or later. +ARG PB_IMAGE=tajniak81/pocketbase-docker-root:latest +FROM ${PB_IMAGE} AS pocketbase + FROM node:22-alpine AS web WORKDIR /src @@ -74,6 +79,8 @@ RUN apk add --no-cache ca-certificates tzdata curl tini && \ adduser -D -u 10001 -h /data warp COPY --from=build /out/warp /out/warp-admin /out/warp-web /usr/local/bin/ +# PocketBase, from the same image the separate-container stacks run. +COPY --from=pocketbase /pb/pocketbase /usr/local/bin/pocketbase COPY warp.yaml.example /etc/warp/warp.yaml.example COPY --chmod=755 Docker-AIO/entrypoint.sh /usr/local/bin/warp-aio @@ -85,7 +92,7 @@ USER warp ENV WARP_LISTEN_ADDRESS=":8008" \ WARP_PRIVATE_KEY="/data/signing.key" \ WARP_DB_DRIVER="pocketbase" \ - WARP_DB_CONNECTION_STRING="/data/pb_data" \ + WARP_DB_CONNECTION_STRING="http://127.0.0.1:8070" \ WARP_LOG_FORMAT="json" \ WARP_ADMIN_LISTEN=":8009" \ WARP_ADMIN_HOMESERVER="http://127.0.0.1:8008" \ @@ -95,9 +102,11 @@ ENV WARP_LISTEN_ADDRESS=":8008" \ WARP_WEB_LOG_FORMAT="json" \ WARP_AIO_ADMIN="true" \ WARP_AIO_WEB="true" \ + WARP_AIO_POCKETBASE="true" \ + WARP_PB_SUPERUSER_FILE="/data/pocketbase-superuser.env" \ WARP_AIO_EMBEDDED_POSTGRES="false" -EXPOSE 8008 8009 8080 +EXPOSE 8008 8009 8080 8070 HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ CMD curl -fsS http://127.0.0.1:8008/healthz && \ diff --git a/Docker-AIO/Dockerfile.PocketBase.SeaweedFS b/Docker-AIO/Dockerfile.PocketBase.SeaweedFS index 358cba7..b928730 100644 --- a/Docker-AIO/Dockerfile.PocketBase.SeaweedFS +++ b/Docker-AIO/Dockerfile.PocketBase.SeaweedFS @@ -1,7 +1,8 @@ # warp-aio-pocketbase-seaweedfs — the homeserver, the admin panel and the # web client on PocketBase, with SeaweedFS in the same image as the media # store: one `weed server -s3` process, master, volume, filer and S3 gateway -# in one. The homeserver keeps its database in /data/pb_data and its media +# in one. PocketBase runs beside the homeserver, its database in +# /data/pb_data; the homeserver keeps its media # in the bucket WARP_PB_S3_BUCKET (default warp) on the gateway at # 127.0.0.1:8333. For per-role processes and SeaweedFS's admin UI, see # Dockerfile.PocketBase.SeaweedFS.split. @@ -31,6 +32,10 @@ # it. ARG SEAWEED_VERSION="4.45" +# PocketBase itself: the binary of this image, PocketBase 0.40 or later. +ARG PB_IMAGE=tajniak81/pocketbase-docker-root:latest +FROM ${PB_IMAGE} AS pocketbase + FROM node:22-alpine AS web WORKDIR /src @@ -98,6 +103,8 @@ RUN apk add --no-cache ca-certificates tzdata curl tini && \ adduser -D -u 10001 -h /data warp COPY --from=build /out/warp /out/warp-admin /out/warp-web /usr/local/bin/ +# PocketBase, from the same image the separate-container stacks run. +COPY --from=pocketbase /pb/pocketbase /usr/local/bin/pocketbase COPY --from=seaweed /usr/bin/weed /usr/local/bin/weed COPY warp.yaml.example /etc/warp/warp.yaml.example COPY --chmod=755 Docker-AIO/entrypoint.sh /usr/local/bin/warp-aio @@ -116,7 +123,7 @@ USER warp ENV WARP_LISTEN_ADDRESS=":8008" \ WARP_PRIVATE_KEY="/data/signing.key" \ WARP_DB_DRIVER="pocketbase" \ - WARP_DB_CONNECTION_STRING="/data/pb_data" \ + WARP_DB_CONNECTION_STRING="http://127.0.0.1:8070" \ WARP_PB_S3_ENABLED="true" \ WARP_PB_S3_ENDPOINT="http://127.0.0.1:8333" \ WARP_PB_S3_BUCKET="warp" \ @@ -131,10 +138,12 @@ ENV WARP_LISTEN_ADDRESS=":8008" \ WARP_WEB_LOG_FORMAT="json" \ WARP_AIO_ADMIN="true" \ WARP_AIO_WEB="true" \ + WARP_AIO_POCKETBASE="true" \ + WARP_PB_SUPERUSER_FILE="/data/pocketbase-superuser.env" \ WARP_AIO_EMBEDDED_POSTGRES="false" \ WARP_AIO_SEAWEEDFS="server" -EXPOSE 8008 8009 8080 8333 +EXPOSE 8008 8009 8080 8070 8333 # Every process must answer, so a wedged one shows in `docker ps`. HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ diff --git a/Docker-AIO/Dockerfile.PocketBase.SeaweedFS.split b/Docker-AIO/Dockerfile.PocketBase.SeaweedFS.split index 828c8d7..085c28b 100644 --- a/Docker-AIO/Dockerfile.PocketBase.SeaweedFS.split +++ b/Docker-AIO/Dockerfile.PocketBase.SeaweedFS.split @@ -38,6 +38,10 @@ # it. ARG SEAWEED_VERSION="4.45" +# PocketBase itself: the binary of this image, PocketBase 0.40 or later. +ARG PB_IMAGE=tajniak81/pocketbase-docker-root:latest +FROM ${PB_IMAGE} AS pocketbase + FROM node:22-alpine AS web WORKDIR /src @@ -105,6 +109,8 @@ RUN apk add --no-cache ca-certificates tzdata curl tini && \ adduser -D -u 10001 -h /data warp COPY --from=build /out/warp /out/warp-admin /out/warp-web /usr/local/bin/ +# PocketBase, from the same image the separate-container stacks run. +COPY --from=pocketbase /pb/pocketbase /usr/local/bin/pocketbase COPY --from=seaweed /usr/bin/weed /usr/local/bin/weed COPY warp.yaml.example /etc/warp/warp.yaml.example COPY --chmod=755 Docker-AIO/entrypoint.sh /usr/local/bin/warp-aio @@ -124,7 +130,7 @@ USER warp ENV WARP_LISTEN_ADDRESS=":8008" \ WARP_PRIVATE_KEY="/data/signing.key" \ WARP_DB_DRIVER="pocketbase" \ - WARP_DB_CONNECTION_STRING="/data/pb_data" \ + WARP_DB_CONNECTION_STRING="http://127.0.0.1:8070" \ WARP_PB_S3_ENABLED="true" \ WARP_PB_S3_ENDPOINT="http://127.0.0.1:8333" \ WARP_PB_S3_BUCKET="warp" \ @@ -139,11 +145,13 @@ ENV WARP_LISTEN_ADDRESS=":8008" \ WARP_WEB_LOG_FORMAT="json" \ WARP_AIO_ADMIN="true" \ WARP_AIO_WEB="true" \ + WARP_AIO_POCKETBASE="true" \ + WARP_PB_SUPERUSER_FILE="/data/pocketbase-superuser.env" \ WARP_AIO_EMBEDDED_POSTGRES="false" \ WARP_AIO_SEAWEEDFS="split" \ WEED_ADMIN_USER="admin" -EXPOSE 8008 8009 8080 8333 23646 +EXPOSE 8008 8009 8080 8070 8333 23646 # Every process must answer, so a wedged one shows in `docker ps`. HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ diff --git a/Docker-AIO/Dockerfile.PocketBase.s3 b/Docker-AIO/Dockerfile.PocketBase.s3 index 6ce331e..933f1b5 100644 --- a/Docker-AIO/Dockerfile.PocketBase.s3 +++ b/Docker-AIO/Dockerfile.PocketBase.s3 @@ -1,7 +1,7 @@ # warp-aio-pocketbase-s3 — the homeserver, the admin panel and the web client # in one image, on PocketBase, with media in an S3 bucket that already exists -# somewhere else: AWS, MinIO, a SeaweedFS on another host. The database stays -# in /data/pb_data. For a SeaweedFS that comes up inside the container, see +# somewhere else: AWS, MinIO, a SeaweedFS on another host. PocketBase runs +# beside the homeserver, its database in /data/pb_data. For a SeaweedFS that comes up inside the container, see # Dockerfile.PocketBase.SeaweedFS. # # Build from the repository root: @@ -19,6 +19,10 @@ # WARP_PB_S3_FORCE_PATH_STYLE=false for AWS S3 proper. The container waits up # to 30 seconds for the endpoint to answer before it starts the homeserver. +# PocketBase itself: the binary of this image, PocketBase 0.40 or later. +ARG PB_IMAGE=tajniak81/pocketbase-docker-root:latest +FROM ${PB_IMAGE} AS pocketbase + FROM node:22-alpine AS web WORKDIR /src @@ -82,6 +86,8 @@ RUN apk add --no-cache ca-certificates tzdata curl tini && \ adduser -D -u 10001 -h /data warp COPY --from=build /out/warp /out/warp-admin /out/warp-web /usr/local/bin/ +# PocketBase, from the same image the separate-container stacks run. +COPY --from=pocketbase /pb/pocketbase /usr/local/bin/pocketbase COPY warp.yaml.example /etc/warp/warp.yaml.example COPY --chmod=755 Docker-AIO/entrypoint.sh /usr/local/bin/warp-aio @@ -94,7 +100,7 @@ USER warp ENV WARP_LISTEN_ADDRESS=":8008" \ WARP_PRIVATE_KEY="/data/signing.key" \ WARP_DB_DRIVER="pocketbase" \ - WARP_DB_CONNECTION_STRING="/data/pb_data" \ + WARP_DB_CONNECTION_STRING="http://127.0.0.1:8070" \ WARP_PB_S3_ENABLED="true" \ WARP_PB_S3_BUCKET="warp" \ WARP_PB_S3_REGION="us-east-1" \ @@ -108,9 +114,11 @@ ENV WARP_LISTEN_ADDRESS=":8008" \ WARP_WEB_LOG_FORMAT="json" \ WARP_AIO_ADMIN="true" \ WARP_AIO_WEB="true" \ + WARP_AIO_POCKETBASE="true" \ + WARP_PB_SUPERUSER_FILE="/data/pocketbase-superuser.env" \ WARP_AIO_EMBEDDED_POSTGRES="false" -EXPOSE 8008 8009 8080 +EXPOSE 8008 8009 8080 8070 HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ CMD curl -fsS http://127.0.0.1:8008/healthz && \ diff --git a/Docker-AIO/README.md b/Docker-AIO/README.md index 1aa92f7..cd9cd68 100644 --- a/Docker-AIO/README.md +++ b/Docker-AIO/README.md @@ -8,7 +8,7 @@ policy brings the set back. | Dockerfile | Image | Database | | --- | --- | --- | | [`Dockerfile`](Dockerfile) | `warp-aio` | SQLite, `/data/warp.db` | -| [`Dockerfile.PocketBase`](Dockerfile.PocketBase) | `warp-aio-pocketbase` | PocketBase (embedded driver), `/data/pb_data` | +| [`Dockerfile.PocketBase`](Dockerfile.PocketBase) | `warp-aio-pocketbase` | PocketBase, a process in the container, `/data/pb_data` | | [`Dockerfile.PostgreSQL`](Dockerfile.PostgreSQL) | `warp-aio-postgresql` | PostgreSQL 17 in the container, `/data/postgres` | | [`Dockerfile.PocketBase.s3`](Dockerfile.PocketBase.s3) | `warp-aio-pocketbase-s3` | PocketBase, media in an existing S3 bucket elsewhere | | [`Dockerfile.PocketBase.SeaweedFS`](Dockerfile.PocketBase.SeaweedFS) | `warp-aio-pocketbase-seaweedfs` | PocketBase, media in SeaweedFS (`weed server -s3`) in the container | @@ -42,6 +42,17 @@ inside the container, so it needs no password and is not reachable from outside. To point that image at a PostgreSQL elsewhere instead, set `WARP_AIO_EMBEDDED_POSTGRES=false` and `WARP_DB_CONNECTION_STRING`. +The four PocketBase images run PocketBase as a process of its own beside +the homeserver, the binary taken from `tajniak81/pocketbase-docker-root` +(the `PB_IMAGE` build argument picks another, PocketBase 0.40 or later). +Its `pb_data` is `/data/pb_data`, and its admin UI is on `:8070`, +published on loopback. The homeserver signs in as the superuser in +`WARP_PB_SUPERUSER_EMAIL` and `WARP_PB_SUPERUSER_PASSWORD`, whose password +is set again on every start; without them, the first start makes one up +and keeps it in `/data/pocketbase-superuser.env`, where the admin UI's +login can be read back. To point an image at a PocketBase elsewhere, set +`WARP_AIO_POCKETBASE=false` and `WARP_DB_CONNECTION_STRING`. + The Dockerfiles share their build stages line for line; only the header, the SeaweedFS stage and the final stage differ. Keep the build stages in step when changing one. diff --git a/Docker-AIO/docker-compose.pocketbase.prod.yml b/Docker-AIO/docker-compose.pocketbase.prod.yml index b818af6..4e2ffbf 100644 --- a/Docker-AIO/docker-compose.pocketbase.prod.yml +++ b/Docker-AIO/docker-compose.pocketbase.prod.yml @@ -5,9 +5,11 @@ # cp .env.pocketbase.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.prod.yml up -d # -# PocketBase is embedded in the homeserver (the pocketbase database driver): -# its data directory, with media in its storage, is /data/pb_data in the -# warp-data volume. Set WARP_PB_S3_* in .env to keep media in an +# PocketBase runs in the container as a server of its own, which the +# homeserver signs in to as the superuser in .env (or one the container +# makes up, kept in /data/pocketbase-superuser.env): its pb_data, with media +# in its storage, is /data/pb_data in the warp-data volume. Its admin UI is +# http://localhost:8070/_/ on loopback. Set WARP_PB_S3_* in .env to keep media in an # S3 bucket instead (docs/configuration.md). # # Homeserver http://localhost:8008, admin panel http://localhost:8009, web @@ -37,6 +39,7 @@ services: - "${WARP_PORT:-8008}:8008" - "${WARP_ADMIN_PORT:-8009}:8009" - "${WARP_WEB_PORT:-8080}:8080" + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" volumes: warp-data: diff --git a/Docker-AIO/docker-compose.pocketbase.s3.prod.yml b/Docker-AIO/docker-compose.pocketbase.s3.prod.yml index 0f0f904..4814f04 100644 --- a/Docker-AIO/docker-compose.pocketbase.s3.prod.yml +++ b/Docker-AIO/docker-compose.pocketbase.s3.prod.yml @@ -6,7 +6,8 @@ # cp .env.pocketbase.s3.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.s3.prod.yml up -d # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# PocketBase runs in the container, its database in /data/pb_data +# (warp-data), admin UI on loopback at http://localhost:8070/_/; media goes to the bucket # WARP_PB_S3_BUCKET (default warp) on WARP_PB_S3_ENDPOINT, which must already # exist: nothing here runs a gateway or creates the bucket. For a gateway on # this Docker host, use http://host.docker.internal:8333; the host entry below @@ -54,6 +55,7 @@ services: - "${WARP_PORT:-8008}:8008" - "${WARP_ADMIN_PORT:-8009}:8009" - "${WARP_WEB_PORT:-8080}:8080" + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" volumes: warp-data: diff --git a/Docker-AIO/docker-compose.pocketbase.s3.yml b/Docker-AIO/docker-compose.pocketbase.s3.yml index 535e639..1f94159 100644 --- a/Docker-AIO/docker-compose.pocketbase.s3.yml +++ b/Docker-AIO/docker-compose.pocketbase.s3.yml @@ -6,7 +6,8 @@ # cp .env.pocketbase.s3.example .env # then edit it # docker compose -f docker-compose.pocketbase.s3.yml up -d --build # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# PocketBase runs in the container, its database in /data/pb_data +# (warp-data), admin UI on loopback at http://localhost:8070/_/; media goes to the bucket # WARP_PB_S3_BUCKET (default warp) on WARP_PB_S3_ENDPOINT, which must already # exist: nothing here runs a gateway or creates the bucket. For a gateway on # this Docker host, use http://host.docker.internal:8333; the host entry below @@ -50,6 +51,7 @@ services: - "${WARP_PORT:-8008}:8008" - "${WARP_ADMIN_PORT:-8009}:8009" - "${WARP_WEB_PORT:-8080}:8080" + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" volumes: warp-data: diff --git a/Docker-AIO/docker-compose.pocketbase.seaweedfs.prod.yml b/Docker-AIO/docker-compose.pocketbase.seaweedfs.prod.yml index 8cb3c08..7e853ee 100644 --- a/Docker-AIO/docker-compose.pocketbase.seaweedfs.prod.yml +++ b/Docker-AIO/docker-compose.pocketbase.seaweedfs.prod.yml @@ -7,7 +7,8 @@ # cp .env.pocketbase.seaweedfs.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.prod.yml up -d # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# PocketBase runs in the container, its database in /data/pb_data +# (warp-data), admin UI on loopback at http://localhost:8070/_/; media goes to the bucket # WARP_PB_S3_BUCKET in SeaweedFS (seaweed-data), whose gateway has exactly # one identity: the homeserver's. Media stored before S3 was on is not moved # into the bucket. @@ -56,6 +57,7 @@ services: - "${WARP_PORT:-8008}:8008" - "${WARP_ADMIN_PORT:-8009}:8009" - "${WARP_WEB_PORT:-8080}:8080" + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" - "${SEAWEED_S3_BIND:-127.0.0.1}:${SEAWEED_S3_PORT:-8333}:8333" volumes: diff --git a/Docker-AIO/docker-compose.pocketbase.seaweedfs.split.prod.yml b/Docker-AIO/docker-compose.pocketbase.seaweedfs.split.prod.yml index 8e51174..05eb9b5 100644 --- a/Docker-AIO/docker-compose.pocketbase.seaweedfs.split.prod.yml +++ b/Docker-AIO/docker-compose.pocketbase.seaweedfs.split.prod.yml @@ -7,7 +7,8 @@ # cp .env.pocketbase.seaweedfs.split.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.split.prod.yml up -d # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# PocketBase runs in the container, its database in /data/pb_data +# (warp-data), admin UI on loopback at http://localhost:8070/_/; media goes to the bucket # WARP_PB_S3_BUCKET in SeaweedFS (seaweed-data). The homeserver's S3 identity # is seeded into the filer's IAM store on every boot, beside any added in # the admin UI under Object Store → Users. Media stored before S3 was on is @@ -64,6 +65,7 @@ services: - "${WARP_PORT:-8008}:8008" - "${WARP_ADMIN_PORT:-8009}:8009" - "${WARP_WEB_PORT:-8080}:8080" + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" - "${SEAWEED_S3_BIND:-127.0.0.1}:${SEAWEED_S3_PORT:-8333}:8333" - "${SEAWEED_ADMIN_BIND:-127.0.0.1}:${SEAWEED_ADMIN_PORT:-23646}:23646" diff --git a/Docker-AIO/docker-compose.pocketbase.seaweedfs.split.yml b/Docker-AIO/docker-compose.pocketbase.seaweedfs.split.yml index 8bedbb3..d0e90dc 100644 --- a/Docker-AIO/docker-compose.pocketbase.seaweedfs.split.yml +++ b/Docker-AIO/docker-compose.pocketbase.seaweedfs.split.yml @@ -7,7 +7,8 @@ # cp .env.pocketbase.seaweedfs.split.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.split.yml up -d --build # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# PocketBase runs in the container, its database in /data/pb_data +# (warp-data), admin UI on loopback at http://localhost:8070/_/; media goes to the bucket # WARP_PB_S3_BUCKET in SeaweedFS (seaweed-data). The homeserver's S3 identity # is seeded into the filer's IAM store on every boot, beside any added in # the admin UI under Object Store → Users. Media stored before S3 was on is @@ -63,6 +64,7 @@ services: - "${WARP_PORT:-8008}:8008" - "${WARP_ADMIN_PORT:-8009}:8009" - "${WARP_WEB_PORT:-8080}:8080" + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" - "${SEAWEED_S3_BIND:-127.0.0.1}:${SEAWEED_S3_PORT:-8333}:8333" - "${SEAWEED_ADMIN_BIND:-127.0.0.1}:${SEAWEED_ADMIN_PORT:-23646}:23646" diff --git a/Docker-AIO/docker-compose.pocketbase.seaweedfs.yml b/Docker-AIO/docker-compose.pocketbase.seaweedfs.yml index f8147eb..5b7ca1a 100644 --- a/Docker-AIO/docker-compose.pocketbase.seaweedfs.yml +++ b/Docker-AIO/docker-compose.pocketbase.seaweedfs.yml @@ -6,7 +6,8 @@ # cp .env.pocketbase.seaweedfs.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.yml up -d --build # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# PocketBase runs in the container, its database in /data/pb_data +# (warp-data), admin UI on loopback at http://localhost:8070/_/; media goes to the bucket # WARP_PB_S3_BUCKET in SeaweedFS (seaweed-data), whose gateway has exactly # one identity: the homeserver's. Media stored before S3 was on is not moved # into the bucket. @@ -54,6 +55,7 @@ services: - "${WARP_PORT:-8008}:8008" - "${WARP_ADMIN_PORT:-8009}:8009" - "${WARP_WEB_PORT:-8080}:8080" + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" - "${SEAWEED_S3_BIND:-127.0.0.1}:${SEAWEED_S3_PORT:-8333}:8333" volumes: diff --git a/Docker-AIO/docker-compose.pocketbase.yml b/Docker-AIO/docker-compose.pocketbase.yml index 1918b07..6318ffb 100644 --- a/Docker-AIO/docker-compose.pocketbase.yml +++ b/Docker-AIO/docker-compose.pocketbase.yml @@ -5,9 +5,11 @@ # cp .env.pocketbase.example .env # then edit it # docker compose -f docker-compose.pocketbase.yml up -d --build # -# PocketBase is embedded in the homeserver (the pocketbase database driver): -# its data directory, with media in its storage, is /data/pb_data in the -# warp-data volume. Set WARP_PB_S3_* in .env to keep media in an +# PocketBase runs in the container as a server of its own, which the +# homeserver signs in to as the superuser in .env (or one the container +# makes up, kept in /data/pocketbase-superuser.env): its pb_data, with media +# in its storage, is /data/pb_data in the warp-data volume. Its admin UI is +# http://localhost:8070/_/ on loopback. Set WARP_PB_S3_* in .env to keep media in an # S3 bucket instead (docs/configuration.md). # # Homeserver http://localhost:8008, admin panel http://localhost:8009, web @@ -33,6 +35,7 @@ services: - "${WARP_PORT:-8008}:8008" - "${WARP_ADMIN_PORT:-8009}:8009" - "${WARP_WEB_PORT:-8080}:8080" + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" volumes: warp-data: diff --git a/Docker-AIO/entrypoint.sh b/Docker-AIO/entrypoint.sh index 7c61161..16f61ba 100644 --- a/Docker-AIO/entrypoint.sh +++ b/Docker-AIO/entrypoint.sh @@ -16,6 +16,10 @@ # $PGDATA (/data/postgres); the default # in warp-aio-postgresql, the only image # that has PostgreSQL +# WARP_AIO_POCKETBASE=true PocketBase in the container, on port +# 8070, its pb_data in /data/pb_data; set +# by the warp-aio-pocketbase* images, the +# only ones that have it # WARP_AIO_SEAWEEDFS=server SeaweedFS as one `weed server -s3` # WARP_AIO_SEAWEEDFS=split SeaweedFS as master, volume, filer, S3 # gateway and admin UI, one process each; @@ -195,6 +199,36 @@ if [ "${WARP_PB_S3_ENABLED:-false}" = true ] && [ -z "$seaweedfs" ] && [ -n "${W done fi +# PocketBase is the homeserver's database server. Its superuser is the one +# the homeserver signs in as; without one in the environment, the first +# start makes one up and keeps it in the volume, so a bare `docker run` +# works and the admin UI can still be signed in to. Every start sets the +# password again, which is how a changed one lands. +if [ "${WARP_AIO_POCKETBASE:-false}" = true ]; then + if ! command -v pocketbase >/dev/null; then + log "WARP_AIO_POCKETBASE=true, but this image has no PocketBase; use warp-aio-pocketbase" + exit 1 + fi + # The homeserver reads the same file through WARP_PB_SUPERUSER_FILE, so + # that `docker exec ... warp create-account` finds the account too. + superuser="${WARP_PB_SUPERUSER_FILE:-/data/pocketbase-superuser.env}" + if [ -z "${WARP_PB_SUPERUSER_EMAIL:-}" ] || [ -z "${WARP_PB_SUPERUSER_PASSWORD:-}" ]; then + if [ ! -f "$superuser" ]; then + log "no WARP_PB_SUPERUSER_EMAIL and WARP_PB_SUPERUSER_PASSWORD; making up a superuser, kept in $superuser" + umask 077 + printf 'WARP_PB_SUPERUSER_EMAIL=warp@warp.local\nWARP_PB_SUPERUSER_PASSWORD=%s\n' \ + "$(head -c 24 /dev/urandom | base64 | tr -d '/+=')" >"$superuser" + fi + set -a + . "$superuser" + set +a + fi + pocketbase superuser upsert "$WARP_PB_SUPERUSER_EMAIL" "$WARP_PB_SUPERUSER_PASSWORD" \ + --dir=/data/pb_data >/dev/null + run pocketbase pocketbase serve --http=0.0.0.0:8070 --dir=/data/pb_data + wait_http PocketBase http://127.0.0.1:8070/api/health +fi + # The signing key is created on first start and kept in the volume: losing # it means other servers no longer recognise this one. if [ ! -f "$WARP_PRIVATE_KEY" ]; then diff --git a/Docker/.env.pocketbase.example b/Docker/.env.pocketbase.example index c027488..5080fe1 100644 --- a/Docker/.env.pocketbase.example +++ b/Docker/.env.pocketbase.example @@ -21,11 +21,31 @@ WARP_PORT=8008 WARP_ADMIN_PORT=8009 WARP_WEB_PORT=8080 +# The PocketBase image; any PocketBase 0.40 or later answers the same. +WARP_PB_IMAGE=tajniak81/pocketbase-docker-root:latest + +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. The pocketbase container creates it on its first start and +# leaves it alone after that: to change the password, change it in the +# admin UI first, then here. It needs 8 characters or more. +WARP_PB_SUPERUSER_EMAIL=admin@example.org +WARP_PB_SUPERUSER_PASSWORD=change-me-now + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. -# The homeserver runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The homeserver runs as UID 10001, so its directory must be writable by +# it; PocketBase runs as root. +# The homeserver's /data (warp-data): its signing key. WARP_DATA_VOLUME= +# PocketBase's pb_data (pb-data): the database, and media unless it is +# in S3. +WARP_PB_DATA_VOLUME= # ------------------------------------------------------------ homeserver diff --git a/Docker/.env.pocketbase.prod.example b/Docker/.env.pocketbase.prod.example index dfef386..71fd30d 100644 --- a/Docker/.env.pocketbase.prod.example +++ b/Docker/.env.pocketbase.prod.example @@ -5,6 +5,9 @@ # cp .env.pocketbase.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.prod.yml up -d # +# Required, with no default: WARP_PB_SUPERUSER_EMAIL and +# WARP_PB_SUPERUSER_PASSWORD. +# # Compose reads .env for the values it puts into the compose file, and hands # all of it to the containers, so a change takes effect on the next `up -d`. # An empty value means the built-in default, given in the comment above it; @@ -26,11 +29,31 @@ WARP_WEB_PORT=8080 WARP_REGISTRY=10.2.1.10:5500/admin WARP_VERSION=latest +# The PocketBase image; any PocketBase 0.40 or later answers the same. +WARP_PB_IMAGE=tajniak81/pocketbase-docker-root:latest + +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. The pocketbase container creates it on its first start and +# leaves it alone after that: to change the password, change it in the +# admin UI first, then here. It needs 8 characters or more. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. -# The homeserver runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The homeserver runs as UID 10001, so its directory must be writable by +# it; PocketBase runs as root. +# The homeserver's /data (warp-data): its signing key. WARP_DATA_VOLUME= +# PocketBase's pb_data (pb-data): the database, and media unless it is +# in S3. +WARP_PB_DATA_VOLUME= # ------------------------------------------------------------ homeserver diff --git a/Docker/.env.pocketbase.s3.example b/Docker/.env.pocketbase.s3.example index 454e3f5..84ff714 100644 --- a/Docker/.env.pocketbase.s3.example +++ b/Docker/.env.pocketbase.s3.example @@ -24,15 +24,35 @@ WARP_PORT=8008 WARP_ADMIN_PORT=8009 WARP_WEB_PORT=8080 +# The PocketBase image; any PocketBase 0.40 or later answers the same. +WARP_PB_IMAGE=tajniak81/pocketbase-docker-root:latest + +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. The pocketbase container creates it on its first start and +# leaves it alone after that: to change the password, change it in the +# admin UI first, then here. It needs 8 characters or more. +WARP_PB_SUPERUSER_EMAIL=admin@example.org +WARP_PB_SUPERUSER_PASSWORD=change-me-now + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. -# The homeserver runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The homeserver runs as UID 10001, so its directory must be writable by +# it; PocketBase runs as root. +# The homeserver's /data (warp-data): its signing key. WARP_DATA_VOLUME= +# PocketBase's pb_data (pb-data): the database, and media unless it is +# in S3. +WARP_PB_DATA_VOLUME= # ------------------------------------------------------------- media (S3) -# Media goes to this bucket; the database stays in /data/pb_data. The bucket +# Media goes to this bucket; the database stays in PocketBase. The bucket # must already exist: nothing here runs a gateway or creates it. For a # gateway on the Docker host, use http://host.docker.internal:8333. Media # stored before S3 was on is not moved into the bucket. diff --git a/Docker/.env.pocketbase.s3.prod.example b/Docker/.env.pocketbase.s3.prod.example index 209fa97..cadf3ba 100644 --- a/Docker/.env.pocketbase.s3.prod.example +++ b/Docker/.env.pocketbase.s3.prod.example @@ -5,8 +5,9 @@ # cp .env.pocketbase.s3.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.s3.prod.yml up -d # -# Required, with no default: WARP_PB_S3_ENDPOINT, WARP_PB_S3_ACCESS_KEY -# and WARP_PB_S3_SECRET. +# Required, with no default: WARP_PB_SUPERUSER_EMAIL, +# WARP_PB_SUPERUSER_PASSWORD, WARP_PB_S3_ENDPOINT, WARP_PB_S3_ACCESS_KEY and +# WARP_PB_S3_SECRET. # # Compose reads .env for the values it puts into the compose file, and hands # all of it to the containers, so a change takes effect on the next `up -d`. @@ -29,15 +30,35 @@ WARP_WEB_PORT=8080 WARP_REGISTRY=10.2.1.10:5500/admin WARP_VERSION=latest +# The PocketBase image; any PocketBase 0.40 or later answers the same. +WARP_PB_IMAGE=tajniak81/pocketbase-docker-root:latest + +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. The pocketbase container creates it on its first start and +# leaves it alone after that: to change the password, change it in the +# admin UI first, then here. It needs 8 characters or more. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. -# The homeserver runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The homeserver runs as UID 10001, so its directory must be writable by +# it; PocketBase runs as root. +# The homeserver's /data (warp-data): its signing key. WARP_DATA_VOLUME= +# PocketBase's pb_data (pb-data): the database, and media unless it is +# in S3. +WARP_PB_DATA_VOLUME= # ------------------------------------------------------------- media (S3) -# Media goes to this bucket; the database stays in /data/pb_data. The bucket +# Media goes to this bucket; the database stays in PocketBase. The bucket # must already exist: nothing here runs a gateway or creates it. For a # gateway on the Docker host, use http://host.docker.internal:8333. Media # stored before S3 was on is not moved into the bucket. diff --git a/Docker/.env.pocketbase.seaweedfs.example b/Docker/.env.pocketbase.seaweedfs.example index e4adbb5..2ab1d59 100644 --- a/Docker/.env.pocketbase.seaweedfs.example +++ b/Docker/.env.pocketbase.seaweedfs.example @@ -23,18 +23,38 @@ WARP_PORT=8008 WARP_ADMIN_PORT=8009 WARP_WEB_PORT=8080 +# The PocketBase image; any PocketBase 0.40 or later answers the same. +WARP_PB_IMAGE=tajniak81/pocketbase-docker-root:latest + +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. The pocketbase container creates it on its first start and +# leaves it alone after that: to change the password, change it in the +# admin UI first, then here. It needs 8 characters or more. +WARP_PB_SUPERUSER_EMAIL=admin@example.org +WARP_PB_SUPERUSER_PASSWORD=change-me-now + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. -# The homeserver runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The homeserver runs as UID 10001, so its directory must be writable by +# it; PocketBase runs as root. +# The homeserver's /data (warp-data): its signing key. WARP_DATA_VOLUME= +# PocketBase's pb_data (pb-data): the database, and media unless it is +# in S3. +WARP_PB_DATA_VOLUME= # SeaweedFS's data (seaweed-data). SEAWEED_DATA_VOLUME= # -------------------------------------------------------------- seaweedfs # Media goes to this bucket in SeaweedFS, created on start; the database -# stays in /data/pb_data. The key pair is the homeserver's S3 identity: +# stays in PocketBase. The key pair is the homeserver's S3 identity: # the gateway's only one, and what the homeserver signs with. # Media stored before S3 was on is not moved into the bucket. WARP_PB_S3_BUCKET=warp diff --git a/Docker/.env.pocketbase.seaweedfs.prod.example b/Docker/.env.pocketbase.seaweedfs.prod.example index 901811f..648da02 100644 --- a/Docker/.env.pocketbase.seaweedfs.prod.example +++ b/Docker/.env.pocketbase.seaweedfs.prod.example @@ -5,7 +5,8 @@ # cp .env.pocketbase.seaweedfs.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.prod.yml up -d # -# Required, with no default: WARP_PB_S3_ACCESS_KEY and WARP_PB_S3_SECRET. +# Required, with no default: WARP_PB_SUPERUSER_EMAIL, +# WARP_PB_SUPERUSER_PASSWORD, WARP_PB_S3_ACCESS_KEY and WARP_PB_S3_SECRET. # # Compose reads .env for the values it puts into the compose file, and hands # all of it to the containers, so a change takes effect on the next `up -d`. @@ -28,18 +29,38 @@ WARP_WEB_PORT=8080 WARP_REGISTRY=10.2.1.10:5500/admin WARP_VERSION=latest +# The PocketBase image; any PocketBase 0.40 or later answers the same. +WARP_PB_IMAGE=tajniak81/pocketbase-docker-root:latest + +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. The pocketbase container creates it on its first start and +# leaves it alone after that: to change the password, change it in the +# admin UI first, then here. It needs 8 characters or more. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. -# The homeserver runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The homeserver runs as UID 10001, so its directory must be writable by +# it; PocketBase runs as root. +# The homeserver's /data (warp-data): its signing key. WARP_DATA_VOLUME= +# PocketBase's pb_data (pb-data): the database, and media unless it is +# in S3. +WARP_PB_DATA_VOLUME= # SeaweedFS's data (seaweed-data). SEAWEED_DATA_VOLUME= # -------------------------------------------------------------- seaweedfs # Media goes to this bucket in SeaweedFS, created on start; the database -# stays in /data/pb_data. The key pair is the homeserver's S3 identity: +# stays in PocketBase. The key pair is the homeserver's S3 identity: # the gateway's only one, and what the homeserver signs with. # Media stored before S3 was on is not moved into the bucket. WARP_PB_S3_BUCKET=warp diff --git a/Docker/.env.pocketbase.seaweedfs.split.example b/Docker/.env.pocketbase.seaweedfs.split.example index b37c37c..da78c07 100644 --- a/Docker/.env.pocketbase.seaweedfs.split.example +++ b/Docker/.env.pocketbase.seaweedfs.split.example @@ -24,11 +24,31 @@ WARP_PORT=8008 WARP_ADMIN_PORT=8009 WARP_WEB_PORT=8080 +# The PocketBase image; any PocketBase 0.40 or later answers the same. +WARP_PB_IMAGE=tajniak81/pocketbase-docker-root:latest + +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. The pocketbase container creates it on its first start and +# leaves it alone after that: to change the password, change it in the +# admin UI first, then here. It needs 8 characters or more. +WARP_PB_SUPERUSER_EMAIL=admin@example.org +WARP_PB_SUPERUSER_PASSWORD=change-me-now + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. -# The homeserver runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The homeserver runs as UID 10001, so its directory must be writable by +# it; PocketBase runs as root. +# The homeserver's /data (warp-data): its signing key. WARP_DATA_VOLUME= +# PocketBase's pb_data (pb-data): the database, and media unless it is +# in S3. +WARP_PB_DATA_VOLUME= # SeaweedFS's data (seaweed-data). SEAWEED_DATA_VOLUME= # SeaweedFS's admin UI state (seaweed-admin). @@ -37,7 +57,7 @@ SEAWEED_ADMIN_VOLUME= # -------------------------------------------------------------- seaweedfs # Media goes to this bucket in SeaweedFS, created on start; the database -# stays in /data/pb_data. The key pair is the homeserver's S3 identity: +# stays in PocketBase. The key pair is the homeserver's S3 identity: # written into the filer's IAM store on every start, beside any identity # added in the admin UI, so a new secret here lands on the next start. # Media stored before S3 was on is not moved into the bucket. diff --git a/Docker/.env.pocketbase.seaweedfs.split.prod.example b/Docker/.env.pocketbase.seaweedfs.split.prod.example index aa85517..6f9ba93 100644 --- a/Docker/.env.pocketbase.seaweedfs.split.prod.example +++ b/Docker/.env.pocketbase.seaweedfs.split.prod.example @@ -5,7 +5,8 @@ # cp .env.pocketbase.seaweedfs.split.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.split.prod.yml up -d # -# Required, with no default: WARP_PB_S3_ACCESS_KEY, WARP_PB_S3_SECRET and +# Required, with no default: WARP_PB_SUPERUSER_EMAIL, +# WARP_PB_SUPERUSER_PASSWORD, WARP_PB_S3_ACCESS_KEY, WARP_PB_S3_SECRET and # SEAWEED_ADMIN_PASSWORD. # # Compose reads .env for the values it puts into the compose file, and hands @@ -29,11 +30,31 @@ WARP_WEB_PORT=8080 WARP_REGISTRY=10.2.1.10:5500/admin WARP_VERSION=latest +# The PocketBase image; any PocketBase 0.40 or later answers the same. +WARP_PB_IMAGE=tajniak81/pocketbase-docker-root:latest + +# PocketBase's port, for its admin UI at http://localhost:8070/_/. It is +# on loopback unless the bind address says otherwise; put the UI behind a +# reverse proxy before binding it to 0.0.0.0. +WARP_PB_BIND=127.0.0.1 +WARP_PB_PORT=8070 + +# The PocketBase superuser the homeserver signs in as, and so can you, in +# the admin UI. The pocketbase container creates it on its first start and +# leaves it alone after that: to change the password, change it in the +# admin UI first, then here. It needs 8 characters or more. +WARP_PB_SUPERUSER_EMAIL= +WARP_PB_SUPERUSER_PASSWORD= + # Where the data lives. Empty keeps the named volume in brackets; a # host path (starting with . or /) bind-mounts that directory instead. -# The homeserver runs as UID 10001, so its directory must be writable by it. -# The homeserver's /data (warp-data). +# The homeserver runs as UID 10001, so its directory must be writable by +# it; PocketBase runs as root. +# The homeserver's /data (warp-data): its signing key. WARP_DATA_VOLUME= +# PocketBase's pb_data (pb-data): the database, and media unless it is +# in S3. +WARP_PB_DATA_VOLUME= # SeaweedFS's data (seaweed-data). SEAWEED_DATA_VOLUME= # SeaweedFS's admin UI state (seaweed-admin). @@ -42,7 +63,7 @@ SEAWEED_ADMIN_VOLUME= # -------------------------------------------------------------- seaweedfs # Media goes to this bucket in SeaweedFS, created on start; the database -# stays in /data/pb_data. The key pair is the homeserver's S3 identity: +# stays in PocketBase. The key pair is the homeserver's S3 identity: # written into the filer's IAM store on every start, beside any identity # added in the admin UI, so a new secret here lands on the next start. # Media stored before S3 was on is not moved into the bucket. diff --git a/Docker/README.md b/Docker/README.md index aa84ac6..ac3ddd8 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -7,7 +7,7 @@ The homeserver (`warp`), the admin panel (`warp-admin`) and the web client | File | Database | Images | | --- | --- | --- | | `docker-compose.sqlite.yml` | SQLite, in the `warp-data` volume | built here | -| `docker-compose.pocketbase.yml` | PocketBase, embedded in the homeserver, in `warp-data` | built here | +| `docker-compose.pocketbase.yml` | PocketBase, a container of its own, in `pb-data` | built here | | `docker-compose.postgresql.yml` | PostgreSQL 17, its own container | built here | | `docker-compose.pocketbase.s3.yml` | PocketBase, media in an existing S3 bucket elsewhere | built here | | `docker-compose.pocketbase.seaweedfs.yml` | PocketBase, media in SeaweedFS, one container | built here | @@ -26,16 +26,22 @@ Homeserver on `:8008`, admin panel on `:8009`, web client on `:8080` TLS-terminating reverse proxy in front before exposing them; see [docs/deployment.md](../docs/deployment.md). -PocketBase is a database driver inside the homeserver, not a separate -server, so the PocketBase files have no PocketBase container. +The PocketBase files run PocketBase as a container of its own, from +`tajniak81/pocketbase-docker-root` (`WARP_PB_IMAGE` picks another; any +PocketBase 0.40 or later will do). Its data is in the `pb-data` volume, and +the homeserver reaches it at `http://pocketbase:8070`, signed in as the +superuser in `.env`. The image creates that superuser on its first start +and leaves it alone after, so a new password is set in the admin UI first, +at `http://localhost:8070/_/` on loopback, and in `.env` after. ## Media in S3 The three `pocketbase.s3` and `pocketbase.seaweedfs` setups keep the -PocketBase database in `/data/pb_data` and put media in an S3 bucket +PocketBase database in `pb-data` and put media in an S3 bucket (`WARP_PB_S3_BUCKET`, default `warp`). Media already stored on the filesystem is not moved into the bucket when S3 is turned on, so start -these on a fresh `warp-data` or copy the files across first. +these on a fresh `pb-data` or copy the files across first. It is the +PocketBase container that talks to the bucket. - **`docker-compose.pocketbase.s3.yml`**: a bucket that already exists elsewhere (AWS, MinIO, a SeaweedFS on another host). Set diff --git a/Docker/docker-compose.pocketbase.prod.yml b/Docker/docker-compose.pocketbase.prod.yml index d1c7ec2..4840d87 100644 --- a/Docker/docker-compose.pocketbase.prod.yml +++ b/Docker/docker-compose.pocketbase.prod.yml @@ -5,10 +5,11 @@ # cp .env.pocketbase.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.prod.yml up -d # -# PocketBase is embedded in the homeserver (the pocketbase database driver), -# so it is not a container of its own: its data directory, with media in -# its storage, is /data/pb_data in the warp-data volume. Set WARP_PB_S3_* -# in .env to keep media in an S3 bucket instead (docs/configuration.md). +# PocketBase is the homeserver's database, in a container of its own: Warp's +# collections, and media in PocketBase's storage, are in the pb-data volume, +# and the homeserver reaches it at http://pocketbase:8070 as the superuser +# set in .env. Set WARP_PB_S3_* in .env to keep media in an S3 bucket +# instead (docs/configuration.md). # # Homeserver http://localhost:8008, admin panel http://localhost:8009, web # client http://localhost:8080. @@ -26,10 +27,38 @@ name: warp-pocketbase services: + # PocketBase, the homeserver's database: Warp's collections, and media in + # PocketBase's storage, are in the pb-data volume. The image creates the + # superuser from .env on its first start, and never changes it after: a + # new password goes in through the admin UI, then into .env. The admin UI + # is http://localhost:8070/_/ on loopback; WARP_PB_BIND=0.0.0.0 exposes + # it, behind a reverse proxy only. + pocketbase: + image: ${WARP_PB_IMAGE:-tajniak81/pocketbase-docker-root:latest} + pull_policy: always + restart: unless-stopped + environment: + PB_PORT: "8070" + PB_ADMIN_EMAIL: ${WARP_PB_SUPERUSER_EMAIL:?set WARP_PB_SUPERUSER_EMAIL in .env} + PB_ADMIN_PASSWORD: ${WARP_PB_SUPERUSER_PASSWORD:?set WARP_PB_SUPERUSER_PASSWORD in .env} + volumes: + - ${WARP_PB_DATA_VOLUME:-pb-data}:/pb/pb_data + ports: + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8070/api/health >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + warp: image: ${WARP_REGISTRY:-10.2.1.10:5500/admin}/warp:${WARP_VERSION:-latest} pull_policy: always restart: unless-stopped + depends_on: + pocketbase: + condition: service_healthy # The signing key is created on first start and kept in the volume: # losing it means other servers no longer recognise this one. entrypoint: ["/bin/sh", "-c"] @@ -38,7 +67,7 @@ services: env_file: .env environment: WARP_DB_DRIVER: pocketbase - WARP_DB_CONNECTION_STRING: /data/pb_data + WARP_DB_CONNECTION_STRING: http://pocketbase:8070 volumes: - ${WARP_DATA_VOLUME:-warp-data}:/data ports: @@ -69,4 +98,7 @@ services: - "${WARP_WEB_PORT:-8080}:8080" volumes: + # The homeserver's signing key. warp-data: + # PocketBase's pb_data: the database, and media unless it is in S3. + pb-data: diff --git a/Docker/docker-compose.pocketbase.s3.prod.yml b/Docker/docker-compose.pocketbase.s3.prod.yml index e1354ac..49030c2 100644 --- a/Docker/docker-compose.pocketbase.s3.prod.yml +++ b/Docker/docker-compose.pocketbase.s3.prod.yml @@ -6,7 +6,7 @@ # cp .env.pocketbase.s3.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.s3.prod.yml up -d # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# The database is PocketBase's, in pb-data; media goes to the bucket # WARP_PB_S3_BUCKET (default warp) on WARP_PB_S3_ENDPOINT, which must already # exist: nothing here runs a gateway or creates the bucket. For a gateway on # this Docker host, use http://host.docker.internal:8333; the host entry below @@ -31,10 +31,42 @@ name: warp-pocketbase-s3 services: + # PocketBase, the homeserver's database: Warp's collections, and media in + # PocketBase's storage, are in the pb-data volume. The image creates the + # superuser from .env on its first start, and never changes it after: a + # new password goes in through the admin UI, then into .env. The admin UI + # is http://localhost:8070/_/ on loopback; WARP_PB_BIND=0.0.0.0 exposes + # it, behind a reverse proxy only. + pocketbase: + image: ${WARP_PB_IMAGE:-tajniak81/pocketbase-docker-root:latest} + pull_policy: always + restart: unless-stopped + extra_hosts: + # Lets WARP_PB_S3_ENDPOINT name the Docker host as host.docker.internal: + # PocketBase is what talks to the bucket. + - "host.docker.internal:host-gateway" + environment: + PB_PORT: "8070" + PB_ADMIN_EMAIL: ${WARP_PB_SUPERUSER_EMAIL:?set WARP_PB_SUPERUSER_EMAIL in .env} + PB_ADMIN_PASSWORD: ${WARP_PB_SUPERUSER_PASSWORD:?set WARP_PB_SUPERUSER_PASSWORD in .env} + volumes: + - ${WARP_PB_DATA_VOLUME:-pb-data}:/pb/pb_data + ports: + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8070/api/health >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + warp: image: ${WARP_REGISTRY:-10.2.1.10:5500/admin}/warp:${WARP_VERSION:-latest} pull_policy: always restart: unless-stopped + depends_on: + pocketbase: + condition: service_healthy extra_hosts: # Lets WARP_PB_S3_ENDPOINT name the Docker host as host.docker.internal. - "host.docker.internal:host-gateway" @@ -53,7 +85,7 @@ services: env_file: .env environment: WARP_DB_DRIVER: pocketbase - WARP_DB_CONNECTION_STRING: /data/pb_data + WARP_DB_CONNECTION_STRING: http://pocketbase:8070 WARP_PB_S3_ENABLED: "true" WARP_PB_S3_ENDPOINT: ${WARP_PB_S3_ENDPOINT:?set WARP_PB_S3_ENDPOINT in .env} WARP_PB_S3_BUCKET: ${WARP_PB_S3_BUCKET:-warp} @@ -93,4 +125,7 @@ services: - "${WARP_WEB_PORT:-8080}:8080" volumes: + # The homeserver's signing key. warp-data: + # PocketBase's pb_data: the database, and media unless it is in S3. + pb-data: diff --git a/Docker/docker-compose.pocketbase.s3.yml b/Docker/docker-compose.pocketbase.s3.yml index cb68db5..7d4637b 100644 --- a/Docker/docker-compose.pocketbase.s3.yml +++ b/Docker/docker-compose.pocketbase.s3.yml @@ -6,7 +6,7 @@ # cp .env.pocketbase.s3.example .env # then edit it # docker compose -f docker-compose.pocketbase.s3.yml up -d --build # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# The database is PocketBase's, in pb-data; media goes to the bucket # WARP_PB_S3_BUCKET (default warp) on WARP_PB_S3_ENDPOINT, which must already # exist: nothing here runs a gateway or creates the bucket. For a gateway on # this Docker host, use http://host.docker.internal:8333; the host entry below @@ -24,12 +24,43 @@ name: warp-pocketbase-s3 services: + # PocketBase, the homeserver's database: Warp's collections, and media in + # PocketBase's storage, are in the pb-data volume. The image creates the + # superuser from .env on its first start, and never changes it after: a + # new password goes in through the admin UI, then into .env. The admin UI + # is http://localhost:8070/_/ on loopback; WARP_PB_BIND=0.0.0.0 exposes + # it, behind a reverse proxy only. + pocketbase: + image: ${WARP_PB_IMAGE:-tajniak81/pocketbase-docker-root:latest} + restart: unless-stopped + extra_hosts: + # Lets WARP_PB_S3_ENDPOINT name the Docker host as host.docker.internal: + # PocketBase is what talks to the bucket. + - "host.docker.internal:host-gateway" + environment: + PB_PORT: "8070" + PB_ADMIN_EMAIL: ${WARP_PB_SUPERUSER_EMAIL:?set WARP_PB_SUPERUSER_EMAIL in .env} + PB_ADMIN_PASSWORD: ${WARP_PB_SUPERUSER_PASSWORD:?set WARP_PB_SUPERUSER_PASSWORD in .env} + volumes: + - ${WARP_PB_DATA_VOLUME:-pb-data}:/pb/pb_data + ports: + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8070/api/health >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + warp: build: context: .. dockerfile: Dockerfile image: warp:latest restart: unless-stopped + depends_on: + pocketbase: + condition: service_healthy extra_hosts: # Lets WARP_PB_S3_ENDPOINT name the Docker host as host.docker.internal. - "host.docker.internal:host-gateway" @@ -48,7 +79,7 @@ services: env_file: .env environment: WARP_DB_DRIVER: pocketbase - WARP_DB_CONNECTION_STRING: /data/pb_data + WARP_DB_CONNECTION_STRING: http://pocketbase:8070 WARP_PB_S3_ENABLED: "true" WARP_PB_S3_ENDPOINT: ${WARP_PB_S3_ENDPOINT:?set WARP_PB_S3_ENDPOINT in .env} WARP_PB_S3_BUCKET: ${WARP_PB_S3_BUCKET:-warp} @@ -92,4 +123,7 @@ services: - "${WARP_WEB_PORT:-8080}:8080" volumes: + # The homeserver's signing key. warp-data: + # PocketBase's pb_data: the database, and media unless it is in S3. + pb-data: diff --git a/Docker/docker-compose.pocketbase.seaweedfs.prod.yml b/Docker/docker-compose.pocketbase.seaweedfs.prod.yml index 84fe18e..fefa0dd 100644 --- a/Docker/docker-compose.pocketbase.seaweedfs.prod.yml +++ b/Docker/docker-compose.pocketbase.seaweedfs.prod.yml @@ -7,7 +7,7 @@ # cp .env.pocketbase.seaweedfs.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.prod.yml up -d # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# The database is PocketBase's, in pb-data; media goes to the bucket # WARP_PB_S3_BUCKET (default warp) in SeaweedFS (seaweed-data), whose gateway # has exactly one identity: the homeserver's. For SeaweedFS's admin UI and # per-role containers, see docker-compose.pocketbase.seaweedfs.split.yml. @@ -72,11 +72,38 @@ services: command: - echo "s3.bucket.create -name $$WARP_PB_S3_BUCKET" | weed shell -master=seaweedfs:9333 >/dev/null + # PocketBase, the homeserver's database: Warp's collections, and media in + # PocketBase's storage, are in the pb-data volume. The image creates the + # superuser from .env on its first start, and never changes it after: a + # new password goes in through the admin UI, then into .env. The admin UI + # is http://localhost:8070/_/ on loopback; WARP_PB_BIND=0.0.0.0 exposes + # it, behind a reverse proxy only. + pocketbase: + image: ${WARP_PB_IMAGE:-tajniak81/pocketbase-docker-root:latest} + pull_policy: always + restart: unless-stopped + environment: + PB_PORT: "8070" + PB_ADMIN_EMAIL: ${WARP_PB_SUPERUSER_EMAIL:?set WARP_PB_SUPERUSER_EMAIL in .env} + PB_ADMIN_PASSWORD: ${WARP_PB_SUPERUSER_PASSWORD:?set WARP_PB_SUPERUSER_PASSWORD in .env} + volumes: + - ${WARP_PB_DATA_VOLUME:-pb-data}:/pb/pb_data + ports: + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8070/api/health >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + warp: image: ${WARP_REGISTRY:-10.2.1.10:5500/admin}/warp:${WARP_VERSION:-latest} pull_policy: always restart: unless-stopped depends_on: + pocketbase: + condition: service_healthy seaweedfs: condition: service_healthy seaweedfs-init: @@ -89,7 +116,7 @@ services: env_file: .env environment: WARP_DB_DRIVER: pocketbase - WARP_DB_CONNECTION_STRING: /data/pb_data + WARP_DB_CONNECTION_STRING: http://pocketbase:8070 # The gateway on the compose network; these override whatever .env # says about S3. WARP_PB_S3_ENABLED: "true" @@ -129,5 +156,8 @@ services: - "${WARP_WEB_PORT:-8080}:8080" volumes: + # The homeserver's signing key. warp-data: + # PocketBase's pb_data: the database, and media unless it is in S3. + pb-data: seaweed-data: diff --git a/Docker/docker-compose.pocketbase.seaweedfs.split.prod.yml b/Docker/docker-compose.pocketbase.seaweedfs.split.prod.yml index a6dfcab..a3c9787 100644 --- a/Docker/docker-compose.pocketbase.seaweedfs.split.prod.yml +++ b/Docker/docker-compose.pocketbase.seaweedfs.split.prod.yml @@ -7,7 +7,7 @@ # cp .env.pocketbase.seaweedfs.split.prod.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.split.prod.yml up -d # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# The database is PocketBase's, in pb-data; media goes to the bucket # WARP_PB_S3_BUCKET (default warp) in SeaweedFS (seaweed-data). Over # docker-compose.pocketbase.seaweedfs.yml this buys SeaweedFS's admin UI, # where S3 identities are created and revoked under Object Store → Users, @@ -177,11 +177,38 @@ services: retries: 12 start_period: 15s + # PocketBase, the homeserver's database: Warp's collections, and media in + # PocketBase's storage, are in the pb-data volume. The image creates the + # superuser from .env on its first start, and never changes it after: a + # new password goes in through the admin UI, then into .env. The admin UI + # is http://localhost:8070/_/ on loopback; WARP_PB_BIND=0.0.0.0 exposes + # it, behind a reverse proxy only. + pocketbase: + image: ${WARP_PB_IMAGE:-tajniak81/pocketbase-docker-root:latest} + pull_policy: always + restart: unless-stopped + environment: + PB_PORT: "8070" + PB_ADMIN_EMAIL: ${WARP_PB_SUPERUSER_EMAIL:?set WARP_PB_SUPERUSER_EMAIL in .env} + PB_ADMIN_PASSWORD: ${WARP_PB_SUPERUSER_PASSWORD:?set WARP_PB_SUPERUSER_PASSWORD in .env} + volumes: + - ${WARP_PB_DATA_VOLUME:-pb-data}:/pb/pb_data + ports: + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8070/api/health >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + warp: image: ${WARP_REGISTRY:-10.2.1.10:5500/admin}/warp:${WARP_VERSION:-latest} pull_policy: always restart: unless-stopped depends_on: + pocketbase: + condition: service_healthy seaweedfs-s3: condition: service_healthy # The signing key is created on first start and kept in the volume: @@ -192,7 +219,7 @@ services: env_file: .env environment: WARP_DB_DRIVER: pocketbase - WARP_DB_CONNECTION_STRING: /data/pb_data + WARP_DB_CONNECTION_STRING: http://pocketbase:8070 # The gateway on the compose network; these override whatever .env # says about S3. WARP_PB_S3_ENABLED: "true" @@ -232,7 +259,10 @@ services: - "${WARP_WEB_PORT:-8080}:8080" volumes: + # The homeserver's signing key. warp-data: + # PocketBase's pb_data: the database, and media unless it is in S3. + pb-data: # Master, volume and filer share it, as `weed server -dir` lays it out. seaweed-data: # The admin UI's session key and maintenance state; no object data. diff --git a/Docker/docker-compose.pocketbase.seaweedfs.split.yml b/Docker/docker-compose.pocketbase.seaweedfs.split.yml index 08d22f1..e4970ac 100644 --- a/Docker/docker-compose.pocketbase.seaweedfs.split.yml +++ b/Docker/docker-compose.pocketbase.seaweedfs.split.yml @@ -7,7 +7,7 @@ # cp .env.pocketbase.seaweedfs.split.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.split.yml up -d --build # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# The database is PocketBase's, in pb-data; media goes to the bucket # WARP_PB_S3_BUCKET (default warp) in SeaweedFS (seaweed-data). Over # docker-compose.pocketbase.seaweedfs.yml this buys SeaweedFS's admin UI, # where S3 identities are created and revoked under Object Store → Users, @@ -170,6 +170,30 @@ services: retries: 12 start_period: 15s + # PocketBase, the homeserver's database: Warp's collections, and media in + # PocketBase's storage, are in the pb-data volume. The image creates the + # superuser from .env on its first start, and never changes it after: a + # new password goes in through the admin UI, then into .env. The admin UI + # is http://localhost:8070/_/ on loopback; WARP_PB_BIND=0.0.0.0 exposes + # it, behind a reverse proxy only. + pocketbase: + image: ${WARP_PB_IMAGE:-tajniak81/pocketbase-docker-root:latest} + restart: unless-stopped + environment: + PB_PORT: "8070" + PB_ADMIN_EMAIL: ${WARP_PB_SUPERUSER_EMAIL:?set WARP_PB_SUPERUSER_EMAIL in .env} + PB_ADMIN_PASSWORD: ${WARP_PB_SUPERUSER_PASSWORD:?set WARP_PB_SUPERUSER_PASSWORD in .env} + volumes: + - ${WARP_PB_DATA_VOLUME:-pb-data}:/pb/pb_data + ports: + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8070/api/health >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + warp: build: context: .. @@ -177,6 +201,8 @@ services: image: warp:latest restart: unless-stopped depends_on: + pocketbase: + condition: service_healthy seaweedfs-s3: condition: service_healthy # The signing key is created on first start and kept in the volume: @@ -187,7 +213,7 @@ services: env_file: .env environment: WARP_DB_DRIVER: pocketbase - WARP_DB_CONNECTION_STRING: /data/pb_data + WARP_DB_CONNECTION_STRING: http://pocketbase:8070 # The gateway on the compose network; these override whatever .env # says about S3. WARP_PB_S3_ENABLED: "true" @@ -231,7 +257,10 @@ services: - "${WARP_WEB_PORT:-8080}:8080" volumes: + # The homeserver's signing key. warp-data: + # PocketBase's pb_data: the database, and media unless it is in S3. + pb-data: # Master, volume and filer share it, as `weed server -dir` lays it out. seaweed-data: # The admin UI's session key and maintenance state; no object data. diff --git a/Docker/docker-compose.pocketbase.seaweedfs.yml b/Docker/docker-compose.pocketbase.seaweedfs.yml index 6c49940..875c142 100644 --- a/Docker/docker-compose.pocketbase.seaweedfs.yml +++ b/Docker/docker-compose.pocketbase.seaweedfs.yml @@ -7,7 +7,7 @@ # cp .env.pocketbase.seaweedfs.example .env # then edit it # docker compose -f docker-compose.pocketbase.seaweedfs.yml up -d --build # -# The database stays in /data/pb_data (warp-data); media goes to the bucket +# The database is PocketBase's, in pb-data; media goes to the bucket # WARP_PB_S3_BUCKET (default warp) in SeaweedFS (seaweed-data), whose gateway # has exactly one identity: the homeserver's. For SeaweedFS's admin UI and # per-role containers, see docker-compose.pocketbase.seaweedfs.split.yml. @@ -65,6 +65,30 @@ services: command: - echo "s3.bucket.create -name $$WARP_PB_S3_BUCKET" | weed shell -master=seaweedfs:9333 >/dev/null + # PocketBase, the homeserver's database: Warp's collections, and media in + # PocketBase's storage, are in the pb-data volume. The image creates the + # superuser from .env on its first start, and never changes it after: a + # new password goes in through the admin UI, then into .env. The admin UI + # is http://localhost:8070/_/ on loopback; WARP_PB_BIND=0.0.0.0 exposes + # it, behind a reverse proxy only. + pocketbase: + image: ${WARP_PB_IMAGE:-tajniak81/pocketbase-docker-root:latest} + restart: unless-stopped + environment: + PB_PORT: "8070" + PB_ADMIN_EMAIL: ${WARP_PB_SUPERUSER_EMAIL:?set WARP_PB_SUPERUSER_EMAIL in .env} + PB_ADMIN_PASSWORD: ${WARP_PB_SUPERUSER_PASSWORD:?set WARP_PB_SUPERUSER_PASSWORD in .env} + volumes: + - ${WARP_PB_DATA_VOLUME:-pb-data}:/pb/pb_data + ports: + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8070/api/health >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + warp: build: context: .. @@ -72,6 +96,8 @@ services: image: warp:latest restart: unless-stopped depends_on: + pocketbase: + condition: service_healthy seaweedfs: condition: service_healthy seaweedfs-init: @@ -84,7 +110,7 @@ services: env_file: .env environment: WARP_DB_DRIVER: pocketbase - WARP_DB_CONNECTION_STRING: /data/pb_data + WARP_DB_CONNECTION_STRING: http://pocketbase:8070 # The gateway on the compose network; these override whatever .env # says about S3. WARP_PB_S3_ENABLED: "true" @@ -128,5 +154,8 @@ services: - "${WARP_WEB_PORT:-8080}:8080" volumes: + # The homeserver's signing key. warp-data: + # PocketBase's pb_data: the database, and media unless it is in S3. + pb-data: seaweed-data: diff --git a/Docker/docker-compose.pocketbase.yml b/Docker/docker-compose.pocketbase.yml index b2f6c82..7b2ab60 100644 --- a/Docker/docker-compose.pocketbase.yml +++ b/Docker/docker-compose.pocketbase.yml @@ -5,10 +5,11 @@ # cp .env.pocketbase.example .env # then edit it # docker compose -f docker-compose.pocketbase.yml up -d --build # -# PocketBase is embedded in the homeserver (the pocketbase database driver), -# so it is not a container of its own: its data directory, with media in -# its storage, is /data/pb_data in the warp-data volume. Set WARP_PB_S3_* -# in .env to keep media in an S3 bucket instead (docs/configuration.md). +# PocketBase is the homeserver's database, in a container of its own: Warp's +# collections, and media in PocketBase's storage, are in the pb-data volume, +# and the homeserver reaches it at http://pocketbase:8070 as the superuser +# set in .env. Set WARP_PB_S3_* in .env to keep media in an S3 bucket +# instead (docs/configuration.md). # # Homeserver http://localhost:8008, admin panel http://localhost:8009, web # client http://localhost:8080. @@ -20,12 +21,39 @@ name: warp-pocketbase services: + # PocketBase, the homeserver's database: Warp's collections, and media in + # PocketBase's storage, are in the pb-data volume. The image creates the + # superuser from .env on its first start, and never changes it after: a + # new password goes in through the admin UI, then into .env. The admin UI + # is http://localhost:8070/_/ on loopback; WARP_PB_BIND=0.0.0.0 exposes + # it, behind a reverse proxy only. + pocketbase: + image: ${WARP_PB_IMAGE:-tajniak81/pocketbase-docker-root:latest} + restart: unless-stopped + environment: + PB_PORT: "8070" + PB_ADMIN_EMAIL: ${WARP_PB_SUPERUSER_EMAIL:?set WARP_PB_SUPERUSER_EMAIL in .env} + PB_ADMIN_PASSWORD: ${WARP_PB_SUPERUSER_PASSWORD:?set WARP_PB_SUPERUSER_PASSWORD in .env} + volumes: + - ${WARP_PB_DATA_VOLUME:-pb-data}:/pb/pb_data + ports: + - "${WARP_PB_BIND:-127.0.0.1}:${WARP_PB_PORT:-8070}:8070" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8070/api/health >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + warp: build: context: .. dockerfile: Dockerfile image: warp:latest restart: unless-stopped + depends_on: + pocketbase: + condition: service_healthy # The signing key is created on first start and kept in the volume: # losing it means other servers no longer recognise this one. entrypoint: ["/bin/sh", "-c"] @@ -34,7 +62,7 @@ services: env_file: .env environment: WARP_DB_DRIVER: pocketbase - WARP_DB_CONNECTION_STRING: /data/pb_data + WARP_DB_CONNECTION_STRING: http://pocketbase:8070 volumes: - ${WARP_DATA_VOLUME:-warp-data}:/data ports: @@ -69,4 +97,7 @@ services: - "${WARP_WEB_PORT:-8080}:8080" volumes: + # The homeserver's signing key. warp-data: + # PocketBase's pb_data: the database, and media unless it is in S3. + pb-data: diff --git a/README.md b/README.md index 8cf74a2..04d8558 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,9 @@ carries its thread summary, latest edit and references in through the federation hierarchy endpoint, and `room_summary`. `/search` runs on SQLite's FTS5 or PostgreSQL's `tsvector`, bounded to what each user may see, with context, profiles, state and grouping. The PocketBase driver -is real: the SQLite schema inside an embedded PocketBase app, with media in -its filesystem — local or S3 — and it passes the same storage tests as -SQLite. OpenTelemetry tracing follows a request from the client through +keeps Warp's data as collections on a PocketBase server of its own, through +its REST API, with media in its storage — local or S3 — and it passes the +same storage tests as SQLite. OpenTelemetry tracing follows a request from the client through federation, push and application service calls; `/metrics` gains event, sync and database pool metrics, with a Grafana dashboard over them. There is a Compose setup, a Helm chart and a load tester (`warp-loadtest`), and diff --git a/docs/configuration.md b/docs/configuration.md index fc5760f..7b4fd81 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -37,19 +37,17 @@ List-valued variables are comma-separated. A single `-` clears a list: | `listen.tls_cert_file` | `WARP_TLS_CERT_FILE` | empty | PEM certificate chain for the federation listener. Required with it. | | `listen.tls_key_file` | `WARP_TLS_KEY_FILE` | empty | PEM key for the federation listener. Required with it. | | `database.driver` | `WARP_DB_DRIVER` | `sqlite` | `sqlite`, `postgres`, `pocketbase`. | -| `database.connection_string` | `WARP_DB_CONNECTION_STRING` | SQLite file in the working directory | A DSN for postgres, a `file:` URI for sqlite, the data directory (PocketBase's `pb_data`) for pocketbase. | +| `database.connection_string` | `WARP_DB_CONNECTION_STRING` | SQLite file in the working directory | A DSN for postgres, a `file:` URI for sqlite, the PocketBase server's URL, e.g. `http://pocketbase:8070`, for pocketbase. | | `database.max_open_conns` | `WARP_DB_MAX_OPEN_CONNS` | `10` | Ignored by SQLite: writes are serialised onto one connection. | | `database.max_idle_conns` | `WARP_DB_MAX_IDLE_CONNS` | `5` | | | `database.conn_max_lifetime` | `WARP_DB_CONN_MAX_LIFETIME` | `1h` | | | `database.auto_migrate` | `WARP_DB_AUTO_MIGRATE` | `true` | `false` makes startup fail while migrations are pending. | -| `database.pocketbase.s3.enabled` | `WARP_PB_S3_ENABLED` | `false` | pocketbase only: media content goes to an S3-compatible bucket instead of `pb_data/storage`. | -| `database.pocketbase.s3.bucket` / `.region` / `.endpoint` | `WARP_PB_S3_BUCKET` / `WARP_PB_S3_REGION` / `WARP_PB_S3_ENDPOINT` | empty | All required when S3 is on. | +| `database.pocketbase.superuser_email` / `.superuser_password` | `WARP_PB_SUPERUSER_EMAIL` / `WARP_PB_SUPERUSER_PASSWORD` | empty | pocketbase only, and required: the PocketBase superuser Warp signs in as. | +| `database.pocketbase.superuser_file` | `WARP_PB_SUPERUSER_FILE` | empty | A file of `WARP_PB_SUPERUSER_EMAIL=` and `WARP_PB_SUPERUSER_PASSWORD=` lines, read for whichever is not set otherwise. The all-in-one images keep the superuser they make up there. | +| `database.pocketbase.s3.enabled` | `WARP_PB_S3_ENABLED` | `false` | pocketbase only: media content goes to an S3-compatible bucket instead of PocketBase's `pb_data/storage`. Warp writes it into PocketBase's settings on every start; PocketBase is what talks to the bucket. | +| `database.pocketbase.s3.bucket` / `.region` / `.endpoint` | `WARP_PB_S3_BUCKET` / `WARP_PB_S3_REGION` / `WARP_PB_S3_ENDPOINT` | empty | All required when S3 is on. The endpoint has to be reachable from the PocketBase server. | | `database.pocketbase.s3.access_key` / `.secret` | `WARP_PB_S3_ACCESS_KEY` / `WARP_PB_S3_SECRET` | empty | All required when S3 is on. | | `database.pocketbase.s3.force_path_style` | `WARP_PB_S3_FORCE_PATH_STYLE` | `false` | For MinIO and other servers without virtual-host buckets. | -| `database.pocketbase.s3.enabled` | `WARP_PB_S3_ENABLED` | `false` | pocketbase only: media content in an S3-compatible bucket instead of `pb_data/storage`. | -| `database.pocketbase.s3.bucket` / `.region` / `.endpoint` | `WARP_PB_S3_BUCKET` / `WARP_PB_S3_REGION` / `WARP_PB_S3_ENDPOINT` | empty | All required with S3 on. | -| `database.pocketbase.s3.access_key` / `.secret` | `WARP_PB_S3_ACCESS_KEY` / `WARP_PB_S3_SECRET` | empty | All required with S3 on. | -| `database.pocketbase.s3.force_path_style` | `WARP_PB_S3_FORCE_PATH_STYLE` | `false` | For MinIO and other servers without virtual-host buckets. | | `client_api.cors_allowed_origins` | `WARP_CORS_ALLOWED_ORIGINS` | `["*"]` | The spec expects any origin. | | `client_api.trust_x_forwarded_for` | `WARP_TRUST_X_FORWARDED_FOR` | `false` | Only behind a proxy that overwrites the header; the rate limiter keys on the result. | | `client_api.access_token_lifetime` | `WARP_ACCESS_TOKEN_LIFETIME` | `0s` | `0` = never expire. With a lifetime, clients may ask for a refresh token at login. | diff --git a/docs/deployment.md b/docs/deployment.md index 26a75ab..cb13a13 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -12,15 +12,24 @@ tester, which is `server/cmd/warp-loadtest`. |---|---|---| | `postgres` | production | Several connections, so reads and writes run side by side. | | `sqlite` | development, a single user, a small server | One writer. Everything is one file plus `media.base_path`. | -| `pocketbase` | an "appliance": one binary, one data directory | The sqlite driver's tables inside an embedded [PocketBase](https://pocketbase.io) app's `data.db`, written through PocketBase's own single-writer connection. Media goes into PocketBase's filesystem, which can be an S3 bucket (`database.pocketbase.s3`). | +| `pocketbase` | a [PocketBase](https://pocketbase.io) server you want to see and manage the data in | Warp's tables as PocketBase collections on a PocketBase server of its own (0.40 or later), reached only through its REST API. Media goes into PocketBase's storage, which can be an S3 bucket (`database.pocketbase.s3`). | -The pocketbase driver is the SQLite schema on the SQLite engine, so it is -at parity with sqlite by construction, and the storage tests run against -both. What it adds is PocketBase around it: its system tables in the same -database, its `auxiliary.db` for its own logs, and its filesystem -abstraction for media. Warp does not serve PocketBase's admin UI or API; -the app is reachable from Go through `pocketbase.Database.App()` for -features that want it later. Back up the whole data directory. +The pocketbase driver connects to PocketBase the way the postgres driver +connects to PostgreSQL: `database.connection_string` is the server's URL, +and Warp signs in as the superuser in `database.pocketbase`. Every table +the SQL drivers have is a superuser-only collection of the same name and +columns, so the PocketBase admin UI shows Warp's accounts, rooms and events +as records. Warp creates and updates the collections itself (`warp +migrate`), reads them through PocketBase's SQL endpoint with the same +queries the SQL drivers run, and writes them as record operations in batch +requests, each one a transaction. It turns batch requests on in +PocketBase's settings at startup. + +Writes go through one lock in the Warp process, so one Warp server per +PocketBase, and every database call is an HTTP request: expect it to be +slower than PostgreSQL. Search has no full-text engine behind it; it +matches the start of words, case-insensitively. The storage tests run +against a real PocketBase. Back up PocketBase's whole `pb_data`. ## The container image diff --git a/server/go.mod b/server/go.mod index 91fa60c..88aa846 100644 --- a/server/go.mod +++ b/server/go.mod @@ -4,7 +4,6 @@ go 1.26.4 require ( github.com/jackc/pgx/v5 v5.11.0 - github.com/pocketbase/dbx v1.12.0 github.com/pocketbase/pocketbase v0.39.11 github.com/prometheus/client_golang v1.25.0 go.opentelemetry.io/otel v1.47.0 @@ -43,6 +42,7 @@ require ( github.com/mattn/go-isatty v0.0.24 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/pocketbase/dbx v1.12.0 // indirect github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect github.com/prometheus/client_model v0.6.3 // indirect github.com/prometheus/common v0.72.0 // indirect diff --git a/server/internal/config/config.go b/server/internal/config/config.go index 61f81d9..7216bc7 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -6,6 +6,7 @@ import ( "crypto/subtle" "errors" "fmt" + "io/fs" "net/url" "os" "path/filepath" @@ -101,7 +102,7 @@ type Database struct { Driver string `yaml:"driver" env:"WARP_DB_DRIVER"` // ConnectionString is driver specific: a DSN for postgres, a file URI - // for sqlite, a directory for pocketbase. + // for sqlite, the server's URL for pocketbase. ConnectionString string `yaml:"connection_string" env:"WARP_DB_CONNECTION_STRING"` MaxOpenConns int `yaml:"max_open_conns" env:"WARP_DB_MAX_OPEN_CONNS"` @@ -112,14 +113,28 @@ type Database struct { // schema changes behind an explicit `warp migrate`. AutoMigrate bool `yaml:"auto_migrate" env:"WARP_DB_AUTO_MIGRATE"` - // PocketBase configures what the pocketbase driver adds over sqlite. + // PocketBase configures the pocketbase driver: how it signs in to the + // server, and where that server keeps media. PocketBase PocketBase `yaml:"pocketbase"` } // PocketBase configures the pocketbase storage driver. type PocketBase struct { + // SuperuserEmail and SuperuserPassword are the PocketBase superuser + // Warp signs in as. Warp's collections are superuser-only, so nothing + // less can read or write them. + SuperuserEmail string `yaml:"superuser_email" env:"WARP_PB_SUPERUSER_EMAIL"` + SuperuserPassword string `yaml:"superuser_password" env:"WARP_PB_SUPERUSER_PASSWORD"` + + // SuperuserFile names a file of WARP_PB_SUPERUSER_EMAIL=... and + // WARP_PB_SUPERUSER_PASSWORD=... lines, read for whichever of the two + // is not set otherwise. The all-in-one images keep the superuser they + // make up there, so that every warp command in the container finds it. + SuperuserFile string `yaml:"superuser_file" env:"WARP_PB_SUPERUSER_FILE"` + // S3 puts media content in an S3-compatible bucket through - // PocketBase's filesystem, instead of under the data directory. + // PocketBase's filesystem, instead of under its pb_data. Warp writes it + // into PocketBase's settings on every start. S3 PocketBaseS3 `yaml:"s3"` } @@ -143,6 +158,9 @@ func (d Database) Storage() storage.Config { MaxOpenConns: d.MaxOpenConns, MaxIdleConns: d.MaxIdleConns, ConnMaxLifetime: d.ConnMaxLifetime, + + SuperuserEmail: d.PocketBase.SuperuserEmail, + SuperuserPassword: d.PocketBase.SuperuserPassword, S3: storage.S3Config{ Enabled: s3.Enabled, Bucket: s3.Bucket, @@ -709,6 +727,14 @@ func (c *Config) Validate() error { bad("tracing.endpoint: must be an http or https URL") } } + if c.Database.Driver == "pocketbase" { + if u, err := url.Parse(c.Database.ConnectionString); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + bad("database.connection_string: must be the PocketBase server's http or https URL for the pocketbase driver") + } + if c.Database.PocketBase.SuperuserEmail == "" || c.Database.PocketBase.SuperuserPassword == "" { + bad("database.pocketbase: superuser_email and superuser_password are required") + } + } if c.Database.Driver == "pocketbase" && c.Database.PocketBase.S3.Enabled { s3 := c.Database.PocketBase.S3 if s3.Bucket == "" || s3.Region == "" || s3.Endpoint == "" || s3.AccessKey == "" || s3.Secret == "" { @@ -889,8 +915,45 @@ func Load(path string) (*Config, error) { if err := ApplyEnv(cfg, os.Getenv); err != nil { return nil, err } + if err := cfg.Database.PocketBase.readSuperuserFile(); err != nil { + return nil, err + } if err := cfg.Validate(); err != nil { return nil, fmt.Errorf("invalid configuration:\n%w", err) } return cfg, nil } + +// readSuperuserFile fills in the superuser from SuperuserFile, for whichever +// of the email and password is still empty. A file that does not exist yet +// is not an error: the all-in-one entrypoint writes it on first start, and +// a command run before that finds nothing either way. +func (p *PocketBase) readSuperuserFile() error { + if p.SuperuserFile == "" || (p.SuperuserEmail != "" && p.SuperuserPassword != "") { + return nil + } + data, err := os.ReadFile(p.SuperuserFile) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read database.pocketbase.superuser_file: %w", err) + } + for _, line := range strings.Split(string(data), "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + switch key { + case "WARP_PB_SUPERUSER_EMAIL": + if p.SuperuserEmail == "" { + p.SuperuserEmail = value + } + case "WARP_PB_SUPERUSER_PASSWORD": + if p.SuperuserPassword == "" { + p.SuperuserPassword = value + } + } + } + return nil +} diff --git a/server/internal/config/pocketbase_test.go b/server/internal/config/pocketbase_test.go new file mode 100644 index 0000000..385e67e --- /dev/null +++ b/server/internal/config/pocketbase_test.go @@ -0,0 +1,37 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestSuperuserFileFillsInWhatIsUnset(t *testing.T) { + path := filepath.Join(t.TempDir(), "superuser.env") + if err := os.WriteFile(path, []byte("WARP_PB_SUPERUSER_EMAIL=warp@warp.local\nWARP_PB_SUPERUSER_PASSWORD=from-the-file\n"), 0o600); err != nil { + t.Fatal(err) + } + + p := PocketBase{SuperuserFile: path} + if err := p.readSuperuserFile(); err != nil { + t.Fatalf("readSuperuserFile: %v", err) + } + if p.SuperuserEmail != "warp@warp.local" || p.SuperuserPassword != "from-the-file" { + t.Errorf("read %q / %q", p.SuperuserEmail, p.SuperuserPassword) + } + + // What the environment set wins. + p = PocketBase{SuperuserFile: path, SuperuserPassword: "from-the-environment"} + if err := p.readSuperuserFile(); err != nil { + t.Fatalf("readSuperuserFile: %v", err) + } + if p.SuperuserEmail != "warp@warp.local" || p.SuperuserPassword != "from-the-environment" { + t.Errorf("read %q / %q", p.SuperuserEmail, p.SuperuserPassword) + } + + // A file not written yet is no error. + p = PocketBase{SuperuserFile: filepath.Join(t.TempDir(), "missing.env")} + if err := p.readSuperuserFile(); err != nil || p.SuperuserEmail != "" { + t.Errorf("missing file: %q, %v", p.SuperuserEmail, err) + } +} diff --git a/server/internal/setup/phase10_test.go b/server/internal/setup/phase10_test.go index 893e84f..c05e3da 100644 --- a/server/internal/setup/phase10_test.go +++ b/server/internal/setup/phase10_test.go @@ -10,21 +10,23 @@ import ( "time" "github.com/dariuszjaniak/warp/server/internal/config" + "github.com/dariuszjaniak/warp/server/internal/storage/pocketbase/pbtest" ) // The Phase 10 tests: the PocketBase driver and the new endpoints, through // the whole stack. -// TestPocketBaseServer runs a server on the pocketbase driver: accounts, -// rooms, search and media, with the media landing in PocketBase's own -// filesystem rather than under media.base_path. +// TestPocketBaseServer runs a server on the pocketbase driver, against a +// PocketBase server of its own: accounts, rooms, search and media, with the +// media landing in PocketBase's storage rather than under media.base_path. func TestPocketBaseServer(t *testing.T) { - dir := t.TempDir() - dataDir := filepath.Join(dir, "pb_data") - mediaDir := filepath.Join(dir, "media") + pb := pbtest.Start(t) + mediaDir := filepath.Join(t.TempDir(), "media") srv := newSigningServer(t, func(cfg *config.Config) { cfg.Database.Driver = "pocketbase" - cfg.Database.ConnectionString = dataDir + cfg.Database.ConnectionString = pb.URL + cfg.Database.PocketBase.SuperuserEmail = pbtest.SuperuserEmail + cfg.Database.PocketBase.SuperuserPassword = pbtest.SuperuserPassword cfg.Registration.Enabled = true cfg.ClientAPI.RateLimit.Enabled = false cfg.Media.BasePath = mediaDir @@ -63,7 +65,7 @@ func TestPocketBaseServer(t *testing.T) { } var found bool - _ = filepath.WalkDir(filepath.Join(dataDir, "storage", "warp_media"), func(_ string, d os.DirEntry, err error) error { + _ = filepath.WalkDir(filepath.Join(pb.DataDir, "storage"), func(_ string, d os.DirEntry, err error) error { if err == nil && !d.IsDir() { found = true } diff --git a/server/internal/storage/pocketbase/admin.go b/server/internal/storage/pocketbase/admin.go new file mode 100644 index 0000000..95d8116 --- /dev/null +++ b/server/internal/storage/pocketbase/admin.go @@ -0,0 +1,342 @@ +package pocketbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Admin is the PocketBase implementation of storage.AdminStore. +type Admin struct{ *base } + +// registrationTokenColumns reads uses_allowed as NULL for a token without a +// limit, as the SQL drivers store it. +const registrationTokenColumns = `token, CASE WHEN has_uses_allowed THEN uses_allowed END, + completed, expires_at_ms, created_at_ms` + +// CreateRegistrationToken implements storage.AdminStore. +func (s *Admin) CreateRegistrationToken(ctx context.Context, rt storage.RegistrationToken) error { + err := s.write(ctx, func(t *tx) error { + taken, err := t.exists("admin_registration_tokens", `token = ?`, rt.Token) + if err != nil { + return err + } + if taken { + return storage.ErrRegistrationTokenExists + } + row := rec{ + "token": rt.Token, "completed": rt.Completed, + "expires_at_ms": rt.ExpiresAtMS, "created_at_ms": rt.CreatedAtMS, + } + setUsesAllowed(row, rt.UsesAllowed) + t.put("admin_registration_tokens", row) + return nil + }) + switch { + case errors.Is(err, storage.ErrRegistrationTokenExists): + return err + case err != nil: + return fmt.Errorf("create a registration token: %w", err) + } + return nil +} + +// setUsesAllowed writes an optional use limit into a row. +func setUsesAllowed(row rec, n *int) { + if n == nil { + row["uses_allowed"], row["has_uses_allowed"] = int64(0), false + return + } + row["uses_allowed"], row["has_uses_allowed"] = int64(*n), true +} + +// RegistrationToken implements storage.AdminStore. +func (s *Admin) RegistrationToken(ctx context.Context, token string) (storage.RegistrationToken, error) { + rt, err := scanRegistrationToken(s.db.QueryRowContext(ctx, `SELECT `+registrationTokenColumns+` + FROM admin_registration_tokens WHERE token = ?`, token).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.RegistrationToken{}, storage.ErrNoSuchRegistrationToken + case err != nil: + return storage.RegistrationToken{}, fmt.Errorf("read a registration token: %w", err) + } + return rt, nil +} + +// RegistrationTokens implements storage.AdminStore. +func (s *Admin) RegistrationTokens(ctx context.Context) ([]storage.RegistrationToken, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+registrationTokenColumns+` + FROM admin_registration_tokens ORDER BY token`) + if err != nil { + return nil, fmt.Errorf("list registration tokens: %w", err) + } + defer func() { _ = rows.Close() }() + + out := []storage.RegistrationToken{} + for rows.Next() { + rt, err := scanRegistrationToken(rows.Scan) + if err != nil { + return nil, fmt.Errorf("list registration tokens: %w", err) + } + out = append(out, rt) + } + return out, rows.Err() +} + +// UpdateRegistrationToken implements storage.AdminStore. +func (s *Admin) UpdateRegistrationToken(ctx context.Context, token string, usesAllowed *int, expiresAtMS int64) error { + err := s.write(ctx, func(t *tx) error { + row := rec{"expires_at_ms": expiresAtMS} + setUsesAllowed(row, usesAllowed) + n, err := t.setWhere("admin_registration_tokens", row, `token = ?`, token) + if err != nil { + return fmt.Errorf("update a registration token: %w", err) + } + if n == 0 { + return storage.ErrNoSuchRegistrationToken + } + return nil + }) + return err +} + +// DeleteRegistrationToken implements storage.AdminStore. +func (s *Admin) DeleteRegistrationToken(ctx context.Context, token string) error { + return s.write(ctx, func(t *tx) error { + found, err := t.delKey("admin_registration_tokens", token) + if err != nil { + return fmt.Errorf("delete a registration token: %w", err) + } + if !found { + return storage.ErrNoSuchRegistrationToken + } + return nil + }) +} + +// UseRegistrationToken implements storage.AdminStore. The check and the +// spend happen under the write lock, so two registrations cannot both take +// the last use. +func (s *Admin) UseRegistrationToken(ctx context.Context, token string, nowMS int64) error { + return s.write(ctx, func(t *tx) error { + var completed int64 + err := t.queryRow(`SELECT completed FROM admin_registration_tokens + WHERE token = ? + AND (NOT has_uses_allowed OR completed < uses_allowed) + AND (expires_at_ms = 0 OR expires_at_ms > ?)`, token, nowMS).Scan(&completed) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.ErrNoSuchRegistrationToken + case err != nil: + return fmt.Errorf("spend a registration token: %w", err) + } + t.put("admin_registration_tokens", rec{"token": token, "completed": completed + 1}) + return nil + }) +} + +// ReleaseRegistrationToken implements storage.AdminStore. A token deleted in +// the meantime has nothing to give back to, which is not an error. +func (s *Admin) ReleaseRegistrationToken(ctx context.Context, token string) error { + err := s.write(ctx, func(t *tx) error { + var completed int64 + err := t.queryRow(`SELECT completed FROM admin_registration_tokens WHERE token = ? AND completed > 0`, token). + Scan(&completed) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil + case err != nil: + return err + } + t.put("admin_registration_tokens", rec{"token": token, "completed": completed - 1}) + return nil + }) + if err != nil { + return fmt.Errorf("release a registration token: %w", err) + } + return nil +} + +func scanRegistrationToken(scan func(...any) error) (storage.RegistrationToken, error) { + var ( + rt storage.RegistrationToken + uses sql.NullInt64 + ) + if err := scan(&rt.Token, &uses, &rt.Completed, &rt.ExpiresAtMS, &rt.CreatedAtMS); err != nil { + return storage.RegistrationToken{}, err + } + if uses.Valid { + n := int(uses.Int64) + rt.UsesAllowed = &n + } + return rt, nil +} + +const ( + auditColumns = `audit_id, ts_ms, actor, action, target, details_json` + + defaultAuditPage = 50 + maxAuditPage = 1000 +) + +// AppendAudit implements storage.AdminStore. +func (s *Admin) AppendAudit(ctx context.Context, e storage.AuditEntry) (int64, error) { + details := string(e.Details) + if details == "" { + details = "{}" + } + var id int64 + err := s.write(ctx, func(t *tx) error { + var err error + if id, err = t.next(adminSequences, seqAudit); err != nil { + return err + } + t.put("admin_audit_log", rec{ + "audit_id": id, "ts_ms": e.TSMS, "actor": e.Actor, + "action": e.Action, "target": e.Target, "details_json": details, + }) + return nil + }) + if err != nil { + return 0, fmt.Errorf("append to the audit log: %w", err) + } + return id, nil +} + +// AuditLog implements storage.AdminStore. +func (s *Admin) AuditLog(ctx context.Context, f storage.AuditFilter) ([]storage.AuditEntry, error) { + var ( + conditions []string + args []any + ) + if f.Before > 0 { + conditions = append(conditions, `audit_id < ?`) + args = append(args, f.Before) + } + if f.Actor != "" { + conditions = append(conditions, `actor = ?`) + args = append(args, f.Actor) + } + if f.Action != "" { + conditions = append(conditions, `action = ?`) + args = append(args, f.Action) + } + limit := f.Limit + switch { + case limit <= 0: + limit = defaultAuditPage + case limit > maxAuditPage: + limit = maxAuditPage + } + args = append(args, limit) + + query := `SELECT ` + auditColumns + ` FROM admin_audit_log` + if len(conditions) > 0 { + query += ` WHERE ` + strings.Join(conditions, " AND ") + } + query += ` ORDER BY audit_id DESC LIMIT ?` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the audit log: %w", err) + } + defer func() { _ = rows.Close() }() + + out := []storage.AuditEntry{} + for rows.Next() { + var ( + e storage.AuditEntry + details string + ) + if err := rows.Scan(&e.ID, &e.TSMS, &e.Actor, &e.Action, &e.Target, &details); err != nil { + return nil, fmt.Errorf("read the audit log: %w", err) + } + e.Details = []byte(details) + out = append(out, e) + } + return out, rows.Err() +} + +// ServerNoticeRoom implements storage.AdminStore. +func (s *Admin) ServerNoticeRoom(ctx context.Context, localpart string) (string, error) { + var roomID string + err := s.db.QueryRowContext(ctx, `SELECT room_id FROM admin_server_notice_rooms WHERE localpart = ?`, localpart). + Scan(&roomID) + switch { + case errors.Is(err, sql.ErrNoRows): + return "", storage.ErrNoServerNoticeRoom + case err != nil: + return "", fmt.Errorf("read the server notice room of %q: %w", localpart, err) + } + return roomID, nil +} + +// SetServerNoticeRoom implements storage.AdminStore. +func (s *Admin) SetServerNoticeRoom(ctx context.Context, localpart, roomID string) error { + err := s.write(ctx, func(t *tx) error { + t.put("admin_server_notice_rooms", rec{"localpart": localpart, "room_id": roomID}) + return nil + }) + if err != nil { + return fmt.Errorf("record the server notice room of %q: %w", localpart, err) + } + return nil +} + +// AppServices is the PocketBase implementation of storage.AppServiceStore. +type AppServices struct{ *base } + +// AppServiceState implements storage.AppServiceStore. +func (s *AppServices) AppServiceState(ctx context.Context, id string, startPos int64) (storage.AppServiceState, error) { + var st storage.AppServiceState + err := s.write(ctx, func(t *tx) error { + err := t.queryRow(`SELECT as_id, stream_pos, txn_id, txn_to FROM appservice_state WHERE as_id = ?`, id). + Scan(&st.ID, &st.StreamPos, &st.TxnID, &st.TxnTo) + switch { + case errors.Is(err, sql.ErrNoRows): + st = storage.AppServiceState{ID: id, StreamPos: startPos} + t.put("appservice_state", rec{"as_id": id, "stream_pos": startPos, "txn_id": int64(0), "txn_to": int64(0)}) + return nil + case err != nil: + return err + } + return nil + }) + if err != nil { + return storage.AppServiceState{}, fmt.Errorf("read the cursor of application service %q: %w", id, err) + } + return st, nil +} + +// BeginTransaction implements storage.AppServiceStore. +func (s *AppServices) BeginTransaction(ctx context.Context, id string, txnID, to int64) error { + err := s.write(ctx, func(t *tx) error { + _, err := t.setWhere("appservice_state", rec{"txn_id": txnID, "txn_to": to}, `as_id = ?`, id) + return err + }) + if err != nil { + return fmt.Errorf("begin transaction %d of application service %q: %w", txnID, id, err) + } + return nil +} + +// CompleteTransaction implements storage.AppServiceStore. +func (s *AppServices) CompleteTransaction(ctx context.Context, id string, to int64) error { + err := s.write(ctx, func(t *tx) error { + _, err := t.setWhere("appservice_state", rec{"stream_pos": to, "txn_to": int64(0)}, `as_id = ?`, id) + return err + }) + if err != nil { + return fmt.Errorf("advance the cursor of application service %q: %w", id, err) + } + return nil +} + +var ( + _ storage.AdminStore = (*Admin)(nil) + _ storage.AppServiceStore = (*AppServices)(nil) +) diff --git a/server/internal/storage/pocketbase/batch.go b/server/internal/storage/pocketbase/batch.go new file mode 100644 index 0000000..62a3d50 --- /dev/null +++ b/server/internal/storage/pocketbase/batch.go @@ -0,0 +1,282 @@ +package pocketbase + +import ( + "context" + "database/sql" + "fmt" + "strings" + "sync" +) + +// base is what every store shares: the client, the read-only SQL pool and +// the write lock. +// +// Writes are serialised by the lock, in this process, the way the sqlite +// driver serialises them onto one connection: PocketBase is SQLite and takes +// one writer at a time anyway, and holding the lock across a store call's +// reads and its batch is what makes "read, decide, write" safe without a +// transaction that spans requests. It does mean one Warp server per +// PocketBase. +type base struct { + c *client + db *sql.DB + lock *sync.RWMutex +} + +// rec is one row's columns, by name. +type rec map[string]any + +// maxBatch is how many operations go into one batch request. PocketBase is +// told to allow this many (see configure); a store call that writes more is +// split, and each part is atomic on its own. +const maxBatch = 1000 + +// tx collects the writes of one store call. +type tx struct { + ctx context.Context + b *base + + // seqs are the counters this call has read or advanced, by table and + // name, and seqOrder the order they were first touched in. + seqs map[string]int64 + seqOrder []string + + ops []batchRequest + deleted map[string]bool +} + +// write runs fn with the write lock held and sends what it collected. +// +// The counters it advanced are written first, in the first batch, so that a +// call split over several batches that fails part-way leaves a gap in a +// stream — which every reader already copes with — and never a position +// handed out twice. +func (b *base) write(ctx context.Context, fn func(*tx) error) error { + b.lock.Lock() + defer b.lock.Unlock() + + t := &tx{ctx: withWriteLock(ctx), b: b, seqs: map[string]int64{}, deleted: map[string]bool{}} + if err := fn(t); err != nil { + return err + } + return t.commit() +} + +// read runs fn with the read lock held, so that its queries see one state: +// no write lands between them. The SQL drivers get that from a +// transaction; here it is the lock, since every write takes it. +func (b *base) read(ctx context.Context, fn func(ctx context.Context) error) error { + b.lock.RLock() + defer b.lock.RUnlock() + return fn(withWriteLock(ctx)) +} + +func (t *tx) commit() error { + ops := make([]batchRequest, 0, len(t.seqOrder)+len(t.ops)) + for _, key := range t.seqOrder { + table, name, _ := strings.Cut(key, "\x00") + ops = append(ops, putRequest(table, rec{"name": name, "value": t.seqs[key]})) + } + ops = append(ops, t.ops...) + for len(ops) > 0 { + n := min(len(ops), maxBatch) + if err := t.b.c.batch(t.ctx, ops[:n]); err != nil { + return err + } + ops = ops[n:] + } + return nil +} + +// putRequest is an upsert of one row: created if its key is new, otherwise +// updated in the columns given and left alone in the rest. +func putRequest(collection string, row rec) batchRequest { + tb, ok := tables[collection] + if !ok { + panic("pocketbase: no collection " + collection) + } + key := make([]any, len(tb.key)) + for i, col := range tb.key { + v, ok := row[col] + if !ok { + panic(fmt.Sprintf("pocketbase: a %s row without its key column %s", collection, col)) + } + key[i] = v + } + return patchRequest(collection, rowID(collection, key...), row) +} + +// patchRequest is an upsert of a row whose id is already known. +func patchRequest(collection, id string, row rec) batchRequest { + body := make(map[string]any, len(row)+1) + for col, v := range row { + body[col] = jsonValue(v) + } + body["id"] = id + return batchRequest{Method: "PUT", URL: "/api/collections/" + collection + "/records", Body: body} +} + +// jsonValue turns a column value into what the record API takes. +func jsonValue(v any) any { + switch v := v.(type) { + case []byte: + return string(v) + case int: + return int64(v) + case int32: + return int64(v) + default: + return v + } +} + +// put upserts a row; see putRequest. +func (t *tx) put(collection string, row rec) { + req := putRequest(collection, row) + delete(t.deleted, collection+"/"+req.Body["id"].(string)) + t.ops = append(t.ops, req) +} + +// patch upserts columns of the row with this id. +func (t *tx) patch(collection, id string, row rec) { + delete(t.deleted, collection+"/"+id) + t.ops = append(t.ops, patchRequest(collection, id, row)) +} + +// del deletes a row that exists. Deleting one that does not fails the whole +// batch, so callers only delete what they have just read. +func (t *tx) del(collection, id string) bool { + key := collection + "/" + id + if t.deleted[key] { + return false + } + t.deleted[key] = true + t.ops = append(t.ops, batchRequest{Method: "DELETE", URL: "/api/collections/" + collection + "/records/" + id}) + return true +} + +// delKey deletes the row with this primary key, if it exists. +func (t *tx) delKey(collection string, key ...any) (bool, error) { + id := rowID(collection, key...) + found, err := t.exists(collection, `id = ?`, id) + if err != nil || !found { + return false, err + } + return t.del(collection, id), nil +} + +// ids reads the ids of the rows matching a condition. +func (t *tx) ids(collection, where string, args ...any) ([]string, error) { + rows, err := t.b.db.QueryContext(t.ctx, `SELECT id FROM `+collection+` WHERE `+where, args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + +// delWhere deletes the rows matching a condition and reports how many there +// were. +func (t *tx) delWhere(collection, where string, args ...any) (int, error) { + ids, err := t.ids(collection, where, args...) + if err != nil { + return 0, fmt.Errorf("find the %s rows to delete: %w", collection, err) + } + n := 0 + for _, id := range ids { + if t.del(collection, id) { + n++ + } + } + return n, nil +} + +// setWhere updates columns of the rows matching a condition and reports how +// many there were. +func (t *tx) setWhere(collection string, row rec, where string, args ...any) (int, error) { + ids, err := t.ids(collection, where, args...) + if err != nil { + return 0, fmt.Errorf("find the %s rows to update: %w", collection, err) + } + n := 0 + for _, id := range ids { + if t.deleted[collection+"/"+id] { + continue + } + t.patch(collection, id, row) + n++ + } + return n, nil +} + +// exists reports whether a row matches a condition. +func (t *tx) exists(collection, where string, args ...any) (bool, error) { + var one int + err := t.b.db.QueryRowContext(t.ctx, `SELECT 1 FROM `+collection+` WHERE `+where+` LIMIT 1`, args...).Scan(&one) + switch { + case err == sql.ErrNoRows: + return false, nil + case err != nil: + return false, err + } + return true, nil +} + +// queryRow reads one row inside the call. +func (t *tx) queryRow(query string, args ...any) *sql.Row { + return t.b.db.QueryRowContext(t.ctx, query, args...) +} + +// query reads rows inside the call. +func (t *tx) query(query string, args ...any) (*sql.Rows, error) { + return t.b.db.QueryContext(t.ctx, query, args...) +} + +// next hands out the next value of a counter. +func (t *tx) next(table, name string) (int64, error) { + return t.advance(table, name, 1) +} + +// advance moves a counter by n and returns the last value of the block, as +// the SQL drivers' advanceSequence does. n may be negative. +func (t *tx) advance(table, name string, n int64) (int64, error) { + cur, err := t.current(table, name) + if err != nil { + return 0, err + } + cur += n + t.seqs[table+"\x00"+name] = cur + return cur, nil +} + +// current reads a counter as this call has left it. +func (t *tx) current(table, name string) (int64, error) { + key := table + "\x00" + name + if v, ok := t.seqs[key]; ok { + return v, nil + } + var v int64 + if err := t.queryRow(`SELECT value FROM `+table+` WHERE name = ?`, name).Scan(&v); err != nil { + return 0, fmt.Errorf("read the %s counter: %w", name, err) + } + t.seqs[key] = v + t.seqOrder = append(t.seqOrder, key) + return v, nil +} + +// currentSequence reads a counter outside any write. +func (b *base) currentSequence(ctx context.Context, table, name string) (int64, error) { + var v int64 + if err := b.db.QueryRowContext(ctx, `SELECT value FROM `+table+` WHERE name = ?`, name).Scan(&v); err != nil { + return 0, fmt.Errorf("read the %s counter: %w", name, err) + } + return v, nil +} diff --git a/server/internal/storage/pocketbase/blobs.go b/server/internal/storage/pocketbase/blobs.go new file mode 100644 index 0000000..24bcc99 --- /dev/null +++ b/server/internal/storage/pocketbase/blobs.go @@ -0,0 +1,210 @@ +package pocketbase + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "strconv" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// mediaCollection holds media content: one record per distinct content, +// keyed by its SHA-256, with the bytes in a protected file field. Where +// PocketBase keeps the file — its pb_data/storage or an S3 bucket — is its +// own storage setting. +const mediaCollection = "warp_media" + +// blobStore is storage.BlobStore over warp_media. +type blobStore struct{ *base } + +// validSum reports whether s is a lower-case hex SHA-256: the only names +// content is stored under, and so the only ones worth asking about. +func validSum(s string) bool { + if len(s) != sha256.Size*2 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +// Put implements storage.BlobStore. +// +// The content is spooled to a temporary file first: its name is its hash, +// which is not known until the last byte has been read, and the upload has +// to say both. The spool is also what keeps an oversize upload from reaching +// PocketBase at all. +func (b *blobStore) Put(ctx context.Context, r io.Reader, limit int64) (storage.BlobInfo, error) { + tmp, err := os.CreateTemp("", "warp-media-*") + if err != nil { + return storage.BlobInfo{}, fmt.Errorf("create a temporary file: %w", err) + } + defer func() { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + }() + + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(tmp, hash), &ctxReader{ctx: ctx, r: io.LimitReader(r, limit+1)}) + if err != nil { + return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) + } + if n > limit { + return storage.BlobInfo{}, storage.ErrBlobTooLarge + } + info := storage.BlobInfo{SHA256: hex.EncodeToString(hash.Sum(nil)), Size: n} + id := rowID(mediaCollection, info.SHA256) + + if _, err := b.c.getRecord(ctx, mediaCollection, id); err == nil { + // The same content is already stored. + return info, nil + } else if !isNotFound(err) { + return storage.BlobInfo{}, fmt.Errorf("look up stored content: %w", err) + } + + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) + } + err = b.c.createRecordWithFile(ctx, mediaCollection, map[string]string{ + "id": id, + "sha256": info.SHA256, + "size": strconv.FormatInt(n, 10), + }, "content", info.SHA256, tmp) + if err != nil { + // Two uploads of the same content can race to create the record; + // the loser finds the winner's. + if _, getErr := b.c.getRecord(ctx, mediaCollection, id); getErr == nil { + return info, nil + } + return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) + } + return info, nil +} + +// Open implements storage.BlobStore. +func (b *blobStore) Open(ctx context.Context, sum string) (io.ReadSeekCloser, error) { + if !validSum(sum) { + return nil, storage.ErrNoSuchBlob + } + id := rowID(mediaCollection, sum) + record, err := b.c.getRecord(ctx, mediaCollection, id) + switch { + case isNotFound(err): + return nil, storage.ErrNoSuchBlob + case err != nil: + return nil, fmt.Errorf("open media content: %w", err) + } + name, _ := record["content"].(string) + size, _ := record["size"].(float64) + if name == "" { + return nil, storage.ErrNoSuchBlob + } + token, err := b.c.fileToken(ctx) + if err != nil { + return nil, fmt.Errorf("open media content: %w", err) + } + return &blobReader{ctx: ctx, c: b.c, id: id, name: name, token: token, size: int64(size)}, nil +} + +// Delete implements storage.BlobStore. +func (b *blobStore) Delete(ctx context.Context, sum string) error { + if !validSum(sum) { + return nil + } + if err := b.c.deleteRecord(ctx, mediaCollection, rowID(mediaCollection, sum)); err != nil && !isNotFound(err) { + return fmt.Errorf("delete media content: %w", err) + } + return nil +} + +// blobReader reads a stored file over HTTP. A seek only moves the offset; +// the next read starts a ranged download from there, so reading a file +// through costs one request however it is read. +type blobReader struct { + ctx context.Context + c *client + id string + name string + token string + size int64 + + offset int64 + body io.ReadCloser +} + +func (r *blobReader) Read(p []byte) (int, error) { + if r.offset >= r.size { + return 0, io.EOF + } + if r.body == nil { + resp, err := r.c.openFile(r.ctx, mediaCollection, r.id, r.name, r.token, r.offset) + if err != nil { + return 0, err + } + if r.offset > 0 && resp.StatusCode != http.StatusPartialContent { + _ = resp.Body.Close() + return 0, fmt.Errorf("pocketbase: the file server ignored a range request (HTTP %d)", resp.StatusCode) + } + r.body = resp.Body + } + n, err := r.body.Read(p) + r.offset += int64(n) + if errors.Is(err, io.EOF) && r.offset < r.size { + err = io.ErrUnexpectedEOF + } + return n, err +} + +func (r *blobReader) Seek(offset int64, whence int) (int64, error) { + var next int64 + switch whence { + case io.SeekStart: + next = offset + case io.SeekCurrent: + next = r.offset + offset + case io.SeekEnd: + next = r.size + offset + default: + return 0, errors.New("pocketbase: invalid whence") + } + if next < 0 { + return 0, errors.New("pocketbase: negative position") + } + if next != r.offset && r.body != nil { + _ = r.body.Close() + r.body = nil + } + r.offset = next + return next, nil +} + +func (r *blobReader) Close() error { + if r.body != nil { + return r.body.Close() + } + return nil +} + +// ctxReader stops a copy when its context ends. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c *ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} + +var _ storage.BlobStore = (*blobStore)(nil) diff --git a/server/internal/storage/pocketbase/client.go b/server/internal/storage/pocketbase/client.go new file mode 100644 index 0000000..124e476 --- /dev/null +++ b/server/internal/storage/pocketbase/client.go @@ -0,0 +1,331 @@ +package pocketbase + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "strings" + "sync" +) + +// client speaks PocketBase's REST API as a superuser. Everything the driver +// does goes through it: the SQL endpoint for reads, the batch endpoint for +// writes, and the record and file endpoints for media. +type client struct { + base *url.URL + email string + password string + http *http.Client + + mu sync.Mutex + token string +} + +// newClient parses the server address. Credentials are checked on first use, +// not here: a PocketBase that is still starting is not a configuration +// error. +func newClient(rawURL, email, password string) (*client, error) { + if rawURL == "" { + return nil, errors.New("pocketbase: connection_string must be the server's URL, e.g. http://pocketbase:8070") + } + base, err := url.Parse(strings.TrimSuffix(rawURL, "/")) + if err != nil || (base.Scheme != "http" && base.Scheme != "https") || base.Host == "" { + return nil, fmt.Errorf("pocketbase: connection_string %q is not an http(s) URL", rawURL) + } + if email == "" || password == "" { + return nil, errors.New("pocketbase: a superuser email and password are required") + } + return &client{base: base, email: email, password: password, http: &http.Client{}}, nil +} + +// apiError is a non-2xx answer from PocketBase. +type apiError struct { + Status int + Message string + Data json.RawMessage +} + +func (e *apiError) Error() string { + msg := fmt.Sprintf("pocketbase: HTTP %d: %s", e.Status, e.Message) + if len(e.Data) > 0 && string(e.Data) != "{}" && string(e.Data) != "null" { + msg += " " + string(e.Data) + } + return msg +} + +// isNotFound reports a 404 from PocketBase. +func isNotFound(err error) bool { + var apiErr *apiError + return errors.As(err, &apiErr) && apiErr.Status == http.StatusNotFound +} + +// url resolves an API path against the server address. +func (c *client) url(path string) string { return c.base.String() + path } + +// authToken returns the current superuser token, signing in if there is +// none yet. +func (c *client) authToken(ctx context.Context) (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.token != "" { + return c.token, nil + } + body, err := json.Marshal(map[string]string{"identity": c.email, "password": c.password}) + if err != nil { + return "", err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.url("/api/collections/_superusers/auth-with-password"), bytes.NewReader(body)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + var out struct { + Token string `json:"token"` + } + if err := c.send(req, &out); err != nil { + return "", fmt.Errorf("pocketbase: sign in as superuser %q: %w", c.email, err) + } + if out.Token == "" { + return "", errors.New("pocketbase: sign in returned no token") + } + c.token = out.Token + return c.token, nil +} + +// forgetToken drops a token PocketBase refused, so the next call signs in +// again. A superuser token outlives most deployments' restarts, but not all. +func (c *client) forgetToken(token string) { + c.mu.Lock() + if c.token == token { + c.token = "" + } + c.mu.Unlock() +} + +// do sends one authenticated request, signing in again once if the token has +// expired. body is JSON-encoded unless it is an io.Reader, which is sent as +// is with contentType. +func (c *client) do(ctx context.Context, method, path string, body any, contentType string, out any) error { + var payload []byte + if body != nil { + if r, ok := body.(io.Reader); ok { + // A streamed body cannot be replayed, so it is sent once and a + // stale token is the caller's error to retry. + return c.doOnce(ctx, method, path, r, contentType, out) + } + var err error + if payload, err = json.Marshal(body); err != nil { + return err + } + contentType = "application/json" + } + for attempt := 0; ; attempt++ { + var r io.Reader + if payload != nil { + r = bytes.NewReader(payload) + } + err := c.doOnce(ctx, method, path, r, contentType, out) + var apiErr *apiError + if attempt == 0 && errors.As(err, &apiErr) && apiErr.Status == http.StatusUnauthorized { + continue + } + return err + } +} + +func (c *client) doOnce(ctx context.Context, method, path string, body io.Reader, contentType string, out any) error { + token, err := c.authToken(ctx) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, method, c.url(path), body) + if err != nil { + return err + } + req.Header.Set("Authorization", token) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + err = c.send(req, out) + var apiErr *apiError + if errors.As(err, &apiErr) && apiErr.Status == http.StatusUnauthorized { + c.forgetToken(token) + } + return err +} + +// send runs a request and decodes a JSON answer into out, if out is set. +func (c *client) send(req *http.Request, out any) error { + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("pocketbase: %s %s: %w", req.Method, req.URL.Path, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readAPIError(resp) + } + if out == nil || resp.StatusCode == http.StatusNoContent { + _, _ = io.Copy(io.Discard, resp.Body) + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("pocketbase: decode the answer to %s %s: %w", req.Method, req.URL.Path, err) + } + return nil +} + +func readAPIError(resp *http.Response) error { + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + apiErr := &apiError{Status: resp.StatusCode} + var body struct { + Message string `json:"message"` + Data json.RawMessage `json:"data"` + } + if json.Unmarshal(raw, &body) == nil && body.Message != "" { + apiErr.Message = body.Message + apiErr.Data = body.Data + } else { + apiErr.Message = strings.TrimSpace(string(raw)) + } + return apiErr +} + +// health checks the server answers at all; it needs no token. +func (c *client) health(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url("/api/health"), nil) + if err != nil { + return err + } + return c.send(req, nil) +} + +// sqlResult is what PocketBase's SQL endpoint answers with. Every value comes +// back as text, or null. +type sqlResult struct { + Columns []struct { + Name string `json:"name"` + } `json:"columns"` + Rows [][]*string `json:"rows"` +} + +// query runs one SELECT through the SQL endpoint. +func (c *client) query(ctx context.Context, query string) (*sqlResult, error) { + var out sqlResult + if err := c.do(ctx, http.MethodPost, "/api/sql", map[string]string{"query": query}, "", &out); err != nil { + return nil, err + } + return &out, nil +} + +// batchRequest is one record operation inside a batch. +type batchRequest struct { + Method string `json:"method"` + URL string `json:"url"` + Body map[string]any `json:"body,omitempty"` +} + +// batch runs record operations in one PocketBase transaction: all of them +// happen, or none do. +func (c *client) batch(ctx context.Context, reqs []batchRequest) error { + if len(reqs) == 0 { + return nil + } + return c.do(ctx, http.MethodPost, "/api/batch", map[string]any{"requests": reqs}, "", nil) +} + +// patchSettings changes part of PocketBase's application settings. +func (c *client) patchSettings(ctx context.Context, settings map[string]any) error { + return c.do(ctx, http.MethodPatch, "/api/settings", settings, "", nil) +} + +// importCollections creates or updates collections, leaving any not named +// alone. +func (c *client) importCollections(ctx context.Context, collections []map[string]any) error { + return c.do(ctx, http.MethodPut, "/api/collections/import", + map[string]any{"collections": collections, "deleteMissing": false}, "", nil) +} + +// getRecord reads one record as JSON fields. +func (c *client) getRecord(ctx context.Context, collection, id string) (map[string]any, error) { + var out map[string]any + if err := c.do(ctx, http.MethodGet, + "/api/collections/"+collection+"/records/"+url.PathEscape(id), nil, "", &out); err != nil { + return nil, err + } + return out, nil +} + +// deleteRecord removes one record. +func (c *client) deleteRecord(ctx context.Context, collection, id string) error { + return c.do(ctx, http.MethodDelete, + "/api/collections/"+collection+"/records/"+url.PathEscape(id), nil, "", nil) +} + +// createRecordWithFile creates a record with one file field, streaming the +// file as multipart so that it is never held in memory whole. +func (c *client) createRecordWithFile(ctx context.Context, collection string, fields map[string]string, + fileField, fileName string, file io.Reader) error { + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + go func() { + err := func() error { + for k, v := range fields { + if err := mw.WriteField(k, v); err != nil { + return err + } + } + part, err := mw.CreateFormFile(fileField, fileName) + if err != nil { + return err + } + if _, err := io.Copy(part, file); err != nil { + return err + } + return mw.Close() + }() + _ = pw.CloseWithError(err) + }() + err := c.do(ctx, http.MethodPost, "/api/collections/"+collection+"/records", pr, mw.FormDataContentType(), nil) + _ = pr.Close() + return err +} + +// fileToken mints the short-lived token protected files are read with. +func (c *client) fileToken(ctx context.Context) (string, error) { + var out struct { + Token string `json:"token"` + } + if err := c.do(ctx, http.MethodPost, "/api/files/token", nil, "", &out); err != nil { + return "", err + } + return out.Token, nil +} + +// openFile starts a download of a stored file from offset on. +func (c *client) openFile(ctx context.Context, collection, id, name, token string, offset int64) (*http.Response, error) { + path := "/api/files/" + collection + "/" + url.PathEscape(id) + "/" + url.PathEscape(name) + + "?token=" + url.QueryEscape(token) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url(path), nil) + if err != nil { + return nil, err + } + if offset > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("pocketbase: download %s: %w", name, err) + } + if resp.StatusCode >= 300 { + defer func() { _ = resp.Body.Close() }() + return nil, readAPIError(resp) + } + return resp, nil +} diff --git a/server/internal/storage/pocketbase/delayed.go b/server/internal/storage/pocketbase/delayed.go new file mode 100644 index 0000000..e3ccaf7 --- /dev/null +++ b/server/internal/storage/pocketbase/delayed.go @@ -0,0 +1,232 @@ +package pocketbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// DelayedEvents is the PocketBase implementation of +// storage.DelayedEventStore. +type DelayedEvents struct{ *base } + +// delayedEventColumns reads state_key as NULL for a message event, as the +// SQL drivers store it. +const delayedEventColumns = ` + delay_id, user_id, device_id, room_id, event_type, CASE WHEN has_state_key THEN state_key END, content, + delay_ms, running_since_ms, send_at_ms, + finalised_at_ms, outcome, reason, event_id, error` + +// InsertDelayedEvent implements storage.DelayedEventStore. +func (s *DelayedEvents) InsertDelayedEvent(ctx context.Context, e storage.DelayedEvent) error { + stateKey, hasStateKey := "", false + if e.StateKey != nil { + stateKey, hasStateKey = *e.StateKey, true + } + err := s.write(ctx, func(t *tx) error { + t.put("delayed_events", rec{ + "delay_id": e.DelayID, "user_id": e.UserID, "device_id": e.DeviceID, "room_id": e.RoomID, + "event_type": e.Type, "state_key": stateKey, "has_state_key": hasStateKey, + "content": string(e.Content), "delay_ms": e.DelayMS, "running_since_ms": e.RunningSinceMS, + "send_at_ms": e.SendAtMS, "finalised_at_ms": int64(0), + "outcome": "", "reason": "", "event_id": "", "error": "", + }) + return nil + }) + if err != nil { + return fmt.Errorf("insert delayed event: %w", err) + } + return nil +} + +// DelayedEvent implements storage.DelayedEventStore. +func (s *DelayedEvents) DelayedEvent(ctx context.Context, userID, delayID string) (storage.DelayedEvent, error) { + return s.one(ctx, `SELECT`+delayedEventColumns+` + FROM delayed_events WHERE user_id = ? AND delay_id = ?`, userID, delayID) +} + +// ScheduledDelayedEvent implements storage.DelayedEventStore. +func (s *DelayedEvents) ScheduledDelayedEvent(ctx context.Context, delayID string) (storage.DelayedEvent, error) { + return s.one(ctx, `SELECT`+delayedEventColumns+` + FROM delayed_events WHERE delay_id = ? AND finalised_at_ms = 0`, delayID) +} + +func (s *DelayedEvents) one(ctx context.Context, query string, args ...any) (storage.DelayedEvent, error) { + events, err := s.list(ctx, query, args...) + if err != nil { + return storage.DelayedEvent{}, err + } + if len(events) == 0 { + return storage.DelayedEvent{}, storage.ErrNoSuchDelayedEvent + } + return events[0], nil +} + +// DelayedEvents implements storage.DelayedEventStore. +func (s *DelayedEvents) DelayedEvents(ctx context.Context, userID string) ([]storage.DelayedEvent, error) { + return s.list(ctx, `SELECT`+delayedEventColumns+` + FROM delayed_events WHERE user_id = ? + ORDER BY finalised_at_ms = 0 DESC, send_at_ms ASC, finalised_at_ms DESC`, userID) +} + +// CountScheduledDelayedEvents implements storage.DelayedEventStore. +func (s *DelayedEvents) CountScheduledDelayedEvents(ctx context.Context, userID string) (int, error) { + var n int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM delayed_events + WHERE user_id = ? AND finalised_at_ms = 0`, userID).Scan(&n); err != nil { + return 0, fmt.Errorf("count delayed events: %w", err) + } + return n, nil +} + +// RestartDelayedEvent implements storage.DelayedEventStore. +func (s *DelayedEvents) RestartDelayedEvent(ctx context.Context, delayID string, nowMS int64) error { + return s.write(ctx, func(t *tx) error { + var delay int64 + err := t.queryRow(`SELECT delay_ms FROM delayed_events WHERE delay_id = ? AND finalised_at_ms = 0`, delayID). + Scan(&delay) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.ErrNoSuchDelayedEvent + case err != nil: + return fmt.Errorf("restart delayed event: %w", err) + } + t.put("delayed_events", rec{"delay_id": delayID, "running_since_ms": nowMS, "send_at_ms": nowMS + delay}) + return nil + }) +} + +// FinaliseDelayedEvent implements storage.DelayedEventStore. +func (s *DelayedEvents) FinaliseDelayedEvent(ctx context.Context, e storage.DelayedEvent) error { + return s.write(ctx, func(t *tx) error { + n, err := t.setWhere("delayed_events", rec{ + "finalised_at_ms": e.FinalisedAtMS, "outcome": e.Outcome, "reason": e.Reason, + "event_id": e.EventID, "error": string(e.Error), + }, `delay_id = ? AND finalised_at_ms = 0`, e.DelayID) + if err != nil { + return fmt.Errorf("finalise delayed event: %w", err) + } + if n == 0 { + return storage.ErrNoSuchDelayedEvent + } + return nil + }) +} + +// DueDelayedEvents implements storage.DelayedEventStore. +func (s *DelayedEvents) DueDelayedEvents(ctx context.Context, nowMS int64, limit int) ([]storage.DelayedEvent, error) { + return s.list(ctx, `SELECT`+delayedEventColumns+` + FROM delayed_events WHERE finalised_at_ms = 0 AND send_at_ms <= ? + ORDER BY send_at_ms LIMIT ?`, nowMS, limit) +} + +// NextDelayedEventDue implements storage.DelayedEventStore. +func (s *DelayedEvents) NextDelayedEventDue(ctx context.Context) (int64, bool, error) { + var next sql.NullInt64 + if err := s.db.QueryRowContext(ctx, `SELECT MIN(send_at_ms) FROM delayed_events + WHERE finalised_at_ms = 0`).Scan(&next); err != nil { + return 0, false, fmt.Errorf("next delayed event: %w", err) + } + return next.Int64, next.Valid, nil +} + +// ScheduledDelayedStateEvents implements storage.DelayedEventStore. +func (s *DelayedEvents) ScheduledDelayedStateEvents(ctx context.Context, roomID, eventType, stateKey string) ([]storage.DelayedEvent, error) { + return s.list(ctx, `SELECT`+delayedEventColumns+` + FROM delayed_events + WHERE room_id = ? AND event_type = ? AND has_state_key AND state_key = ? AND finalised_at_ms = 0`, + roomID, eventType, stateKey) +} + +// PurgeFinalisedDelayedEvents implements storage.DelayedEventStore. +func (s *DelayedEvents) PurgeFinalisedDelayedEvents(ctx context.Context, beforeMS int64) error { + err := s.write(ctx, func(t *tx) error { + _, err := t.delWhere("delayed_events", `finalised_at_ms <> 0 AND finalised_at_ms < ?`, beforeMS) + return err + }) + if err != nil { + return fmt.Errorf("purge delayed events: %w", err) + } + return nil +} + +func (s *DelayedEvents) list(ctx context.Context, query string, args ...any) ([]storage.DelayedEvent, error) { + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read delayed events: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []storage.DelayedEvent + for rows.Next() { + var ( + e storage.DelayedEvent + stateKey sql.NullString + content, errorContent string + ) + if err := rows.Scan(&e.DelayID, &e.UserID, &e.DeviceID, &e.RoomID, &e.Type, &stateKey, &content, + &e.DelayMS, &e.RunningSinceMS, &e.SendAtMS, + &e.FinalisedAtMS, &e.Outcome, &e.Reason, &e.EventID, &errorContent, + ); err != nil { + return nil, fmt.Errorf("read delayed event: %w", err) + } + if stateKey.Valid { + e.StateKey = &stateKey.String + } + e.Content = []byte(content) + if errorContent != "" { + e.Error = []byte(errorContent) + } + out = append(out, e) + } + return out, rows.Err() +} + +// OpenIDTokens is the PocketBase implementation of storage.OpenIDTokenStore. +type OpenIDTokens struct{ *base } + +// InsertOpenIDToken implements storage.OpenIDTokenStore. +func (s *OpenIDTokens) InsertOpenIDToken(ctx context.Context, tokenHash, userID string, expiresAtMS int64) error { + err := s.write(ctx, func(t *tx) error { + t.put("openid_tokens", rec{"token_hash": tokenHash, "user_id": userID, "expires_at_ms": expiresAtMS}) + return nil + }) + if err != nil { + return fmt.Errorf("insert OpenID token: %w", err) + } + return nil +} + +// OpenIDTokenUser implements storage.OpenIDTokenStore. +func (s *OpenIDTokens) OpenIDTokenUser(ctx context.Context, tokenHash string, nowMS int64) (string, error) { + var userID string + err := s.db.QueryRowContext(ctx, `SELECT user_id FROM openid_tokens + WHERE token_hash = ? AND expires_at_ms > ?`, tokenHash, nowMS).Scan(&userID) + if errors.Is(err, sql.ErrNoRows) { + return "", storage.ErrNoSuchOpenIDToken + } + if err != nil { + return "", fmt.Errorf("read OpenID token: %w", err) + } + return userID, nil +} + +// PurgeOpenIDTokens implements storage.OpenIDTokenStore. +func (s *OpenIDTokens) PurgeOpenIDTokens(ctx context.Context, beforeMS int64) error { + err := s.write(ctx, func(t *tx) error { + _, err := t.delWhere("openid_tokens", `expires_at_ms < ?`, beforeMS) + return err + }) + if err != nil { + return fmt.Errorf("purge OpenID tokens: %w", err) + } + return nil +} + +var ( + _ storage.DelayedEventStore = (*DelayedEvents)(nil) + _ storage.OpenIDTokenStore = (*OpenIDTokens)(nil) +) diff --git a/server/internal/storage/pocketbase/federation.go b/server/internal/storage/pocketbase/federation.go new file mode 100644 index 0000000..c940620 --- /dev/null +++ b/server/internal/storage/pocketbase/federation.go @@ -0,0 +1,247 @@ +package pocketbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Federation is the PocketBase implementation of storage.FederationStore. +type Federation struct{ *base } + +// StoreServerKeys implements storage.FederationStore. +func (s *Federation) StoreServerKeys(ctx context.Context, keys []storage.ServerKey) error { + if len(keys) == 0 { + return nil + } + return s.write(ctx, func(t *tx) error { + for _, k := range keys { + t.put("federation_server_keys", rec{ + "server_name": k.ServerName, "key_id": k.KeyID, "public_key": k.PublicKey, + "valid_until_ts": k.ValidUntilTS, "expired_ts": k.ExpiredTS, "fetched_at_ms": k.FetchedAtMS, + }) + } + return nil + }) +} + +// ServerKeys implements storage.FederationStore. +func (s *Federation) ServerKeys(ctx context.Context, serverName string, keyIDs []string) ([]storage.ServerKey, error) { + query := ` + SELECT server_name, key_id, public_key, valid_until_ts, expired_ts, fetched_at_ms + FROM federation_server_keys WHERE server_name = ?` + args := []any{serverName} + if len(keyIDs) > 0 { + query += ` AND key_id IN ` + inList + args = append(args, jsonList(keyIDs)) + } + query += ` ORDER BY key_id` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the keys of %q: %w", serverName, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.ServerKey + for rows.Next() { + var k storage.ServerKey + if err := rows.Scan(&k.ServerName, &k.KeyID, &k.PublicKey, + &k.ValidUntilTS, &k.ExpiredTS, &k.FetchedAtMS); err != nil { + return nil, fmt.Errorf("read the keys of %q: %w", serverName, err) + } + out = append(out, k) + } + return out, rows.Err() +} + +// Enqueue implements storage.FederationStore. +func (s *Federation) Enqueue(ctx context.Context, items []storage.QueuedItem) ([]storage.QueuedItem, error) { + if len(items) == 0 { + return nil, nil + } + out := make([]storage.QueuedItem, len(items)) + copy(out, items) + err := s.write(ctx, func(t *tx) error { + top, err := t.advance(federationSequences, seqQueue, int64(len(out))) + if err != nil { + return err + } + first := top - int64(len(out)) + 1 + for i := range out { + out[i].ID = first + int64(i) + item := out[i] + t.put("federation_queue", rec{ + "queue_id": item.ID, "destination": item.Destination, "kind": item.Kind, + "room_id": item.RoomID, "event_id": item.EventID, + "payload": string(item.Payload), "queued_at_ms": item.QueuedAtMS, + }) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("enqueue %d items: %w", len(items), err) + } + return out, nil +} + +// Queued implements storage.FederationStore. +func (s *Federation) Queued(ctx context.Context, destination, kind string, limit int) ([]storage.QueuedItem, error) { + if limit <= 0 { + return nil, nil + } + rows, err := s.db.QueryContext(ctx, ` + SELECT queue_id, destination, kind, room_id, event_id, payload, queued_at_ms + FROM federation_queue + WHERE destination = ? AND kind = ? + ORDER BY queue_id + LIMIT ?`, destination, kind, limit) + if err != nil { + return nil, fmt.Errorf("read the queue for %q: %w", destination, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.QueuedItem + for rows.Next() { + var ( + item storage.QueuedItem + payload string + ) + if err := rows.Scan(&item.ID, &item.Destination, &item.Kind, &item.RoomID, + &item.EventID, &payload, &item.QueuedAtMS); err != nil { + return nil, fmt.Errorf("read the queue for %q: %w", destination, err) + } + item.Payload = []byte(payload) + out = append(out, item) + } + return out, rows.Err() +} + +// Dequeue implements storage.FederationStore. +func (s *Federation) Dequeue(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + err := s.write(ctx, func(t *tx) error { + _, err := t.delWhere("federation_queue", `queue_id IN `+inList, jsonInts(ids)) + return err + }) + if err != nil { + return fmt.Errorf("dequeue %d items: %w", len(ids), err) + } + return nil +} + +// QueuedDestinations implements storage.FederationStore. +func (s *Federation) QueuedDestinations(ctx context.Context) ([]string, error) { + out, err := s.queryStrings(ctx, `SELECT DISTINCT destination FROM federation_queue ORDER BY destination`) + if err != nil { + return nil, fmt.Errorf("read the queued destinations: %w", err) + } + return out, nil +} + +const destinationColumns = ` + server_name, failures, retry_at_ms, last_success_ms, last_failure_ms, last_error` + +// Destination implements storage.FederationStore. +func (s *Federation) Destination(ctx context.Context, serverName string) (storage.Destination, error) { + var d storage.Destination + err := s.db.QueryRowContext(ctx, `SELECT `+destinationColumns+` + FROM federation_destinations WHERE server_name = ?`, serverName). + Scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS, &d.LastFailureMS, &d.LastError) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.Destination{ServerName: serverName}, nil + case err != nil: + return storage.Destination{}, fmt.Errorf("read destination %q: %w", serverName, err) + } + return d, nil +} + +// SetDestination implements storage.FederationStore. +func (s *Federation) SetDestination(ctx context.Context, d storage.Destination) error { + err := s.write(ctx, func(t *tx) error { + t.put("federation_destinations", rec{ + "server_name": d.ServerName, "failures": d.Failures, "retry_at_ms": d.RetryAtMS, + "last_success_ms": d.LastSuccessMS, "last_failure_ms": d.LastFailureMS, "last_error": d.LastError, + }) + return nil + }) + if err != nil { + return fmt.Errorf("record destination %q: %w", d.ServerName, err) + } + return nil +} + +// Destinations implements storage.FederationStore: every server with a +// health row or something queued. +func (s *Federation) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT n.server_name, + COALESCE(d.failures, 0), COALESCE(d.retry_at_ms, 0), + COALESCE(d.last_success_ms, 0), COALESCE(d.last_failure_ms, 0), + COALESCE(d.last_error, ''), + (SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'pdu'), + (SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'edu') + FROM ( + SELECT server_name FROM federation_destinations + UNION + SELECT DISTINCT destination FROM federation_queue + ) n + LEFT JOIN federation_destinations d ON d.server_name = n.server_name + ORDER BY n.server_name`) + if err != nil { + return nil, fmt.Errorf("read the destinations: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []storage.DestinationSummary + for rows.Next() { + var d storage.DestinationSummary + if err := rows.Scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS, + &d.LastFailureMS, &d.LastError, &d.QueuedPDUs, &d.QueuedEDUs); err != nil { + return nil, fmt.Errorf("read the destinations: %w", err) + } + out = append(out, d) + } + return out, rows.Err() +} + +// InboundTransaction implements storage.FederationStore. +func (s *Federation) InboundTransaction(ctx context.Context, origin, txnID string) ([]byte, bool, error) { + var response string + err := s.db.QueryRowContext(ctx, `SELECT response_json FROM federation_inbound_txns + WHERE origin = ? AND txn_id = ?`, origin, txnID).Scan(&response) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, false, nil + case err != nil: + return nil, false, fmt.Errorf("read transaction %q from %q: %w", txnID, origin, err) + } + return []byte(response), true, nil +} + +// RecordInboundTransaction implements storage.FederationStore. A +// transaction already recorded keeps its first response. +func (s *Federation) RecordInboundTransaction(ctx context.Context, origin, txnID string, response []byte, atMS int64) error { + err := s.write(ctx, func(t *tx) error { + found, err := t.exists("federation_inbound_txns", `origin = ? AND txn_id = ?`, origin, txnID) + if err != nil || found { + return err + } + t.put("federation_inbound_txns", rec{ + "origin": origin, "txn_id": txnID, "response_json": string(response), "received_at_ms": atMS, + }) + return nil + }) + if err != nil { + return fmt.Errorf("record transaction %q from %q: %w", txnID, origin, err) + } + return nil +} + +var _ storage.FederationStore = (*Federation)(nil) diff --git a/server/internal/storage/pocketbase/helpers.go b/server/internal/storage/pocketbase/helpers.go new file mode 100644 index 0000000..7394105 --- /dev/null +++ b/server/internal/storage/pocketbase/helpers.go @@ -0,0 +1,83 @@ +package pocketbase + +import ( + "encoding/json" + "strings" +) + +// The counter tables and the names in them, as the SQL drivers have them. +const ( + roomSequences = "roomserver_sequences" + syncSequences = "sync_sequences" + federationSequences = "federation_sequences" + keySequences = "keyserver_sequences" + pushSequences = "push_sequences" + adminSequences = "admin_sequences" + + seqStreamOrdering = "stream_ordering" + seqStateSnapshot = "state_snapshot" + seqBackfillOrdering = "backfill_ordering" + + seqReceipt = "receipt" + seqToDevice = "to_device" + seqAccountData = "account_data" + seqPresence = "presence" + + seqQueue = "queue" + seqDeviceList = "device_list" + seqNotification = "notification" + seqAudit = "audit" +) + +// inList is the IN-list form every store uses: one JSON array argument, +// expanded by json_each, instead of a placeholder per element. The SQL +// endpoint takes no parameters and at most 5000 characters, and a single +// argument is what the driver can move out of a query that would be too +// long; a few hundred separate ones would each have to be moved. +const inList = `(SELECT value FROM json_each(?))` + +// jsonList renders strings as the JSON array inList takes. +func jsonList(values []string) string { + if values == nil { + values = []string{} + } + raw, _ := json.Marshal(values) + return string(raw) +} + +// jsonInts renders integers as the JSON array inList takes. +func jsonInts(values []int64) string { + if values == nil { + values = []int64{} + } + raw, _ := json.Marshal(values) + return string(raw) +} + +// pageBounds clamps a caller's limit and offset. +func pageBounds(limit, offset, defaultLimit, maxLimit int) (int, int) { + if limit <= 0 { + limit = defaultLimit + } + if limit > maxLimit { + limit = maxLimit + } + if offset < 0 { + offset = 0 + } + return limit, offset +} + +// escapeLike makes a literal string safe to embed in a LIKE pattern that +// declares '\' as its escape character. +func escapeLike(s string) string { + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + if r == '\\' || r == '%' || r == '_' { + b.WriteRune('\\') + } + b.WriteRune(r) + } + return b.String() +} diff --git a/server/internal/storage/pocketbase/keys.go b/server/internal/storage/pocketbase/keys.go new file mode 100644 index 0000000..b37a3b9 --- /dev/null +++ b/server/internal/storage/pocketbase/keys.go @@ -0,0 +1,766 @@ +package pocketbase + +import ( + "cmp" + "context" + "database/sql" + "errors" + "fmt" + "slices" + "strconv" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Keys is the PocketBase implementation of storage.KeyStore. +// +// Callers hand it key JSON already in canonical form, so "the same key +// uploaded again" is a byte comparison. Where the SQL store reads back what +// it has written inside its transaction — a device list position moved +// earlier in the call, a backup's key count and etag after a change — this +// one carries the value in the call instead. +type Keys struct{ *base } + +// maxInList caps an IN list, as the SQL store does. +const maxInList = 500 + +// keysWrite is one write call's device list positions, as it has left them. +type keysWrite struct { + *tx + lists map[string]int64 +} + +func newKeysWrite(t *tx) *keysWrite { return &keysWrite{tx: t, lists: map[string]int64{}} } + +// --- the device list stream --------------------------------------------------- + +// bumpDeviceList moves a user's device list once per device named, or once +// for the user when none is, and returns an update per move. Each device +// gets a position of its own, because a remote server treats a stream ID it +// has already seen as a duplicate. +func (w *keysWrite) bumpDeviceList(userID string, deviceIDs []string) ([]storage.DeviceListUpdate, error) { + if len(deviceIDs) == 0 { + deviceIDs = []string{""} + } + n := int64(len(deviceIDs)) + last, err := w.advance(keySequences, seqDeviceList, n) + if err != nil { + return nil, err + } + + prev, ok := w.lists[userID] + if !ok { + err = w.queryRow(`SELECT stream_pos FROM keyserver_device_lists WHERE user_id = ?`, userID).Scan(&prev) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("read the device list position of %q: %w", userID, err) + } + } + + out := make([]storage.DeviceListUpdate, 0, n) + for i, deviceID := range deviceIDs { + pos := last - n + 1 + int64(i) + out = append(out, storage.DeviceListUpdate{ + UserID: userID, DeviceID: deviceID, StreamPos: pos, PrevPos: prev, + }) + prev = pos + } + w.put("keyserver_device_lists", rec{"user_id": userID, "stream_pos": last}) + w.lists[userID] = last + return out, nil +} + +// MarkDeviceListsChanged implements storage.KeyStore. +func (s *Keys) MarkDeviceListsChanged(ctx context.Context, userIDs []string) (int64, error) { + var top int64 + err := s.write(ctx, func(t *tx) error { + w := newKeysWrite(t) + for _, userID := range dedupe(userIDs) { + updates, err := w.bumpDeviceList(userID, nil) + if err != nil { + return err + } + top = updates[len(updates)-1].StreamPos + } + return nil + }) + return top, err +} + +// DeviceListChanges implements storage.KeyStore. +func (s *Keys) DeviceListChanges(ctx context.Context, userIDs []string, from, to int64) ([]storage.DeviceListChange, error) { + var out []storage.DeviceListChange + for chunk := range slices.Chunk(dedupe(userIDs), maxInList) { + query := ` + SELECT user_id, stream_pos FROM keyserver_device_lists + WHERE user_id IN ` + inList + ` AND stream_pos > ?` + args := []any{jsonList(chunk), from} + if to > 0 { + query += ` AND stream_pos <= ?` + args = append(args, to) + } + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read device list changes: %w", err) + } + for rows.Next() { + var c storage.DeviceListChange + if err := rows.Scan(&c.UserID, &c.StreamPos); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("read device list changes: %w", err) + } + out = append(out, c) + } + _ = rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read device list changes: %w", err) + } + } + slices.SortFunc(out, func(a, b storage.DeviceListChange) int { return cmp.Compare(a.StreamPos, b.StreamPos) }) + return out, nil +} + +// DeviceListPosition implements storage.KeyStore. +func (s *Keys) DeviceListPosition(ctx context.Context, userID string) (int64, error) { + var pos int64 + err := s.db.QueryRowContext(ctx, `SELECT stream_pos FROM keyserver_device_lists WHERE user_id = ?`, userID).Scan(&pos) + switch { + case errors.Is(err, sql.ErrNoRows): + return 0, nil + case err != nil: + return 0, fmt.Errorf("read the device list position of %q: %w", userID, err) + } + return pos, nil +} + +// --- device keys -------------------------------------------------------------- + +// SetDeviceKeys implements storage.KeyStore. +func (s *Keys) SetDeviceKeys(ctx context.Context, keys storage.DeviceKeys) (storage.DeviceListUpdate, error) { + var update storage.DeviceListUpdate + err := s.write(ctx, func(t *tx) error { + var existing string + err := t.queryRow(`SELECT key_json FROM keyserver_device_keys WHERE user_id = ? AND device_id = ?`, + keys.UserID, keys.DeviceID).Scan(&existing) + switch { + case err == nil && existing == string(keys.JSON): + return nil + case err != nil && !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("read the keys of %s/%s: %w", keys.UserID, keys.DeviceID, err) + } + + t.put("keyserver_device_keys", rec{"user_id": keys.UserID, "device_id": keys.DeviceID, "key_json": string(keys.JSON)}) + updates, err := newKeysWrite(t).bumpDeviceList(keys.UserID, []string{keys.DeviceID}) + if err != nil { + return err + } + update = updates[0] + return nil + }) + return update, err +} + +// DeviceKeys implements storage.KeyStore. +func (s *Keys) DeviceKeys(ctx context.Context, userID string, deviceIDs []string) ([]storage.DeviceKeys, error) { + query := `SELECT device_id, key_json FROM keyserver_device_keys WHERE user_id = ?` + args := []any{userID} + if len(deviceIDs) > 0 { + deviceIDs = dedupe(deviceIDs) + if len(deviceIDs) > maxInList { + deviceIDs = deviceIDs[:maxInList] + } + query += ` AND device_id IN ` + inList + args = append(args, jsonList(deviceIDs)) + } + query += ` ORDER BY device_id` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the device keys of %q: %w", userID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.DeviceKeys + for rows.Next() { + k := storage.DeviceKeys{UserID: userID} + var keyJSON string + if err := rows.Scan(&k.DeviceID, &keyJSON); err != nil { + return nil, fmt.Errorf("read the device keys of %q: %w", userID, err) + } + k.JSON = []byte(keyJSON) + out = append(out, k) + } + return out, rows.Err() +} + +// DeleteDeviceKeys implements storage.KeyStore. +func (s *Keys) DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []string) ([]storage.DeviceListUpdate, error) { + var updates []storage.DeviceListUpdate + err := s.write(ctx, func(t *tx) error { + where := `user_id = ?` + args := []any{userID} + if len(deviceIDs) > 0 { + where += ` AND device_id IN ` + inList + args = append(args, jsonList(dedupe(deviceIDs))) + } + + // The devices that lose identity keys are the ones whose deletion + // moves the stream. + had, err := s.queryStringsTx(t, `SELECT device_id FROM keyserver_device_keys WHERE `+where+` ORDER BY device_id`, args...) + if err != nil { + return fmt.Errorf("read the devices of %q: %w", userID, err) + } + + for _, table := range []string{"keyserver_device_keys", "keyserver_one_time_keys", "keyserver_fallback_keys"} { + if _, err := t.delWhere(table, where, args...); err != nil { + return fmt.Errorf("delete from %s for %q: %w", table, userID, err) + } + } + if len(had) == 0 { + return nil + } + updates, err = newKeysWrite(t).bumpDeviceList(userID, had) + return err + }) + return updates, err +} + +// queryStringsTx runs a one-column query inside a write. +func (s *Keys) queryStringsTx(t *tx, query string, args ...any) ([]string, error) { + rows, err := t.query(query, args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} + +// --- one-time and fallback keys ----------------------------------------------- + +// AddOneTimeKeys implements storage.KeyStore. The same key again is not an +// error; another key under the same ID is a conflict, and nothing is +// stored. +func (s *Keys) AddOneTimeKeys(ctx context.Context, userID, deviceID string, keys []storage.OneTimeKey) error { + if len(keys) == 0 { + return nil + } + return s.write(ctx, func(t *tx) error { + ids := make([]string, len(keys)) + for i, k := range keys { + ids[i] = k.KeyID + } + held := map[string]string{} + rows, err := t.query(`SELECT key_id, key_json FROM keyserver_one_time_keys + WHERE user_id = ? AND device_id = ? AND key_id IN `+inList, userID, deviceID, jsonList(ids)) + if err != nil { + return fmt.Errorf("read the one-time keys of %s/%s: %w", userID, deviceID, err) + } + for rows.Next() { + var id, keyJSON string + if err := rows.Scan(&id, &keyJSON); err != nil { + _ = rows.Close() + return fmt.Errorf("read the one-time keys of %s/%s: %w", userID, deviceID, err) + } + held[id] = keyJSON + } + _ = rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("read the one-time keys of %s/%s: %w", userID, deviceID, err) + } + + for _, k := range keys { + if existing, ok := held[k.KeyID]; ok { + if existing == string(k.JSON) { + continue + } + return fmt.Errorf("%w: %s", storage.ErrKeyConflict, k.KeyID) + } + t.put("keyserver_one_time_keys", rec{ + "user_id": userID, "device_id": deviceID, "key_id": k.KeyID, + "algorithm": k.Algorithm(), "key_json": string(k.JSON), + }) + held[k.KeyID] = string(k.JSON) + } + return nil + }) +} + +// OneTimeKeyCounts implements storage.KeyStore. +func (s *Keys) OneTimeKeyCounts(ctx context.Context, userID, deviceID string) (map[string]int, error) { + rows, err := s.db.QueryContext(ctx, `SELECT algorithm, COUNT(*) FROM keyserver_one_time_keys + WHERE user_id = ? AND device_id = ? GROUP BY algorithm`, userID, deviceID) + if err != nil { + return nil, fmt.Errorf("count the one-time keys of %s/%s: %w", userID, deviceID, err) + } + defer func() { _ = rows.Close() }() + + out := map[string]int{} + for rows.Next() { + var ( + algorithm string + n int + ) + if err := rows.Scan(&algorithm, &n); err != nil { + return nil, fmt.Errorf("count the one-time keys of %s/%s: %w", userID, deviceID, err) + } + out[algorithm] = n + } + return out, rows.Err() +} + +const selectFallbackKeySQL = ` + SELECT key_id, key_json FROM keyserver_fallback_keys + WHERE user_id = ? AND device_id = ? AND algorithm = ?` + +// SetFallbackKeys implements storage.KeyStore. The same key again keeps its +// used flag; a new one starts unused. +func (s *Keys) SetFallbackKeys(ctx context.Context, userID, deviceID string, keys []storage.OneTimeKey) error { + if len(keys) == 0 { + return nil + } + return s.write(ctx, func(t *tx) error { + for _, k := range keys { + var existingID, existingJSON string + err := t.queryRow(selectFallbackKeySQL, userID, deviceID, k.Algorithm()).Scan(&existingID, &existingJSON) + switch { + case err == nil && existingID == k.KeyID && existingJSON == string(k.JSON): + continue + case err != nil && !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("read the fallback key of %s/%s: %w", userID, deviceID, err) + } + t.put("keyserver_fallback_keys", rec{ + "user_id": userID, "device_id": deviceID, "algorithm": k.Algorithm(), + "key_id": k.KeyID, "key_json": string(k.JSON), "used": false, + }) + } + return nil + }) +} + +// UnusedFallbackKeyTypes implements storage.KeyStore. +func (s *Keys) UnusedFallbackKeyTypes(ctx context.Context, userID, deviceID string) ([]string, error) { + out, err := s.queryStrings(ctx, `SELECT algorithm FROM keyserver_fallback_keys + WHERE user_id = ? AND device_id = ? AND used = ? ORDER BY algorithm`, userID, deviceID, false) + if err != nil { + return nil, fmt.Errorf("read the fallback keys of %s/%s: %w", userID, deviceID, err) + } + return out, nil +} + +// ClaimKey implements storage.KeyStore. The read and the delete happen under +// the write lock, so a claim cannot lose the key it read to another one. +func (s *Keys) ClaimKey(ctx context.Context, userID, deviceID, algorithm string) (storage.ClaimedKey, bool, error) { + var ( + claimed storage.ClaimedKey + found bool + ) + err := s.write(ctx, func(t *tx) error { + var keyID, keyJSON string + err := t.queryRow(`SELECT key_id, key_json FROM keyserver_one_time_keys + WHERE user_id = ? AND device_id = ? AND algorithm = ? + ORDER BY key_id LIMIT 1`, userID, deviceID, algorithm).Scan(&keyID, &keyJSON) + switch { + case err == nil: + t.del("keyserver_one_time_keys", rowID("keyserver_one_time_keys", userID, deviceID, keyID)) + claimed = storage.ClaimedKey{OneTimeKey: storage.OneTimeKey{KeyID: keyID, JSON: []byte(keyJSON)}} + found = true + return nil + case !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("claim a one-time key of %s/%s: %w", userID, deviceID, err) + } + + err = t.queryRow(selectFallbackKeySQL, userID, deviceID, algorithm).Scan(&keyID, &keyJSON) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil + case err != nil: + return fmt.Errorf("claim the fallback key of %s/%s: %w", userID, deviceID, err) + } + t.put("keyserver_fallback_keys", rec{"user_id": userID, "device_id": deviceID, "algorithm": algorithm, "used": true}) + claimed = storage.ClaimedKey{OneTimeKey: storage.OneTimeKey{KeyID: keyID, JSON: []byte(keyJSON)}, Fallback: true} + found = true + return nil + }) + if err != nil { + return storage.ClaimedKey{}, false, err + } + return claimed, found, nil +} + +// --- cross-signing ------------------------------------------------------------ + +// SetCrossSigningKeys implements storage.KeyStore. A nil key deletes that +// purpose. +func (s *Keys) SetCrossSigningKeys(ctx context.Context, userID string, keys map[string][]byte) (storage.DeviceListUpdate, error) { + var update storage.DeviceListUpdate + err := s.write(ctx, func(t *tx) error { + for _, purpose := range sortedKeys(keys) { + keyJSON := keys[purpose] + if keyJSON == nil { + if _, err := t.delKey("keyserver_cross_signing_keys", userID, purpose); err != nil { + return fmt.Errorf("store the %s key of %q: %w", purpose, userID, err) + } + continue + } + t.put("keyserver_cross_signing_keys", rec{"user_id": userID, "purpose": purpose, "key_json": string(keyJSON)}) + } + updates, err := newKeysWrite(t).bumpDeviceList(userID, nil) + if err != nil { + return err + } + update = updates[0] + return nil + }) + return update, err +} + +// CrossSigningKeys implements storage.KeyStore. +func (s *Keys) CrossSigningKeys(ctx context.Context, userID string) (map[string][]byte, error) { + rows, err := s.db.QueryContext(ctx, `SELECT purpose, key_json FROM keyserver_cross_signing_keys WHERE user_id = ?`, userID) + if err != nil { + return nil, fmt.Errorf("read the cross-signing keys of %q: %w", userID, err) + } + defer func() { _ = rows.Close() }() + + out := map[string][]byte{} + for rows.Next() { + var purpose, keyJSON string + if err := rows.Scan(&purpose, &keyJSON); err != nil { + return nil, fmt.Errorf("read the cross-signing keys of %q: %w", userID, err) + } + out[purpose] = []byte(keyJSON) + } + return out, rows.Err() +} + +// --- signatures --------------------------------------------------------------- + +// AddSignatures implements storage.KeyStore. +func (s *Keys) AddSignatures(ctx context.Context, sigs []storage.KeySignature) ([]storage.DeviceListUpdate, error) { + if len(sigs) == 0 { + return nil, nil + } + var updates []storage.DeviceListUpdate + err := s.write(ctx, func(t *tx) error { + w := newKeysWrite(t) + targets := map[string][]string{} + for _, sig := range sigs { + t.put("keyserver_signatures", rec{ + "target_user_id": sig.TargetUserID, "target_key_id": sig.TargetKeyID, + "signer_user_id": sig.SignerUserID, "signer_key_id": sig.SignerKeyID, + "signature": sig.Signature, + }) + if !slices.Contains(targets[sig.TargetUserID], sig.TargetKeyID) { + targets[sig.TargetUserID] = append(targets[sig.TargetUserID], sig.TargetKeyID) + } + } + for _, userID := range sortedKeys(targets) { + moved, err := w.bumpDeviceList(userID, targets[userID]) + if err != nil { + return err + } + updates = append(updates, moved...) + } + return nil + }) + return updates, err +} + +// Signatures implements storage.KeyStore. +func (s *Keys) Signatures(ctx context.Context, targetUserID string) ([]storage.KeySignature, error) { + rows, err := s.db.QueryContext(ctx, `SELECT target_key_id, signer_user_id, signer_key_id, signature + FROM keyserver_signatures WHERE target_user_id = ? + ORDER BY target_key_id, signer_user_id, signer_key_id`, targetUserID) + if err != nil { + return nil, fmt.Errorf("read the signatures of %q's keys: %w", targetUserID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.KeySignature + for rows.Next() { + sig := storage.KeySignature{TargetUserID: targetUserID} + if err := rows.Scan(&sig.TargetKeyID, &sig.SignerUserID, &sig.SignerKeyID, &sig.Signature); err != nil { + return nil, fmt.Errorf("read the signatures of %q's keys: %w", targetUserID, err) + } + out = append(out, sig) + } + return out, rows.Err() +} + +// --- key backup --------------------------------------------------------------- + +// parseBackupVersion reads a version number. A version that is not one is a +// version that does not exist. +func parseBackupVersion(version string) (int64, bool) { + n, err := strconv.ParseInt(version, 10, 64) + return n, err == nil && n > 0 +} + +// rowQuerier is what readBackup reads through: the pool, or a write. +type rowQuerier func(query string, args ...any) *sql.Row + +func (s *Keys) poolRow(ctx context.Context) rowQuerier { + return func(query string, args ...any) *sql.Row { return s.db.QueryRowContext(ctx, query, args...) } +} + +// readBackup reads one live version, with its key count. +func readBackup(row rowQuerier, userID, version string) (storage.BackupVersion, int64, int64, error) { + var n int64 + if version == "" { + if err := row(`SELECT COALESCE(MAX(version), 0) FROM keyserver_backup_versions + WHERE user_id = ? AND deleted = ?`, userID, false).Scan(&n); err != nil { + return storage.BackupVersion{}, 0, 0, fmt.Errorf("read the backup version of %q: %w", userID, err) + } + if n == 0 { + return storage.BackupVersion{}, 0, 0, storage.ErrNoSuchBackup + } + } else { + var ok bool + if n, ok = parseBackupVersion(version); !ok { + return storage.BackupVersion{}, 0, 0, storage.ErrNoSuchBackup + } + } + + out := storage.BackupVersion{UserID: userID, Version: strconv.FormatInt(n, 10)} + var ( + authData string + etag int64 + ) + err := row(`SELECT algorithm, auth_data, etag FROM keyserver_backup_versions + WHERE user_id = ? AND version = ? AND deleted = ?`, userID, n, false).Scan(&out.Algorithm, &authData, &etag) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.BackupVersion{}, 0, 0, storage.ErrNoSuchBackup + case err != nil: + return storage.BackupVersion{}, 0, 0, fmt.Errorf("read backup version %d of %q: %w", n, userID, err) + } + out.AuthData = []byte(authData) + out.ETag = strconv.FormatInt(etag, 10) + if err := row(`SELECT COUNT(*) FROM keyserver_backup_keys WHERE user_id = ? AND version = ?`, userID, n). + Scan(&out.Count); err != nil { + return storage.BackupVersion{}, 0, 0, fmt.Errorf("count the keys of backup version %d of %q: %w", n, userID, err) + } + return out, n, etag, nil +} + +// CreateBackup implements storage.KeyStore. +func (s *Keys) CreateBackup(ctx context.Context, userID, algorithm string, authData []byte) (string, error) { + var version int64 + err := s.write(ctx, func(t *tx) error { + if err := t.queryRow(`SELECT COALESCE(MAX(version), 0) FROM keyserver_backup_versions WHERE user_id = ?`, + userID).Scan(&version); err != nil { + return fmt.Errorf("read the backup versions of %q: %w", userID, err) + } + version++ + t.put("keyserver_backup_versions", rec{ + "user_id": userID, "version": version, "algorithm": algorithm, + "auth_data": string(authData), "etag": int64(0), "deleted": false, + }) + return nil + }) + if err != nil { + return "", err + } + return strconv.FormatInt(version, 10), nil +} + +// Backup implements storage.KeyStore. +func (s *Keys) Backup(ctx context.Context, userID, version string) (storage.BackupVersion, error) { + out, _, _, err := readBackup(s.poolRow(ctx), userID, version) + return out, err +} + +// UpdateBackup implements storage.KeyStore. +func (s *Keys) UpdateBackup(ctx context.Context, userID, version string, authData []byte) error { + n, ok := parseBackupVersion(version) + if !ok { + return storage.ErrNoSuchBackup + } + return s.write(ctx, func(t *tx) error { + affected, err := t.setWhere("keyserver_backup_versions", rec{"auth_data": string(authData)}, + `user_id = ? AND version = ? AND deleted = ?`, userID, n, false) + if err != nil { + return fmt.Errorf("update backup version %d of %q: %w", n, userID, err) + } + if affected == 0 { + return storage.ErrNoSuchBackup + } + return nil + }) +} + +// DeleteBackup implements storage.KeyStore. +func (s *Keys) DeleteBackup(ctx context.Context, userID, version string) error { + n, ok := parseBackupVersion(version) + if !ok { + return storage.ErrNoSuchBackup + } + return s.write(ctx, func(t *tx) error { + affected, err := t.setWhere("keyserver_backup_versions", rec{"deleted": true}, + `user_id = ? AND version = ? AND deleted = ?`, userID, n, false) + if err != nil { + return fmt.Errorf("delete backup version %d of %q: %w", n, userID, err) + } + if affected == 0 { + return storage.ErrNoSuchBackup + } + if _, err := t.delWhere("keyserver_backup_keys", `user_id = ? AND version = ?`, userID, n); err != nil { + return fmt.Errorf("delete the keys of backup version %d of %q: %w", n, userID, err) + } + return nil + }) +} + +// PutBackupKeys implements storage.KeyStore. +func (s *Keys) PutBackupKeys(ctx context.Context, userID, version string, keys []storage.BackupKey) (storage.BackupVersion, error) { + var out storage.BackupVersion + err := s.write(ctx, func(t *tx) error { + current, n, etag, err := readBackup(t.queryRow, userID, version) + if err != nil { + return err + } + + changed := false + written := map[string]storage.BackupKey{} + for _, k := range keys { + slot := k.RoomID + "\x00" + k.SessionID + existing, have := written[slot] + if !have { + var sessionData string + err := t.queryRow(`SELECT first_message_index, forwarded_count, is_verified, session_data + FROM keyserver_backup_keys + WHERE user_id = ? AND version = ? AND room_id = ? AND session_id = ?`, userID, n, k.RoomID, k.SessionID). + Scan(&existing.FirstMessageIndex, &existing.ForwardedCount, &existing.IsVerified, &sessionData) + switch { + case err == nil: + have = true + case !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("read backed-up key %s/%s: %w", k.RoomID, k.SessionID, err) + } + if !have { + current.Count++ + } + } + if have && !k.Replaces(existing) { + written[slot] = existing + continue + } + t.put("keyserver_backup_keys", rec{ + "user_id": userID, "version": n, "room_id": k.RoomID, "session_id": k.SessionID, + "first_message_index": k.FirstMessageIndex, "forwarded_count": k.ForwardedCount, + "is_verified": k.IsVerified, "session_data": string(k.SessionData), + }) + written[slot] = k + changed = true + } + if changed { + etag++ + t.put("keyserver_backup_versions", rec{"user_id": userID, "version": n, "etag": etag}) + } + current.ETag = strconv.FormatInt(etag, 10) + out = current + return nil + }) + return out, err +} + +// backupKeyScope is the WHERE clause the key reads and deletes share. +func backupKeyScope(userID string, version int64, roomID, sessionID string) (string, []any) { + where := `user_id = ? AND version = ?` + args := []any{userID, version} + if roomID != "" { + where += ` AND room_id = ?` + args = append(args, roomID) + if sessionID != "" { + where += ` AND session_id = ?` + args = append(args, sessionID) + } + } + return where, args +} + +// BackupKeys implements storage.KeyStore. +func (s *Keys) BackupKeys(ctx context.Context, userID, version, roomID, sessionID string) ([]storage.BackupKey, error) { + _, n, _, err := readBackup(s.poolRow(ctx), userID, version) + if err != nil { + return nil, err + } + where, args := backupKeyScope(userID, n, roomID, sessionID) + rows, err := s.db.QueryContext(ctx, ` + SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data + FROM keyserver_backup_keys WHERE `+where+` ORDER BY room_id, session_id`, args...) + if err != nil { + return nil, fmt.Errorf("read the keys of backup version %d of %q: %w", n, userID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.BackupKey + for rows.Next() { + var ( + k storage.BackupKey + sessionData string + ) + if err := rows.Scan(&k.RoomID, &k.SessionID, &k.FirstMessageIndex, &k.ForwardedCount, + &k.IsVerified, &sessionData); err != nil { + return nil, fmt.Errorf("read the keys of backup version %d of %q: %w", n, userID, err) + } + k.SessionData = []byte(sessionData) + out = append(out, k) + } + return out, rows.Err() +} + +// DeleteBackupKeys implements storage.KeyStore. +func (s *Keys) DeleteBackupKeys(ctx context.Context, userID, version, roomID, sessionID string) (storage.BackupVersion, error) { + var out storage.BackupVersion + err := s.write(ctx, func(t *tx) error { + current, n, etag, err := readBackup(t.queryRow, userID, version) + if err != nil { + return err + } + where, args := backupKeyScope(userID, n, roomID, sessionID) + deleted, err := t.delWhere("keyserver_backup_keys", where, args...) + if err != nil { + return fmt.Errorf("delete keys from backup version %d of %q: %w", n, userID, err) + } + if deleted > 0 { + etag++ + t.put("keyserver_backup_versions", rec{"user_id": userID, "version": n, "etag": etag}) + } + current.Count -= deleted + current.ETag = strconv.FormatInt(etag, 10) + out = current + return nil + }) + return out, err +} + +// --- helpers ------------------------------------------------------------------ + +// dedupe returns the distinct values of a list, sorted. +func dedupe(values []string) []string { + out := slices.Clone(values) + slices.Sort(out) + return slices.Compact(out) +} + +// sortedKeys is the keys of a map in order, which makes the order of writes +// — and so of the positions handed out — deterministic. +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + slices.Sort(out) + return out +} + +var _ storage.KeyStore = (*Keys)(nil) diff --git a/server/internal/storage/pocketbase/media.go b/server/internal/storage/pocketbase/media.go new file mode 100644 index 0000000..5675e64 --- /dev/null +++ b/server/internal/storage/pocketbase/media.go @@ -0,0 +1,195 @@ +package pocketbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Media is the PocketBase implementation of storage.MediaStore: the +// metadata of media. The content is the blob store's. +type Media struct{ *base } + +// CreateMedia implements storage.MediaStore. +func (s *Media) CreateMedia(ctx context.Context, m storage.Media) error { + return s.write(ctx, func(t *tx) error { + taken, err := t.exists("media_repository", `media_origin = ? AND media_id = ?`, m.Origin, m.MediaID) + if err != nil { + return fmt.Errorf("store media %s/%s: %w", m.Origin, m.MediaID, err) + } + if taken { + return storage.ErrMediaExists + } + t.put("media_repository", mediaRow(m)) + return nil + }) +} + +func mediaRow(m storage.Media) rec { + return rec{ + "media_origin": m.Origin, "media_id": m.MediaID, "content_type": m.ContentType, + "upload_name": m.UploadName, "file_size": m.Size, "sha256": m.SHA256, + "user_id": m.UserID, "created_at_ms": m.CreatedAtMS, "pending": m.Pending, + "expires_at_ms": m.ExpiresAtMS, + } +} + +// Media implements storage.MediaStore. +func (s *Media) Media(ctx context.Context, origin, mediaID string) (storage.Media, error) { + var m storage.Media + err := s.db.QueryRowContext(ctx, ` + SELECT media_origin, media_id, content_type, upload_name, file_size, sha256, + user_id, created_at_ms, pending, expires_at_ms + FROM media_repository WHERE media_origin = ? AND media_id = ?`, origin, mediaID).Scan( + &m.Origin, &m.MediaID, &m.ContentType, &m.UploadName, &m.Size, &m.SHA256, + &m.UserID, &m.CreatedAtMS, &m.Pending, &m.ExpiresAtMS) + switch { + case err == nil: + return m, nil + case errors.Is(err, sql.ErrNoRows): + return storage.Media{}, storage.ErrNoSuchMedia + default: + return storage.Media{}, fmt.Errorf("read media %s/%s: %w", origin, mediaID, err) + } +} + +// CompleteUpload implements storage.MediaStore. Only pending media is +// completed, so that two uploads for the same ID cannot both win. +func (s *Media) CompleteUpload(ctx context.Context, origin, mediaID string, c storage.MediaContent) error { + var completed bool + err := s.write(ctx, func(t *tx) error { + n, err := t.setWhere("media_repository", rec{ + "content_type": c.ContentType, "upload_name": c.UploadName, "file_size": c.Size, + "sha256": c.SHA256, "pending": false, "expires_at_ms": int64(0), + }, `media_origin = ? AND media_id = ? AND pending = ?`, origin, mediaID, true) + completed = n == 1 + return err + }) + if err != nil { + return fmt.Errorf("complete upload %s/%s: %w", origin, mediaID, err) + } + if completed { + return nil + } + // Nothing pending matched: either there is no such media, or its + // content is already there. + if _, err := s.Media(ctx, origin, mediaID); err != nil { + return err + } + return storage.ErrMediaUploaded +} + +// PendingUploads implements storage.MediaStore. +func (s *Media) PendingUploads(ctx context.Context, userID string, nowMS int64) (int, error) { + var n int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM media_repository + WHERE pending = ? AND user_id = ? AND expires_at_ms > ?`, true, userID, nowMS).Scan(&n); err != nil { + return 0, fmt.Errorf("count the pending uploads of %s: %w", userID, err) + } + return n, nil +} + +// DeleteExpiredUploads implements storage.MediaStore. +func (s *Media) DeleteExpiredUploads(ctx context.Context, nowMS int64) (int, error) { + var n int + err := s.write(ctx, func(t *tx) error { + var err error + n, err = t.delWhere("media_repository", `pending = ? AND expires_at_ms <= ?`, true, nowMS) + return err + }) + if err != nil { + return 0, fmt.Errorf("delete expired uploads: %w", err) + } + return n, nil +} + +// Thumbnail implements storage.MediaStore. +func (s *Media) Thumbnail(ctx context.Context, origin, mediaID string, width, height int, method string) (storage.Thumbnail, error) { + var t storage.Thumbnail + err := s.db.QueryRowContext(ctx, ` + SELECT media_origin, media_id, width, height, method, content_type, + file_size, sha256, created_at_ms + FROM media_thumbnails + WHERE media_origin = ? AND media_id = ? AND width = ? AND height = ? AND method = ?`, + origin, mediaID, width, height, method).Scan( + &t.Origin, &t.MediaID, &t.Width, &t.Height, &t.Method, &t.ContentType, + &t.Size, &t.SHA256, &t.CreatedAtMS) + switch { + case err == nil: + return t, nil + case errors.Is(err, sql.ErrNoRows): + return storage.Thumbnail{}, storage.ErrNoSuchMedia + default: + return storage.Thumbnail{}, fmt.Errorf("read a thumbnail of %s/%s: %w", origin, mediaID, err) + } +} + +// StoreThumbnail implements storage.MediaStore. +func (s *Media) StoreThumbnail(ctx context.Context, t storage.Thumbnail) error { + err := s.write(ctx, func(w *tx) error { + w.put("media_thumbnails", rec{ + "media_origin": t.Origin, "media_id": t.MediaID, "width": t.Width, "height": t.Height, + "method": t.Method, "content_type": t.ContentType, "file_size": t.Size, + "sha256": t.SHA256, "created_at_ms": t.CreatedAtMS, + }) + return nil + }) + if err != nil { + return fmt.Errorf("store a thumbnail of %s/%s: %w", t.Origin, t.MediaID, err) + } + return nil +} + +// MediaUsage implements storage.MediaStore. It counts every upload at its +// own size, even when two of them are the same bytes. +func (s *Media) MediaUsage(ctx context.Context, origin, userID string) (storage.MediaUsage, error) { + var u storage.MediaUsage + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*), COALESCE(SUM(file_size), 0) FROM media_repository + WHERE user_id = ? AND media_origin = ? AND pending = ?`, userID, origin, false). + Scan(&u.Count, &u.Bytes); err != nil { + return storage.MediaUsage{}, fmt.Errorf("total the uploads of %s: %w", userID, err) + } + return u, nil +} + +// MediaQuota implements storage.MediaStore. +func (s *Media) MediaQuota(ctx context.Context, userID string) (int64, bool, error) { + var maxBytes int64 + err := s.db.QueryRowContext(ctx, `SELECT max_bytes FROM media_quotas WHERE user_id = ?`, userID).Scan(&maxBytes) + switch { + case errors.Is(err, sql.ErrNoRows): + return 0, false, nil + case err != nil: + return 0, false, fmt.Errorf("read the media quota of %s: %w", userID, err) + } + return maxBytes, true, nil +} + +// SetMediaQuota implements storage.MediaStore. +func (s *Media) SetMediaQuota(ctx context.Context, userID string, maxBytes int64) error { + err := s.write(ctx, func(t *tx) error { + t.put("media_quotas", rec{"user_id": userID, "max_bytes": maxBytes}) + return nil + }) + if err != nil { + return fmt.Errorf("set the media quota of %s: %w", userID, err) + } + return nil +} + +// DeleteMediaQuota implements storage.MediaStore. +func (s *Media) DeleteMediaQuota(ctx context.Context, userID string) error { + err := s.write(ctx, func(t *tx) error { + _, err := t.delKey("media_quotas", userID) + return err + }) + if err != nil { + return fmt.Errorf("delete the media quota of %s: %w", userID, err) + } + return nil +} + +var _ storage.MediaStore = (*Media)(nil) diff --git a/server/internal/storage/pocketbase/pbtest/pbtest.go b/server/internal/storage/pocketbase/pbtest/pbtest.go new file mode 100644 index 0000000..8b81e19 --- /dev/null +++ b/server/internal/storage/pocketbase/pbtest/pbtest.go @@ -0,0 +1,64 @@ +// Package pbtest runs a real PocketBase server in-process for tests, so the +// pocketbase driver is tested against the same REST API it uses in +// production, with no container to start. +package pbtest + +import ( + "net/http/httptest" + "testing" + + "github.com/pocketbase/pocketbase/apis" + "github.com/pocketbase/pocketbase/core" + _ "github.com/pocketbase/pocketbase/migrations" // PocketBase's system migrations; without them its bootstrap has no tables +) + +// Superuser is the account the server is started with. +const ( + SuperuserEmail = "warp-tests@example.com" + SuperuserPassword = "warp-tests-password" +) + +// Server is a running PocketBase. +type Server struct { + // URL is the server's address, for database.connection_string. + URL string + + // DataDir is its pb_data, where a test can look for what it stored. + DataDir string +} + +// Start runs a fresh PocketBase with an empty data directory. It is stopped +// when the test ends. +func Start(tb testing.TB) Server { + tb.Helper() + + dataDir := tb.TempDir() + app := core.NewBaseApp(core.BaseAppConfig{DataDir: dataDir}) + if err := app.Bootstrap(); err != nil { + tb.Fatalf("pocketbase: bootstrap: %v", err) + } + tb.Cleanup(func() { _ = app.ResetBootstrapState() }) + + superusers, err := app.FindCollectionByNameOrId(core.CollectionNameSuperusers) + if err != nil { + tb.Fatalf("pocketbase: %v", err) + } + su := core.NewRecord(superusers) + su.SetEmail(SuperuserEmail) + su.SetPassword(SuperuserPassword) + if err := app.Save(su); err != nil { + tb.Fatalf("pocketbase: create the superuser: %v", err) + } + + router, err := apis.NewRouter(app) + if err != nil { + tb.Fatalf("pocketbase: router: %v", err) + } + mux, err := router.BuildMux() + if err != nil { + tb.Fatalf("pocketbase: router: %v", err) + } + server := httptest.NewServer(mux) + tb.Cleanup(server.Close) + return Server{URL: server.URL, DataDir: dataDir} +} diff --git a/server/internal/storage/pocketbase/pocketbase.go b/server/internal/storage/pocketbase/pocketbase.go index 27341a4..d1fb461 100644 --- a/server/internal/storage/pocketbase/pocketbase.go +++ b/server/internal/storage/pocketbase/pocketbase.go @@ -1,278 +1,338 @@ -// Package pocketbase is the PocketBase storage driver: Warp on an embedded -// PocketBase app, for "appliance" deployments that want one binary and one -// data directory. +// Package pocketbase is the PocketBase storage driver: Warp's data in a +// standalone PocketBase server, as collections, reached only through +// PocketBase's REST API — the way the postgres driver reaches PostgreSQL +// over its wire protocol. // -// PLAN.md section 4 set out the shape, and open question 4 asked how far -// to take it. The answer here is the cheaper of the two: the event DAG and -// everything else Warp stores are a poor fit for PocketBase's record -// model, so the Warp tables are the sqlite driver's own, migrated by Warp -// and written through the same shared store, on PocketBase's underlying -// SQLite connection rather than through its collection API. What PocketBase -// adds is everything around that: its data directory and its system -// tables, which a PocketBase admin or a later Warp feature can build on, and -// its filesystem abstraction, which puts media content on local disk or in -// an S3 bucket. -// -// The result is at parity with the sqlite driver by construction — it is -// the same SQL on the same engine — and the storage tests run against it -// to keep it that way. +// Each of the SQL drivers' tables is a collection of the same name and +// columns (see schema.go), so a PocketBase admin sees Warp's rooms, events +// and accounts as records. The stores read them through the superuser SQL +// endpoint with the same queries the SQL drivers use (sqldriver.go), and +// write them as record operations in batch requests (batch.go), which +// PocketBase runs as one transaction each. Media content goes into a file +// field, which is PocketBase's own storage: its pb_data/storage, or an S3 +// bucket. package pocketbase import ( "context" - "crypto/sha256" - "encoding/hex" + "database/sql" "errors" "fmt" - "io" "log/slog" - "os" - "path/filepath" - - "github.com/pocketbase/dbx" - "github.com/pocketbase/pocketbase/core" - _ "github.com/pocketbase/pocketbase/migrations" // PocketBase's system migrations; without them its bootstrap has no tables - "github.com/pocketbase/pocketbase/tools/filesystem" + "sync" + "time" "github.com/dariuszjaniak/warp/server/internal/storage" - "github.com/dariuszjaniak/warp/server/internal/storage/sqlite" - "github.com/dariuszjaniak/warp/server/internal/storage/sqlstore" ) func init() { storage.Register(storage.DriverPocketBase, Open) } -// mediaPrefix is where media content lives in PocketBase's filesystem, -// apart from the files of PocketBase's own collections. -const mediaPrefix = "warp_media" - // Database is the PocketBase implementation of storage.Database. type Database struct { - *sqlstore.Store - app *core.BaseApp - blobs *blobStore + *base + log *slog.Logger + s3 storage.S3Config + + settings *Settings + users *Users + rooms *Rooms + sync *Sync + fed *Federation + media *Media + keys *Keys + push *Push + admin *Admin + apps *AppServices + delayed *DelayedEvents + openID *OpenIDTokens + blobs *blobStore } -// Open bootstraps a PocketBase app in the directory named by -// cfg.ConnectionString — PocketBase's pb_data — and opens the Warp store on -// its database. -func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Database, error) { - dir := cfg.ConnectionString - if dir == "" { - return nil, errors.New("pocketbase: connection_string must name the data directory") - } - if err := ctx.Err(); err != nil { - return nil, err - } - - app := core.NewBaseApp(core.BaseAppConfig{ - DataDir: dir, - DataMaxOpenConns: cfg.MaxOpenConns, - DataMaxIdleConns: cfg.MaxIdleConns, - }) - if err := app.Bootstrap(); err != nil { - _ = app.ResetBootstrapState() - return nil, fmt.Errorf("pocketbase: bootstrap %s: %w", dir, err) - } - if cfg.S3.Enabled { - // Applied in memory on every start rather than saved: the Warp - // config is the authority, and PocketBase's own settings would - // otherwise be a second place for the bucket to be set, and to - // disagree. - app.Settings().S3 = core.S3Config{ - Enabled: true, - Bucket: cfg.S3.Bucket, - Region: cfg.S3.Region, - Endpoint: cfg.S3.Endpoint, - AccessKey: cfg.S3.AccessKey, - Secret: cfg.S3.Secret, - ForcePathStyle: cfg.S3.ForcePathStyle, - } - } - - // The writer pool: one connection, the way the sqlite driver runs, and - // the way PocketBase itself writes. Warp's store reads in the same - // transactions it writes in, so it is given the one pool for both. - writer, ok := app.NonconcurrentDB().(*dbx.DB) - if !ok { - _ = app.ResetBootstrapState() - return nil, errors.New("pocketbase: the app's database is not a *dbx.DB") - } - - opts := sqlite.StoreOptions(writer.DB(), log) - opts.DriverName = storage.DriverPocketBase - store, err := sqlstore.New(opts) - if err != nil { - _ = app.ResetBootstrapState() - return nil, err - } - return &Database{ - Store: store, - app: app, - blobs: &blobStore{app: app, tmpDir: filepath.Join(app.DataDir(), ".warp_tmp")}, - }, nil -} - -// App is the PocketBase app, for code that wants PocketBase's own -// features: its collections, its auth, its realtime. -func (d *Database) App() core.App { return d.app } - -// Blobs implements storage.BlobProvider: media content goes into -// PocketBase's filesystem. -func (d *Database) Blobs() storage.BlobStore { return d.blobs } - -// Close implements storage.Database. The app owns the connection pools. -func (d *Database) Close() error { return d.app.ResetBootstrapState() } - -// blobStore is storage.BlobStore over PocketBase's filesystem: the local -// pb_data/storage directory, or the S3 bucket the settings name. +// Open connects to the PocketBase server at cfg.ConnectionString and signs +// in as the configured superuser. // -// The filesystem can only be written a whole file at a time, and a blob's -// name is its hash, which is not known until the last byte has been read. -// So an upload is spooled to a temporary file first, hashed on the way, and -// handed over once it is complete — which is also what keeps a half-written -// object from ever appearing under a hash. -type blobStore struct { - app *core.BaseApp - tmpDir string -} - -func blobKey(sum string) (string, bool) { - if len(sum) != sha256.Size*2 { - return "", false - } - for _, c := range sum { - if (c < '0' || c > '9') && (c < 'a' || c > 'f') { - return "", false - } - } - return mediaPrefix + "/" + sum[0:2] + "/" + sum[2:4] + "/" + sum, true -} - -// fs opens the filesystem for one operation. It is cheap for both kinds, -// and an instance per operation is what lets each carry its own context. -func (b *blobStore) fs(ctx context.Context) (*filesystem.System, error) { - fsys, err := b.app.NewFilesystem() - if err != nil { - return nil, fmt.Errorf("open the media filesystem: %w", err) - } - fsys.SetContext(ctx) - return fsys, nil -} - -// Put implements storage.BlobStore. -func (b *blobStore) Put(ctx context.Context, r io.Reader, limit int64) (storage.BlobInfo, error) { - if err := os.MkdirAll(b.tmpDir, 0o750); err != nil { - return storage.BlobInfo{}, fmt.Errorf("create the media spool directory: %w", err) - } - tmp, err := os.CreateTemp(b.tmpDir, "upload-*") - if err != nil { - return storage.BlobInfo{}, fmt.Errorf("create a temporary file: %w", err) - } - defer func() { - _ = tmp.Close() - _ = os.Remove(tmp.Name()) - }() - - hash := sha256.New() - n, err := io.Copy(io.MultiWriter(tmp, hash), &ctxReader{ctx: ctx, r: io.LimitReader(r, limit+1)}) - if err != nil { - return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) - } - if n > limit { - return storage.BlobInfo{}, storage.ErrBlobTooLarge - } - if err := tmp.Close(); err != nil { - return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) - } - - info := storage.BlobInfo{SHA256: hex.EncodeToString(hash.Sum(nil)), Size: n} - key, _ := blobKey(info.SHA256) - - fsys, err := b.fs(ctx) - if err != nil { - return storage.BlobInfo{}, err - } - defer func() { _ = fsys.Close() }() - - if exists, err := fsys.Exists(key); err == nil && exists { - // The same content is already stored. - return info, nil - } - file, err := filesystem.NewFileFromPath(tmp.Name()) - if err != nil { - return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) - } - if err := fsys.UploadFile(file, key); err != nil { - return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) - } - return info, nil -} - -// Open implements storage.BlobStore. -func (b *blobStore) Open(ctx context.Context, sum string) (io.ReadSeekCloser, error) { - key, ok := blobKey(sum) - if !ok { - return nil, storage.ErrNoSuchBlob - } - fsys, err := b.fs(ctx) +// It configures the server for Warp before anything else — batch requests +// on, and the S3 bucket when one is configured — so a server that cannot +// be reached or refuses the account fails here, at startup. +func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Database, error) { + c, err := newClient(cfg.ConnectionString, cfg.SuperuserEmail, cfg.SuperuserPassword) if err != nil { return nil, err } - reader, err := fsys.GetReader(key) - if err != nil { - _ = fsys.Close() - if errors.Is(err, filesystem.ErrNotFound) { - return nil, storage.ErrNoSuchBlob - } - return nil, fmt.Errorf("open media content: %w", err) + lock := &sync.RWMutex{} + db := sql.OpenDB(&sqlConnector{c: c, lock: lock}) + // The connections hold nothing, so the pool only bounds how many reads + // are in flight at once. + maxConns := cfg.MaxOpenConns + if maxConns <= 0 { + maxConns = 16 } - return &blobReader{ReadSeekCloser: reader, fsys: fsys}, nil + db.SetMaxOpenConns(maxConns) + + b := &base{c: c, db: db, lock: lock} + d := &Database{ + base: b, + log: log, + s3: cfg.S3, + settings: &Settings{base: b}, + users: &Users{base: b}, + rooms: &Rooms{base: b}, + sync: &Sync{base: b}, + fed: &Federation{base: b}, + media: &Media{base: b}, + keys: &Keys{base: b}, + push: &Push{base: b}, + admin: &Admin{base: b}, + apps: &AppServices{base: b}, + delayed: &DelayedEvents{base: b}, + openID: &OpenIDTokens{base: b}, + blobs: &blobStore{base: b}, + } + if err := d.configure(ctx); err != nil { + _ = db.Close() + return nil, err + } + return d, nil } -// Delete implements storage.BlobStore. -func (b *blobStore) Delete(ctx context.Context, sum string) error { - key, ok := blobKey(sum) - if !ok { - return nil +// configure applies the PocketBase settings Warp depends on. +// +// The S3 settings are written from Warp's configuration on every start +// rather than left to the PocketBase admin UI: Warp's config is the +// authority, and a second place for the bucket to be set would be a second +// place for it to disagree. A deployment without S3 in Warp's config leaves +// PocketBase's own setting as it is. +func (d *Database) configure(ctx context.Context) error { + settings := map[string]any{ + "batch": map[string]any{ + "enabled": true, + "maxRequests": maxBatch, + "timeout": 300, + "maxBodySize": 0, + }, } - fsys, err := b.fs(ctx) - if err != nil { - return err + if d.s3.Enabled { + settings["s3"] = map[string]any{ + "enabled": true, + "bucket": d.s3.Bucket, + "region": d.s3.Region, + "endpoint": d.s3.Endpoint, + "accessKey": d.s3.AccessKey, + "secret": d.s3.Secret, + "forcePathStyle": d.s3.ForcePathStyle, + } } - defer func() { _ = fsys.Close() }() - if err := fsys.Delete(key); err != nil && !errors.Is(err, filesystem.ErrNotFound) { - return fmt.Errorf("delete media content: %w", err) + if err := d.c.patchSettings(ctx, settings); err != nil { + return fmt.Errorf("pocketbase: configure the server: %w", err) } return nil } -// blobReader closes the filesystem with the reader it opened. -type blobReader struct { - io.ReadSeekCloser - fsys *filesystem.System -} +// DB exposes the read-only SQL pool, for tests and diagnostics: queries run +// on the PocketBase server through its SQL endpoint, and anything else is +// refused. +func (d *Database) DB() *sql.DB { return d.db } -func (r *blobReader) Close() error { - return errors.Join(r.ReadSeekCloser.Close(), r.fsys.Close()) -} +// Driver implements storage.Database. +func (d *Database) Driver() string { return storage.DriverPocketBase } -// ctxReader stops a copy when its context ends. -type ctxReader struct { - ctx context.Context - r io.Reader -} +// Ping implements storage.Database. +func (d *Database) Ping(ctx context.Context) error { return d.db.PingContext(ctx) } -func (c *ctxReader) Read(p []byte) (int, error) { - if err := c.ctx.Err(); err != nil { - return 0, err +// Close implements storage.Database. +func (d *Database) Close() error { return d.db.Close() } + +// Settings implements storage.Database. +func (d *Database) Settings() storage.SettingsStore { return d.settings } + +// Users implements storage.Database. +func (d *Database) Users() storage.UserStore { return d.users } + +// Rooms implements storage.Database. +func (d *Database) Rooms() storage.RoomStore { return d.rooms } + +// Sync implements storage.Database. +func (d *Database) Sync() storage.SyncStore { return d.sync } + +// Federation implements storage.Database. +func (d *Database) Federation() storage.FederationStore { return d.fed } + +// Media implements storage.Database. +func (d *Database) Media() storage.MediaStore { return d.media } + +// Keys implements storage.Database. +func (d *Database) Keys() storage.KeyStore { return d.keys } + +// Push implements storage.Database. +func (d *Database) Push() storage.PushStore { return d.push } + +// Admin implements storage.Database. +func (d *Database) Admin() storage.AdminStore { return d.admin } + +// AppServices implements storage.Database. +func (d *Database) AppServices() storage.AppServiceStore { return d.apps } + +// DelayedEvents implements storage.Database. +func (d *Database) DelayedEvents() storage.DelayedEventStore { return d.delayed } + +// OpenIDTokens implements storage.Database. +func (d *Database) OpenIDTokens() storage.OpenIDTokenStore { return d.openID } + +// Blobs implements storage.BlobProvider: media content goes into +// PocketBase's storage. +func (d *Database) Blobs() storage.BlobStore { return d.blobs } + +// schemaMetaKey is the warp_meta row holding the schema version applied. +const schemaMetaKey = "schema" + +// Migrate implements storage.Database. +// +// There are no numbered migrations: the schema is declared in schema.go +// and imported as it is, which PocketBase applies as a diff — a new +// collection is created, a new field added, an unchanged one left alone. +// The version stamped afterwards is a hash of the declaration, so any +// change to it is a pending migration. +func (d *Database) Migrate(ctx context.Context) error { + pending, err := d.PendingMigrations(ctx) + if err != nil { + return err } - return c.r.Read(p) + if len(pending) == 0 { + d.log.Debug("database schema is up to date", "version", schemaVersion) + return nil + } + + collections := make([]map[string]any, len(schema)) + for i := range schema { + collections[i] = schema[i].collectionJSON() + } + if err := d.c.importCollections(ctx, collections); err != nil { + return fmt.Errorf("pocketbase: import the schema: %w", err) + } + + err = d.write(ctx, func(t *tx) error { + for table, names := range sequenceRows { + for _, name := range names { + found, err := t.exists(table, `name = ?`, name) + if err != nil { + return err + } + if !found { + t.put(table, rec{"name": name, "value": int64(0)}) + } + } + } + // Arguments a crashed query left behind. + if _, err := t.delWhere(paramsCollection, `1 = 1`); err != nil { + return err + } + t.put("warp_meta", rec{"meta_key": schemaMetaKey, "meta_value": schemaVersion}) + return nil + }) + if err != nil { + return fmt.Errorf("pocketbase: initialise the schema: %w", err) + } + d.log.Info("applied the database schema", "version", schemaVersion) + return nil +} + +// PendingMigrations implements storage.Database. +func (d *Database) PendingMigrations(ctx context.Context) ([]string, error) { + pending := []string{"pocketbase_schema_" + schemaVersion} + + var n int + if err := d.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'warp_meta'`).Scan(&n); err != nil { + return nil, fmt.Errorf("pocketbase: read the schema version: %w", err) + } + if n == 0 { + return pending, nil + } + var version string + err := d.db.QueryRowContext(ctx, + `SELECT meta_value FROM warp_meta WHERE meta_key = ?`, schemaMetaKey).Scan(&version) + switch { + case errors.Is(err, sql.ErrNoRows): + return pending, nil + case err != nil: + return nil, fmt.Errorf("pocketbase: read the schema version: %w", err) + case version != schemaVersion: + return pending, nil + } + return nil, nil +} + +// Settings is the server_settings store. +type Settings struct{ *base } + +// Get implements storage.SettingsStore. +func (s *Settings) Get(ctx context.Context, key string) (string, bool, error) { + var value string + err := s.db.QueryRowContext(ctx, + `SELECT setting_value FROM server_settings WHERE setting_key = ?`, key).Scan(&value) + switch { + case errors.Is(err, sql.ErrNoRows): + return "", false, nil + case err != nil: + return "", false, fmt.Errorf("get setting %q: %w", key, err) + } + return value, true, nil +} + +// GetAll implements storage.SettingsStore. +func (s *Settings) GetAll(ctx context.Context) (map[string]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT setting_key, setting_value FROM server_settings`) + if err != nil { + return nil, fmt.Errorf("list settings: %w", err) + } + defer func() { _ = rows.Close() }() + + out := map[string]string{} + for rows.Next() { + var k, v string + if err := rows.Scan(&k, &v); err != nil { + return nil, fmt.Errorf("scan setting: %w", err) + } + out[k] = v + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list settings: %w", err) + } + return out, nil +} + +// Set implements storage.SettingsStore. +func (s *Settings) Set(ctx context.Context, key, value string) error { + err := s.write(ctx, func(t *tx) error { + t.put("server_settings", rec{ + "setting_key": key, + "setting_value": value, + "updated_at": time.Now().UTC().Format(time.RFC3339Nano), + }) + return nil + }) + if err != nil { + return fmt.Errorf("set setting %q: %w", key, err) + } + return nil +} + +// Delete implements storage.SettingsStore. +func (s *Settings) Delete(ctx context.Context, key string) error { + err := s.write(ctx, func(t *tx) error { + _, err := t.delKey("server_settings", key) + return err + }) + if err != nil { + return fmt.Errorf("delete setting %q: %w", key, err) + } + return nil } var ( - _ storage.Database = (*Database)(nil) - _ storage.BlobProvider = (*Database)(nil) - _ storage.BlobStore = (*blobStore)(nil) + _ storage.Database = (*Database)(nil) + _ storage.BlobProvider = (*Database)(nil) + _ storage.SettingsStore = (*Settings)(nil) ) diff --git a/server/internal/storage/pocketbase/push.go b/server/internal/storage/pocketbase/push.go new file mode 100644 index 0000000..fdc6fe9 --- /dev/null +++ b/server/internal/storage/pocketbase/push.go @@ -0,0 +1,328 @@ +package pocketbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Push is the PocketBase implementation of storage.PushStore. +type Push struct{ *base } + +const pusherColumns = `localpart, app_id, pushkey, device_id, kind, app_display_name, + device_display_name, profile_tag, lang, data_json, pushkey_ts_ms` + +// Pushers implements storage.PushStore. +func (s *Push) Pushers(ctx context.Context, localpart string) ([]storage.Pusher, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+pusherColumns+` FROM push_pushers + WHERE localpart = ? ORDER BY app_id, pushkey`, localpart) + if err != nil { + return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.Pusher + for rows.Next() { + var p storage.Pusher + var data string + if err := rows.Scan(&p.Localpart, &p.AppID, &p.PushKey, &p.DeviceID, &p.Kind, &p.AppDisplayName, + &p.DeviceDisplayName, &p.ProfileTag, &p.Lang, &data, &p.PushKeyTSMS); err != nil { + return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err) + } + p.Data = []byte(data) + out = append(out, p) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err) + } + return out, nil +} + +// SetPusher implements storage.PushStore. +func (s *Push) SetPusher(ctx context.Context, p storage.Pusher, appendOthers bool) error { + return s.write(ctx, func(t *tx) error { + if !appendOthers { + if _, err := t.delWhere("push_pushers", `app_id = ? AND pushkey = ? AND localpart <> ?`, + p.AppID, p.PushKey, p.Localpart); err != nil { + return fmt.Errorf("remove other users' pushers for the key: %w", err) + } + } + t.put("push_pushers", rec{ + "localpart": p.Localpart, "app_id": p.AppID, "pushkey": p.PushKey, "device_id": p.DeviceID, + "kind": p.Kind, "app_display_name": p.AppDisplayName, "device_display_name": p.DeviceDisplayName, + "profile_tag": p.ProfileTag, "lang": p.Lang, "data_json": string(p.Data), "pushkey_ts_ms": p.PushKeyTSMS, + }) + return nil + }) +} + +// DeletePusher implements storage.PushStore. +func (s *Push) DeletePusher(ctx context.Context, localpart, appID, pushKey string) error { + err := s.write(ctx, func(t *tx) error { + _, err := t.delKey("push_pushers", localpart, appID, pushKey) + return err + }) + if err != nil { + return fmt.Errorf("delete a pusher of %q: %w", localpart, err) + } + return nil +} + +// DeletePushersForDevices implements storage.PushStore. +func (s *Push) DeletePushersForDevices(ctx context.Context, localpart string, deviceIDs []string) error { + where, args := `localpart = ?`, []any{localpart} + if len(deviceIDs) > 0 { + where += ` AND device_id IN ` + inList + args = append(args, jsonList(deviceIDs)) + } + err := s.write(ctx, func(t *tx) error { + _, err := t.delWhere("push_pushers", where, args...) + return err + }) + if err != nil { + return fmt.Errorf("delete the pushers of %q's devices: %w", localpart, err) + } + return nil +} + +const notificationColumns = `notification_id, user_id, room_id, event_id, event_pos, thread_id, ts_ms, highlight, actions, is_read` + +// roomCounts is one room's counts for one user, as a write leaves them. +type roomCounts struct{ notifications, highlights int64 } + +// AddNotifications implements storage.PushStore. Every notification takes a +// position of its own, which is both its ID and the stamp on its room's +// counts. +func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotification) (int64, error) { + if len(ns) == 0 { + return s.currentSequence(ctx, pushSequences, seqNotification) + } + var last int64 + err := s.write(ctx, func(t *tx) error { + counts := map[[2]string]roomCounts{} + for _, n := range ns { + pos, err := t.next(pushSequences, seqNotification) + if err != nil { + return err + } + t.put("push_notifications", rec{ + "notification_id": pos, "user_id": n.UserID, "room_id": n.RoomID, "event_id": n.EventID, + "event_pos": n.EventPos, "thread_id": n.ThreadID, "ts_ms": n.TimestampMS, + "highlight": n.Highlight, "actions": string(n.Actions), "is_read": false, + }) + + key := [2]string{n.UserID, n.RoomID} + c, ok := counts[key] + if !ok { + err := t.queryRow(`SELECT notification_count, highlight_count FROM push_room_counts + WHERE user_id = ? AND room_id = ?`, n.UserID, n.RoomID).Scan(&c.notifications, &c.highlights) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("count a notification for %q: %w", n.UserID, err) + } + } + c.notifications++ + if n.Highlight { + c.highlights++ + } + counts[key] = c + t.put("push_room_counts", rec{ + "user_id": n.UserID, "room_id": n.RoomID, + "notification_count": c.notifications, "highlight_count": c.highlights, "stream_pos": pos, + }) + last = pos + } + return nil + }) + return last, err +} + +// MarkNotificationsRead implements storage.PushStore. +// +// The counts are recounted from the rows that stay unread rather than +// decremented, so whatever got them out of step is put right by the next +// receipt. The room's counts stay the room's total across every thread. +func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID, thread string, upTo int64) (int64, bool, error) { + var ( + pos int64 + moved bool + ) + err := s.write(ctx, func(t *tx) error { + marked := `event_pos <= ?` + markedArgs := []any{upTo} + if thread != storage.AllThreads { + marked += ` AND thread_id = ?` + markedArgs = append(markedArgs, thread) + } + n, err := t.setWhere("push_notifications", rec{"is_read": true}, + `user_id = ? AND room_id = ? AND is_read = ? AND `+marked, + append([]any{userID, roomID, false}, markedArgs...)...) + if err != nil { + return fmt.Errorf("mark %q's notifications read: %w", userID, err) + } + if n == 0 { + pos, err = t.current(pushSequences, seqNotification) + return err + } + + var notifications, highlights int64 + if err := t.queryRow(`SELECT COUNT(*), CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT) + FROM push_notifications WHERE user_id = ? AND room_id = ? AND is_read = ? AND NOT (`+marked+`)`, + append([]any{userID, roomID, false}, markedArgs...)...). + Scan(¬ifications, &highlights); err != nil { + return fmt.Errorf("count %q's unread notifications: %w", userID, err) + } + if pos, err = t.next(pushSequences, seqNotification); err != nil { + return err + } + t.put("push_room_counts", rec{ + "user_id": userID, "room_id": roomID, + "notification_count": notifications, "highlight_count": highlights, "stream_pos": pos, + }) + moved = true + return nil + }) + return pos, moved, err +} + +// NotificationCounts implements storage.PushStore. +func (s *Push) NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]storage.NotificationCounts, error) { + out := make(map[string]storage.NotificationCounts, len(roomIDs)) + if len(roomIDs) == 0 { + return out, nil + } + for _, id := range roomIDs { + out[id] = storage.NotificationCounts{} + } + + rows, err := s.db.QueryContext(ctx, `SELECT room_id, notification_count, highlight_count FROM push_room_counts + WHERE user_id = ? AND room_id IN `+inList, userID, jsonList(roomIDs)) + if err != nil { + return nil, fmt.Errorf("read %q's notification counts: %w", userID, err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var roomID string + var c storage.NotificationCounts + if err := rows.Scan(&roomID, &c.Notifications, &c.Highlights); err != nil { + return nil, fmt.Errorf("read %q's notification counts: %w", userID, err) + } + out[roomID] = c + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read %q's notification counts: %w", userID, err) + } + return out, nil +} + +// ThreadNotificationCounts implements storage.PushStore. These are counted +// from the unread rows rather than kept like the room totals. +func (s *Push) ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error) { + out := make(map[string]map[string]storage.NotificationCounts, len(roomIDs)) + if len(roomIDs) == 0 { + return out, nil + } + + rows, err := s.db.QueryContext(ctx, `SELECT room_id, thread_id, COUNT(*), + CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT) + FROM push_notifications + WHERE user_id = ? AND is_read = ? AND room_id IN `+inList+` + GROUP BY room_id, thread_id`, userID, false, jsonList(roomIDs)) + if err != nil { + return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var roomID, thread string + var notifications, highlights int64 + if err := rows.Scan(&roomID, &thread, ¬ifications, &highlights); err != nil { + return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err) + } + if out[roomID] == nil { + out[roomID] = map[string]storage.NotificationCounts{} + } + out[roomID][thread] = storage.NotificationCounts{ + Notifications: int(notifications), + Highlights: int(highlights), + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err) + } + return out, nil +} + +// NotificationCountChanges implements storage.PushStore. +func (s *Push) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) { + out, err := s.queryStrings(ctx, `SELECT room_id FROM push_room_counts + WHERE user_id = ? AND stream_pos > ? AND stream_pos <= ? ORDER BY room_id`, userID, from, to) + if err != nil { + return nil, fmt.Errorf("read %q's notification count changes: %w", userID, err) + } + return out, nil +} + +// UnreadNotifications implements storage.PushStore. +func (s *Push) UnreadNotifications(ctx context.Context, userID string) (int, error) { + var n int64 + if err := s.db.QueryRowContext(ctx, `SELECT CAST(COALESCE(SUM(notification_count), 0) AS BIGINT) + FROM push_room_counts WHERE user_id = ?`, userID).Scan(&n); err != nil { + return 0, fmt.Errorf("count %q's unread notifications: %w", userID, err) + } + return int(n), nil +} + +// Notifications implements storage.PushStore. +func (s *Push) Notifications(ctx context.Context, userID string, before int64, limit int, onlyHighlight bool) ([]storage.Notification, error) { + query := `SELECT ` + notificationColumns + ` FROM push_notifications WHERE user_id = ?` + args := []any{userID} + if before > 0 { + query += ` AND notification_id < ?` + args = append(args, before) + } + if onlyHighlight { + query += ` AND highlight = ?` + args = append(args, true) + } + query += ` ORDER BY notification_id DESC LIMIT ?` + args = append(args, limit) + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read %q's notifications: %w", userID, err) + } + defer func() { _ = rows.Close() }() + var out []storage.Notification + for rows.Next() { + var n storage.Notification + var actions string + if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.ThreadID, &n.TimestampMS, + &n.Highlight, &actions, &n.Read); err != nil { + return nil, fmt.Errorf("read %q's notifications: %w", userID, err) + } + n.Actions = []byte(actions) + out = append(out, n) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read %q's notifications: %w", userID, err) + } + return out, nil +} + +// PruneNotifications implements storage.PushStore. +func (s *Push) PruneNotifications(ctx context.Context, beforeMS int64) (int64, error) { + var n int + err := s.write(ctx, func(t *tx) error { + var err error + n, err = t.delWhere("push_notifications", `is_read = ? AND ts_ms < ?`, true, beforeMS) + return err + }) + if err != nil { + return 0, fmt.Errorf("prune read notifications: %w", err) + } + return int64(n), nil +} + +var _ storage.PushStore = (*Push)(nil) diff --git a/server/internal/storage/pocketbase/relations.go b/server/internal/storage/pocketbase/relations.go new file mode 100644 index 0000000..cd1a42e --- /dev/null +++ b/server/internal/storage/pocketbase/relations.go @@ -0,0 +1,369 @@ +package pocketbase + +import ( + "context" + "fmt" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The relation and search half of the room store. +// +// Relations are the SQL store's queries. Search has no full-text engine to +// lean on — PocketBase does not expose FTS5 — so roomserver_event_search +// holds each event's searchable text as its words, lower-cased and padded +// with spaces, and a search matches every word of the term as the start of +// a word: what the SQLite driver's prefix query matches, short of folding +// diacritics. Every match ranks the same, so a ranked search comes back +// newest first. + +// searchText is how an event's text is stored for searching: its words, +// lower-cased, each with a space before it and the whole with one after. +func searchText(text string) string { + words := storage.SearchWords(text) + if len(words) == 0 { + return "" + } + return " " + strings.Join(words, " ") + " " +} + +// indexEvent adds an event that has just joined a timeline to the search +// index, if it has anything to find it by. +func (w *roomWrite) indexEvent(e storage.StoredEvent, ordering int64) { + body := searchText(e.SearchText) + if e.SearchKey == "" || body == "" { + return + } + w.put("roomserver_event_search", rec{ + "stream_ordering": ordering, + "room_id": e.RoomID, + "event_id": e.EventID, + "search_key": e.SearchKey, + "sender": e.Sender, + "event_type": e.Type, + "body": body, + }) +} + +// Relations implements storage.RoomStore. +func (s *Rooms) Relations(ctx context.Context, req storage.RelationsRequest) ([]storage.StoredEvent, error) { + if req.Limit <= 0 || len(req.ParentIDs) == 0 { + return nil, nil + } + + conditions := []string{ + `e.room_id = ?`, + `e.relates_to_id IN ` + inList, + `NOT e.outlier`, + } + args := []any{req.RoomID, jsonList(req.ParentIDs)} + if req.RelType != "" { + conditions = append(conditions, `e.rel_type = ?`) + args = append(args, req.RelType) + } else { + conditions = append(conditions, `e.rel_type <> ''`) + } + if req.EventType != "" { + conditions = append(conditions, `e.event_type = ?`) + args = append(args, req.EventType) + } + if req.Low != 0 { + conditions = append(conditions, `e.stream_ordering > ?`) + args = append(args, req.Low) + } + if req.High != 0 { + conditions = append(conditions, `e.stream_ordering <= ?`) + args = append(args, req.High) + } + order := `ASC` + if req.Direction == storage.DirectionBackward { + order = `DESC` + } + args = append(args, req.Limit) + + rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_events e + WHERE `+strings.Join(conditions, " AND ")+` + ORDER BY e.stream_ordering `+order+` + LIMIT ?`, args...) + if err != nil { + return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err) + } + return events, nil +} + +// ChildEventIDs implements storage.RoomStore. +func (s *Rooms) ChildEventIDs(ctx context.Context, roomID string, parentIDs []string, relType, eventType string) ([]string, error) { + if len(parentIDs) == 0 { + return nil, nil + } + query := `SELECT event_id FROM roomserver_events + WHERE room_id = ? AND relates_to_id IN ` + inList + ` + AND rel_type <> '' AND NOT outlier` + args := []any{roomID, jsonList(parentIDs)} + if relType != "" { + query += ` AND rel_type = ?` + args = append(args, relType) + } + if eventType != "" { + query += ` AND event_type = ?` + args = append(args, eventType) + } + query += ` ORDER BY stream_ordering` + + ids, err := s.queryStrings(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the children of %v in %q: %w", parentIDs, roomID, err) + } + return ids, nil +} + +// RelationAggregates implements storage.RoomStore. +func (s *Rooms) RelationAggregates(ctx context.Context, roomID string, parentIDs []string, high int64) ([]storage.RelationAggregate, error) { + if len(parentIDs) == 0 { + return nil, nil + } + inner := `SELECT relates_to_id, rel_type, sender, COUNT(*) AS n, MAX(stream_ordering) AS latest + FROM roomserver_events + WHERE room_id = ? AND relates_to_id IN ` + inList + ` + AND rel_type IN (?, ?, ?) AND NOT outlier` + args := []any{roomID, jsonList(parentIDs), storage.RelThread, storage.RelReplace, storage.RelReference} + if high != 0 { + inner += ` AND stream_ordering <= ?` + args = append(args, high) + } + inner += ` GROUP BY relates_to_id, rel_type, sender` + + rows, err := s.db.QueryContext(ctx, `SELECT g.relates_to_id, g.rel_type, g.sender, g.n, g.latest, e.event_id + FROM (`+inner+`) g + JOIN roomserver_events e ON e.stream_ordering = g.latest`, args...) + if err != nil { + return nil, fmt.Errorf("summarise the relations of %d events in %q: %w", len(parentIDs), roomID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.RelationAggregate + for rows.Next() { + var a storage.RelationAggregate + if err := rows.Scan(&a.ParentID, &a.RelType, &a.Sender, &a.Count, &a.LatestOrdering, &a.LatestEventID); err != nil { + return nil, fmt.Errorf("summarise the relations in %q: %w", roomID, err) + } + out = append(out, a) + } + return out, rows.Err() +} + +// RelationEventIDs implements storage.RoomStore. +func (s *Rooms) RelationEventIDs(ctx context.Context, roomID string, parentIDs []string, relType string, high int64, limit int) (map[string][]string, error) { + if len(parentIDs) == 0 || limit <= 0 { + return nil, nil + } + query := `SELECT relates_to_id, event_id FROM roomserver_events + WHERE room_id = ? AND relates_to_id IN ` + inList + ` + AND rel_type = ? AND NOT outlier` + args := []any{roomID, jsonList(parentIDs), relType} + if high != 0 { + query += ` AND stream_ordering <= ?` + args = append(args, high) + } + query += ` ORDER BY stream_ordering LIMIT ?` + args = append(args, limit*len(parentIDs)) + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err) + } + defer func() { _ = rows.Close() }() + + out := map[string][]string{} + for rows.Next() { + var parent, id string + if err := rows.Scan(&parent, &id); err != nil { + return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err) + } + if len(out[parent]) < limit { + out[parent] = append(out[parent], id) + } + } + return out, rows.Err() +} + +// Threads implements storage.RoomStore. +func (s *Rooms) Threads(ctx context.Context, req storage.ThreadsRequest) ([]storage.Thread, error) { + if req.Limit <= 0 { + return nil, nil + } + inner := `SELECT relates_to_id AS root, MAX(stream_ordering) AS latest + FROM roomserver_events + WHERE room_id = ? AND rel_type = ? AND NOT outlier` + args := []any{req.RoomID, storage.RelThread} + if req.High != 0 { + inner += ` AND stream_ordering <= ?` + args = append(args, req.High) + } + inner += ` GROUP BY relates_to_id` + + conditions := []string{`r.room_id = ?`, `NOT r.outlier`} + args = append(args, req.RoomID) + if req.Before != 0 { + conditions = append(conditions, `g.latest < ?`) + args = append(args, req.Before) + } + if req.ParticipatedBy != "" { + conditions = append(conditions, `(r.sender = ? OR EXISTS ( + SELECT 1 FROM roomserver_events p + WHERE p.room_id = r.room_id AND p.relates_to_id = g.root AND p.rel_type = ? AND p.sender = ?))`) + args = append(args, req.ParticipatedBy, storage.RelThread, req.ParticipatedBy) + } + args = append(args, req.Limit) + + rows, err := s.db.QueryContext(ctx, `SELECT g.root, g.latest + FROM (`+inner+`) g + JOIN roomserver_events r ON r.event_id = g.root + WHERE `+strings.Join(conditions, " AND ")+` + ORDER BY g.latest DESC + LIMIT ?`, args...) + if err != nil { + return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.Thread + for rows.Next() { + var t storage.Thread + if err := rows.Scan(&t.RootID, &t.LatestOrdering); err != nil { + return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err) + } + out = append(out, t) + } + return out, rows.Err() +} + +// searchKeys are the fields the index holds. +var searchKeys = []string{"content.body", "content.name", "content.topic"} + +// Search implements storage.RoomStore. +func (s *Rooms) Search(ctx context.Context, req storage.SearchRequest) ([]storage.SearchResult, int, error) { + if req.Limit <= 0 || len(req.Rooms) == 0 { + return nil, 0, nil + } + words := storage.SearchWords(req.Term) + if len(words) == 0 { + return nil, 0, nil + } + + var ( + conditions []string + args []any + ) + for _, w := range words { + conditions = append(conditions, `body LIKE ? ESCAPE '\'`) + args = append(args, `% `+escapeLike(w)+`%`) + } + + // One bounded clause per room: what a user may search of a room is a + // window of its stream, and the windows differ from room to room. + rooms := make([]string, 0, len(req.Rooms)) + for _, r := range req.Rooms { + clause := `room_id = ?` + args = append(args, r.RoomID) + if r.Low != 0 { + clause += ` AND stream_ordering > ?` + args = append(args, r.Low) + } + if r.High != 0 { + clause += ` AND stream_ordering <= ?` + args = append(args, r.High) + } + rooms = append(rooms, `(`+clause+`)`) + } + conditions = append(conditions, `(`+strings.Join(rooms, ` OR `)+`)`) + + keys := req.Keys + if len(keys) == 0 { + keys = searchKeys + } + conditions = append(conditions, `search_key IN `+inList) + args = append(args, jsonList(keys)) + + if clause, filterArgs := filterConditionsOn(`event_type`, `sender`, req.Filter); clause != "" { + conditions = append(conditions, clause) + args = append(args, filterArgs...) + } + where := strings.Join(conditions, " AND ") + + var total int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM roomserver_event_search WHERE `+where, args...). + Scan(&total); err != nil { + return nil, 0, fmt.Errorf("count search results: %w", err) + } + + pageWhere := where + pageArgs := append([]any{}, args...) + if req.Order == storage.SearchByRecent && req.Before != 0 { + pageWhere += ` AND stream_ordering < ?` + pageArgs = append(pageArgs, req.Before) + } + offset := req.Offset + if req.Order == storage.SearchByRecent { + offset = 0 + } + pageArgs = append(pageArgs, req.Limit, offset) + + positions, err := s.queryInts(ctx, `SELECT stream_ordering FROM roomserver_event_search + WHERE `+pageWhere+` + ORDER BY stream_ordering DESC + LIMIT ? OFFSET ?`, pageArgs...) + if err != nil { + return nil, 0, fmt.Errorf("search: %w", err) + } + if len(positions) == 0 { + return nil, total, nil + } + + eventRows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_events e WHERE e.stream_ordering IN `+inList, jsonInts(positions)) + if err != nil { + return nil, 0, fmt.Errorf("read search results: %w", err) + } + events, err := scanEvents(eventRows) + if err != nil { + return nil, 0, fmt.Errorf("read search results: %w", err) + } + byPos := make(map[int64]storage.StoredEvent, len(events)) + for _, e := range events { + byPos[e.StreamOrdering] = e + } + + out := make([]storage.SearchResult, 0, len(positions)) + for _, pos := range positions { + if e, ok := byPos[pos]; ok { + out = append(out, storage.SearchResult{Event: e, Rank: 1}) + } + } + return out, total, nil +} + +// queryInts runs a query returning one integer column. +func (b *base) queryInts(ctx context.Context, query string, args ...any) ([]int64, error) { + rows, err := b.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var out []int64 + for rows.Next() { + var v int64 + if err := rows.Scan(&v); err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} diff --git a/server/internal/storage/pocketbase/remotekeys.go b/server/internal/storage/pocketbase/remotekeys.go new file mode 100644 index 0000000..76e1011 --- /dev/null +++ b/server/internal/storage/pocketbase/remotekeys.go @@ -0,0 +1,247 @@ +package pocketbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + "slices" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The cache of other servers' users' keys, as the SQL store keeps it. + +var remoteKeyTables = []string{ + "keyserver_remote_users", "keyserver_remote_device_keys", "keyserver_remote_cross_signing_keys", +} + +// remoteMarker reads a cached user's marker row. +func remoteMarker(row rowQuerier, userID string) (streamID int64, stale, ok bool, err error) { + err = row(`SELECT stream_id, stale FROM keyserver_remote_users WHERE user_id = ?`, userID).Scan(&streamID, &stale) + switch { + case errors.Is(err, sql.ErrNoRows): + return 0, false, false, nil + case err != nil: + return 0, false, false, fmt.Errorf("read the cached keys of %q: %w", userID, err) + } + return streamID, stale, true, nil +} + +// RemoteKeys implements storage.KeyStore. The marker and the keys are read +// under one read lock, so that a copy halfway through its replacement is +// never served. +func (s *Keys) RemoteKeys(ctx context.Context, userID string) (storage.RemoteKeys, bool, error) { + out := storage.RemoteKeys{UserID: userID, CrossSigning: map[string][]byte{}} + var found bool + err := s.read(ctx, func(ctx context.Context) error { + var err error + out.StreamID, out.Stale, found, err = remoteMarker(s.poolRow(ctx), userID) + if err != nil || !found { + return err + } + if out.Devices, err = s.remoteDevices(ctx, userID); err != nil { + return err + } + out.CrossSigning, err = s.remoteCrossSigning(ctx, userID) + return err + }) + if err != nil || !found { + return storage.RemoteKeys{}, false, err + } + return out, true, nil +} + +func (s *Keys) remoteDevices(ctx context.Context, userID string) ([]storage.RemoteDevice, error) { + rows, err := s.db.QueryContext(ctx, `SELECT device_id, display_name, key_json FROM keyserver_remote_device_keys + WHERE user_id = ? ORDER BY device_id`, userID) + if err != nil { + return nil, fmt.Errorf("read the cached devices of %q: %w", userID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.RemoteDevice + for rows.Next() { + var d storage.RemoteDevice + var keyJSON string + if err := rows.Scan(&d.DeviceID, &d.DisplayName, &keyJSON); err != nil { + return nil, fmt.Errorf("read the cached devices of %q: %w", userID, err) + } + d.JSON = []byte(keyJSON) + out = append(out, d) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read the cached devices of %q: %w", userID, err) + } + return out, nil +} + +func (s *Keys) remoteCrossSigning(ctx context.Context, userID string) (map[string][]byte, error) { + rows, err := s.db.QueryContext(ctx, `SELECT purpose, key_json FROM keyserver_remote_cross_signing_keys + WHERE user_id = ?`, userID) + if err != nil { + return nil, fmt.Errorf("read the cached cross-signing keys of %q: %w", userID, err) + } + defer func() { _ = rows.Close() }() + + out := map[string][]byte{} + for rows.Next() { + var purpose, keyJSON string + if err := rows.Scan(&purpose, &keyJSON); err != nil { + return nil, fmt.Errorf("read the cached cross-signing keys of %q: %w", userID, err) + } + out[purpose] = []byte(keyJSON) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read the cached cross-signing keys of %q: %w", userID, err) + } + return out, nil +} + +// SetRemoteKeys implements storage.KeyStore. A snapshot older than what is +// held is not stored: it was fetched while an update was on its way in. +func (s *Keys) SetRemoteKeys(ctx context.Context, keys storage.RemoteKeys) (bool, error) { + stored := false + err := s.write(ctx, func(t *tx) error { + held, _, ok, err := remoteMarker(t.queryRow, keys.UserID) + if err != nil { + return err + } + if ok && held > keys.StreamID { + return nil + } + + if err := deleteRemote(t, keys.UserID); err != nil { + return err + } + t.put("keyserver_remote_users", rec{"user_id": keys.UserID, "stream_id": keys.StreamID, "stale": false}) + for _, d := range keys.Devices { + t.put("keyserver_remote_device_keys", rec{ + "user_id": keys.UserID, "device_id": d.DeviceID, + "display_name": d.DisplayName, "key_json": string(d.JSON), + }) + } + putRemoteCrossSigning(t, keys.UserID, keys.CrossSigning) + stored = true + return nil + }) + return stored, err +} + +// UpdateRemoteDevice implements storage.KeyStore. +func (s *Keys) UpdateRemoteDevice(ctx context.Context, u storage.RemoteDeviceUpdate) (storage.RemoteUpdateResult, error) { + var result storage.RemoteUpdateResult + err := s.write(ctx, func(t *tx) error { + held, stale, ok, err := remoteMarker(t.queryRow, u.UserID) + if err != nil { + return err + } + + // A stale copy keeps the highest position it has heard of, so that + // a snapshot fetched before this update cannot be stored over it. + markStale := func(r storage.RemoteUpdateResult) error { + result = r + t.put("keyserver_remote_users", rec{"user_id": u.UserID, "stream_id": max(held, u.StreamID), "stale": true}) + return nil + } + + switch { + case !ok: + return markStale(storage.RemoteUpdateNotCached) + case stale: + return markStale(storage.RemoteUpdateStale) + case u.StreamID <= held: + result = storage.RemoteUpdateDuplicate + return nil + case !followsOn(held, u.PrevIDs): + return markStale(storage.RemoteUpdateStale) + case !u.Deleted && u.JSON == nil: + return markStale(storage.RemoteUpdateStale) + } + + if u.Deleted { + if _, err := t.delKey("keyserver_remote_device_keys", u.UserID, u.DeviceID); err != nil { + return fmt.Errorf("apply an update to the cached keys of %s/%s: %w", u.UserID, u.DeviceID, err) + } + } else { + t.put("keyserver_remote_device_keys", rec{ + "user_id": u.UserID, "device_id": u.DeviceID, + "display_name": u.DisplayName, "key_json": string(u.JSON), + }) + } + t.put("keyserver_remote_users", rec{"user_id": u.UserID, "stream_id": u.StreamID, "stale": false}) + result = storage.RemoteUpdateApplied + return nil + }) + return result, err +} + +// followsOn reports whether an update with the given prev_id comes right +// after the position held. +func followsOn(held int64, prevIDs []int64) bool { + if len(prevIDs) == 0 { + return held == 0 + } + return slices.Contains(prevIDs, held) +} + +// SetRemoteCrossSigningKeys implements storage.KeyStore. +func (s *Keys) SetRemoteCrossSigningKeys(ctx context.Context, userID string, keys map[string][]byte) (bool, error) { + stored := false + err := s.write(ctx, func(t *tx) error { + _, stale, ok, err := remoteMarker(t.queryRow, userID) + if err != nil || !ok || stale { + return err + } + if _, err := t.delWhere("keyserver_remote_cross_signing_keys", `user_id = ?`, userID); err != nil { + return fmt.Errorf("replace the cached cross-signing keys of %q: %w", userID, err) + } + putRemoteCrossSigning(t, userID, keys) + stored = true + return nil + }) + return stored, err +} + +func putRemoteCrossSigning(t *tx, userID string, keys map[string][]byte) { + for _, purpose := range sortedKeys(keys) { + if keys[purpose] == nil { + continue + } + t.put("keyserver_remote_cross_signing_keys", rec{"user_id": userID, "purpose": purpose, "key_json": string(keys[purpose])}) + } +} + +// RemoteKeyUsers implements storage.KeyStore. +func (s *Keys) RemoteKeyUsers(ctx context.Context) ([]string, error) { + out, err := s.queryStrings(ctx, `SELECT user_id FROM keyserver_remote_users ORDER BY user_id`) + if err != nil { + return nil, fmt.Errorf("read the users with cached keys: %w", err) + } + return out, nil +} + +// DeleteRemoteKeys implements storage.KeyStore. +func (s *Keys) DeleteRemoteKeys(ctx context.Context, userIDs []string) error { + if len(userIDs) == 0 { + return nil + } + return s.write(ctx, func(t *tx) error { + for _, userID := range dedupe(userIDs) { + if err := deleteRemote(t, userID); err != nil { + return err + } + } + return nil + }) +} + +// deleteRemote drops every row cached for a user. +func deleteRemote(t *tx, userID string) error { + for _, table := range remoteKeyTables { + if _, err := t.delWhere(table, `user_id = ?`, userID); err != nil { + return fmt.Errorf("delete from %s for %q: %w", table, userID, err) + } + } + return nil +} diff --git a/server/internal/storage/pocketbase/rooms.go b/server/internal/storage/pocketbase/rooms.go new file mode 100644 index 0000000..7cc49a0 --- /dev/null +++ b/server/internal/storage/pocketbase/rooms.go @@ -0,0 +1,1149 @@ +package pocketbase + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/base64" + "errors" + "fmt" + "slices" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Rooms is the PocketBase implementation of storage.RoomStore. +// +// The reads are the SQL store's, with the event's nullable state_key read +// through has_state_key. A write is the SQL store's transaction turned into +// one batch, with the one difference that a batch cannot read what it has +// not yet written: whatever the SQL store would read back inside its +// transaction — the member count after a state change, a snapshot or an +// event written earlier in the same call — is tracked here, in the call. +type Rooms struct{ *base } + +const ( + defaultRoomPageSize = 100 + maxRoomPageSize = 1000 +) + +// memberEventType is the one event type the store has to recognise. +const memberEventType = "m.room.member" + +// CreateRoom implements storage.RoomStore. +func (s *Rooms) CreateRoom(ctx context.Context, room storage.NewRoom) error { + err := s.write(ctx, func(t *tx) error { + taken, err := t.exists("roomserver_rooms", `room_id = ?`, room.ID) + if err != nil { + return err + } + if taken { + return storage.ErrRoomExists + } + t.put("roomserver_rooms", rec{ + "room_id": room.ID, + "room_version": room.Version, + "creator": room.Creator, + "is_published": room.Published, + "created_at_ms": room.CreatedAtMS, + "name": "", + "topic": "", + "avatar_url": "", + "canonical_alias": "", + "join_rule": "", + "history_visibility": "", + "guest_access": "", + "room_type": "", + "joined_members": int64(0), + }) + return nil + }) + if err != nil { + return fmt.Errorf("create room %q: %w", room.ID, err) + } + return nil +} + +// Room implements storage.RoomStore. +func (s *Rooms) Room(ctx context.Context, roomID string) (storage.Room, error) { + var room storage.Room + err := s.db.QueryRowContext(ctx, `SELECT room_id, room_version, creator, is_published, created_at_ms + FROM roomserver_rooms WHERE room_id = ?`, roomID).Scan( + &room.ID, &room.Version, &room.Creator, &room.Published, &room.CreatedAtMS) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.Room{}, fmt.Errorf("room %q: %w", roomID, storage.ErrNoSuchRoom) + case err != nil: + return storage.Room{}, fmt.Errorf("read room %q: %w", roomID, err) + } + return room, nil +} + +// roomWrite is one write call's view of the room tables: what it has +// written so far, which a later step of the same call has to see. +type roomWrite struct { + *tx + + // held are the events the call knows about, read or written: whether + // each is an outlier, and whether it has been redacted. + held map[string]heldEvent + + // snapshots are the state snapshots by room and hash. + snapshots map[string]int64 +} + +type heldEvent struct { + outlier bool + redacted bool +} + +func newRoomWrite(t *tx) *roomWrite { + return &roomWrite{tx: t, held: map[string]heldEvent{}, snapshots: map[string]int64{}} +} + +// loadHeld reads which of these events are already stored, in one query. +func (w *roomWrite) loadHeld(eventIDs []string) error { + var unknown []string + for _, id := range eventIDs { + if _, ok := w.held[id]; !ok { + unknown = append(unknown, id) + } + } + if len(unknown) == 0 { + return nil + } + rows, err := w.query(`SELECT event_id, outlier, redacted_by FROM roomserver_events + WHERE event_id IN `+inList, jsonList(unknown)) + if err != nil { + return fmt.Errorf("look up events: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var ( + id string + outlier bool + redactedBy string + ) + if err := rows.Scan(&id, &outlier, &redactedBy); err != nil { + return fmt.Errorf("look up events: %w", err) + } + w.held[id] = heldEvent{outlier: outlier, redacted: redactedBy != ""} + } + return rows.Err() +} + +// Write implements storage.RoomStore. +func (s *Rooms) Write(ctx context.Context, rw storage.RoomWrite) (int64, error) { + var ordering int64 + err := s.write(ctx, func(t *tx) error { + w := newRoomWrite(t) + var err error + if ordering, err = t.next(roomSequences, seqStreamOrdering); err != nil { + return err + } + + var snapshotID int64 + if !rw.Outlier { + if snapshotID, err = w.stateSnapshot(rw.Event.RoomID, rw.StateBefore); err != nil { + return err + } + } + + if err := w.loadHeld([]string{rw.Event.EventID}); err != nil { + return err + } + w.insertEvent(rw.Event, ordering, snapshotID, rw.Outlier) + if rw.Event.Type == memberEventType && rw.Event.StateKey != nil { + if err := w.replaceInviteState(rw.Event.RoomID, *rw.Event.StateKey, rw.InviteState); err != nil { + return err + } + } + if rw.Redacts != "" { + if err := w.applyRedaction(rw.Event.RoomID, rw.Redacts, rw.Event.EventID, rw.RedactedJSON); err != nil { + return err + } + } + if err := w.applyStateDelta(rw); err != nil { + return err + } + if err := w.replaceExtremities(rw.Event.RoomID, rw.Extremities); err != nil { + return err + } + if rw.Profile != nil { + p := rw.Profile + if _, err := t.setWhere("roomserver_rooms", rec{ + "name": p.Name, "topic": p.Topic, "avatar_url": p.AvatarURL, "canonical_alias": p.CanonicalAlias, + "join_rule": p.JoinRule, "history_visibility": p.HistoryVisibility, + "guest_access": p.GuestAccess, "room_type": p.RoomType, + }, `room_id = ?`, rw.Event.RoomID); err != nil { + return fmt.Errorf("update room profile: %w", err) + } + } + // The member count is recomputed whenever current state moved, + // because that is the only thing that can change it. + if len(rw.StateDelta) > 0 || rw.StateDeltaIsWholeState { + if err := w.recountMembers(rw); err != nil { + return err + } + } + if rw.Transaction != nil { + ref := *rw.Transaction + found, err := t.exists("roomserver_transactions", `localpart = ? AND device_id = ? AND txn_id = ?`, + ref.Localpart, ref.DeviceID, ref.TxnID) + if err != nil { + return fmt.Errorf("record transaction %q: %w", ref.TxnID, err) + } + if !found { + t.put("roomserver_transactions", rec{ + "localpart": ref.Localpart, "device_id": ref.DeviceID, "txn_id": ref.TxnID, + "event_id": rw.Event.EventID, "room_id": rw.Event.RoomID, "sent_at_ms": rw.Event.OriginServerTS, + }) + } + } + return nil + }) + if err != nil { + return 0, fmt.Errorf("write event %s: %w", rw.Event.EventID, err) + } + return ordering, nil +} + +// stateSnapshot stores a state set, re-using the row of an identical one. +// An empty state set is snapshot 0 and stores nothing. +func (w *roomWrite) stateSnapshot(roomID string, entries []storage.StateEntry) (int64, error) { + if len(entries) == 0 { + return 0, nil + } + hash := stateHash(entries) + if id, ok := w.snapshots[roomID+"\x00"+hash]; ok { + return id, nil + } + + var id int64 + err := w.queryRow(`SELECT snapshot_id FROM roomserver_state_snapshots + WHERE room_id = ? AND state_hash = ?`, roomID, hash).Scan(&id) + switch { + case err == nil: + w.snapshots[roomID+"\x00"+hash] = id + return id, nil + case !errors.Is(err, sql.ErrNoRows): + return 0, fmt.Errorf("look up state snapshot: %w", err) + } + + if id, err = w.next(roomSequences, seqStateSnapshot); err != nil { + return 0, err + } + w.put("roomserver_state_snapshots", rec{"snapshot_id": id, "room_id": roomID, "state_hash": hash}) + for _, e := range entries { + w.put("roomserver_state_entries", rec{ + "snapshot_id": id, "event_type": e.Type, "state_key": e.StateKey, "event_id": e.EventID, + }) + } + w.snapshots[roomID+"\x00"+hash] = id + return id, nil +} + +// stateHash identifies a state set by its contents, as the SQL store does. +func stateHash(entries []storage.StateEntry) string { + sorted := slices.Clone(entries) + slices.SortFunc(sorted, func(a, b storage.StateEntry) int { + if c := strings.Compare(a.Type, b.Type); c != 0 { + return c + } + return strings.Compare(a.StateKey, b.StateKey) + }) + + h := sha256.New() + for _, e := range sorted { + for _, field := range [...]string{e.Type, e.StateKey, e.EventID} { + _, _ = fmt.Fprintf(h, "%d:%s", len(field), field) + } + } + return base64.RawURLEncoding.EncodeToString(h.Sum(nil)) +} + +// eventRow is the roomserver_events row of an event. +func eventRow(e storage.StoredEvent, ordering, snapshotID int64, outlier bool) rec { + stateKey, hasStateKey := "", false + if e.StateKey != nil { + stateKey, hasStateKey = *e.StateKey, true + } + return rec{ + "event_id": e.EventID, + "room_id": e.RoomID, + "stream_ordering": ordering, + "event_type": e.Type, + "state_key": stateKey, + "has_state_key": hasStateKey, + "sender": e.Sender, + "depth": e.Depth, + "origin_server_ts": e.OriginServerTS, + "event_json": string(e.JSON), + "state_before_id": snapshotID, + "redacted_by": "", + "outlier": outlier, + "relates_to_id": e.RelatesTo, + "rel_type": e.RelType, + } +} + +// insertEvent writes the event row; loadHeld must have been called for it. +// +// An event already held is not written twice, except that an outlier +// arriving as an ordinary event is promoted in place: it gets the new stream +// position and state snapshot, and joins the timeline, with its stored JSON +// left as it was. +func (w *roomWrite) insertEvent(e storage.StoredEvent, ordering, snapshotID int64, outlier bool) { + if held, ok := w.held[e.EventID]; ok { + if held.outlier && !outlier { + w.promote(e, ordering, snapshotID, held.redacted) + } + return + } + w.put("roomserver_events", eventRow(e, ordering, snapshotID, outlier)) + w.held[e.EventID] = heldEvent{outlier: outlier} + if !outlier { + w.indexEvent(e, ordering) + } +} + +// promote moves an outlier into the timeline. It joins the search index +// too, unless the copy held was redacted, whose text is gone. +func (w *roomWrite) promote(e storage.StoredEvent, ordering, snapshotID int64, redacted bool) { + w.patch("roomserver_events", rowID("roomserver_events", e.EventID), rec{ + "stream_ordering": ordering, + "state_before_id": snapshotID, + "outlier": false, + }) + w.held[e.EventID] = heldEvent{outlier: false, redacted: redacted} + if !redacted { + w.indexEvent(e, ordering) + } +} + +// replaceInviteState clears a user's stored invite state and, for an invite +// that came with some, stores the new one. +func (w *roomWrite) replaceInviteState(roomID, userID string, state []byte) error { + if len(state) > 0 { + w.put("roomserver_invite_state", rec{"room_id": roomID, "user_id": userID, "state_json": string(state)}) + return nil + } + if _, err := w.delKey("roomserver_invite_state", roomID, userID); err != nil { + return fmt.Errorf("clear invite state: %w", err) + } + return nil +} + +// InviteState implements storage.RoomStore. +func (s *Rooms) InviteState(ctx context.Context, roomID, userID string) ([]byte, error) { + var state string + err := s.db.QueryRowContext(ctx, `SELECT state_json FROM roomserver_invite_state + WHERE room_id = ? AND user_id = ?`, roomID, userID).Scan(&state) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, nil + case err != nil: + return nil, fmt.Errorf("read the invite state of %q in %q: %w", userID, roomID, err) + } + return []byte(state), nil +} + +// WriteOutliers implements storage.RoomStore. +// +// Which of the events are already held is asked once for the whole batch, +// and the stream positions are taken as one block. +func (s *Rooms) WriteOutliers(ctx context.Context, roomID string, events []storage.StoredEvent) error { + if len(events) == 0 { + return nil + } + err := s.write(ctx, func(t *tx) error { + w := newRoomWrite(t) + top, err := t.advance(roomSequences, seqStreamOrdering, int64(len(events))) + if err != nil { + return err + } + ids := make([]string, len(events)) + for i, e := range events { + ids[i] = e.EventID + } + if err := w.loadHeld(ids); err != nil { + return err + } + first := top - int64(len(events)) + 1 + for i, e := range events { + if e.RoomID == "" { + e.RoomID = roomID + } + w.insertEvent(e, first+int64(i), 0, true) + } + return nil + }) + if err != nil { + return fmt.Errorf("write %d outliers to %q: %w", len(events), roomID, err) + } + return nil +} + +// WriteBackfill implements storage.RoomStore. +// +// The positions come from their own counter, advanced downwards by the +// number of events actually written, the batch taking the block oldest +// first. +func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []storage.BackfillEvent) error { + if len(events) == 0 { + return nil + } + err := s.write(ctx, func(t *tx) error { + w := newRoomWrite(t) + ids := make([]string, len(events)) + for i, b := range events { + ids[i] = b.Event.EventID + } + if err := w.loadHeld(ids); err != nil { + return err + } + + // Which of the batch is new, which is an outlier to promote, and + // which is already in the timeline and left alone. + var write []storage.BackfillEvent + seen := map[string]bool{} + for _, b := range events { + if seen[b.Event.EventID] { + continue + } + seen[b.Event.EventID] = true + if held, ok := w.held[b.Event.EventID]; !ok || held.outlier { + write = append(write, b) + } + } + if len(write) == 0 { + return nil + } + + bottom, err := t.advance(roomSequences, seqBackfillOrdering, -int64(len(write))) + if err != nil { + return err + } + for i, b := range write { + e := b.Event + if e.RoomID == "" { + e.RoomID = roomID + } + ordering := bottom + int64(i) + snapshotID, err := w.stateSnapshot(e.RoomID, b.StateBefore) + if err != nil { + return err + } + if held, ok := w.held[e.EventID]; ok { + w.promote(e, ordering, snapshotID, held.redacted) + continue + } + w.insertEvent(e, ordering, snapshotID, false) + } + return nil + }) + if err != nil { + return fmt.Errorf("backfill %d events into %q: %w", len(events), roomID, err) + } + return nil +} + +// applyRedaction replaces a stored event with its redacted form. A second +// redaction of the same event is a no-op. +func (w *roomWrite) applyRedaction(roomID, targetID, redactionID string, redacted []byte) error { + n, err := w.setWhere("roomserver_events", rec{ + "event_json": string(redacted), + "redacted_by": redactionID, + "relates_to_id": "", + "rel_type": "", + }, `room_id = ? AND event_id = ? AND redacted_by = ''`, roomID, targetID) + if err != nil { + return fmt.Errorf("apply redaction to %s: %w", targetID, err) + } + if n > 0 { + if held, ok := w.held[targetID]; ok { + held.redacted = true + w.held[targetID] = held + } + } + // What a redacted event said is gone, and so is its way of being found + // by what it said. + if _, err := w.delWhere("roomserver_event_search", `room_id = ? AND event_id = ?`, roomID, targetID); err != nil { + return fmt.Errorf("remove event %s from the search index: %w", targetID, err) + } + return nil +} + +// applyStateDelta moves the room's current state. A whole-state write +// clears the room's rows first, because tuples the resolution dropped have +// to go. +func (w *roomWrite) applyStateDelta(rw storage.RoomWrite) error { + if len(rw.StateDelta) == 0 && !rw.StateDeltaIsWholeState { + return nil + } + if rw.StateDeltaIsWholeState { + if _, err := w.delWhere("roomserver_current_state", `room_id = ?`, rw.Event.RoomID); err != nil { + return fmt.Errorf("clear current state: %w", err) + } + } + for _, e := range rw.StateDelta { + w.put("roomserver_current_state", rec{ + "room_id": rw.Event.RoomID, "event_type": e.Type, "state_key": e.StateKey, + "event_id": e.EventID, "membership": e.Membership, + }) + } + return nil +} + +// recountMembers sets the room's joined member count from its current state +// as this write leaves it: the stored joins, with the delta applied. +func (w *roomWrite) recountMembers(rw storage.RoomWrite) error { + joined := map[string]bool{} + if !rw.StateDeltaIsWholeState { + rows, err := w.query(`SELECT state_key FROM roomserver_current_state + WHERE room_id = ? AND event_type = 'm.room.member' AND membership = 'join'`, rw.Event.RoomID) + if err != nil { + return fmt.Errorf("recount joined members: %w", err) + } + for rows.Next() { + var user string + if err := rows.Scan(&user); err != nil { + _ = rows.Close() + return fmt.Errorf("recount joined members: %w", err) + } + joined[user] = true + } + _ = rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("recount joined members: %w", err) + } + } + for _, e := range rw.StateDelta { + if e.Type != memberEventType { + continue + } + joined[e.StateKey] = e.Membership == "join" + } + n := 0 + for _, j := range joined { + if j { + n++ + } + } + if _, err := w.setWhere("roomserver_rooms", rec{"joined_members": int64(n)}, `room_id = ?`, rw.Event.RoomID); err != nil { + return fmt.Errorf("recount joined members: %w", err) + } + return nil +} + +func (w *roomWrite) replaceExtremities(roomID string, extremities []string) error { + if len(extremities) == 0 { + return nil + } + if _, err := w.delWhere("roomserver_extremities", `room_id = ?`, roomID); err != nil { + return fmt.Errorf("clear forward extremities: %w", err) + } + for _, id := range extremities { + w.put("roomserver_extremities", rec{"room_id": roomID, "event_id": id}) + } + return nil +} + +// eventColumns is the select list every event read shares, in the order +// scanEvent expects. state_key reads as NULL for a message event, as the SQL +// drivers store it. +const eventColumns = ` + e.room_id, e.event_id, e.event_type, CASE WHEN e.has_state_key THEN e.state_key END, e.sender, + e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier, + e.relates_to_id, e.rel_type` + +// scanEvent reads one row of eventColumns. +func scanEvent(scan func(...any) error) (storage.StoredEvent, error) { + var ( + e storage.StoredEvent + stateKey sql.NullString + body string + ) + err := scan(&e.RoomID, &e.EventID, &e.Type, &stateKey, &e.Sender, + &e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier, + &e.RelatesTo, &e.RelType) + if err != nil { + return storage.StoredEvent{}, err + } + if stateKey.Valid { + e.StateKey = &stateKey.String + } + e.JSON = []byte(body) + return e, nil +} + +// scanEvents drains a result set of eventColumns rows. +func scanEvents(rows *sql.Rows) ([]storage.StoredEvent, error) { + defer func() { _ = rows.Close() }() + + var out []storage.StoredEvent + for rows.Next() { + e, err := scanEvent(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, e) + } + return out, rows.Err() +} + +// Event implements storage.RoomStore. +func (s *Rooms) Event(ctx context.Context, roomID, eventID string) (storage.StoredEvent, error) { + e, err := scanEvent(s.db.QueryRowContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_events e WHERE e.room_id = ? AND e.event_id = ?`, roomID, eventID).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.StoredEvent{}, fmt.Errorf("event %q: %w", eventID, storage.ErrNoSuchEvent) + case err != nil: + return storage.StoredEvent{}, fmt.Errorf("read event %q: %w", eventID, err) + } + return e, nil +} + +// EventsByID implements storage.RoomStore. +func (s *Rooms) EventsByID(ctx context.Context, roomID string, eventIDs []string) ([]storage.StoredEvent, error) { + if len(eventIDs) == 0 { + return nil, nil + } + rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_events e + WHERE e.room_id = ? AND e.event_id IN `+inList, roomID, jsonList(eventIDs)) + if err != nil { + return nil, fmt.Errorf("read events by ID: %w", err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read events by ID: %w", err) + } + return events, nil +} + +// CurrentState implements storage.RoomStore. +func (s *Rooms) CurrentState(ctx context.Context, roomID string) ([]storage.StateEntry, error) { + rows, err := s.db.QueryContext(ctx, `SELECT event_type, state_key, event_id, membership + FROM roomserver_current_state WHERE room_id = ? + ORDER BY event_type, state_key`, roomID) + if err != nil { + return nil, fmt.Errorf("read current state of %q: %w", roomID, err) + } + entries, err := scanStateEntries(rows) + if err != nil { + return nil, fmt.Errorf("read current state of %q: %w", roomID, err) + } + return entries, nil +} + +func scanStateEntries(rows *sql.Rows) ([]storage.StateEntry, error) { + defer func() { _ = rows.Close() }() + + var out []storage.StateEntry + for rows.Next() { + var e storage.StateEntry + if err := rows.Scan(&e.Type, &e.StateKey, &e.EventID, &e.Membership); err != nil { + return nil, err + } + out = append(out, e) + } + return out, rows.Err() +} + +// CurrentStateEvents implements storage.RoomStore. +func (s *Rooms) CurrentStateEvents(ctx context.Context, roomID string) ([]storage.StoredEvent, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_current_state cs + JOIN roomserver_events e ON e.event_id = cs.event_id + WHERE cs.room_id = ? + ORDER BY cs.event_type, cs.state_key`, roomID) + if err != nil { + return nil, fmt.Errorf("read current state events of %q: %w", roomID, err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read current state events of %q: %w", roomID, err) + } + return events, nil +} + +// CurrentStateEvent implements storage.RoomStore. +func (s *Rooms) CurrentStateEvent(ctx context.Context, roomID, eventType, stateKey string) (storage.StoredEvent, error) { + e, err := scanEvent(s.db.QueryRowContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_current_state cs + JOIN roomserver_events e ON e.event_id = cs.event_id + WHERE cs.room_id = ? AND cs.event_type = ? AND cs.state_key = ?`, roomID, eventType, stateKey).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.StoredEvent{}, fmt.Errorf("state %s/%s of %q: %w", + eventType, stateKey, roomID, storage.ErrNoSuchEvent) + case err != nil: + return storage.StoredEvent{}, fmt.Errorf("read state %s/%s of %q: %w", + eventType, stateKey, roomID, err) + } + return e, nil +} + +// StateBefore implements storage.RoomStore. +func (s *Rooms) StateBefore(ctx context.Context, roomID, eventID string) ([]storage.StateEntry, error) { + rows, err := s.db.QueryContext(ctx, `SELECT se.event_type, se.state_key, se.event_id, '' + FROM roomserver_events e + JOIN roomserver_state_entries se ON se.snapshot_id = e.state_before_id + WHERE e.room_id = ? AND e.event_id = ? + ORDER BY se.event_type, se.state_key`, roomID, eventID) + if err != nil { + return nil, fmt.Errorf("read the state before %q: %w", eventID, err) + } + entries, err := scanStateEntries(rows) + if err != nil { + return nil, fmt.Errorf("read the state before %q: %w", eventID, err) + } + return entries, nil +} + +// Extremities implements storage.RoomStore. +func (s *Rooms) Extremities(ctx context.Context, roomID string) ([]storage.StoredEvent, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_extremities x + JOIN roomserver_events e ON e.event_id = x.event_id + WHERE x.room_id = ? + ORDER BY e.stream_ordering`, roomID) + if err != nil { + return nil, fmt.Errorf("read the forward extremities of %q: %w", roomID, err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read the forward extremities of %q: %w", roomID, err) + } + return events, nil +} + +// Members implements storage.RoomStore. +func (s *Rooms) Members(ctx context.Context, roomID string, memberships []string) ([]storage.StoredEvent, error) { + query := `SELECT ` + eventColumns + ` + FROM roomserver_current_state cs + JOIN roomserver_events e ON e.event_id = cs.event_id + WHERE cs.room_id = ? AND cs.event_type = 'm.room.member'` + args := []any{roomID} + if len(memberships) > 0 { + query += ` AND cs.membership IN ` + inList + args = append(args, jsonList(memberships)) + } + query += ` ORDER BY cs.state_key` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the members of %q: %w", roomID, err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read the members of %q: %w", roomID, err) + } + return events, nil +} + +// MembershipEntries implements storage.RoomStore. +func (s *Rooms) MembershipEntries(ctx context.Context, roomID string, memberships []string) ([]storage.MembershipEntry, error) { + query := ` + SELECT state_key, membership, event_id FROM roomserver_current_state + WHERE room_id = ? AND event_type = 'm.room.member'` + args := []any{roomID} + if len(memberships) > 0 { + query += ` AND membership IN ` + inList + args = append(args, jsonList(memberships)) + } + query += ` ORDER BY state_key` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the memberships of %q: %w", roomID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.MembershipEntry + for rows.Next() { + var m storage.MembershipEntry + if err := rows.Scan(&m.UserID, &m.Membership, &m.EventID); err != nil { + return nil, fmt.Errorf("read the memberships of %q: %w", roomID, err) + } + out = append(out, m) + } + return out, rows.Err() +} + +// Membership implements storage.RoomStore. +func (s *Rooms) Membership(ctx context.Context, roomID, userID string) (storage.MembershipEntry, error) { + var m storage.MembershipEntry + err := s.db.QueryRowContext(ctx, `SELECT state_key, membership, event_id FROM roomserver_current_state + WHERE room_id = ? AND event_type = 'm.room.member' AND state_key = ?`, roomID, userID). + Scan(&m.UserID, &m.Membership, &m.EventID) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.MembershipEntry{UserID: userID}, nil + case err != nil: + return storage.MembershipEntry{}, fmt.Errorf("read the membership of %q in %q: %w", userID, roomID, err) + } + return m, nil +} + +// RoomsForUser implements storage.RoomStore. +func (s *Rooms) RoomsForUser(ctx context.Context, userID string, memberships []string) ([]string, error) { + query := ` + SELECT room_id FROM roomserver_current_state + WHERE event_type = 'm.room.member' AND state_key = ?` + args := []any{userID} + if len(memberships) > 0 { + query += ` AND membership IN ` + inList + args = append(args, jsonList(memberships)) + } + query += ` ORDER BY room_id` + + out, err := s.queryStrings(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the rooms of %q: %w", userID, err) + } + return out, nil +} + +// TransactionEvent implements storage.RoomStore. +func (s *Rooms) TransactionEvent(ctx context.Context, ref storage.TransactionRef) (string, error) { + var eventID string + err := s.db.QueryRowContext(ctx, `SELECT event_id FROM roomserver_transactions + WHERE localpart = ? AND device_id = ? AND txn_id = ?`, + ref.Localpart, ref.DeviceID, ref.TxnID).Scan(&eventID) + switch { + case errors.Is(err, sql.ErrNoRows): + return "", fmt.Errorf("transaction %q: %w", ref.TxnID, storage.ErrNoSuchEvent) + case err != nil: + return "", fmt.Errorf("read transaction %q: %w", ref.TxnID, err) + } + return eventID, nil +} + +// CreateAlias implements storage.RoomStore. +func (s *Rooms) CreateAlias(ctx context.Context, alias storage.RoomAlias) error { + err := s.write(ctx, func(t *tx) error { + taken, err := t.exists("roomserver_aliases", `alias = ?`, alias.Alias) + if err != nil { + return err + } + if taken { + return storage.ErrAliasExists + } + t.put("roomserver_aliases", rec{ + "alias": alias.Alias, "room_id": alias.RoomID, + "creator": alias.Creator, "created_at_ms": alias.CreatedAtMS, + }) + return nil + }) + if err != nil { + return fmt.Errorf("create alias %q: %w", alias.Alias, err) + } + return nil +} + +// DeleteAlias implements storage.RoomStore. +func (s *Rooms) DeleteAlias(ctx context.Context, alias string) error { + err := s.write(ctx, func(t *tx) error { + found, err := t.delKey("roomserver_aliases", alias) + if err != nil { + return err + } + if !found { + return fmt.Errorf("%q: %w", alias, storage.ErrNoSuchAlias) + } + return nil + }) + if err != nil { + return fmt.Errorf("delete alias %q: %w", alias, err) + } + return nil +} + +// Alias implements storage.RoomStore. +func (s *Rooms) Alias(ctx context.Context, alias string) (storage.RoomAlias, error) { + var out storage.RoomAlias + err := s.db.QueryRowContext(ctx, `SELECT alias, room_id, creator, created_at_ms + FROM roomserver_aliases WHERE alias = ?`, alias). + Scan(&out.Alias, &out.RoomID, &out.Creator, &out.CreatedAtMS) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.RoomAlias{}, fmt.Errorf("alias %q: %w", alias, storage.ErrNoSuchAlias) + case err != nil: + return storage.RoomAlias{}, fmt.Errorf("resolve alias %q: %w", alias, err) + } + return out, nil +} + +// AliasesForRoom implements storage.RoomStore. +func (s *Rooms) AliasesForRoom(ctx context.Context, roomID string) ([]string, error) { + out, err := s.queryStrings(ctx, `SELECT alias FROM roomserver_aliases WHERE room_id = ? ORDER BY alias`, roomID) + if err != nil { + return nil, fmt.Errorf("read the aliases of %q: %w", roomID, err) + } + return out, nil +} + +// SetPublished implements storage.RoomStore. +func (s *Rooms) SetPublished(ctx context.Context, roomID string, published bool) error { + err := s.write(ctx, func(t *tx) error { + n, err := t.setWhere("roomserver_rooms", rec{"is_published": published}, `room_id = ?`, roomID) + if err != nil { + return err + } + if n == 0 { + return fmt.Errorf("%q: %w", roomID, storage.ErrNoSuchRoom) + } + return nil + }) + if err != nil { + return fmt.Errorf("set the published flag of %q: %w", roomID, err) + } + return nil +} + +// roomSummaryColumns is the select list a room listing shares. +const roomSummaryColumns = ` + room_id, room_version, creator, is_published, created_at_ms, + name, topic, avatar_url, canonical_alias, + join_rule, history_visibility, guest_access, room_type, joined_members` + +func scanRoomSummary(scan func(...any) error) (storage.RoomSummary, error) { + var r storage.RoomSummary + err := scan(&r.ID, &r.Version, &r.Creator, &r.Published, &r.CreatedAtMS, + &r.Name, &r.Topic, &r.AvatarURL, &r.CanonicalAlias, + &r.JoinRule, &r.HistoryVisibility, &r.GuestAccess, &r.RoomType, &r.JoinedMembers) + return r, err +} + +// PublishedRooms implements storage.RoomStore. +func (s *Rooms) PublishedRooms(ctx context.Context, filter storage.RoomFilter) ([]storage.RoomSummary, int, error) { + return s.listRooms(ctx, filter, true) +} + +// ListRooms implements storage.RoomStore. +func (s *Rooms) ListRooms(ctx context.Context, filter storage.RoomFilter) ([]storage.RoomSummary, int, error) { + return s.listRooms(ctx, filter, false) +} + +// listRooms is the body of both listings, ordered by descending member +// count and then by room ID. +func (s *Rooms) listRooms(ctx context.Context, filter storage.RoomFilter, publishedOnly bool) ([]storage.RoomSummary, int, error) { + where := ` WHERE 1 = 1` + var args []any + if publishedOnly { + where += ` AND is_published = ?` + args = append(args, true) + } + if search := strings.TrimSpace(filter.Search); search != "" { + pattern := "%" + strings.ToLower(search) + "%" + where += ` AND (LOWER(room_id) LIKE ? OR LOWER(name) LIKE ? + OR LOWER(topic) LIKE ? OR LOWER(canonical_alias) LIKE ?)` + args = append(args, pattern, pattern, pattern, pattern) + } + + var total int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM roomserver_rooms`+where, args...).Scan(&total); err != nil { + return nil, 0, fmt.Errorf("count rooms: %w", err) + } + + limit, offset := pageBounds(filter.Limit, filter.Offset, defaultRoomPageSize, maxRoomPageSize) + query := `SELECT ` + roomSummaryColumns + ` FROM roomserver_rooms` + where + + ` ORDER BY joined_members DESC, room_id LIMIT ? OFFSET ?` + + rows, err := s.db.QueryContext(ctx, query, append(args, limit, offset)...) + if err != nil { + return nil, 0, fmt.Errorf("list rooms: %w", err) + } + defer func() { _ = rows.Close() }() + + out := make([]storage.RoomSummary, 0, limit) + for rows.Next() { + r, err := scanRoomSummary(rows.Scan) + if err != nil { + return nil, 0, fmt.Errorf("list rooms: %w", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, 0, fmt.Errorf("list rooms: %w", err) + } + return out, total, nil +} + +// RoomSummary implements storage.RoomStore. +func (s *Rooms) RoomSummary(ctx context.Context, roomID string) (storage.RoomSummary, error) { + r, err := scanRoomSummary(s.db.QueryRowContext(ctx, `SELECT `+roomSummaryColumns+ + ` FROM roomserver_rooms WHERE room_id = ?`, roomID).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.RoomSummary{}, fmt.Errorf("room %q: %w", roomID, storage.ErrNoSuchRoom) + case err != nil: + return storage.RoomSummary{}, fmt.Errorf("read room %q: %w", roomID, err) + } + return r, nil +} + +// purgeTables are cleared of a room's rows, children before parents, the +// room row last. +var purgeTables = []struct{ table, where string }{ + {"roomserver_state_entries", `snapshot_id IN ( + SELECT snapshot_id FROM roomserver_state_snapshots WHERE room_id = ?)`}, + {"roomserver_state_snapshots", `room_id = ?`}, + {"roomserver_current_state", `room_id = ?`}, + {"roomserver_extremities", `room_id = ?`}, + {"roomserver_transactions", `room_id = ?`}, + {"roomserver_aliases", `room_id = ?`}, + {"roomserver_invite_state", `room_id = ?`}, + {"roomserver_event_search", `room_id = ?`}, + {"roomserver_events", `room_id = ?`}, + {"roomserver_rooms", `room_id = ?`}, +} + +// PurgeRoom implements storage.RoomStore. +func (s *Rooms) PurgeRoom(ctx context.Context, roomID string) error { + err := s.write(ctx, func(t *tx) error { + found, err := t.exists("roomserver_rooms", `room_id = ?`, roomID) + if err != nil { + return err + } + if !found { + return fmt.Errorf("%q: %w", roomID, storage.ErrNoSuchRoom) + } + for _, p := range purgeTables { + if _, err := t.delWhere(p.table, p.where, roomID); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("purge room %q: %w", roomID, err) + } + return nil +} + +// RoomStats implements storage.RoomStore. +func (s *Rooms) RoomStats(ctx context.Context) (storage.RoomStats, error) { + var stats storage.RoomStats + err := s.db.QueryRowContext(ctx, ` + SELECT + (SELECT COUNT(*) FROM roomserver_rooms), + (SELECT COUNT(*) FROM roomserver_rooms WHERE is_published = ?), + (SELECT COUNT(*) FROM roomserver_events)`, true). + Scan(&stats.Rooms, &stats.Published, &stats.Events) + if err != nil { + return storage.RoomStats{}, fmt.Errorf("read room statistics: %w", err) + } + return stats, nil +} + +// EventByID implements storage.RoomStore. +func (s *Rooms) EventByID(ctx context.Context, eventID string) (storage.StoredEvent, error) { + e, err := scanEvent(s.db.QueryRowContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_events e WHERE e.event_id = ?`, eventID).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.StoredEvent{}, fmt.Errorf("event %q: %w", eventID, storage.ErrNoSuchEvent) + case err != nil: + return storage.StoredEvent{}, fmt.Errorf("read event %q: %w", eventID, err) + } + return e, nil +} + +// HasJoinedMemberOn implements storage.RoomStore. A user ID's server is +// everything after its first colon, and a localpart cannot contain one. +func (s *Rooms) HasJoinedMemberOn(ctx context.Context, roomID, serverName string) (bool, error) { + var one int + err := s.db.QueryRowContext(ctx, `SELECT 1 FROM roomserver_current_state + WHERE room_id = ? AND event_type = 'm.room.member' AND membership = 'join' + AND state_key LIKE ? ESCAPE '\' + LIMIT 1`, roomID, `%:`+escapeLike(serverName)).Scan(&one) + switch { + case errors.Is(err, sql.ErrNoRows): + return false, nil + case err != nil: + return false, fmt.Errorf("look for members of %q in %q: %w", serverName, roomID, err) + } + return true, nil +} + +// UsersSharingRooms implements storage.RoomStore. +func (s *Rooms) UsersSharingRooms(ctx context.Context, userID string) ([]string, error) { + out, err := s.queryStrings(ctx, ` + SELECT DISTINCT other.state_key + FROM roomserver_current_state mine + JOIN roomserver_current_state other + ON other.room_id = mine.room_id + AND other.event_type = 'm.room.member' + AND other.membership = 'join' + WHERE mine.event_type = 'm.room.member' + AND mine.state_key = ? + AND mine.membership = 'join' + ORDER BY other.state_key`, userID) + if err != nil { + return nil, fmt.Errorf("read the users sharing a room with %q: %w", userID, err) + } + return out, nil +} + +// MemberChanges implements storage.RoomStore. +func (s *Rooms) MemberChanges(ctx context.Context, roomIDs []string, from, to int64) ([]storage.MemberChange, error) { + if len(roomIDs) == 0 { + return nil, nil + } + query := ` + SELECT e.room_id, e.state_key, e.stream_ordering, COALESCE(c.membership, '') + FROM roomserver_events e + LEFT JOIN roomserver_current_state c + ON c.room_id = e.room_id AND c.event_type = 'm.room.member' AND c.state_key = e.state_key + WHERE e.event_type = 'm.room.member' AND e.has_state_key AND NOT e.outlier + AND e.room_id IN ` + inList + ` + AND e.stream_ordering > ?` + args := []any{jsonList(roomIDs), from} + if to != 0 { + query += ` AND e.stream_ordering <= ?` + args = append(args, to) + } + query += ` ORDER BY e.stream_ordering` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read membership changes: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []storage.MemberChange + for rows.Next() { + var c storage.MemberChange + if err := rows.Scan(&c.RoomID, &c.UserID, &c.StreamOrdering, &c.Membership); err != nil { + return nil, fmt.Errorf("read membership changes: %w", err) + } + out = append(out, c) + } + return out, rows.Err() +} + +// queryStrings runs a query returning one string column. +func (b *base) queryStrings(ctx context.Context, query string, args ...any) ([]string, error) { + rows, err := b.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var out []string + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} + +var _ storage.RoomStore = (*Rooms)(nil) diff --git a/server/internal/storage/pocketbase/schema.go b/server/internal/storage/pocketbase/schema.go new file mode 100644 index 0000000..787e051 --- /dev/null +++ b/server/internal/storage/pocketbase/schema.go @@ -0,0 +1,449 @@ +package pocketbase + +import ( + "crypto/sha256" + "encoding/base32" + "encoding/hex" + "encoding/json" + "fmt" + "strconv" + "strings" +) + +// The schema is the SQL drivers' tables as PocketBase collections: the same +// names and columns, so that the reads can be the same SQL, with three +// differences PocketBase forces. +// +// - Every record has a text `id`, which is the primary key. Warp's own keys +// are kept as ordinary fields under a unique index, and a record's id is +// derived from them (see rowID): a row is always addressed by its key. +// - Nothing can be NULL. The two nullable state_key columns and +// admin_registration_tokens.uses_allowed get a has_* flag beside them. +// - `id` is taken, so push_notifications.id is notification_id and +// admin_audit_log.id is audit_id. +// +// Full-text search has no FTS5 here — it is a plain collection searched with +// LIKE — and media content lives in warp_media's file field, which is +// PocketBase's own storage: its pb_data/storage or an S3 bucket. + +type fieldKind int + +const ( + text fieldKind = iota + number + boolean + file +) + +type field struct { + name string + kind fieldKind +} + +// table is one collection. +type table struct { + name string + + // key is the primary key the SQL drivers declare; the record id is a + // hash of its values. Empty for a collection whose ids are chosen by + // the writer (warp_params). + key []string + + fields []field + + // indexes are column lists; a leading "unique:" makes one unique. + indexes []string +} + +func tbl(name string, key []string, fields []field, indexes ...string) table { + return table{name: name, key: key, fields: fields, indexes: indexes} +} + +func pk(cols ...string) []string { return cols } + +func fld(name string, kind fieldKind) field { return field{name: name, kind: kind} } + +func texts(names ...string) []field { + out := make([]field, len(names)) + for i, n := range names { + out[i] = field{name: n, kind: text} + } + return out +} + +func numbers(names ...string) []field { + out := make([]field, len(names)) + for i, n := range names { + out[i] = field{name: n, kind: number} + } + return out +} + +func fields(groups ...[]field) []field { + var out []field + for _, g := range groups { + out = append(out, g...) + } + return out +} + +func sequenceTable(name string) table { + return tbl(name, pk("name"), fields(texts("name"), numbers("value"))) +} + +// The sequence rows each table starts with, as the SQL migrations insert +// them. +var sequenceRows = map[string][]string{ + "roomserver_sequences": {"stream_ordering", "state_snapshot", "backfill_ordering"}, + "sync_sequences": {"receipt", "to_device", "account_data", "presence"}, + "federation_sequences": {"queue"}, + "keyserver_sequences": {"device_list"}, + "push_sequences": {"notification"}, + "admin_sequences": {"audit"}, +} + +var schema = []table{ + tbl("warp_meta", pk("meta_key"), texts("meta_key", "meta_value")), + {name: paramsCollection, fields: texts("value")}, + tbl("warp_media", pk("sha256"), fields(texts("sha256"), numbers("size"), []field{fld("content", file)})), + + tbl("server_settings", pk("setting_key"), texts("setting_key", "setting_value", "updated_at")), + + // userapi + tbl("user_accounts", pk("localpart"), fields( + texts("localpart", "password_hash", "display_name", "avatar_url"), + []field{fld("admin", boolean), fld("deactivated", boolean)}, + numbers("created_at_ms"))), + tbl("user_devices", pk("localpart", "device_id"), fields( + texts("localpart", "device_id", "display_name"), + numbers("created_at_ms", "last_seen_ms"), + texts("last_seen_ip", "user_agent"))), + tbl("user_access_tokens", pk("token_hash"), fields( + texts("token_hash", "localpart", "device_id"), numbers("created_at_ms", "expires_at_ms")), + "localpart, device_id"), + tbl("user_refresh_tokens", pk("token_hash"), fields( + texts("token_hash", "localpart", "device_id"), numbers("created_at_ms", "expires_at_ms")), + "localpart, device_id"), + tbl("user_account_data", pk("localpart", "room_id", "data_type"), fields( + texts("localpart", "room_id", "data_type", "content"), numbers("stream_pos")), + "localpart, stream_pos"), + + // roomserver + tbl("roomserver_rooms", pk("room_id"), fields( + texts("room_id", "room_version", "creator"), + []field{fld("is_published", boolean)}, + numbers("created_at_ms"), + texts("name", "topic", "avatar_url", "canonical_alias", + "join_rule", "history_visibility", "guest_access", "room_type"), + numbers("joined_members")), + "is_published, joined_members"), + sequenceTable("roomserver_sequences"), + tbl("roomserver_state_snapshots", pk("snapshot_id"), fields( + numbers("snapshot_id"), texts("room_id", "state_hash")), + "unique: room_id, state_hash"), + tbl("roomserver_state_entries", pk("snapshot_id", "event_type", "state_key"), fields( + numbers("snapshot_id"), texts("event_type", "state_key", "event_id"))), + tbl("roomserver_events", pk("event_id"), fields( + texts("event_id", "room_id"), + numbers("stream_ordering"), + texts("event_type", "state_key"), + []field{fld("has_state_key", boolean)}, + texts("sender"), + numbers("depth", "origin_server_ts"), + texts("event_json"), + numbers("state_before_id"), + texts("redacted_by"), + []field{fld("outlier", boolean)}, + texts("relates_to_id", "rel_type")), + "unique: stream_ordering", + "room_id, stream_ordering", + "room_id, relates_to_id, rel_type, stream_ordering", + "event_type, state_key"), + tbl("roomserver_extremities", pk("room_id", "event_id"), texts("room_id", "event_id")), + tbl("roomserver_current_state", pk("room_id", "event_type", "state_key"), + texts("room_id", "event_type", "state_key", "event_id", "membership"), + "event_type, state_key, membership"), + tbl("roomserver_transactions", pk("localpart", "device_id", "txn_id"), fields( + texts("localpart", "device_id", "txn_id", "event_id", "room_id"), numbers("sent_at_ms"))), + tbl("roomserver_aliases", pk("alias"), fields( + texts("alias", "room_id", "creator"), numbers("created_at_ms")), + "room_id"), + tbl("roomserver_invite_state", pk("room_id", "user_id"), texts("room_id", "user_id", "state_json")), + tbl("roomserver_event_search", pk("stream_ordering"), fields( + numbers("stream_ordering"), + texts("room_id", "event_id", "search_key", "sender", "event_type", "body")), + "room_id, stream_ordering"), + + // syncapi + sequenceTable("sync_sequences"), + tbl("sync_receipts", pk("room_id", "user_id", "receipt_type", "thread_id"), fields( + texts("room_id", "user_id", "receipt_type", "thread_id", "event_id"), numbers("ts_ms", "stream_pos")), + "stream_pos", "room_id, stream_pos"), + tbl("sync_to_device", pk("stream_pos"), fields( + numbers("stream_pos"), + texts("localpart", "device_id", "sender", "event_type", "content"), + numbers("sent_at_ms")), + "localpart, device_id, stream_pos"), + tbl("sync_presence", pk("user_id"), fields( + texts("user_id", "presence", "status_msg"), numbers("last_active_ms", "stream_pos")), + "stream_pos"), + tbl("sync_filters", pk("localpart", "filter_id"), fields( + texts("localpart", "filter_id", "filter_json"), numbers("created_at_ms"))), + + // federation + tbl("federation_server_keys", pk("server_name", "key_id"), fields( + texts("server_name", "key_id", "public_key"), + numbers("valid_until_ts", "expired_ts", "fetched_at_ms"))), + sequenceTable("federation_sequences"), + tbl("federation_queue", pk("queue_id"), fields( + numbers("queue_id"), + texts("destination", "kind", "room_id", "event_id", "payload"), + numbers("queued_at_ms")), + "destination, kind, queue_id"), + tbl("federation_destinations", pk("server_name"), fields( + texts("server_name"), + numbers("failures", "retry_at_ms", "last_success_ms", "last_failure_ms"), + texts("last_error"))), + tbl("federation_inbound_txns", pk("origin", "txn_id"), fields( + texts("origin", "txn_id", "response_json"), numbers("received_at_ms"))), + + // mediaapi + tbl("media_repository", pk("media_origin", "media_id"), fields( + texts("media_origin", "media_id", "content_type", "upload_name"), + numbers("file_size"), + texts("sha256", "user_id"), + numbers("created_at_ms"), + []field{fld("pending", boolean)}, + numbers("expires_at_ms")), + "pending, user_id, expires_at_ms", "user_id, media_origin", "sha256"), + tbl("media_thumbnails", pk("media_origin", "media_id", "width", "height", "method"), fields( + texts("media_origin", "media_id"), + numbers("width", "height"), + texts("method", "content_type"), + numbers("file_size"), + texts("sha256"), + numbers("created_at_ms")), + "sha256"), + tbl("media_quotas", pk("user_id"), fields(texts("user_id"), numbers("max_bytes"))), + + // keyserver + sequenceTable("keyserver_sequences"), + tbl("keyserver_device_keys", pk("user_id", "device_id"), texts("user_id", "device_id", "key_json")), + tbl("keyserver_one_time_keys", pk("user_id", "device_id", "key_id"), + texts("user_id", "device_id", "key_id", "algorithm", "key_json"), + "user_id, device_id, algorithm"), + tbl("keyserver_fallback_keys", pk("user_id", "device_id", "algorithm"), fields( + texts("user_id", "device_id", "algorithm", "key_id", "key_json"), + []field{fld("used", boolean)})), + tbl("keyserver_cross_signing_keys", pk("user_id", "purpose"), texts("user_id", "purpose", "key_json")), + tbl("keyserver_signatures", pk("target_user_id", "target_key_id", "signer_user_id", "signer_key_id"), + texts("target_user_id", "target_key_id", "signer_user_id", "signer_key_id", "signature")), + tbl("keyserver_device_lists", pk("user_id"), fields(texts("user_id"), numbers("stream_pos")), + "stream_pos"), + tbl("keyserver_backup_versions", pk("user_id", "version"), fields( + texts("user_id"), numbers("version"), texts("algorithm", "auth_data"), numbers("etag"), + []field{fld("deleted", boolean)})), + tbl("keyserver_backup_keys", pk("user_id", "version", "room_id", "session_id"), fields( + texts("user_id"), numbers("version"), texts("room_id", "session_id"), + numbers("first_message_index", "forwarded_count"), + []field{fld("is_verified", boolean)}, + texts("session_data"))), + tbl("keyserver_remote_users", pk("user_id"), fields( + texts("user_id"), numbers("stream_id"), []field{fld("stale", boolean)})), + tbl("keyserver_remote_device_keys", pk("user_id", "device_id"), + texts("user_id", "device_id", "display_name", "key_json")), + tbl("keyserver_remote_cross_signing_keys", pk("user_id", "purpose"), texts("user_id", "purpose", "key_json")), + + // push + sequenceTable("push_sequences"), + tbl("push_pushers", pk("localpart", "app_id", "pushkey"), fields( + texts("localpart", "app_id", "pushkey", "device_id", "kind", "app_display_name", + "device_display_name", "profile_tag", "lang", "data_json"), + numbers("pushkey_ts_ms")), + "app_id, pushkey"), + tbl("push_notifications", pk("notification_id"), fields( + numbers("notification_id"), + texts("user_id", "room_id", "event_id"), + numbers("event_pos", "ts_ms"), + []field{fld("highlight", boolean)}, + texts("actions"), + []field{fld("is_read", boolean)}, + texts("thread_id")), + "user_id, notification_id", + "user_id, room_id, is_read, event_pos", + "is_read, ts_ms", + "user_id, room_id, thread_id, is_read, event_pos"), + tbl("push_room_counts", pk("user_id", "room_id"), fields( + texts("user_id", "room_id"), numbers("notification_count", "highlight_count", "stream_pos")), + "user_id, stream_pos"), + + // admin and application services + tbl("admin_registration_tokens", pk("token"), fields( + texts("token"), + numbers("uses_allowed"), + []field{fld("has_uses_allowed", boolean)}, + numbers("completed", "expires_at_ms", "created_at_ms"))), + tbl("admin_server_notice_rooms", pk("localpart"), texts("localpart", "room_id")), + sequenceTable("admin_sequences"), + tbl("admin_audit_log", pk("audit_id"), fields( + numbers("audit_id", "ts_ms"), texts("actor", "action", "target", "details_json")), + "actor, audit_id", "action, audit_id"), + tbl("appservice_state", pk("as_id"), fields(texts("as_id"), numbers("stream_pos", "txn_id", "txn_to"))), + + // delayed events and OpenID tokens + tbl("delayed_events", pk("delay_id"), fields( + texts("delay_id", "user_id", "device_id", "room_id", "event_type", "state_key"), + []field{fld("has_state_key", boolean)}, + texts("content"), + numbers("delay_ms", "running_since_ms", "send_at_ms", "finalised_at_ms"), + texts("outcome", "reason", "event_id", "error")), + "user_id, finalised_at_ms", + "finalised_at_ms, send_at_ms", + "room_id, event_type, state_key, finalised_at_ms"), + tbl("openid_tokens", pk("token_hash"), fields(texts("token_hash", "user_id"), numbers("expires_at_ms")), + "expires_at_ms"), +} + +// tables indexes the schema by collection name. +var tables = func() map[string]*table { + out := make(map[string]*table, len(schema)) + for i := range schema { + out[schema[i].name] = &schema[i] + } + return out +}() + +// maxText is the longest a text field may be. PocketBase's default is 5000 +// characters, which an event can pass; this is far above the 64 KiB a +// Matrix event may be, and above any account data a client sends. +const maxText = 1 << 24 + +// maxFileSize bounds one media file. The media API enforces the configured +// upload limit before content ever gets here. +const maxFileSize = 1 << 40 + +// shortHash is a lower-case base32 digest, which fits PocketBase's id +// alphabet of [a-z0-9]. +func shortHash(n int, parts ...string) string { + h := sha256.New() + for _, p := range parts { + _, _ = fmt.Fprintf(h, "%d:%s", len(p), p) + } + enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h.Sum(nil)) + return strings.ToLower(enc)[:n] +} + +// rowID is the record id of the row whose primary key has these values. +func rowID(collection string, key ...any) string { + parts := make([]string, 0, len(key)+1) + parts = append(parts, collection) + for _, v := range key { + parts = append(parts, keyString(v)) + } + return shortHash(15, parts...) +} + +func keyString(v any) string { + switch v := v.(type) { + case string: + return v + case int64: + return strconv.FormatInt(v, 10) + case int: + return strconv.Itoa(v) + case bool: + return strconv.FormatBool(v) + default: + return fmt.Sprint(v) + } +} + +// collectionJSON renders a table as PocketBase's collection import format. +// Collection and field ids are derived from their names, so that importing +// the same schema again changes nothing, and a later version that adds a +// field adds just that field. +func (tb *table) collectionJSON() map[string]any { + out := []map[string]any{{ + "id": "text3208210256", "name": "id", "type": "text", "system": true, + "primaryKey": true, "required": true, "hidden": false, "presentable": false, + "min": 15, "max": 15, "pattern": "^[a-z0-9]+$", "autogeneratePattern": "[a-z0-9]{15}", + }} + for _, fd := range tb.fields { + m := map[string]any{ + "id": "f" + shortHash(12, tb.name, fd.name), + "name": fd.name, + "system": false, "required": false, "hidden": false, "presentable": false, + } + switch fd.kind { + case text: + m["type"] = "text" + m["min"] = 0 + m["max"] = maxText + m["pattern"] = "" + m["autogeneratePattern"] = "" + m["primaryKey"] = false + case number: + m["type"] = "number" + m["onlyInt"] = true + m["min"] = nil + m["max"] = nil + case boolean: + m["type"] = "bool" + case file: + m["type"] = "file" + m["maxSelect"] = 1 + m["maxSize"] = maxFileSize + m["mimeTypes"] = []string{} + m["thumbs"] = []string{} + m["protected"] = true + } + out = append(out, m) + } + + indexes := []string{} + if len(tb.key) > 0 { + indexes = append(indexes, indexSQL(tb.name, "unique: "+strings.Join(tb.key, ", "))) + } + for _, idx := range tb.indexes { + indexes = append(indexes, indexSQL(tb.name, idx)) + } + + return map[string]any{ + "id": "warp" + shortHash(11, tb.name), + "name": tb.name, + "type": "base", + "system": false, + "listRule": nil, + "viewRule": nil, + "createRule": nil, + "updateRule": nil, + "deleteRule": nil, + "fields": out, + "indexes": indexes, + } +} + +func indexSQL(tableName, spec string) string { + unique := "" + if cols, ok := strings.CutPrefix(spec, "unique:"); ok { + unique, spec = "UNIQUE ", cols + } + var cols []string + for _, c := range strings.Split(spec, ",") { + cols = append(cols, strings.TrimSpace(c)) + } + name := "idx_" + shortHash(10, tableName, unique, strings.Join(cols, ",")) + return fmt.Sprintf("CREATE %sINDEX `%s` ON `%s` (%s)", unique, name, tableName, "`"+strings.Join(cols, "`, `")+"`") +} + +// schemaVersion identifies the schema: any change to a table changes it, and +// a database stamped with another one has a migration pending. +var schemaVersion = func() string { + cols := make([]map[string]any, len(schema)) + for i := range schema { + cols[i] = schema[i].collectionJSON() + } + raw, _ := json.Marshal(cols) + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:])[:12] +}() diff --git a/server/internal/storage/pocketbase/sqldriver.go b/server/internal/storage/pocketbase/sqldriver.go new file mode 100644 index 0000000..82e943c --- /dev/null +++ b/server/internal/storage/pocketbase/sqldriver.go @@ -0,0 +1,362 @@ +package pocketbase + +import ( + "context" + "crypto/rand" + "database/sql/driver" + "encoding/hex" + "errors" + "fmt" + "io" + "slices" + "strconv" + "strings" + "sync" + "time" + "unicode/utf8" +) + +// The read side of the driver is a database/sql driver over PocketBase's SQL +// endpoint, so that the stores can read the way the SQL drivers do: with +// `?` placeholders, joins and aggregates, and rows.Scan. It cannot write — +// every write goes through the batch endpoint, as record operations — and +// it cannot begin a transaction. +// +// The endpoint has three limits the driver hides. It takes no parameters, so +// every argument is inlined as a SQL literal. It takes at most 5000 +// characters, so a query that would be longer has its longest arguments +// moved into temporary records of warp_params and read back by subquery. And +// it returns at most 1000 rows, so a result that fills a page is read on in +// further pages. + +// errReadOnly is returned for anything but a query. +var errReadOnly = errors.New("pocketbase: the SQL connection is read-only; writes go through the batch API") + +const ( + // maxQueryLength is the SQL endpoint's limit, with room for the paging + // wrapper. + maxQueryLength = 5000 - len(pageWrapperPrefix) - len(pageWrapperSuffix) - 32 + + // pageSize is how many rows the SQL endpoint returns at most. + pageSize = 1000 + + pageWrapperPrefix = `SELECT * FROM (` + pageWrapperSuffix = `) LIMIT 1000 OFFSET ` + + // paramsCollection holds the arguments too long to inline. + paramsCollection = "warp_params" +) + +// sqlConnector opens connections to one PocketBase. +type sqlConnector struct { + c *client + + // lock is held for reading around every query that runs outside a + // write, so that a query read in several pages is not split by one. + lock *sync.RWMutex +} + +func (k *sqlConnector) Connect(context.Context) (driver.Conn, error) { return &sqlConn{k: k}, nil } +func (k *sqlConnector) Driver() driver.Driver { return sqlDriver{k: k} } + +// sqlDriver exists because database/sql wants one; connections come from the +// connector. +type sqlDriver struct{ k *sqlConnector } + +func (d sqlDriver) Open(string) (driver.Conn, error) { return &sqlConn{k: d.k}, nil } + +type sqlConn struct{ k *sqlConnector } + +func (c *sqlConn) Prepare(query string) (driver.Stmt, error) { + return &sqlStmt{c: c, query: query}, nil +} +func (c *sqlConn) Close() error { return nil } +func (c *sqlConn) Begin() (driver.Tx, error) { return nil, errReadOnly } + +// Ping implements driver.Pinger. +func (c *sqlConn) Ping(ctx context.Context) error { return c.k.c.health(ctx) } + +// QueryContext implements driver.QueryerContext. +func (c *sqlConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if !holdsWriteLock(ctx) { + c.k.lock.RLock() + defer c.k.lock.RUnlock() + } + return c.k.run(ctx, query, args) +} + +type sqlStmt struct { + c *sqlConn + query string +} + +func (s *sqlStmt) Close() error { return nil } +func (s *sqlStmt) NumInput() int { return -1 } +func (s *sqlStmt) Exec([]driver.Value) (driver.Result, error) { return nil, errReadOnly } +func (s *sqlStmt) Query(args []driver.Value) (driver.Rows, error) { + named := make([]driver.NamedValue, len(args)) + for i, v := range args { + named[i] = driver.NamedValue{Ordinal: i + 1, Value: v} + } + return s.c.QueryContext(context.Background(), s.query, named) +} + +// writeLockKey marks a context whose caller already holds the store's lock, +// for writing or for a multi-query read, so that its queries do not take +// the read lock again behind it. +type writeLockKey struct{} + +func withWriteLock(ctx context.Context) context.Context { + return context.WithValue(ctx, writeLockKey{}, true) +} + +func holdsWriteLock(ctx context.Context) bool { + held, _ := ctx.Value(writeLockKey{}).(bool) + return held +} + +// run inlines the arguments, moves what does not fit into warp_params, and +// reads every page of the result. +func (k *sqlConnector) run(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + parts, literals, err := splitPlaceholders(query, args) + if err != nil { + return nil, err + } + + params, err := k.fitLiterals(ctx, parts, literals) + if err != nil { + return nil, err + } + if len(params) > 0 { + defer k.dropParams(params) + } + full := joinLiterals(parts, literals) + + res, err := k.c.query(ctx, full) + if err != nil { + return nil, fmt.Errorf("%w\n\tin query: %s", err, abbreviate(full)) + } + rows := &sqlRows{columns: make([]string, len(res.Columns)), data: res.Rows} + for i, col := range res.Columns { + rows.columns[i] = col.Name + } + for page := 1; len(res.Rows) == pageSize; page++ { + res, err = k.c.query(ctx, pageWrapperPrefix+full+pageWrapperSuffix+strconv.Itoa(page*pageSize)) + if err != nil { + return nil, fmt.Errorf("%w\n\tin page %d of query: %s", err, page, abbreviate(full)) + } + rows.data = append(rows.data, res.Rows...) + } + return rows, nil +} + +// splitPlaceholders cuts a query at its `?` placeholders, skipping those +// inside quoted strings and identifiers, and renders each argument as a SQL +// literal. +func splitPlaceholders(query string, args []driver.NamedValue) ([]string, []string, error) { + var ( + parts []string + literals []string + start int + quote byte + ) + for i := 0; i < len(query); i++ { + ch := query[i] + switch { + case quote != 0: + if ch == quote { + quote = 0 + } + case ch == '\'' || ch == '"': + quote = ch + case ch == '?': + n := len(literals) + if n >= len(args) { + return nil, nil, fmt.Errorf("pocketbase: query has more placeholders than its %d arguments", len(args)) + } + lit, err := literal(args[n].Value) + if err != nil { + return nil, nil, err + } + parts = append(parts, query[start:i]) + literals = append(literals, lit) + start = i + 1 + } + } + if len(literals) != len(args) { + return nil, nil, fmt.Errorf("pocketbase: query has %d placeholders for %d arguments", len(literals), len(args)) + } + return append(parts, query[start:]), literals, nil +} + +// literal renders one value as SQLite would read it back. +func literal(v any) (string, error) { + switch v := v.(type) { + case nil: + return "NULL", nil + case bool: + if v { + return "1", nil + } + return "0", nil + case int64: + return strconv.FormatInt(v, 10), nil + case float64: + return strconv.FormatFloat(v, 'g', -1, 64), nil + case string: + return textLiteral(v), nil + case []byte: + return textLiteral(string(v)), nil + case time.Time: + return textLiteral(v.UTC().Format(time.RFC3339Nano)), nil + default: + return "", fmt.Errorf("pocketbase: cannot inline a %T into SQL", v) + } +} + +// textLiteral quotes a string. Text that is not valid UTF-8, or holds a NUL, +// goes in as hex and is cast back, so that nothing in it can end the literal +// early. +func textLiteral(s string) string { + if utf8.ValidString(s) && !strings.ContainsRune(s, 0) { + return "'" + strings.ReplaceAll(s, "'", "''") + "'" + } + return "CAST(X'" + hex.EncodeToString([]byte(s)) + "' AS TEXT)" +} + +func joinLiterals(parts, literals []string) string { + var b strings.Builder + for i, p := range parts { + b.WriteString(p) + if i < len(literals) { + b.WriteString(literals[i]) + } + } + return b.String() +} + +// queryLength is what the SQL endpoint counts: characters, not bytes. +func queryLength(parts, literals []string) int { + n := 0 + for _, p := range parts { + n += utf8.RuneCountInString(p) + } + for _, l := range literals { + n += utf8.RuneCountInString(l) + } + return n +} + +// fitLiterals moves the longest text literals into warp_params until the +// query fits, rewriting each into a subquery reading it back. It returns the +// IDs of the records it created. +func (k *sqlConnector) fitLiterals(ctx context.Context, parts, literals []string) ([]string, error) { + if queryLength(parts, literals) <= maxQueryLength { + return nil, nil + } + order := make([]int, len(literals)) + for i := range order { + order[i] = i + } + slices.SortFunc(order, func(a, b int) int { return len(literals[b]) - len(literals[a]) }) + + var ( + ids []string + reqs []batchRequest + ) + for _, i := range order { + if queryLength(parts, literals) <= maxQueryLength { + break + } + text, ok := unquote(literals[i]) + if !ok { + continue + } + id, err := randomID() + if err != nil { + return nil, err + } + reqs = append(reqs, batchRequest{ + Method: "POST", + URL: "/api/collections/" + paramsCollection + "/records", + Body: map[string]any{"id": id, "value": text}, + }) + ids = append(ids, id) + literals[i] = `(SELECT value FROM ` + paramsCollection + ` WHERE id = '` + id + `')` + } + if queryLength(parts, literals) > maxQueryLength { + return nil, fmt.Errorf("pocketbase: query is too long for the SQL endpoint even with its arguments moved out") + } + if err := k.c.batch(ctx, reqs); err != nil { + return nil, fmt.Errorf("pocketbase: store query arguments: %w", err) + } + return ids, nil +} + +// dropParams deletes the records a query's arguments were moved into. It +// runs after the query whatever happened to it, on a context of its own, and +// a failure only leaves rows behind for the next start to clear. +func (k *sqlConnector) dropParams(ids []string) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + reqs := make([]batchRequest, len(ids)) + for i, id := range ids { + reqs[i] = batchRequest{Method: "DELETE", URL: "/api/collections/" + paramsCollection + "/records/" + id} + } + _ = k.c.batch(ctx, reqs) +} + +// unquote recovers the text of a literal textLiteral made. +func unquote(lit string) (string, bool) { + if strings.HasPrefix(lit, "'") { + return strings.ReplaceAll(lit[1:len(lit)-1], "''", "'"), true + } + if hexText, ok := strings.CutPrefix(lit, "CAST(X'"); ok { + raw, err := hex.DecodeString(strings.TrimSuffix(hexText, "' AS TEXT)")) + return string(raw), err == nil + } + return "", false +} + +// randomID makes a record ID in PocketBase's default alphabet. +func randomID() (string, error) { + var raw [8]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", err + } + return "p" + hex.EncodeToString(raw[:])[:14], nil +} + +func abbreviate(query string) string { + query = strings.Join(strings.Fields(query), " ") + if len(query) > 500 { + return query[:500] + "…" + } + return query +} + +// sqlRows is a whole result, already read. +type sqlRows struct { + columns []string + data [][]*string + next int +} + +func (r *sqlRows) Columns() []string { return r.columns } +func (r *sqlRows) Close() error { return nil } + +func (r *sqlRows) Next(dest []driver.Value) error { + if r.next >= len(r.data) { + return io.EOF + } + row := r.data[r.next] + r.next++ + for i := range dest { + if i < len(row) && row[i] != nil { + dest[i] = *row[i] + } else { + dest[i] = nil + } + } + return nil +} diff --git a/server/internal/storage/pocketbase/sync.go b/server/internal/storage/pocketbase/sync.go new file mode 100644 index 0000000..b38f41a --- /dev/null +++ b/server/internal/storage/pocketbase/sync.go @@ -0,0 +1,461 @@ +package pocketbase + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/base64" + "errors" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Sync is the PocketBase implementation of storage.SyncStore. Every stream +// here is a counter row advanced in the same batch as the row it stamps. +type Sync struct{ *base } + +// maxToDeviceBatch bounds how many to-device messages one sync carries. +const maxToDeviceBatch = 100 + +// StreamPositions implements storage.SyncStore. The positions are read under +// one read lock, so that a caller gets one consistent set. +func (s *Sync) StreamPositions(ctx context.Context) (storage.StreamPositions, error) { + var out storage.StreamPositions + err := s.read(ctx, func(ctx context.Context) error { + err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(stream_ordering), 0) FROM roomserver_events`).Scan(&out.PDU) + if err != nil { + return fmt.Errorf("read the top of the event stream: %w", err) + } + + rows, err := s.db.QueryContext(ctx, `SELECT name, value FROM sync_sequences`) + if err != nil { + return fmt.Errorf("read the stream positions: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var ( + name string + value int64 + ) + if err := rows.Scan(&name, &value); err != nil { + return fmt.Errorf("read the stream positions: %w", err) + } + switch name { + case seqReceipt: + out.Receipt = value + case seqToDevice: + out.ToDevice = value + case seqAccountData: + out.AccountData = value + case seqPresence: + out.Presence = value + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("read the stream positions: %w", err) + } + + if out.DeviceList, err = s.currentSequence(ctx, keySequences, seqDeviceList); err != nil { + return err + } + out.Notification, err = s.currentSequence(ctx, pushSequences, seqNotification) + return err + }) + if err != nil { + return storage.StreamPositions{}, err + } + return out, nil +} + +// SetReceipt implements storage.SyncStore. A receipt that would move +// backwards is dropped: the comparison is on the position of the event +// being acknowledged, not on when the receipt arrived. +func (s *Sync) SetReceipt(ctx context.Context, r storage.Receipt) (int64, bool, error) { + var ( + pos int64 + moved bool + ) + err := s.write(ctx, func(t *tx) error { + var target int64 + err := t.queryRow(`SELECT e.stream_ordering FROM roomserver_events e + WHERE e.room_id = ? AND e.event_id = ?`, r.RoomID, r.EventID).Scan(&target) + switch { + case errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("event %q: %w", r.EventID, storage.ErrNoSuchEvent) + case err != nil: + return fmt.Errorf("locate the event a receipt names: %w", err) + } + + var ( + haveEventID string + havePos sql.NullInt64 + haveStream int64 + ) + err = t.queryRow(`SELECT r.event_id, e.stream_ordering, r.stream_pos + FROM sync_receipts r + LEFT JOIN roomserver_events e ON e.event_id = r.event_id + WHERE r.room_id = ? AND r.user_id = ? AND r.receipt_type = ? AND r.thread_id = ?`, + r.RoomID, r.UserID, r.ReceiptType, r.ThreadID).Scan(&haveEventID, &havePos, &haveStream) + switch { + case err == nil: + if havePos.Valid && havePos.Int64 >= target { + pos, moved = haveStream, false + return nil + } + case !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("read the current receipt: %w", err) + } + + if pos, err = t.next(syncSequences, seqReceipt); err != nil { + return err + } + t.put("sync_receipts", rec{ + "room_id": r.RoomID, "user_id": r.UserID, "receipt_type": r.ReceiptType, "thread_id": r.ThreadID, + "event_id": r.EventID, "ts_ms": r.TimestampMS, "stream_pos": pos, + }) + moved = true + return nil + }) + if err != nil { + return 0, false, fmt.Errorf("set receipt in %q: %w", r.RoomID, err) + } + return pos, moved, nil +} + +const receiptColumns = ` + room_id, user_id, receipt_type, thread_id, event_id, ts_ms, stream_pos` + +func scanReceipts(rows *sql.Rows) ([]storage.Receipt, error) { + defer func() { _ = rows.Close() }() + + var out []storage.Receipt + for rows.Next() { + var r storage.Receipt + if err := rows.Scan(&r.RoomID, &r.UserID, &r.ReceiptType, &r.ThreadID, + &r.EventID, &r.TimestampMS, &r.StreamPos); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// Receipts implements storage.SyncStore. +func (s *Sync) Receipts(ctx context.Context, roomIDs []string, from, to int64) ([]storage.Receipt, error) { + if len(roomIDs) == 0 { + return nil, nil + } + + query := `SELECT ` + receiptColumns + ` FROM sync_receipts + WHERE room_id IN ` + inList + ` AND stream_pos > ?` + args := []any{jsonList(roomIDs), from} + if to > 0 { + query += ` AND stream_pos <= ?` + args = append(args, to) + } + query += ` ORDER BY stream_pos` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read receipts: %w", err) + } + receipts, err := scanReceipts(rows) + if err != nil { + return nil, fmt.Errorf("read receipts: %w", err) + } + return receipts, nil +} + +// ReceiptsForUser implements storage.SyncStore. +func (s *Sync) ReceiptsForUser(ctx context.Context, userID string, roomIDs []string) ([]storage.Receipt, error) { + if len(roomIDs) == 0 { + return nil, nil + } + rows, err := s.db.QueryContext(ctx, `SELECT `+receiptColumns+` FROM sync_receipts + WHERE user_id = ? AND room_id IN `+inList+` + ORDER BY stream_pos`, userID, jsonList(roomIDs)) + if err != nil { + return nil, fmt.Errorf("read the receipts of %q: %w", userID, err) + } + receipts, err := scanReceipts(rows) + if err != nil { + return nil, fmt.Errorf("read the receipts of %q: %w", userID, err) + } + return receipts, nil +} + +// SendToDevice implements storage.SyncStore. Each message takes a position +// of its own, so that one device acknowledging its copy does not drop +// another's. +func (s *Sync) SendToDevice(ctx context.Context, msgs []storage.NewToDeviceMessage) (int64, error) { + if len(msgs) == 0 { + top, err := s.currentSequence(ctx, syncSequences, seqToDevice) + if err != nil { + return 0, fmt.Errorf("send to-device messages: %w", err) + } + return top, nil + } + var top int64 + err := s.write(ctx, func(t *tx) error { + for _, m := range msgs { + pos, err := t.next(syncSequences, seqToDevice) + if err != nil { + return err + } + t.put("sync_to_device", rec{ + "stream_pos": pos, "localpart": m.Localpart, "device_id": m.DeviceID, + "sender": m.Sender, "event_type": m.Type, "content": string(m.Content), "sent_at_ms": m.SentAtMS, + }) + top = pos + } + return nil + }) + if err != nil { + return 0, fmt.Errorf("send to-device messages: %w", err) + } + return top, nil +} + +// ToDeviceMessages implements storage.SyncStore. +func (s *Sync) ToDeviceMessages(ctx context.Context, localpart, deviceID string, from, to int64, limit int) ([]storage.ToDeviceMessage, error) { + if limit <= 0 || limit > maxToDeviceBatch { + limit = maxToDeviceBatch + } + + query := `SELECT stream_pos, localpart, device_id, sender, event_type, content, sent_at_ms + FROM sync_to_device + WHERE localpart = ? AND device_id = ? AND stream_pos > ?` + args := []any{localpart, deviceID, from} + if to > 0 { + query += ` AND stream_pos <= ?` + args = append(args, to) + } + query += ` ORDER BY stream_pos LIMIT ?` + args = append(args, limit) + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read to-device messages for %s/%s: %w", localpart, deviceID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.ToDeviceMessage + for rows.Next() { + var ( + m storage.ToDeviceMessage + content string + ) + if err := rows.Scan(&m.StreamPos, &m.Localpart, &m.DeviceID, + &m.Sender, &m.Type, &content, &m.SentAtMS); err != nil { + return nil, fmt.Errorf("read to-device messages for %s/%s: %w", localpart, deviceID, err) + } + m.Content = []byte(content) + out = append(out, m) + } + return out, rows.Err() +} + +// DeleteToDeviceMessages implements storage.SyncStore. +func (s *Sync) DeleteToDeviceMessages(ctx context.Context, localpart, deviceID string, pos int64) error { + if pos <= 0 { + return nil + } + err := s.write(ctx, func(t *tx) error { + _, err := t.delWhere("sync_to_device", `localpart = ? AND device_id = ? AND stream_pos <= ?`, + localpart, deviceID, pos) + return err + }) + if err != nil { + return fmt.Errorf("delete to-device messages for %s/%s: %w", localpart, deviceID, err) + } + return nil +} + +const presenceColumns = `user_id, presence, status_msg, last_active_ms, stream_pos` + +// SetPresence implements storage.SyncStore. A write that changes nothing a +// peer would notice refreshes the last-active timestamp and leaves the +// stream where it was. +func (s *Sync) SetPresence(ctx context.Context, p storage.PresenceState) (int64, bool, error) { + var ( + pos int64 + moved bool + ) + err := s.write(ctx, func(t *tx) error { + var existing storage.PresenceState + err := t.queryRow(`SELECT `+presenceColumns+` FROM sync_presence WHERE user_id = ?`, p.UserID).Scan( + &existing.UserID, &existing.Presence, &existing.StatusMsg, + &existing.LastActiveMS, &existing.StreamPos) + switch { + case err == nil: + if existing.Presence == p.Presence && existing.StatusMsg == p.StatusMsg { + pos = existing.StreamPos + t.put("sync_presence", rec{"user_id": p.UserID, "last_active_ms": p.LastActiveMS}) + return nil + } + case !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("read the current presence: %w", err) + } + + if pos, err = t.next(syncSequences, seqPresence); err != nil { + return err + } + t.put("sync_presence", rec{ + "user_id": p.UserID, "presence": p.Presence, "status_msg": p.StatusMsg, + "last_active_ms": p.LastActiveMS, "stream_pos": pos, + }) + moved = true + return nil + }) + if err != nil { + return 0, false, fmt.Errorf("set the presence of %q: %w", p.UserID, err) + } + return pos, moved, nil +} + +// Presence implements storage.SyncStore. +func (s *Sync) Presence(ctx context.Context, userID string) (storage.PresenceState, error) { + var p storage.PresenceState + err := s.db.QueryRowContext(ctx, `SELECT `+presenceColumns+` FROM sync_presence WHERE user_id = ?`, userID). + Scan(&p.UserID, &p.Presence, &p.StatusMsg, &p.LastActiveMS, &p.StreamPos) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.PresenceState{}, fmt.Errorf("presence of %q: %w", userID, storage.ErrNoSuchUser) + case err != nil: + return storage.PresenceState{}, fmt.Errorf("read the presence of %q: %w", userID, err) + } + return p, nil +} + +// PresenceForUsers implements storage.SyncStore. +func (s *Sync) PresenceForUsers(ctx context.Context, userIDs []string, from, to int64) ([]storage.PresenceState, error) { + if len(userIDs) == 0 { + return nil, nil + } + + query := `SELECT ` + presenceColumns + ` FROM sync_presence + WHERE user_id IN ` + inList + ` AND stream_pos > ?` + args := []any{jsonList(userIDs), from} + if to > 0 { + query += ` AND stream_pos <= ?` + args = append(args, to) + } + query += ` ORDER BY stream_pos` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read presence: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []storage.PresenceState + for rows.Next() { + var p storage.PresenceState + if err := rows.Scan(&p.UserID, &p.Presence, &p.StatusMsg, &p.LastActiveMS, &p.StreamPos); err != nil { + return nil, fmt.Errorf("read presence: %w", err) + } + out = append(out, p) + } + return out, rows.Err() +} + +// AccountDataChanges implements storage.SyncStore. +func (s *Sync) AccountDataChanges(ctx context.Context, localpart string, from, to int64) ([]storage.AccountDataEntry, error) { + query := `SELECT room_id, data_type, stream_pos FROM user_account_data + WHERE localpart = ? AND stream_pos > ?` + args := []any{localpart, from} + if to > 0 { + query += ` AND stream_pos <= ?` + args = append(args, to) + } + query += ` ORDER BY stream_pos` + return s.accountData(ctx, localpart, query, args) +} + +// AccountDataAt implements storage.SyncStore. +func (s *Sync) AccountDataAt(ctx context.Context, localpart string, to int64) ([]storage.AccountDataEntry, error) { + query := `SELECT room_id, data_type, stream_pos FROM user_account_data WHERE localpart = ?` + args := []any{localpart} + if to > 0 { + query += ` AND stream_pos <= ?` + args = append(args, to) + } + query += ` ORDER BY room_id, data_type` + return s.accountData(ctx, localpart, query, args) +} + +func (s *Sync) accountData(ctx context.Context, localpart, query string, args []any) ([]storage.AccountDataEntry, error) { + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the account data of %q: %w", localpart, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.AccountDataEntry + for rows.Next() { + var e storage.AccountDataEntry + if err := rows.Scan(&e.RoomID, &e.DataType, &e.StreamPos); err != nil { + return nil, fmt.Errorf("read the account data of %q: %w", localpart, err) + } + out = append(out, e) + } + return out, rows.Err() +} + +// SaveFilter implements storage.SyncStore. The ID is random rather than a +// counter, so that it says nothing about whose filter it is. +func (s *Sync) SaveFilter(ctx context.Context, localpart string, filter []byte, createdAtMS int64) (string, error) { + var filterID string + err := s.write(ctx, func(t *tx) error { + for range 3 { + id, err := newFilterID() + if err != nil { + return err + } + taken, err := t.exists("sync_filters", `localpart = ? AND filter_id = ?`, localpart, id) + if err != nil { + return err + } + if taken { + continue + } + t.put("sync_filters", rec{ + "localpart": localpart, "filter_id": id, + "filter_json": string(filter), "created_at_ms": createdAtMS, + }) + filterID = id + return nil + } + return errors.New("could not find an unused filter ID") + }) + if err != nil { + return "", fmt.Errorf("save a filter for %q: %w", localpart, err) + } + return filterID, nil +} + +// newFilterID mints a filter ID: 12 random bytes, URL-safe base64. +func newFilterID() (string, error) { + var raw [12]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", fmt.Errorf("generate a filter ID: %w", err) + } + return base64.RawURLEncoding.EncodeToString(raw[:]), nil +} + +// Filter implements storage.SyncStore. +func (s *Sync) Filter(ctx context.Context, localpart, filterID string) ([]byte, error) { + var body string + err := s.db.QueryRowContext(ctx, `SELECT filter_json FROM sync_filters + WHERE localpart = ? AND filter_id = ?`, localpart, filterID).Scan(&body) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, fmt.Errorf("filter %q: %w", filterID, storage.ErrNoSuchFilter) + case err != nil: + return nil, fmt.Errorf("read filter %q: %w", filterID, err) + } + return []byte(body), nil +} + +var _ storage.SyncStore = (*Sync)(nil) diff --git a/server/internal/storage/pocketbase/timeline.go b/server/internal/storage/pocketbase/timeline.go new file mode 100644 index 0000000..08a6652 --- /dev/null +++ b/server/internal/storage/pocketbase/timeline.go @@ -0,0 +1,334 @@ +package pocketbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The stream reads of the room store: the queries the sync API and +// /messages are built on. They are the SQL store's. + +// MaxStreamOrdering implements storage.RoomStore. It reads the events, not +// the counter: a batch writes both at once, but an event skipped as a +// duplicate leaves the counter above every event there is. +func (s *Rooms) MaxStreamOrdering(ctx context.Context) (int64, error) { + var top int64 + if err := s.db.QueryRowContext(ctx, + `SELECT COALESCE(MAX(stream_ordering), 0) FROM roomserver_events`).Scan(&top); err != nil { + return 0, fmt.Errorf("read the top of the event stream: %w", err) + } + return top, nil +} + +// MembershipsForUser implements storage.RoomStore. +func (s *Rooms) MembershipsForUser(ctx context.Context, userID string, memberships []string) ([]storage.SyncMembership, error) { + query := ` + SELECT cs.room_id, cs.membership, cs.event_id, e.stream_ordering + FROM roomserver_current_state cs + JOIN roomserver_events e ON e.event_id = cs.event_id + WHERE cs.event_type = 'm.room.member' AND cs.state_key = ?` + args := []any{userID} + if len(memberships) > 0 { + query += ` AND cs.membership IN ` + inList + args = append(args, jsonList(memberships)) + } + query += ` ORDER BY cs.room_id` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the memberships of %q: %w", userID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.SyncMembership + for rows.Next() { + var m storage.SyncMembership + if err := rows.Scan(&m.RoomID, &m.Membership, &m.EventID, &m.StreamOrdering); err != nil { + return nil, fmt.Errorf("read the memberships of %q: %w", userID, err) + } + out = append(out, m) + } + return out, rows.Err() +} + +// UserMembershipEvents implements storage.RoomStore. +func (s *Rooms) UserMembershipEvents(ctx context.Context, roomID, userID string) ([]storage.StoredEvent, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_events e + WHERE e.room_id = ? AND e.event_type = 'm.room.member' AND e.has_state_key AND e.state_key = ? + AND NOT e.outlier + ORDER BY e.stream_ordering`, roomID, userID) + if err != nil { + return nil, fmt.Errorf("read the membership events of %q in %q: %w", userID, roomID, err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read the membership events of %q in %q: %w", userID, roomID, err) + } + return events, nil +} + +// Timeline implements storage.RoomStore. +func (s *Rooms) Timeline(ctx context.Context, req storage.TimelineRequest) ([]storage.StoredEvent, error) { + if req.Limit <= 0 { + return nil, nil + } + + conditions := []string{`e.room_id = ?`, `NOT e.outlier`} + args := []any{req.RoomID} + if req.Low != 0 { + conditions = append(conditions, `e.stream_ordering > ?`) + args = append(args, req.Low) + } + if req.High != 0 { + conditions = append(conditions, `e.stream_ordering <= ?`) + args = append(args, req.High) + } + + order := `ASC` + if req.Direction == storage.DirectionBackward { + order = `DESC` + } + + if clause, filterArgs := filterConditions(req.Filter); clause != "" { + conditions = append(conditions, clause) + args = append(args, filterArgs...) + } + args = append(args, req.Limit) + + rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_events e + WHERE `+strings.Join(conditions, " AND ")+` + ORDER BY e.stream_ordering `+order+` + LIMIT ?`, args...) + if err != nil { + return nil, fmt.Errorf("read the timeline of %q: %w", req.RoomID, err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read the timeline of %q: %w", req.RoomID, err) + } + return events, nil +} + +// StreamEvents implements storage.RoomStore. +func (s *Rooms) StreamEvents(ctx context.Context, after, upTo int64, limit int) ([]storage.StoredEvent, error) { + if limit <= 0 { + return nil, nil + } + conditions := []string{`e.stream_ordering > ?`, `NOT e.outlier`} + args := []any{after} + if upTo != 0 { + conditions = append(conditions, `e.stream_ordering <= ?`) + args = append(args, upTo) + } + args = append(args, limit) + + rows, err := s.db.QueryContext(ctx, `SELECT `+eventColumns+` + FROM roomserver_events e + WHERE `+strings.Join(conditions, " AND ")+` + ORDER BY e.stream_ordering + LIMIT ?`, args...) + if err != nil { + return nil, fmt.Errorf("read the event stream after %d: %w", after, err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read the event stream after %d: %w", after, err) + } + return events, nil +} + +// filterConditions renders the part of an event filter the store applies in +// SQL. The exclusions are ANDed in after the inclusions, which is what makes +// not_types win over types. +func filterConditions(f storage.TimelineFilter) (string, []any) { + return filterConditionsOn(`e.event_type`, `e.sender`, f) +} + +// filterConditionsOn is filterConditions against named columns. +func filterConditionsOn(typeColumn, senderColumn string, f storage.TimelineFilter) (string, []any) { + var ( + clauses []string + args []any + ) + add := func(column string, patterns []string, negate bool) { + if len(patterns) == 0 { + return + } + clause, clauseArgs := patternMatch(column, patterns) + if negate { + clause = `NOT (` + clause + `)` + } + clauses = append(clauses, clause) + args = append(args, clauseArgs...) + } + add(typeColumn, f.Types, false) + add(typeColumn, f.NotTypes, true) + add(senderColumn, f.Senders, false) + add(senderColumn, f.NotSenders, true) + + if len(clauses) == 0 { + return "", nil + } + return strings.Join(clauses, " AND "), args +} + +// patternMatch renders "this column matches any of these patterns", where a +// trailing '*' matches any suffix. +func patternMatch(column string, patterns []string) (string, []any) { + var ( + exact []string + wildcard []string + args []any + ) + for _, p := range patterns { + if prefix, isWildcard := strings.CutSuffix(p, "*"); isWildcard { + wildcard = append(wildcard, prefix) + continue + } + exact = append(exact, p) + } + + var parts []string + if len(exact) > 0 { + parts = append(parts, column+` IN `+inList) + args = append(args, jsonList(exact)) + } + for _, prefix := range wildcard { + parts = append(parts, column+` LIKE ? ESCAPE '\'`) + args = append(args, escapeLike(prefix)+`%`) + } + return strings.Join(parts, " OR "), args +} + +// RoomsWithEvents implements storage.RoomStore. +func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to int64) ([]string, error) { + if len(roomIDs) == 0 { + return nil, nil + } + + query := ` + SELECT DISTINCT room_id FROM roomserver_events + WHERE room_id IN ` + inList + ` AND NOT outlier` + args := []any{jsonList(roomIDs)} + if from != 0 { + query += ` AND stream_ordering > ?` + args = append(args, from) + } + if to != 0 { + query += ` AND stream_ordering <= ?` + args = append(args, to) + } + query += ` ORDER BY room_id` + + out, err := s.queryStrings(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("find the rooms with events after %d: %w", from, err) + } + return out, nil +} + +// LatestEventPositions implements storage.RoomStore. +func (s *Rooms) LatestEventPositions(ctx context.Context, roomIDs, types []string, upTo int64) (map[string]int64, error) { + if len(roomIDs) == 0 || len(types) == 0 { + return map[string]int64{}, nil + } + query := ` + SELECT room_id, MAX(stream_ordering) FROM roomserver_events + WHERE room_id IN ` + inList + ` + AND event_type IN ` + inList + ` AND NOT outlier` + args := []any{jsonList(roomIDs), jsonList(types)} + if upTo != 0 { + query += ` AND stream_ordering <= ?` + args = append(args, upTo) + } + query += ` GROUP BY room_id` + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err) + } + defer func() { _ = rows.Close() }() + + out := make(map[string]int64, len(roomIDs)) + for rows.Next() { + var ( + roomID string + pos int64 + ) + if err := rows.Scan(&roomID, &pos); err != nil { + return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err) + } + out[roomID] = pos + } + return out, rows.Err() +} + +// StateAtStream implements storage.RoomStore: the snapshot recorded against +// the last event at or before the position, plus that event itself when it +// is a state event. +func (s *Rooms) StateAtStream(ctx context.Context, roomID string, pos int64) ([]storage.StateEntry, error) { + if pos == 0 { + return nil, nil + } + + var ( + eventID string + eventType string + stateKey sql.NullString + snapshotID int64 + ) + err := s.db.QueryRowContext(ctx, ` + SELECT e.event_id, e.event_type, CASE WHEN e.has_state_key THEN e.state_key END, e.state_before_id + FROM roomserver_events e + WHERE e.room_id = ? AND NOT e.outlier AND e.stream_ordering <= ? + ORDER BY e.stream_ordering DESC + LIMIT 1`, roomID, pos). + Scan(&eventID, &eventType, &stateKey, &snapshotID) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, nil + case err != nil: + return nil, fmt.Errorf("read the state of %q at %d: %w", roomID, pos, err) + } + + var entries []storage.StateEntry + if snapshotID != 0 { + rows, qErr := s.db.QueryContext(ctx, ` + SELECT se.event_type, se.state_key, se.event_id, '' + FROM roomserver_state_entries se + WHERE se.snapshot_id = ? + ORDER BY se.event_type, se.state_key`, snapshotID) + if qErr != nil { + return nil, fmt.Errorf("read the state of %q at %d: %w", roomID, pos, qErr) + } + if entries, err = scanStateEntries(rows); err != nil { + return nil, fmt.Errorf("read the state of %q at %d: %w", roomID, pos, err) + } + } + if stateKey.Valid { + entries = upsertStateEntry(entries, storage.StateEntry{ + Type: eventType, + StateKey: stateKey.String, + EventID: eventID, + }) + } + return entries, nil +} + +// upsertStateEntry replaces the matching tuple in entries, or appends it. +func upsertStateEntry(entries []storage.StateEntry, entry storage.StateEntry) []storage.StateEntry { + for i, existing := range entries { + if existing.Type == entry.Type && existing.StateKey == entry.StateKey { + entries[i] = entry + return entries + } + } + return append(entries, entry) +} diff --git a/server/internal/storage/pocketbase/users.go b/server/internal/storage/pocketbase/users.go new file mode 100644 index 0000000..01754d9 --- /dev/null +++ b/server/internal/storage/pocketbase/users.go @@ -0,0 +1,557 @@ +package pocketbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Users is the PocketBase implementation of storage.UserStore. +// +// The reads are the SQL store's statements. The writes are record +// operations, and where the SQL store lets a primary key or an UPDATE's +// affected-row count answer a question — is this localpart taken, does +// this account exist — this one asks first: the write lock is held from +// the question to the batch, so the answer cannot change in between. +type Users struct{ *base } + +const ( + defaultAccountPageSize = 100 + maxAccountPageSize = 1000 +) + +// CreateAccount implements storage.UserStore. +func (u *Users) CreateAccount(ctx context.Context, acc storage.NewAccount) (storage.Account, error) { + err := u.write(ctx, func(t *tx) error { + taken, err := t.exists("user_accounts", `localpart = ?`, acc.Localpart) + if err != nil { + return err + } + if taken { + return storage.ErrUserExists + } + t.put("user_accounts", rec{ + "localpart": acc.Localpart, + "password_hash": acc.PasswordHash, + "display_name": acc.DisplayName, + "avatar_url": "", + "admin": acc.Admin, + "deactivated": false, + "created_at_ms": acc.CreatedAtMS, + }) + return nil + }) + if err != nil { + return storage.Account{}, fmt.Errorf("create account %q: %w", acc.Localpart, err) + } + return storage.Account{ + Localpart: acc.Localpart, + DisplayName: acc.DisplayName, + Admin: acc.Admin, + CreatedAtMS: acc.CreatedAtMS, + }, nil +} + +const selectAccountSQL = ` + SELECT localpart, display_name, avatar_url, admin, deactivated, created_at_ms + FROM user_accounts WHERE localpart = ?` + +// Account implements storage.UserStore. +func (u *Users) Account(ctx context.Context, localpart string) (storage.Account, error) { + var acc storage.Account + err := u.db.QueryRowContext(ctx, selectAccountSQL, localpart).Scan( + &acc.Localpart, &acc.DisplayName, &acc.AvatarURL, + &acc.Admin, &acc.Deactivated, &acc.CreatedAtMS) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.Account{}, fmt.Errorf("account %q: %w", localpart, storage.ErrNoSuchUser) + case err != nil: + return storage.Account{}, fmt.Errorf("account %q: %w", localpart, err) + } + return acc, nil +} + +// PasswordHash implements storage.UserStore. +func (u *Users) PasswordHash(ctx context.Context, localpart string) (string, error) { + var hash string + err := u.db.QueryRowContext(ctx, `SELECT password_hash FROM user_accounts WHERE localpart = ?`, localpart).Scan(&hash) + switch { + case errors.Is(err, sql.ErrNoRows): + return "", fmt.Errorf("password for %q: %w", localpart, storage.ErrNoSuchUser) + case err != nil: + return "", fmt.Errorf("password for %q: %w", localpart, err) + } + return hash, nil +} + +// SetPasswordHash implements storage.UserStore. +func (u *Users) SetPasswordHash(ctx context.Context, localpart, hash string) error { + return u.updateAccount(ctx, localpart, rec{"password_hash": hash}) +} + +// SetAdmin implements storage.UserStore. +func (u *Users) SetAdmin(ctx context.Context, localpart string, admin bool) error { + return u.updateAccount(ctx, localpart, rec{"admin": admin}) +} + +// SetDeactivated implements storage.UserStore. +func (u *Users) SetDeactivated(ctx context.Context, localpart string, deactivated bool) error { + return u.updateAccount(ctx, localpart, rec{"deactivated": deactivated}) +} + +// SetDisplayName implements storage.UserStore. +func (u *Users) SetDisplayName(ctx context.Context, localpart, displayName string) error { + return u.updateAccount(ctx, localpart, rec{"display_name": displayName}) +} + +// SetAvatarURL implements storage.UserStore. +func (u *Users) SetAvatarURL(ctx context.Context, localpart, avatarURL string) error { + return u.updateAccount(ctx, localpart, rec{"avatar_url": avatarURL}) +} + +// updateAccount changes columns of one account, returning ErrNoSuchUser for +// an account that is not there. +func (u *Users) updateAccount(ctx context.Context, localpart string, row rec) error { + err := u.write(ctx, func(t *tx) error { + n, err := t.setWhere("user_accounts", row, `localpart = ?`, localpart) + if err != nil { + return err + } + if n == 0 { + return storage.ErrNoSuchUser + } + return nil + }) + if err != nil { + return fmt.Errorf("update account %q: %w", localpart, err) + } + return nil +} + +// likeEscape is the escape character the search patterns use. +const likeEscape = '!' + +const ( + countAccountsSQL = ` + SELECT COUNT(*) FROM user_accounts + WHERE LOWER(localpart) LIKE ? ESCAPE '!' OR LOWER(display_name) LIKE ? ESCAPE '!'` + + listAccountsSQL = ` + SELECT localpart, display_name, avatar_url, admin, deactivated, created_at_ms + FROM user_accounts + WHERE LOWER(localpart) LIKE ? ESCAPE '!' OR LOWER(display_name) LIKE ? ESCAPE '!' + ORDER BY localpart + LIMIT ? OFFSET ?` +) + +// ListAccounts implements storage.UserStore. +func (u *Users) ListAccounts(ctx context.Context, filter storage.AccountFilter) ([]storage.Account, int, error) { + pattern := likeContains(filter.Search) + + limit := filter.Limit + switch { + case limit <= 0: + limit = defaultAccountPageSize + case limit > maxAccountPageSize: + limit = maxAccountPageSize + } + offset := max(filter.Offset, 0) + + var total int + if err := u.db.QueryRowContext(ctx, countAccountsSQL, pattern, pattern).Scan(&total); err != nil { + return nil, 0, fmt.Errorf("count accounts: %w", err) + } + + rows, err := u.db.QueryContext(ctx, listAccountsSQL, pattern, pattern, limit, offset) + if err != nil { + return nil, 0, fmt.Errorf("list accounts: %w", err) + } + defer func() { _ = rows.Close() }() + + accounts := []storage.Account{} + for rows.Next() { + var acc storage.Account + if err := rows.Scan(&acc.Localpart, &acc.DisplayName, &acc.AvatarURL, + &acc.Admin, &acc.Deactivated, &acc.CreatedAtMS); err != nil { + return nil, 0, fmt.Errorf("scan account: %w", err) + } + accounts = append(accounts, acc) + } + if err := rows.Err(); err != nil { + return nil, 0, fmt.Errorf("list accounts: %w", err) + } + return accounts, total, nil +} + +// likeContains turns a search term into a LIKE pattern matching it +// anywhere, with the wildcards and the escape character escaped. +func likeContains(search string) string { + search = strings.ToLower(strings.TrimSpace(search)) + var b strings.Builder + b.Grow(len(search) + 2) + b.WriteByte('%') + for _, r := range search { + if r == '%' || r == '_' || r == likeEscape { + b.WriteByte(likeEscape) + } + b.WriteRune(r) + } + b.WriteByte('%') + return b.String() +} + +const statsSQL = ` + SELECT + (SELECT COUNT(*) FROM user_accounts), + (SELECT COUNT(*) FROM user_accounts WHERE admin = ?), + (SELECT COUNT(*) FROM user_accounts WHERE deactivated = ?), + (SELECT COUNT(*) FROM user_devices)` + +// Stats implements storage.UserStore. +func (u *Users) Stats(ctx context.Context) (storage.Stats, error) { + var s storage.Stats + err := u.db.QueryRowContext(ctx, statsSQL, true, true). + Scan(&s.Users, &s.Admins, &s.Deactivated, &s.Devices) + if err != nil { + return storage.Stats{}, fmt.Errorf("user stats: %w", err) + } + return s, nil +} + +// CreateSession implements storage.UserStore. +// +// Logging in on a device ID that already exists keeps the device and its +// display name but revokes the tokens it held, which is what the spec +// requires. +func (u *Users) CreateSession(ctx context.Context, s storage.NewSession) (storage.Device, error) { + device := storage.Device{ + Localpart: s.Localpart, + ID: s.DeviceID, + DisplayName: s.DisplayName, + CreatedAtMS: s.CreatedAtMS, + LastSeenMS: s.CreatedAtMS, + LastSeenIP: s.LastSeenIP, + UserAgent: s.UserAgent, + } + + err := u.write(ctx, func(t *tx) error { + found, err := t.exists("user_accounts", `localpart = ?`, s.Localpart) + if err != nil { + return fmt.Errorf("create session for %q: %w", s.Localpart, err) + } + if !found { + return fmt.Errorf("create session for %q: %w", s.Localpart, storage.ErrNoSuchUser) + } + + var existingName string + var existingCreated int64 + err = t.queryRow(`SELECT display_name, created_at_ms FROM user_devices + WHERE localpart = ? AND device_id = ?`, s.Localpart, s.DeviceID). + Scan(&existingName, &existingCreated) + switch { + case errors.Is(err, sql.ErrNoRows): + t.put("user_devices", rec{ + "localpart": s.Localpart, + "device_id": s.DeviceID, + "display_name": s.DisplayName, + "created_at_ms": s.CreatedAtMS, + "last_seen_ms": s.CreatedAtMS, + "last_seen_ip": s.LastSeenIP, + "user_agent": s.UserAgent, + }) + case err != nil: + return fmt.Errorf("read device %q: %w", s.DeviceID, err) + default: + device.DisplayName = existingName + device.CreatedAtMS = existingCreated + t.put("user_devices", rec{ + "localpart": s.Localpart, + "device_id": s.DeviceID, + "last_seen_ms": s.CreatedAtMS, + "last_seen_ip": s.LastSeenIP, + "user_agent": s.UserAgent, + }) + if err := revokeDeviceTokens(t, `localpart = ? AND device_id = ?`, s.Localpart, s.DeviceID); err != nil { + return err + } + } + + t.put("user_access_tokens", rec{ + "token_hash": s.AccessTokenHash, + "localpart": s.Localpart, + "device_id": s.DeviceID, + "created_at_ms": s.CreatedAtMS, + "expires_at_ms": s.AccessTokenExpiresAtMS, + }) + if s.RefreshTokenHash != "" { + t.put("user_refresh_tokens", rec{ + "token_hash": s.RefreshTokenHash, + "localpart": s.Localpart, + "device_id": s.DeviceID, + "created_at_ms": s.CreatedAtMS, + "expires_at_ms": s.RefreshTokenExpiresAtMS, + }) + } + return nil + }) + if err != nil { + return storage.Device{}, err + } + return device, nil +} + +// revokeDeviceTokens deletes the access and refresh tokens matching a +// condition on (localpart, device_id). +func revokeDeviceTokens(t *tx, where string, args ...any) error { + for _, table := range []string{"user_access_tokens", "user_refresh_tokens"} { + if _, err := t.delWhere(table, where, args...); err != nil { + return fmt.Errorf("revoke tokens: %w", err) + } + } + return nil +} + +// RefreshSession implements storage.UserStore. +// +// The spent refresh token and the access token it had minted both go away +// immediately. +func (u *Users) RefreshSession(ctx context.Context, r storage.RefreshedSession) (storage.TokenInfo, error) { + var info storage.TokenInfo + var expired bool + + err := u.write(ctx, func(t *tx) error { + var expiresAt int64 + err := t.queryRow(`SELECT localpart, device_id, expires_at_ms FROM user_refresh_tokens WHERE token_hash = ?`, + r.RefreshTokenHash).Scan(&info.Localpart, &info.DeviceID, &expiresAt) + switch { + case errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("refresh token: %w", storage.ErrNoSuchToken) + case err != nil: + return fmt.Errorf("refresh token: %w", err) + } + if _, err := t.delKey("user_refresh_tokens", r.RefreshTokenHash); err != nil { + return fmt.Errorf("spend refresh token: %w", err) + } + if expiresAt != 0 && expiresAt <= r.RefreshedAtMS { + // The dead token is cleaned up on the way past: the batch is + // still sent, and the error reported after it. + expired = true + return nil + } + + if _, err := t.delWhere("user_access_tokens", `localpart = ? AND device_id = ?`, + info.Localpart, info.DeviceID); err != nil { + return fmt.Errorf("revoke previous access token: %w", err) + } + t.put("user_access_tokens", rec{ + "token_hash": r.NewAccessTokenHash, + "localpart": info.Localpart, + "device_id": info.DeviceID, + "created_at_ms": r.RefreshedAtMS, + "expires_at_ms": r.AccessTokenExpiresAtMS, + }) + if r.NewRefreshTokenHash != "" { + t.put("user_refresh_tokens", rec{ + "token_hash": r.NewRefreshTokenHash, + "localpart": info.Localpart, + "device_id": info.DeviceID, + "created_at_ms": r.RefreshedAtMS, + "expires_at_ms": r.RefreshTokenExpiresAtMS, + }) + } + + if err := t.queryRow(`SELECT admin, deactivated FROM user_accounts WHERE localpart = ?`, info.Localpart). + Scan(&info.Admin, &info.Deactivated); err != nil { + return fmt.Errorf("read account %q: %w", info.Localpart, err) + } + info.ExpiresAtMS = r.AccessTokenExpiresAtMS + return nil + }) + if err != nil { + return storage.TokenInfo{}, err + } + if expired { + return storage.TokenInfo{}, fmt.Errorf("refresh token has expired: %w", storage.ErrNoSuchToken) + } + return info, nil +} + +const selectTokenInfoSQL = ` + SELECT t.localpart, t.device_id, t.expires_at_ms, a.admin, a.deactivated + FROM user_access_tokens t + JOIN user_accounts a ON a.localpart = t.localpart + WHERE t.token_hash = ?` + +// TokenInfo implements storage.UserStore. +func (u *Users) TokenInfo(ctx context.Context, tokenHash string) (storage.TokenInfo, error) { + var info storage.TokenInfo + err := u.db.QueryRowContext(ctx, selectTokenInfoSQL, tokenHash).Scan( + &info.Localpart, &info.DeviceID, &info.ExpiresAtMS, &info.Admin, &info.Deactivated) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.TokenInfo{}, storage.ErrNoSuchToken + case err != nil: + return storage.TokenInfo{}, fmt.Errorf("look up access token: %w", err) + } + return info, nil +} + +// TouchDevice implements storage.UserStore. A device that has gone away +// between authenticating and recording the visit is not an error. +func (u *Users) TouchDevice(ctx context.Context, localpart, deviceID, ip, userAgent string, seenAtMS int64) error { + err := u.write(ctx, func(t *tx) error { + _, err := t.setWhere("user_devices", rec{ + "last_seen_ms": seenAtMS, + "last_seen_ip": ip, + "user_agent": userAgent, + }, `localpart = ? AND device_id = ?`, localpart, deviceID) + return err + }) + if err != nil { + return fmt.Errorf("touch device %q: %w", deviceID, err) + } + return nil +} + +const deviceColumns = `localpart, device_id, display_name, created_at_ms, last_seen_ms, last_seen_ip, user_agent` + +// Device implements storage.UserStore. +func (u *Users) Device(ctx context.Context, localpart, deviceID string) (storage.Device, error) { + var d storage.Device + err := u.db.QueryRowContext(ctx, `SELECT `+deviceColumns+` + FROM user_devices WHERE localpart = ? AND device_id = ?`, localpart, deviceID).Scan( + &d.Localpart, &d.ID, &d.DisplayName, &d.CreatedAtMS, + &d.LastSeenMS, &d.LastSeenIP, &d.UserAgent) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.Device{}, fmt.Errorf("device %q: %w", deviceID, storage.ErrNoSuchDevice) + case err != nil: + return storage.Device{}, fmt.Errorf("device %q: %w", deviceID, err) + } + return d, nil +} + +// Devices implements storage.UserStore. +func (u *Users) Devices(ctx context.Context, localpart string) ([]storage.Device, error) { + rows, err := u.db.QueryContext(ctx, `SELECT `+deviceColumns+` + FROM user_devices WHERE localpart = ? ORDER BY device_id`, localpart) + if err != nil { + return nil, fmt.Errorf("list devices of %q: %w", localpart, err) + } + defer func() { _ = rows.Close() }() + + devices := []storage.Device{} + for rows.Next() { + var d storage.Device + if err := rows.Scan(&d.Localpart, &d.ID, &d.DisplayName, &d.CreatedAtMS, + &d.LastSeenMS, &d.LastSeenIP, &d.UserAgent); err != nil { + return nil, fmt.Errorf("scan device: %w", err) + } + devices = append(devices, d) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list devices of %q: %w", localpart, err) + } + return devices, nil +} + +// SetDeviceDisplayName implements storage.UserStore. +func (u *Users) SetDeviceDisplayName(ctx context.Context, localpart, deviceID, displayName string) error { + err := u.write(ctx, func(t *tx) error { + n, err := t.setWhere("user_devices", rec{"display_name": displayName}, + `localpart = ? AND device_id = ?`, localpart, deviceID) + if err != nil { + return err + } + if n == 0 { + return storage.ErrNoSuchDevice + } + return nil + }) + if err != nil { + return fmt.Errorf("rename device %q: %w", deviceID, err) + } + return nil +} + +// DeleteDevices implements storage.UserStore. +func (u *Users) DeleteDevices(ctx context.Context, localpart string, deviceIDs []string) (int, error) { + if len(deviceIDs) == 0 { + return 0, nil + } + var deleted int + err := u.write(ctx, func(t *tx) error { + where := `localpart = ? AND device_id IN (SELECT value FROM json_each(?))` + list := jsonList(deviceIDs) + if err := revokeDeviceTokens(t, where, localpart, list); err != nil { + return err + } + n, err := t.delWhere("user_devices", where, localpart, list) + if err != nil { + return fmt.Errorf("delete devices: %w", err) + } + deleted = n + return nil + }) + return deleted, err +} + +// DeleteAllDevices implements storage.UserStore. +func (u *Users) DeleteAllDevices(ctx context.Context, localpart string) (int, error) { + var deleted int + err := u.write(ctx, func(t *tx) error { + if err := revokeDeviceTokens(t, `localpart = ?`, localpart); err != nil { + return err + } + n, err := t.delWhere("user_devices", `localpart = ?`, localpart) + if err != nil { + return fmt.Errorf("delete devices of %q: %w", localpart, err) + } + deleted = n + return nil + }) + return deleted, err +} + +// AccountData implements storage.UserStore. +func (u *Users) AccountData(ctx context.Context, localpart, roomID, dataType string) ([]byte, error) { + var content string + err := u.db.QueryRowContext(ctx, `SELECT content FROM user_account_data + WHERE localpart = ? AND room_id = ? AND data_type = ?`, localpart, roomID, dataType).Scan(&content) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, fmt.Errorf("account data %q: %w", dataType, storage.ErrNoSuchAccountData) + case err != nil: + return nil, fmt.Errorf("account data %q: %w", dataType, err) + } + return []byte(content), nil +} + +// SetAccountData implements storage.UserStore. The write stamps the row with +// a position in the account data stream, in the same batch as the content. +func (u *Users) SetAccountData(ctx context.Context, localpart, roomID, dataType string, content []byte) error { + err := u.write(ctx, func(t *tx) error { + pos, err := t.next(syncSequences, seqAccountData) + if err != nil { + return err + } + t.put("user_account_data", rec{ + "localpart": localpart, + "room_id": roomID, + "data_type": dataType, + "content": string(content), + "stream_pos": pos, + }) + return nil + }) + if err != nil { + return fmt.Errorf("set account data %q: %w", dataType, err) + } + return nil +} + +var _ storage.UserStore = (*Users)(nil) diff --git a/server/internal/storage/storage.go b/server/internal/storage/storage.go index 34762f3..71b2e64 100644 --- a/server/internal/storage/storage.go +++ b/server/internal/storage/storage.go @@ -27,7 +27,7 @@ const ( ) // ErrNotImplemented is returned by a driver that is compiled in but not -// finished — currently PocketBase. +// finished. var ErrNotImplemented = errors.New("not implemented by this storage driver") // ErrNoSuchSetting is returned by SettingsStore.Get for an unknown key. @@ -41,6 +41,12 @@ type Config struct { MaxIdleConns int ConnMaxLifetime time.Duration + // SuperuserEmail and SuperuserPassword are the account the pocketbase + // driver signs in to its server with. Only the pocketbase driver reads + // them. + SuperuserEmail string + SuperuserPassword string + // S3 is the bucket a driver with its own file storage puts media in. // Only the pocketbase driver reads it. S3 S3Config diff --git a/server/internal/storage/storage_test.go b/server/internal/storage/storage_test.go index 9df88f0..3b8c490 100644 --- a/server/internal/storage/storage_test.go +++ b/server/internal/storage/storage_test.go @@ -11,11 +11,13 @@ import ( "github.com/dariuszjaniak/warp/server/internal/storage" _ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" + "github.com/dariuszjaniak/warp/server/internal/storage/pocketbase/pbtest" ) // openTestDB opens each driver that can run without external services. The // postgres driver joins in when WARP_TEST_POSTGRES_DSN is set, which is how -// CI runs the same tests against both engines. +// CI runs the same tests against both engines; the pocketbase driver gets a +// real PocketBase server of its own, run in-process. func openTestDB(t *testing.T, driver string) storage.Database { t.Helper() @@ -31,7 +33,9 @@ func openTestDB(t *testing.T, driver string) storage.Database { } cfg.ConnectionString = dsn case storage.DriverPocketBase: - cfg.ConnectionString = filepath.Join(t.TempDir(), "pb_data") + cfg.ConnectionString = pbtest.Start(t).URL + cfg.SuperuserEmail = pbtest.SuperuserEmail + cfg.SuperuserPassword = pbtest.SuperuserPassword default: t.Fatalf("no test setup for driver %q", driver) } @@ -45,7 +49,7 @@ func openTestDB(t *testing.T, driver string) storage.Database { } // testDrivers are run by every storage test. PocketBase is here because -// parity with sqlite is its whole contract. +// parity with the SQL drivers is its whole contract. var testDrivers = []string{storage.DriverSQLite, storage.DriverPostgres, storage.DriverPocketBase} func TestDriversRegistered(t *testing.T) { @@ -64,14 +68,26 @@ func TestOpenUnknownDriver(t *testing.T) { } } -func TestPocketBaseNeedsADataDirectory(t *testing.T) { +func TestPocketBaseNeedsAServer(t *testing.T) { if _, err := storage.Open(t.Context(), storage.Config{Driver: storage.DriverPocketBase}, nil); err == nil { - t.Fatal("expected an error for a pocketbase driver with no data directory") + t.Fatal("expected an error for a pocketbase driver with no server URL") + } + _, err := storage.Open(t.Context(), storage.Config{ + Driver: storage.DriverPocketBase, + ConnectionString: pbtest.Start(t).URL, + SuperuserEmail: pbtest.SuperuserEmail, + SuperuserPassword: "not-the-password", + }, nil) + if err == nil { + t.Fatal("expected an error for a superuser password the server refuses") } } -func TestPocketBaseKeepsMediaInItsFilesystem(t *testing.T) { +func TestPocketBaseKeepsMediaInItsStorage(t *testing.T) { db := openTestDB(t, storage.DriverPocketBase) + if err := db.Migrate(t.Context()); err != nil { + t.Fatalf("Migrate: %v", err) + } provider, ok := db.(storage.BlobProvider) if !ok { t.Fatal("the pocketbase driver does not provide its own blob store") diff --git a/warp.yaml.example b/warp.yaml.example index 772a3c6..c93cd5a 100644 --- a/warp.yaml.example +++ b/warp.yaml.example @@ -51,13 +51,13 @@ listen: database: # sqlite (dev, single user), postgres (production) or pocketbase - # (appliance: sqlite inside an embedded PocketBase, media on its - # filesystem, local or S3). WARP_DB_DRIVER + # (a PocketBase server of its own, media in its storage, local or + # S3). WARP_DB_DRIVER driver: sqlite # sqlite: file:warp.db?_pragma=journal_mode(WAL) # postgres: postgres://warp:secret@localhost/warp?sslmode=disable - # pocketbase: the data directory, e.g. ./pb_data + # pocketbase: the server's URL, e.g. http://localhost:8070 # WARP_DB_CONNECTION_STRING connection_string: "file:warp.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_pragma=foreign_keys(1)" @@ -69,10 +69,15 @@ database: # behind an explicit `warp migrate`. WARP_DB_AUTO_MIGRATE auto_migrate: true - # The pocketbase driver only. With S3 on, media content goes to the - # bucket instead of /storage; media.base_path is not - # used by this driver either way. + # The pocketbase driver only. Warp signs in as this PocketBase + # superuser. With S3 on, media content goes to the bucket instead of + # PocketBase's pb_data/storage, and Warp writes the bucket into + # PocketBase's settings on every start; media.base_path is not used by + # this driver either way. pocketbase: + superuser_email: "" # WARP_PB_SUPERUSER_EMAIL + superuser_password: "" # WARP_PB_SUPERUSER_PASSWORD + superuser_file: "" # WARP_PB_SUPERUSER_FILE: KEY=value lines for either s3: enabled: false # WARP_PB_S3_ENABLED bucket: "" # WARP_PB_S3_BUCKET