optimized search count queries to use rowid by default

Co-authored-by: Yuxiang Gao <yuxiang-gao@outlook.com>

.github/workflows/release.yaml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: '>=1.20.0'
+          go-version: '>=1.20.3'
 
       # This step usually is not needed because the /ui/dist is pregenerated locally
       # but its here to ensure that each release embeds the latest admin ui artifacts.

CHANGELOG.md

@@ -1,3 +1,224 @@
+## v0.16.7
+
+- Minor optimiaztion for the list/search queries to use `rowid` with the `COUNT` statement when available.
+  _This eliminates the temp B-TREE step when executing the query and for large datasets (eg. 150k) it could have 10x improvement (from ~580ms to ~60ms)._
+
+
+## v0.16.6
+
+- Fixed collection index column sort normalization in the Admin UI ([#2681](https://github.com/pocketbase/pocketbase/pull/2681); thanks @SimonLoir).
+
+- Removed unnecessary admins count in `apis.RequireAdminAuthOnlyIfAny()` middleware ([#2726](https://github.com/pocketbase/pocketbase/pull/2726); thanks @svekko).
+
+- Fixed `multipart/form-data` request bind not populating map array values ([#2763](https://github.com/pocketbase/pocketbase/discussions/2763#discussioncomment-6278902)).
+
+- Upgraded npm and Go dependencies.
+
+
+## v0.16.5
+
+- Fixed the Admin UI serialization of implicit relation display fields ([#2675](https://github.com/pocketbase/pocketbase/issues/2675)).
+
+- Reset the Admin UI sort in case the active sort collection field is renamed or deleted.
+
+
+## v0.16.4
+
+- Fixed the selfupdate command not working on Windows due to missing `.exe` in the extracted binary path ([#2589](https://github.com/pocketbase/pocketbase/discussions/2589)).
+  _Note that the command on Windows will work on v0.16.4+ onwards, meaning that you still will have to update manually one more time to v0.16.4._
+
+- Added `int64`, `int32`, `uint`, `uint64` and `uint32` support when scanning `types.DateTime` ([#2602](https://github.com/pocketbase/pocketbase/discussions/2602))
+
+- Updated dependencies.
+
+
+## v0.16.3
+
+- Fixed schema fields sort not working on Safari/Gnome Web ([#2567](https://github.com/pocketbase/pocketbase/issues/2567)).
+
+- Fixed default `PRAGMA`s not being applied for new connections ([#2570](https://github.com/pocketbase/pocketbase/discussions/2570)).
+
+
+## v0.16.2
+
+- Fixed backups archive not excluding the local `backups` directory on Windows ([#2548](https://github.com/pocketbase/pocketbase/discussions/2548#discussioncomment-5979712)).
+
+- Changed file field to not use `dataTransfer.effectAllowed` when dropping files since it is not reliable and consistent across different OS and browsers ([#2541](https://github.com/pocketbase/pocketbase/issues/2541)).
+
+- Auto register the initial generated snapshot migration to prevent incorrectly reapplying the snapshot on Docker restart ([#2551](https://github.com/pocketbase/pocketbase/discussions/2551)).
+
+- Fixed missing view id field error message typo.
+
+
+## v0.16.1
+
+- Fixed backup restore not working in a container environment when `pb_data` is mounted as volume ([#2519](https://github.com/pocketbase/pocketbase/issues/2519)).
+
+- Fixed Dart SDK realtime API preview example ([#2523](https://github.com/pocketbase/pocketbase/pull/2523); thanks @xFrann).
+
+- Fixed typo in the backups create panel ([#2526](https://github.com/pocketbase/pocketbase/pull/2526); thanks @dschissler).
+
+- Removed unnecessary slice length check in `list.ExistInSlice` ([#2527](https://github.com/pocketbase/pocketbase/pull/2527); thanks @KunalSin9h).
+
+- Avoid mutating the cached request data on OAuth2 user create ([#2535](https://github.com/pocketbase/pocketbase/discussions/2535)).
+
+- Fixed Export Collections "Download as JSON" ([#2540](https://github.com/pocketbase/pocketbase/issues/2540)).
+
+- Fixed file field drag and drop not working in Firefox and Safari ([#2541](https://github.com/pocketbase/pocketbase/issues/2541)).
+
+
+## v0.16.0
+
+- Added automated backups (_+ cron rotation_) APIs and UI for the `pb_data` directory.
+  The backups can be also initialized programmatically using `app.CreateBackup("backup.zip")`.
+  There is also experimental restore method - `app.RestoreBackup("backup.zip")` (_currently works only on UNIX systems as it relies on execve_).
+  The backups can be stored locally or in external S3 storage (_it has its own configuration, separate from the file uploads storage filesystem_).
+
+- Added option to limit the returned API fields using the `?fields` query parameter.
+  The "fields picker" is applied for `SearchResult.Items` and every other JSON response. For example:
+  ```js
+  // original: {"id": "RECORD_ID", "name": "abc", "description": "...something very big...", "items": ["id1", "id2"], "expand": {"items": [{"id": "id1", "name": "test1"}, {"id": "id2", "name": "test2"}]}}
+  // output:   {"name": "abc", "expand": {"items": [{"name": "test1"}, {"name": "test2"}]}}
+  const result = await pb.collection("example").getOne("RECORD_ID", {
+    expand: "items",
+    fields: "name,expand.items.name",
+  })
+  ```
+
+- Added new `./pocketbase update` command to selfupdate the prebuilt executable (with option to generate a backup of your `pb_data`).
+
+- Added new `./pocketbase admin` console command:
+  ```sh
+  // creates new admin account
+  ./pocketbase admin create test@example.com 123456890
+
+  // changes the password of an existing admin account
+  ./pocketbase admin update test@example.com 0987654321
+
+  // deletes single admin account (if exists)
+  ./pocketbase admin delete test@example.com
+  ```
+
+- Added `apis.Serve(app, options)` helper to allow starting the API server programmatically.
+
+- Updated the schema fields Admin UI for "tidier" fields visualization.
+
+- Updated the logs "real" user IP to check for `Fly-Client-IP` header and changed the `X-Forward-For` header to use the first non-empty leftmost-ish IP as it the closest to the "real IP".
+
+- Added new `tools/archive` helper subpackage for managing archives (_currently works only with zip_).
+
+- Added new `tools/cron` helper subpackage for scheduling task using cron-like syntax (_this eventually may get exported in the future in a separate repo_).
+
+- Added new `Filesystem.List(prefix)` helper to retrieve a flat list with all files under the provided prefix.
+
+- Added new `App.NewBackupsFilesystem()` helper to create a dedicated filesystem abstraction for managing app data backups.
+
+- Added new `App.OnTerminate()` hook (_executed right before app termination, eg. on `SIGTERM` signal_).
+
+- Added `accept` file field attribute with the field MIME types ([#2466](https://github.com/pocketbase/pocketbase/pull/2466); thanks @Nikhil1920).
+
+- Added support for multiple files sort in the Admin UI ([#2445](https://github.com/pocketbase/pocketbase/issues/2445)).
+
+- Added support for multiple relations sort in the Admin UI.
+
+- Added `meta.isNew` to the OAuth2 auth JSON response to indicate a newly OAuth2 created PocketBase user.
+
+
+## v0.15.3
+
+- Updated the Admin UI to use the latest JS SDK to resolve the `isNew` record field conflict ([#2385](https://github.com/pocketbase/pocketbase/discussions/2385)).
+
+- Fixed `editor` field fullscreen `z-index` ([#2410](https://github.com/pocketbase/pocketbase/issues/2410)).
+
+- Inserts the default app settings as part of the system init migration so that they are always available when accessed from within a user defined migration ([#2423](https://github.com/pocketbase/pocketbase/discussions/2423)).
+
+
+## v0.15.2
+
+- Fixed View query `SELECT DISTINCT` identifiers parsing ([#2349-5706019](https://github.com/pocketbase/pocketbase/discussions/2349#discussioncomment-5706019)).
+
+- Fixed View collection schema incorrectly resolving multiple aliased fields originating from the same field source ([#2349-5707675](https://github.com/pocketbase/pocketbase/discussions/2349#discussioncomment-5707675)).
+
+- Added OAuth2 redirect fallback message to notify the user to go back to the app in case the browser window is not auto closed.
+
+
+## v0.15.1
+
+- Trigger the related `Record` model realtime subscription events on [custom model struct](https://pocketbase.io/docs/custom-models/) save ([#2325](https://github.com/pocketbase/pocketbase/discussions/2325)).
+
+- Fixed `Ctrl + S` in the `editor` field not propagating the quick save shortcut to the parent form.
+
+- Added `⌘ + S` alias for the record quick save shortcut (_I have no Mac device to test it but it should work based on [`e.metaKey` docs](https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/metaKey)_).
+
+- Enabled RTL for the TinyMCE editor ([#2327](https://github.com/pocketbase/pocketbase/issues/2327)).
+
+- Reduced the record form vertical layout shifts and slightly improved the rendering speed when loading multiple `relation` fields.
+
+- Enabled Admin UI assets cache.
+
+
+## v0.15.0
+
+- Simplified the OAuth2 authentication flow in a single "all in one" call ([#55](https://github.com/pocketbase/pocketbase/issues/55)).
+  Requires JS SDK v0.14.0+ or Dart SDK v0.9.0+.
+  The manual code-token exchange flow is still supported but the SDK method is renamed to `authWithOAuth2Code()` (_to minimize the breaking changes the JS SDK has a function overload that will proxy the existing `authWithOauth2` calls to `authWithOAuth2Code`_).
+  For more details and example, you could check https://pocketbase.io/docs/authentication/#oauth2-integration.
+
+- Added support for protected files ([#215](https://github.com/pocketbase/pocketbase/issues/215)).
+  Requires JS SDK v0.14.0+ or Dart SDK v0.9.0+.
+  It works with a short lived (~5min) file token passed as query param with the file url.
+  For more details and example, you could check https://pocketbase.io/docs/files-handling/#protected-files.
+
+- **!** Fixed typo in `Record.WithUnkownData()` -> `Record.WithUnknownData()`.
+
+- Added simple loose wildcard search term support in the Admin UI.
+
+- Added auto "draft" to allow restoring previous record state in case of accidental reload or power outage.
+
+- Added `Ctrl + S` shortcut to save the record changes without closing the panel.
+
+- Added "drop files" support for the file upload field.
+
+- Refreshed the OAuth2 Admin UI.
+
+
+## v0.14.5
+
+- Added checks for `nil` hooks in `forms.RecordUpsert` when used with custom `Dao` ([#2277](https://github.com/pocketbase/pocketbase/issues/2277)).
+
+- Fixed unique detailed field error not returned on record create failure ([#2287](https://github.com/pocketbase/pocketbase/discussions/2287)).
+
+
+## v0.14.4
+
+- Fixed concurrent map write pannic on `list.ExistInSliceWithRegex()` cache ([#2272](https://github.com/pocketbase/pocketbase/issues/2272)).
+
+
+## v0.14.3
+
+- Fixed Admin UI Logs `meta` visualization in Firefox ([#2221](https://github.com/pocketbase/pocketbase/issues/2221)).
+
+- Downgraded to v1 of the `aws/aws-sdk-go` package since v2 has compatibility issues with GCS ([#2231](https://github.com/pocketbase/pocketbase/issues/2231)).
+
+- Upgraded the GitHub action to use [min Go 1.20.3](https://github.com/golang/go/issues?q=milestone%3AGo1.20.3+label%3ACherryPickApproved) for the prebuilt executable since it contains some minor `net/http` security fixes.
+
+
+## v0.14.2
+
+- Reverted part of the old `COALESCE` handling as a fallback to support empty string comparison with missing joined relation fields.
+
+
+## v0.14.1
+
+- Fixed realtime events firing before the files upload completion.
+
+- Updated the underlying S3 lib to use `aws-sdk-go-v2` ([#1346](https://github.com/pocketbase/pocketbase/pull/1346); thanks @yuxiang-gao).
+
+- Updated TinyMCE to v6.4.1.
+
+- Updated the godoc of `Dao.Save*` methods.
+
+
 ## v0.14.0
 
 - Added _experimental_ Apple OAuth2 integration.
@@ -1134,7 +1355,7 @@ Please check the individual SDK package changelog and apply the necessary change
 - Added option to return serialized custom `models.Record` fields data:
   ```go
   func (m *Record) UnknownData() map[string]any
-  func (m *Record) WithUnkownData(state bool)
+  func (m *Record) WithUnknownData(state bool)
   ```
 
 - Deleted `model.User`. Now the user data is stored as an auth `models.Record`.

apis/backup.go

@@ -0,0 +1,192 @@
+package apis
+
+import (
+	"context"
+	"log"
+	"net/http"
+	"net/url"
+	"path/filepath"
+	"time"
+
+	"github.com/labstack/echo/v5"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/forms"
+	"github.com/pocketbase/pocketbase/models"
+	"github.com/pocketbase/pocketbase/tools/types"
+	"github.com/spf13/cast"
+)
+
+// bindBackupApi registers the file api endpoints and the corresponding handlers.
+//
+// @todo add hooks once the app hooks api restructuring is finalized
+func bindBackupApi(app core.App, rg *echo.Group) {
+	api := backupApi{app: app}
+
+	subGroup := rg.Group("/backups", ActivityLogger(app))
+	subGroup.GET("", api.list, RequireAdminAuth())
+	subGroup.POST("", api.create, RequireAdminAuth())
+	subGroup.GET("/:key", api.download)
+	subGroup.DELETE("/:key", api.delete, RequireAdminAuth())
+	subGroup.POST("/:key/restore", api.restore, RequireAdminAuth())
+}
+
+type backupApi struct {
+	app core.App
+}
+
+func (api *backupApi) list(c echo.Context) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	fsys, err := api.app.NewBackupsFilesystem()
+	if err != nil {
+		return NewBadRequestError("Failed to load backups filesystem.", err)
+	}
+	defer fsys.Close()
+
+	fsys.SetContext(ctx)
+
+	backups, err := fsys.List("")
+	if err != nil {
+		return NewBadRequestError("Failed to retrieve backup items. Raw error: \n"+err.Error(), nil)
+	}
+
+	result := make([]models.BackupFileInfo, len(backups))
+
+	for i, obj := range backups {
+		modified, _ := types.ParseDateTime(obj.ModTime)
+
+		result[i] = models.BackupFileInfo{
+			Key:      obj.Key,
+			Size:     obj.Size,
+			Modified: modified,
+		}
+	}
+
+	return c.JSON(http.StatusOK, result)
+}
+
+func (api *backupApi) create(c echo.Context) error {
+	if api.app.Cache().Has(core.CacheKeyActiveBackup) {
+		return NewBadRequestError("Try again later - another backup/restore process has already been started", nil)
+	}
+
+	form := forms.NewBackupCreate(api.app)
+	if err := c.Bind(form); err != nil {
+		return NewBadRequestError("An error occurred while loading the submitted data.", err)
+	}
+
+	return form.Submit(func(next forms.InterceptorNextFunc[string]) forms.InterceptorNextFunc[string] {
+		return func(name string) error {
+			if err := next(name); err != nil {
+				return NewBadRequestError("Failed to create backup.", err)
+			}
+
+			// we don't retrieve the generated backup file because it may not be
+			// available yet due to the eventually consistent nature of some S3 providers
+			return c.NoContent(http.StatusNoContent)
+		}
+	})
+}
+
+func (api *backupApi) download(c echo.Context) error {
+	fileToken := c.QueryParam("token")
+
+	_, err := api.app.Dao().FindAdminByToken(
+		fileToken,
+		api.app.Settings().AdminFileToken.Secret,
+	)
+	if err != nil {
+		return NewForbiddenError("Insufficient permissions to access the resource.", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+
+	fsys, err := api.app.NewBackupsFilesystem()
+	if err != nil {
+		return NewBadRequestError("Failed to load backups filesystem.", err)
+	}
+	defer fsys.Close()
+
+	fsys.SetContext(ctx)
+
+	key := c.PathParam("key")
+
+	br, err := fsys.GetFile(key)
+	if err != nil {
+		return NewBadRequestError("Failed to retrieve backup item. Raw error: \n"+err.Error(), nil)
+	}
+	defer br.Close()
+
+	return fsys.Serve(
+		c.Response(),
+		c.Request(),
+		key,
+		filepath.Base(key), // without the path prefix (if any)
+	)
+}
+
+func (api *backupApi) restore(c echo.Context) error {
+	if api.app.Cache().Has(core.CacheKeyActiveBackup) {
+		return NewBadRequestError("Try again later - another backup/restore process has already been started.", nil)
+	}
+
+	// @todo remove the extra unescape after https://github.com/labstack/echo/issues/2447
+	key, _ := url.PathUnescape(c.PathParam("key"))
+
+	existsCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	fsys, err := api.app.NewBackupsFilesystem()
+	if err != nil {
+		return NewBadRequestError("Failed to load backups filesystem.", err)
+	}
+	defer fsys.Close()
+
+	fsys.SetContext(existsCtx)
+
+	if exists, err := fsys.Exists(key); !exists {
+		return NewBadRequestError("Missing or invalid backup file.", err)
+	}
+
+	go func() {
+		// wait max 15 minutes to fetch the backup
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+		defer cancel()
+
+		// give some optimistic time to write the response
+		time.Sleep(1 * time.Second)
+
+		if err := api.app.RestoreBackup(ctx, key); err != nil && api.app.IsDebug() {
+			log.Println(err)
+		}
+	}()
+
+	return c.NoContent(http.StatusNoContent)
+}
+
+func (api *backupApi) delete(c echo.Context) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	fsys, err := api.app.NewBackupsFilesystem()
+	if err != nil {
+		return NewBadRequestError("Failed to load backups filesystem.", err)
+	}
+	defer fsys.Close()
+
+	fsys.SetContext(ctx)
+
+	key := c.PathParam("key")
+
+	if key != "" && cast.ToString(api.app.Cache().Get(core.CacheKeyActiveBackup)) == key {
+		return NewBadRequestError("The backup is currently being used and cannot be deleted.", nil)
+	}
+
+	if err := fsys.Delete(key); err != nil {
+		return NewBadRequestError("Invalid or already deleted backup file. Raw error: \n"+err.Error(), nil)
+	}
+
+	return c.NoContent(http.StatusNoContent)
+}

apis/backup_test.go

@@ -0,0 +1,571 @@
+package apis_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/labstack/echo/v5"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tests"
+	"gocloud.dev/blob"
+)
+
+func TestBackupsList(t *testing.T) {
+	scenarios := []tests.ApiScenario{
+		{
+			Name:   "unauthorized",
+			Method: http.MethodGet,
+			Url:    "/api/backups",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  401,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as auth record",
+			Method: http.MethodGet,
+			Url:    "/api/backups",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsInR5cGUiOiJhdXRoUmVjb3JkIiwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwiZXhwIjoyMjA4OTg1MjYxfQ.UwD8JvkbQtXpymT09d7J6fdA0aP9g4FJ1GPh_ggEkzc",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  401,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as admin (empty list)",
+			Method: http.MethodGet,
+			Url:    "/api/backups",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			ExpectedStatus: 200,
+			ExpectedContent: []string{
+				`[]`,
+			},
+		},
+		{
+			Name:   "authorized as admin",
+			Method: http.MethodGet,
+			Url:    "/api/backups",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus: 200,
+			ExpectedContent: []string{
+				`"test1.zip"`,
+				`"test2.zip"`,
+				`"test3.zip"`,
+			},
+		},
+	}
+
+	for _, scenario := range scenarios {
+		scenario.Test(t)
+	}
+}
+
+func TestBackupsCreate(t *testing.T) {
+	scenarios := []tests.ApiScenario{
+		{
+			Name:   "unauthorized",
+			Method: http.MethodPost,
+			Url:    "/api/backups",
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				ensureNoBackups(t, app)
+			},
+			ExpectedStatus:  401,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as auth record",
+			Method: http.MethodPost,
+			Url:    "/api/backups",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsInR5cGUiOiJhdXRoUmVjb3JkIiwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwiZXhwIjoyMjA4OTg1MjYxfQ.UwD8JvkbQtXpymT09d7J6fdA0aP9g4FJ1GPh_ggEkzc",
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				ensureNoBackups(t, app)
+			},
+			ExpectedStatus:  401,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as admin (pending backup)",
+			Method: http.MethodPost,
+			Url:    "/api/backups",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				app.Cache().Set(core.CacheKeyActiveBackup, "")
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				ensureNoBackups(t, app)
+			},
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as admin (autogenerated name)",
+			Method: http.MethodPost,
+			Url:    "/api/backups",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				files, err := getBackupFiles(app)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if total := len(files); total != 1 {
+					t.Fatalf("Expected 1 backup file, got %d", total)
+				}
+
+				expected := "pb_backup_"
+				if !strings.HasPrefix(files[0].Key, expected) {
+					t.Fatalf("Expected backup file with prefix %q, got %q", expected, files[0].Key)
+				}
+			},
+			ExpectedStatus: 204,
+		},
+		{
+			Name:   "authorized as admin (invalid name)",
+			Method: http.MethodPost,
+			Url:    "/api/backups",
+			Body:   strings.NewReader(`{"name":"!test.zip"}`),
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				ensureNoBackups(t, app)
+			},
+			ExpectedStatus: 400,
+			ExpectedContent: []string{
+				`"data":{`,
+				`"name":{"code":"validation_match_invalid"`,
+			},
+		},
+		{
+			Name:   "authorized as admin (valid name)",
+			Method: http.MethodPost,
+			Url:    "/api/backups",
+			Body:   strings.NewReader(`{"name":"test.zip"}`),
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				files, err := getBackupFiles(app)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if total := len(files); total != 1 {
+					t.Fatalf("Expected 1 backup file, got %d", total)
+				}
+
+				expected := "test.zip"
+				if files[0].Key != expected {
+					t.Fatalf("Expected backup file %q, got %q", expected, files[0].Key)
+				}
+			},
+			ExpectedStatus: 204,
+		},
+	}
+
+	for _, scenario := range scenarios {
+		scenario.Test(t)
+	}
+}
+
+func TestBackupsDownload(t *testing.T) {
+	scenarios := []tests.ApiScenario{
+		{
+			Name:   "unauthorized",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with record auth header",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsInR5cGUiOiJhdXRoUmVjb3JkIiwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwiZXhwIjoyMjA4OTg1MjYxfQ.UwD8JvkbQtXpymT09d7J6fdA0aP9g4FJ1GPh_ggEkzc",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with admin auth header",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with empty or invalid token",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip?token=",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with valid record auth token",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip?token=eyJhbGciOiJIUzI1NiJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsInR5cGUiOiJhdXRoUmVjb3JkIiwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwiZXhwIjoyMjA4OTg1MjYxfQ.UwD8JvkbQtXpymT09d7J6fdA0aP9g4FJ1GPh_ggEkzc",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with valid record file token",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsImV4cCI6MTg5MzQ1MjQ2MSwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwidHlwZSI6ImF1dGhSZWNvcmQifQ.0d_0EO6kfn9ijZIQWAqgRi8Bo1z7MKcg1LQpXhQsEPk",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with valid admin auth token",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with expired admin file token",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsImV4cCI6MTY0MDk5MTY2MSwidHlwZSI6ImFkbWluIn0.g7Q_3UX6H--JWJ7yt1Hoe-1ugTX1KpbKzdt0zjGSe-E",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with valid admin file token but missing backup name",
+			Method: http.MethodGet,
+			Url:    "/api/backups/mizzing?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsImV4cCI6MTg5MzQ1MjQ2MSwidHlwZSI6ImFkbWluIn0.LyAMpSfaHVsuUqIlqqEbhDQSdFzoPz_EIDcb2VJMBsU",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "with valid admin file token",
+			Method: http.MethodGet,
+			Url:    "/api/backups/test1.zip?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsImV4cCI6MTg5MzQ1MjQ2MSwidHlwZSI6ImFkbWluIn0.LyAMpSfaHVsuUqIlqqEbhDQSdFzoPz_EIDcb2VJMBsU",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus: 200,
+			ExpectedContent: []string{
+				`storage/`,
+				`data.db`,
+				`logs.db`,
+			},
+		},
+	}
+
+	for _, scenario := range scenarios {
+		scenario.Test(t)
+	}
+}
+
+func TestBackupsDelete(t *testing.T) {
+	noTestBackupFilesChanges := func(t *testing.T, app *tests.TestApp) {
+		files, err := getBackupFiles(app)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expected := 3
+		if total := len(files); total != expected {
+			t.Fatalf("Expected %d backup(s), got %d", expected, total)
+		}
+	}
+
+	scenarios := []tests.ApiScenario{
+		{
+			Name:   "unauthorized",
+			Method: http.MethodDelete,
+			Url:    "/api/backups/test1.zip",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				noTestBackupFilesChanges(t, app)
+			},
+			ExpectedStatus:  401,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as auth record",
+			Method: http.MethodDelete,
+			Url:    "/api/backups/test1.zip",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsInR5cGUiOiJhdXRoUmVjb3JkIiwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwiZXhwIjoyMjA4OTg1MjYxfQ.UwD8JvkbQtXpymT09d7J6fdA0aP9g4FJ1GPh_ggEkzc",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				noTestBackupFilesChanges(t, app)
+			},
+			ExpectedStatus:  401,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as admin (missing file)",
+			Method: http.MethodDelete,
+			Url:    "/api/backups/missing.zip",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				noTestBackupFilesChanges(t, app)
+			},
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as admin (existing file with matching active backup)",
+			Method: http.MethodDelete,
+			Url:    "/api/backups/test1.zip",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+
+				// mock active backup with the same name to delete
+				app.Cache().Set(core.CacheKeyActiveBackup, "test1.zip")
+
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				noTestBackupFilesChanges(t, app)
+			},
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as admin (existing file and no matching active backup)",
+			Method: http.MethodDelete,
+			Url:    "/api/backups/test1.zip",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+
+				// mock active backup with different name
+				app.Cache().Set(core.CacheKeyActiveBackup, "new.zip")
+			},
+			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				files, err := getBackupFiles(app)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if total := len(files); total != 2 {
+					t.Fatalf("Expected 2 backup files, got %d", total)
+				}
+
+				deletedFile := "test1.zip"
+
+				for _, f := range files {
+					if f.Key == deletedFile {
+						t.Fatalf("Expected backup %q to be deleted", deletedFile)
+					}
+				}
+			},
+			ExpectedStatus: 204,
+		},
+	}
+
+	for _, scenario := range scenarios {
+		scenario.Test(t)
+	}
+}
+
+func TestBackupsRestore(t *testing.T) {
+	scenarios := []tests.ApiScenario{
+		{
+			Name:   "unauthorized",
+			Method: http.MethodPost,
+			Url:    "/api/backups/test1.zip/restore",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  401,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as auth record",
+			Method: http.MethodPost,
+			Url:    "/api/backups/test1.zip/restore",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsInR5cGUiOiJhdXRoUmVjb3JkIiwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwiZXhwIjoyMjA4OTg1MjYxfQ.UwD8JvkbQtXpymT09d7J6fdA0aP9g4FJ1GPh_ggEkzc",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  401,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as admin (missing file)",
+			Method: http.MethodPost,
+			Url:    "/api/backups/missing.zip/restore",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+			},
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "authorized as admin (active backup process)",
+			Method: http.MethodPost,
+			Url:    "/api/backups/test1.zip/restore",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				if err := createTestBackups(app); err != nil {
+					t.Fatal(err)
+				}
+
+				app.Cache().Set(core.CacheKeyActiveBackup, "")
+			},
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+		},
+	}
+
+	for _, scenario := range scenarios {
+		scenario.Test(t)
+	}
+}
+
+// -------------------------------------------------------------------
+
+func createTestBackups(app core.App) error {
+	ctx := context.Background()
+
+	if err := app.CreateBackup(ctx, "test1.zip"); err != nil {
+		return err
+	}
+
+	if err := app.CreateBackup(ctx, "test2.zip"); err != nil {
+		return err
+	}
+
+	if err := app.CreateBackup(ctx, "test3.zip"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getBackupFiles(app core.App) ([]*blob.ListObject, error) {
+	fsys, err := app.NewBackupsFilesystem()
+	if err != nil {
+		return nil, err
+	}
+	defer fsys.Close()
+
+	return fsys.List("")
+}
+
+func ensureNoBackups(t *testing.T, app *tests.TestApp) {
+	files, err := getBackupFiles(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if total := len(files); total != 0 {
+		t.Fatalf("Expected 0 backup files, got %d", total)
+	}
+}

apis/base.go

@@ -14,6 +14,7 @@ import (
 	"github.com/labstack/echo/v5"
 	"github.com/labstack/echo/v5/middleware"
 	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tools/rest"
 	"github.com/pocketbase/pocketbase/ui"
 	"github.com/spf13/cast"
 )
@@ -25,6 +26,9 @@ const trailedAdminPath = "/_/"
 func InitApi(app core.App) (*echo.Echo, error) {
 	e := echo.New()
 	e.Debug = app.IsDebug()
+	e.JSONSerializer = &rest.Serializer{
+		FieldsParam: "fields",
+	}
 
 	// configure a custom router
 	e.ResetRouterCreator(func(ec *echo.Echo) echo.Router {
@@ -110,6 +114,7 @@ func InitApi(app core.App) (*echo.Echo, error) {
 	bindRealtimeApi(app, api)
 	bindLogsApi(app, api)
 	bindHealthApi(app, api)
+	bindBackupApi(app, api)
 
 	// trigger the custom BeforeServe hook for the created api router
 	// allowing users to further adjust its options or register new routes
@@ -121,6 +126,10 @@ func InitApi(app core.App) (*echo.Echo, error) {
 		return nil, err
 	}
 
+	// note: it is after the OnBeforeServe hook to ensure that the implicit
+	// cache is after any user custom defined middlewares
+	e.Use(eagerRequestDataCache(app))
+
 	// catch all any route
 	api.Any("/*", func(c echo.Context) error {
 		return echo.ErrNotFound
@@ -176,13 +185,14 @@ func bindStaticAdminUI(app core.App, e *echo.Echo) error {
 		trailedAdminPath+"*",
 		echo.StaticDirectoryHandler(ui.DistDirFS, false),
 		installerRedirect(app),
+		uiCacheControl(),
 		middleware.Gzip(),
 	)
 
 	return nil
 }
 
-const totalAdminsCacheKey = "totalAdmins"
+const totalAdminsCacheKey = "@totalAdmins"
 
 func updateTotalAdminsCache(app core.App) error {
 	total, err := app.Dao().TotalAdmins()
@@ -195,6 +205,20 @@ func updateTotalAdminsCache(app core.App) error {
 	return nil
 }
 
+func uiCacheControl() echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			// add default Cache-Control header for all Admin UI resources
+			// (ignoring the root admin path)
+			if c.Request().URL.Path != trailedAdminPath {
+				c.Response().Header().Set("Cache-Control", "max-age=1209600, stale-while-revalidate=86400")
+			}
+
+			return next(c)
+		}
+	}
+}
+
 // installerRedirect redirects the user to the installer admin UI page
 // when the application needs some preliminary configurations to be done.
 func installerRedirect(app core.App) echo.MiddlewareFunc {

apis/base_test.go

@@ -2,12 +2,15 @@ package apis_test
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/labstack/echo/v5"
 	"github.com/pocketbase/pocketbase/apis"
 	"github.com/pocketbase/pocketbase/tests"
+	"github.com/spf13/cast"
 )
 
 func Test404(t *testing.T) {
@@ -209,3 +212,83 @@ func TestRemoveTrailingSlashMiddleware(t *testing.T) {
 		scenario.Test(t)
 	}
 }
+
+func TestEagerRequestDataCache(t *testing.T) {
+	scenarios := []tests.ApiScenario{
+		{
+			Name:   "[UNKNOWN] unsupported eager cached request method",
+			Method: "UNKNOWN",
+			Url:    "/custom",
+			Body:   strings.NewReader(`{"name":"test123"}`),
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				e.AddRoute(echo.Route{
+					Method: "UNKNOWN",
+					Path:   "/custom",
+					Handler: func(c echo.Context) error {
+						data := &struct {
+							Name string `json:"name"`
+						}{}
+
+						if err := c.Bind(data); err != nil {
+							return err
+						}
+
+						// since the unknown method is not eager cache support
+						// it should fail reading the json body twice
+						r := apis.RequestData(c)
+						if v := cast.ToString(r.Data["name"]); v != "" {
+							t.Fatalf("Expected empty request data body, got, %v", r.Data)
+						}
+
+						return c.String(200, data.Name)
+					},
+				})
+			},
+			ExpectedStatus:  200,
+			ExpectedContent: []string{"test123"},
+		},
+	}
+
+	// supported eager cache request methods
+	supportedMethods := []string{"POST", "PUT", "PATCH", "DELETE"}
+	for _, m := range supportedMethods {
+		scenarios = append(
+			scenarios,
+			tests.ApiScenario{
+				Name:   fmt.Sprintf("[%s] valid cached json body request", m),
+				Method: http.MethodPost,
+				Url:    "/custom",
+				Body:   strings.NewReader(`{"name":"test123"}`),
+				BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+					e.AddRoute(echo.Route{
+						Method: http.MethodPost,
+						Path:   "/custom",
+						Handler: func(c echo.Context) error {
+							data := &struct {
+								Name string `json:"name"`
+							}{}
+
+							if err := c.Bind(data); err != nil {
+								return err
+							}
+
+							// try to read the body again
+							r := apis.RequestData(c)
+							if v := cast.ToString(r.Data["name"]); v != "test123" {
+								t.Fatalf("Expected request data with name %q, got, %q", "test123", v)
+							}
+
+							return c.String(200, data.Name)
+						},
+					})
+				},
+				ExpectedStatus:  200,
+				ExpectedContent: []string{"test123"},
+			},
+		)
+	}
+
+	for _, scenario := range scenarios {
+		scenario.Test(t)
+	}
+}

apis/collection_test.go

@@ -1097,7 +1097,7 @@ func TestCollectionUpdate(t *testing.T) {
 	}
 }
 
-func TestCollectionImport(t *testing.T) {
+func TestCollectionsImport(t *testing.T) {
 	totalCollections := 10
 
 	scenarios := []tests.ApiScenario{
@@ -1157,7 +1157,7 @@ func TestCollectionImport(t *testing.T) {
 			},
 			ExpectedEvents: map[string]int{
 				"OnCollectionsBeforeImportRequest": 1,
-				"OnModelBeforeDelete":              7,
+				"OnModelBeforeDelete":              4,
 			},
 			AfterTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
 				collections := []*models.Collection{}

apis/file.go

@@ -1,13 +1,23 @@
 package apis
 
 import (
+	"errors"
 	"fmt"
+	"log"
+	"net/http"
+	"strings"
 
 	"github.com/labstack/echo/v5"
+	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/core"
 	"github.com/pocketbase/pocketbase/models"
 	"github.com/pocketbase/pocketbase/models/schema"
+	"github.com/pocketbase/pocketbase/resolvers"
+	"github.com/pocketbase/pocketbase/tokens"
 	"github.com/pocketbase/pocketbase/tools/list"
+	"github.com/pocketbase/pocketbase/tools/search"
+	"github.com/pocketbase/pocketbase/tools/security"
+	"github.com/spf13/cast"
 )
 
 var imageContentTypes = []string{"image/png", "image/jpg", "image/jpeg", "image/gif"}
@@ -18,6 +28,7 @@ func bindFileApi(app core.App, rg *echo.Group) {
 	api := fileApi{app: app}
 
 	subGroup := rg.Group("/files", ActivityLogger(app))
+	subGroup.POST("/token", api.fileToken)
 	subGroup.HEAD("/:collection/:recordId/:filename", api.download, LoadCollectionContext(api.app))
 	subGroup.GET("/:collection/:recordId/:filename", api.download, LoadCollectionContext(api.app))
 }
@@ -26,6 +37,37 @@ type fileApi struct {
 	app core.App
 }
 
+func (api *fileApi) fileToken(c echo.Context) error {
+	event := new(core.FileTokenEvent)
+	event.HttpContext = c
+
+	if admin, _ := c.Get(ContextAdminKey).(*models.Admin); admin != nil {
+		event.Model = admin
+		event.Token, _ = tokens.NewAdminFileToken(api.app, admin)
+	} else if record, _ := c.Get(ContextAuthRecordKey).(*models.Record); record != nil {
+		event.Model = record
+		event.Token, _ = tokens.NewRecordFileToken(api.app, record)
+	}
+
+	handlerErr := api.app.OnFileBeforeTokenRequest().Trigger(event, func(e *core.FileTokenEvent) error {
+		if e.Model == nil || e.Token == "" {
+			return NewBadRequestError("Failed to generate file token.", nil)
+		}
+
+		return e.HttpContext.JSON(http.StatusOK, map[string]string{
+			"token": e.Token,
+		})
+	})
+
+	if handlerErr == nil {
+		if err := api.app.OnFileAfterTokenRequest().Trigger(event); err != nil && api.app.IsDebug() {
+			log.Println(err)
+		}
+	}
+
+	return handlerErr
+}
+
 func (api *fileApi) download(c echo.Context) error {
 	collection, _ := c.Get(ContextCollectionKey).(*models.Collection)
 	if collection == nil {
@@ -49,7 +91,21 @@ func (api *fileApi) download(c echo.Context) error {
 		return NewNotFoundError("", nil)
 	}
 
-	options, _ := fileField.Options.(*schema.FileOptions)
+	options, ok := fileField.Options.(*schema.FileOptions)
+	if !ok {
+		return NewBadRequestError("", errors.New("Failed to load file options."))
+	}
+
+	// check whether the request is authorized to view the protected file
+	if options.Protected {
+		token := c.QueryParam("token")
+
+		adminOrAuthRecord, _ := api.findAdminOrAuthRecordByFileToken(token)
+
+		if !api.canAccessRecord(adminOrAuthRecord, record, record.Collection().ViewRule) {
+			return NewForbiddenError("Insufficient permissions to access the file resource.", nil)
+		}
+	}
 
 	baseFilesPath := record.BaseFilesPath()
 
@@ -87,11 +143,8 @@ func (api *fileApi) download(c echo.Context) error {
 			servedName = thumbSize + "_" + filename
 			servedPath = baseFilesPath + "/thumbs_" + filename + "/" + servedName
 
-			// check if the thumb exists:
-			// - if doesn't exist - create a new thumb with the specified thumb size
-			// - if exists - compare last modified dates to determine whether the thumb should be recreated
-			tAttrs, tAttrsErr := fs.Attributes(servedPath)
-			if tAttrsErr != nil || oAttrs.ModTime.After(tAttrs.ModTime) {
+			// create a new thumb if it doesn exists
+			if exists, _ := fs.Exists(servedPath); !exists {
 				if err := fs.CreateThumb(originalPath, servedPath, thumbSize); err != nil {
 					servedPath = originalPath // fallback to the original
 				}
@@ -122,3 +175,77 @@ func (api *fileApi) download(c echo.Context) error {
 		return nil
 	})
 }
+
+func (api *fileApi) findAdminOrAuthRecordByFileToken(fileToken string) (models.Model, error) {
+	fileToken = strings.TrimSpace(fileToken)
+	if fileToken == "" {
+		return nil, errors.New("missing file token")
+	}
+
+	claims, _ := security.ParseUnverifiedJWT(strings.TrimSpace(fileToken))
+	tokenType := cast.ToString(claims["type"])
+
+	switch tokenType {
+	case tokens.TypeAdmin:
+		admin, err := api.app.Dao().FindAdminByToken(
+			fileToken,
+			api.app.Settings().AdminFileToken.Secret,
+		)
+		if err == nil && admin != nil {
+			return admin, nil
+		}
+	case tokens.TypeAuthRecord:
+		record, err := api.app.Dao().FindAuthRecordByToken(
+			fileToken,
+			api.app.Settings().RecordFileToken.Secret,
+		)
+		if err == nil && record != nil {
+			return record, nil
+		}
+	}
+
+	return nil, errors.New("missing or invalid file token")
+}
+
+// @todo move to a helper and maybe combine with the realtime checks when refactoring the realtime service
+func (api *fileApi) canAccessRecord(adminOrAuthRecord models.Model, record *models.Record, accessRule *string) bool {
+	admin, _ := adminOrAuthRecord.(*models.Admin)
+	if admin != nil {
+		// admins can access everything
+		return true
+	}
+
+	if accessRule == nil {
+		// only admins can access this record
+		return false
+	}
+
+	ruleFunc := func(q *dbx.SelectQuery) error {
+		if *accessRule == "" {
+			return nil // empty public rule
+		}
+
+		// mock request data
+		requestData := &models.RequestData{
+			Method: "GET",
+		}
+		requestData.AuthRecord, _ = adminOrAuthRecord.(*models.Record)
+
+		resolver := resolvers.NewRecordFieldResolver(api.app.Dao(), record.Collection(), requestData, true)
+		expr, err := search.FilterData(*accessRule).BuildExpr(resolver)
+		if err != nil {
+			return err
+		}
+		resolver.UpdateQuery(q)
+		q.AndWhere(expr)
+
+		return nil
+	}
+
+	foundRecord, err := api.app.Dao().FindRecordById(record.Collection().Id, record.Id, ruleFunc)
+	if err == nil && foundRecord != nil {
+		return true
+	}
+
+	return false
+}

apis/file_test.go

@@ -8,9 +8,85 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/labstack/echo/v5"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/daos"
 	"github.com/pocketbase/pocketbase/tests"
+	"github.com/pocketbase/pocketbase/tools/types"
 )
 
+func TestFileToken(t *testing.T) {
+	scenarios := []tests.ApiScenario{
+		{
+			Name:            "unauthorized",
+			Method:          http.MethodPost,
+			Url:             "/api/files/token",
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+			ExpectedEvents: map[string]int{
+				"OnFileBeforeTokenRequest": 1,
+			},
+		},
+		{
+			Name:   "unauthorized with model and token via hook",
+			Method: http.MethodPost,
+			Url:    "/api/files/token",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				app.OnFileBeforeTokenRequest().Add(func(e *core.FileTokenEvent) error {
+					record, _ := app.Dao().FindAuthRecordByEmail("users", "test@example.com")
+					e.Model = record
+					e.Token = "test"
+					return nil
+				})
+			},
+			ExpectedStatus: 200,
+			ExpectedContent: []string{
+				`"token":"test"`,
+			},
+			ExpectedEvents: map[string]int{
+				"OnFileBeforeTokenRequest": 1,
+				"OnFileAfterTokenRequest":  1,
+			},
+		},
+		{
+			Name:   "auth record",
+			Method: http.MethodPost,
+			Url:    "/api/files/token",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsInR5cGUiOiJhdXRoUmVjb3JkIiwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwiZXhwIjoyMjA4OTg1MjYxfQ.UwD8JvkbQtXpymT09d7J6fdA0aP9g4FJ1GPh_ggEkzc",
+			},
+			ExpectedStatus: 200,
+			ExpectedContent: []string{
+				`"token":"`,
+			},
+			ExpectedEvents: map[string]int{
+				"OnFileBeforeTokenRequest": 1,
+				"OnFileAfterTokenRequest":  1,
+			},
+		},
+		{
+			Name:   "admin",
+			Method: http.MethodPost,
+			Url:    "/api/files/token",
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			ExpectedStatus: 200,
+			ExpectedContent: []string{
+				`"token":"`,
+			},
+			ExpectedEvents: map[string]int{
+				"OnFileBeforeTokenRequest": 1,
+				"OnFileAfterTokenRequest":  1,
+			},
+		},
+	}
+
+	for _, scenario := range scenarios {
+		scenario.Test(t)
+	}
+}
+
 func TestFileDownload(t *testing.T) {
 	_, currentFile, _, _ := runtime.Caller(0)
 	dataDirRelPath := "../tests/data/"
@@ -176,6 +252,125 @@ func TestFileDownload(t *testing.T) {
 				"OnFileDownloadRequest": 1,
 			},
 		},
+
+		// protected file access checks
+		{
+			Name:            "protected file - expired token",
+			Method:          http.MethodGet,
+			Url:             "/api/files/_pb_users_auth_/oap640cot4yru2s/test_kfd2wYLxkz.txt?thumb=100x100",
+			ExpectedStatus:  200,
+			ExpectedContent: []string{string(testFile)},
+			ExpectedEvents: map[string]int{
+				"OnFileDownloadRequest": 1,
+			},
+		},
+		{
+			Name:            "protected file - admin with expired file token",
+			Method:          http.MethodGet,
+			Url:             "/api/files/demo1/al1h9ijdeojtsjy/300_Jsjq7RdBgA.png?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsImV4cCI6MTY0MDk5MTY2MSwidHlwZSI6ImFkbWluIn0.g7Q_3UX6H--JWJ7yt1Hoe-1ugTX1KpbKzdt0zjGSe-E",
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:            "protected file - admin with valid file token",
+			Method:          http.MethodGet,
+			Url:             "/api/files/demo1/al1h9ijdeojtsjy/300_Jsjq7RdBgA.png?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsImV4cCI6MTg5MzQ1MjQ2MSwidHlwZSI6ImFkbWluIn0.LyAMpSfaHVsuUqIlqqEbhDQSdFzoPz_EIDcb2VJMBsU",
+			ExpectedStatus:  200,
+			ExpectedContent: []string{"PNG"},
+			ExpectedEvents: map[string]int{
+				"OnFileDownloadRequest": 1,
+			},
+		},
+		{
+			Name:            "protected file - guest without view access",
+			Method:          http.MethodGet,
+			Url:             "/api/files/demo1/al1h9ijdeojtsjy/300_Jsjq7RdBgA.png",
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "protected file - guest with view access",
+			Method: http.MethodGet,
+			Url:    "/api/files/demo1/al1h9ijdeojtsjy/300_Jsjq7RdBgA.png",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				dao := daos.New(app.Dao().DB())
+
+				// mock public view access
+				c, err := dao.FindCollectionByNameOrId("demo1")
+				if err != nil {
+					t.Fatalf("Failed to fetch mock collection: %v", err)
+				}
+				c.ViewRule = types.Pointer("")
+				if err := dao.SaveCollection(c); err != nil {
+					t.Fatalf("Failed to update mock collection: %v", err)
+				}
+			},
+			ExpectedStatus:  200,
+			ExpectedContent: []string{"PNG"},
+			ExpectedEvents: map[string]int{
+				"OnFileDownloadRequest": 1,
+			},
+		},
+		{
+			Name:   "protected file - auth record without view access",
+			Method: http.MethodGet,
+			Url:    "/api/files/demo1/al1h9ijdeojtsjy/300_Jsjq7RdBgA.png?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsImV4cCI6MTg5MzQ1MjQ2MSwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwidHlwZSI6ImF1dGhSZWNvcmQifQ.0d_0EO6kfn9ijZIQWAqgRi8Bo1z7MKcg1LQpXhQsEPk",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				dao := daos.New(app.Dao().DB())
+
+				// mock restricted user view access
+				c, err := dao.FindCollectionByNameOrId("demo1")
+				if err != nil {
+					t.Fatalf("Failed to fetch mock collection: %v", err)
+				}
+				c.ViewRule = types.Pointer("@request.auth.verified = true")
+				if err := dao.SaveCollection(c); err != nil {
+					t.Fatalf("Failed to update mock collection: %v", err)
+				}
+			},
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "protected file - auth record with view access",
+			Method: http.MethodGet,
+			Url:    "/api/files/demo1/al1h9ijdeojtsjy/300_Jsjq7RdBgA.png?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsImV4cCI6MTg5MzQ1MjQ2MSwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwidHlwZSI6ImF1dGhSZWNvcmQifQ.0d_0EO6kfn9ijZIQWAqgRi8Bo1z7MKcg1LQpXhQsEPk",
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				dao := daos.New(app.Dao().DB())
+
+				// mock user view access
+				c, err := dao.FindCollectionByNameOrId("demo1")
+				if err != nil {
+					t.Fatalf("Failed to fetch mock collection: %v", err)
+				}
+				c.ViewRule = types.Pointer("@request.auth.verified = false")
+				if err := dao.SaveCollection(c); err != nil {
+					t.Fatalf("Failed to update mock collection: %v", err)
+				}
+			},
+			ExpectedStatus:  200,
+			ExpectedContent: []string{"PNG"},
+			ExpectedEvents: map[string]int{
+				"OnFileDownloadRequest": 1,
+			},
+		},
+		{
+			Name:            "protected file in view (view's View API rule failure)",
+			Method:          http.MethodGet,
+			Url:             "/api/files/view1/al1h9ijdeojtsjy/300_Jsjq7RdBgA.png?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsImV4cCI6MTg5MzQ1MjQ2MSwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwidHlwZSI6ImF1dGhSZWNvcmQifQ.0d_0EO6kfn9ijZIQWAqgRi8Bo1z7MKcg1LQpXhQsEPk",
+			ExpectedStatus:  403,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:            "protected file in view (view's View API rule success)",
+			Method:          http.MethodGet,
+			Url:             "/api/files/view1/84nmscqy84lsi1t/test_d61b33QdDU.txt?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjRxMXhsY2xtZmxva3UzMyIsImV4cCI6MTg5MzQ1MjQ2MSwiY29sbGVjdGlvbklkIjoiX3BiX3VzZXJzX2F1dGhfIiwidHlwZSI6ImF1dGhSZWNvcmQifQ.0d_0EO6kfn9ijZIQWAqgRi8Bo1z7MKcg1LQpXhQsEPk",
+			ExpectedStatus:  200,
+			ExpectedContent: []string{"test"},
+			ExpectedEvents: map[string]int{
+				"OnFileDownloadRequest": 1,
+			},
+		},
 	}
 
 	for _, scenario := range scenarios {

apis/health.go

@@ -19,12 +19,20 @@ type healthApi struct {
 	app core.App
 }
 
+type healthCheckResponse struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+	Data    struct {
+		CanBackup bool `json:"canBackup"`
+	} `json:"data"`
+}
+
 // healthCheck returns a 200 OK response if the server is healthy.
 func (api *healthApi) healthCheck(c echo.Context) error {
-	payload := map[string]any{
-		"code":    http.StatusOK,
-		"message": "API is healthy.",
-	}
+	resp := new(healthCheckResponse)
+	resp.Code = http.StatusOK
+	resp.Message = "API is healthy."
+	resp.Data.CanBackup = !api.app.Cache().Has(core.CacheKeyActiveBackup)
 
-	return c.JSON(http.StatusOK, payload)
+	return c.JSON(http.StatusOK, resp)
 }

apis/health_test.go

@@ -16,6 +16,8 @@ func TestHealthAPI(t *testing.T) {
 			ExpectedStatus: 200,
 			ExpectedContent: []string{
 				`"code":200`,
+				`"data":{`,
+				`"canBackup":true`,
 			},
 		},
 	}

apis/middlewares.go

@@ -129,14 +129,17 @@ func RequireAdminAuth() echo.MiddlewareFunc {
 func RequireAdminAuthOnlyIfAny(app core.App) echo.MiddlewareFunc {
 	return func(next echo.HandlerFunc) echo.HandlerFunc {
 		return func(c echo.Context) error {
+			admin, _ := c.Get(ContextAdminKey).(*models.Admin)
+			if admin != nil {
+				return next(c)
+			}
+
 			totalAdmins, err := app.Dao().TotalAdmins()
 			if err != nil {
 				return NewBadRequestError("Failed to fetch admins info.", err)
 			}
 
-			admin, _ := c.Get(ContextAdminKey).(*models.Admin)
-
-			if admin != nil || totalAdmins == 0 {
+			if totalAdmins == 0 {
 				return next(c)
 			}
 
@@ -368,15 +371,19 @@ func realUserIp(r *http.Request, fallbackIp string) string {
 		return ip
 	}
 
+	if ip := r.Header.Get("Fly-Client-IP"); ip != "" {
+		return ip
+	}
+
 	if ip := r.Header.Get("X-Real-IP"); ip != "" {
 		return ip
 	}
 
 	if ipsList := r.Header.Get("X-Forwarded-For"); ipsList != "" {
+		// extract the first non-empty leftmost-ish ip
 		ips := strings.Split(ipsList, ",")
-		// extract the rightmost ip
-		for i := len(ips) - 1; i >= 0; i-- {
-			ip := strings.TrimSpace(ips[i])
+		for _, ip := range ips {
+			ip = strings.TrimSpace(ip)
 			if ip != "" {
 				return ip
 			}
@@ -385,3 +392,19 @@ func realUserIp(r *http.Request, fallbackIp string) string {
 
 	return fallbackIp
 }
+
+// eagerRequestDataCache ensures that the request data is cached in the request
+// context to allow reading for example the json request body data more than once.
+func eagerRequestDataCache(app core.App) echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			switch c.Request().Method {
+			// currently we are eagerly caching only the requests with body
+			case "POST", "PUT", "PATCH", "DELETE":
+				RequestData(c)
+			}
+
+			return next(c)
+		}
+	}
+}

apis/realtime.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/labstack/echo/v5"
@@ -215,7 +216,9 @@ func (api *realtimeApi) setSubscriptions(c echo.Context) error {
 func (api *realtimeApi) updateClientsAuthModel(contextKey string, newModel models.Model) error {
 	for _, client := range api.app.SubscriptionsBroker().Clients() {
 		clientModel, _ := client.Get(contextKey).(models.Model)
-		if clientModel != nil && clientModel.GetId() == newModel.GetId() {
+		if clientModel != nil &&
+			clientModel.TableName() == newModel.TableName() &&
+			clientModel.GetId() == newModel.GetId() {
 			client.Set(contextKey, newModel)
 		}
 	}
@@ -227,7 +230,9 @@ func (api *realtimeApi) updateClientsAuthModel(contextKey string, newModel model
 func (api *realtimeApi) unregisterClientsByAuthModel(contextKey string, model models.Model) error {
 	for _, client := range api.app.SubscriptionsBroker().Clients() {
 		clientModel, _ := client.Get(contextKey).(models.Model)
-		if clientModel != nil && clientModel.GetId() == model.GetId() {
+		if clientModel != nil &&
+			clientModel.TableName() == model.TableName() &&
+			clientModel.GetId() == model.GetId() {
 			api.app.SubscriptionsBroker().Unregister(client.Id())
 		}
 	}
@@ -238,7 +243,7 @@ func (api *realtimeApi) unregisterClientsByAuthModel(contextKey string, model mo
 func (api *realtimeApi) bindEvents() {
 	// update the clients that has admin or auth record association
 	api.app.OnModelAfterUpdate().PreAdd(func(e *core.ModelEvent) error {
-		if record, ok := e.Model.(*models.Record); ok && record != nil && record.Collection().IsAuth() {
+		if record := api.resolveRecord(e.Model); record != nil && record.Collection().IsAuth() {
 			return api.updateClientsAuthModel(ContextAuthRecordKey, record)
 		}
 
@@ -251,8 +256,8 @@ func (api *realtimeApi) bindEvents() {
 
 	// remove the client(s) associated to the deleted admin or auth record
 	api.app.OnModelAfterDelete().PreAdd(func(e *core.ModelEvent) error {
-		if record, ok := e.Model.(*models.Record); ok && record != nil && record.Collection().IsAuth() {
-			return api.unregisterClientsByAuthModel(ContextAuthRecordKey, record)
+		if collection := api.resolveRecordCollection(e.Model); collection != nil && collection.IsAuth() {
+			return api.unregisterClientsByAuthModel(ContextAuthRecordKey, e.Model)
 		}
 
 		if admin, ok := e.Model.(*models.Admin); ok && admin != nil {
@@ -263,7 +268,7 @@ func (api *realtimeApi) bindEvents() {
 	})
 
 	api.app.OnModelAfterCreate().PreAdd(func(e *core.ModelEvent) error {
-		if record, ok := e.Model.(*models.Record); ok {
+		if record := api.resolveRecord(e.Model); record != nil {
 			if err := api.broadcastRecord("create", record); err != nil && api.app.IsDebug() {
 				log.Println(err)
 			}
@@ -272,7 +277,7 @@ func (api *realtimeApi) bindEvents() {
 	})
 
 	api.app.OnModelAfterUpdate().PreAdd(func(e *core.ModelEvent) error {
-		if record, ok := e.Model.(*models.Record); ok {
+		if record := api.resolveRecord(e.Model); record != nil {
 			if err := api.broadcastRecord("update", record); err != nil && api.app.IsDebug() {
 				log.Println(err)
 			}
@@ -281,7 +286,7 @@ func (api *realtimeApi) bindEvents() {
 	})
 
 	api.app.OnModelBeforeDelete().Add(func(e *core.ModelEvent) error {
-		if record, ok := e.Model.(*models.Record); ok {
+		if record := api.resolveRecord(e.Model); record != nil {
 			if err := api.broadcastRecord("delete", record); err != nil && api.app.IsDebug() {
 				log.Println(err)
 			}
@@ -290,6 +295,33 @@ func (api *realtimeApi) bindEvents() {
 	})
 }
 
+// resolveRecord converts *if possible* the provided model interface to a Record.
+// This is usually helpful if the provided model is a custom Record model struct.
+func (api *realtimeApi) resolveRecord(model models.Model) (record *models.Record) {
+	record, _ = model.(*models.Record)
+
+	// check if it is custom Record model struct (ignore "private" tables)
+	if record == nil && !strings.HasPrefix(model.TableName(), "_") {
+		record, _ = api.app.Dao().FindRecordById(model.TableName(), model.GetId())
+	}
+
+	return record
+}
+
+// resolveRecordCollection extracts *if possible* the Collection model from the provided model interface.
+// This is usually helpful if the provided model is a custom Record model struct.
+func (api *realtimeApi) resolveRecordCollection(model models.Model) (collection *models.Collection) {
+	if record, ok := model.(*models.Record); ok {
+		collection = record.Collection()
+	} else if !strings.HasPrefix(model.TableName(), "_") {
+		// check if it is custom Record model struct (ignore "private" tables)
+		collection, _ = api.app.Dao().FindCollectionByNameOrId(model.TableName())
+	}
+
+	return collection
+}
+
+// canAccessRecord checks if the subscription client has access to the specified record model.
 func (api *realtimeApi) canAccessRecord(client subscriptions.Client, record *models.Record, accessRule *string) bool {
 	admin, _ := client.Get(ContextAdminKey).(*models.Admin)
 	if admin != nil {
@@ -307,7 +339,7 @@ func (api *realtimeApi) canAccessRecord(client subscriptions.Client, record *mod
 			return nil // empty public rule
 		}
 
-		// emulate request data
+		// mock request data
 		requestData := &models.RequestData{
 			Method: "GET",
 		}

apis/realtime_test.go

@@ -7,8 +7,10 @@ import (
 	"testing"
 
 	"github.com/labstack/echo/v5"
+	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/apis"
 	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/daos"
 	"github.com/pocketbase/pocketbase/models"
 	"github.com/pocketbase/pocketbase/tests"
 	"github.com/pocketbase/pocketbase/tools/hook"
@@ -253,7 +255,10 @@ func TestRealtimeAuthRecordDeleteEvent(t *testing.T) {
 	client.Set(apis.ContextAuthRecordKey, authRecord)
 	testApp.SubscriptionsBroker().Register(client)
 
-	testApp.OnModelAfterDelete().Trigger(&core.ModelEvent{Dao: testApp.Dao(), Model: authRecord})
+	e := new(core.ModelEvent)
+	e.Dao = testApp.Dao()
+	e.Model = authRecord
+	testApp.OnModelAfterDelete().Trigger(e)
 
 	if len(testApp.SubscriptionsBroker().Clients()) != 0 {
 		t.Fatalf("Expected no subscription clients, found %d", len(testApp.SubscriptionsBroker().Clients()))
@@ -282,7 +287,10 @@ func TestRealtimeAuthRecordUpdateEvent(t *testing.T) {
 	}
 	authRecord2.SetEmail("new@example.com")
 
-	testApp.OnModelAfterUpdate().Trigger(&core.ModelEvent{Dao: testApp.Dao(), Model: authRecord2})
+	e := new(core.ModelEvent)
+	e.Dao = testApp.Dao()
+	e.Model = authRecord2
+	testApp.OnModelAfterUpdate().Trigger(e)
 
 	clientAuthRecord, _ := client.Get(apis.ContextAuthRecordKey).(*models.Record)
 	if clientAuthRecord.Email() != authRecord2.Email() {
@@ -305,7 +313,10 @@ func TestRealtimeAdminDeleteEvent(t *testing.T) {
 	client.Set(apis.ContextAdminKey, admin)
 	testApp.SubscriptionsBroker().Register(client)
 
-	testApp.OnModelAfterDelete().Trigger(&core.ModelEvent{Dao: testApp.Dao(), Model: admin})
+	e := new(core.ModelEvent)
+	e.Dao = testApp.Dao()
+	e.Model = admin
+	testApp.OnModelAfterDelete().Trigger(e)
 
 	if len(testApp.SubscriptionsBroker().Clients()) != 0 {
 		t.Fatalf("Expected no subscription clients, found %d", len(testApp.SubscriptionsBroker().Clients()))
@@ -334,10 +345,106 @@ func TestRealtimeAdminUpdateEvent(t *testing.T) {
 	}
 	admin2.Email = "new@example.com"
 
-	testApp.OnModelAfterUpdate().Trigger(&core.ModelEvent{Dao: testApp.Dao(), Model: admin2})
+	e := new(core.ModelEvent)
+	e.Dao = testApp.Dao()
+	e.Model = admin2
+	testApp.OnModelAfterUpdate().Trigger(e)
 
 	clientAdmin, _ := client.Get(apis.ContextAdminKey).(*models.Admin)
 	if clientAdmin.Email != admin2.Email {
 		t.Fatalf("Expected authRecord with email %q, got %q", admin2.Email, clientAdmin.Email)
 	}
 }
+
+// Custom auth record model struct
+// -------------------------------------------------------------------
+var _ models.Model = (*CustomUser)(nil)
+
+type CustomUser struct {
+	models.BaseModel
+
+	Email string `db:"email" json:"email"`
+}
+
+func (m *CustomUser) TableName() string {
+	return "users" // the name of your collection
+}
+
+func findCustomUserByEmail(dao *daos.Dao, email string) (*CustomUser, error) {
+	model := &CustomUser{}
+
+	err := dao.ModelQuery(model).
+		AndWhere(dbx.HashExp{"email": email}).
+		Limit(1).
+		One(model)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return model, nil
+}
+
+func TestRealtimeCustomAuthModelDeleteEvent(t *testing.T) {
+	testApp, _ := tests.NewTestApp()
+	defer testApp.Cleanup()
+
+	apis.InitApi(testApp)
+
+	authRecord, err := testApp.Dao().FindFirstRecordByData("users", "email", "test@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := subscriptions.NewDefaultClient()
+	client.Set(apis.ContextAuthRecordKey, authRecord)
+	testApp.SubscriptionsBroker().Register(client)
+
+	// refetch the authRecord as CustomUser
+	customUser, err := findCustomUserByEmail(testApp.Dao(), "test@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// delete the custom user (should unset the client auth record)
+	if err := testApp.Dao().Delete(customUser); err != nil {
+		t.Fatal(err)
+	}
+
+	if len(testApp.SubscriptionsBroker().Clients()) != 0 {
+		t.Fatalf("Expected no subscription clients, found %d", len(testApp.SubscriptionsBroker().Clients()))
+	}
+}
+
+func TestRealtimeCustomAuthModelUpdateEvent(t *testing.T) {
+	testApp, _ := tests.NewTestApp()
+	defer testApp.Cleanup()
+
+	apis.InitApi(testApp)
+
+	authRecord, err := testApp.Dao().FindFirstRecordByData("users", "email", "test@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := subscriptions.NewDefaultClient()
+	client.Set(apis.ContextAuthRecordKey, authRecord)
+	testApp.SubscriptionsBroker().Register(client)
+
+	// refetch the authRecord as CustomUser
+	customUser, err := findCustomUserByEmail(testApp.Dao(), "test@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// change its email
+	customUser.Email = "new@example.com"
+	if err := testApp.Dao().Save(customUser); err != nil {
+		t.Fatal(err)
+	}
+
+	clientAuthRecord, _ := client.Get(apis.ContextAuthRecordKey).(*models.Record)
+	if clientAuthRecord.Email() != customUser.Email {
+		t.Fatalf("Expected authRecord with email %q, got %q", customUser.Email, clientAuthRecord.Email())
+	}
+}

apis/record_auth.go

@@ -1,6 +1,7 @@
 package apis
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log"
@@ -17,6 +18,7 @@ import (
 	"github.com/pocketbase/pocketbase/tools/routine"
 	"github.com/pocketbase/pocketbase/tools/search"
 	"github.com/pocketbase/pocketbase/tools/security"
+	"github.com/pocketbase/pocketbase/tools/subscriptions"
 	"golang.org/x/oauth2"
 )
 
@@ -25,12 +27,15 @@ import (
 func bindRecordAuthApi(app core.App, rg *echo.Group) {
 	api := recordAuthApi{app: app}
 
+	// global oauth2 subscription redirect handler
+	rg.GET("/oauth2-redirect", api.oauth2SubscriptionRedirect)
+
+	// common collection record related routes
 	subGroup := rg.Group(
 		"/collections/:collection",
 		ActivityLogger(app),
 		LoadCollectionContext(app, models.CollectionTypeAuth),
 	)
-
 	subGroup.GET("/auth-methods", api.authMethods)
 	subGroup.POST("/auth-refresh", api.authRefresh, RequireSameContextRecordAuth())
 	subGroup.POST("/auth-with-oauth2", api.authWithOAuth2)
@@ -185,9 +190,10 @@ func (api *recordAuthApi) authWithOAuth2(c echo.Context) error {
 
 	form.SetBeforeNewRecordCreateFunc(func(createForm *forms.RecordUpsert, authRecord *models.Record, authUser *auth.AuthUser) error {
 		return createForm.DrySubmit(func(txDao *daos.Dao) error {
-			requestData := RequestData(c)
-			requestData.Data = form.CreateData
 			event.IsNewRecord = true
+			// clone the current request data and assign the form create data as its body data
+			requestData := *RequestData(c)
+			requestData.Data = form.CreateData
 
 			createRuleFunc := func(q *dbx.SelectQuery) error {
 				admin, _ := c.Get(ContextAdminKey).(*models.Admin)
@@ -200,7 +206,7 @@ func (api *recordAuthApi) authWithOAuth2(c echo.Context) error {
 				}
 
 				if *collection.CreateRule != "" {
-					resolver := resolvers.NewRecordFieldResolver(txDao, collection, requestData, true)
+					resolver := resolvers.NewRecordFieldResolver(txDao, collection, &requestData, true)
 					expr, err := search.FilterData(*collection.CreateRule).BuildExpr(resolver)
 					if err != nil {
 						return err
@@ -237,7 +243,15 @@ func (api *recordAuthApi) authWithOAuth2(c echo.Context) error {
 				e.Record = data.Record
 				e.OAuth2User = data.OAuth2User
 
-				return RecordAuthResponse(api.app, e.HttpContext, e.Record, e.OAuth2User)
+				meta := struct {
+					*auth.AuthUser
+					IsNew bool `json:"isNew"`
+				}{
+					AuthUser: e.OAuth2User,
+					IsNew:    event.IsNewRecord,
+				}
+
+				return RecordAuthResponse(api.app, e.HttpContext, e.Record, meta)
 			})
 		}
 	})
@@ -628,3 +642,40 @@ func (api *recordAuthApi) unlinkExternalAuth(c echo.Context) error {
 
 	return handlerErr
 }
+
+// -------------------------------------------------------------------
+
+const oauth2SubscriptionTopic = "@oauth2"
+
+func (api *recordAuthApi) oauth2SubscriptionRedirect(c echo.Context) error {
+	state := c.QueryParam("state")
+	code := c.QueryParam("code")
+
+	if code == "" || state == "" {
+		return NewBadRequestError("Invalid OAuth2 redirect parameters.", nil)
+	}
+
+	client, err := api.app.SubscriptionsBroker().ClientById(state)
+	if err != nil || client.IsDiscarded() || !client.HasSubscription(oauth2SubscriptionTopic) {
+		return NewNotFoundError("Missing or invalid OAuth2 subscription client.", err)
+	}
+
+	data := map[string]string{
+		"state": state,
+		"code":  code,
+	}
+
+	encodedData, err := json.Marshal(data)
+	if err != nil {
+		return NewBadRequestError("Failed to marshalize OAuth2 redirect data.", err)
+	}
+
+	msg := subscriptions.Message{
+		Name: oauth2SubscriptionTopic,
+		Data: string(encodedData),
+	}
+
+	client.Channel() <- msg
+
+	return c.Redirect(http.StatusTemporaryRedirect, "../_/#/auth/oauth2-redirect")
+}

apis/record_auth_test.go

@@ -1,6 +1,7 @@
 package apis_test
 
 import (
+	"context"
 	"net/http"
 	"strings"
 	"testing"
@@ -9,6 +10,7 @@ import (
 	"github.com/labstack/echo/v5"
 	"github.com/pocketbase/pocketbase/daos"
 	"github.com/pocketbase/pocketbase/tests"
+	"github.com/pocketbase/pocketbase/tools/subscriptions"
 	"github.com/pocketbase/pocketbase/tools/types"
 )
 
@@ -1144,3 +1146,120 @@ func TestRecordAuthUnlinkExternalsAuth(t *testing.T) {
 		scenario.Test(t)
 	}
 }
+
+func TestRecordAuthOAuth2Redirect(t *testing.T) {
+	c1 := subscriptions.NewDefaultClient()
+
+	c2 := subscriptions.NewDefaultClient()
+	c2.Subscribe("@oauth2")
+
+	c3 := subscriptions.NewDefaultClient()
+	c3.Subscribe("test1", "@oauth2")
+
+	c4 := subscriptions.NewDefaultClient()
+	c4.Subscribe("test1", "test2")
+
+	c5 := subscriptions.NewDefaultClient()
+	c5.Subscribe("@oauth2")
+	c5.Discard()
+
+	beforeTestFunc := func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+		app.SubscriptionsBroker().Register(c1)
+		app.SubscriptionsBroker().Register(c2)
+		app.SubscriptionsBroker().Register(c3)
+		app.SubscriptionsBroker().Register(c4)
+		app.SubscriptionsBroker().Register(c5)
+	}
+
+	scenarios := []tests.ApiScenario{
+		{
+			Name:            "no state query param",
+			Method:          http.MethodGet,
+			Url:             "/api/oauth2-redirect?code=123",
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:            "no code query param",
+			Method:          http.MethodGet,
+			Url:             "/api/oauth2-redirect?state=" + c3.Id(),
+			ExpectedStatus:  400,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:            "missing client",
+			Method:          http.MethodGet,
+			Url:             "/api/oauth2-redirect?code=123&state=missing",
+			ExpectedStatus:  404,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:            "discarded client with @oauth2 subscription",
+			Method:          http.MethodGet,
+			Url:             "/api/oauth2-redirect?code=123&state=" + c5.Id(),
+			BeforeTestFunc:  beforeTestFunc,
+			ExpectedStatus:  404,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:            "client without @oauth2 subscription",
+			Method:          http.MethodGet,
+			Url:             "/api/oauth2-redirect?code=123&state=" + c4.Id(),
+			BeforeTestFunc:  beforeTestFunc,
+			ExpectedStatus:  404,
+			ExpectedContent: []string{`"data":{}`},
+		},
+		{
+			Name:   "client with @oauth2 subscription",
+			Method: http.MethodGet,
+			Url:    "/api/oauth2-redirect?code=123&state=" + c3.Id(),
+			BeforeTestFunc: func(t *testing.T, app *tests.TestApp, e *echo.Echo) {
+				beforeTestFunc(t, app, e)
+
+				ctx, cancelFunc := context.WithTimeout(context.Background(), 1*time.Second)
+
+				go func() {
+					defer cancelFunc()
+				L:
+					for {
+						select {
+						case <-c1.Channel():
+							t.Error("Unexpected c1 message")
+							break L
+						case <-c2.Channel():
+							t.Error("Unexpected c2 message")
+							break L
+						case msg := <-c3.Channel():
+							if msg.Name != "@oauth2" {
+								t.Errorf("Expected @oauth2 msg.Name, got %q", msg.Name)
+							}
+
+							expectedParams := []string{`"state"`, `"code"`}
+							for _, p := range expectedParams {
+								if !strings.Contains(msg.Data, p) {
+									t.Errorf("Couldn't find %s in \n%v", p, msg.Data)
+								}
+							}
+
+							break L
+						case <-c4.Channel():
+							t.Error("Unexpected c4 message")
+							break L
+						case <-c5.Channel():
+							t.Error("Unexpected c5 message")
+							break L
+						case <-ctx.Done():
+							t.Error("Context timeout reached")
+							break L
+						}
+					}
+				}()
+			},
+			ExpectedStatus: http.StatusTemporaryRedirect,
+		},
+	}
+
+	for _, scenario := range scenarios {
+		scenario.Test(t)
+	}
+}

apis/record_crud.go

@@ -68,6 +68,11 @@ func (api *recordApi) list(c echo.Context) error {
 	searchProvider := search.NewProvider(fieldsResolver).
 		Query(api.app.Dao().RecordQuery(collection))
 
+	// views don't have "rowid" so we fallback to "id"
+	if collection.IsView() {
+		searchProvider.CountCol("id")
+	}
+
 	if requestData.Admin == nil && collection.ListRule != nil {
 		searchProvider.AddFilter(search.FilterData(*collection.ListRule))
 	}

apis/record_crud_test.go

@@ -1173,6 +1173,20 @@ func TestRecordCrudCreate(t *testing.T) {
 				"OnModelAfterCreate":          1,
 			},
 		},
+		{
+			Name:   "unique field error check",
+			Method: http.MethodPost,
+			Url:    "/api/collections/demo2/records",
+			Body: strings.NewReader(`{
+				"title":"test2"
+			}`),
+			ExpectedStatus: 400,
+			ExpectedContent: []string{
+				`"data":{`,
+				`"title":{`,
+				`"code":"validation_not_unique"`,
+			},
+		},
 
 		// ID checks
 		// -----------------------------------------------------------
@@ -1747,6 +1761,24 @@ func TestRecordCrudUpdate(t *testing.T) {
 				`"id":{"code":"validation_in_invalid"`,
 			},
 		},
+		{
+			Name:   "unique field error check",
+			Method: http.MethodPatch,
+			Url:    "/api/collections/demo2/records/llvuca81nly1qls",
+			Body: strings.NewReader(`{
+				"title":"test2"
+			}`),
+			ExpectedStatus: 400,
+			ExpectedContent: []string{
+				`"data":{`,
+				`"title":{`,
+				`"code":"validation_not_unique"`,
+			},
+			ExpectedEvents: map[string]int{
+				"OnRecordBeforeUpdateRequest": 1,
+				"OnModelBeforeUpdate":         1,
+			},
+		},
 
 		// check whether if @request.data modifer fields are properly resolved
 		// -----------------------------------------------------------

apis/record_helpers.go

@@ -121,7 +121,9 @@ func EnrichRecords(c echo.Context, dao *daos.Dao, records []*models.Record, defa
 	}
 
 	expands := defaultExpands
-	expands = append(expands, strings.Split(c.QueryParam(expandQueryParam), ",")...)
+	if param := c.QueryParam(expandQueryParam); param != "" {
+		expands = append(expands, strings.Split(param, ",")...)
+	}
 	if len(expands) == 0 {
 		return nil // nothing to expand
 	}

apis/serve.go

@@ -0,0 +1,174 @@
+package apis
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"net"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/labstack/echo/v5/middleware"
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/migrations"
+	"github.com/pocketbase/pocketbase/migrations/logs"
+	"github.com/pocketbase/pocketbase/tools/migrate"
+	"golang.org/x/crypto/acme"
+	"golang.org/x/crypto/acme/autocert"
+)
+
+// ServeOptions defines an optional struct for apis.Serve().
+type ServeOptions struct {
+	ShowStartBanner bool
+	HttpAddr        string
+	HttpsAddr       string
+	AllowedOrigins  []string // optional list of CORS origins (default to "*")
+	BeforeServeFunc func(server *http.Server) error
+}
+
+// Serve starts a new app web server.
+func Serve(app core.App, options *ServeOptions) error {
+	if options == nil {
+		options = &ServeOptions{}
+	}
+
+	if len(options.AllowedOrigins) == 0 {
+		options.AllowedOrigins = []string{"*"}
+	}
+
+	// ensure that the latest migrations are applied before starting the server
+	if err := runMigrations(app); err != nil {
+		return err
+	}
+
+	// reload app settings in case a new default value was set with a migration
+	// (or if this is the first time the init migration was executed)
+	if err := app.RefreshSettings(); err != nil {
+		color.Yellow("=====================================")
+		color.Yellow("WARNING: Settings load error! \n%v", err)
+		color.Yellow("Fallback to the application defaults.")
+		color.Yellow("=====================================")
+	}
+
+	router, err := InitApi(app)
+	if err != nil {
+		return err
+	}
+
+	// configure cors
+	router.Use(middleware.CORSWithConfig(middleware.CORSConfig{
+		Skipper:      middleware.DefaultSkipper,
+		AllowOrigins: options.AllowedOrigins,
+		AllowMethods: []string{http.MethodGet, http.MethodHead, http.MethodPut, http.MethodPatch, http.MethodPost, http.MethodDelete},
+	}))
+
+	// start http server
+	// ---
+	mainAddr := options.HttpAddr
+	if options.HttpsAddr != "" {
+		mainAddr = options.HttpsAddr
+	}
+
+	mainHost, _, _ := net.SplitHostPort(mainAddr)
+
+	certManager := autocert.Manager{
+		Prompt:     autocert.AcceptTOS,
+		Cache:      autocert.DirCache(filepath.Join(app.DataDir(), ".autocert_cache")),
+		HostPolicy: autocert.HostWhitelist(mainHost, "www."+mainHost),
+	}
+
+	serverConfig := &http.Server{
+		TLSConfig: &tls.Config{
+			GetCertificate: certManager.GetCertificate,
+			NextProtos:     []string{acme.ALPNProto},
+		},
+		ReadTimeout:       10 * time.Minute,
+		ReadHeaderTimeout: 30 * time.Second,
+		// WriteTimeout: 60 * time.Second, // breaks sse!
+		Handler: router,
+		Addr:    mainAddr,
+	}
+
+	if options.BeforeServeFunc != nil {
+		if err := options.BeforeServeFunc(serverConfig); err != nil {
+			return err
+		}
+	}
+
+	if options.ShowStartBanner {
+		schema := "http"
+		if options.HttpsAddr != "" {
+			schema = "https"
+		}
+
+		date := new(strings.Builder)
+		log.New(date, "", log.LstdFlags).Print()
+
+		bold := color.New(color.Bold).Add(color.FgGreen)
+		bold.Printf(
+			"%s Server started at %s\n",
+			strings.TrimSpace(date.String()),
+			color.CyanString("%s://%s", schema, serverConfig.Addr),
+		)
+
+		regular := color.New()
+		regular.Printf(" ➜ REST API: %s\n", color.CyanString("%s://%s/api/", schema, serverConfig.Addr))
+		regular.Printf(" ➜ Admin UI: %s\n", color.CyanString("%s://%s/_/", schema, serverConfig.Addr))
+	}
+
+	// try to gracefully shutdown the server on app termination
+	app.OnTerminate().Add(func(e *core.TerminateEvent) error {
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		defer cancel()
+		serverConfig.Shutdown(ctx)
+		return nil
+	})
+
+	// start HTTPS server
+	if options.HttpsAddr != "" {
+		// if httpAddr is set, start an HTTP server to redirect the traffic to the HTTPS version
+		if options.HttpAddr != "" {
+			go http.ListenAndServe(options.HttpAddr, certManager.HTTPHandler(nil))
+		}
+
+		return serverConfig.ListenAndServeTLS("", "")
+	}
+
+	// OR start HTTP server
+	return serverConfig.ListenAndServe()
+}
+
+type migrationsConnection struct {
+	DB             *dbx.DB
+	MigrationsList migrate.MigrationsList
+}
+
+func runMigrations(app core.App) error {
+	connections := []migrationsConnection{
+		{
+			DB:             app.DB(),
+			MigrationsList: migrations.AppMigrations,
+		},
+		{
+			DB:             app.LogsDB(),
+			MigrationsList: logs.LogsMigrations,
+		},
+	}
+
+	for _, c := range connections {
+		runner, err := migrate.NewRunner(c.DB, c.MigrationsList)
+		if err != nil {
+			return err
+		}
+
+		if _, err := runner.Up(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

apis/settings.go

@@ -1,7 +1,6 @@
 package apis
 
 import (
-	"fmt"
 	"log"
 	"net/http"
 
@@ -10,7 +9,6 @@ import (
 	"github.com/pocketbase/pocketbase/core"
 	"github.com/pocketbase/pocketbase/forms"
 	"github.com/pocketbase/pocketbase/models/settings"
-	"github.com/pocketbase/pocketbase/tools/security"
 )
 
 // bindSettingsApi registers the settings api endpoints.
@@ -86,27 +84,22 @@ func (api *settingsApi) set(c echo.Context) error {
 }
 
 func (api *settingsApi) testS3(c echo.Context) error {
-	if !api.app.Settings().S3.Enabled {
-		return NewBadRequestError("S3 storage is not enabled.", nil)
+	form := forms.NewTestS3Filesystem(api.app)
+
+	// load request
+	if err := c.Bind(form); err != nil {
+		return NewBadRequestError("An error occurred while loading the submitted data.", err)
 	}
 
-	fs, err := api.app.NewFilesystem()
-	if err != nil {
-		return NewBadRequestError("Failed to initialize the S3 storage. Raw error: \n"+err.Error(), nil)
-	}
-	defer fs.Close()
+	// send
+	if err := form.Submit(); err != nil {
+		// form error
+		if fErr, ok := err.(validation.Errors); ok {
+			return NewBadRequestError("Failed to test the S3 filesystem.", fErr)
+		}
 
-	testPrefix := "pb_settings_test_" + security.PseudorandomString(5)
-	testFileKey := testPrefix + "/test.txt"
-
-	// try to upload a test file
-	if err := fs.Upload([]byte("test"), testFileKey); err != nil {
-		return NewBadRequestError("Failed to upload a test file. Raw error: \n"+err.Error(), nil)
-	}
-
-	// test prefix deletion (ensures that both bucket list and delete works)
-	if errs := fs.DeletePrefix(testPrefix); len(errs) > 0 {
-		return NewBadRequestError(fmt.Sprintf("Failed to delete a test file. Raw error: %v", errs), nil)
+		// mailer error
+		return NewBadRequestError("Failed to test the S3 filesystem. Raw error: \n"+err.Error(), nil)
 	}
 
 	return c.NoContent(http.StatusNoContent)

apis/settings_test.go

@@ -47,12 +47,15 @@ func TestSettingsList(t *testing.T) {
 				`"logs":{`,
 				`"smtp":{`,
 				`"s3":{`,
+				`"backups":{`,
 				`"adminAuthToken":{`,
 				`"adminPasswordResetToken":{`,
+				`"adminFileToken":{`,
 				`"recordAuthToken":{`,
 				`"recordPasswordResetToken":{`,
 				`"recordEmailChangeToken":{`,
 				`"recordVerificationToken":{`,
+				`"recordFileToken":{`,
 				`"emailAuth":{`,
 				`"googleAuth":{`,
 				`"facebookAuth":{`,
@@ -123,12 +126,15 @@ func TestSettingsSet(t *testing.T) {
 				`"logs":{`,
 				`"smtp":{`,
 				`"s3":{`,
+				`"backups":{`,
 				`"adminAuthToken":{`,
 				`"adminPasswordResetToken":{`,
+				`"adminFileToken":{`,
 				`"recordAuthToken":{`,
 				`"recordPasswordResetToken":{`,
 				`"recordEmailChangeToken":{`,
 				`"recordVerificationToken":{`,
+				`"recordFileToken":{`,
 				`"emailAuth":{`,
 				`"googleAuth":{`,
 				`"facebookAuth":{`,
@@ -186,12 +192,15 @@ func TestSettingsSet(t *testing.T) {
 				`"logs":{`,
 				`"smtp":{`,
 				`"s3":{`,
+				`"backups":{`,
 				`"adminAuthToken":{`,
 				`"adminPasswordResetToken":{`,
+				`"adminFileToken":{`,
 				`"recordAuthToken":{`,
 				`"recordPasswordResetToken":{`,
 				`"recordEmailChangeToken":{`,
 				`"recordVerificationToken":{`,
+				`"recordFileToken":{`,
 				`"emailAuth":{`,
 				`"googleAuth":{`,
 				`"facebookAuth":{`,
@@ -249,14 +258,44 @@ func TestSettingsTestS3(t *testing.T) {
 			ExpectedContent: []string{`"data":{}`},
 		},
 		{
-			Name:   "authorized as admin (no s3)",
+			Name:   "authorized as admin (missing body + no s3)",
 			Method: http.MethodPost,
 			Url:    "/api/settings/test/s3",
 			RequestHeaders: map[string]string{
 				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
 			},
-			ExpectedStatus:  400,
-			ExpectedContent: []string{`"data":{}`},
+			ExpectedStatus: 400,
+			ExpectedContent: []string{
+				`"data":{`,
+				`"filesystem":{`,
+			},
+		},
+		{
+			Name:   "authorized as admin (invalid filesystem)",
+			Method: http.MethodPost,
+			Url:    "/api/settings/test/s3",
+			Body:   strings.NewReader(`{"filesystem":"invalid"}`),
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			ExpectedStatus: 400,
+			ExpectedContent: []string{
+				`"data":{`,
+				`"filesystem":{`,
+			},
+		},
+		{
+			Name:   "authorized as admin (valid filesystem and no s3)",
+			Method: http.MethodPost,
+			Url:    "/api/settings/test/s3",
+			Body:   strings.NewReader(`{"filesystem":"storage"}`),
+			RequestHeaders: map[string]string{
+				"Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6InN5d2JoZWNuaDQ2cmhtMCIsInR5cGUiOiJhZG1pbiIsImV4cCI6MjIwODk4NTI2MX0.M1m--VOqGyv0d23eeUc0r9xE8ZzHaYVmVFw1VZW6gT8",
+			},
+			ExpectedStatus: 400,
+			ExpectedContent: []string{
+				`"data":{}`,
+			},
 		},
 	}
 

cmd/admin.go

@@ -0,0 +1,135 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/fatih/color"
+	"github.com/go-ozzo/ozzo-validation/v4/is"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/models"
+	"github.com/spf13/cobra"
+)
+
+// NewAdminCommand creates and returns new command for managing
+// admin accounts (create, update, delete).
+func NewAdminCommand(app core.App) *cobra.Command {
+	command := &cobra.Command{
+		Use:   "admin",
+		Short: "Manages admin accounts",
+	}
+
+	command.AddCommand(adminCreateCommand(app))
+	command.AddCommand(adminUpdateCommand(app))
+	command.AddCommand(adminDeleteCommand(app))
+
+	return command
+}
+
+func adminCreateCommand(app core.App) *cobra.Command {
+	command := &cobra.Command{
+		Use:     "create",
+		Example: "admin create test@example.com 1234567890",
+		Short:   "Creates a new admin account",
+		// prevents printing the error log twice
+		SilenceErrors: true,
+		SilenceUsage:  true,
+		RunE: func(command *cobra.Command, args []string) error {
+			if len(args) != 2 {
+				return errors.New("Missing email and password arguments.")
+			}
+
+			if args[0] == "" || is.EmailFormat.Validate(args[0]) != nil {
+				return errors.New("Missing or invalid email address.")
+			}
+
+			if len(args[1]) < 8 {
+				return errors.New("The password must be at least 8 chars long.")
+			}
+
+			admin := &models.Admin{}
+			admin.Email = args[0]
+			admin.SetPassword(args[1])
+
+			if err := app.Dao().SaveAdmin(admin); err != nil {
+				return fmt.Errorf("Failed to create new admin account: %v", err)
+			}
+
+			color.Green("Successfully created new admin %s!", admin.Email)
+			return nil
+		},
+	}
+
+	return command
+}
+
+func adminUpdateCommand(app core.App) *cobra.Command {
+	command := &cobra.Command{
+		Use:     "update",
+		Example: "admin update test@example.com 1234567890",
+		Short:   "Changes the password of a single admin account",
+		// prevents printing the error log twice
+		SilenceErrors: true,
+		SilenceUsage:  true,
+		RunE: func(command *cobra.Command, args []string) error {
+			if len(args) != 2 {
+				return errors.New("Missing email and password arguments.")
+			}
+
+			if args[0] == "" || is.EmailFormat.Validate(args[0]) != nil {
+				return errors.New("Missing or invalid email address.")
+			}
+
+			if len(args[1]) < 8 {
+				return errors.New("The new password must be at least 8 chars long.")
+			}
+
+			admin, err := app.Dao().FindAdminByEmail(args[0])
+			if err != nil {
+				return fmt.Errorf("Admin with email %s doesn't exist.", args[0])
+			}
+
+			admin.SetPassword(args[1])
+
+			if err := app.Dao().SaveAdmin(admin); err != nil {
+				return fmt.Errorf("Failed to change admin %s password: %v", admin.Email, err)
+			}
+
+			color.Green("Successfully changed admin %s password!", admin.Email)
+			return nil
+		},
+	}
+
+	return command
+}
+
+func adminDeleteCommand(app core.App) *cobra.Command {
+	command := &cobra.Command{
+		Use:     "delete",
+		Example: "admin delete test@example.com",
+		Short:   "Deletes an existing admin account",
+		// prevents printing the error log twice
+		SilenceErrors: true,
+		SilenceUsage:  true,
+		RunE: func(command *cobra.Command, args []string) error {
+			if len(args) == 0 || args[0] == "" || is.EmailFormat.Validate(args[0]) != nil {
+				return errors.New("Invalid or missing email address.")
+			}
+
+			admin, err := app.Dao().FindAdminByEmail(args[0])
+			if err != nil {
+				color.Yellow("Admin %s is already deleted.", args[0])
+				return nil
+			}
+
+			if err := app.Dao().DeleteAdmin(admin); err != nil {
+				return fmt.Errorf("Failed to delete admin %s: %v", admin.Email, err)
+			}
+
+			color.Green("Successfully deleted admin %s!", admin.Email)
+			return nil
+		},
+	}
+
+	return command
+}

cmd/admin_test.go

@@ -0,0 +1,219 @@
+package cmd_test
+
+import (
+	"testing"
+
+	"github.com/pocketbase/pocketbase/cmd"
+	"github.com/pocketbase/pocketbase/tests"
+)
+
+func TestAdminCreateCommand(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	scenarios := []struct {
+		name        string
+		email       string
+		password    string
+		expectError bool
+	}{
+		{
+			"empty email and password",
+			"",
+			"",
+			true,
+		},
+		{
+			"empty email",
+			"",
+			"1234567890",
+			true,
+		},
+		{
+			"invalid email",
+			"invalid",
+			"1234567890",
+			true,
+		},
+		{
+			"duplicated email",
+			"test@example.com",
+			"1234567890",
+			true,
+		},
+		{
+			"empty password",
+			"test@example.com",
+			"",
+			true,
+		},
+		{
+			"short password",
+			"test_new@example.com",
+			"1234567",
+			true,
+		},
+		{
+			"valid email and password",
+			"test_new@example.com",
+			"12345678",
+			false,
+		},
+	}
+
+	for _, s := range scenarios {
+		command := cmd.NewAdminCommand(app)
+		command.SetArgs([]string{"create", s.email, s.password})
+
+		err := command.Execute()
+
+		hasErr := err != nil
+		if s.expectError != hasErr {
+			t.Errorf("[%s] Expected hasErr %v, got %v (%v)", s.name, s.expectError, hasErr, err)
+		}
+
+		if hasErr {
+			continue
+		}
+
+		// check whether the admin account was actually created
+		admin, err := app.Dao().FindAdminByEmail(s.email)
+		if err != nil {
+			t.Errorf("[%s] Failed to fetch created admin %s: %v", s.name, s.email, err)
+		} else if !admin.ValidatePassword(s.password) {
+			t.Errorf("[%s] Expected the admin password to match", s.name)
+		}
+	}
+}
+
+func TestAdminUpdateCommand(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	scenarios := []struct {
+		name        string
+		email       string
+		password    string
+		expectError bool
+	}{
+		{
+			"empty email and password",
+			"",
+			"",
+			true,
+		},
+		{
+			"empty email",
+			"",
+			"1234567890",
+			true,
+		},
+		{
+			"invalid email",
+			"invalid",
+			"1234567890",
+			true,
+		},
+		{
+			"nonexisting admin",
+			"test_missing@example.com",
+			"1234567890",
+			true,
+		},
+		{
+			"empty password",
+			"test@example.com",
+			"",
+			true,
+		},
+		{
+			"short password",
+			"test_new@example.com",
+			"1234567",
+			true,
+		},
+		{
+			"valid email and password",
+			"test@example.com",
+			"12345678",
+			false,
+		},
+	}
+
+	for _, s := range scenarios {
+		command := cmd.NewAdminCommand(app)
+		command.SetArgs([]string{"update", s.email, s.password})
+
+		err := command.Execute()
+
+		hasErr := err != nil
+		if s.expectError != hasErr {
+			t.Errorf("[%s] Expected hasErr %v, got %v (%v)", s.name, s.expectError, hasErr, err)
+		}
+
+		if hasErr {
+			continue
+		}
+
+		// check whether the admin password was actually changed
+		admin, err := app.Dao().FindAdminByEmail(s.email)
+		if err != nil {
+			t.Errorf("[%s] Failed to fetch admin %s: %v", s.name, s.email, err)
+		} else if !admin.ValidatePassword(s.password) {
+			t.Errorf("[%s] Expected the admin password to match", s.name)
+		}
+	}
+}
+
+func TestAdminDeleteCommand(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	scenarios := []struct {
+		name        string
+		email       string
+		expectError bool
+	}{
+		{
+			"empty email",
+			"",
+			true,
+		},
+		{
+			"invalid email",
+			"invalid",
+			true,
+		},
+		{
+			"nonexisting admin",
+			"test_missing@example.com",
+			false,
+		},
+		{
+			"existing admin",
+			"test@example.com",
+			false,
+		},
+	}
+
+	for _, s := range scenarios {
+		command := cmd.NewAdminCommand(app)
+		command.SetArgs([]string{"delete", s.email})
+
+		err := command.Execute()
+
+		hasErr := err != nil
+		if s.expectError != hasErr {
+			t.Errorf("[%s] Expected hasErr %v, got %v (%v)", s.name, s.expectError, hasErr, err)
+		}
+
+		if hasErr {
+			continue
+		}
+
+		// check whether the admin account was actually deleted
+		if _, err := app.Dao().FindAdminByEmail(s.email); err == nil {
+			t.Errorf("[%s] Expected the admin account to be deleted", s.name)
+		}
+	}
+}

cmd/serve.go

@@ -1,25 +1,12 @@
 package cmd
 
 import (
-	"crypto/tls"
 	"log"
-	"net"
 	"net/http"
-	"path/filepath"
-	"strings"
-	"time"
 
-	"github.com/fatih/color"
-	"github.com/labstack/echo/v5/middleware"
-	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/apis"
 	"github.com/pocketbase/pocketbase/core"
-	"github.com/pocketbase/pocketbase/migrations"
-	"github.com/pocketbase/pocketbase/migrations/logs"
-	"github.com/pocketbase/pocketbase/tools/migrate"
 	"github.com/spf13/cobra"
-	"golang.org/x/crypto/acme"
-	"golang.org/x/crypto/acme/autocert"
 )
 
 // NewServeCommand creates and returns new command responsible for
@@ -33,96 +20,15 @@ func NewServeCommand(app core.App, showStartBanner bool) *cobra.Command {
 		Use:   "serve",
 		Short: "Starts the web server (default to 127.0.0.1:8090)",
 		Run: func(command *cobra.Command, args []string) {
-			// ensure that the latest migrations are applied before starting the server
-			if err := runMigrations(app); err != nil {
-				panic(err)
-			}
+			err := apis.Serve(app, &apis.ServeOptions{
+				HttpAddr:        httpAddr,
+				HttpsAddr:       httpsAddr,
+				ShowStartBanner: showStartBanner,
+				AllowedOrigins:  allowedOrigins,
+			})
 
-			// reload app settings in case a new default value was set with a migration
-			// (or if this is the first time the init migration was executed)
-			if err := app.RefreshSettings(); err != nil {
-				color.Yellow("=====================================")
-				color.Yellow("WARNING: Settings load error! \n%v", err)
-				color.Yellow("Fallback to the application defaults.")
-				color.Yellow("=====================================")
-			}
-
-			router, err := apis.InitApi(app)
-			if err != nil {
-				panic(err)
-			}
-
-			// configure cors
-			router.Use(middleware.CORSWithConfig(middleware.CORSConfig{
-				Skipper:      middleware.DefaultSkipper,
-				AllowOrigins: allowedOrigins,
-				AllowMethods: []string{http.MethodGet, http.MethodHead, http.MethodPut, http.MethodPatch, http.MethodPost, http.MethodDelete},
-			}))
-
-			// start http server
-			// ---
-			mainAddr := httpAddr
-			if httpsAddr != "" {
-				mainAddr = httpsAddr
-			}
-
-			mainHost, _, _ := net.SplitHostPort(mainAddr)
-
-			certManager := autocert.Manager{
-				Prompt:     autocert.AcceptTOS,
-				Cache:      autocert.DirCache(filepath.Join(app.DataDir(), ".autocert_cache")),
-				HostPolicy: autocert.HostWhitelist(mainHost, "www."+mainHost),
-			}
-
-			serverConfig := &http.Server{
-				TLSConfig: &tls.Config{
-					GetCertificate: certManager.GetCertificate,
-					NextProtos:     []string{acme.ALPNProto},
-				},
-				ReadTimeout:       5 * time.Minute,
-				ReadHeaderTimeout: 30 * time.Second,
-				// WriteTimeout: 60 * time.Second, // breaks sse!
-				Handler: router,
-				Addr:    mainAddr,
-			}
-
-			if showStartBanner {
-				schema := "http"
-				if httpsAddr != "" {
-					schema = "https"
-				}
-
-				date := new(strings.Builder)
-				log.New(date, "", log.LstdFlags).Print()
-
-				bold := color.New(color.Bold).Add(color.FgGreen)
-				bold.Printf(
-					"%s Server started at %s\n",
-					strings.TrimSpace(date.String()),
-					color.CyanString("%s://%s", schema, serverConfig.Addr),
-				)
-
-				regular := color.New()
-				regular.Printf(" ➜ REST API: %s\n", color.CyanString("%s://%s/api/", schema, serverConfig.Addr))
-				regular.Printf(" ➜ Admin UI: %s\n", color.CyanString("%s://%s/_/", schema, serverConfig.Addr))
-			}
-
-			var serveErr error
-			if httpsAddr != "" {
-				// if httpAddr is set, start an HTTP server to redirect the traffic to the HTTPS version
-				if httpAddr != "" {
-					go http.ListenAndServe(httpAddr, certManager.HTTPHandler(nil))
-				}
-
-				// start HTTPS server
-				serveErr = serverConfig.ListenAndServeTLS("", "")
-			} else {
-				// start HTTP server
-				serveErr = serverConfig.ListenAndServe()
-			}
-
-			if serveErr != http.ErrServerClosed {
-				log.Fatalln(serveErr)
+			if err != http.ErrServerClosed {
+				log.Fatalln(err)
 			}
 		},
 	}
@@ -150,34 +56,3 @@ func NewServeCommand(app core.App, showStartBanner bool) *cobra.Command {
 
 	return command
 }
-
-type migrationsConnection struct {
-	DB             *dbx.DB
-	MigrationsList migrate.MigrationsList
-}
-
-func runMigrations(app core.App) error {
-	connections := []migrationsConnection{
-		{
-			DB:             app.DB(),
-			MigrationsList: migrations.AppMigrations,
-		},
-		{
-			DB:             app.LogsDB(),
-			MigrationsList: logs.LogsMigrations,
-		},
-	}
-
-	for _, c := range connections {
-		runner, err := migrate.NewRunner(c.DB, c.MigrationsList)
-		if err != nil {
-			return err
-		}
-
-		if _, err := runner.Up(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

cmd/temp_upgrade.go

@@ -1,444 +0,0 @@
-package cmd
-
-import (
-	"errors"
-	"fmt"
-	"regexp"
-	"strings"
-
-	"github.com/fatih/color"
-	"github.com/pocketbase/dbx"
-	"github.com/pocketbase/pocketbase/core"
-	"github.com/pocketbase/pocketbase/daos"
-	"github.com/pocketbase/pocketbase/models"
-	"github.com/pocketbase/pocketbase/models/schema"
-	"github.com/pocketbase/pocketbase/tools/types"
-	"github.com/spf13/cobra"
-)
-
-// Temporary console command to update the pb_data structure to be compatible with the v0.8.0 changes.
-//
-// NB! It will be removed in v0.9+
-func NewTempUpgradeCommand(app core.App) *cobra.Command {
-	command := &cobra.Command{
-		Use:   "upgrade",
-		Short: "Upgrades your existing pb_data to be compatible with the v0.8.x changes",
-		Long: `
-Upgrades your existing pb_data to be compatible with the v0.8.x changes
-Prerequisites and caveats:
-- already upgraded to v0.7.*
-- no existing users collection
-- existing profiles collection fields like email, username, verified, etc. will be renamed to username2, email2, etc.
-`,
-		Run: func(command *cobra.Command, args []string) {
-			if err := upgrade(app); err != nil {
-				color.Red("Error: %v", err)
-			}
-		},
-	}
-
-	return command
-}
-
-func upgrade(app core.App) error {
-	if _, err := app.Dao().FindCollectionByNameOrId("users"); err == nil {
-		return errors.New("It seems that you've already upgraded or have an existing 'users' collection.")
-	}
-
-	return app.Dao().RunInTransaction(func(txDao *daos.Dao) error {
-		if err := migrateCollections(txDao); err != nil {
-			return err
-		}
-
-		if err := migrateUsers(app, txDao); err != nil {
-			return err
-		}
-
-		if err := resetMigrationsTable(txDao); err != nil {
-			return err
-		}
-
-		bold := color.New(color.Bold).Add(color.FgGreen)
-		bold.Println("The pb_data upgrade completed successfully!")
-		bold.Println("You can now start the application as usual with the 'serve' command.")
-		bold.Println("Please review the migrated collection API rules and fields in the Admin UI and apply the necessary changes in your client-side code.")
-		fmt.Println()
-
-		return nil
-	})
-}
-
-// -------------------------------------------------------------------
-
-func migrateCollections(txDao *daos.Dao) error {
-	// add new collection columns
-	if _, err := txDao.DB().AddColumn("_collections", "type", "TEXT DEFAULT 'base' NOT NULL").Execute(); err != nil {
-		return err
-	}
-	if _, err := txDao.DB().AddColumn("_collections", "options", "JSON DEFAULT '{}' NOT NULL").Execute(); err != nil {
-		return err
-	}
-
-	ruleReplacements := []struct {
-		old string
-		new string
-	}{
-		{"expand", "expand2"},
-		{"collecitonId", "collectionId2"},
-		{"collecitonName", "collectionName2"},
-		{"profile.userId", "profile.id"},
-
-		// @collection.*
-		{"@collection.profiles.userId", "@collection.users.id"},
-		{"@collection.profiles.username", "@collection.users.username2"},
-		{"@collection.profiles.email", "@collection.users.email2"},
-		{"@collection.profiles.emailVisibility", "@collection.users.emailVisibility2"},
-		{"@collection.profiles.verified", "@collection.users.verified2"},
-		{"@collection.profiles.tokenKey", "@collection.users.tokenKey2"},
-		{"@collection.profiles.passwordHash", "@collection.users.passwordHash2"},
-		{"@collection.profiles.lastResetSentAt", "@collection.users.lastResetSentAt2"},
-		{"@collection.profiles.lastVerificationSentAt", "@collection.users.lastVerificationSentAt2"},
-		{"@collection.profiles.", "@collection.users."},
-
-		// @request.*
-		{"@request.user.profile.userId", "@request.auth.id"},
-		{"@request.user.profile.username", "@request.auth.username2"},
-		{"@request.user.profile.email", "@request.auth.email2"},
-		{"@request.user.profile.emailVisibility", "@request.auth.emailVisibility2"},
-		{"@request.user.profile.verified", "@request.auth.verified2"},
-		{"@request.user.profile.tokenKey", "@request.auth.tokenKey2"},
-		{"@request.user.profile.passwordHash", "@request.auth.passwordHash2"},
-		{"@request.user.profile.lastResetSentAt", "@request.auth.lastResetSentAt2"},
-		{"@request.user.profile.lastVerificationSentAt", "@request.auth.lastVerificationSentAt2"},
-		{"@request.user.profile.", "@request.auth."},
-		{"@request.user", "@request.auth"},
-	}
-
-	collections := []*models.Collection{}
-	if err := txDao.CollectionQuery().All(&collections); err != nil {
-		return err
-	}
-
-	for _, collection := range collections {
-		collection.Type = models.CollectionTypeBase
-		collection.NormalizeOptions()
-
-		// rename profile fields
-		// ---
-		fieldsToRename := []string{
-			"collectionId",
-			"collectionName",
-			"expand",
-		}
-		if collection.Name == "profiles" {
-			fieldsToRename = append(fieldsToRename,
-				"username",
-				"email",
-				"emailVisibility",
-				"verified",
-				"tokenKey",
-				"passwordHash",
-				"lastResetSentAt",
-				"lastVerificationSentAt",
-			)
-		}
-		for _, name := range fieldsToRename {
-			f := collection.Schema.GetFieldByName(name)
-			if f != nil {
-				color.Blue("[%s - renamed field]", collection.Name)
-				color.Yellow(" - old: %s", f.Name)
-				color.Green(" - new: %s2", f.Name)
-				fmt.Println()
-				f.Name += "2"
-			}
-		}
-		// ---
-
-		// replace rule fields
-		// ---
-		rules := map[string]*string{
-			"ListRule":   collection.ListRule,
-			"ViewRule":   collection.ViewRule,
-			"CreateRule": collection.CreateRule,
-			"UpdateRule": collection.UpdateRule,
-			"DeleteRule": collection.DeleteRule,
-		}
-
-		for ruleKey, rule := range rules {
-			if rule == nil || *rule == "" {
-				continue
-			}
-
-			originalRule := *rule
-
-			for _, replacement := range ruleReplacements {
-				re := regexp.MustCompile(regexp.QuoteMeta(replacement.old) + `\b`)
-				*rule = re.ReplaceAllString(*rule, replacement.new)
-			}
-
-			*rule = replaceReversedLikes(*rule)
-
-			if originalRule != *rule {
-				color.Blue("[%s - replaced %s]:", collection.Name, ruleKey)
-				color.Yellow(" - old: %s", strings.TrimSpace(originalRule))
-				color.Green(" - new: %s", strings.TrimSpace(*rule))
-				fmt.Println()
-			}
-		}
-		// ---
-
-		if err := txDao.SaveCollection(collection); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func migrateUsers(app core.App, txDao *daos.Dao) error {
-	color.Blue(`[merging "_users" and "profiles"]:`)
-
-	profilesCollection, err := txDao.FindCollectionByNameOrId("profiles")
-	if err != nil {
-		return err
-	}
-
-	originalProfilesCollectionId := profilesCollection.Id
-
-	// change the profiles collection id to something else since we will be using
-	// it for the new users collection in order to avoid renaming the storage dir
-	_, idRenameErr := txDao.DB().NewQuery(fmt.Sprintf(
-		`UPDATE {{_collections}}
-			SET id = '%s'
-			WHERE id = '%s';
-		`,
-		(originalProfilesCollectionId + "__old__"),
-		originalProfilesCollectionId,
-	)).Execute()
-	if idRenameErr != nil {
-		return idRenameErr
-	}
-
-	// refresh profiles collection
-	profilesCollection, err = txDao.FindCollectionByNameOrId("profiles")
-	if err != nil {
-		return err
-	}
-
-	usersSchema, _ := profilesCollection.Schema.Clone()
-	userIdField := usersSchema.GetFieldByName("userId")
-	if userIdField != nil {
-		usersSchema.RemoveField(userIdField.Id)
-	}
-
-	usersCollection := &models.Collection{}
-	usersCollection.MarkAsNew()
-	usersCollection.Id = originalProfilesCollectionId
-	usersCollection.Name = "users"
-	usersCollection.Type = models.CollectionTypeAuth
-	usersCollection.Schema = *usersSchema
-	usersCollection.CreateRule = types.Pointer("")
-	if profilesCollection.ListRule != nil && *profilesCollection.ListRule != "" {
-		*profilesCollection.ListRule = strings.ReplaceAll(*profilesCollection.ListRule, "userId", "id")
-		usersCollection.ListRule = profilesCollection.ListRule
-	}
-	if profilesCollection.ViewRule != nil && *profilesCollection.ViewRule != "" {
-		*profilesCollection.ViewRule = strings.ReplaceAll(*profilesCollection.ViewRule, "userId", "id")
-		usersCollection.ViewRule = profilesCollection.ViewRule
-	}
-	if profilesCollection.UpdateRule != nil && *profilesCollection.UpdateRule != "" {
-		*profilesCollection.UpdateRule = strings.ReplaceAll(*profilesCollection.UpdateRule, "userId", "id")
-		usersCollection.UpdateRule = profilesCollection.UpdateRule
-	}
-	if profilesCollection.DeleteRule != nil && *profilesCollection.DeleteRule != "" {
-		*profilesCollection.DeleteRule = strings.ReplaceAll(*profilesCollection.DeleteRule, "userId", "id")
-		usersCollection.DeleteRule = profilesCollection.DeleteRule
-	}
-
-	// set auth options
-	settings := app.Settings()
-	authOptions := usersCollection.AuthOptions()
-	authOptions.ManageRule = nil
-	authOptions.AllowOAuth2Auth = true
-	authOptions.AllowUsernameAuth = false
-	authOptions.AllowEmailAuth = settings.EmailAuth.Enabled
-	authOptions.MinPasswordLength = settings.EmailAuth.MinPasswordLength
-	authOptions.OnlyEmailDomains = settings.EmailAuth.OnlyDomains
-	authOptions.ExceptEmailDomains = settings.EmailAuth.ExceptDomains
-	// twitter currently is the only provider that doesn't return an email
-	authOptions.RequireEmail = !settings.TwitterAuth.Enabled
-
-	usersCollection.SetOptions(authOptions)
-
-	if err := txDao.SaveCollection(usersCollection); err != nil {
-		return err
-	}
-
-	// copy the original users
-	_, usersErr := txDao.DB().NewQuery(`
-		INSERT INTO {{users}} (id, created, updated, username, email, emailVisibility, verified, tokenKey, passwordHash, lastResetSentAt, lastVerificationSentAt)
-		SELECT id, created, updated, ("u_" || id), email, false, verified, tokenKey, passwordHash, lastResetSentAt, lastVerificationSentAt
-		FROM {{_users}};
-	`).Execute()
-	if usersErr != nil {
-		return usersErr
-	}
-
-	// generate the profile fields copy statements
-	sets := []string{"id = p.id"}
-	for _, f := range usersSchema.Fields() {
-		sets = append(sets, fmt.Sprintf("%s = p.%s", f.Name, f.Name))
-	}
-
-	// copy profile fields
-	_, copyProfileErr := txDao.DB().NewQuery(fmt.Sprintf(`
-		UPDATE {{users}} as u
-		SET %s
-		FROM {{profiles}} as p
-		WHERE u.id = p.userId;
-	`, strings.Join(sets, ", "))).Execute()
-	if copyProfileErr != nil {
-		return copyProfileErr
-	}
-
-	profileRecords, err := txDao.FindRecordsByExpr("profiles")
-	if err != nil {
-		return err
-	}
-
-	// update all profiles and users fields to point to the new users collection
-	collections := []*models.Collection{}
-	if err := txDao.CollectionQuery().All(&collections); err != nil {
-		return err
-	}
-	for _, collection := range collections {
-		var hasChanges bool
-
-		for _, f := range collection.Schema.Fields() {
-			f.InitOptions()
-
-			if f.Type == schema.FieldTypeUser {
-				if collection.Name == "profiles" && f.Name == "userId" {
-					continue
-				}
-
-				hasChanges = true
-
-				// change the user field to a relation field
-				options, _ := f.Options.(*schema.UserOptions)
-				f.Type = schema.FieldTypeRelation
-				f.Options = &schema.RelationOptions{
-					CollectionId:  usersCollection.Id,
-					MaxSelect:     &options.MaxSelect,
-					CascadeDelete: options.CascadeDelete,
-				}
-
-				for _, p := range profileRecords {
-					pId := p.Id
-					pUserId := p.GetString("userId")
-					// replace all user record id references with the profile id
-					_, replaceErr := txDao.DB().NewQuery(fmt.Sprintf(`
-						UPDATE %s
-						SET [[%s]] = REPLACE([[%s]], '%s', '%s')
-						WHERE [[%s]] LIKE ('%%%s%%');
-					`, collection.Name, f.Name, f.Name, pUserId, pId, f.Name, pUserId)).Execute()
-					if replaceErr != nil {
-						return replaceErr
-					}
-				}
-			}
-		}
-
-		if hasChanges {
-			if err := txDao.Save(collection); err != nil {
-				return err
-			}
-		}
-	}
-
-	if err := migrateExternalAuths(txDao, originalProfilesCollectionId); err != nil {
-		return err
-	}
-
-	// drop _users table
-	if _, err := txDao.DB().DropTable("_users").Execute(); err != nil {
-		return err
-	}
-
-	// drop profiles table
-	if _, err := txDao.DB().DropTable("profiles").Execute(); err != nil {
-		return err
-	}
-
-	// delete profiles collection
-	if err := txDao.Delete(profilesCollection); err != nil {
-		return err
-	}
-
-	color.Green(` - Successfully merged "_users" and "profiles" into a new collection "users".`)
-	fmt.Println()
-
-	return nil
-}
-
-func migrateExternalAuths(txDao *daos.Dao, userCollectionId string) error {
-	_, alterErr := txDao.DB().NewQuery(`
-		-- crate new externalAuths table
-		CREATE TABLE {{_newExternalAuths}} (
-			[[id]]           TEXT PRIMARY KEY,
-			[[collectionId]] TEXT NOT NULL,
-			[[recordId]]     TEXT NOT NULL,
-			[[provider]]     TEXT NOT NULL,
-			[[providerId]]   TEXT NOT NULL,
-			[[created]]      TEXT DEFAULT "" NOT NULL,
-			[[updated]]      TEXT DEFAULT "" NOT NULL,
-			---
-			FOREIGN KEY ([[collectionId]]) REFERENCES {{_collections}} ([[id]]) ON UPDATE CASCADE ON DELETE CASCADE
-		);
-
-		-- copy all data from the old table to the new one
-		INSERT INTO {{_newExternalAuths}}
-		SELECT auth.id, "` + userCollectionId + `" as collectionId, [[profiles.id]] as recordId, auth.provider, auth.providerId, auth.created, auth.updated
-		FROM {{_externalAuths}} auth
-		INNER JOIN {{profiles}} on [[profiles.userId]] = [[auth.userId]];
-
-		-- drop old table
-		DROP TABLE {{_externalAuths}};
-
-		-- rename new table
-		ALTER TABLE {{_newExternalAuths}} RENAME TO {{_externalAuths}};
-
-		-- create named indexes
-		CREATE UNIQUE INDEX _externalAuths_record_provider_idx on {{_externalAuths}} ([[collectionId]], [[recordId]], [[provider]]);
-		CREATE UNIQUE INDEX _externalAuths_provider_providerId_idx on {{_externalAuths}} ([[provider]], [[providerId]]);
-	`).Execute()
-
-	return alterErr
-}
-
-func resetMigrationsTable(txDao *daos.Dao) error {
-	// reset the migration state to the new init
-	_, err := txDao.DB().Delete("_migrations", dbx.HashExp{
-		"file": "1661586591_add_externalAuths_table.go",
-	}).Execute()
-
-	return err
-}
-
-var reverseLikeRegex = regexp.MustCompile(`(['"]\w*['"])\s*(\~|!~)\s*([\w\@\.]*)`)
-
-func replaceReversedLikes(rule string) string {
-	parts := reverseLikeRegex.FindAllStringSubmatch(rule, -1)
-
-	for _, p := range parts {
-		if len(p) != 4 {
-			continue
-		}
-
-		newPart := fmt.Sprintf("%s %s %s", p[3], p[2], p[1])
-
-		rule = strings.ReplaceAll(rule, p[0], newPart)
-	}
-
-	return rule
-}

core/app.go

@@ -4,6 +4,8 @@
 package core
 
 import (
+	"context"
+
 	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/daos"
 	"github.com/pocketbase/pocketbase/models/settings"
@@ -69,12 +71,20 @@ type App interface {
 	// NewMailClient creates and returns a configured app mail client.
 	NewMailClient() mailer.Mailer
 
-	// NewFilesystem creates and returns a configured filesystem.System instance.
+	// NewFilesystem creates and returns a configured filesystem.System instance
+	// for managing regular app files (eg. collection uploads).
 	//
 	// NB! Make sure to call `Close()` on the returned result
 	// after you are done working with it.
 	NewFilesystem() (*filesystem.System, error)
 
+	// NewBackupsFilesystem creates and returns a configured filesystem.System instance
+	// for managing app backups.
+	//
+	// NB! Make sure to call `Close()` on the returned result
+	// after you are done working with it.
+	NewBackupsFilesystem() (*filesystem.System, error)
+
 	// RefreshSettings reinitializes and reloads the stored application settings.
 	RefreshSettings() error
 
@@ -92,6 +102,31 @@ type App interface {
 	// (eg. closing db connections).
 	ResetBootstrapState() error
 
+	// CreateBackup creates a new backup of the current app pb_data directory.
+	//
+	// Backups can be stored on S3 if it is configured in app.Settings().Backups.
+	//
+	// Please refer to the godoc of the specific core.App implementation
+	// for details on the backup procedures.
+	CreateBackup(ctx context.Context, name string) error
+
+	// RestoreBackup restores the backup with the specified name and restarts
+	// the current running application process.
+	//
+	// The safely perform the restore it is recommended to have free disk space
+	// for at least 2x the size of the restored pb_data backup.
+	//
+	// Please refer to the godoc of the specific core.App implementation
+	// for details on the restore procedures.
+	//
+	// NB! This feature is experimental and currently is expected to work only on UNIX based systems.
+	RestoreBackup(ctx context.Context, name string) error
+
+	// Restart restarts the current running application process.
+	//
+	// Currently it is relying on execve so it is supported only on UNIX based systems.
+	Restart() error
+
 	// ---------------------------------------------------------------
 	// App event hooks
 	// ---------------------------------------------------------------
@@ -118,6 +153,10 @@ type App interface {
 	// It could be used to log the final API error in external services.
 	OnAfterApiError() *hook.Hook[*ApiErrorEvent]
 
+	// OnTerminate hook is triggered when the app is in the process
+	// of being terminated (eg. on SIGTERM signal).
+	OnTerminate() *hook.Hook[*TerminateEvent]
+
 	// ---------------------------------------------------------------
 	// Dao event hooks
 	// ---------------------------------------------------------------
@@ -307,6 +346,25 @@ type App interface {
 	// returning it to the client.
 	OnFileDownloadRequest(tags ...string) *hook.TaggedHook[*FileDownloadEvent]
 
+	// OnFileBeforeTokenRequest hook is triggered before each file
+	// token API request.
+	//
+	// If no token or model was submitted, e.Model and e.Token will be empty,
+	// allowing you to implement your own custom model file auth implementation.
+	//
+	// If the optional "tags" list (Collection ids or names) is specified,
+	// then all event handlers registered via the created hook will be
+	// triggered and called only if their event data origin matches the tags.
+	OnFileBeforeTokenRequest(tags ...string) *hook.TaggedHook[*FileTokenEvent]
+
+	// OnFileAfterTokenRequest hook is triggered after each
+	// successful file token API request.
+	//
+	// If the optional "tags" list (Collection ids or names) is specified,
+	// then all event handlers registered via the created hook will be
+	// triggered and called only if their event data origin matches the tags.
+	OnFileAfterTokenRequest(tags ...string) *hook.TaggedHook[*FileTokenEvent]
+
 	// ---------------------------------------------------------------
 	// Admin API event hooks
 	// ---------------------------------------------------------------

core/base.go

@@ -7,6 +7,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"runtime"
+	"syscall"
 	"time"
 
 	"github.com/fatih/color"
@@ -27,6 +29,10 @@ const (
 	DefaultDataMaxIdleConns int = 20
 	DefaultLogsMaxOpenConns int = 10
 	DefaultLogsMaxIdleConns int = 2
+
+	LocalStorageDirName string = "storage"
+	LocalBackupsDirName string = "backups"
+	LocalTempDirName    string = ".pb_temp_to_delete" // temp pb_data sub directory that will be deleted on each app.Bootstrap()
 )
 
 var _ App = (*BaseApp)(nil)
@@ -55,6 +61,7 @@ type BaseApp struct {
 	onBeforeServe     *hook.Hook[*ServeEvent]
 	onBeforeApiError  *hook.Hook[*ApiErrorEvent]
 	onAfterApiError   *hook.Hook[*ApiErrorEvent]
+	onTerminate       *hook.Hook[*TerminateEvent]
 
 	// dao event hooks
 	onModelBeforeCreate *hook.Hook[*ModelEvent]
@@ -88,7 +95,9 @@ type BaseApp struct {
 	onSettingsAfterUpdateRequest  *hook.Hook[*SettingsUpdateEvent]
 
 	// file api event hooks
-	onFileDownloadRequest *hook.Hook[*FileDownloadEvent]
+	onFileDownloadRequest    *hook.Hook[*FileDownloadEvent]
+	onFileBeforeTokenRequest *hook.Hook[*FileTokenEvent]
+	onFileAfterTokenRequest  *hook.Hook[*FileTokenEvent]
 
 	// admin api event hooks
 	onAdminsListRequest                      *hook.Hook[*AdminsListEvent]
@@ -190,6 +199,7 @@ func NewBaseApp(config *BaseAppConfig) *BaseApp {
 		onBeforeServe:     &hook.Hook[*ServeEvent]{},
 		onBeforeApiError:  &hook.Hook[*ApiErrorEvent]{},
 		onAfterApiError:   &hook.Hook[*ApiErrorEvent]{},
+		onTerminate:       &hook.Hook[*TerminateEvent]{},
 
 		// dao event hooks
 		onModelBeforeCreate: &hook.Hook[*ModelEvent]{},
@@ -223,7 +233,9 @@ func NewBaseApp(config *BaseAppConfig) *BaseApp {
 		onSettingsAfterUpdateRequest:  &hook.Hook[*SettingsUpdateEvent]{},
 
 		// file API event hooks
-		onFileDownloadRequest: &hook.Hook[*FileDownloadEvent]{},
+		onFileDownloadRequest:    &hook.Hook[*FileDownloadEvent]{},
+		onFileBeforeTokenRequest: &hook.Hook[*FileTokenEvent]{},
+		onFileAfterTokenRequest:  &hook.Hook[*FileTokenEvent]{},
 
 		// admin API event hooks
 		onAdminsListRequest:                      &hook.Hook[*AdminsListEvent]{},
@@ -334,6 +346,9 @@ func (app *BaseApp) Bootstrap() error {
 	// we don't check for an error because the db migrations may have not been executed yet
 	app.RefreshSettings()
 
+	// cleanup the pb_data temp directory (if any)
+	os.RemoveAll(filepath.Join(app.DataDir(), LocalTempDirName))
+
 	if err := app.OnAfterBootstrap().Trigger(event); err != nil && app.IsDebug() {
 		log.Println(err)
 	}
@@ -467,6 +482,7 @@ func (app *BaseApp) NewMailClient() mailer.Mailer {
 }
 
 // NewFilesystem creates a new local or S3 filesystem instance
+// for managing regular app files (eg. collection uploads)
 // based on the current app settings.
 //
 // NB! Make sure to call `Close()` on the returned result
@@ -484,7 +500,54 @@ func (app *BaseApp) NewFilesystem() (*filesystem.System, error) {
 	}
 
 	// fallback to local filesystem
-	return filesystem.NewLocal(filepath.Join(app.DataDir(), "storage"))
+	return filesystem.NewLocal(filepath.Join(app.DataDir(), LocalStorageDirName))
+}
+
+// NewFilesystem creates a new local or S3 filesystem instance
+// for managing app backups based on the current app settings.
+//
+// NB! Make sure to call `Close()` on the returned result
+// after you are done working with it.
+func (app *BaseApp) NewBackupsFilesystem() (*filesystem.System, error) {
+	if app.settings != nil && app.settings.Backups.S3.Enabled {
+		return filesystem.NewS3(
+			app.settings.Backups.S3.Bucket,
+			app.settings.Backups.S3.Region,
+			app.settings.Backups.S3.Endpoint,
+			app.settings.Backups.S3.AccessKey,
+			app.settings.Backups.S3.Secret,
+			app.settings.Backups.S3.ForcePathStyle,
+		)
+	}
+
+	// fallback to local filesystem
+	return filesystem.NewLocal(filepath.Join(app.DataDir(), LocalBackupsDirName))
+}
+
+// Restart restarts (aka. replaces) the current running application process.
+//
+// NB! It relies on execve which is supported only on UNIX based systems.
+func (app *BaseApp) Restart() error {
+	if runtime.GOOS == "windows" {
+		return errors.New("restart is not supported on windows")
+	}
+
+	execPath, err := os.Executable()
+	if err != nil {
+		return err
+	}
+
+	// optimistically reset the app bootstrap state
+	app.ResetBootstrapState()
+
+	if err := syscall.Exec(execPath, os.Args, os.Environ()); err != nil {
+		// restart the app bootstrap state
+		app.Bootstrap()
+
+		return err
+	}
+
+	return nil
 }
 
 // RefreshSettings reinitializes and reloads the stored application settings.
@@ -537,6 +600,10 @@ func (app *BaseApp) OnAfterApiError() *hook.Hook[*ApiErrorEvent] {
 	return app.onAfterApiError
 }
 
+func (app *BaseApp) OnTerminate() *hook.Hook[*TerminateEvent] {
+	return app.onTerminate
+}
+
 // -------------------------------------------------------------------
 // Dao event hooks
 // -------------------------------------------------------------------
@@ -653,6 +720,14 @@ func (app *BaseApp) OnFileDownloadRequest(tags ...string) *hook.TaggedHook[*File
 	return hook.NewTaggedHook(app.onFileDownloadRequest, tags...)
 }
 
+func (app *BaseApp) OnFileBeforeTokenRequest(tags ...string) *hook.TaggedHook[*FileTokenEvent] {
+	return hook.NewTaggedHook(app.onFileBeforeTokenRequest, tags...)
+}
+
+func (app *BaseApp) OnFileAfterTokenRequest(tags ...string) *hook.TaggedHook[*FileTokenEvent] {
+	return hook.NewTaggedHook(app.onFileAfterTokenRequest, tags...)
+}
+
 // -------------------------------------------------------------------
 // Admin API event hooks
 // -------------------------------------------------------------------
@@ -979,34 +1054,55 @@ func (app *BaseApp) createDaoWithHooks(concurrentDB, nonconcurrentDB dbx.Builder
 	dao := daos.NewMultiDB(concurrentDB, nonconcurrentDB)
 
 	dao.BeforeCreateFunc = func(eventDao *daos.Dao, m models.Model) error {
-		return app.OnModelBeforeCreate().Trigger(&ModelEvent{eventDao, m})
+		e := new(ModelEvent)
+		e.Dao = eventDao
+		e.Model = m
+
+		return app.OnModelBeforeCreate().Trigger(e)
 	}
 
 	dao.AfterCreateFunc = func(eventDao *daos.Dao, m models.Model) {
-		err := app.OnModelAfterCreate().Trigger(&ModelEvent{eventDao, m})
-		if err != nil && app.isDebug {
+		e := new(ModelEvent)
+		e.Dao = eventDao
+		e.Model = m
+
+		if err := app.OnModelAfterCreate().Trigger(e); err != nil && app.isDebug {
 			log.Println(err)
 		}
 	}
 
 	dao.BeforeUpdateFunc = func(eventDao *daos.Dao, m models.Model) error {
-		return app.OnModelBeforeUpdate().Trigger(&ModelEvent{eventDao, m})
+		e := new(ModelEvent)
+		e.Dao = eventDao
+		e.Model = m
+
+		return app.OnModelBeforeUpdate().Trigger(e)
 	}
 
 	dao.AfterUpdateFunc = func(eventDao *daos.Dao, m models.Model) {
-		err := app.OnModelAfterUpdate().Trigger(&ModelEvent{eventDao, m})
-		if err != nil && app.isDebug {
+		e := new(ModelEvent)
+		e.Dao = eventDao
+		e.Model = m
+
+		if err := app.OnModelAfterUpdate().Trigger(e); err != nil && app.isDebug {
 			log.Println(err)
 		}
 	}
 
 	dao.BeforeDeleteFunc = func(eventDao *daos.Dao, m models.Model) error {
-		return app.OnModelBeforeDelete().Trigger(&ModelEvent{eventDao, m})
+		e := new(ModelEvent)
+		e.Dao = eventDao
+		e.Model = m
+
+		return app.OnModelBeforeDelete().Trigger(e)
 	}
 
 	dao.AfterDeleteFunc = func(eventDao *daos.Dao, m models.Model) {
-		err := app.OnModelAfterDelete().Trigger(&ModelEvent{eventDao, m})
-		if err != nil && app.isDebug {
+		e := new(ModelEvent)
+		e.Dao = eventDao
+		e.Model = m
+
+		if err := app.OnModelAfterDelete().Trigger(e); err != nil && app.isDebug {
 			log.Println(err)
 		}
 	}
@@ -1051,4 +1147,13 @@ func (app *BaseApp) registerDefaultHooks() {
 
 		return nil
 	})
+
+	app.OnTerminate().Add(func(e *TerminateEvent) error {
+		app.ResetBootstrapState()
+		return nil
+	})
+
+	if err := app.initAutobackupHooks(); err != nil && app.IsDebug() {
+		log.Println(err)
+	}
 }

core/base_backup.go

@@ -0,0 +1,325 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"time"
+
+	"github.com/pocketbase/pocketbase/daos"
+	"github.com/pocketbase/pocketbase/models"
+	"github.com/pocketbase/pocketbase/tools/archive"
+	"github.com/pocketbase/pocketbase/tools/cron"
+	"github.com/pocketbase/pocketbase/tools/filesystem"
+	"github.com/pocketbase/pocketbase/tools/osutils"
+	"github.com/pocketbase/pocketbase/tools/security"
+)
+
+const CacheKeyActiveBackup string = "@activeBackup"
+
+// CreateBackup creates a new backup of the current app pb_data directory.
+//
+// If name is empty, it will be autogenerated.
+// If backup with the same name exists, the new backup file will replace it.
+//
+// The backup is executed within a transaction, meaning that new writes
+// will be temporary "blocked" until the backup file is generated.
+//
+// By default backups are stored in pb_data/backups
+// (the backups directory itself is excluded from the generated backup).
+//
+// When using S3 storage for the uploaded collection files, you have to
+// take care manually to backup those since they are not part of the pb_data.
+//
+// Backups can be stored on S3 if it is configured in app.Settings().Backups.
+func (app *BaseApp) CreateBackup(ctx context.Context, name string) error {
+	if app.Cache().Has(CacheKeyActiveBackup) {
+		return errors.New("try again later - another backup/restore operation has already been started")
+	}
+
+	// auto generate backup name
+	if name == "" {
+		name = fmt.Sprintf(
+			"pb_backup_%s.zip",
+			time.Now().UTC().Format("20060102150405"),
+		)
+	}
+
+	app.Cache().Set(CacheKeyActiveBackup, name)
+	defer app.Cache().Remove(CacheKeyActiveBackup)
+
+	// Archive pb_data in a temp directory, exluding the "backups" dir itself (if exist).
+	//
+	// Run in transaction to temporary block other writes (transactions uses the NonconcurrentDB connection).
+	// ---
+	tempPath := filepath.Join(os.TempDir(), "pb_backup_"+security.PseudorandomString(4))
+	createErr := app.Dao().RunInTransaction(func(txDao *daos.Dao) error {
+		if err := archive.Create(app.DataDir(), tempPath, LocalBackupsDirName); err != nil {
+			return err
+		}
+		return nil
+	})
+	if createErr != nil {
+		return createErr
+	}
+	defer os.Remove(tempPath)
+
+	// Persist the backup in the backups filesystem.
+	// ---
+	fsys, err := app.NewBackupsFilesystem()
+	if err != nil {
+		return err
+	}
+	defer fsys.Close()
+
+	fsys.SetContext(ctx)
+
+	file, err := filesystem.NewFileFromPath(tempPath)
+	if err != nil {
+		return err
+	}
+	file.OriginalName = name
+	file.Name = file.OriginalName
+
+	if err := fsys.UploadFile(file, file.Name); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RestoreBackup restores the backup with the specified name and restarts
+// the current running application process.
+//
+// NB! This feature is experimental and currently is expected to work only on UNIX based systems.
+//
+// To safely perform the restore it is recommended to have free disk space
+// for at least 2x the size of the restored pb_data backup.
+//
+// The performed steps are:
+//
+//  1. Download the backup with the specified name in a temp location
+//     (this is in case of S3; otherwise it creates a temp copy of the zip)
+//
+//  2. Extract the backup in a temp directory inside the app "pb_data"
+//     (eg. "pb_data/.pb_temp_to_delete/pb_restore").
+//
+//  3. Move the current app "pb_data" content (excluding the local backups and the special temp dir)
+//     under another temp sub dir that will be deleted on the next app start up
+//     (eg. "pb_data/.pb_temp_to_delete/old_pb_data").
+//     This is because on some environments it may not be allowed
+//     to delete the currently open "pb_data" files.
+//
+//  4. Move the extracted dir content to the app "pb_data".
+//
+//  5. Restart the app (on successfull app bootstap it will also remove the old pb_data).
+//
+// If a failure occure during the restore process the dir changes are reverted.
+// If for whatever reason the revert is not possible, it panics.
+func (app *BaseApp) RestoreBackup(ctx context.Context, name string) error {
+	if runtime.GOOS == "windows" {
+		return errors.New("restore is not supported on windows")
+	}
+
+	if app.Cache().Has(CacheKeyActiveBackup) {
+		return errors.New("try again later - another backup/restore operation has already been started")
+	}
+
+	app.Cache().Set(CacheKeyActiveBackup, name)
+	defer app.Cache().Remove(CacheKeyActiveBackup)
+
+	fsys, err := app.NewBackupsFilesystem()
+	if err != nil {
+		return err
+	}
+	defer fsys.Close()
+
+	fsys.SetContext(ctx)
+
+	// fetch the backup file in a temp location
+	br, err := fsys.GetFile(name)
+	if err != nil {
+		return err
+	}
+	defer br.Close()
+
+	tempZip, err := os.CreateTemp(os.TempDir(), "pb_restore")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tempZip.Name())
+
+	if _, err := io.Copy(tempZip, br); err != nil {
+		return err
+	}
+
+	// make sure that the special temp directory
+	if err := os.MkdirAll(filepath.Join(app.DataDir(), LocalTempDirName), os.ModePerm); err != nil {
+		return fmt.Errorf("failed to create a temp dir: %w", err)
+	}
+
+	// note: it needs to be inside the current pb_data to avoid "cross-device link" errors
+	extractedDataDir := filepath.Join(app.DataDir(), LocalTempDirName, "pb_restore_"+security.PseudorandomString(4))
+	defer os.RemoveAll(extractedDataDir)
+	if err := archive.Extract(tempZip.Name(), extractedDataDir); err != nil {
+		return err
+	}
+
+	// ensure that a database file exists
+	extractedDB := filepath.Join(extractedDataDir, "data.db")
+	if _, err := os.Stat(extractedDB); err != nil {
+		return fmt.Errorf("data.db file is missing or invalid: %w", err)
+	}
+
+	// remove the extracted zip file since we no longer need it
+	// (this is in case the app restarts and the defer calls are not called)
+	if err := os.Remove(tempZip.Name()); err != nil && app.IsDebug() {
+		log.Println(err)
+	}
+
+	// root dir entries to exclude from the backup restore
+	exclude := []string{LocalBackupsDirName, LocalTempDirName}
+
+	// move the current pb_data content to a special temp location
+	// that will hold the old data between dirs replace
+	// (the temp dir will be automatically removed on the next app start)
+	oldTempDataDir := filepath.Join(app.DataDir(), LocalTempDirName, "old_pb_data_"+security.PseudorandomString(4))
+	if err := osutils.MoveDirContent(app.DataDir(), oldTempDataDir, exclude...); err != nil {
+		return fmt.Errorf("failed to move the current pb_data content to a temp location: %w", err)
+	}
+
+	// move the extracted archive content to the app's pb_data
+	if err := osutils.MoveDirContent(extractedDataDir, app.DataDir(), exclude...); err != nil {
+		return fmt.Errorf("failed to move the extracted archive content to pb_data: %w", err)
+	}
+
+	revertDataDirChanges := func() error {
+		if err := osutils.MoveDirContent(app.DataDir(), extractedDataDir, exclude...); err != nil {
+			return fmt.Errorf("failed to revert the extracted dir change: %w", err)
+		}
+
+		if err := osutils.MoveDirContent(oldTempDataDir, app.DataDir(), exclude...); err != nil {
+			return fmt.Errorf("failed to revert old pb_data dir change: %w", err)
+		}
+
+		return nil
+	}
+
+	// restart the app
+	if err := app.Restart(); err != nil {
+		if err := revertDataDirChanges(); err != nil {
+			panic(err)
+		}
+
+		return fmt.Errorf("failed to restart the app process: %w", err)
+	}
+
+	return nil
+}
+
+// initAutobackupHooks registers the autobackup app serve hooks.
+// @todo add tests
+func (app *BaseApp) initAutobackupHooks() error {
+	c := cron.New()
+	isServe := false
+
+	loadJob := func() {
+		c.Stop()
+
+		rawSchedule := app.Settings().Backups.Cron
+		if rawSchedule == "" || !isServe || !app.IsBootstrapped() {
+			return
+		}
+
+		c.Add("@autobackup", rawSchedule, func() {
+			autoPrefix := "@auto_pb_backup_"
+
+			name := fmt.Sprintf(
+				"%s%s.zip",
+				autoPrefix,
+				time.Now().UTC().Format("20060102150405"),
+			)
+
+			if err := app.CreateBackup(context.Background(), name); err != nil && app.IsDebug() {
+				// @todo replace after logs generalization
+				log.Println(err)
+			}
+
+			maxKeep := app.Settings().Backups.CronMaxKeep
+
+			if maxKeep == 0 {
+				return // no explicit limit
+			}
+
+			fsys, err := app.NewBackupsFilesystem()
+			if err != nil && app.IsDebug() {
+				// @todo replace after logs generalization
+				log.Println(err)
+				return
+			}
+			defer fsys.Close()
+
+			files, err := fsys.List(autoPrefix)
+			if err != nil && app.IsDebug() {
+				// @todo replace after logs generalization
+				log.Println(err)
+				return
+			}
+
+			if maxKeep >= len(files) {
+				return // nothing to remove
+			}
+
+			// sort desc
+			sort.Slice(files, func(i, j int) bool {
+				return files[i].ModTime.After(files[j].ModTime)
+			})
+
+			// keep only the most recent n auto backup files
+			toRemove := files[maxKeep:]
+
+			for _, f := range toRemove {
+				if err := fsys.Delete(f.Key); err != nil && app.IsDebug() {
+					// @todo replace after logs generalization
+					log.Println(err)
+				}
+			}
+		})
+
+		// restart the ticker
+		c.Start()
+	}
+
+	// load on app serve
+	app.OnBeforeServe().Add(func(e *ServeEvent) error {
+		isServe = true
+		loadJob()
+		return nil
+	})
+
+	// stop the ticker on app termination
+	app.OnTerminate().Add(func(e *TerminateEvent) error {
+		c.Stop()
+		return nil
+	})
+
+	// reload on app settings change
+	app.OnModelAfterUpdate((&models.Param{}).TableName()).Add(func(e *ModelEvent) error {
+		p := e.Model.(*models.Param)
+		if p == nil || p.Key != models.ParamAppSettings {
+			return nil
+		}
+
+		loadJob()
+
+		return nil
+	})
+
+	return nil
+}

core/base_backup_test.go

@@ -0,0 +1,159 @@
+package core_test
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tests"
+	"github.com/pocketbase/pocketbase/tools/archive"
+	"github.com/pocketbase/pocketbase/tools/list"
+)
+
+func TestCreateBackup(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	// test pending error
+	app.Cache().Set(core.CacheKeyActiveBackup, "")
+	if err := app.CreateBackup(context.Background(), "test.zip"); err == nil {
+		t.Fatal("Expected pending error, got nil")
+	}
+	app.Cache().Remove(core.CacheKeyActiveBackup)
+
+	// create with auto generated name
+	if err := app.CreateBackup(context.Background(), ""); err != nil {
+		t.Fatal("Failed to create a backup with autogenerated name")
+	}
+
+	// create with custom name
+	if err := app.CreateBackup(context.Background(), "custom"); err != nil {
+		t.Fatal("Failed to create a backup with custom name")
+	}
+
+	// create new with the same name (aka. replace)
+	if err := app.CreateBackup(context.Background(), "custom"); err != nil {
+		t.Fatal("Failed to create and replace a backup with the same name")
+	}
+
+	backupsDir := filepath.Join(app.DataDir(), core.LocalBackupsDirName)
+
+	entries, err := os.ReadDir(backupsDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedFiles := []string{
+		`^pb_backup_\w+\.zip$`,
+		`^pb_backup_\w+\.zip.attrs$`,
+		"custom",
+		"custom.attrs",
+	}
+
+	if len(entries) != len(expectedFiles) {
+		names := getEntryNames(entries)
+		t.Fatalf("Expected %d backup files, got %d: \n%v", len(expectedFiles), len(entries), names)
+	}
+
+	for i, entry := range entries {
+		if !list.ExistInSliceWithRegex(entry.Name(), expectedFiles) {
+			t.Fatalf("[%d] Missing backup file %q", i, entry.Name())
+		}
+
+		if strings.HasSuffix(entry.Name(), ".attrs") {
+			continue
+		}
+
+		path := filepath.Join(backupsDir, entry.Name())
+
+		if err := verifyBackupContent(app, path); err != nil {
+			t.Fatalf("[%d] Failed to verify backup content: %v", i, err)
+		}
+	}
+}
+
+func TestRestoreBackup(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	// create a initial test backup to ensure that there are at least 1
+	// backup file and that the generated zip doesn't contain the backups dir
+	if err := app.CreateBackup(context.Background(), "initial"); err != nil {
+		t.Fatal("Failed to create test initial backup")
+	}
+
+	// create test backup
+	if err := app.CreateBackup(context.Background(), "test"); err != nil {
+		t.Fatal("Failed to create test backup")
+	}
+
+	// test pending error
+	app.Cache().Set(core.CacheKeyActiveBackup, "")
+	if err := app.RestoreBackup(context.Background(), "test"); err == nil {
+		t.Fatal("Expected pending error, got nil")
+	}
+	app.Cache().Remove(core.CacheKeyActiveBackup)
+
+	// missing backup
+	if err := app.RestoreBackup(context.Background(), "missing"); err == nil {
+		t.Fatal("Expected missing error, got nil")
+	}
+}
+
+// -------------------------------------------------------------------
+
+func verifyBackupContent(app core.App, path string) error {
+	dir, err := os.MkdirTemp("", "backup_test")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(dir)
+
+	if err := archive.Extract(path, dir); err != nil {
+		return err
+	}
+
+	expectedRootEntries := []string{
+		"storage",
+		"data.db",
+		"data.db-shm",
+		"data.db-wal",
+		"logs.db",
+		"logs.db-shm",
+		"logs.db-wal",
+		".gitignore",
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+
+	if len(entries) != len(expectedRootEntries) {
+		names := getEntryNames(entries)
+		return fmt.Errorf("Expected %d backup files, got %d: \n%v", len(expectedRootEntries), len(entries), names)
+	}
+
+	for _, entry := range entries {
+		if !list.ExistInSliceWithRegex(entry.Name(), expectedRootEntries) {
+			return fmt.Errorf("Didn't expect %q entry", entry.Name())
+		}
+	}
+
+	return nil
+}
+
+func getEntryNames(entries []fs.DirEntry) []string {
+	names := make([]string, len(entries))
+
+	for i, entry := range entries {
+		names[i] = entry.Name()
+	}
+
+	return names
+}

core/base_test.go

@@ -236,3 +236,33 @@ func TestBaseAppNewFilesystem(t *testing.T) {
 		t.Fatalf("Expected nil s3 filesystem, got %v", s3)
 	}
 }
+
+func TestBaseAppNewBackupsFilesystem(t *testing.T) {
+	const testDataDir = "./pb_base_app_test_data_dir/"
+	defer os.RemoveAll(testDataDir)
+
+	app := NewBaseApp(&BaseAppConfig{
+		DataDir:       testDataDir,
+		EncryptionEnv: "pb_test_env",
+		IsDebug:       false,
+	})
+
+	// local
+	local, localErr := app.NewBackupsFilesystem()
+	if localErr != nil {
+		t.Fatal(localErr)
+	}
+	if local == nil {
+		t.Fatal("Expected local backups filesystem instance, got nil")
+	}
+
+	// misconfigured s3
+	app.Settings().Backups.S3.Enabled = true
+	s3, s3Err := app.NewBackupsFilesystem()
+	if s3Err == nil {
+		t.Fatal("Expected S3 error, got nil")
+	}
+	if s3 != nil {
+		t.Fatalf("Expected nil s3 backups filesystem, got %v", s3)
+	}
+}

core/db.go

@@ -1,20 +0,0 @@
-package core
-
-import (
-	"github.com/pocketbase/dbx"
-)
-
-func initPragmas(db *dbx.DB) error {
-	// note: the busy_timeout pragma must be first because
-	// the connection needs to be set to block on busy before WAL mode
-	// is set in case it hasn't been already set by another connection
-	_, err := db.NewQuery(`
-		PRAGMA busy_timeout       = 10000;
-		PRAGMA journal_mode       = WAL;
-		PRAGMA journal_size_limit = 200000000;
-		PRAGMA synchronous        = NORMAL;
-		PRAGMA foreign_keys       = TRUE;
-	`).Execute()
-
-	return err
-}

core/db_cgo.go

@@ -3,18 +3,44 @@
 package core
 
 import (
+	"database/sql"
+
+	"github.com/mattn/go-sqlite3"
 	"github.com/pocketbase/dbx"
-	_ "github.com/mattn/go-sqlite3"
 )
 
-func connectDB(dbPath string) (*dbx.DB, error) {
-	db, err := dbx.Open("sqlite3", dbPath)
-	if err != nil {
-		return nil, err
-	}
+func init() {
+	// Registers the sqlite3 driver with a ConnectHook so that we can
+	// initialize the default PRAGMAs.
+	//
+	// Note 1: we don't define the PRAGMA as part of the dsn string
+	// because not all pragmas are available.
+	//
+	// Note 2: the busy_timeout pragma must be first because
+	// the connection needs to be set to block on busy before WAL mode
+	// is set in case it hasn't been already set by another connection.
+	sql.Register("pb_sqlite3",
+		&sqlite3.SQLiteDriver{
+			ConnectHook: func(conn *sqlite3.SQLiteConn) error {
+				_, err := conn.Exec(`
+					PRAGMA busy_timeout       = 10000;
+					PRAGMA journal_mode       = WAL;
+					PRAGMA journal_size_limit = 200000000;
+					PRAGMA synchronous        = NORMAL;
+					PRAGMA foreign_keys       = ON;
+				`, nil)
 
-	if err := initPragmas(db); err != nil {
-		db.Close()
+				return err
+			},
+		},
+	)
+
+	dbx.BuilderFuncMap["pb_sqlite3"] = dbx.BuilderFuncMap["sqlite3"]
+}
+
+func connectDB(dbPath string) (*dbx.DB, error) {
+	db, err := dbx.Open("pb_sqlite3", dbPath)
+	if err != nil {
 		return nil, err
 	}
 

core/db_nocgo.go

@@ -8,13 +8,13 @@ import (
 )
 
 func connectDB(dbPath string) (*dbx.DB, error) {
-	db, err := dbx.Open("sqlite", dbPath)
-	if err != nil {
-		return nil, err
-	}
+	// Note: the busy_timeout pragma must be first because
+	// the connection needs to be set to block on busy before WAL mode
+	// is set in case it hasn't been already set by another connection.
+	pragmas := "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=journal_size_limit(200000000)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)"
 
-	if err := initPragmas(db); err != nil {
-		db.Close()
+	db, err := dbx.Open("sqlite", dbPath+pragmas)
+	if err != nil {
 		return nil, err
 	}
 

core/events.go

@@ -14,6 +14,27 @@ import (
 	"github.com/pocketbase/pocketbase/tools/subscriptions"
 )
 
+var (
+	_ hook.Tagger = (*BaseModelEvent)(nil)
+	_ hook.Tagger = (*BaseCollectionEvent)(nil)
+)
+
+type BaseModelEvent struct {
+	Model models.Model
+}
+
+func (e *BaseModelEvent) Tags() []string {
+	if e.Model == nil {
+		return nil
+	}
+
+	if r, ok := e.Model.(*models.Record); ok && r.Collection() != nil {
+		return []string{r.Collection().Id, r.Collection().Name}
+	}
+
+	return []string{e.Model.TableName()}
+}
+
 type BaseCollectionEvent struct {
 	Collection *models.Collection
 }
@@ -44,6 +65,10 @@ type BootstrapEvent struct {
 	App App
 }
 
+type TerminateEvent struct {
+	App App
+}
+
 type ServeEvent struct {
 	App    App
 	Router *echo.Echo
@@ -58,23 +83,10 @@ type ApiErrorEvent struct {
 // Model DAO events data
 // -------------------------------------------------------------------
 
-var _ hook.Tagger = (*ModelEvent)(nil)
-
 type ModelEvent struct {
-	Dao   *daos.Dao
-	Model models.Model
-}
+	BaseModelEvent
 
-func (e *ModelEvent) Tags() []string {
-	if e.Model == nil {
-		return nil
-	}
-
-	if r, ok := e.Model.(*models.Record); ok && r.Collection() != nil {
-		return []string{r.Collection().Id, r.Collection().Name}
-	}
-
-	return []string{e.Model.TableName()}
+	Dao *daos.Dao
 }
 
 // -------------------------------------------------------------------
@@ -379,6 +391,13 @@ type CollectionsImportEvent struct {
 // File API events data
 // -------------------------------------------------------------------
 
+type FileTokenEvent struct {
+	BaseModelEvent
+
+	HttpContext echo.Context
+	Token       string
+}
+
 type FileDownloadEvent struct {
 	BaseCollectionEvent
 

daos/base.go

@@ -74,6 +74,13 @@ func (dao *Dao) NonconcurrentDB() dbx.Builder {
 	return dao.nonconcurrentDB
 }
 
+// Clone returns a new Dao with the same configuration options as the current one.
+func (dao *Dao) Clone() *Dao {
+	clone := *dao
+
+	return &clone
+}
+
 // ModelQuery creates a new preconfigured select query with preset
 // SELECT, FROM and other common fields based on the provided model.
 func (dao *Dao) ModelQuery(m models.Model) *dbx.SelectQuery {
@@ -110,6 +117,8 @@ func (dao *Dao) RunInTransaction(fn func(txDao *Dao) error) error {
 		// ---
 		// create a new dao with the same hooks to avoid semaphore deadlock when nesting
 		txDao := New(txOrDB)
+		txDao.MaxLockRetries = dao.MaxLockRetries
+		txDao.ModelQueryTimeout = dao.ModelQueryTimeout
 		txDao.BeforeCreateFunc = dao.BeforeCreateFunc
 		txDao.BeforeUpdateFunc = dao.BeforeUpdateFunc
 		txDao.BeforeDeleteFunc = dao.BeforeDeleteFunc
@@ -205,7 +214,10 @@ func (dao *Dao) Delete(m models.Model) error {
 	})
 }
 
-// Save upserts (update or create if primary key is not set) the provided model.
+// Save persists the provided model in the database.
+//
+// If m.IsNew() is true, the method will perform a create, otherwise an update.
+// To explicitly mark a model for update you can use m.MarkAsNotNew().
 func (dao *Dao) Save(m models.Model) error {
 	if m.IsNew() {
 		return dao.lockRetry(func(retryDao *Dao) error {

daos/base_test.go

@@ -40,6 +40,86 @@ func TestNewMultiDB(t *testing.T) {
 	}
 }
 
+func TestDaoClone(t *testing.T) {
+	testApp, _ := tests.NewTestApp()
+	defer testApp.Cleanup()
+
+	hookCalls := map[string]int{}
+
+	dao := daos.NewMultiDB(testApp.Dao().ConcurrentDB(), testApp.Dao().NonconcurrentDB())
+	dao.MaxLockRetries = 1
+	dao.ModelQueryTimeout = 2
+	dao.BeforeDeleteFunc = func(eventDao *daos.Dao, m models.Model) error {
+		hookCalls["BeforeDeleteFunc"]++
+		return nil
+	}
+	dao.BeforeUpdateFunc = func(eventDao *daos.Dao, m models.Model) error {
+		hookCalls["BeforeUpdateFunc"]++
+		return nil
+	}
+	dao.BeforeCreateFunc = func(eventDao *daos.Dao, m models.Model) error {
+		hookCalls["BeforeCreateFunc"]++
+		return nil
+	}
+	dao.AfterDeleteFunc = func(eventDao *daos.Dao, m models.Model) {
+		hookCalls["AfterDeleteFunc"]++
+	}
+	dao.AfterUpdateFunc = func(eventDao *daos.Dao, m models.Model) {
+		hookCalls["AfterUpdateFunc"]++
+	}
+	dao.AfterCreateFunc = func(eventDao *daos.Dao, m models.Model) {
+		hookCalls["AfterCreateFunc"]++
+	}
+
+	clone := dao.Clone()
+	clone.MaxLockRetries = 3
+	clone.ModelQueryTimeout = 4
+	clone.AfterCreateFunc = func(eventDao *daos.Dao, m models.Model) {
+		hookCalls["NewAfterCreateFunc"]++
+	}
+
+	if dao.MaxLockRetries == clone.MaxLockRetries {
+		t.Fatal("Expected different MaxLockRetries")
+	}
+
+	if dao.ModelQueryTimeout == clone.ModelQueryTimeout {
+		t.Fatal("Expected different ModelQueryTimeout")
+	}
+
+	// trigger hooks
+	dao.BeforeDeleteFunc(nil, nil)
+	dao.BeforeUpdateFunc(nil, nil)
+	dao.BeforeCreateFunc(nil, nil)
+	dao.AfterDeleteFunc(nil, nil)
+	dao.AfterUpdateFunc(nil, nil)
+	dao.AfterCreateFunc(nil, nil)
+	clone.BeforeDeleteFunc(nil, nil)
+	clone.BeforeUpdateFunc(nil, nil)
+	clone.BeforeCreateFunc(nil, nil)
+	clone.AfterDeleteFunc(nil, nil)
+	clone.AfterUpdateFunc(nil, nil)
+	clone.AfterCreateFunc(nil, nil)
+
+	expectations := []struct {
+		hook  string
+		total int
+	}{
+		{"BeforeDeleteFunc", 2},
+		{"BeforeUpdateFunc", 2},
+		{"BeforeCreateFunc", 2},
+		{"AfterDeleteFunc", 2},
+		{"AfterUpdateFunc", 2},
+		{"AfterCreateFunc", 1},
+		{"NewAfterCreateFunc", 1},
+	}
+
+	for _, e := range expectations {
+		if hookCalls[e.hook] != e.total {
+			t.Errorf("Expected %s to be caleed %d", e.hook, e.total)
+		}
+	}
+}
+
 func TestDaoModelQuery(t *testing.T) {
 	testApp, _ := tests.NewTestApp()
 	defer testApp.Cleanup()

daos/collection.go

@@ -159,8 +159,11 @@ func (dao *Dao) DeleteCollection(collection *models.Collection) error {
 	})
 }
 
-// SaveCollection upserts the provided Collection model and updates
+// SaveCollection persists the provided Collection model and updates
 // its related records table schema.
+//
+// If collecction.IsNew() is true, the method will perform a create, otherwise an update.
+// To explicitly mark a collection for update you can use collecction.MarkAsNotNew().
 func (dao *Dao) SaveCollection(collection *models.Collection) error {
 	var oldCollection *models.Collection
 

daos/collection_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/daos"
 	"github.com/pocketbase/pocketbase/models"
 	"github.com/pocketbase/pocketbase/models/schema"
@@ -166,9 +167,9 @@ func TestDeleteCollection(t *testing.T) {
 	app, _ := tests.NewTestApp()
 	defer app.Cleanup()
 
-	colEmpty := &models.Collection{}
+	colUnsaved := &models.Collection{}
 
-	colAuth, err := app.Dao().FindCollectionByNameOrId("clients")
+	colAuth, err := app.Dao().FindCollectionByNameOrId("users")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -187,6 +188,11 @@ func TestDeleteCollection(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	colBase, err := app.Dao().FindCollectionByNameOrId("demo1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	colView1, err := app.Dao().FindCollectionByNameOrId("view1")
 	if err != nil {
 		t.Fatal(err)
@@ -201,12 +207,14 @@ func TestDeleteCollection(t *testing.T) {
 		model       *models.Collection
 		expectError bool
 	}{
-		{colEmpty, true},
-		{colAuth, false},
+		{colUnsaved, true},
 		{colReferenced, true},
 		{colSystem, true},
 		{colView1, true}, // view2 depend on it
 		{colView2, false},
+		{colView1, false}, // no longer has dependent collections
+		{colBase, false},
+		{colAuth, false}, // should delete also its related external auths
 	}
 
 	for i, s := range scenarios {
@@ -225,6 +233,19 @@ func TestDeleteCollection(t *testing.T) {
 		if app.Dao().HasTable(s.model.Name) {
 			t.Errorf("[%d] Expected table/view %s to be deleted", i, s.model.Name)
 		}
+
+		// check if the external auths were deleted
+		if s.model.IsAuth() {
+			var total int
+			err := app.Dao().ExternalAuthQuery().
+				Select("count(*)").
+				AndWhere(dbx.HashExp{"collectionId": s.model.Id}).
+				Row(&total)
+
+			if err != nil || total > 0 {
+				t.Fatalf("[%d] Expected external auths to be deleted, got %v (%v)", i, total, err)
+			}
+		}
 	}
 }
 

daos/record.go

@@ -386,7 +386,10 @@ func (dao *Dao) SuggestUniqueAuthRecordUsername(
 	return username
 }
 
-// SaveRecord upserts the provided Record model.
+// SaveRecord persists the provided Record model in the database.
+//
+// If record.IsNew() is true, the method will perform a create, otherwise an update.
+// To explicitly mark a record for update you can use record.MarkAsNotNew().
 func (dao *Dao) SaveRecord(record *models.Record) error {
 	if record.Collection().IsAuth() {
 		if record.Username() == "" {

daos/view.go

@@ -127,7 +127,7 @@ func (dao *Dao) CreateViewSchema(selectQuery string) (schema.Schema, error) {
 		}
 
 		if !hasId {
-			return errors.New("missing required id column (you ca use `(ROW_NUMBER() OVER()) as id` if you don't have one)")
+			return errors.New("missing required id column (you can use `(ROW_NUMBER() OVER()) as id` if you don't have one)")
 		}
 
 		return nil
@@ -311,6 +311,7 @@ func (dao *Dao) parseQueryToFields(selectQuery string) (map[string]*queryField,
 
 		if field != nil {
 			clone := *field
+			clone.Id = "" // unset to prevent duplications if the same field is aliased multiple times
 			clone.Name = col.alias
 			result[col.alias] = &queryField{
 				field:      &clone,
@@ -462,6 +463,8 @@ func (p *identifiersParser) parse(selectQuery string) error {
 			skip = false
 			partType = "select"
 			activeBuilder = &selectParts
+		case "distinct":
+			continue // ignore as it is not important for the identifiers parsing
 		case "from":
 			skip = false
 			partType = "from"
@@ -477,7 +480,7 @@ func (p *identifiersParser) parse(selectQuery string) error {
 			partType = "join"
 			activeBuilder = &joinParts
 		case "_discard_":
-			// do nothing...
+			// skip following tokens
 			skip = true
 		default:
 			isJoin := partType == "join"

daos/view_test.go

@@ -395,8 +395,7 @@ func TestCreateViewSchema(t *testing.T) {
 				created created_alias,
 				updated updated_alias,
 				123 as custom
-			from demo1
-			`,
+			from demo1`,
 			false,
 			map[string]string{
 				"id2":           schema.FieldTypeRelation,
@@ -409,6 +408,37 @@ func TestCreateViewSchema(t *testing.T) {
 				"custom":        schema.FieldTypeJson,
 			},
 		},
+		{
+			"query with distinct and reordered id column",
+			`select distinct
+				id as id2,
+				id,
+				123 as custom
+			from demo1`,
+			false,
+			map[string]string{
+				"id2":    schema.FieldTypeRelation,
+				"custom": schema.FieldTypeJson,
+			},
+		},
+		{
+			"query with aliasing the same field multiple times",
+			`select
+				a.id as id,
+				a.text as alias1,
+				a.text as alias2,
+				b.text as alias3,
+				b.text as alias4
+			from demo1 a
+			left join demo1 as b`,
+			false,
+			map[string]string{
+				"alias1": schema.FieldTypeText,
+				"alias2": schema.FieldTypeText,
+				"alias3": schema.FieldTypeText,
+				"alias4": schema.FieldTypeText,
+			},
+		},
 	}
 
 	for _, s := range scenarios {

examples/base/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pocketbase/pocketbase"
 	"github.com/pocketbase/pocketbase/apis"
 	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/plugins/ghupdate"
 	"github.com/pocketbase/pocketbase/plugins/jsvm"
 	"github.com/pocketbase/pocketbase/plugins/migratecmd"
 )
@@ -79,6 +80,9 @@ func main() {
 		Dir:          migrationsDir,
 	})
 
+	// GitHub selfupdate
+	ghupdate.MustRegister(app, app.RootCmd, nil)
+
 	app.OnAfterBootstrap().Add(func(e *core.BootstrapEvent) error {
 		app.Dao().ModelQueryTimeout = time.Duration(queryTimeout) * time.Second
 		return nil

forms/backup_create.go

@@ -0,0 +1,79 @@
+package forms
+
+import (
+	"context"
+	"regexp"
+
+	validation "github.com/go-ozzo/ozzo-validation/v4"
+	"github.com/pocketbase/pocketbase/core"
+)
+
+var backupNameRegex = regexp.MustCompile(`^[a-z0-9_-]+\.zip$`)
+
+// BackupCreate is a request form for creating a new app backup.
+type BackupCreate struct {
+	app core.App
+	ctx context.Context
+
+	Name string `form:"name" json:"name"`
+}
+
+// NewBackupCreate creates new BackupCreate request form.
+func NewBackupCreate(app core.App) *BackupCreate {
+	return &BackupCreate{
+		app: app,
+		ctx: context.Background(),
+	}
+}
+
+// SetContext replaces the default form context with the provided one.
+func (form *BackupCreate) SetContext(ctx context.Context) {
+	form.ctx = ctx
+}
+
+// Validate makes the form validatable by implementing [validation.Validatable] interface.
+func (form *BackupCreate) Validate() error {
+	return validation.ValidateStruct(form,
+		validation.Field(
+			&form.Name,
+			validation.Length(1, 100),
+			validation.Match(backupNameRegex),
+			validation.By(form.checkUniqueName),
+		),
+	)
+}
+
+func (form *BackupCreate) checkUniqueName(value any) error {
+	v, _ := value.(string)
+	if v == "" {
+		return nil // nothing to check
+	}
+
+	fsys, err := form.app.NewBackupsFilesystem()
+	if err != nil {
+		return err
+	}
+	defer fsys.Close()
+
+	fsys.SetContext(form.ctx)
+
+	if exists, err := fsys.Exists(v); err != nil || exists {
+		return validation.NewError("validation_backup_name_exists", "The backup file name is invalid or already exists.")
+	}
+
+	return nil
+}
+
+// Submit validates the form and creates the app backup.
+//
+// You can optionally provide a list of InterceptorFunc to further
+// modify the form behavior before creating the backup.
+func (form *BackupCreate) Submit(interceptors ...InterceptorFunc[string]) error {
+	if err := form.Validate(); err != nil {
+		return err
+	}
+
+	return runInterceptors(form.Name, func(name string) error {
+		return form.app.CreateBackup(form.ctx, name)
+	}, interceptors...)
+}

forms/backup_create_test.go

@@ -0,0 +1,102 @@
+package forms_test
+
+import (
+	"strings"
+	"testing"
+
+	validation "github.com/go-ozzo/ozzo-validation/v4"
+	"github.com/pocketbase/pocketbase/forms"
+	"github.com/pocketbase/pocketbase/tests"
+)
+
+func TestBackupCreateValidateAndSubmit(t *testing.T) {
+	scenarios := []struct {
+		name           string
+		backupName     string
+		expectedErrors []string
+	}{
+		{
+			"invalid length",
+			strings.Repeat("a", 97) + ".zip",
+			[]string{"name"},
+		},
+		{
+			"valid length + invalid format",
+			strings.Repeat("a", 96),
+			[]string{"name"},
+		},
+		{
+			"valid length + valid format",
+			strings.Repeat("a", 96) + ".zip",
+			[]string{},
+		},
+		{
+			"auto generated name",
+			"",
+			[]string{},
+		},
+	}
+
+	for _, s := range scenarios {
+		func() {
+			app, _ := tests.NewTestApp()
+			defer app.Cleanup()
+
+			fsys, err := app.NewBackupsFilesystem()
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer fsys.Close()
+
+			form := forms.NewBackupCreate(app)
+			form.Name = s.backupName
+
+			result := form.Submit()
+
+			// parse errors
+			errs, ok := result.(validation.Errors)
+			if !ok && result != nil {
+				t.Errorf("[%s] Failed to parse errors %v", s.name, result)
+				return
+			}
+
+			// check errors
+			if len(errs) > len(s.expectedErrors) {
+				t.Errorf("[%s] Expected error keys %v, got %v", s.name, s.expectedErrors, errs)
+			}
+			for _, k := range s.expectedErrors {
+				if _, ok := errs[k]; !ok {
+					t.Errorf("[%s] Missing expected error key %q in %v", s.name, k, errs)
+				}
+			}
+
+			// retrieve all created backup files
+			files, err := fsys.List("")
+			if err != nil {
+				t.Errorf("[%s] Failed to retrieve backup files", s.name)
+				return
+			}
+
+			if result != nil {
+				if total := len(files); total != 0 {
+					t.Errorf("[%s] Didn't expected backup files, found %d", s.name, total)
+				}
+				return
+			}
+
+			if total := len(files); total != 1 {
+				t.Errorf("[%s] Expected 1 backup file, got %d", s.name, total)
+				return
+			}
+
+			if s.backupName == "" {
+				prefix := "pb_backup_"
+				if !strings.HasPrefix(files[0].Key, prefix) {
+					t.Errorf("[%s] Expected the backup file, to have prefix %q: %q", s.name, prefix, files[0].Key)
+				}
+			} else if s.backupName != files[0].Key {
+				t.Errorf("[%s] Expected backup file %q, got %q", s.name, s.backupName, files[0].Key)
+			}
+		}()
+	}
+}

forms/collections_import_test.go

@@ -206,7 +206,7 @@ func TestCollectionsImportSubmit(t *testing.T) {
 			expectError:            true,
 			expectCollectionsCount: totalCollections,
 			expectEvents: map[string]int{
-				"OnModelBeforeDelete": 7,
+				"OnModelBeforeDelete": 4,
 			},
 		},
 		{

forms/record_upsert.go

@@ -707,7 +707,7 @@ func (form *RecordUpsert) DrySubmit(callback func(txDao *daos.Dao) error) error
 		defer tx.Rollback()
 
 		if err := txDao.SaveRecord(form.record); err != nil {
-			return err
+			return form.prepareError(err)
 		}
 
 		// restore record isNew state
@@ -728,8 +728,6 @@ func (form *RecordUpsert) DrySubmit(callback func(txDao *daos.Dao) error) error
 // You can optionally provide a list of InterceptorFunc to further
 // modify the form behavior before persisting it.
 func (form *RecordUpsert) Submit(interceptors ...InterceptorFunc[*models.Record]) error {
-	oldRecord := form.record.OriginalCopy()
-
 	if err := form.ValidateAndFill(); err != nil {
 		return err
 	}
@@ -742,33 +740,46 @@ func (form *RecordUpsert) Submit(interceptors ...InterceptorFunc[*models.Record]
 			form.record.MarkAsNew()
 		}
 
-		// persist the record model
-		if err := form.dao.SaveRecord(form.record); err != nil {
-			preparedErr := form.prepareError(err)
-			if _, ok := preparedErr.(validation.Errors); ok {
-				return preparedErr
-			}
-			return fmt.Errorf("failed to save the record: %w", err)
-		}
+		dao := form.dao.Clone()
 
 		// upload new files (if any)
 		//
-		// note1: executed outside of transaction to avoid keeping a lock for too long
-		// note2: executed after the record save to allow record id change with a before model hook
-		if err := form.processFilesToUpload(); err != nil {
-			if oldRecord.IsNew() {
-				// delete previously inserted record
-				if err := form.dao.DeleteRecord(form.record); err != nil && form.app.IsDebug() {
-					log.Println(err)
-				}
-			} else {
-				// revert record changes
-				if err := form.dao.SaveRecord(oldRecord); err != nil && form.app.IsDebug() {
-					log.Println(err)
+		// note: executed after the default BeforeCreateFunc and BeforeUpdateFunc hooks
+		// to allow uploading AFTER the before app model hooks (eg. in case of an id change)
+		// but BEFORE the actual record db persistence
+		// ---
+		dao.BeforeCreateFunc = func(eventDao *daos.Dao, m models.Model) error {
+			if form.dao.BeforeCreateFunc != nil {
+				if err := form.dao.BeforeCreateFunc(eventDao, m); err != nil {
+					return err
 				}
 			}
 
-			return fmt.Errorf("failed to process the uploaded files: %w", err)
+			if m.TableName() == form.record.TableName() && m.GetId() == form.record.GetId() {
+				return form.processFilesToUpload()
+			}
+
+			return nil
+		}
+
+		dao.BeforeUpdateFunc = func(eventDao *daos.Dao, m models.Model) error {
+			if form.dao.BeforeUpdateFunc != nil {
+				if err := form.dao.BeforeUpdateFunc(eventDao, m); err != nil {
+					return err
+				}
+			}
+
+			if m.TableName() == form.record.TableName() && m.GetId() == form.record.GetId() {
+				return form.processFilesToUpload()
+			}
+
+			return nil
+		}
+		// ---
+
+		// persist the record model
+		if err := dao.SaveRecord(form.record); err != nil {
+			return form.prepareError(err)
 		}
 
 		// delete old files (if any)
@@ -789,7 +800,7 @@ func (form *RecordUpsert) processFilesToUpload() error {
 	}
 
 	if !form.record.HasId() {
-		return errors.New("the record is not persisted yet")
+		return errors.New("the record doesn't have an id")
 	}
 
 	fs, err := form.app.NewFilesystem()
@@ -837,7 +848,7 @@ func (form *RecordUpsert) deleteFilesByNamesList(filenames []string) ([]string,
 	}
 
 	if !form.record.HasId() {
-		return filenames, errors.New("the record doesn't have a unique ID")
+		return filenames, errors.New("the record doesn't have an id")
 	}
 
 	fs, err := form.app.NewFilesystem()

forms/record_upsert_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -1047,41 +1048,54 @@ func TestRecordUpsertUploadFailure(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// create with invalid file
-	{
-		new := models.NewRecord(collection)
-		new.Id = "123456789012341"
-
-		form := forms.NewRecordUpsert(app, new)
-		form.LoadData(map[string]any{"title": "new_test"})
-		form.AddFiles("files", &filesystem.File{Reader: &filesystem.PathReader{Path: "/tmp/__missing__"}})
-
-		if err := form.Submit(); err == nil {
-			t.Fatal("Expected error, got nil")
-		}
-
-		if r, err := app.Dao().FindRecordById(collection.Id, new.Id); err == nil {
-			t.Fatalf("Expected the inserted record to be deleted, found \n%v", r.PublicExport())
-		}
+	testDaos := []*daos.Dao{
+		app.Dao(),                // with hooks
+		daos.New(app.Dao().DB()), // without hooks
 	}
 
-	// update with invalid file
-	{
-		record, err := app.Dao().FindRecordById(collection.Id, "1tmknxy2868d869")
-		if err != nil {
-			t.Fatal(err)
+	for i, dao := range testDaos {
+		// create with invalid file
+		{
+			prefix := fmt.Sprintf("%d-create", i)
+
+			new := models.NewRecord(collection)
+			new.Id = "123456789012341"
+
+			form := forms.NewRecordUpsert(app, new)
+			form.SetDao(dao)
+			form.LoadData(map[string]any{"title": "new_test"})
+			form.AddFiles("files", &filesystem.File{Reader: &filesystem.PathReader{Path: "/tmp/__missing__"}})
+
+			if err := form.Submit(); err == nil {
+				t.Fatalf("[%s] Expected error, got nil", prefix)
+			}
+
+			if r, err := app.Dao().FindRecordById(collection.Id, new.Id); err == nil {
+				t.Fatalf("[%s] Expected the inserted record to be deleted, found \n%v", prefix, r.PublicExport())
+			}
 		}
 
-		form := forms.NewRecordUpsert(app, record)
-		form.LoadData(map[string]any{"title": "update_test"})
-		form.AddFiles("files", &filesystem.File{Reader: &filesystem.PathReader{Path: "/tmp/__missing__"}})
+		// update with invalid file
+		{
+			prefix := fmt.Sprintf("%d-update", i)
 
-		if err := form.Submit(); err == nil {
-			t.Fatal("Expected error, got nil")
-		}
+			record, err := app.Dao().FindRecordById(collection.Id, "1tmknxy2868d869")
+			if err != nil {
+				t.Fatal(err)
+			}
 
-		if r, _ := app.Dao().FindRecordById(collection.Id, record.Id); r == nil || r.GetString("title") == "update_test" {
-			t.Fatalf("Expected the record changes to be reverted, got \n%v", r.PublicExport())
+			form := forms.NewRecordUpsert(app, record)
+			form.SetDao(dao)
+			form.LoadData(map[string]any{"title": "update_test"})
+			form.AddFiles("files", &filesystem.File{Reader: &filesystem.PathReader{Path: "/tmp/__missing__"}})
+
+			if err := form.Submit(); err == nil {
+				t.Fatalf("[%s] Expected error, got nil", prefix)
+			}
+
+			if r, _ := app.Dao().FindRecordById(collection.Id, record.Id); r == nil || r.GetString("title") == "update_test" {
+				t.Fatalf("[%s] Expected the record changes to be reverted, got \n%v", prefix, r.PublicExport())
+			}
 		}
 	}
 }

forms/settings_upsert.go

@@ -58,8 +58,22 @@ func (form *SettingsUpsert) Submit(interceptors ...InterceptorFunc[*settings.Set
 	return runInterceptors(form.Settings, func(s *settings.Settings) error {
 		form.Settings = s
 
+		oldSettings, err := form.app.Settings().Clone();
+		if err != nil {
+			return err
+		}
+
+		// eagerly merge the application settings with the form ones
+		if err := form.app.Settings().Merge(form.Settings); err != nil {
+			return err
+		}
+
+		// persists settings change
 		encryptionKey := os.Getenv(form.app.EncryptionEnv())
 		if err := form.dao.SaveSettings(form.Settings, encryptionKey); err != nil {
+			// try to revert app settings
+			form.app.Settings().Merge(oldSettings)
+
 			return err
 		}
 
@@ -73,7 +87,6 @@ func (form *SettingsUpsert) Submit(interceptors ...InterceptorFunc[*settings.Set
 			form.app.LogsDao().Vacuum()
 		}
 
-		// merge the application settings with the form ones
-		return form.app.Settings().Merge(form.Settings)
+		return nil
 	}, interceptors...)
 }

forms/test_email_send_test.go

@@ -24,56 +24,58 @@ func TestEmailSendValidateAndSubmit(t *testing.T) {
 	}
 
 	for i, s := range scenarios {
-		app, _ := tests.NewTestApp()
-		defer app.Cleanup()
+		func() {
+			app, _ := tests.NewTestApp()
+			defer app.Cleanup()
 
-		form := forms.NewTestEmailSend(app)
-		form.Email = s.email
-		form.Template = s.template
+			form := forms.NewTestEmailSend(app)
+			form.Email = s.email
+			form.Template = s.template
 
-		result := form.Submit()
+			result := form.Submit()
 
-		// parse errors
-		errs, ok := result.(validation.Errors)
-		if !ok && result != nil {
-			t.Errorf("(%d) Failed to parse errors %v", i, result)
-			continue
-		}
-
-		// check errors
-		if len(errs) > len(s.expectedErrors) {
-			t.Errorf("(%d) Expected error keys %v, got %v", i, s.expectedErrors, errs)
-			continue
-		}
-		for _, k := range s.expectedErrors {
-			if _, ok := errs[k]; !ok {
-				t.Errorf("(%d) Missing expected error key %q in %v", i, k, errs)
-				continue
+			// parse errors
+			errs, ok := result.(validation.Errors)
+			if !ok && result != nil {
+				t.Errorf("(%d) Failed to parse errors %v", i, result)
+				return
 			}
-		}
 
-		expectedEmails := 1
-		if len(s.expectedErrors) > 0 {
-			expectedEmails = 0
-		}
+			// check errors
+			if len(errs) > len(s.expectedErrors) {
+				t.Errorf("(%d) Expected error keys %v, got %v", i, s.expectedErrors, errs)
+				return
+			}
+			for _, k := range s.expectedErrors {
+				if _, ok := errs[k]; !ok {
+					t.Errorf("(%d) Missing expected error key %q in %v", i, k, errs)
+					return
+				}
+			}
 
-		if app.TestMailer.TotalSend != expectedEmails {
-			t.Errorf("(%d) Expected %d email(s) to be sent, got %d", i, expectedEmails, app.TestMailer.TotalSend)
-		}
+			expectedEmails := 1
+			if len(s.expectedErrors) > 0 {
+				expectedEmails = 0
+			}
 
-		if len(s.expectedErrors) > 0 {
-			continue
-		}
+			if app.TestMailer.TotalSend != expectedEmails {
+				t.Errorf("(%d) Expected %d email(s) to be sent, got %d", i, expectedEmails, app.TestMailer.TotalSend)
+			}
 
-		expectedContent := "Verify"
-		if s.template == "password-reset" {
-			expectedContent = "Reset password"
-		} else if s.template == "email-change" {
-			expectedContent = "Confirm new email"
-		}
+			if len(s.expectedErrors) > 0 {
+				return
+			}
 
-		if !strings.Contains(app.TestMailer.LastMessage.HTML, expectedContent) {
-			t.Errorf("(%d) Expected the email to contains %s, got \n%v", i, expectedContent, app.TestMailer.LastMessage.HTML)
-		}
+			expectedContent := "Verify"
+			if s.template == "password-reset" {
+				expectedContent = "Reset password"
+			} else if s.template == "email-change" {
+				expectedContent = "Confirm new email"
+			}
+
+			if !strings.Contains(app.TestMailer.LastMessage.HTML, expectedContent) {
+				t.Errorf("(%d) Expected the email to contains %s, got \n%v", i, expectedContent, app.TestMailer.LastMessage.HTML)
+			}
+		}()
 	}
 }

forms/test_s3_filesystem.go

@@ -0,0 +1,88 @@
+package forms
+
+import (
+	"errors"
+	"fmt"
+
+	validation "github.com/go-ozzo/ozzo-validation/v4"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/models/settings"
+	"github.com/pocketbase/pocketbase/tools/filesystem"
+	"github.com/pocketbase/pocketbase/tools/security"
+)
+
+const (
+	s3FilesystemStorage = "storage"
+	s3FilesystemBackups = "backups"
+)
+
+// TestS3Filesystem defines a S3 filesystem connection test.
+type TestS3Filesystem struct {
+	app core.App
+
+	// The name of the filesystem - storage or backups
+	Filesystem string `form:"filesystem" json:"filesystem"`
+}
+
+// NewTestS3Filesystem creates and initializes new TestS3Filesystem form.
+func NewTestS3Filesystem(app core.App) *TestS3Filesystem {
+	return &TestS3Filesystem{app: app}
+}
+
+// Validate makes the form validatable by implementing [validation.Validatable] interface.
+func (form *TestS3Filesystem) Validate() error {
+	return validation.ValidateStruct(form,
+		validation.Field(
+			&form.Filesystem,
+			validation.Required,
+			validation.In(s3FilesystemStorage, s3FilesystemBackups),
+		),
+	)
+}
+
+// Submit validates and performs a S3 filesystem connection test.
+func (form *TestS3Filesystem) Submit() error {
+	if err := form.Validate(); err != nil {
+		return err
+	}
+
+	var s3Config settings.S3Config
+
+	if form.Filesystem == s3FilesystemBackups {
+		s3Config = form.app.Settings().Backups.S3
+	} else {
+		s3Config = form.app.Settings().S3
+	}
+
+	if !s3Config.Enabled {
+		return errors.New("S3 storage filesystem is not enabled")
+	}
+
+	fsys, err := filesystem.NewS3(
+		s3Config.Bucket,
+		s3Config.Region,
+		s3Config.Endpoint,
+		s3Config.AccessKey,
+		s3Config.Secret,
+		s3Config.ForcePathStyle,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to initialize the S3 filesystem: %w", err)
+	}
+	defer fsys.Close()
+
+	testPrefix := "pb_settings_test_" + security.PseudorandomString(5)
+	testFileKey := testPrefix + "/test.txt"
+
+	// try to upload a test file
+	if err := fsys.Upload([]byte("test"), testFileKey); err != nil {
+		return fmt.Errorf("failed to upload a test file: %w", err)
+	}
+
+	// test prefix deletion (ensures that both bucket list and delete works)
+	if errs := fsys.DeletePrefix(testPrefix); len(errs) > 0 {
+		return fmt.Errorf("failed to delete a test file: %w", errs[0])
+	}
+
+	return nil
+}

forms/test_s3_filesystem_test.go

@@ -0,0 +1,103 @@
+package forms_test
+
+import (
+	"testing"
+
+	validation "github.com/go-ozzo/ozzo-validation/v4"
+	"github.com/pocketbase/pocketbase/forms"
+	"github.com/pocketbase/pocketbase/tests"
+)
+
+func TestS3FilesystemValidate(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	scenarios := []struct {
+		name           string
+		filesystem     string
+		expectedErrors []string
+	}{
+		{
+			"empty filesystem",
+			"",
+			[]string{"filesystem"},
+		},
+		{
+			"invalid filesystem",
+			"something",
+			[]string{"filesystem"},
+		},
+		{
+			"backups filesystem",
+			"backups",
+			[]string{},
+		},
+		{
+			"storage filesystem",
+			"storage",
+			[]string{},
+		},
+	}
+
+	for _, s := range scenarios {
+		form := forms.NewTestS3Filesystem(app)
+		form.Filesystem = s.filesystem
+
+		result := form.Validate()
+
+		// parse errors
+		errs, ok := result.(validation.Errors)
+		if !ok && result != nil {
+			t.Errorf("[%s] Failed to parse errors %v", s.name, result)
+			continue
+		}
+
+		// check errors
+		if len(errs) > len(s.expectedErrors) {
+			t.Errorf("[%s] Expected error keys %v, got %v", s.name, s.expectedErrors, errs)
+			continue
+		}
+		for _, k := range s.expectedErrors {
+			if _, ok := errs[k]; !ok {
+				t.Errorf("[%s] Missing expected error key %q in %v", s.name, k, errs)
+			}
+		}
+	}
+}
+
+func TestS3FilesystemSubmitFailure(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	// check if validate was called
+	{
+		form := forms.NewTestS3Filesystem(app)
+		form.Filesystem = ""
+
+		result := form.Submit()
+
+		if result == nil {
+			t.Fatal("Expected error, got nil")
+		}
+
+		if _, ok := result.(validation.Errors); !ok {
+			t.Fatalf("Expected validation.Error, got %v", result)
+		}
+	}
+
+	// check with valid storage and disabled s3
+	{
+		form := forms.NewTestS3Filesystem(app)
+		form.Filesystem = "storage"
+
+		result := form.Submit()
+
+		if result == nil {
+			t.Fatal("Expected error, got nil")
+		}
+
+		if _, ok := result.(validation.Error); ok {
+			t.Fatalf("Didn't expect validation.Error, got %v", result)
+		}
+	}
+}

go.mod

@@ -3,88 +3,89 @@ module github.com/pocketbase/pocketbase
 go 1.18
 
 require (
-	github.com/AlecAivazis/survey/v2 v2.3.6
-	github.com/aws/aws-sdk-go v1.44.229
+	github.com/AlecAivazis/survey/v2 v2.3.7
+	github.com/aws/aws-sdk-go v1.44.289
 	github.com/disintegration/imaging v1.6.2
-	github.com/domodwyer/mailyak/v3 v3.5.0
-	github.com/dop251/goja v0.0.0-20221118162653-d4bf6fde1b86
-	github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6
+	github.com/domodwyer/mailyak/v3 v3.6.0
+	github.com/dop251/goja v0.0.0-20230427124612-428fc442ff5f
+	github.com/dop251/goja_nodejs v0.0.0-20230322100729-2550c7b6c124
 	github.com/fatih/color v1.15.0
 	github.com/gabriel-vasile/mimetype v1.4.2
 	github.com/ganigeorgiev/fexpr v0.3.0
 	github.com/go-ozzo/ozzo-validation/v4 v4.3.0
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/labstack/echo/v5 v5.0.0-20220201181537-ed2888cfa198
-	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/pocketbase/dbx v1.10.0
-	github.com/spf13/cast v1.5.0
-	github.com/spf13/cobra v1.6.1
-	gocloud.dev v0.29.0
-	golang.org/x/crypto v0.7.0
-	golang.org/x/net v0.8.0
-	golang.org/x/oauth2 v0.6.0
-	modernc.org/sqlite v1.21.0
+	github.com/spf13/cast v1.5.1
+	github.com/spf13/cobra v1.7.0
+	gocloud.dev v0.30.0
+	golang.org/x/crypto v0.10.0
+	golang.org/x/net v0.11.0
+	golang.org/x/oauth2 v0.9.0
+	modernc.org/sqlite v1.23.1
 )
 
 require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.27 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.70 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.31.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect
-	github.com/dlclark/regexp2 v1.7.0 // indirect
+	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.8.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/image v0.6.0 // indirect
-	golang.org/x/mod v0.9.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/image v0.8.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/term v0.9.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/tools v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.114.0 // indirect
+	google.golang.org/api v0.128.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 // indirect
-	google.golang.org/grpc v1.54.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/grpc v1.56.1 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	lukechampine.com/uint128 v1.3.0 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.3 // indirect
+	modernc.org/cc/v3 v3.41.0 // indirect
+	modernc.org/ccgo/v3 v3.16.14 // indirect
+	modernc.org/libc v1.24.1 // indirect
 	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
+	modernc.org/memory v1.6.0 // indirect
 	modernc.org/opt v0.1.3 // indirect
 	modernc.org/strutil v1.1.3 // indirect
 	modernc.org/token v1.1.0 // indirect

migrations/1640988000_init.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pocketbase/pocketbase/daos"
 	"github.com/pocketbase/pocketbase/models"
 	"github.com/pocketbase/pocketbase/models/schema"
+	"github.com/pocketbase/pocketbase/models/settings"
 	"github.com/pocketbase/pocketbase/tools/migrate"
 	"github.com/pocketbase/pocketbase/tools/types"
 )
@@ -90,6 +91,15 @@ func init() {
 			return tablesErr
 		}
 
+		dao := daos.New(db)
+
+		// inserts default settings
+		// -----------------------------------------------------------
+		defaultSettings := settings.New()
+		if err := dao.SaveSettings(defaultSettings); err != nil {
+			return err
+		}
+
 		// inserts the system profiles collection
 		// -----------------------------------------------------------
 		usersCollection := &models.Collection{}
@@ -139,7 +149,7 @@ func init() {
 			},
 		)
 
-		return daos.New(db).SaveCollection(usersCollection)
+		return dao.SaveCollection(usersCollection)
 	}, func(db dbx.Builder) error {
 		tables := []string{
 			"users",

migrations/1679943781_add_indexes_column.go

@@ -42,14 +42,19 @@ func init() {
 			return err
 		}
 
+		var hasIndexesColumn bool
 		for _, col := range cols {
 			if col == "indexes" {
-				return nil // already existing (probably via the init migration)
+				// already existing (probably via the init migration)
+				hasIndexesColumn = true
+				break
 			}
 		}
 
-		if _, err := db.AddColumn("_collections", "indexes", `JSON DEFAULT "[]" NOT NULL`).Execute(); err != nil {
-			return err
+		if !hasIndexesColumn {
+			if _, err := db.AddColumn("_collections", "indexes", `JSON DEFAULT "[]" NOT NULL`).Execute(); err != nil {
+				return err
+			}
 		}
 
 		collections := []*models.Collection{}

migrations/1685164450_check_fk.go

@@ -0,0 +1,20 @@
+package migrations
+
+import (
+	"github.com/pocketbase/dbx"
+)
+
+// Cleanup dangling deleted collections references
+// (see https://github.com/pocketbase/pocketbase/discussions/2570).
+func init() {
+	AppMigrations.Register(func(db dbx.Builder) error {
+		_, err := db.NewQuery(`
+			DELETE FROM {{_externalAuths}}
+			WHERE [[collectionId]] NOT IN (SELECT [[id]] FROM {{_collections}})
+		`).Execute()
+
+		return err
+	}, func(db dbx.Builder) error {
+		return nil
+	})
+}

models/backup_file_info.go

@@ -0,0 +1,9 @@
+package models
+
+import "github.com/pocketbase/pocketbase/tools/types"
+
+type BackupFileInfo struct {
+	Key      string         `json:"key"`
+	Size     int64          `json:"size"`
+	Modified types.DateTime `json:"modified"`
+}

models/record.go

@@ -279,9 +279,9 @@ func (m *Record) IgnoreEmailVisibility(state bool) {
 	m.ignoreEmailVisibility = state
 }
 
-// WithUnkownData toggles the export/serialization of unknown data fields
+// WithUnknownData toggles the export/serialization of unknown data fields
 // (false by default).
-func (m *Record) WithUnkownData(state bool) {
+func (m *Record) WithUnknownData(state bool) {
 	m.exportUnknown = state
 }
 

models/record_test.go

@@ -380,7 +380,7 @@ func TestRecordCleanCopy(t *testing.T) {
 	// allow the special flags and options to check whether they will be ignored
 	m.SetExpand(map[string]any{"test": 123})
 	m.IgnoreEmailVisibility(true)
-	m.WithUnkownData(true)
+	m.WithUnknownData(true)
 
 	copy := m.CleanCopy()
 	copyExport, _ := copy.MarshalJSON()
@@ -1343,7 +1343,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 
 		m.Load(data)
 		m.IgnoreEmailVisibility(s.exportHidden)
-		m.WithUnkownData(s.exportUnknown)
+		m.WithUnknownData(s.exportUnknown)
 
 		exportResult, err := json.Marshal(m.PublicExport())
 		if err != nil {

models/schema/schema_field.go

@@ -190,7 +190,7 @@ func (f SchemaField) Validate() error {
 
 	excludeNames := BaseModelFieldNames()
 	// exclude special filter literals
-	excludeNames = append(excludeNames, "null", "true", "false")
+	excludeNames = append(excludeNames, "null", "true", "false", "_rowid_")
 	// exclude system literals
 	excludeNames = append(excludeNames, SystemFieldNames()...)
 
@@ -597,6 +597,7 @@ type FileOptions struct {
 	MaxSize   int      `form:"maxSize" json:"maxSize"` // in bytes
 	MimeTypes []string `form:"mimeTypes" json:"mimeTypes"`
 	Thumbs    []string `form:"thumbs" json:"thumbs"`
+	Protected bool     `form:"protected" json:"protected"`
 }
 
 func (o FileOptions) Validate() error {

models/schema/schema_field_test.go

@@ -313,6 +313,15 @@ func TestSchemaFieldValidate(t *testing.T) {
 			},
 			[]string{"name"},
 		},
+		{
+			"reserved name (_rowid_)",
+			schema.SchemaField{
+				Type: schema.FieldTypeText,
+				Id:   "1234567890",
+				Name: "_rowid_",
+			},
+			[]string{"name"},
+		},
 		{
 			"reserved name (id)",
 			schema.SchemaField{
@@ -504,7 +513,7 @@ func TestSchemaFieldInitOptions(t *testing.T) {
 		{
 			schema.SchemaField{Type: schema.FieldTypeFile},
 			false,
-			`{"system":false,"id":"","name":"","type":"file","required":false,"unique":false,"options":{"maxSelect":0,"maxSize":0,"mimeTypes":null,"thumbs":null}}`,
+			`{"system":false,"id":"","name":"","type":"file","required":false,"unique":false,"options":{"maxSelect":0,"maxSize":0,"mimeTypes":null,"thumbs":null,"protected":false}}`,
 		},
 		{
 			schema.SchemaField{Type: schema.FieldTypeRelation},

models/settings/settings.go

@@ -10,6 +10,7 @@ import (
 	validation "github.com/go-ozzo/ozzo-validation/v4"
 	"github.com/go-ozzo/ozzo-validation/v4/is"
 	"github.com/pocketbase/pocketbase/tools/auth"
+	"github.com/pocketbase/pocketbase/tools/cron"
 	"github.com/pocketbase/pocketbase/tools/mailer"
 	"github.com/pocketbase/pocketbase/tools/rest"
 	"github.com/pocketbase/pocketbase/tools/security"
@@ -23,17 +24,20 @@ const SecretMask string = "******"
 type Settings struct {
 	mux sync.RWMutex
 
-	Meta MetaConfig `form:"meta" json:"meta"`
-	Logs LogsConfig `form:"logs" json:"logs"`
-	Smtp SmtpConfig `form:"smtp" json:"smtp"`
-	S3   S3Config   `form:"s3" json:"s3"`
+	Meta    MetaConfig    `form:"meta" json:"meta"`
+	Logs    LogsConfig    `form:"logs" json:"logs"`
+	Smtp    SmtpConfig    `form:"smtp" json:"smtp"`
+	S3      S3Config      `form:"s3" json:"s3"`
+	Backups BackupsConfig `form:"backups" json:"backups"`
 
 	AdminAuthToken           TokenConfig `form:"adminAuthToken" json:"adminAuthToken"`
 	AdminPasswordResetToken  TokenConfig `form:"adminPasswordResetToken" json:"adminPasswordResetToken"`
+	AdminFileToken           TokenConfig `form:"adminFileToken" json:"adminFileToken"`
 	RecordAuthToken          TokenConfig `form:"recordAuthToken" json:"recordAuthToken"`
 	RecordPasswordResetToken TokenConfig `form:"recordPasswordResetToken" json:"recordPasswordResetToken"`
 	RecordEmailChangeToken   TokenConfig `form:"recordEmailChangeToken" json:"recordEmailChangeToken"`
 	RecordVerificationToken  TokenConfig `form:"recordVerificationToken" json:"recordVerificationToken"`
+	RecordFileToken          TokenConfig `form:"recordFileToken" json:"recordFileToken"`
 
 	// Deprecated: Will be removed in v0.9+
 	EmailAuth EmailAuthConfig `form:"emailAuth" json:"emailAuth"`
@@ -82,29 +86,40 @@ func New() *Settings {
 			Password: "",
 			Tls:      false,
 		},
+		Backups: BackupsConfig{
+			CronMaxKeep: 3,
+		},
 		AdminAuthToken: TokenConfig{
 			Secret:   security.RandomString(50),
-			Duration: 1209600, // 14 days,
+			Duration: 1209600, // 14 days
 		},
 		AdminPasswordResetToken: TokenConfig{
 			Secret:   security.RandomString(50),
-			Duration: 1800, // 30 minutes,
+			Duration: 1800, // 30 minutes
+		},
+		AdminFileToken: TokenConfig{
+			Secret:   security.RandomString(50),
+			Duration: 120, // 2 minutes
 		},
 		RecordAuthToken: TokenConfig{
 			Secret:   security.RandomString(50),
-			Duration: 1209600, // 14 days,
+			Duration: 1209600, // 14 days
 		},
 		RecordPasswordResetToken: TokenConfig{
 			Secret:   security.RandomString(50),
-			Duration: 1800, // 30 minutes,
+			Duration: 1800, // 30 minutes
 		},
 		RecordVerificationToken: TokenConfig{
 			Secret:   security.RandomString(50),
-			Duration: 604800, // 7 days,
+			Duration: 604800, // 7 days
+		},
+		RecordFileToken: TokenConfig{
+			Secret:   security.RandomString(50),
+			Duration: 120, // 2 minutes
 		},
 		RecordEmailChangeToken: TokenConfig{
 			Secret:   security.RandomString(50),
-			Duration: 1800, // 30 minutes,
+			Duration: 1800, // 30 minutes
 		},
 		GoogleAuth: AuthProviderConfig{
 			Enabled: false,
@@ -173,12 +188,15 @@ func (s *Settings) Validate() error {
 		validation.Field(&s.Logs),
 		validation.Field(&s.AdminAuthToken),
 		validation.Field(&s.AdminPasswordResetToken),
+		validation.Field(&s.AdminFileToken),
 		validation.Field(&s.RecordAuthToken),
 		validation.Field(&s.RecordPasswordResetToken),
 		validation.Field(&s.RecordEmailChangeToken),
 		validation.Field(&s.RecordVerificationToken),
+		validation.Field(&s.RecordFileToken),
 		validation.Field(&s.Smtp),
 		validation.Field(&s.S3),
+		validation.Field(&s.Backups),
 		validation.Field(&s.GoogleAuth),
 		validation.Field(&s.FacebookAuth),
 		validation.Field(&s.GithubAuth),
@@ -233,12 +251,15 @@ func (s *Settings) RedactClone() (*Settings, error) {
 	sensitiveFields := []*string{
 		&clone.Smtp.Password,
 		&clone.S3.Secret,
+		&clone.Backups.S3.Secret,
 		&clone.AdminAuthToken.Secret,
 		&clone.AdminPasswordResetToken.Secret,
+		&clone.AdminFileToken.Secret,
 		&clone.RecordAuthToken.Secret,
 		&clone.RecordPasswordResetToken.Secret,
 		&clone.RecordEmailChangeToken.Secret,
 		&clone.RecordVerificationToken.Secret,
+		&clone.RecordFileToken.Secret,
 		&clone.GoogleAuth.ClientSecret,
 		&clone.FacebookAuth.ClientSecret,
 		&clone.GithubAuth.ClientSecret,
@@ -379,6 +400,51 @@ func (c S3Config) Validate() error {
 
 // -------------------------------------------------------------------
 
+type BackupsConfig struct {
+	// Cron is a cron expression to schedule auto backups, eg. "* * * * *".
+	//
+	// Leave it empty to disable the auto backups functionality.
+	Cron string `form:"cron" json:"cron"`
+
+	// CronMaxKeep is the the max number of cron generated backups to
+	// keep before removing older entries.
+	//
+	// This field works only when the cron config has valid cron expression.
+	CronMaxKeep int `form:"cronMaxKeep" json:"cronMaxKeep"`
+
+	// S3 is an optional S3 storage config specifying where to store the app backups.
+	S3 S3Config `form:"s3" json:"s3"`
+}
+
+// Validate makes BackupsConfig validatable by implementing [validation.Validatable] interface.
+func (c BackupsConfig) Validate() error {
+	return validation.ValidateStruct(&c,
+		validation.Field(&c.S3),
+		validation.Field(&c.Cron, validation.By(checkCronExpression)),
+		validation.Field(
+			&c.CronMaxKeep,
+			validation.When(c.Cron != "", validation.Required),
+			validation.Min(1),
+		),
+	)
+}
+
+func checkCronExpression(value any) error {
+	v, _ := value.(string)
+	if v == "" {
+		return nil // nothing to check
+	}
+
+	_, err := cron.NewSchedule(v)
+	if err != nil {
+		return validation.NewError("validation_invalid_cron", err.Error())
+	}
+
+	return nil
+}
+
+// -------------------------------------------------------------------
+
 type MetaConfig struct {
 	AppName                    string        `form:"appName" json:"appName"`
 	AppUrl                     string        `form:"appUrl" json:"appUrl"`

models/settings/settings_test.go

@@ -25,10 +25,12 @@ func TestSettingsValidate(t *testing.T) {
 	s.S3.Endpoint = "invalid"
 	s.AdminAuthToken.Duration = -10
 	s.AdminPasswordResetToken.Duration = -10
+	s.AdminFileToken.Duration = -10
 	s.RecordAuthToken.Duration = -10
 	s.RecordPasswordResetToken.Duration = -10
 	s.RecordEmailChangeToken.Duration = -10
 	s.RecordVerificationToken.Duration = -10
+	s.RecordFileToken.Duration = -10
 	s.GoogleAuth.Enabled = true
 	s.GoogleAuth.ClientId = ""
 	s.FacebookAuth.Enabled = true
@@ -79,10 +81,12 @@ func TestSettingsValidate(t *testing.T) {
 		`"s3":{`,
 		`"adminAuthToken":{`,
 		`"adminPasswordResetToken":{`,
+		`"adminFileToken":{`,
 		`"recordAuthToken":{`,
 		`"recordPasswordResetToken":{`,
 		`"recordEmailChangeToken":{`,
 		`"recordVerificationToken":{`,
+		`"recordFileToken":{`,
 		`"googleAuth":{`,
 		`"facebookAuth":{`,
 		`"githubAuth":{`,
@@ -123,12 +127,15 @@ func TestSettingsMerge(t *testing.T) {
 	s2.Smtp.Enabled = true
 	s2.S3.Enabled = true
 	s2.S3.Endpoint = "test"
+	s2.Backups.Cron = "* * * * *"
 	s2.AdminAuthToken.Duration = 1
 	s2.AdminPasswordResetToken.Duration = 2
+	s2.AdminFileToken.Duration = 2
 	s2.RecordAuthToken.Duration = 3
 	s2.RecordPasswordResetToken.Duration = 4
 	s2.RecordEmailChangeToken.Duration = 5
 	s2.RecordVerificationToken.Duration = 6
+	s2.RecordFileToken.Duration = 7
 	s2.GoogleAuth.Enabled = true
 	s2.GoogleAuth.ClientId = "google_test"
 	s2.FacebookAuth.Enabled = true
@@ -225,12 +232,15 @@ func TestSettingsRedactClone(t *testing.T) {
 	// secrets
 	s1.Smtp.Password = testSecret
 	s1.S3.Secret = testSecret
+	s1.Backups.S3.Secret = testSecret
 	s1.AdminAuthToken.Secret = testSecret
 	s1.AdminPasswordResetToken.Secret = testSecret
+	s1.AdminFileToken.Secret = testSecret
 	s1.RecordAuthToken.Secret = testSecret
 	s1.RecordPasswordResetToken.Secret = testSecret
 	s1.RecordEmailChangeToken.Secret = testSecret
 	s1.RecordVerificationToken.Secret = testSecret
+	s1.RecordFileToken.Secret = testSecret
 	s1.GoogleAuth.ClientSecret = testSecret
 	s1.FacebookAuth.ClientSecret = testSecret
 	s1.GithubAuth.ClientSecret = testSecret
@@ -602,6 +612,74 @@ func TestMetaConfigValidate(t *testing.T) {
 	}
 }
 
+func TestBackupsConfigValidate(t *testing.T) {
+	scenarios := []struct {
+		name           string
+		config         settings.BackupsConfig
+		expectedErrors []string
+	}{
+		{
+			"zero value",
+			settings.BackupsConfig{},
+			[]string{},
+		},
+		{
+			"invalid cron",
+			settings.BackupsConfig{
+				Cron:        "invalid",
+				CronMaxKeep: 0,
+			},
+			[]string{"cron", "cronMaxKeep"},
+		},
+		{
+			"invalid enabled S3",
+			settings.BackupsConfig{
+				S3: settings.S3Config{
+					Enabled: true,
+				},
+			},
+			[]string{"s3"},
+		},
+		{
+			"valid data",
+			settings.BackupsConfig{
+				S3: settings.S3Config{
+					Enabled:   true,
+					Endpoint:  "example.com",
+					Bucket:    "test",
+					Region:    "test",
+					AccessKey: "test",
+					Secret:    "test",
+				},
+				Cron:        "*/10 * * * *",
+				CronMaxKeep: 1,
+			},
+			[]string{},
+		},
+	}
+
+	for _, s := range scenarios {
+		result := s.config.Validate()
+
+		// parse errors
+		errs, ok := result.(validation.Errors)
+		if !ok && result != nil {
+			t.Errorf("[%s] Failed to parse errors %v", s.name, result)
+			continue
+		}
+
+		// check errors
+		if len(errs) > len(s.expectedErrors) {
+			t.Errorf("[%s] Expected error keys %v, got %v", s.name, s.expectedErrors, errs)
+		}
+		for _, k := range s.expectedErrors {
+			if _, ok := errs[k]; !ok {
+				t.Errorf("[%s] Missing expected error key %q in %v", s.name, k, errs)
+			}
+		}
+	}
+}
+
 func TestEmailTemplateValidate(t *testing.T) {
 	scenarios := []struct {
 		emailTemplate  settings.EmailTemplate

plugins/ghupdate/ghupdate.go

@@ -0,0 +1,372 @@
+// Package ghupdate implements a new command to selfupdate the current
+// PocketBase executable with the latest GitHub release.
+package ghupdate
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tools/archive"
+	"github.com/spf13/cobra"
+)
+
+// HttpClient is a base HTTP client interface (usually used for test purposes).
+type HttpClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+// Options defines optional struct to customize the default plugin behavior.
+//
+// NB! This plugin is considered experimental and its options may change in the future.
+type Options struct {
+	// Owner specifies the account owner of the repository (default to "pocketbase").
+	Owner string
+
+	// Repo specifies the name of the repository (default to "pocketbase").
+	Repo string
+
+	// ArchiveExecutable specifies the name of the executable file in the release archive
+	// (default to "pocketbase"; an additional ".exe" check is also performed as a fallback).
+	ArchiveExecutable string
+
+	// Optional context to use when fetching and downloading the latest release.
+	Context context.Context
+
+	// The HTTP client to use when fetching and downloading the latest release.
+	// Defaults to `http.DefaultClient`.
+	HttpClient HttpClient
+}
+
+// MustRegister registers the ghupdate plugin to the provided app instance
+// and panic if it fails.
+func MustRegister(app core.App, rootCmd *cobra.Command, options *Options) {
+	if err := Register(app, rootCmd, options); err != nil {
+		panic(err)
+	}
+}
+
+// Register registers the ghupdate plugin to the provided app instance.
+func Register(app core.App, rootCmd *cobra.Command, options *Options) error {
+	p := &plugin{
+		app:            app,
+		currentVersion: rootCmd.Version,
+	}
+
+	if options != nil {
+		p.options = options
+	} else {
+		p.options = &Options{}
+	}
+
+	if p.options.Owner == "" {
+		p.options.Owner = "pocketbase"
+	}
+
+	if p.options.Repo == "" {
+		p.options.Repo = "pocketbase"
+	}
+
+	if p.options.ArchiveExecutable == "" {
+		p.options.ArchiveExecutable = "pocketbase"
+	}
+
+	if p.options.HttpClient == nil {
+		p.options.HttpClient = http.DefaultClient
+	}
+
+	if p.options.Context == nil {
+		p.options.Context = context.Background()
+	}
+
+	rootCmd.AddCommand(p.updateCmd())
+
+	return nil
+}
+
+type plugin struct {
+	app            core.App
+	currentVersion string
+	options        *Options
+}
+
+func (p *plugin) updateCmd() *cobra.Command {
+	var withBackup bool
+
+	command := &cobra.Command{
+		Use:   "update",
+		Short: "Automatically updates the current PocketBase executable with the latest available version",
+		// @todo remove after logs generalization
+		// prevents printing the error log twice
+		SilenceErrors: true,
+		SilenceUsage:  true,
+		RunE: func(command *cobra.Command, args []string) error {
+			return p.update(withBackup)
+		},
+	}
+
+	command.PersistentFlags().BoolVar(
+		&withBackup,
+		"backup",
+		true,
+		"Creates a pb_data backup at the end of the update process",
+	)
+
+	return command
+}
+
+func (p *plugin) update(withBackup bool) error {
+	color.Yellow("Fetching release information...")
+
+	latest, err := fetchLatestRelease(
+		p.options.Context,
+		p.options.HttpClient,
+		p.options.Owner,
+		p.options.Repo,
+	)
+	if err != nil {
+		return err
+	}
+
+	if compareVersions(strings.TrimPrefix(p.currentVersion, "v"), strings.TrimPrefix(latest.Tag, "v")) <= 0 {
+		color.Green("You already have the latest PocketBase %s.", p.currentVersion)
+		return nil
+	}
+
+	suffix := archiveSuffix(runtime.GOOS, runtime.GOARCH)
+	if suffix == "" {
+		return errors.New("unsupported platform")
+	}
+
+	asset, err := latest.findAssetBySuffix(suffix)
+	if err != nil {
+		return err
+	}
+
+	releaseDir := filepath.Join(p.app.DataDir(), core.LocalTempDirName)
+	defer os.RemoveAll(releaseDir)
+
+	color.Yellow("Downloading %s...", asset.Name)
+
+	// download the release asset
+	assetZip := filepath.Join(releaseDir, asset.Name)
+	if err := downloadFile(p.options.Context, p.options.HttpClient, asset.DownloadUrl, assetZip); err != nil {
+		return err
+	}
+
+	color.Yellow("Extracting %s...", asset.Name)
+
+	extractDir := filepath.Join(releaseDir, "extracted_"+asset.Name)
+	defer os.RemoveAll(extractDir)
+
+	if err := archive.Extract(assetZip, extractDir); err != nil {
+		return err
+	}
+
+	color.Yellow("Replacing the executable...")
+
+	oldExec, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	renamedOldExec := oldExec + ".old"
+	defer os.Remove(renamedOldExec)
+
+	newExec := filepath.Join(extractDir, p.options.ArchiveExecutable)
+	if _, err := os.Stat(newExec); err != nil {
+		// try again with an .exe extension
+		newExec = newExec + ".exe"
+		if _, fallbackErr := os.Stat(newExec); fallbackErr != nil {
+			return fmt.Errorf("The executable in the extracted path is missing or it is inaccessible: %v, %v", err, fallbackErr)
+		}
+	}
+
+	// rename the current executable
+	if err := os.Rename(oldExec, renamedOldExec); err != nil {
+		return fmt.Errorf("Failed to rename the current executable: %w", err)
+	}
+
+	tryToRevertExecChanges := func() {
+		if revertErr := os.Rename(renamedOldExec, oldExec); revertErr != nil && p.app.IsDebug() {
+			log.Println(revertErr)
+		}
+	}
+
+	// replace with the extracted binary
+	if err := os.Rename(newExec, oldExec); err != nil {
+		tryToRevertExecChanges()
+		return fmt.Errorf("Failed replacing the executable: %w", err)
+	}
+
+	if withBackup {
+		color.Yellow("Creating pb_data backup...")
+
+		backupName := fmt.Sprintf("@update_%s.zip", latest.Tag)
+		if err := p.app.CreateBackup(p.options.Context, backupName); err != nil {
+			tryToRevertExecChanges()
+			return err
+		}
+	}
+
+	color.HiBlack("---")
+	color.Green("Update completed sucessfully! You can start the executable as usual.")
+
+	return nil
+}
+
+func fetchLatestRelease(
+	ctx context.Context,
+	client HttpClient,
+	owner string,
+	repo string,
+) (*release, error) {
+	url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo)
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	rawBody, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// http.Client doesn't treat non 2xx responses as error
+	if res.StatusCode >= 400 {
+		return nil, fmt.Errorf(
+			"(%d) failed to fetch latest releases:\n%s",
+			res.StatusCode,
+			string(rawBody),
+		)
+	}
+
+	result := &release{}
+	if err := json.Unmarshal(rawBody, result); err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func downloadFile(
+	ctx context.Context,
+	client HttpClient,
+	url string,
+	destPath string,
+) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return err
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	// http.Client doesn't treat non 2xx responses as error
+	if res.StatusCode >= 400 {
+		return fmt.Errorf("(%d) failed to send download file request", res.StatusCode)
+	}
+
+	// ensure that the dest parent dir(s) exist
+	if err := os.MkdirAll(filepath.Dir(destPath), os.ModePerm); err != nil {
+		return err
+	}
+
+	dest, err := os.Create(destPath)
+	if err != nil {
+		return err
+	}
+	defer dest.Close()
+
+	if _, err := io.Copy(dest, res.Body); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func archiveSuffix(goos, goarch string) string {
+	switch goos {
+	case "linux":
+		switch goarch {
+		case "amd64":
+			return "_linux_amd64.zip"
+		case "arm64":
+			return "_linux_arm64.zip"
+		case "arm":
+			return "_linux_armv7.zip"
+		}
+	case "darwin":
+		switch goarch {
+		case "amd64":
+			return "_darwin_amd64.zip"
+		case "arm64":
+			return "_darwin_arm64.zip"
+		}
+	case "windows":
+		switch goarch {
+		case "amd64":
+			return "_windows_amd64.zip"
+		case "arm64":
+			return "_windows_arm64.zip"
+		}
+	}
+
+	return ""
+}
+
+func compareVersions(a, b string) int {
+	aSplit := strings.Split(a, ".")
+	aTotal := len(aSplit)
+
+	bSplit := strings.Split(b, ".")
+	bTotal := len(bSplit)
+
+	limit := aTotal
+	if bTotal > aTotal {
+		limit = bTotal
+	}
+
+	for i := 0; i < limit; i++ {
+		var x, y int
+
+		if i < aTotal {
+			x, _ = strconv.Atoi(aSplit[i])
+		}
+
+		if i < bTotal {
+			y, _ = strconv.Atoi(bSplit[i])
+		}
+
+		if x < y {
+			return 1 // b is newer
+		}
+
+		if x > y {
+			return -1 // a is newer
+		}
+	}
+
+	return 0 // equal
+}

plugins/ghupdate/ghupdate_test.go

@@ -0,0 +1,32 @@
+package ghupdate
+
+import "testing"
+
+func TestCompareVersions(t *testing.T) {
+	scenarios := []struct {
+		a        string
+		b        string
+		expected int
+	}{
+		{"", "", 0},
+		{"0", "", 0},
+		{"1", "1.0.0", 0},
+		{"1.1", "1.1.0", 0},
+		{"1.1", "1.1.1", 1},
+		{"1.1", "1.0.1", -1},
+		{"1.0", "1.0.1", 1},
+		{"1.10", "1.9", -1},
+		{"1.2", "1.12", 1},
+		{"3.2", "1.6", -1},
+		{"3.2", "4.0", 1},
+		{"3.2.4", "3.2.3", -1},
+	}
+
+	for i, s := range scenarios {
+		result := compareVersions(s.a, s.b)
+
+		if result != s.expected {
+			t.Fatalf("[%d] Expected %q vs %q to result in %d, got %d", i, s.a, s.b, s.expected, result)
+		}
+	}
+}

plugins/ghupdate/release.go

@@ -0,0 +1,35 @@
+package ghupdate
+
+import (
+	"errors"
+	"strings"
+)
+
+type releaseAsset struct {
+	Id          int    `json:"id"`
+	Name        string `json:"name"`
+	Size        int    `json:"size"`
+	DownloadUrl string `json:"browser_download_url"`
+}
+
+type release struct {
+	Id        int             `json:"id"`
+	Name      string          `json:"name"`
+	Tag       string          `json:"tag_name"`
+	Published string          `json:"published_at"`
+	Url       string          `json:"html_url"`
+	Assets    []*releaseAsset `json:"assets"`
+}
+
+// findAssetBySuffix returns the first available asset containing the specified suffix.
+func (r *release) findAssetBySuffix(suffix string) (*releaseAsset, error) {
+	if suffix != "" {
+		for _, asset := range r.Assets {
+			if strings.HasSuffix(asset.Name, suffix) {
+				return asset, nil
+			}
+		}
+	}
+
+	return nil, errors.New("missing asset containing " + suffix)
+}

plugins/ghupdate/release_test.go

@@ -0,0 +1,23 @@
+package ghupdate
+
+import "testing"
+
+func TestReleaseFindAssetBySuffix(t *testing.T) {
+	r := release{
+		Assets: []*releaseAsset{
+			{Name: "test1.zip", Id: 1},
+			{Name: "test2.zip", Id: 2},
+			{Name: "test22.zip", Id: 22},
+			{Name: "test3.zip", Id: 3},
+		},
+	}
+
+	asset, err := r.findAssetBySuffix("2.zip")
+	if err != nil {
+		t.Fatalf("Expected nil, got err: %v", err)
+	}
+
+	if asset.Id != 2 {
+		t.Fatalf("Expected asset with id %d, got %v", 2, asset)
+	}
+}

plugins/migratecmd/migratecmd.go

@@ -17,13 +17,13 @@ package migratecmd
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
 	"time"
 
 	"github.com/AlecAivazis/survey/v2"
+	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/core"
 	"github.com/pocketbase/pocketbase/migrations"
 	"github.com/pocketbase/pocketbase/models"
@@ -103,7 +103,19 @@ func Register(app core.App, rootCmd *cobra.Command, options *Options) error {
 			// migrations but there is already at least 1 collection created,
 			// to ensure that the automigrate will work with up-to-date collections data
 			if !p.hasCustomMigrations() && len(cachedCollections) > 1 {
-				p.migrateCollectionsHandler(nil, false)
+				snapshotFile, err := p.migrateCollectionsHandler(nil, false)
+				if err != nil {
+					return err
+				}
+
+				// insert the snapshot migration entry
+				_, insertErr := p.app.Dao().NonconcurrentDB().Insert(migrate.DefaultMigrationsTable, dbx.Params{
+					"file":    snapshotFile,
+					"applied": time.Now().Unix(),
+				}).Execute()
+				if insertErr != nil {
+					return insertErr
+				}
 			}
 
 			return nil
@@ -129,9 +141,12 @@ func (p *plugin) createCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:       "migrate",
 		Short:     "Executes app DB migration scripts",
-		ValidArgs: []string{"up", "down", "create", "collections"},
 		Long:      cmdDesc,
-		Run: func(command *cobra.Command, args []string) {
+		ValidArgs: []string{"up", "down", "create", "collections"},
+		// prevents printing the error log twice
+		SilenceErrors: true,
+		SilenceUsage:  true,
+		RunE: func(command *cobra.Command, args []string) error {
 			cmd := ""
 			if len(args) > 0 {
 				cmd = args[0]
@@ -139,41 +154,42 @@ func (p *plugin) createCommand() *cobra.Command {
 
 			switch cmd {
 			case "create":
-				if err := p.migrateCreateHandler("", args[1:], true); err != nil {
-					log.Fatal(err)
+				if _, err := p.migrateCreateHandler("", args[1:], true); err != nil {
+					return err
 				}
 			case "collections":
-				if err := p.migrateCollectionsHandler(args[1:], true); err != nil {
-					log.Fatal(err)
+				if _, err := p.migrateCollectionsHandler(args[1:], true); err != nil {
+					return err
 				}
 			default:
 				runner, err := migrate.NewRunner(p.app.DB(), migrations.AppMigrations)
 				if err != nil {
-					log.Fatal(err)
+					return err
 				}
 
 				if err := runner.Run(args...); err != nil {
-					log.Fatal(err)
+					return err
 				}
 			}
+
+			return nil
 		},
 	}
 
 	return command
 }
 
-func (p *plugin) migrateCreateHandler(template string, args []string, interactive bool) error {
+func (p *plugin) migrateCreateHandler(template string, args []string, interactive bool) (string, error) {
 	if len(args) < 1 {
-		return fmt.Errorf("Missing migration file name")
+		return "", fmt.Errorf("Missing migration file name")
 	}
 
 	name := args[0]
 	dir := p.options.Dir
 
-	resultFilePath := path.Join(
-		dir,
-		fmt.Sprintf("%d_%s.%s", time.Now().Unix(), inflector.Snakecase(name), p.options.TemplateLang),
-	)
+	filename := fmt.Sprintf("%d_%s.%s", time.Now().Unix(), inflector.Snakecase(name), p.options.TemplateLang)
+
+	resultFilePath := path.Join(dir, filename)
 
 	if interactive {
 		confirm := false
@@ -183,7 +199,7 @@ func (p *plugin) migrateCreateHandler(template string, args []string, interactiv
 		survey.AskOne(prompt, &confirm)
 		if !confirm {
 			fmt.Println("The command has been cancelled")
-			return nil
+			return "", nil
 		}
 	}
 
@@ -196,34 +212,34 @@ func (p *plugin) migrateCreateHandler(template string, args []string, interactiv
 			template, templateErr = p.goBlankTemplate()
 		}
 		if templateErr != nil {
-			return fmt.Errorf("Failed to resolve create template: %v\n", templateErr)
+			return "", fmt.Errorf("Failed to resolve create template: %v\n", templateErr)
 		}
 	}
 
 	// ensure that the migrations dir exist
 	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
-		return err
+		return "", err
 	}
 
 	// save the migration file
 	if err := os.WriteFile(resultFilePath, []byte(template), 0644); err != nil {
-		return fmt.Errorf("Failed to save migration file %q: %v\n", resultFilePath, err)
+		return "", fmt.Errorf("Failed to save migration file %q: %v\n", resultFilePath, err)
 	}
 
 	if interactive {
 		fmt.Printf("Successfully created file %q\n", resultFilePath)
 	}
 
-	return nil
+	return filename, nil
 }
 
-func (p *plugin) migrateCollectionsHandler(args []string, interactive bool) error {
+func (p *plugin) migrateCollectionsHandler(args []string, interactive bool) (string, error) {
 	createArgs := []string{"collections_snapshot"}
 	createArgs = append(createArgs, args...)
 
 	collections := []*models.Collection{}
 	if err := p.app.Dao().CollectionQuery().OrderBy("created ASC").All(&collections); err != nil {
-		return fmt.Errorf("Failed to fetch migrations list: %v", err)
+		return "", fmt.Errorf("Failed to fetch migrations list: %v", err)
 	}
 
 	var template string
@@ -234,7 +250,7 @@ func (p *plugin) migrateCollectionsHandler(args []string, interactive bool) erro
 		template, templateErr = p.goSnapshotTemplate(collections)
 	}
 	if templateErr != nil {
-		return fmt.Errorf("Failed to resolve template: %v", templateErr)
+		return "", fmt.Errorf("Failed to resolve template: %v", templateErr)
 	}
 
 	return p.migrateCreateHandler(template, createArgs, interactive)

plugins/migratecmd/migratecmd_test.go

@@ -6,11 +6,14 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
 	"github.com/pocketbase/pocketbase/daos"
 	"github.com/pocketbase/pocketbase/models"
 	"github.com/pocketbase/pocketbase/models/schema"
 	"github.com/pocketbase/pocketbase/plugins/migratecmd"
 	"github.com/pocketbase/pocketbase/tests"
+	"github.com/pocketbase/pocketbase/tools/migrate"
 	"github.com/pocketbase/pocketbase/tools/types"
 )
 
@@ -776,3 +779,36 @@ func TestAutomigrateCollectionNoChanges(t *testing.T) {
 		}
 	}
 }
+
+func TestInitialAutoSnapshot(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	migrationsDir := filepath.Join(app.DataDir(), "_test_auto_snapshot_")
+
+	migratecmd.MustRegister(app, nil, &migratecmd.Options{
+		TemplateLang: migratecmd.TemplateLangJS,
+		Automigrate:  true,
+		Dir:          migrationsDir,
+	})
+
+	app.Bootstrap()
+
+	app.OnBeforeServe().Trigger(&core.ServeEvent{
+		App: app,
+	})
+
+	var foundFiles []string
+
+	err := app.Dao().NonconcurrentDB().Select("file").
+		From(migrate.DefaultMigrationsTable).
+		Where(dbx.NewExp("file like '%collections_snapshot.js'")).
+		Column(&foundFiles)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(foundFiles) != 1 {
+		t.Fatalf("Expected 1 collections_snapshot migration, found %v", foundFiles)
+	}
+}

pocketbase.go

@@ -1,14 +1,13 @@
 package pocketbase
 
 import (
-	"log"
 	"os"
 	"os/signal"
 	"path/filepath"
 	"strings"
-	"sync"
 	"syscall"
 
+	"github.com/fatih/color"
 	"github.com/pocketbase/pocketbase/cmd"
 	"github.com/pocketbase/pocketbase/core"
 	"github.com/pocketbase/pocketbase/tools/list"
@@ -136,8 +135,8 @@ func NewWithConfig(config *Config) *PocketBase {
 // commands (serve, migrate, version) and executes pb.RootCmd.
 func (pb *PocketBase) Start() error {
 	// register system commands
+	pb.RootCmd.AddCommand(cmd.NewAdminCommand(pb))
 	pb.RootCmd.AddCommand(cmd.NewServeCommand(pb, !pb.hideStartBanner))
-	pb.RootCmd.AddCommand(cmd.NewTempUpgradeCommand(pb))
 
 	return pb.Execute()
 }
@@ -154,35 +153,33 @@ func (pb *PocketBase) Execute() error {
 		}
 	}
 
-	var wg sync.WaitGroup
+	done := make(chan bool, 1)
 
-	wg.Add(1)
-
-	// wait for interrupt signal to gracefully shutdown the application
+	// listen for interrupt signal to gracefully shutdown the application
 	go func() {
-		defer wg.Done()
-		quit := make(chan os.Signal, 1) // we need to reserve to buffer size 1, so the notifier are not blocked
-		signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
-		<-quit
+		sigch := make(chan os.Signal, 1)
+		signal.Notify(sigch, os.Interrupt, syscall.SIGTERM)
+		<-sigch
+		done <- true
 	}()
 
 	// execute the root command
 	go func() {
-		defer wg.Done()
 		if err := pb.RootCmd.Execute(); err != nil {
-			log.Println(err)
+			// @todo replace with db log once generalized logs are added
+			// (note may need to update the existing commands to not silence errors)
+			color.Red(err.Error())
 		}
+
+		done <- true
 	}()
 
-	wg.Wait()
+	<-done
 
-	// cleanup
-	return pb.onTerminate()
-}
-
-// onTerminate tries to release the app resources on app termination.
-func (pb *PocketBase) onTerminate() error {
-	return pb.ResetBootstrapState()
+	// trigger app cleanups
+	return pb.OnTerminate().Trigger(&core.TerminateEvent{
+		App: pb,
+	})
 }
 
 // eagerParseFlags parses the global app flags before calling pb.RootCmd.Execute().
@@ -212,7 +209,7 @@ func (pb *PocketBase) eagerParseFlags(config *Config) error {
 	return pb.RootCmd.ParseFlags(os.Args[1:])
 }
 
-// skipBootstrap eagerly checks if the app should skip the bootstap process:
+// skipBootstrap eagerly checks if the app should skip the bootstrap process:
 // - already bootstrapped
 // - is unknown command
 // - is the default help command

resolvers/record_field_resolver_test.go

@@ -250,7 +250,7 @@ func TestRecordFieldResolverUpdateQuery(t *testing.T) {
 				"select_many:each ?< select_one:each &&" +
 				"select_many:each = @request.data.select_many:each",
 			false,
-			"SELECT DISTINCT `demo1`.* FROM `demo1` LEFT JOIN json_each(CASE WHEN json_valid([[demo1.select_one]]) THEN [[demo1.select_one]] ELSE json_array([[demo1.select_one]]) END) `demo1_select_one_je` LEFT JOIN json_each(CASE WHEN json_valid([[demo1.select_many]]) THEN [[demo1.select_many]] ELSE json_array([[demo1.select_many]]) END) `demo1_select_many_je` LEFT JOIN json_each({:TEST}) `__dataSelect_select_many_je` WHERE ((([[demo1_select_one_je.value]] != [[demo1_select_many_je.value]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_select_many_je.value]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.select_many]]) THEN [[__mm_demo1.select_many]] ELSE json_array([[__mm_demo1.select_many]]) END) `__mm_demo1_select_many_je` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__smTEST}} WHERE ((NOT ([[demo1_select_one_je.value]] != [[__smTEST.multiMatchValue]])) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND (([[demo1_select_many_je.value]] > [[demo1_select_one_je.value]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_select_many_je.value]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.select_many]]) THEN [[__mm_demo1.select_many]] ELSE json_array([[__mm_demo1.select_many]]) END) `__mm_demo1_select_many_je` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] > [[demo1_select_one_je.value]])) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND [[demo1_select_many_je.value]] < [[demo1_select_one_je.value]] AND (([[demo1_select_many_je.value]] = [[__dataSelect_select_many_je.value]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_select_many_je.value]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.select_many]]) THEN [[__mm_demo1.select_many]] ELSE json_array([[__mm_demo1.select_many]]) END) `__mm_demo1_select_many_je` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mlTEST}} LEFT JOIN (SELECT [[__mm__dataSelect_select_many_je.value]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each({:TEST}) `__mm__dataSelect_select_many_je` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mrTEST}} WHERE NOT ([[__mlTEST.multiMatchValue]] = [[__mrTEST.multiMatchValue]])))))",
+			"SELECT DISTINCT `demo1`.* FROM `demo1` LEFT JOIN json_each(CASE WHEN json_valid([[demo1.select_one]]) THEN [[demo1.select_one]] ELSE json_array([[demo1.select_one]]) END) `demo1_select_one_je` LEFT JOIN json_each(CASE WHEN json_valid([[demo1.select_many]]) THEN [[demo1.select_many]] ELSE json_array([[demo1.select_many]]) END) `demo1_select_many_je` LEFT JOIN json_each({:dataSelectTEST}) `__dataSelect_select_many_je` WHERE (((COALESCE([[demo1_select_one_je.value]], '') != COALESCE([[demo1_select_many_je.value]], '')) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_select_many_je.value]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.select_many]]) THEN [[__mm_demo1.select_many]] ELSE json_array([[__mm_demo1.select_many]]) END) `__mm_demo1_select_many_je` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__smTEST}} WHERE ((NOT (COALESCE([[demo1_select_one_je.value]], '') != COALESCE([[__smTEST.multiMatchValue]], ''))) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND (([[demo1_select_many_je.value]] > [[demo1_select_one_je.value]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_select_many_je.value]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.select_many]]) THEN [[__mm_demo1.select_many]] ELSE json_array([[__mm_demo1.select_many]]) END) `__mm_demo1_select_many_je` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] > [[demo1_select_one_je.value]])) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND [[demo1_select_many_je.value]] < [[demo1_select_one_je.value]] AND (([[demo1_select_many_je.value]] = [[__dataSelect_select_many_je.value]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_select_many_je.value]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.select_many]]) THEN [[__mm_demo1.select_many]] ELSE json_array([[__mm_demo1.select_many]]) END) `__mm_demo1_select_many_je` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mlTEST}} LEFT JOIN (SELECT [[__mm__dataSelect_select_many_je.value]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each({:mmdataSelectTEST}) `__mm__dataSelect_select_many_je` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mrTEST}} WHERE NOT (COALESCE([[__mlTEST.multiMatchValue]], '') = COALESCE([[__mrTEST.multiMatchValue]], ''))))))",
 		},
 		{
 			"mixed multi-match vs multi-match",
@@ -262,7 +262,7 @@ func TestRecordFieldResolverUpdateQuery(t *testing.T) {
 				"@collection.demo2.active ?= rel_many.rel.active &&" +
 				"rel_many.email > @request.data.rel_many.email",
 			false,
-			"SELECT DISTINCT `demo1`.* FROM `demo1` LEFT JOIN json_each(CASE WHEN json_valid([[demo1.rel_many]]) THEN [[demo1.rel_many]] ELSE json_array([[demo1.rel_many]]) END) `demo1_rel_many_je` LEFT JOIN `users` `demo1_rel_many` ON [[demo1_rel_many.id]] = [[demo1_rel_many_je.value]] LEFT JOIN `demo2` `demo1_rel_many_rel` ON [[demo1_rel_many_rel.id]] = [[demo1_rel_many.rel]] LEFT JOIN `demo1` `demo1_rel_one` ON [[demo1_rel_one.id]] = [[demo1.rel_one]] LEFT JOIN `demo2` `__collection_demo2` LEFT JOIN `users` `__data_users` ON [[__data_users.id]] IN ({:TEST}, {:TEST}) WHERE ((([[demo1_rel_many_rel.active]] != [[demo1_rel_many.name]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_rel_many_rel.active]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] LEFT JOIN `demo2` `__mm_demo1_rel_many_rel` ON [[__mm_demo1_rel_many_rel.id]] = [[__mm_demo1_rel_many.rel]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mlTEST}} LEFT JOIN (SELECT [[__mm_demo1_rel_many.name]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mrTEST}} WHERE ((NOT ([[__mlTEST.multiMatchValue]] != [[__mrTEST.multiMatchValue]])) OR ([[__mlTEST.multiMatchValue]] IS NULL) OR ([[__mrTEST.multiMatchValue]] IS NULL))))) AND [[demo1_rel_many_rel.active]] = [[demo1_rel_many.name]] AND (([[demo1_rel_many_rel.title]] LIKE ('%' || [[demo1_rel_one.email]] || '%') ESCAPE '\\') AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_rel_many_rel.title]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] LEFT JOIN `demo2` `__mm_demo1_rel_many_rel` ON [[__mm_demo1_rel_many_rel.id]] = [[__mm_demo1_rel_many.rel]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] LIKE ('%' || [[demo1_rel_one.email]] || '%') ESCAPE '\\')) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND (([[__collection_demo2.active]] = [[demo1_rel_many_rel.active]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm__collection_demo2.active]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN `demo2` `__mm__collection_demo2` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mlTEST}} LEFT JOIN (SELECT [[__mm_demo1_rel_many_rel.active]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] LEFT JOIN `demo2` `__mm_demo1_rel_many_rel` ON [[__mm_demo1_rel_many_rel.id]] = [[__mm_demo1_rel_many.rel]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mrTEST}} WHERE NOT ([[__mlTEST.multiMatchValue]] = [[__mrTEST.multiMatchValue]])))) AND [[__collection_demo2.active]] = [[demo1_rel_many_rel.active]] AND (((([[demo1_rel_many.email]] > [[__data_users.email]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_rel_many.email]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mlTEST}} LEFT JOIN (SELECT [[__data_mm_users.email]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN `users` `__data_mm_users` ON `__data_mm_users`.`id` IN ({:TEST}, {:TEST}) WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mrTEST}} WHERE ((NOT ([[__mlTEST.multiMatchValue]] > [[__mrTEST.multiMatchValue]])) OR ([[__mlTEST.multiMatchValue]] IS NULL) OR ([[__mrTEST.multiMatchValue]] IS NULL)))))) AND ([[demo1_rel_many.emailVisibility]] = TRUE)))",
+			"SELECT DISTINCT `demo1`.* FROM `demo1` LEFT JOIN json_each(CASE WHEN json_valid([[demo1.rel_many]]) THEN [[demo1.rel_many]] ELSE json_array([[demo1.rel_many]]) END) `demo1_rel_many_je` LEFT JOIN `users` `demo1_rel_many` ON [[demo1_rel_many.id]] = [[demo1_rel_many_je.value]] LEFT JOIN `demo2` `demo1_rel_many_rel` ON [[demo1_rel_many_rel.id]] = [[demo1_rel_many.rel]] LEFT JOIN `demo1` `demo1_rel_one` ON [[demo1_rel_one.id]] = [[demo1.rel_one]] LEFT JOIN `demo2` `__collection_demo2` LEFT JOIN `users` `__data_users` ON [[__data_users.id]] IN ({:TEST}, {:TEST}) WHERE (((COALESCE([[demo1_rel_many_rel.active]], '') != COALESCE([[demo1_rel_many.name]], '')) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_rel_many_rel.active]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] LEFT JOIN `demo2` `__mm_demo1_rel_many_rel` ON [[__mm_demo1_rel_many_rel.id]] = [[__mm_demo1_rel_many.rel]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mlTEST}} LEFT JOIN (SELECT [[__mm_demo1_rel_many.name]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mrTEST}} WHERE ((NOT (COALESCE([[__mlTEST.multiMatchValue]], '') != COALESCE([[__mrTEST.multiMatchValue]], ''))) OR ([[__mlTEST.multiMatchValue]] IS NULL) OR ([[__mrTEST.multiMatchValue]] IS NULL))))) AND COALESCE([[demo1_rel_many_rel.active]], '') = COALESCE([[demo1_rel_many.name]], '') AND (([[demo1_rel_many_rel.title]] LIKE ('%' || [[demo1_rel_one.email]] || '%') ESCAPE '\\') AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_rel_many_rel.title]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] LEFT JOIN `demo2` `__mm_demo1_rel_many_rel` ON [[__mm_demo1_rel_many_rel.id]] = [[__mm_demo1_rel_many.rel]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] LIKE ('%' || [[demo1_rel_one.email]] || '%') ESCAPE '\\')) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND ((COALESCE([[__collection_demo2.active]], '') = COALESCE([[demo1_rel_many_rel.active]], '')) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm__collection_demo2.active]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN `demo2` `__mm__collection_demo2` WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mlTEST}} LEFT JOIN (SELECT [[__mm_demo1_rel_many_rel.active]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] LEFT JOIN `demo2` `__mm_demo1_rel_many_rel` ON [[__mm_demo1_rel_many_rel.id]] = [[__mm_demo1_rel_many.rel]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mrTEST}} WHERE NOT (COALESCE([[__mlTEST.multiMatchValue]], '') = COALESCE([[__mrTEST.multiMatchValue]], ''))))) AND COALESCE([[__collection_demo2.active]], '') = COALESCE([[demo1_rel_many_rel.active]], '') AND (((([[demo1_rel_many.email]] > [[__data_users.email]]) AND (NOT EXISTS (SELECT 1 FROM (SELECT [[__mm_demo1_rel_many.email]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo1.rel_many]]) THEN [[__mm_demo1.rel_many]] ELSE json_array([[__mm_demo1.rel_many]]) END) `__mm_demo1_rel_many_je` LEFT JOIN `users` `__mm_demo1_rel_many` ON [[__mm_demo1_rel_many.id]] = [[__mm_demo1_rel_many_je.value]] WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mlTEST}} LEFT JOIN (SELECT [[__data_mm_users.email]] as [[multiMatchValue]] FROM `demo1` `__mm_demo1` LEFT JOIN `users` `__data_mm_users` ON `__data_mm_users`.`id` IN ({:TEST}, {:TEST}) WHERE `__mm_demo1`.`id` = `demo1`.`id`) {{__mrTEST}} WHERE ((NOT ([[__mlTEST.multiMatchValue]] > [[__mrTEST.multiMatchValue]])) OR ([[__mlTEST.multiMatchValue]] IS NULL) OR ([[__mrTEST.multiMatchValue]] IS NULL)))))) AND ([[demo1_rel_many.emailVisibility]] = TRUE)))",
 		},
 		{
 			"@request.data.arrayable:length fields",
@@ -294,14 +294,14 @@ func TestRecordFieldResolverUpdateQuery(t *testing.T) {
 				"self_rel_one.rel_many_cascade.files:length != 7 &&" +
 				"self_rel_one.rel_many_cascade.files:length ?!= 8",
 			false,
-			"SELECT DISTINCT `demo4`.* FROM `demo4` LEFT JOIN `demo4` `__data_demo4` ON [[__data_demo4.id]]={:TEST} LEFT JOIN `demo3` `__data_demo3` ON [[__data_demo3.id]] IN ({:TEST}, {:TEST}) LEFT JOIN `demo4` `demo4_self_rel_one` ON [[demo4_self_rel_one.id]] = [[demo4.self_rel_one]] LEFT JOIN json_each(CASE WHEN json_valid([[demo4_self_rel_one.rel_many_cascade]]) THEN [[demo4_self_rel_one.rel_many_cascade]] ELSE json_array([[demo4_self_rel_one.rel_many_cascade]]) END) `demo4_self_rel_one_rel_many_cascade_je` LEFT JOIN `demo3` `demo4_self_rel_one_rel_many_cascade` ON [[demo4_self_rel_one_rel_many_cascade.id]] = [[demo4_self_rel_one_rel_many_cascade_je.value]] WHERE (json_array_length(CASE WHEN json_valid([[__data_demo4.self_rel_many]]) THEN [[__data_demo4.self_rel_many]] ELSE json_array([[__data_demo4.self_rel_many]]) END) > {:TEST} AND json_array_length(CASE WHEN json_valid([[__data_demo4.self_rel_many]]) THEN [[__data_demo4.self_rel_many]] ELSE json_array([[__data_demo4.self_rel_many]]) END) > {:TEST} AND json_array_length(CASE WHEN json_valid([[__data_demo3.files]]) THEN [[__data_demo3.files]] ELSE json_array([[__data_demo3.files]]) END) < {:TEST} AND ((json_array_length(CASE WHEN json_valid([[__data_demo3.files]]) THEN [[__data_demo3.files]] ELSE json_array([[__data_demo3.files]]) END) < {:TEST}) AND (NOT EXISTS (SELECT 1 FROM (SELECT json_array_length(CASE WHEN json_valid([[__data_mm_demo3.files]]) THEN [[__data_mm_demo3.files]] ELSE json_array([[__data_mm_demo3.files]]) END) as [[multiMatchValue]] FROM `demo4` `__mm_demo4` LEFT JOIN `demo3` `__data_mm_demo3` ON `__data_mm_demo3`.`id` IN ({:TEST}, {:TEST}) WHERE `__mm_demo4`.`id` = `demo4`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] < {:TEST})) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND COALESCE(json_array_length(CASE WHEN json_valid([[demo4_self_rel_one.self_rel_many]]) THEN [[demo4_self_rel_one.self_rel_many]] ELSE json_array([[demo4_self_rel_one.self_rel_many]]) END), '') = {:TEST} AND COALESCE(json_array_length(CASE WHEN json_valid([[demo4_self_rel_one.self_rel_many]]) THEN [[demo4_self_rel_one.self_rel_many]] ELSE json_array([[demo4_self_rel_one.self_rel_many]]) END), '') = {:TEST} AND ((COALESCE(json_array_length(CASE WHEN json_valid([[demo4_self_rel_one_rel_many_cascade.files]]) THEN [[demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[demo4_self_rel_one_rel_many_cascade.files]]) END), '') != {:TEST}) AND (NOT EXISTS (SELECT 1 FROM (SELECT json_array_length(CASE WHEN json_valid([[__mm_demo4_self_rel_one_rel_many_cascade.files]]) THEN [[__mm_demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[__mm_demo4_self_rel_one_rel_many_cascade.files]]) END) as [[multiMatchValue]] FROM `demo4` `__mm_demo4` LEFT JOIN `demo4` `__mm_demo4_self_rel_one` ON [[__mm_demo4_self_rel_one.id]] = [[__mm_demo4.self_rel_one]] LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo4_self_rel_one.rel_many_cascade]]) THEN [[__mm_demo4_self_rel_one.rel_many_cascade]] ELSE json_array([[__mm_demo4_self_rel_one.rel_many_cascade]]) END) `__mm_demo4_self_rel_one_rel_many_cascade_je` LEFT JOIN `demo3` `__mm_demo4_self_rel_one_rel_many_cascade` ON [[__mm_demo4_self_rel_one_rel_many_cascade.id]] = [[__mm_demo4_self_rel_one_rel_many_cascade_je.value]] WHERE `__mm_demo4`.`id` = `demo4`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] != {:TEST})) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND COALESCE(json_array_length(CASE WHEN json_valid([[demo4_self_rel_one_rel_many_cascade.files]]) THEN [[demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[demo4_self_rel_one_rel_many_cascade.files]]) END), '') != {:TEST})",
+			"SELECT DISTINCT `demo4`.* FROM `demo4` LEFT JOIN `demo4` `__data_demo4` ON [[__data_demo4.id]]={:TEST} LEFT JOIN `demo3` `__data_demo3` ON [[__data_demo3.id]] IN ({:TEST}, {:TEST}) LEFT JOIN `demo4` `demo4_self_rel_one` ON [[demo4_self_rel_one.id]] = [[demo4.self_rel_one]] LEFT JOIN json_each(CASE WHEN json_valid([[demo4_self_rel_one.rel_many_cascade]]) THEN [[demo4_self_rel_one.rel_many_cascade]] ELSE json_array([[demo4_self_rel_one.rel_many_cascade]]) END) `demo4_self_rel_one_rel_many_cascade_je` LEFT JOIN `demo3` `demo4_self_rel_one_rel_many_cascade` ON [[demo4_self_rel_one_rel_many_cascade.id]] = [[demo4_self_rel_one_rel_many_cascade_je.value]] WHERE (json_array_length(CASE WHEN json_valid([[__data_demo4.self_rel_many]]) THEN [[__data_demo4.self_rel_many]] ELSE json_array([[__data_demo4.self_rel_many]]) END) > {:TEST} AND json_array_length(CASE WHEN json_valid([[__data_demo4.self_rel_many]]) THEN [[__data_demo4.self_rel_many]] ELSE json_array([[__data_demo4.self_rel_many]]) END) > {:TEST} AND json_array_length(CASE WHEN json_valid([[__data_demo3.files]]) THEN [[__data_demo3.files]] ELSE json_array([[__data_demo3.files]]) END) < {:TEST} AND ((json_array_length(CASE WHEN json_valid([[__data_demo3.files]]) THEN [[__data_demo3.files]] ELSE json_array([[__data_demo3.files]]) END) < {:TEST}) AND (NOT EXISTS (SELECT 1 FROM (SELECT json_array_length(CASE WHEN json_valid([[__data_mm_demo3.files]]) THEN [[__data_mm_demo3.files]] ELSE json_array([[__data_mm_demo3.files]]) END) as [[multiMatchValue]] FROM `demo4` `__mm_demo4` LEFT JOIN `demo3` `__data_mm_demo3` ON `__data_mm_demo3`.`id` IN ({:TEST}, {:TEST}) WHERE `__mm_demo4`.`id` = `demo4`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] < {:TEST})) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one.self_rel_many]]) THEN [[demo4_self_rel_one.self_rel_many]] ELSE json_array([[demo4_self_rel_one.self_rel_many]]) END) = {:TEST} AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one.self_rel_many]]) THEN [[demo4_self_rel_one.self_rel_many]] ELSE json_array([[demo4_self_rel_one.self_rel_many]]) END) = {:TEST} AND ((json_array_length(CASE WHEN json_valid([[demo4_self_rel_one_rel_many_cascade.files]]) THEN [[demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[demo4_self_rel_one_rel_many_cascade.files]]) END) != {:TEST}) AND (NOT EXISTS (SELECT 1 FROM (SELECT json_array_length(CASE WHEN json_valid([[__mm_demo4_self_rel_one_rel_many_cascade.files]]) THEN [[__mm_demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[__mm_demo4_self_rel_one_rel_many_cascade.files]]) END) as [[multiMatchValue]] FROM `demo4` `__mm_demo4` LEFT JOIN `demo4` `__mm_demo4_self_rel_one` ON [[__mm_demo4_self_rel_one.id]] = [[__mm_demo4.self_rel_one]] LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo4_self_rel_one.rel_many_cascade]]) THEN [[__mm_demo4_self_rel_one.rel_many_cascade]] ELSE json_array([[__mm_demo4_self_rel_one.rel_many_cascade]]) END) `__mm_demo4_self_rel_one_rel_many_cascade_je` LEFT JOIN `demo3` `__mm_demo4_self_rel_one_rel_many_cascade` ON [[__mm_demo4_self_rel_one_rel_many_cascade.id]] = [[__mm_demo4_self_rel_one_rel_many_cascade_je.value]] WHERE `__mm_demo4`.`id` = `demo4`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] != {:TEST})) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one_rel_many_cascade.files]]) THEN [[demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[demo4_self_rel_one_rel_many_cascade.files]]) END) != {:TEST})",
 		},
 		{
 			"json_extract and json_array_length COALESCE equal normalizations",
 			"demo4",
-			"json_object.a.b = 1 && self_rel_many:length != 2 && json_object.a.b > 3 && self_rel_many:length <= 4",
+			"json_object.a.b = '' && self_rel_many:length != 2 && json_object.a.b > 3 && self_rel_many:length <= 4",
 			false,
-			"SELECT `demo4`.* FROM `demo4` WHERE (COALESCE(JSON_EXTRACT([[demo4.json_object]], '$.a.b'), '') = {:TEST} AND COALESCE(json_array_length(CASE WHEN json_valid([[demo4.self_rel_many]]) THEN [[demo4.self_rel_many]] ELSE json_array([[demo4.self_rel_many]]) END), '') != {:TEST} AND JSON_EXTRACT([[demo4.json_object]], '$.a.b') > {:TEST} AND json_array_length(CASE WHEN json_valid([[demo4.self_rel_many]]) THEN [[demo4.self_rel_many]] ELSE json_array([[demo4.self_rel_many]]) END) <= {:TEST})",
+			"SELECT `demo4`.* FROM `demo4` WHERE ((JSON_EXTRACT([[demo4.json_object]], '$.a.b') = '' OR JSON_EXTRACT([[demo4.json_object]], '$.a.b') IS NULL) AND json_array_length(CASE WHEN json_valid([[demo4.self_rel_many]]) THEN [[demo4.self_rel_many]] ELSE json_array([[demo4.self_rel_many]]) END) != {:TEST} AND JSON_EXTRACT([[demo4.json_object]], '$.a.b') > {:TEST} AND json_array_length(CASE WHEN json_valid([[demo4.self_rel_many]]) THEN [[demo4.self_rel_many]] ELSE json_array([[demo4.self_rel_many]]) END) <= {:TEST})",
 		},
 	}
 

tests/api.go

@@ -159,6 +159,7 @@ func (scenario *ApiScenario) Test(t *testing.T) {
 		}
 	}
 
+	// @todo consider adding the response body to the AfterTestFunc args
 	if scenario.AfterTestFunc != nil {
 		scenario.AfterTestFunc(t, testApp, e)
 	}

tests/app.go

@@ -44,6 +44,7 @@ func (t *TestApp) Cleanup() {
 	}
 }
 
+// NewMailClient initializes test app mail client.
 func (t *TestApp) NewMailClient() mailer.Mailer {
 	t.mux.Lock()
 	defer t.mux.Unlock()
@@ -104,6 +105,14 @@ func NewTestApp(optTestDataDir ...string) (*TestApp, error) {
 		return nil, err
 	}
 
+	// ensure that the Dao and DB configurations are properly loaded
+	if _, err := app.Dao().DB().NewQuery("Select 1").Execute(); err != nil {
+		return nil, err
+	}
+	if _, err := app.LogsDao().DB().NewQuery("Select 1").Execute(); err != nil {
+		return nil, err
+	}
+
 	// force disable request logs because the logs db call execute in a separate
 	// go routine and it is possible to panic due to earlier api test completion.
 	app.Settings().Logs.MaxDays = 0
@@ -446,6 +455,14 @@ func NewTestApp(optTestDataDir ...string) (*TestApp, error) {
 		return t.registerEventCall("OnFileDownloadRequest")
 	})
 
+	t.OnFileBeforeTokenRequest().Add(func(e *core.FileTokenEvent) error {
+		return t.registerEventCall("OnFileBeforeTokenRequest")
+	})
+
+	t.OnFileAfterTokenRequest().Add(func(e *core.FileTokenEvent) error {
+		return t.registerEventCall("OnFileAfterTokenRequest")
+	})
+
 	return t, nil
 }
 

tokens/admin.go

@@ -24,3 +24,12 @@ func NewAdminResetPasswordToken(app core.App, admin *models.Admin) (string, erro
 		app.Settings().AdminPasswordResetToken.Duration,
 	)
 }
+
+// NewAdminFileToken generates and returns a new admin private file access token.
+func NewAdminFileToken(app core.App, admin *models.Admin) (string, error) {
+	return security.NewToken(
+		jwt.MapClaims{"id": admin.Id, "type": TypeAdmin},
+		(admin.TokenKey + app.Settings().AdminFileToken.Secret),
+		app.Settings().AdminFileToken.Duration,
+	)
+}

tokens/admin_test.go

@@ -52,3 +52,26 @@ func TestNewAdminResetPasswordToken(t *testing.T) {
 		t.Fatalf("Expected admin %v, got %v", admin, tokenAdmin)
 	}
 }
+
+func TestNewAdminFileToken(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	admin, err := app.Dao().FindAdminByEmail("test@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	token, err := tokens.NewAdminFileToken(app, admin)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tokenAdmin, _ := app.Dao().FindAdminByToken(
+		token,
+		app.Settings().AdminFileToken.Secret,
+	)
+	if tokenAdmin == nil || tokenAdmin.Id != admin.Id {
+		t.Fatalf("Expected admin %v, got %v", admin, tokenAdmin)
+	}
+}

tokens/record.go

@@ -76,3 +76,20 @@ func NewRecordChangeEmailToken(app core.App, record *models.Record, newEmail str
 		app.Settings().RecordEmailChangeToken.Duration,
 	)
 }
+
+// NewRecordFileToken generates and returns a new record private file access token.
+func NewRecordFileToken(app core.App, record *models.Record) (string, error) {
+	if !record.Collection().IsAuth() {
+		return "", errors.New("The record is not from an auth collection.")
+	}
+
+	return security.NewToken(
+		jwt.MapClaims{
+			"id":           record.Id,
+			"type":         TypeAuthRecord,
+			"collectionId": record.Collection().Id,
+		},
+		(record.TokenKey() + app.Settings().RecordFileToken.Secret),
+		app.Settings().RecordFileToken.Duration,
+	)
+}

tokens/record_test.go

@@ -98,3 +98,26 @@ func TestNewRecordChangeEmailToken(t *testing.T) {
 		t.Fatalf("Expected auth record %v, got %v", user, tokenRecord)
 	}
 }
+
+func TestNewRecordFileToken(t *testing.T) {
+	app, _ := tests.NewTestApp()
+	defer app.Cleanup()
+
+	user, err := app.Dao().FindAuthRecordByEmail("users", "test@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	token, err := tokens.NewRecordFileToken(app, user)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tokenRecord, _ := app.Dao().FindAuthRecordByToken(
+		token,
+		app.Settings().RecordFileToken.Secret,
+	)
+	if tokenRecord == nil || tokenRecord.Id != user.Id {
+		t.Fatalf("Expected auth record %v, got %v", user, tokenRecord)
+	}
+}

tools/archive/create.go

@@ -0,0 +1,87 @@
+package archive
+
+import (
+	"archive/zip"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// Create creates a new zip archive from src dir content and saves it in dest path.
+//
+// You can specify skipPaths to skip/ignore certain directories and files (relative to src)
+// preventing adding them in the final archive.
+func Create(src string, dest string, skipPaths ...string) error {
+	if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
+		return err
+	}
+
+	zf, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer zf.Close()
+
+	zw := zip.NewWriter(zf)
+	defer zw.Close()
+
+	if err := zipAddFS(zw, os.DirFS(src), skipPaths...); err != nil {
+		// try to cleanup at least the created zip file
+		os.Remove(dest)
+
+		return err
+	}
+
+	return nil
+}
+
+// note remove after similar method is added in the std lib (https://github.com/golang/go/issues/54898)
+func zipAddFS(w *zip.Writer, fsys fs.FS, skipPaths ...string) error {
+	return fs.WalkDir(fsys, ".", func(name string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		// skip
+		for _, ignore := range skipPaths {
+			if ignore == name ||
+				strings.HasPrefix(filepath.Clean(name)+string(os.PathSeparator), filepath.Clean(ignore)+string(os.PathSeparator)) {
+				return nil
+			}
+		}
+
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+
+		h, err := zip.FileInfoHeader(info)
+		if err != nil {
+			return err
+		}
+
+		h.Name = name
+		h.Method = zip.Deflate
+
+		fw, err := w.CreateHeader(h)
+		if err != nil {
+			return err
+		}
+
+		f, err := fsys.Open(name)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		_, err = io.Copy(fw, f)
+
+		return err
+	})
+}

tools/archive/create_test.go

@@ -0,0 +1,120 @@
+package archive_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pocketbase/pocketbase/tools/archive"
+)
+
+func TestCreateFailure(t *testing.T) {
+	testDir := createTestDir(t)
+	defer os.RemoveAll(testDir)
+
+	zipPath := filepath.Join(os.TempDir(), "pb_test.zip")
+	defer os.RemoveAll(zipPath)
+
+	missingDir := filepath.Join(os.TempDir(), "missing")
+
+	if err := archive.Create(missingDir, zipPath); err == nil {
+		t.Fatal("Expected to fail due to missing directory or file")
+	}
+
+	if _, err := os.Stat(zipPath); err == nil {
+		t.Fatalf("Expected the zip file not to be created")
+	}
+}
+
+func TestCreateSuccess(t *testing.T) {
+	testDir := createTestDir(t)
+	defer os.RemoveAll(testDir)
+
+	zipName := "pb_test.zip"
+	zipPath := filepath.Join(os.TempDir(), zipName)
+	defer os.RemoveAll(zipPath)
+
+	// zip testDir content (excluding test and a/b/c dir)
+	if err := archive.Create(testDir, zipPath, "a/b/c", "test"); err != nil {
+		t.Fatalf("Failed to create archive: %v", err)
+	}
+
+	info, err := os.Stat(zipPath)
+	if err != nil {
+		t.Fatalf("Failed to retrieve the generated zip file: %v", err)
+	}
+
+	if name := info.Name(); name != zipName {
+		t.Fatalf("Expected zip with name %q, got %q", zipName, name)
+	}
+
+	expectedSize := int64(405)
+	if size := info.Size(); size != expectedSize {
+		t.Fatalf("Expected zip with size %d, got %d", expectedSize, size)
+	}
+}
+
+// -------------------------------------------------------------------
+
+// note: make sure to call os.RemoveAll(dir) after you are done
+// working with the created test dir.
+func createTestDir(t *testing.T) string {
+	dir, err := os.MkdirTemp(os.TempDir(), "pb_zip_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.MkdirAll(filepath.Join(dir, "a/b/c"), os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "test"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "test2"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "a/test"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "a/b/sub1"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "a/b/c/sub2"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "a/b/c/sub3"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	return dir
+}

tools/archive/extract.go

@@ -0,0 +1,72 @@
+package archive
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// Extract extracts the zip archive at src to dest.
+func Extract(src, dest string) error {
+	zr, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer zr.Close()
+
+	// normalize dest path to check later for Zip Slip
+	dest = filepath.Clean(dest) + string(os.PathSeparator)
+
+	for _, f := range zr.File {
+		err := extractFile(f, dest)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// extractFile extracts the provided zipFile into "basePath/zipFileName" path,
+// creating all the necessary path directories.
+func extractFile(zipFile *zip.File, basePath string) error {
+	path := filepath.Join(basePath, zipFile.Name)
+
+	// check for Zip Slip
+	if !strings.HasPrefix(path, basePath) {
+		return fmt.Errorf("invalid file path: %s", path)
+	}
+
+	r, err := zipFile.Open()
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	if zipFile.FileInfo().IsDir() {
+		if err := os.MkdirAll(path, os.ModePerm); err != nil {
+			return err
+		}
+	} else {
+		// ensure that the file path directories are created
+		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+			return err
+		}
+
+		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, zipFile.Mode())
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		_, err = io.Copy(f, r)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

tools/archive/extract_test.go

@@ -0,0 +1,87 @@
+package archive_test
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pocketbase/pocketbase/tools/archive"
+)
+
+func TestExtractFailure(t *testing.T) {
+	testDir := createTestDir(t)
+	defer os.RemoveAll(testDir)
+
+	missingZipPath := filepath.Join(os.TempDir(), "pb_missing_test.zip")
+	extractedPath := filepath.Join(os.TempDir(), "pb_zip_extract")
+	defer os.RemoveAll(extractedPath)
+
+	if err := archive.Extract(missingZipPath, extractedPath); err == nil {
+		t.Fatal("Expected Extract to fail due to missing zipPath")
+	}
+
+	if _, err := os.Stat(extractedPath); err == nil {
+		t.Fatalf("Expected %q to not be created", extractedPath)
+	}
+}
+
+func TestExtractSuccess(t *testing.T) {
+	testDir := createTestDir(t)
+	defer os.RemoveAll(testDir)
+
+	zipPath := filepath.Join(os.TempDir(), "pb_test.zip")
+	defer os.RemoveAll(zipPath)
+
+	extractedPath := filepath.Join(os.TempDir(), "pb_zip_extract")
+	defer os.RemoveAll(extractedPath)
+
+	// zip testDir content (with exclude)
+	if err := archive.Create(testDir, zipPath, "a/b/c", "test", "sub2"); err != nil {
+		t.Fatalf("Failed to create archive: %v", err)
+	}
+
+	if err := archive.Extract(zipPath, extractedPath); err != nil {
+		t.Fatalf("Failed to extract %q in %q", zipPath, extractedPath)
+	}
+
+	availableFiles := []string{}
+
+	walkErr := filepath.WalkDir(extractedPath, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		availableFiles = append(availableFiles, path)
+
+		return nil
+	})
+	if walkErr != nil {
+		t.Fatalf("Failed to read the extracted dir: %v", walkErr)
+	}
+
+	expectedFiles := []string{
+		filepath.Join(extractedPath, "test2"),
+		filepath.Join(extractedPath, "a/test"),
+		filepath.Join(extractedPath, "a/b/sub1"),
+	}
+
+	if len(availableFiles) != len(expectedFiles) {
+		t.Fatalf("Expected \n%v, \ngot \n%v", expectedFiles, availableFiles)
+	}
+
+ExpectedLoop:
+	for _, expected := range expectedFiles {
+		for _, available := range availableFiles {
+			if available == expected {
+				continue ExpectedLoop
+			}
+		}
+
+		t.Fatalf("Missing file %q in \n%v", expected, availableFiles)
+	}
+}

tools/cron/cron.go

@@ -0,0 +1,173 @@
+// Package cron implements a crontab-like service to execute and schedule
+// repeative tasks/jobs.
+//
+// Example:
+//
+//	c := cron.New()
+//	c.MustAdd("dailyReport", "0 0 * * *", func() { ... })
+//	c.Start()
+package cron
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type job struct {
+	schedule *Schedule
+	run      func()
+}
+
+// Cron is a crontab-like struct for tasks/jobs scheduling.
+type Cron struct {
+	sync.RWMutex
+
+	interval time.Duration
+	timezone *time.Location
+	ticker   *time.Ticker
+	jobs     map[string]*job
+}
+
+// New create a new Cron struct with default tick interval of 1 minute
+// and timezone in UTC.
+//
+// You can change the default tick interval with Cron.SetInterval().
+// You can change the default timezone with Cron.SetTimezone().
+func New() *Cron {
+	return &Cron{
+		interval: 1 * time.Minute,
+		timezone: time.UTC,
+		jobs:     map[string]*job{},
+	}
+}
+
+// SetInterval changes the current cron tick interval
+// (it usually should be >= 1 minute).
+func (c *Cron) SetInterval(d time.Duration) {
+	// update interval
+	c.Lock()
+	wasStarted := c.ticker != nil
+	c.interval = d
+	c.Unlock()
+
+	// restart the ticker
+	if wasStarted {
+		c.Start()
+	}
+}
+
+// SetTimezone changes the current cron tick timezone.
+func (c *Cron) SetTimezone(l *time.Location) {
+	c.Lock()
+	defer c.Unlock()
+
+	c.timezone = l
+}
+
+// MustAdd is similar to Add() but panic on failure.
+func (c *Cron) MustAdd(jobId string, cronExpr string, run func()) {
+	if err := c.Add(jobId, cronExpr, run); err != nil {
+		panic(err)
+	}
+}
+
+// Add registers a single cron job.
+//
+// If there is already a job with the provided id, then the old job
+// will be replaced with the new one.
+//
+// cronExpr is a regular cron expression, eg. "0 */3 * * *" (aka. at minute 0 past every 3rd hour).
+// Check cron.NewSchedule() for the supported tokens.
+func (c *Cron) Add(jobId string, cronExpr string, run func()) error {
+	if run == nil {
+		return errors.New("failed to add new cron job: run must be non-nil function")
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	schedule, err := NewSchedule(cronExpr)
+	if err != nil {
+		return fmt.Errorf("failed to add new cron job: %w", err)
+	}
+
+	c.jobs[jobId] = &job{
+		schedule: schedule,
+		run:      run,
+	}
+
+	return nil
+}
+
+// Remove removes a single cron job by its id.
+func (c *Cron) Remove(jobId string) {
+	c.Lock()
+	defer c.Unlock()
+
+	delete(c.jobs, jobId)
+}
+
+// RemoveAll removes all registered cron jobs.
+func (c *Cron) RemoveAll() {
+	c.Lock()
+	defer c.Unlock()
+
+	c.jobs = map[string]*job{}
+}
+
+// Stop stops the current cron ticker (if not already).
+//
+// You can resume the ticker by calling Start().
+func (c *Cron) Stop() {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.ticker == nil {
+		return // already stopped
+	}
+
+	c.ticker.Stop()
+	c.ticker = nil
+}
+
+// Start starts the cron ticker.
+//
+// Calling Start() on already started cron will restart the ticker.
+func (c *Cron) Start() {
+	c.Stop()
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.ticker = time.NewTicker(c.interval)
+
+	go func() {
+		for t := range c.ticker.C {
+			c.runDue(t)
+		}
+	}()
+}
+
+// HasStarted checks whether the current Cron ticker has been started.
+func (c *Cron) HasStarted() bool {
+	c.RLock()
+	defer c.RUnlock()
+
+	return c.ticker != nil
+}
+
+// runDue runs all registered jobs that are scheduled for the provided time.
+func (c *Cron) runDue(t time.Time) {
+	c.RLock()
+	defer c.RUnlock()
+
+	moment := NewMoment(t.In(c.timezone))
+
+	for _, j := range c.jobs {
+		if j.schedule.IsDue(moment) {
+			go j.run()
+		}
+	}
+}

tools/cron/cron_test.go

@@ -0,0 +1,220 @@
+package cron
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+func TestCronNew(t *testing.T) {
+	c := New()
+
+	expectedInterval := 1 * time.Minute
+	if c.interval != expectedInterval {
+		t.Fatalf("Expected default interval %v, got %v", expectedInterval, c.interval)
+	}
+
+	expectedTimezone := time.UTC
+	if c.timezone.String() != expectedTimezone.String() {
+		t.Fatalf("Expected default timezone %v, got %v", expectedTimezone, c.timezone)
+	}
+
+	if len(c.jobs) != 0 {
+		t.Fatalf("Expected no jobs by default, got \n%v", c.jobs)
+	}
+
+	if c.ticker != nil {
+		t.Fatal("Expected the ticker NOT to be initialized")
+	}
+}
+
+func TestCronSetInterval(t *testing.T) {
+	c := New()
+
+	interval := 2 * time.Minute
+
+	c.SetInterval(interval)
+
+	if c.interval != interval {
+		t.Fatalf("Expected interval %v, got %v", interval, c.interval)
+	}
+}
+
+func TestCronSetTimezone(t *testing.T) {
+	c := New()
+
+	timezone, _ := time.LoadLocation("Asia/Tokyo")
+
+	c.SetTimezone(timezone)
+
+	if c.timezone.String() != timezone.String() {
+		t.Fatalf("Expected timezone %v, got %v", timezone, c.timezone)
+	}
+}
+
+func TestCronAddAndRemove(t *testing.T) {
+	c := New()
+
+	if err := c.Add("test0", "* * * * *", nil); err == nil {
+		t.Fatal("Expected nil function error")
+	}
+
+	if err := c.Add("test1", "invalid", func() {}); err == nil {
+		t.Fatal("Expected invalid cron expression error")
+	}
+
+	if err := c.Add("test2", "* * * * *", func() {}); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := c.Add("test3", "* * * * *", func() {}); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := c.Add("test4", "* * * * *", func() {}); err != nil {
+		t.Fatal(err)
+	}
+
+	// overwrite test2
+	if err := c.Add("test2", "1 2 3 4 5", func() {}); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := c.Add("test5", "1 2 3 4 5", func() {}); err != nil {
+		t.Fatal(err)
+	}
+
+	// mock job deletion
+	c.Remove("test4")
+
+	// try to remove non-existing (should be no-op)
+	c.Remove("missing")
+
+	// check job keys
+	{
+		expectedKeys := []string{"test3", "test2", "test5"}
+
+		if v := len(c.jobs); v != len(expectedKeys) {
+			t.Fatalf("Expected %d jobs, got %d", len(expectedKeys), v)
+		}
+
+		for _, k := range expectedKeys {
+			if c.jobs[k] == nil {
+				t.Fatalf("Expected job with key %s, got nil", k)
+			}
+		}
+	}
+
+	// check the jobs schedule
+	{
+		expectedSchedules := map[string]string{
+			"test2": `{"minutes":{"1":{}},"hours":{"2":{}},"days":{"3":{}},"months":{"4":{}},"daysOfWeek":{"5":{}}}`,
+			"test3": `{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+			"test5": `{"minutes":{"1":{}},"hours":{"2":{}},"days":{"3":{}},"months":{"4":{}},"daysOfWeek":{"5":{}}}`,
+		}
+		for k, v := range expectedSchedules {
+			raw, err := json.Marshal(c.jobs[k].schedule)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if string(raw) != v {
+				t.Fatalf("Expected %q schedule \n%s, \ngot \n%s", k, v, raw)
+			}
+		}
+	}
+}
+
+func TestCronMustAdd(t *testing.T) {
+	c := New()
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Errorf("test1 didn't panic")
+		}
+	}()
+
+	c.MustAdd("test1", "* * * * *", nil)
+
+	c.MustAdd("test2", "* * * * *", func() {})
+
+	if _, ok := c.jobs["test2"]; !ok {
+		t.Fatal("Couldn't find job test2")
+	}
+}
+
+func TestCronRemoveAll(t *testing.T) {
+	c := New()
+
+	if err := c.Add("test1", "* * * * *", func() {}); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := c.Add("test2", "* * * * *", func() {}); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := c.Add("test3", "* * * * *", func() {}); err != nil {
+		t.Fatal(err)
+	}
+
+	if v := len(c.jobs); v != 3 {
+		t.Fatalf("Expected %d jobs, got %d", 3, v)
+	}
+
+	c.RemoveAll()
+
+	if v := len(c.jobs); v != 0 {
+		t.Fatalf("Expected %d jobs, got %d", 0, v)
+	}
+}
+
+func TestCronStartStop(t *testing.T) {
+	c := New()
+
+	c.SetInterval(1 * time.Second)
+
+	test1 := 0
+	test2 := 0
+
+	c.Add("test1", "* * * * *", func() {
+		test1++
+	})
+
+	c.Add("test2", "* * * * *", func() {
+		test2++
+	})
+
+	expectedCalls := 3
+
+	// call twice Start to check if the previous ticker will be reseted
+	c.Start()
+	c.Start()
+
+	time.Sleep(3250 * time.Millisecond)
+
+	// call twice Stop to ensure that the second stop is no-op
+	c.Stop()
+	c.Stop()
+
+	if test1 != expectedCalls {
+		t.Fatalf("Expected %d test1, got %d", expectedCalls, test1)
+	}
+	if test2 != expectedCalls {
+		t.Fatalf("Expected %d test2, got %d", expectedCalls, test2)
+	}
+
+	// resume for ~5 seconds
+	c.Start()
+	time.Sleep(5250 * time.Millisecond)
+	c.Stop()
+
+	expectedCalls += 5
+
+	if test1 != expectedCalls {
+		t.Fatalf("Expected %d test1, got %d", expectedCalls, test1)
+	}
+	if test2 != expectedCalls {
+		t.Fatalf("Expected %d test2, got %d", expectedCalls, test2)
+	}
+}

tools/cron/schedule.go

@@ -0,0 +1,194 @@
+package cron
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Moment represents a parsed single time moment.
+type Moment struct {
+	Minute    int `json:"minute"`
+	Hour      int `json:"hour"`
+	Day       int `json:"day"`
+	Month     int `json:"month"`
+	DayOfWeek int `json:"dayOfWeek"`
+}
+
+// NewMoment creates a new Moment from the specified time.
+func NewMoment(t time.Time) *Moment {
+	return &Moment{
+		Minute:    t.Minute(),
+		Hour:      t.Hour(),
+		Day:       t.Day(),
+		Month:     int(t.Month()),
+		DayOfWeek: int(t.Weekday()),
+	}
+}
+
+// Schedule stores parsed information for each time component when a cron job should run.
+type Schedule struct {
+	Minutes    map[int]struct{} `json:"minutes"`
+	Hours      map[int]struct{} `json:"hours"`
+	Days       map[int]struct{} `json:"days"`
+	Months     map[int]struct{} `json:"months"`
+	DaysOfWeek map[int]struct{} `json:"daysOfWeek"`
+}
+
+// IsDue checks whether the provided Moment satisfies the current Schedule.
+func (s *Schedule) IsDue(m *Moment) bool {
+	if _, ok := s.Minutes[m.Minute]; !ok {
+		return false
+	}
+
+	if _, ok := s.Hours[m.Hour]; !ok {
+		return false
+	}
+
+	if _, ok := s.Days[m.Day]; !ok {
+		return false
+	}
+
+	if _, ok := s.DaysOfWeek[m.DayOfWeek]; !ok {
+		return false
+	}
+
+	if _, ok := s.Months[m.Month]; !ok {
+		return false
+	}
+
+	return true
+}
+
+// NewSchedule creates a new Schedule from a cron expression.
+//
+// A cron expression is consisted of 5 segments separated by space,
+// representing: minute, hour, day of the month, month and day of the week.
+//
+// Each segment could be in the following formats:
+//   - wildcard: *
+//   - range:    1-30
+//   - step:     */n or 1-30/n
+//   - list:     1,2,3,10-20/n
+func NewSchedule(cronExpr string) (*Schedule, error) {
+	segments := strings.Split(cronExpr, " ")
+	if len(segments) != 5 {
+		return nil, errors.New("invalid cron expression - must have exactly 5 space separated segments")
+	}
+
+	minutes, err := parseCronSegment(segments[0], 0, 59)
+	if err != nil {
+		return nil, err
+	}
+
+	hours, err := parseCronSegment(segments[1], 0, 23)
+	if err != nil {
+		return nil, err
+	}
+
+	days, err := parseCronSegment(segments[2], 1, 31)
+	if err != nil {
+		return nil, err
+	}
+
+	months, err := parseCronSegment(segments[3], 1, 12)
+	if err != nil {
+		return nil, err
+	}
+
+	daysOfWeek, err := parseCronSegment(segments[4], 0, 6)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Schedule{
+		Minutes:    minutes,
+		Hours:      hours,
+		Days:       days,
+		Months:     months,
+		DaysOfWeek: daysOfWeek,
+	}, nil
+}
+
+// parseCronSegment parses a single cron expression segment and
+// returns its time schedule slots.
+func parseCronSegment(segment string, min int, max int) (map[int]struct{}, error) {
+	slots := map[int]struct{}{}
+
+	list := strings.Split(segment, ",")
+	for _, p := range list {
+		stepParts := strings.Split(p, "/")
+
+		// step (*/n, 1-30/n)
+		var step int
+		switch len(stepParts) {
+		case 1:
+			step = 1
+		case 2:
+			parsedStep, err := strconv.Atoi(stepParts[1])
+			if err != nil {
+				return nil, err
+			}
+			if parsedStep < 1 || parsedStep > max {
+				return nil, fmt.Errorf("invalid segment step boundary - the step must be between 1 and the %d", max)
+			}
+			step = parsedStep
+		default:
+			return nil, errors.New("invalid segment step format - must be in the format */n or 1-30/n")
+		}
+
+		// find the min and max range of the segment part
+		var rangeMin, rangeMax int
+		if stepParts[0] == "*" {
+			rangeMin = min
+			rangeMax = max
+		} else {
+			// single digit (1) or range (1-30)
+			rangeParts := strings.Split(stepParts[0], "-")
+			switch len(rangeParts) {
+			case 1:
+				if step != 1 {
+					return nil, errors.New("invalid segement step - step > 1 could be used only with the wildcard or range format")
+				}
+				parsed, err := strconv.Atoi(rangeParts[0])
+				if err != nil {
+					return nil, err
+				}
+				if parsed < min || parsed > max {
+					return nil, errors.New("invalid segment value - must be between the min and max of the segment")
+				}
+				rangeMin = parsed
+				rangeMax = rangeMin
+			case 2:
+				parsedMin, err := strconv.Atoi(rangeParts[0])
+				if err != nil {
+					return nil, err
+				}
+				if parsedMin < min || parsedMin > max {
+					return nil, fmt.Errorf("invalid segment range minimum - must be between %d and %d", min, max)
+				}
+				rangeMin = parsedMin
+
+				parsedMax, err := strconv.Atoi(rangeParts[1])
+				if err != nil {
+					return nil, err
+				}
+				if parsedMax < parsedMin || parsedMax > max {
+					return nil, fmt.Errorf("invalid segment range maximum - must be between %d and %d", rangeMin, max)
+				}
+				rangeMax = parsedMax
+			default:
+				return nil, errors.New("invalid segment range format - the range must have 1 or 2 parts")
+			}
+		}
+
+		// fill the slots
+		for i := rangeMin; i <= rangeMax; i += step {
+			slots[i] = struct{}{}
+		}
+	}
+
+	return slots, nil
+}

tools/cron/schedule_test.go

@@ -0,0 +1,361 @@
+package cron_test
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/pocketbase/pocketbase/tools/cron"
+)
+
+func TestNewMoment(t *testing.T) {
+	date, err := time.Parse("2006-01-02 15:04", "2023-05-09 15:20")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := cron.NewMoment(date)
+
+	if m.Minute != 20 {
+		t.Fatalf("Expected m.Minute %d, got %d", 20, m.Minute)
+	}
+
+	if m.Hour != 15 {
+		t.Fatalf("Expected m.Hour %d, got %d", 15, m.Hour)
+	}
+
+	if m.Day != 9 {
+		t.Fatalf("Expected m.Day %d, got %d", 9, m.Day)
+	}
+
+	if m.Month != 5 {
+		t.Fatalf("Expected m.Month %d, got %d", 5, m.Month)
+	}
+
+	if m.DayOfWeek != 2 {
+		t.Fatalf("Expected m.DayOfWeek %d, got %d", 2, m.DayOfWeek)
+	}
+}
+
+func TestNewSchedule(t *testing.T) {
+	scenarios := []struct {
+		cronExpr       string
+		expectError    bool
+		expectSchedule string
+	}{
+		{
+			"invalid",
+			true,
+			"",
+		},
+		{
+			"* * * *",
+			true,
+			"",
+		},
+		{
+			"* * * * * *",
+			true,
+			"",
+		},
+		{
+			"2/3 * * * *",
+			true,
+			"",
+		},
+		{
+			"* * * * *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"*/2 */3 */5 */4 */2",
+			false,
+			`{"minutes":{"0":{},"10":{},"12":{},"14":{},"16":{},"18":{},"2":{},"20":{},"22":{},"24":{},"26":{},"28":{},"30":{},"32":{},"34":{},"36":{},"38":{},"4":{},"40":{},"42":{},"44":{},"46":{},"48":{},"50":{},"52":{},"54":{},"56":{},"58":{},"6":{},"8":{}},"hours":{"0":{},"12":{},"15":{},"18":{},"21":{},"3":{},"6":{},"9":{}},"days":{"1":{},"11":{},"16":{},"21":{},"26":{},"31":{},"6":{}},"months":{"1":{},"5":{},"9":{}},"daysOfWeek":{"0":{},"2":{},"4":{},"6":{}}}`,
+		},
+
+		// minute segment
+		{
+			"-1 * * * *",
+			true,
+			"",
+		},
+		{
+			"60 * * * *",
+			true,
+			"",
+		},
+		{
+			"0 * * * *",
+			false,
+			`{"minutes":{"0":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"59 * * * *",
+			false,
+			`{"minutes":{"59":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"1,2,5,7,40-50/2 * * * *",
+			false,
+			`{"minutes":{"1":{},"2":{},"40":{},"42":{},"44":{},"46":{},"48":{},"5":{},"50":{},"7":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+
+		// hour segment
+		{
+			"* -1 * * *",
+			true,
+			"",
+		},
+		{
+			"* 24 * * *",
+			true,
+			"",
+		},
+		{
+			"* 0 * * *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"* 23 * * *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"23":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"* 3,4,8-16/3,7 * * *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"11":{},"14":{},"3":{},"4":{},"7":{},"8":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+
+		// day segment
+		{
+			"* * 0 * *",
+			true,
+			"",
+		},
+		{
+			"* * 32 * *",
+			true,
+			"",
+		},
+		{
+			"* * 1 * *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"* * 31 * *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"31":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"* * 5,6,20-30/3,1 * *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"20":{},"23":{},"26":{},"29":{},"5":{},"6":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+
+		// month segment
+		{
+			"* * * 0 *",
+			true,
+			"",
+		},
+		{
+			"* * * 13 *",
+			true,
+			"",
+		},
+		{
+			"* * * 1 *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"* * * 12 *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"12":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+		{
+			"* * * 1,4,5-10/2 *",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"4":{},"5":{},"7":{},"9":{}},"daysOfWeek":{"0":{},"1":{},"2":{},"3":{},"4":{},"5":{},"6":{}}}`,
+		},
+
+		// day of week segment
+		{
+			"* * * * -1",
+			true,
+			"",
+		},
+		{
+			"* * * * 7",
+			true,
+			"",
+		},
+		{
+			"* * * * 0",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"0":{}}}`,
+		},
+		{
+			"* * * * 6",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"6":{}}}`,
+		},
+		{
+			"* * * * 1,2-5/2",
+			false,
+			`{"minutes":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"32":{},"33":{},"34":{},"35":{},"36":{},"37":{},"38":{},"39":{},"4":{},"40":{},"41":{},"42":{},"43":{},"44":{},"45":{},"46":{},"47":{},"48":{},"49":{},"5":{},"50":{},"51":{},"52":{},"53":{},"54":{},"55":{},"56":{},"57":{},"58":{},"59":{},"6":{},"7":{},"8":{},"9":{}},"hours":{"0":{},"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"days":{"1":{},"10":{},"11":{},"12":{},"13":{},"14":{},"15":{},"16":{},"17":{},"18":{},"19":{},"2":{},"20":{},"21":{},"22":{},"23":{},"24":{},"25":{},"26":{},"27":{},"28":{},"29":{},"3":{},"30":{},"31":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"months":{"1":{},"10":{},"11":{},"12":{},"2":{},"3":{},"4":{},"5":{},"6":{},"7":{},"8":{},"9":{}},"daysOfWeek":{"1":{},"2":{},"4":{}}}`,
+		},
+	}
+
+	for _, s := range scenarios {
+		schedule, err := cron.NewSchedule(s.cronExpr)
+
+		hasErr := err != nil
+		if hasErr != s.expectError {
+			t.Fatalf("[%s] Expected hasErr to be %v, got %v (%v)", s.cronExpr, s.expectError, hasErr, err)
+		}
+
+		if hasErr {
+			continue
+		}
+
+		encoded, err := json.Marshal(schedule)
+		if err != nil {
+			t.Fatalf("[%s] Failed to marshalize the result schedule: %v", s.cronExpr, err)
+		}
+		encodedStr := string(encoded)
+
+		if encodedStr != s.expectSchedule {
+			t.Fatalf("[%s] Expected \n%s, \ngot \n%s", s.cronExpr, s.expectSchedule, encodedStr)
+		}
+	}
+}
+
+func TestScheduleIsDue(t *testing.T) {
+	scenarios := []struct {
+		cronExpr string
+		moment   *cron.Moment
+		expected bool
+	}{
+		{
+			"* * * * *",
+			&cron.Moment{},
+			false,
+		},
+		{
+			"* * * * *",
+			&cron.Moment{
+				Minute:    1,
+				Hour:      1,
+				Day:       1,
+				Month:     1,
+				DayOfWeek: 1,
+			},
+			true,
+		},
+		{
+			"5 * * * *",
+			&cron.Moment{
+				Minute:    1,
+				Hour:      1,
+				Day:       1,
+				Month:     1,
+				DayOfWeek: 1,
+			},
+			false,
+		},
+		{
+			"5 * * * *",
+			&cron.Moment{
+				Minute:    5,
+				Hour:      1,
+				Day:       1,
+				Month:     1,
+				DayOfWeek: 1,
+			},
+			true,
+		},
+		{
+			"* 2-6 * * 2,3",
+			&cron.Moment{
+				Minute:    1,
+				Hour:      2,
+				Day:       1,
+				Month:     1,
+				DayOfWeek: 1,
+			},
+			false,
+		},
+		{
+			"* 2-6 * * 2,3",
+			&cron.Moment{
+				Minute:    1,
+				Hour:      2,
+				Day:       1,
+				Month:     1,
+				DayOfWeek: 3,
+			},
+			true,
+		},
+		{
+			"* * 1,2,5,15-18 * *",
+			&cron.Moment{
+				Minute:    1,
+				Hour:      1,
+				Day:       6,
+				Month:     1,
+				DayOfWeek: 1,
+			},
+			false,
+		},
+		{
+			"* * 1,2,5,15-18/2 * *",
+			&cron.Moment{
+				Minute:    1,
+				Hour:      1,
+				Day:       2,
+				Month:     1,
+				DayOfWeek: 1,
+			},
+			true,
+		},
+		{
+			"* * 1,2,5,15-18/2 * *",
+			&cron.Moment{
+				Minute:    1,
+				Hour:      1,
+				Day:       18,
+				Month:     1,
+				DayOfWeek: 1,
+			},
+			false,
+		},
+		{
+			"* * 1,2,5,15-18/2 * *",
+			&cron.Moment{
+				Minute:    1,
+				Hour:      1,
+				Day:       17,
+				Month:     1,
+				DayOfWeek: 1,
+			},
+			true,
+		},
+	}
+
+	for i, s := range scenarios {
+		schedule, err := cron.NewSchedule(s.cronExpr)
+		if err != nil {
+			t.Fatalf("[%d-%s] Unexpected cron error: %v", i, s.cronExpr, err)
+		}
+
+		result := schedule.IsDue(s.moment)
+
+		if result != s.expected {
+			t.Fatalf("[%d-%s] Expected %v, got %v", i, s.cronExpr, s.expected, result)
+		}
+	}
+}

tools/filesystem/filesystem.go

@@ -82,6 +82,11 @@ func NewLocal(dirPath string) (*System, error) {
 	return &System{ctx: ctx, bucket: bucket}, nil
 }
 
+// SetContext assigns the specified context to the current filesystem.
+func (s *System) SetContext(ctx context.Context) {
+	s.ctx = ctx
+}
+
 // Close releases any resources used for the related filesystem.
 func (s *System) Close() error {
 	return s.bucket.Close()
@@ -109,6 +114,28 @@ func (s *System) GetFile(fileKey string) (*blob.Reader, error) {
 	return br, nil
 }
 
+// List returns a flat list with info for all files under the specified prefix.
+func (s *System) List(prefix string) ([]*blob.ListObject, error) {
+	files := []*blob.ListObject{}
+
+	iter := s.bucket.List(&blob.ListOptions{
+		Prefix: prefix,
+	})
+
+	for {
+		obj, err := iter.Next(s.ctx)
+		if err != nil {
+			if err != io.EOF {
+				return nil, err
+			}
+			break
+		}
+		files = append(files, obj)
+	}
+
+	return files, nil
+}
+
 // Upload writes content into the fileKey location.
 func (s *System) Upload(content []byte, fileKey string) error {
 	opts := &blob.WriterOptions{
@@ -398,8 +425,12 @@ func (s *System) CreateThumb(originalKey string, thumbKey, thumbSize string) err
 		}
 	}
 
+	opts := &blob.WriterOptions{
+		ContentType: r.ContentType(),
+	}
+
 	// open a thumb storage writer (aka. prepare for upload)
-	w, writerErr := s.bucket.NewWriter(s.ctx, thumbKey, nil)
+	w, writerErr := s.bucket.NewWriter(s.ctx, thumbKey, opts)
 	if writerErr != nil {
 		return writerErr
 	}

tools/filesystem/filesystem_test.go

@@ -401,6 +401,67 @@ func TestFileSystemGetFile(t *testing.T) {
 	}
 }
 
+func TestFileSystemList(t *testing.T) {
+	dir := createTestDir(t)
+	defer os.RemoveAll(dir)
+
+	fs, err := filesystem.NewLocal(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fs.Close()
+
+	scenarios := []struct {
+		prefix   string
+		expected []string
+	}{
+		{
+			"",
+			[]string{
+				"image.png",
+				"image.svg",
+				"image_! noext",
+				"style.css",
+				"test/sub1.txt",
+				"test/sub2.txt",
+			},
+		},
+		{
+			"test",
+			[]string{
+				"test/sub1.txt",
+				"test/sub2.txt",
+			},
+		},
+		{
+			"missing",
+			[]string{},
+		},
+	}
+
+	for _, s := range scenarios {
+		objs, err := fs.List(s.prefix)
+		if err != nil {
+			t.Fatalf("[%s] %v", s.prefix, err)
+		}
+
+		if len(s.expected) != len(objs) {
+			t.Fatalf("[%s] Expected %d files, got \n%v", s.prefix, len(s.expected), objs)
+		}
+
+	ObjsLoop:
+		for _, obj := range objs {
+			for _, name := range s.expected {
+				if name == obj.Key {
+					continue ObjsLoop
+				}
+			}
+
+			t.Fatalf("[%s] Unexpected file %q", s.prefix, obj.Key)
+		}
+	}
+}
+
 func TestFileSystemServeSingleRange(t *testing.T) {
 	dir := createTestDir(t)
 	defer os.RemoveAll(dir)

tools/hook/hook.go

@@ -37,6 +37,8 @@ func (h *Hook[T]) Add(fn Handler[T]) {
 }
 
 // Reset removes all registered handlers.
+//
+// @todo for consistency with other Go methods consider renaming it to Clear.
 func (h *Hook[T]) Reset() {
 	h.mux.Lock()
 	defer h.mux.Unlock()

tools/list/list.go

@@ -5,10 +5,11 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/pocketbase/pocketbase/tools/store"
 	"github.com/spf13/cast"
 )
 
-var cachedPatterns = map[string]*regexp.Regexp{}
+var cachedPatterns = store.New[*regexp.Regexp](nil)
 
 // SubtractSlice returns a new slice with only the "base" elements
 // that don't exist in "subtract".
@@ -26,9 +27,6 @@ func SubtractSlice[T comparable](base []T, subtract []T) []T {
 
 // ExistInSlice checks whether a comparable element exists in a slice of the same type.
 func ExistInSlice[T comparable](item T, list []T) bool {
-	if len(list) == 0 {
-		return false
-	}
 
 	for _, v := range list {
 		if v == item {
@@ -56,15 +54,18 @@ func ExistInSliceWithRegex(str string, list []string) bool {
 		}
 
 		// check for regex match
-		pattern, ok := cachedPatterns[field]
-		if !ok {
+		pattern := cachedPatterns.Get(field)
+		if pattern == nil {
 			var err error
 			pattern, err = regexp.Compile(field)
 			if err != nil {
 				continue
 			}
 			// "cache" the pattern to avoid compiling it every time
-			cachedPatterns[field] = pattern
+			// (the limit size is arbitrary and it is there to prevent the cache growing too big)
+			//
+			// @todo consider replacing with TTL or LRU type cache
+			cachedPatterns.SetIfLessThanLimit(field, pattern, 5000)
 		}
 
 		if pattern != nil && pattern.MatchString(str) {

tools/migrate/runner.go

@@ -50,7 +50,6 @@ func (r *Runner) Run(args ...string) error {
 	case "up":
 		applied, err := r.Up()
 		if err != nil {
-			color.Red(err.Error())
 			return err
 		}
 
@@ -75,7 +74,6 @@ func (r *Runner) Run(args ...string) error {
 
 		names, err := r.lastAppliedMigrations(toRevertCount)
 		if err != nil {
-			color.Red(err.Error())
 			return err
 		}
 
@@ -95,7 +93,6 @@ func (r *Runner) Run(args ...string) error {
 
 		reverted, err := r.Down(toRevertCount)
 		if err != nil {
-			color.Red(err.Error())
 			return err
 		}
 
@@ -110,7 +107,6 @@ func (r *Runner) Run(args ...string) error {
 		return nil
 	case "history-sync":
 		if err := r.removeMissingAppliedMigrations(); err != nil {
-			color.Red(err.Error())
 			return err
 		}
 

tools/osutils/dir.go

@@ -0,0 +1,80 @@
+package osutils
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/pocketbase/pocketbase/tools/list"
+)
+
+// MoveDirContent moves the src dir content, that is not listed in the exclide list,
+// to dest dir (it will be created if missing).
+//
+// The rootExclude argument is used to specify a list of src root entries to exclude.
+//
+// Note that this method doesn't delete the old src dir.
+//
+// It is an alternative to os.Rename() for the cases where we can't
+// rename/delete the src dir (see https://github.com/pocketbase/pocketbase/issues/2519).
+func MoveDirContent(src string, dest string, rootExclude ...string) error {
+	entries, err := os.ReadDir(src)
+	if err != nil {
+		return err
+	}
+
+	// make sure that the dest dir exist
+	manuallyCreatedDestDir := false
+	if _, err := os.Stat(dest); err != nil {
+		if err := os.Mkdir(dest, os.ModePerm); err != nil {
+			return err
+		}
+		manuallyCreatedDestDir = true
+	}
+
+	moved := map[string]string{}
+
+	tryRollback := func() ([]error) {
+		errs := []error{}
+
+		for old, new := range moved {
+			if err := os.Rename(new, old); err != nil{
+				errs = append(errs, err)
+			}
+		}
+
+		// try to delete manually the created dest dir if all moved files were restored
+		if manuallyCreatedDestDir && len(errs) == 0 {
+			if err := os.Remove(dest); err != nil {
+				errs = append(errs, err)
+			}
+		}
+
+		return errs
+	}
+
+	for _, entry := range entries {
+		basename := entry.Name()
+
+		if list.ExistInSlice(basename, rootExclude) {
+			continue
+		}
+
+		old := filepath.Join(src, basename)
+		new := filepath.Join(dest, basename)
+
+		if err := os.Rename(old, new); err != nil {
+			if errs := tryRollback(); len(errs) > 0  {
+				// currently just log the rollback errors
+				// in the future we may require go 1.20+ to use errors.Join()
+				log.Println(errs)
+			}
+
+			return err
+		}
+
+		moved[old] = new
+	}
+
+	return nil
+}

tools/osutils/dir_test.go

@@ -0,0 +1,144 @@
+package osutils_test
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pocketbase/pocketbase/tools/list"
+	"github.com/pocketbase/pocketbase/tools/osutils"
+	"github.com/pocketbase/pocketbase/tools/security"
+)
+
+func TestMoveDirContent(t *testing.T) {
+	testDir := createTestDir(t)
+	defer os.RemoveAll(testDir)
+
+	exclude := []string{
+		"missing",
+		"test2",
+		"b",
+	}
+
+	// missing dest path
+	// ---
+	dir1 := filepath.Join(filepath.Dir(testDir), "a", "b", "c", "d", "_pb_move_dir_content_test_" + security.PseudorandomString(4))
+	defer os.RemoveAll(dir1)
+
+	if err := osutils.MoveDirContent(testDir, dir1, exclude...); err == nil {
+		t.Fatal("Expected path error, got nil")
+	}
+
+	// existing parent dir
+	// ---
+	dir2 := filepath.Join(filepath.Dir(testDir), "_pb_move_dir_content_test_" + security.PseudorandomString(4))
+	defer os.RemoveAll(dir2)
+
+	if err := osutils.MoveDirContent(testDir, dir2, exclude...); err != nil {
+		t.Fatalf("Expected dir2 to be created, got error: %v", err)
+	}
+
+
+	// find all files
+	files := []string{}
+	filepath.WalkDir(dir2, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		files = append(files, path)
+
+		return nil
+	})
+
+	expectedFiles := []string{
+		filepath.Join(dir2, "test1"),
+		filepath.Join(dir2, "a", "a1"),
+		filepath.Join(dir2, "a", "a2"),
+	};
+
+	if len(files) != len(expectedFiles) {
+		t.Fatalf("Expected %d files, got %d: \n%v", len(expectedFiles), len(files), files)
+	}
+
+	for _, expected := range expectedFiles {
+		if !list.ExistInSlice(expected, files) {
+			t.Fatalf("Missing expected file %q in \n%v", expected, files)
+		}
+	}
+}
+
+// -------------------------------------------------------------------
+
+// note: make sure to call os.RemoveAll(dir) after you are done
+// working with the created test dir.
+func createTestDir(t *testing.T) string {
+	dir, err := os.MkdirTemp(os.TempDir(), "test_dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create sub directories
+	if err := os.MkdirAll(filepath.Join(dir, "a"), os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Join(dir, "b"), os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "test1"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "test2"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "a/a1"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "a/a2"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "b/b2"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	{
+		f, err := os.OpenFile(filepath.Join(dir, "b/b2"), os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
+		f.Close()
+	}
+
+	return dir
+}

tools/rest/json_serializer.go

@@ -0,0 +1,121 @@
+package rest
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/labstack/echo/v5"
+	"github.com/pocketbase/pocketbase/tools/search"
+)
+
+// Serializer represents custom REST JSON serializer based on echo.DefaultJSONSerializer,
+// with support for additional generic response data transformation (eg. fields picker).
+type Serializer struct {
+	echo.DefaultJSONSerializer
+
+	FieldsParam string
+}
+
+// Serialize converts an interface into a json and writes it to the response.
+//
+// It also provides a generic response data fields picker via the FieldsParam query parameter (default to "fields").
+func (s *Serializer) Serialize(c echo.Context, i any, indent string) error {
+	fieldsParam := s.FieldsParam
+	if fieldsParam == "" {
+		fieldsParam = "fields"
+	}
+
+	param := c.QueryParam(fieldsParam)
+	if param == "" {
+		return s.DefaultJSONSerializer.Serialize(c, i, indent)
+	}
+
+	fields := strings.Split(param, ",")
+	for i, f := range fields {
+		fields[i] = strings.TrimSpace(f)
+	}
+
+	encoded, err := json.Marshal(i)
+	if err != nil {
+		return err
+	}
+
+	var decoded any
+
+	if err := json.Unmarshal(encoded, &decoded); err != nil {
+		return err
+	}
+
+	var isSearchResult bool
+
+	switch i.(type) {
+	case search.Result, *search.Result:
+		isSearchResult = true
+	}
+
+	if isSearchResult {
+		if decodedMap, ok := decoded.(map[string]any); ok {
+			pickFields(decodedMap["items"], fields)
+		}
+	} else {
+		pickFields(decoded, fields)
+	}
+
+	return s.DefaultJSONSerializer.Serialize(c, decoded, indent)
+}
+
+func pickFields(data any, fields []string) {
+	switch v := data.(type) {
+	case map[string]any:
+		pickMapFields(v, fields)
+	case []map[string]any:
+		for _, item := range v {
+			pickMapFields(item, fields)
+		}
+	case []any:
+		if len(v) == 0 {
+			return // nothing to pick
+		}
+
+		if _, ok := v[0].(map[string]any); !ok {
+			return // for now ignore non-map values
+		}
+
+		for _, item := range v {
+			pickMapFields(item.(map[string]any), fields)
+		}
+	}
+}
+
+func pickMapFields(data map[string]any, fields []string) {
+	if len(fields) == 0 {
+		return // nothing to pick
+	}
+
+DataLoop:
+	for k := range data {
+		matchingFields := make([]string, 0, len(fields))
+		for _, f := range fields {
+			if strings.HasPrefix(f+".", k+".") {
+				matchingFields = append(matchingFields, f)
+				continue
+			}
+		}
+
+		if len(matchingFields) == 0 {
+			delete(data, k)
+			continue DataLoop
+		}
+
+		// trim the key from the fields
+		for i, v := range matchingFields {
+			trimmed := strings.TrimSuffix(strings.TrimPrefix(v+".", k+"."), ".")
+			if trimmed == "" {
+				continue DataLoop
+			}
+			matchingFields[i] = trimmed
+		}
+
+		pickFields(data[k], matchingFields)
+	}
+}

tools/rest/json_serializer_test.go

@@ -0,0 +1,195 @@
+package rest_test
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/labstack/echo/v5"
+	"github.com/pocketbase/pocketbase/tools/rest"
+	"github.com/pocketbase/pocketbase/tools/search"
+)
+
+func TestSerialize(t *testing.T) {
+	scenarios := []struct {
+		name       string
+		serializer rest.Serializer
+		data       any
+		query      string
+		expected   string
+	}{
+		{
+			"empty query",
+			rest.Serializer{},
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"",
+			`{"a":1,"b":2,"c":"test"}`,
+		},
+		{
+			"empty fields",
+			rest.Serializer{},
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"fields=",
+			`{"a":1,"b":2,"c":"test"}`,
+		},
+		{
+			"missing fields",
+			rest.Serializer{},
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"fields=missing",
+			`{}`,
+		},
+		{
+			"non map response",
+			rest.Serializer{},
+			"test",
+			"fields=a,b,test",
+			`"test"`,
+		},
+		{
+			"non slice of map response",
+			rest.Serializer{},
+			[]any{"a", "b", "test"},
+			"fields=a,test",
+			`["a","b","test"]`,
+		},
+		{
+			"map with existing and missing fields",
+			rest.Serializer{},
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"fields=a,  c  ,missing", // test individual fields trim
+			`{"a":1,"c":"test"}`,
+		},
+		{
+			"custom fields param",
+			rest.Serializer{FieldsParam: "custom"},
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"custom=a,  c  ,missing", // test individual fields trim
+			`{"a":1,"c":"test"}`,
+		},
+		{
+			"slice of maps with existing and missing fields",
+			rest.Serializer{},
+			[]any{
+				map[string]any{"a": 11, "b": 11, "c": "test1"},
+				map[string]any{"a": 22, "b": 22, "c": "test2"},
+			},
+			"fields=a,  c  ,missing", // test individual fields trim
+			`[{"a":11,"c":"test1"},{"a":22,"c":"test2"}]`,
+		},
+		{
+			"nested fields with mixed map and any slices",
+			rest.Serializer{},
+			map[string]any{
+				"a": 1,
+				"b": 2,
+				"c": "test",
+				"anySlice": []any{
+					map[string]any{
+						"A": []int{1, 2, 3},
+						"B": []any{"1", "2", 3},
+						"C": "test",
+						"D": map[string]any{
+							"DA": 1,
+							"DB": 2,
+						},
+					},
+					map[string]any{
+						"A": "test",
+					},
+				},
+				"mapSlice": []map[string]any{
+					{
+						"A": []int{1, 2, 3},
+						"B": []any{"1", "2", 3},
+						"C": "test",
+						"D": []any{
+							map[string]any{"DA": 1},
+						},
+					},
+					{
+						"B": []any{"1", "2", 3},
+						"D": []any{
+							map[string]any{"DA": 2},
+							map[string]any{"DA": 3},
+							map[string]any{"DB": 4}, // will result to empty since there is no DA
+						},
+					},
+				},
+				"fullMap": []map[string]any{
+					{
+						"A": []int{1, 2, 3},
+						"B": []any{"1", "2", 3},
+						"C": "test",
+					},
+					{
+						"B": []any{"1", "2", 3},
+						"D": []any{
+							map[string]any{"DA": 2},
+							map[string]any{"DA": 3}, // will result to empty since there is no DA
+						},
+					},
+				},
+			},
+			"fields=a, c, anySlice.A, mapSlice.C, mapSlice.D.DA, anySlice.D,fullMap",
+			`{"a":1,"anySlice":[{"A":[1,2,3],"D":{"DA":1,"DB":2}},{"A":"test"}],"c":"test","fullMap":[{"A":[1,2,3],"B":["1","2",3],"C":"test"},{"B":["1","2",3],"D":[{"DA":2},{"DA":3}]}],"mapSlice":[{"C":"test","D":[{"DA":1}]},{"D":[{"DA":2},{"DA":3},{}]}]}`,
+		},
+		{
+			"SearchResult",
+			rest.Serializer{},
+			search.Result{
+				Page:       1,
+				PerPage:    10,
+				TotalItems: 20,
+				TotalPages: 30,
+				Items: []any{
+					map[string]any{"a": 11, "b": 11, "c": "test1"},
+					map[string]any{"a": 22, "b": 22, "c": "test2"},
+				},
+			},
+			"fields=a,c,missing",
+			`{"items":[{"a":11,"c":"test1"},{"a":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`,
+		},
+		{
+			"*SearchResult",
+			rest.Serializer{},
+			&search.Result{
+				Page:       1,
+				PerPage:    10,
+				TotalItems: 20,
+				TotalPages: 30,
+				Items: []any{
+					map[string]any{"a": 11, "b": 11, "c": "test1"},
+					map[string]any{"a": 22, "b": 22, "c": "test2"},
+				},
+			},
+			"fields=a,c",
+			`{"items":[{"a":11,"c":"test1"},{"a":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`,
+		},
+	}
+
+	for _, s := range scenarios {
+		e := echo.New()
+		req := httptest.NewRequest(http.MethodPost, "/", nil)
+		req.URL.RawQuery = s.query
+		rec := httptest.NewRecorder()
+		c := e.NewContext(req, rec)
+
+		if err := s.serializer.Serialize(c, s.data, ""); err != nil {
+			t.Errorf("[%s] Serialize failure: %v", s.name, err)
+			continue
+		}
+
+		rawBody, err := io.ReadAll(rec.Result().Body)
+		if err != nil {
+			t.Errorf("[%s] Failed to read request body: %v", s.name, err)
+			continue
+		}
+
+		if v := strings.TrimSpace(string(rawBody)); v != s.expected {
+			t.Fatalf("[%s] Expected body\n%v \ngot: \n%v", s.name, s.expected, v)
+		}
+	}
+}

tools/rest/multi_binder.go

@@ -5,9 +5,11 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"reflect"
 	"strings"
 
 	"github.com/labstack/echo/v5"
+	"github.com/spf13/cast"
 )
 
 // BindBody binds request body content to i.
@@ -28,10 +30,12 @@ func BindBody(c echo.Context, i interface{}) error {
 			return echo.NewHTTPErrorWithInternal(http.StatusBadRequest, err, err.Error())
 		}
 		return nil
-	default:
-		// fallback to the default binder
-		return echo.BindBody(c, i)
+	case strings.HasPrefix(ctype, echo.MIMEApplicationForm), strings.HasPrefix(ctype, echo.MIMEMultipartForm):
+		return bindFormData(c, i)
 	}
+
+	// fallback to the default binder
+	return echo.BindBody(c, i)
 }
 
 // CopyJsonBody reads the request body into i by
@@ -57,3 +61,71 @@ func CopyJsonBody(r *http.Request, i interface{}) error {
 
 	return err
 }
+
+// This is temp hotfix for properly binding multipart/form-data array values
+// when a map destination is used.
+//
+// It should be replaced with echo.BindBody(c, i) once the issue is fixed in echo.
+func bindFormData(c echo.Context, i interface{}) error {
+	if i == nil {
+		return nil
+	}
+
+	values, err := c.FormValues()
+	if err != nil {
+		return echo.NewHTTPErrorWithInternal(http.StatusBadRequest, err, err.Error())
+	}
+
+	if len(values) == 0 {
+		return nil
+	}
+
+	rt := reflect.TypeOf(i).Elem()
+
+	// map
+	if rt.Kind() == reflect.Map {
+		rv := reflect.ValueOf(i).Elem()
+
+		for k, v := range values {
+			if total := len(v); total == 1 {
+				rv.SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(normalizeMultipartValue(v[0])))
+			} else {
+				normalized := make([]any, total)
+				for i, vItem := range v {
+					normalized[i] = normalizeMultipartValue(vItem)
+				}
+				rv.SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(normalized))
+			}
+		}
+
+		return nil
+	}
+
+	// anything else
+	return echo.BindBody(c, i)
+}
+
+// In order to support more seamlessly both json and multipart/form-data requests,
+// the following normalization rules are applied for plain multipart string values:
+// - "true" is converted to the json `true`
+// - "false" is converted to the json `false`
+// - numeric (non-scientific) strings are converted to json number
+// - any other string (empty string too) is left as it is
+func normalizeMultipartValue(raw string) any {
+	switch raw {
+	case "":
+		return raw
+	case "true":
+		return true
+	case "false":
+		return false
+	default:
+		if raw[0] == '-' || (raw[0] >= '0' && raw[0] <= '9') {
+			if v, err := cast.ToFloat64E(raw); err == nil {
+				return v
+			}
+		}
+
+		return raw
+	}
+}

tools/rest/multi_binder_test.go

@@ -1,6 +1,7 @@
 package rest_test
 
 import (
+	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -16,31 +17,40 @@ func TestBindBody(t *testing.T) {
 	scenarios := []struct {
 		body        io.Reader
 		contentType string
-		result      map[string]string
+		expectBody  string
 		expectError bool
 	}{
 		{
 			strings.NewReader(""),
 			echo.MIMEApplicationJSON,
-			map[string]string{},
+			`{}`,
 			false,
 		},
 		{
 			strings.NewReader(`{"test":"invalid`),
 			echo.MIMEApplicationJSON,
-			map[string]string{},
+			`{}`,
 			true,
 		},
 		{
-			strings.NewReader(`{"test":"test123"}`),
+			strings.NewReader(`{"test":123}`),
 			echo.MIMEApplicationJSON,
-			map[string]string{"test": "test123"},
+			`{"test":123}`,
 			false,
 		},
 		{
-			strings.NewReader(url.Values{"test": []string{"test123"}}.Encode()),
+			strings.NewReader(
+				url.Values{
+					"string":  []string{"str"},
+					"stings":  []string{"str1", "str2", ""},
+					"number":  []string{"-123"},
+					"numbers": []string{"123", "456.789"},
+					"bool":    []string{"true"},
+					"bools":   []string{"true", "false"},
+				}.Encode(),
+			),
 			echo.MIMEApplicationForm,
-			map[string]string{"test": "test123"},
+			`{"bool":true,"bools":[true,false],"number":-123,"numbers":[123,456.789],"stings":["str1","str2",""],"string":"str"}`,
 			false,
 		},
 	}
@@ -52,25 +62,22 @@ func TestBindBody(t *testing.T) {
 		rec := httptest.NewRecorder()
 		c := e.NewContext(req, rec)
 
-		result := map[string]string{}
-		err := rest.BindBody(c, &result)
+		data := map[string]any{}
+		err := rest.BindBody(c, &data)
 
-		if err == nil && scenario.expectError {
-			t.Errorf("(%d) Expected error, got nil", i)
+		hasErr := err != nil
+		if hasErr != scenario.expectError {
+			t.Errorf("[%d] Expected hasErr %v, got %v", i, scenario.expectError, hasErr)
 		}
 
-		if err != nil && !scenario.expectError {
-			t.Errorf("(%d) Expected nil, got error %v", i, err)
+		rawBody, err := json.Marshal(data)
+		if err != nil {
+			t.Errorf("[%d] Failed to marshal binded body: %v", i, err)
+
 		}
 
-		if len(result) != len(scenario.result) {
-			t.Errorf("(%d) Expected %v, got %v", i, scenario.result, result)
-		}
-
-		for k, v := range result {
-			if sv, ok := scenario.result[k]; !ok || v != sv {
-				t.Errorf("(%d) Expected value %v for key %s, got %v", i, sv, k, v)
-			}
+		if scenario.expectBody != string(rawBody) {
+			t.Errorf("[%d] Expected body \n%s, \ngot \n%s", i, scenario.expectBody, rawBody)
 		}
 	}
 }

tools/search/filter.go

@@ -107,15 +107,9 @@ func buildExpr(
 
 	switch op {
 	case fexpr.SignEq, fexpr.SignAnyEq:
-		expr = dbx.NewExp(
-			fmt.Sprintf("%s = %s", normalizeNullIdentifier(left), normalizeNullIdentifier(right)),
-			mergeParams(left.Params, right.Params),
-		)
+		expr = resolveEqualExpr(true, left, right)
 	case fexpr.SignNeq, fexpr.SignAnyNeq:
-		expr = dbx.NewExp(
-			fmt.Sprintf("%s != %s", normalizeNullIdentifier(left), normalizeNullIdentifier(right)),
-			mergeParams(left.Params, right.Params),
-		)
+		expr = resolveEqualExpr(false, left, right)
 	case fexpr.SignLike, fexpr.SignAnyLike:
 		// the right side is a column and therefor wrap it with "%" for contains like behavior
 		if len(right.Params) == 0 {
@@ -238,18 +232,108 @@ func resolveToken(token fexpr.Token, fieldResolver FieldResolver) (*ResolverResu
 	return nil, errors.New("unresolvable token type")
 }
 
-func normalizeNullIdentifier(result *ResolverResult) string {
-	lower := strings.ToLower(result.Identifier)
+// Resolves = and != expressions in an attempt to minimize the COALESCE
+// usage and to gracefully handle null vs empty string normalizations.
+//
+// The expression `a = "" OR a is null` tends to perform better than
+// `COALESCE(a, "") = ""` since the direct match can be accomplished
+// with a seek while the COALESCE will induce a table scan.
+func resolveEqualExpr(equal bool, left, right *ResolverResult) dbx.Expression {
+	isLeftEmpty := isEmptyIdentifier(left) || (len(left.Params) == 1 && hasEmptyParamValue(left))
+	isRightEmpty := isEmptyIdentifier(right) || (len(right.Params) == 1 && hasEmptyParamValue(right))
 
-	if lower == "null" {
-		return "''"
+	equalOp := "="
+	concatOp := "OR"
+	nullExpr := "IS NULL"
+	if !equal {
+		equalOp = "!="
+		concatOp = "AND"
+		nullExpr = "IS NOT NULL"
 	}
 
-	if strings.Contains(lower, "json_extract(") || strings.Contains(lower, "json_array_length(") {
-		return fmt.Sprintf("COALESCE(%s, '')", result.Identifier)
+	// both operands are empty
+	if isLeftEmpty && isRightEmpty {
+		return dbx.NewExp(fmt.Sprintf("'' %s ''", equalOp), mergeParams(left.Params, right.Params))
 	}
 
-	return result.Identifier
+	// direct compare since at least one of the operands is known to be non-empty
+	// eg. a = 'example'
+	if isKnownNonEmptyIdentifier(left) || isKnownNonEmptyIdentifier(right) {
+		leftIdentifier := left.Identifier
+		if isLeftEmpty {
+			leftIdentifier = "''"
+		}
+		rightIdentifier := right.Identifier
+		if isRightEmpty {
+			rightIdentifier = "''"
+		}
+		return dbx.NewExp(
+			fmt.Sprintf("%s %s %s", leftIdentifier, equalOp, rightIdentifier),
+			mergeParams(left.Params, right.Params),
+		)
+	}
+
+	// "" = b OR b IS NULL
+	// "" != b AND b IS NOT NULL
+	if isLeftEmpty {
+		return dbx.NewExp(
+			fmt.Sprintf("('' %s %s %s %s %s)", equalOp, right.Identifier, concatOp, right.Identifier, nullExpr),
+			mergeParams(left.Params, right.Params),
+		)
+	}
+
+	// a = "" OR a IS NULL
+	// a != "" AND a IS NOT NULL
+	if isRightEmpty {
+		return dbx.NewExp(
+			fmt.Sprintf("(%s %s '' %s %s %s)", left.Identifier, equalOp, concatOp, left.Identifier, nullExpr),
+			mergeParams(left.Params, right.Params),
+		)
+	}
+
+	// fallback to a COALESCE comparison
+	return dbx.NewExp(
+		fmt.Sprintf(
+			"COALESCE(%s, '') %s COALESCE(%s, '')",
+			left.Identifier,
+			equalOp,
+			right.Identifier,
+		),
+		mergeParams(left.Params, right.Params),
+	)
+}
+
+func hasEmptyParamValue(result *ResolverResult) bool {
+	for _, p := range result.Params {
+		switch v := p.(type) {
+		case nil:
+			return true
+		case string:
+			if v == "" {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func isKnownNonEmptyIdentifier(result *ResolverResult) bool {
+	switch strings.ToLower(result.Identifier) {
+	case "1", "0", "false", `true`:
+		return true
+	}
+
+	return len(result.Params) > 0 && !hasEmptyParamValue(result) && !isEmptyIdentifier(result)
+}
+
+func isEmptyIdentifier(result *ResolverResult) bool {
+	switch strings.ToLower(result.Identifier) {
+	case "", "null", "''", `""`, "``":
+		return true
+	default:
+		return false
+	}
 }
 
 func isAnyMatchOp(op fexpr.SignOp) bool {