From 0e9dcfe1ab587de385cdc325646dc975fda6614f Mon Sep 17 00:00:00 2001 From: Gani Georgiev Date: Fri, 21 Aug 2026 18:50:16 +0300 Subject: [PATCH] added experimental backup optimizations --- CHANGELOG.md | 11 +- core/app.go | 13 + core/backup.go | 137 +++++++ core/backup_create.go | 375 +++++++++++++++++++ core/{base_backup.go => backup_restore.go} | 226 ----------- core/{base_backup_test.go => backup_test.go} | 4 - core/base.go | 67 +++- core/events.go | 19 + plugins/jsvm/binds_test.go | 2 +- tools/filesystem/filesystem.go | 87 ++++- 10 files changed, 696 insertions(+), 245 deletions(-) create mode 100644 core/backup.go create mode 100644 core/backup_create.go rename core/{base_backup.go => backup_restore.go} (51%) rename core/{base_backup_test.go => backup_test.go} (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index addc7d11..3e6a198d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,17 @@ _⚠️ Note that this could be a slight breaking change in case you are chaining PocketBase commands and relied on the previous `0` exit status for `Command.RunE` returned errors._ _Or in other words, if you have `./pocketbase invalid && someothercommand` and previously relied that `someothercommand` will be always executed then this is no longer the case and you'll have to adjust it or replace `&&` with `;`._ -- Added `filesystem.NewWriter(key, opts)` low-level helper to allow direct file create from an `io.Reader` value. +- Added helper `filesystem` methods: + - `filesystem.NewWriter(key, opts)` to allow direct file create from an `io.Reader` value. + - `filesystem.OnNewWriter()` hook to allow listening for new/to-be-creaded files (app level equivalent `app.OnFilesystemNewWriter()` hook is also available). + - `filesystem.OnDelete()` hook to allow listening for deleted files (app level equivalent `app.OnFilesystemDelete()` hook is also available). - Added new `DELETE /api/logs` endpoint and UI control to delete all logs without changing the `maxDays` retention setting. - Added `Record.GetInt64(field)` helper (note that the serializable max safe integer of the `number` field is ~2^53-1). +- Added `Store.Keys()` method that returns a slice with all of the store keys. + - Added quotes around the default `Content-Disposition` serving filename in case custom name with special characters is provided. - Added `Cross-Origin-Opener-Policy:same-origin` to the default security response headers. @@ -20,6 +25,8 @@ _If the resulting `Log.Data` json is above the limit, it is truncated to the last valid decoded character and an extra `"__pb_truncated__":true` log data entry will be added.`_ _Additionally, for just in case the log message is also truncated at max 8k characters._ +- (@todo tests and docs) Refactored backups create to minimize the DB lock times. + - Updated `modernc.org/sqlite` to 1.57 and registered by default the new `_defensive=1` DSN query parameter to enable [SQLite's defensive mode](https://sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigdefensive). -- (@todo) Bumped the min Go version to 1.27.0 and migrated to the new `encoding/json/v2` package. +- (@todo docs) Bumped the min Go version to 1.27.0 and migrated to the new `encoding/json/v2` package. diff --git a/core/app.go b/core/app.go index 089c4e07..1351f7de 100644 --- a/core/app.go +++ b/core/app.go @@ -1262,6 +1262,19 @@ type App interface { // triggered and called only if their event data origin matches the tags. OnMailerRecordOTPSend(tags ...string) *hook.TaggedHook[*MailerRecordEvent] + // --------------------------------------------------------------- + // Filesystem event hooks + // --------------------------------------------------------------- + + // OnFilesystemNewWriter is a low level hook for app.NewFilesystem() + // instances that is triggered on every storage filesystem writer initialization + // (aka. whenever attempting to create a new file). + OnFilesystemNewWriter() *hook.Hook[*FilesystemNewWriterEvent] + + // OnFilesystemDelete is a low level hook for app.NewFilesystem() + // instances that is triggered for every storage file delete call. + OnFilesystemDelete() *hook.Hook[*FilesystemDeleteEvent] + // --------------------------------------------------------------- // Realtime API event hooks // --------------------------------------------------------------- diff --git a/core/backup.go b/core/backup.go new file mode 100644 index 00000000..f4a9be33 --- /dev/null +++ b/core/backup.go @@ -0,0 +1,137 @@ +package core + +import ( + "context" + "fmt" + "log/slog" + "sort" + "time" + + "github.com/pocketbase/pocketbase/tools/inflector" +) + +const ( + StoreKeyActiveBackup = "@activeBackup" +) + +// generateBackupName generates a new backup name based on the app name and current date. +func generateBackupName(app App, prefix string) string { + appName := inflector.Snakecase(app.Settings().Meta.AppName) + if len(appName) > 50 { + appName = appName[:50] + } + + return fmt.Sprintf( + "%s%s_%s.zip", + prefix, + appName, + time.Now().UTC().Format("20060102150405"), + ) +} + +// registerAutobackupHooks registers the autobackup app serve hooks. +func (app *BaseApp) registerAutobackupHooks() { + const jobId = "__pbAutoBackup__" + + loadJob := func() { + rawSchedule := app.Settings().Backups.Cron + if rawSchedule == "" { + app.Cron().Remove(jobId) + return + } + + app.Cron().Add(jobId, rawSchedule, func() { + const autoPrefix = "@auto_pb_backup_" + + name := generateBackupName(app, autoPrefix) + + if err := app.CreateBackup(context.Background(), name); err != nil { + app.Logger().Error( + "[Backup cron] Failed to create backup", + slog.String("name", name), + slog.String("error", err.Error()), + ) + + alertError := sendSystemAlertToAllSuperusers( + app, + "Autobackup failure", + "Failed to create/upload automated backup. Raw error:\n"+err.Error(), + ) + if alertError != nil { + app.Logger().Warn( + "[Backup cron] Failed to send backup error alerts", + slog.String("name", name), + slog.String("error", alertError.Error()), + ) + } + } + + maxKeep := app.Settings().Backups.CronMaxKeep + + if maxKeep == 0 { + return // no explicit limit + } + + fsys, err := app.NewBackupsFilesystem() + if err != nil { + app.Logger().Error( + "[Backup cron] Failed to initialize the backup filesystem", + slog.String("error", err.Error()), + ) + return + } + defer fsys.Close() + + files, err := fsys.List(autoPrefix) + if err != nil { + app.Logger().Error( + "[Backup cron] Failed to list autogenerated backups", + slog.String("error", err.Error()), + ) + return + } + + if maxKeep >= len(files) { + return // nothing to remove + } + + // sort desc + sort.Slice(files, func(i, j int) bool { + return files[i].ModTime.After(files[j].ModTime) + }) + + // keep only the most recent n auto backup files + toRemove := files[maxKeep:] + + for _, f := range toRemove { + if err := fsys.Delete(f.Key); err != nil { + app.Logger().Error( + "[Backup cron] Failed to remove old autogenerated backup", + slog.String("key", f.Key), + slog.String("error", err.Error()), + ) + } + } + }) + } + + app.OnBootstrap().BindFunc(func(e *BootstrapEvent) error { + if err := e.Next(); err != nil { + return err + } + + loadJob() + + return nil + }) + + app.OnSettingsReload().BindFunc(func(e *SettingsReloadEvent) error { + if err := e.Next(); err != nil { + return err + } + + loadJob() + + return nil + }) +} diff --git a/core/backup_create.go b/core/backup_create.go new file mode 100644 index 00000000..d5395393 --- /dev/null +++ b/core/backup_create.go @@ -0,0 +1,375 @@ +package core + +import ( + "archive/zip" + "compress/flate" + "context" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/tools/filesystem" + "github.com/pocketbase/pocketbase/tools/hook" + "github.com/pocketbase/pocketbase/tools/security" + "github.com/pocketbase/pocketbase/tools/store" +) + +// CreateBackup creates a new backup of the current app pb_data directory. +// +// If name is empty, it will be autogenerated. +// If backup with the same name exists, the new backup file will replace it. +// +// To safely perform the backup, it is recommended to have free disk space +// for at least 2x the size of the pb_data directory. +// +// By default backups are stored in pb_data/backups +// (the backups directory itself is excluded from the generated backup). +// +// Backups can be stored on S3 if it is configured in app.Settings().Backups. +// When using S3 storage for the uploaded collection files, you have to +// take care manually to backup those since they are not part of the pb_data. +// +// DB write locks are minimal and isolated only for the duration of the +// VACUUM INTO statement that creates a live copy of the app database. +// +// The backup works as follow: +// +// 1. Start listening for DELETED storage files. +// In case a file is being deleted while the backup is still ongoing, +// we directly copy it in the zip before the deletion and mark it as "excluded". +// +// 2. Copy the main database with VACUUM INTO, write it in the zip and mark it as "excluded". +// +// 3. Stop listening for DELETED files. +// +// 4. Start listening for NEW storage files and mark all new files as excluded. +// +// 5. Copy the logs database with VACUUM INTO, write it in the zip and mark it as "excluded". +// +// 6. Copy the rest of the pb_data files in the zip while ignoring the "excluded" list (it should be concurrent safe). +// +// 7. Stop listening for NEW storage files. +// +// While there is a risk for a race condition between steps 1, 2 and 3, it is an +// acceptable trade-off between performance and correctness because in +// the worst case there will be some unused storage files in the backup that don't do any harm. +func (app *BaseApp) CreateBackup(ctx context.Context, name string) error { + if app.Store().Has(StoreKeyActiveBackup) { + return errors.New("try again later - another backup/restore operation has already been started") + } + + app.Store().Set(StoreKeyActiveBackup, name) + defer app.Store().Remove(StoreKeyActiveBackup) + + startTime := time.Now() + + event := new(BackupEvent) + event.App = app + event.Context = ctx + event.Name = name + // default root dir entries to exclude from the backup generation + event.Exclude = []string{ + LocalBackupsDirName, + LocalTempDirName, + LocalNotifyDirName, + LocalAutocertCacheDirName, + lostFoundDirName, + } + + return app.OnBackupCreate().Trigger(event, func(e *BackupEvent) error { + if e.Name == "" { + e.Name = generateBackupName(e.App, "pb_backup_") + } + + // create backup zip + // (it needs to be inside the current pb_data to avoid "cross-device link" errors) + // ----------------------------------------------------------- + tempZipPath := filepath.Join(app.DataDir(), LocalTempDirName, "pb_backup_"+security.PseudorandomString(6)) + err := createZip(e, tempZipPath) + if err != nil { + return err + } + defer os.Remove(tempZipPath) + + // persist the backup in the backups filesystem + // ----------------------------------------------------------- + fsys, err := e.App.NewBackupsFilesystem() + if err != nil { + return err + } + defer fsys.Close() + + fsys.SetContext(e.Context) + + file, err := filesystem.NewFileFromPath(tempZipPath) + if err != nil { + return err + } + file.OriginalName = e.Name + file.Name = file.OriginalName + + err = fsys.UploadFile(file, file.Name) + if err != nil { + return err + } + + app.Logger().Debug( + "["+e.Name+"] zip archive completed", + slog.Float64("execTime", float64(time.Since(startTime))/float64(time.Millisecond)), + ) + + return nil + }) +} + +func createZip(be *BackupEvent, tempZipPath string) error { + logPrefix := "[" + be.Name + "] " + + // make sure that the special temp directory exists + localTempDir := filepath.Dir(tempZipPath) + if err := os.MkdirAll(localTempDir, os.ModePerm); err != nil { + return fmt.Errorf(logPrefix+"failed to create temp dir: %w", err) + } + + const tempFilesHookId = "__pbTempBackupFilesystemWatcher__" + defer func() { + // unbind again in cacase of an error + be.App.OnFilesystemDelete().Unbind(tempFilesHookId) + be.App.OnFilesystemNewWriter().Unbind(tempFilesHookId) + }() + + zf, err := os.Create(tempZipPath) + if err != nil { + return err + } + + zw := zip.NewWriter(zf) + zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(out, flate.BestSpeed) + }) + + closeZip := func() error { + return errors.Join(zw.Close(), zf.Close()) + } + // call defer even though zf.Close will error if invoked multiple times + // because otherwise the code become too brittle + defer closeZip() + + excluded := store.New[string, struct{}](nil) + for _, name := range be.Exclude { + excluded.Set(normalizePathExclude(name), struct{}{}) + } + + // init deleted files tracker + // --------------------------------------------------------------- + be.App.OnFilesystemDelete().Bind(&hook.Handler[*FilesystemDeleteEvent]{ + Id: tempFilesHookId, + Func: func(e *FilesystemDeleteEvent) error { + // note: the zip header name allow only forward slashes + zipPath := path.Join(LocalStorageDirName, e.FileKey) + + if excluded.Has(normalizePathExclude(zipPath)) || be.App.Settings().S3.Enabled { + return e.Next() + } + + localPath := filepath.Join( + be.App.DataDir(), + LocalStorageDirName, + e.FileKey, + ) + + // copy to zip before delete + err := copyFileToZip(zw, localPath, zipPath) + if err != nil { + be.App.Logger().Warn( + logPrefix+"failed to copy file in backup zip before delete", + slog.Any("error", err), + slog.String("file", e.FileKey), + ) + } else { + // mark that it was already copied + excluded.Set(normalizePathExclude(zipPath), struct{}{}) + } + + // proceed with the normal deletion + return e.Next() + }, + }) + + // copy data.db + // --------------------------------------------------------------- + dataStartTime := time.Now() + tempDataDBPath := filepath.Join(localTempDir, dataDBFilename) + + _, err = be.App.NonconcurrentDB().NewQuery("VACUUM INTO {:path}").Bind(dbx.Params{"path": tempDataDBPath}).Execute() + if err != nil { + return err + } + + // eageerly stop listenining for deleted files since we already have what we needed + be.App.OnFilesystemDelete().Unbind(tempFilesHookId) + + be.App.Logger().Debug( + logPrefix+dataDBFilename+" copy completed", + slog.Float64("execTime", float64(time.Since(dataStartTime))/float64(time.Millisecond)), + ) + + err = copyFileToZip(zw, tempDataDBPath, dataDBFilename) + if err != nil { + _ = os.Remove(tempDataDBPath) + return err + } + _ = os.Remove(tempDataDBPath) + + excluded.Set(normalizePathExclude(dataDBFilename), struct{}{}) + excluded.Set(normalizePathExclude(dataDBFilename+"-wal"), struct{}{}) + excluded.Set(normalizePathExclude(dataDBFilename+"-shm"), struct{}{}) + + // init to-be-created files tracker + // --------------------------------------------------------------- + be.App.OnFilesystemNewWriter().Bind(&hook.Handler[*FilesystemNewWriterEvent]{ + Id: tempFilesHookId, + Func: func(e *FilesystemNewWriterEvent) error { + if !be.App.Settings().S3.Enabled { + // mark for exclude even if the writer eventually fails + // (all record files have random name so collusions are unlikely) + name := normalizePathExclude(filepath.Join(LocalStorageDirName, e.FileKey)) + excluded.Set(name, struct{}{}) + } + + return e.Next() + }, + }) + + // copy auxiliary.db + // --------------------------------------------------------------- + auxStartTime := time.Now() + tempAuxDBPath := filepath.Join(localTempDir, auxDBFilename) + + _, err = be.App.AuxNonconcurrentDB().NewQuery("VACUUM INTO {:path}").Bind(dbx.Params{"path": tempAuxDBPath}).Execute() + if err != nil { + return err + } + + be.App.Logger().Debug( + logPrefix+auxDBFilename+" copy completed", + slog.Float64("execTime", float64(time.Since(auxStartTime))/float64(time.Millisecond)), + ) + + err = copyFileToZip(zw, tempAuxDBPath, auxDBFilename) + if err != nil { + _ = os.Remove(tempAuxDBPath) + return err + } + _ = os.Remove(tempAuxDBPath) + + excluded.Set(normalizePathExclude(auxDBFilename), struct{}{}) + excluded.Set(normalizePathExclude(auxDBFilename+"-wal"), struct{}{}) + excluded.Set(normalizePathExclude(auxDBFilename+"-shm"), struct{}{}) + + // copy the rest of the pb_data + // --------------------------------------------------------------- + err = copyDirToZip(zw, os.DirFS(be.App.DataDir()), excluded) + if err != nil { + return err + } + + return closeZip() +} + +// normalize the provided file path to always use and end with forward slash +func normalizePathExclude(filePath string) string { + return path.Clean(filePath) + "/" +} + +func copyFileToZip(w *zip.Writer, localPath string, zipPath string) error { + info, err := os.Stat(localPath) + if err != nil { + return err + } + + h, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + + h.Name = zipPath + h.Method = zip.Deflate + + fw, err := w.CreateHeader(h) + if err != nil { + return err + } + + f, err := os.Open(localPath) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(fw, f) + + return err +} + +func copyDirToZip(w *zip.Writer, fsys fs.FS, excludedPrefixes *store.Store[string, struct{}]) error { + return fs.WalkDir(fsys, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // skip excluded prefixes + if excludedPrefixes != nil { + check := normalizePathExclude(name) + prefixes := excludedPrefixes.Keys() // refetch in case to avoid races + for _, prefix := range prefixes { + if strings.HasPrefix(check, prefix) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + } + } + + if d.IsDir() { + return nil + } + + info, err := d.Info() + if err != nil { + return err + } + + h, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + + h.Name = name + h.Method = zip.Deflate + + fw, err := w.CreateHeader(h) + if err != nil { + return err + } + + f, err := fsys.Open(name) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(fw, f) + + return err + }) +} diff --git a/core/base_backup.go b/core/backup_restore.go similarity index 51% rename from core/base_backup.go rename to core/backup_restore.go index 12d66c10..c4a17cca 100644 --- a/core/base_backup.go +++ b/core/backup_restore.go @@ -9,117 +9,12 @@ import ( "os" "path/filepath" "runtime" - "sort" - "time" "github.com/pocketbase/pocketbase/tools/archive" - "github.com/pocketbase/pocketbase/tools/filesystem" - "github.com/pocketbase/pocketbase/tools/inflector" "github.com/pocketbase/pocketbase/tools/osutils" "github.com/pocketbase/pocketbase/tools/security" ) -const ( - StoreKeyActiveBackup = "@activeBackup" -) - -// CreateBackup creates a new backup of the current app pb_data directory. -// -// If name is empty, it will be autogenerated. -// If backup with the same name exists, the new backup file will replace it. -// -// The backup is executed within a transaction, meaning that new writes -// will be temporary "blocked" until the backup file is generated. -// -// To safely perform the backup, it is recommended to have free disk space -// for at least 2x the size of the pb_data directory. -// -// By default backups are stored in pb_data/backups -// (the backups directory itself is excluded from the generated backup). -// -// When using S3 storage for the uploaded collection files, you have to -// take care manually to backup those since they are not part of the pb_data. -// -// Backups can be stored on S3 if it is configured in app.Settings().Backups. -func (app *BaseApp) CreateBackup(ctx context.Context, name string) error { - if app.Store().Has(StoreKeyActiveBackup) { - return errors.New("try again later - another backup/restore operation has already been started") - } - - app.Store().Set(StoreKeyActiveBackup, name) - defer app.Store().Remove(StoreKeyActiveBackup) - - event := new(BackupEvent) - event.App = app - event.Context = ctx - event.Name = name - // default root dir entries to exclude from the backup generation - event.Exclude = []string{ - LocalBackupsDirName, - LocalTempDirName, - LocalNotifyDirName, - LocalAutocertCacheDirName, - lostFoundDirName, - } - - return app.OnBackupCreate().Trigger(event, func(e *BackupEvent) error { - // generate a default name if missing - if e.Name == "" { - e.Name = generateBackupName(e.App, "pb_backup_") - } - - // make sure that the special temp directory exists - // note: it needs to be inside the current pb_data to avoid "cross-device link" errors - localTempDir := filepath.Join(e.App.DataDir(), LocalTempDirName) - if err := os.MkdirAll(localTempDir, os.ModePerm); err != nil { - return fmt.Errorf("failed to create a temp dir: %w", err) - } - - // archive pb_data in a temp directory, excluding the "backups" and the temp dirs - // - // run in transaction to temporary block other writes (transactions uses the NonconcurrentDB connection) - // --- - tempPath := filepath.Join(localTempDir, "pb_backup_"+security.PseudorandomString(6)) - createErr := e.App.RunInTransaction(func(txApp App) error { - return txApp.AuxRunInTransaction(func(txApp App) error { - // run manual checkpoint and truncate the WAL files - // (errors are ignored because it is not that important and the PRAGMA may not be supported by the used driver) - txApp.DB().NewQuery("PRAGMA wal_checkpoint(TRUNCATE)").Execute() - txApp.AuxDB().NewQuery("PRAGMA wal_checkpoint(TRUNCATE)").Execute() - - return archive.Create(txApp.DataDir(), tempPath, e.Exclude...) - }) - }) - if createErr != nil { - return createErr - } - defer os.Remove(tempPath) - - // persist the backup in the backups filesystem - // --- - fsys, err := e.App.NewBackupsFilesystem() - if err != nil { - return err - } - defer fsys.Close() - - fsys.SetContext(e.Context) - - file, err := filesystem.NewFileFromPath(tempPath) - if err != nil { - return err - } - file.OriginalName = e.Name - file.Name = file.OriginalName - - if err := fsys.UploadFile(file, file.Name); err != nil { - return err - } - - return nil - }) -} - // RestoreBackup restores the backup with the specified name and restarts // the current running application process. // @@ -299,124 +194,3 @@ func (app *BaseApp) RestoreBackup(ctx context.Context, name string) error { return nil }) } - -// registerAutobackupHooks registers the autobackup app serve hooks. -func (app *BaseApp) registerAutobackupHooks() { - const jobId = "__pbAutoBackup__" - - loadJob := func() { - rawSchedule := app.Settings().Backups.Cron - if rawSchedule == "" { - app.Cron().Remove(jobId) - return - } - - app.Cron().Add(jobId, rawSchedule, func() { - const autoPrefix = "@auto_pb_backup_" - - name := generateBackupName(app, autoPrefix) - - if err := app.CreateBackup(context.Background(), name); err != nil { - app.Logger().Error( - "[Backup cron] Failed to create backup", - slog.String("name", name), - slog.String("error", err.Error()), - ) - - alertError := sendSystemAlertToAllSuperusers( - app, - "Autobackup failure", - "Failed to create/upload automated backup. Raw error:\n"+err.Error(), - ) - if alertError != nil { - app.Logger().Warn( - "[Backup cron] Failed to send backup error alerts", - slog.String("name", name), - slog.String("error", alertError.Error()), - ) - } - } - - maxKeep := app.Settings().Backups.CronMaxKeep - - if maxKeep == 0 { - return // no explicit limit - } - - fsys, err := app.NewBackupsFilesystem() - if err != nil { - app.Logger().Error( - "[Backup cron] Failed to initialize the backup filesystem", - slog.String("error", err.Error()), - ) - return - } - defer fsys.Close() - - files, err := fsys.List(autoPrefix) - if err != nil { - app.Logger().Error( - "[Backup cron] Failed to list autogenerated backups", - slog.String("error", err.Error()), - ) - return - } - - if maxKeep >= len(files) { - return // nothing to remove - } - - // sort desc - sort.Slice(files, func(i, j int) bool { - return files[i].ModTime.After(files[j].ModTime) - }) - - // keep only the most recent n auto backup files - toRemove := files[maxKeep:] - - for _, f := range toRemove { - if err := fsys.Delete(f.Key); err != nil { - app.Logger().Error( - "[Backup cron] Failed to remove old autogenerated backup", - slog.String("key", f.Key), - slog.String("error", err.Error()), - ) - } - } - }) - } - - app.OnBootstrap().BindFunc(func(e *BootstrapEvent) error { - if err := e.Next(); err != nil { - return err - } - - loadJob() - - return nil - }) - - app.OnSettingsReload().BindFunc(func(e *SettingsReloadEvent) error { - if err := e.Next(); err != nil { - return err - } - - loadJob() - - return nil - }) -} - -func generateBackupName(app App, prefix string) string { - appName := inflector.Snakecase(app.Settings().Meta.AppName) - if len(appName) > 50 { - appName = appName[:50] - } - - return fmt.Sprintf( - "%s%s_%s.zip", - prefix, - appName, - time.Now().UTC().Format("20060102150405"), - ) -} diff --git a/core/base_backup_test.go b/core/backup_test.go similarity index 98% rename from core/base_backup_test.go rename to core/backup_test.go index 47d00e33..9a1eb85b 100644 --- a/core/base_backup_test.go +++ b/core/backup_test.go @@ -126,11 +126,7 @@ func verifyBackupContent(app core.App, path string) error { expectedRootEntries := []string{ "storage", "data.db", - "data.db-shm", - "data.db-wal", "auxiliary.db", - "auxiliary.db-shm", - "auxiliary.db-wal", ".gitignore", } diff --git a/core/base.go b/core/base.go index 0fbcb6aa..425d871c 100644 --- a/core/base.go +++ b/core/base.go @@ -44,6 +44,9 @@ const ( // @todo consider removing after backups refactoring lostFoundDirName string = "lost+found" + + dataDBFilename string = "data.db" + auxDBFilename string = "auxiliary.db" ) // FilesManager defines an interface with common methods that files manager models should implement. @@ -146,6 +149,10 @@ type BaseApp struct { onMailerRecordOTPSend *hook.Hook[*MailerRecordEvent] onMailerRecordAuthAlertSend *hook.Hook[*MailerRecordEvent] + // filesystem event hooks + onFilesystemNewWriter *hook.Hook[*FilesystemNewWriterEvent] + onFilesystemDelete *hook.Hook[*FilesystemDeleteEvent] + // realtime api event hooks onRealtimeConnectRequest *hook.Hook[*RealtimeConnectRequestEvent] onRealtimeMessageSend *hook.Hook[*RealtimeMessageEvent] @@ -294,6 +301,10 @@ func (app *BaseApp) initHooks() { app.onMailerRecordOTPSend = &hook.Hook[*MailerRecordEvent]{} app.onMailerRecordAuthAlertSend = &hook.Hook[*MailerRecordEvent]{} + // filesystem event hooks + app.onFilesystemNewWriter = &hook.Hook[*FilesystemNewWriterEvent]{} + app.onFilesystemDelete = &hook.Hook[*FilesystemDeleteEvent]{} + // realtime API event hooks app.onRealtimeConnectRequest = &hook.Hook[*RealtimeConnectRequestEvent]{} app.onRealtimeMessageSend = &hook.Hook[*RealtimeMessageEvent]{} @@ -712,9 +723,10 @@ func (app *BaseApp) NewMailClient() mailer.Mailer { // // NB! Make sure to call Close() on the returned result // after you are done working with it. -func (app *BaseApp) NewFilesystem() (*filesystem.System, error) { +func (app *BaseApp) NewFilesystem() (fsys *filesystem.System, err error) { if app.settings != nil && app.settings.S3.Enabled { - return filesystem.NewS3( + // S3 + fsys, err = filesystem.NewS3( app.settings.S3.Bucket, app.settings.S3.Region, app.settings.S3.Endpoint, @@ -722,10 +734,41 @@ func (app *BaseApp) NewFilesystem() (*filesystem.System, error) { app.settings.S3.Secret, app.settings.S3.ForcePathStyle, ) + } else { + // local filesystem + fsys, err = filesystem.NewLocal(filepath.Join(app.DataDir(), LocalStorageDirName)) + } + if err != nil { + return nil, err } - // fallback to local filesystem - return filesystem.NewLocal(filepath.Join(app.DataDir(), LocalStorageDirName)) + // attach delete hook + if app.onFilesystemDelete.Length() > 0 { + fsys.OnDelete().BindFunc(func(originalEvent *filesystem.DeleteEvent) error { + appEvent := new(FilesystemDeleteEvent) + appEvent.DeleteEvent = originalEvent + appEvent.App = app + + return app.onFilesystemDelete.Trigger(appEvent, func(fde *FilesystemDeleteEvent) error { + return originalEvent.Next() + }) + }) + } + + // attach write hook + if app.onFilesystemNewWriter.Length() > 0 { + fsys.OnNewWriter().BindFunc(func(originalEvent *filesystem.NewWriterEvent) error { + appEvent := new(FilesystemNewWriterEvent) + appEvent.NewWriterEvent = originalEvent + appEvent.App = app + + return app.onFilesystemNewWriter.Trigger(appEvent, func(fwe *FilesystemNewWriterEvent) error { + return originalEvent.Next() + }) + }) + } + + return fsys, nil } // NewBackupsFilesystem creates a new local or S3 filesystem instance @@ -1016,6 +1059,18 @@ func (app *BaseApp) OnMailerRecordAuthAlertSend(tags ...string) *hook.TaggedHook return hook.NewTaggedHook(app.onMailerRecordAuthAlertSend, tags...) } +// ------------------------------------------------------------------- +// Filesystem event hooks +// ------------------------------------------------------------------- + +func (app *BaseApp) OnFilesystemNewWriter() *hook.Hook[*FilesystemNewWriterEvent] { + return app.onFilesystemNewWriter +} + +func (app *BaseApp) OnFilesystemDelete() *hook.Hook[*FilesystemDeleteEvent] { + return app.onFilesystemDelete +} + // ------------------------------------------------------------------- // Realtime API event hooks // ------------------------------------------------------------------- @@ -1173,7 +1228,7 @@ func (app *BaseApp) OnBatchRequest() *hook.Hook[*BatchRequestEvent] { // ------------------------------------------------------------------- func (app *BaseApp) initDataDB() error { - dbPath := filepath.Join(app.DataDir(), "data.db") + dbPath := filepath.Join(app.DataDir(), dataDBFilename) concurrentDB, err := app.config.DBConnect(dbPath) if err != nil { @@ -1235,7 +1290,7 @@ func normalizeSQLLog(sql string) string { func (app *BaseApp) initAuxDB() error { // note: renamed to "auxiliary" because "aux" is a reserved Windows filename // (see https://github.com/pocketbase/pocketbase/issues/5607) - dbPath := filepath.Join(app.DataDir(), "auxiliary.db") + dbPath := filepath.Join(app.DataDir(), auxDBFilename) concurrentDB, err := app.config.DBConnect(dbPath) if err != nil { diff --git a/core/events.go b/core/events.go index efb22664..81b95955 100644 --- a/core/events.go +++ b/core/events.go @@ -8,6 +8,7 @@ import ( "time" "github.com/pocketbase/pocketbase/tools/auth" + "github.com/pocketbase/pocketbase/tools/filesystem" "github.com/pocketbase/pocketbase/tools/hook" "github.com/pocketbase/pocketbase/tools/mailer" "github.com/pocketbase/pocketbase/tools/router" @@ -188,6 +189,24 @@ type MailerRecordEvent struct { Meta map[string]any } +// ------------------------------------------------------------------- +// Filesystem events data +// ------------------------------------------------------------------- + +type FilesystemNewWriterEvent struct { + hook.Event + *filesystem.NewWriterEvent + + App App +} + +type FilesystemDeleteEvent struct { + hook.Event + *filesystem.DeleteEvent + + App App +} + // ------------------------------------------------------------------- // Model events data // ------------------------------------------------------------------- diff --git a/plugins/jsvm/binds_test.go b/plugins/jsvm/binds_test.go index 0d8ca9b9..f138a787 100644 --- a/plugins/jsvm/binds_test.go +++ b/plugins/jsvm/binds_test.go @@ -1613,7 +1613,7 @@ func TestHooksBindsCount(t *testing.T) { vm := goja.New() hooksBinds(app, vm, nil) - testBindsCount(vm, "this", 82, t) + testBindsCount(vm, "this", 84, t) } func TestHooksBinds(t *testing.T) { diff --git a/tools/filesystem/filesystem.go b/tools/filesystem/filesystem.go index e2db386b..a343569e 100644 --- a/tools/filesystem/filesystem.go +++ b/tools/filesystem/filesystem.go @@ -22,6 +22,7 @@ import ( "github.com/pocketbase/pocketbase/tools/filesystem/internal/fileblob" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob" "github.com/pocketbase/pocketbase/tools/filesystem/internal/s3blob/s3" + "github.com/pocketbase/pocketbase/tools/hook" "github.com/pocketbase/pocketbase/tools/list" "github.com/pocketbase/pocketbase/tools/routine" @@ -34,9 +35,28 @@ var ErrNotFound = blob.ErrNotFound const MetadataOriginalName = "original-filename" +type DeleteEvent struct { + hook.Event + Filesystem *System + FileKey string +} + +type NewWriterEvent struct { + hook.Event + Filesystem *System + FileKey string + Options *blob.WriterOptions + Writer *blob.Writer // filled only after e.Next() +} + +// @todo consider renaming + type System struct { ctx context.Context bucket *blob.Bucket + + onNewWriter *hook.Hook[*NewWriterEvent] + onDelete *hook.Hook[*DeleteEvent] } // NewS3 initializes a new S3 filesystem instance. @@ -97,9 +117,31 @@ func (s *System) SetContext(ctx context.Context) { // Close releases any resources used for the related filesystem. func (s *System) Close() error { + s.onNewWriter = nil + s.onDelete = nil + return s.bucket.Close() } +// OnNewWriter is a low level hook that is triggered on every writer initialization +// (aka. when attempting to create a new file with [system.NewWriter], [system.Upload], etc.). +func (s *System) OnNewWriter() *hook.Hook[*NewWriterEvent] { + if s.onNewWriter == nil { + s.onNewWriter = &hook.Hook[*NewWriterEvent]{} + } + + return s.onNewWriter +} + +// OnDelete is a low level hook that is triggered on every [System.Delete] call. +func (s *System) OnDelete() *hook.Hook[*DeleteEvent] { + if s.onDelete == nil { + s.onDelete = &hook.Hook[*DeleteEvent]{} + } + + return s.onDelete +} + // Exists checks if file with fileKey path exists or not. func (s *System) Exists(fileKey string) (bool, error) { return s.bucket.Exists(s.ctx, fileKey) @@ -203,7 +245,7 @@ func (s *System) Upload(content []byte, fileKey string) error { ContentType: mimetype.Detect(content).String(), } - w, writerErr := s.bucket.NewWriter(s.ctx, fileKey, opts) + w, writerErr := s.NewWriter(fileKey, opts) if writerErr != nil { return writerErr } @@ -244,7 +286,7 @@ func (s *System) UploadFile(file *File, fileKey string) error { }, } - w, err := s.bucket.NewWriter(s.ctx, fileKey, opts) + w, err := s.NewWriter(fileKey, opts) if err != nil { return err } @@ -286,7 +328,7 @@ func (s *System) UploadMultipart(fh *multipart.FileHeader, fileKey string) error }, } - w, err := s.bucket.NewWriter(s.ctx, fileKey, opts) + w, err := s.NewWriter(fileKey, opts) if err != nil { return err } @@ -321,14 +363,47 @@ func (s *System) UploadMultipart(fh *multipart.FileHeader, fileKey string) error // w.ReadFrom(content) // w.Close() func (s *System) NewWriter(fileKey string, opts *blob.WriterOptions) (*blob.Writer, error) { - return s.bucket.NewWriter(s.ctx, fileKey, opts) + if s.onNewWriter == nil { + return s.bucket.NewWriter(s.ctx, fileKey, opts) + } + + event := new(NewWriterEvent) + event.Filesystem = s + event.FileKey = fileKey + event.Options = opts + + err := s.onNewWriter.Trigger(event, func(e *NewWriterEvent) error { + writer, err := e.Filesystem.bucket.NewWriter(e.Filesystem.ctx, e.FileKey, e.Options) + if err != nil { + return err + } + + e.Writer = writer + + return nil + }) + if err != nil { + return nil, err + } + + return event.Writer, nil } // Delete deletes stored file at fileKey location. // // If the file doesn't exist returns ErrNotFound. func (s *System) Delete(fileKey string) error { - return s.bucket.Delete(s.ctx, fileKey) + if s.onDelete == nil { + return s.bucket.Delete(s.ctx, fileKey) + } + + event := new(DeleteEvent) + event.Filesystem = s + event.FileKey = fileKey + + return s.onDelete.Trigger(event, func(e *DeleteEvent) error { + return e.Filesystem.bucket.Delete(e.Filesystem.ctx, e.FileKey) + }) } // DeletePrefix deletes everything starting with the specified prefix. @@ -603,7 +678,7 @@ func (s *System) createThumb(originalKey, thumbKey, thumbSize string) error { } // open a thumb storage writer (aka. prepare for upload) - w, err := s.bucket.NewWriter(s.ctx, thumbKey, opts) + w, err := s.NewWriter(thumbKey, opts) if err != nil { return err }