[#7836] fixed migration logs write deadlock and added app.ClearBootstrap/OnClearBootstrap helpers

core/base.go

@@ -12,6 +12,7 @@ import (
 	"regexp"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/fatih/color"
@@ -89,11 +90,12 @@ type BaseApp struct {
 	auxNonconcurrentDB  dbx.Builder
 
 	// app event hooks
-	onBootstrap     *hook.Hook[*BootstrapEvent]
-	onServe         *hook.Hook[*ServeEvent]
-	onTerminate     *hook.Hook[*TerminateEvent]
-	onBackupCreate  *hook.Hook[*BackupEvent]
-	onBackupRestore *hook.Hook[*BackupEvent]
+	onBootstrap      *hook.Hook[*BootstrapEvent]
+	onClearBootstrap *hook.Hook[*BootstrapEvent]
+	onServe          *hook.Hook[*ServeEvent]
+	onTerminate      *hook.Hook[*TerminateEvent]
+	onBackupCreate   *hook.Hook[*BackupEvent]
+	onBackupRestore  *hook.Hook[*BackupEvent]
 
 	// db model hooks
 	onModelValidate           *hook.Hook[*ModelEvent]
@@ -249,6 +251,7 @@ func NewBaseApp(config BaseAppConfig) *BaseApp {
 func (app *BaseApp) initHooks() {
 	// app event hooks
 	app.onBootstrap = &hook.Hook[*BootstrapEvent]{}
+	app.onClearBootstrap = &hook.Hook[*BootstrapEvent]{}
 	app.onServe = &hook.Hook[*ServeEvent]{}
 	app.onTerminate = &hook.Hook[*TerminateEvent]{}
 	app.onBackupCreate = &hook.Hook[*BackupEvent]{}
@@ -405,14 +408,14 @@ func (app *BaseApp) IsBootstrapped() bool {
 // Bootstrap initializes the application
 // (aka. create data dir, open db connections, load settings, etc.).
 //
-// It will call ResetBootstrapState() if the application was already bootstrapped.
+// It calls ClearBootstrap() if the application was already bootstrapped.
 func (app *BaseApp) Bootstrap() error {
 	event := &BootstrapEvent{}
 	event.App = app
 
 	err := app.OnBootstrap().Trigger(event, func(e *BootstrapEvent) error {
-		// clear resources of previous core state (if any)
-		if err := app.ResetBootstrapState(); err != nil {
+		// clear previous bootstrap state (if any)
+		if err := app.ClearBootstrap(); err != nil {
 			return err
 		}
 
@@ -460,41 +463,55 @@ func (app *BaseApp) Bootstrap() error {
 	return err
 }
 
-type closer interface {
-	Close() error
+// Deprecated: use [ClearBootstrap].
+func (app *BaseApp) ResetBootstrapState() error {
+	return app.ClearBootstrap()
 }
 
-// ResetBootstrapState releases the initialized core app resources
+// ClearBootstrap releases the initialized core app resources
 // (closing db connections, stopping cron ticker, etc.).
-func (app *BaseApp) ResetBootstrapState() error {
-	app.Cron().Stop()
-
-	var errs []error
-
-	dbs := []*dbx.Builder{
-		&app.concurrentDB,
-		&app.nonconcurrentDB,
-		&app.auxConcurrentDB,
-		&app.auxNonconcurrentDB,
+//
+// This method is no-op if the application is not bootstrapped yet.
+func (app *BaseApp) ClearBootstrap() error {
+	if !app.IsBootstrapped() {
+		return nil
 	}
 
-	for _, db := range dbs {
-		if db == nil {
-			continue
+	event := &BootstrapEvent{}
+	event.App = app
+
+	return app.OnClearBootstrap().Trigger(event, func(e *BootstrapEvent) error {
+		type closer interface {
+			Close() error
 		}
-		if v, ok := (*db).(closer); ok {
-			if err := v.Close(); err != nil {
-				errs = append(errs, err)
+
+		var errs []error
+
+		dbs := []*dbx.Builder{
+			&app.concurrentDB,
+			&app.nonconcurrentDB,
+			&app.auxConcurrentDB,
+			&app.auxNonconcurrentDB,
+		}
+
+		for _, db := range dbs {
+			if db == nil {
+				continue
 			}
+			if v, ok := (*db).(closer); ok {
+				if err := v.Close(); err != nil {
+					errs = append(errs, err)
+				}
+			}
+			*db = nil
 		}
-		*db = nil
-	}
 
-	if len(errs) > 0 {
-		return errors.Join(errs...)
-	}
+		if len(errs) > 0 {
+			return errors.Join(errs...)
+		}
 
-	return nil
+		return nil
+	})
 }
 
 // DB returns the default app data.db builder instance.
@@ -817,7 +834,7 @@ func (app *BaseApp) Restart() error {
 	event.IsRestart = true
 
 	return app.OnTerminate().Trigger(event, func(e *TerminateEvent) error {
-		_ = e.App.ResetBootstrapState()
+		_ = e.App.ClearBootstrap()
 
 		// attempt to restart the bootstrap process in case execve returns an error for some reason
 		defer func() {
@@ -860,6 +877,10 @@ func (app *BaseApp) OnBootstrap() *hook.Hook[*BootstrapEvent] {
 	return app.onBootstrap
 }
 
+func (app *BaseApp) OnClearBootstrap() *hook.Hook[*BootstrapEvent] {
+	return app.onClearBootstrap
+}
+
 func (app *BaseApp) OnServe() *hook.Hook[*ServeEvent] {
 	return app.onServe
 }
@@ -1413,7 +1434,15 @@ func (app *BaseApp) registerBaseHooks() {
 		Id: "__pbCronStart__",
 		Func: func(e *ServeEvent) error {
 			app.Cron().Start()
+			return e.Next()
+		},
+		Priority: 999,
+	})
 
+	app.OnClearBootstrap().Bind(&hook.Handler[*BootstrapEvent]{
+		Id: "__pbCronStop__",
+		Func: func(e *BootstrapEvent) error {
+			app.Cron().Stop()
 			return e.Next()
 		},
 		Priority: 999,
@@ -1470,9 +1499,41 @@ func getLoggerMinLevel(app App) slog.Level {
 }
 
 func (app *BaseApp) initLogger() error {
+	var stopped atomic.Bool
+
 	duration := 3 * time.Second
 	ticker := time.NewTicker(duration)
-	done := make(chan bool, 1)
+
+	done := make(chan struct{}, 1)
+
+	runLogsWrite := func(logs []*logger.Log) {
+		if !app.IsBootstrapped() || app.Settings().Logs.MaxDays == 0 {
+			return
+		}
+
+		// write the accumulated logs
+		//
+		// note: based on several local tests there is no
+		// significant performance difference between small number
+		// of separate write queries vs 1 big INSERT
+		app.AuxRunInTransaction(func(txApp App) error {
+			model := &Log{}
+			for _, l := range logs {
+				model.MarkAsNew()
+				model.Id = GenerateDefaultRandomId()
+				model.Level = int(l.Level)
+				model.Message = l.Message
+				model.Data = l.Data
+				model.Created, _ = types.ParseDateTime(l.Time)
+
+				if err := txApp.AuxSave(model); err != nil {
+					log.Println("Failed to write log", model, err)
+				}
+			}
+
+			return nil
+		})
+	}
 
 	handler := logger.NewBatchHandler(logger.BatchOptions{
 		Level:     getLoggerMinLevel(app),
@@ -1487,35 +1548,25 @@ func (app *BaseApp) initLogger() error {
 				}
 			}
 
-			ticker.Reset(duration)
+			if !stopped.Load() {
+				ticker.Reset(duration)
+			}
 
 			return app.Settings().Logs.MaxDays > 0
 		},
 		WriteFunc: func(ctx context.Context, logs []*logger.Log) error {
-			if !app.IsBootstrapped() || app.Settings().Logs.MaxDays == 0 {
-				return nil
+			// don't block and wait for the write transaction to complete
+			// when we can't be sure if the logs write wasn't triggered while
+			// inside another AUX db transaction (ticker or batch threshold reached)
+			// which can block indefinitely and cause deadlock
+			// (https://github.com/pocketbase/pocketbase/issues/7836)
+			shouldBlock, _ := ctx.Value(logger.BlockKey).(bool)
+			if shouldBlock {
+				runLogsWrite(logs)
+			} else {
+				routine.FireAndForget(func() { runLogsWrite(logs) })
 			}
 
-			// write the accumulated logs
-			// (note: based on several local tests there is no significant performance difference between small number of separate write queries vs 1 big INSERT)
-			app.AuxRunInTransaction(func(txApp App) error {
-				model := &Log{}
-				for _, l := range logs {
-					model.MarkAsNew()
-					model.Id = GenerateDefaultRandomId()
-					model.Level = int(l.Level)
-					model.Message = l.Message
-					model.Data = l.Data
-					model.Created, _ = types.ParseDateTime(l.Time)
-
-					if err := txApp.AuxSave(model); err != nil {
-						log.Println("Failed to write log", model, err)
-					}
-				}
-
-				return nil
-			})
-
 			return nil
 		},
 	})
@@ -1535,17 +1586,21 @@ func (app *BaseApp) initLogger() error {
 
 	app.logger = slog.New(handler)
 
-	// write all remaining logs before ticker.Stop to avoid races with ResetBootstrap user calls
-	app.OnTerminate().Bind(&hook.Handler[*TerminateEvent]{
-		Id: "__pbAppLoggerOnTerminate__",
-		Func: func(e *TerminateEvent) error {
-			handler.WriteAll(context.Background())
+	// attempt to write all queued logs before clearing the application bootstrap state
+	app.OnClearBootstrap().Bind(&hook.Handler[*BootstrapEvent]{
+		Id: "__pbAppLoggerFlushBeforeStop__",
+		Func: func(e *BootstrapEvent) error {
+			// extra precaution in case the hook was manually triggered while inside aux db transaction
+			_, isTx := e.App.AuxNonconcurrentDB().(*dbx.Tx)
+			ctx := context.WithValue(context.Background(), logger.BlockKey, !isTx)
+			handler.WriteAll(ctx)
 
+			stopped.Store(true)
 			ticker.Stop()
 
-			// don't block in case OnTerminate is triggered more than once
+			// don't block in case the hook is triggered more than once
 			select {
-			case done <- true:
+			case done <- struct{}{}:
 			default:
 			}