make raw json scanning determinisctic and added setting option to limit log data size

CHANGELOG.md

@@ -11,6 +11,10 @@
 - Added `Cross-Origin-Opener-Policy:same-origin` to the default security response headers.
     _This is just an extra precaution to prevent tab-nabbing in case custom UI plugins use `target="_blank"` without `rel="noopener"`._
 
+- Added new log settings option to limit the max `Log.Data` size (default to ~16KB).
+    _This is an extra precaution for the cases when logging user supplied data without validating it beforehand._
+    _If the resulting `Log.Data` json is above the limit, it is truncated to the last valid decoded character and an extra `"__pb_truncated__":true` log data entry will be added.`_
+
 
 ## v0.39.11
 

core/log_model.go

@@ -1,13 +1,20 @@
 package core
 
-import "github.com/pocketbase/pocketbase/tools/types"
+import (
+	"encoding/json/v2"
+
+	"github.com/pocketbase/pocketbase/tools/types"
+)
 
 var (
-	_ Model = (*Log)(nil)
+	_ Model      = (*Log)(nil)
+	_ DBExporter = (*Log)(nil)
 )
 
 const LogsTableName = "_logs"
 
+const defaultMaxLogDataSize = 16 << 10 // ~16kb
+
 type Log struct {
 	BaseModel
 
@@ -17,6 +24,57 @@ type Log struct {
 	Level   int                `db:"level" json:"level"`
 }
 
-func (m *Log) TableName() string {
+func (l *Log) TableName() string {
 	return LogsTableName
 }
+
+// DBExport prepares and exports the current log model for db persistence.
+//
+// It also truncates the log's message and data to ensure that it is
+// under app.Settings().Logs.MaxDataSize.
+func (l *Log) DBExport(app App) (map[string]any, error) {
+	result := map[string]any{
+		"id":      l.Id,
+		"created": l.Created,
+		"level":   l.Level,
+	}
+
+	maxDataSize := app.Settings().Logs.MaxDataSize
+	if maxDataSize == 0 {
+		maxDataSize = defaultMaxLogDataSize
+	}
+
+	// truncate the raw message bytes
+	// (this is expected to be very rare so it is ok even if multi-byte chars)
+	if int64(len(l.Message)) > maxDataSize {
+		result["message"] = l.Message[:maxDataSize]
+	} else {
+		result["message"] = l.Message
+	}
+
+	if len(l.Data) == 0 {
+		result["data"] = l.Data
+	} else {
+		rawData, err := l.Data.MarshalJSON()
+		if int64(len(rawData)) > maxDataSize {
+			truncatedData := types.JSONMap[any]{}
+
+			// ignore syntax errors in case of truncated incomplete json
+			//
+			// jsonv2 stream decodes and all "valid" attrs read up to the
+			// invalid part will be populated in truncatedData
+			_ = json.Unmarshal(rawData[:maxDataSize], &truncatedData)
+
+			truncatedData["__pb_truncated__"] = true
+
+			rawData, err = truncatedData.MarshalJSON()
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		result["data"] = types.JSONRaw(rawData)
+	}
+
+	return result, nil
+}

core/log_model_test.go

@@ -0,0 +1,118 @@
+package core_test
+
+import (
+	"bytes"
+	"encoding/json/v2"
+	"strings"
+	"testing"
+
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tests"
+	"github.com/pocketbase/pocketbase/tools/types"
+)
+
+func TestLogTableName(t *testing.T) {
+	t.Parallel()
+
+	var log core.Log
+
+	if name := log.TableName(); name != core.LogsTableName {
+		t.Fatalf("Expected Log table name %q, got %q", core.LogsTableName, name)
+	}
+}
+
+func TestLogDBExport(t *testing.T) {
+	t.Parallel()
+
+	testApp, _ := tests.NewTestApp()
+	defer testApp.Cleanup()
+
+	date, err := types.ParseDateTime("2026-08-18 10:20:30.456Z")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defaultLimit := 16 << 10
+
+	scenarios := []struct {
+		name       string
+		log        core.Log
+		limit      int64 // 0 -> use default
+		expectJSON string
+	}{
+		{
+			"empty log",
+			core.Log{},
+			0,
+			`{"created":"","data":{},"id":"","level":0,"message":""}`,
+		},
+		{
+			"with message and data below the default limit",
+			core.Log{
+				BaseModel: core.BaseModel{Id: "test_id"},
+				Created:   date,
+				Level:     123,
+				Message:   "test_message",
+				Data:      types.JSONMap[any]{"a": "test1", "b": "test2"},
+			},
+			0,
+			`{"created":"2026-08-18 10:20:30.456Z","data":{"a":"test1","b":"test2"},"id":"test_id","level":123,"message":"test_message"}`,
+		},
+		{
+			"with message and data exactly the default limit",
+			core.Log{
+				BaseModel: core.BaseModel{Id: "test_id"},
+				Created:   date,
+				Level:     123,
+				Message:   strings.Repeat("a", defaultLimit),
+				Data:      types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", defaultLimit-32)},
+			},
+			0,
+			`{"created":"2026-08-18 10:20:30.456Z","data":{"a":"test1","b":"test2","c":"` + strings.Repeat("a", defaultLimit-32) + `"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", defaultLimit) + `"}`,
+		},
+		{
+			"with message and data above the default limit",
+			core.Log{
+				BaseModel: core.BaseModel{Id: "test_id"},
+				Created:   date,
+				Level:     123,
+				Message:   strings.Repeat("a", defaultLimit) + "x",                                                         // "x" should be omitted
+				Data:      types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", defaultLimit-32) + "x"}, // the end will be incomplete and something like `"c":"...aaaaaax`
+			},
+			0,
+			`{"created":"2026-08-18 10:20:30.456Z","data":{"__pb_truncated__":true,"a":"test1","b":"test2","c":"` + strings.Repeat("a", defaultLimit-32) + `x"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", defaultLimit) + `"}`,
+		},
+		{
+			"with message and data above custom limit",
+			core.Log{
+				BaseModel: core.BaseModel{Id: "test_id"},
+				Created:   date,
+				Level:     123,
+				Message:   strings.Repeat("a", 2<<10) + "x",                                                           // "x" should be omitted
+				Data:      types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", (2<<10)-32) + "x"}, // the end will be incomplete and something like `"c":"...aaaaaax`
+			},
+			2 << 10,
+			`{"created":"2026-08-18 10:20:30.456Z","data":{"__pb_truncated__":true,"a":"test1","b":"test2","c":"` + strings.Repeat("a", (2<<10)-32) + `x"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", 2<<10) + `"}`,
+		},
+	}
+
+	for _, s := range scenarios {
+		t.Run(s.name, func(t *testing.T) {
+			testApp.Settings().Logs.MaxDataSize = s.limit
+
+			result, err := s.log.DBExport(testApp)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			raw, err := json.Marshal(result, json.Deterministic(true))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !bytes.Equal(raw, []byte(s.expectJSON)) {
+				t.Fatalf("Expected export data\n%s\ngot\n%s", s.expectJSON, raw)
+			}
+		})
+	}
+}

core/settings_model.go

@@ -559,6 +559,12 @@ func (c MetaConfig) Validate() error {
 // -------------------------------------------------------------------
 
 type LogsConfig struct {
+	// MaxDataSize specifies the maximum allowed serialized log data
+	// size before it gets truncated (see [Log.DBExport]).
+	//
+	// If zero, fallbacks to ~16kb by default.
+	MaxDataSize int64 `form:"maxDataSize" json:"maxDataSize"`
+
 	MaxDays   int  `form:"maxDays" json:"maxDays"`
 	MinLevel  int  `form:"minLevel" json:"minLevel"`
 	LogIP     bool `form:"logIP" json:"logIP"`
@@ -568,7 +574,9 @@ type LogsConfig struct {
 // Validate makes LogsConfig validatable by implementing [validation.Validatable] interface.
 func (c LogsConfig) Validate() error {
 	return validation.ValidateStruct(&c,
-		validation.Field(&c.MaxDays, validation.Min(0)),
+		validation.Field(&c.MaxDataSize, validation.Min(0), validation.Max(maxSafeJSONInt)),
+		validation.Field(&c.MaxDays, validation.Min(0), validation.Max(maxSafeJSONInt)),
+		validation.Field(&c.MinLevel, validation.Max(maxSafeJSONInt)),
 	)
 }
 

core/settings_model_test.go

@@ -84,7 +84,7 @@ func TestSettings_DBExport(t *testing.T) {
 				valueStr = string(export["value"].([]byte))
 			}
 
-			expected := `{"superuserIPs":[],"smtp":{"enabled":false,"port":0,"host":"smtp_host","username":"smtp_username","password":"","authMethod":"","tls":false,"localName":""},"backups":{"cron":"* * * * *","cronMaxKeep":0,"s3":{"enabled":true,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false}},"s3":{"enabled":false,"bucket":"","region":"","endpoint":"s3_endpoint","accessKey":"","secret":"s3_secret","forcePathStyle":false},"meta":{"accentColor":"","appName":"test_app_name","appURL":"","senderName":"","senderAddress":"","hideControls":false},"rateLimits":{"rules":[],"excludedIPs":[],"enabled":true},"trustedProxy":{"headers":[],"useLeftmostIP":true},"batch":{"enabled":false,"maxRequests":0,"timeout":15,"maxBodySize":0},"logs":{"maxDays":123,"minLevel":0,"logIP":false,"logAuthId":false}}`
+			expected := `{"superuserIPs":[],"smtp":{"enabled":false,"port":0,"host":"smtp_host","username":"smtp_username","password":"","authMethod":"","tls":false,"localName":""},"backups":{"cron":"* * * * *","cronMaxKeep":0,"s3":{"enabled":true,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false}},"s3":{"enabled":false,"bucket":"","region":"","endpoint":"s3_endpoint","accessKey":"","secret":"s3_secret","forcePathStyle":false},"meta":{"accentColor":"","appName":"test_app_name","appURL":"","senderName":"","senderAddress":"","hideControls":false},"rateLimits":{"rules":[],"excludedIPs":[],"enabled":true},"trustedProxy":{"headers":[],"useLeftmostIP":true},"batch":{"enabled":false,"maxRequests":0,"timeout":15,"maxBodySize":0},"logs":{"maxDataSize":0,"maxDays":123,"minLevel":0,"logIP":false,"logAuthId":false}}`
 			if valueStr != expected {
 				t.Fatalf("Expected exported settings\n%s\ngot\n%s", expected, valueStr)
 			}
@@ -180,7 +180,7 @@ func TestSettingsMarshalJSON(t *testing.T) {
 	}
 	rawStr := string(raw)
 
-	expected := `{"superuserIPs":[],"smtp":{"enabled":false,"port":0,"host":"","username":"abc","authMethod":"","tls":false,"localName":""},"backups":{"cron":"","cronMaxKeep":0,"s3":{"enabled":false,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false}},"s3":{"enabled":false,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false},"meta":{"accentColor":"","appName":"test123","appURL":"","senderName":"","senderAddress":"","hideControls":false},"rateLimits":{"rules":[],"excludedIPs":[],"enabled":false},"trustedProxy":{"headers":[],"useLeftmostIP":false},"batch":{"enabled":false,"maxRequests":0,"timeout":0,"maxBodySize":0},"logs":{"maxDays":0,"minLevel":0,"logIP":false,"logAuthId":false}}`
+	expected := `{"superuserIPs":[],"smtp":{"enabled":false,"port":0,"host":"","username":"abc","authMethod":"","tls":false,"localName":""},"backups":{"cron":"","cronMaxKeep":0,"s3":{"enabled":false,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false}},"s3":{"enabled":false,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false},"meta":{"accentColor":"","appName":"test123","appURL":"","senderName":"","senderAddress":"","hideControls":false},"rateLimits":{"rules":[],"excludedIPs":[],"enabled":false},"trustedProxy":{"headers":[],"useLeftmostIP":false},"batch":{"enabled":false,"maxRequests":0,"timeout":0,"maxBodySize":0},"logs":{"maxDataSize":0,"maxDays":0,"minLevel":0,"logIP":false,"logAuthId":false}}`
 
 	if rawStr != expected {
 		t.Fatalf("Expected\n%v\ngot\n%v", expected, rawStr)
@@ -309,8 +309,11 @@ func TestLogsConfigValidate(t *testing.T) {
 		},
 		{
 			"invalid data",
-			core.LogsConfig{MaxDays: -1},
+			core.LogsConfig{
-			[]string{"maxDays"},
+				MaxDays:     -1,
+				MaxDataSize: -1,
+			},
+			[]string{"maxDays", "maxDataSize"},
 		},
 		{
 			"valid data",

tools/types/json_raw.go

@@ -73,7 +73,7 @@ func (j *JSONRaw) Scan(value any) error {
 			data = []byte(v)
 		}
 	default:
-		bytes, err := json.Marshal(v)
+		bytes, err := json.Marshal(v, json.Deterministic(true))
 		if err != nil {
 			return err
 		}