Github-Mirror/pocketbase
1
0
Fork 0
mirror of https://github.com/pocketbase/pocketbase.git synced 2026-09-08 15:41:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

make raw json scanning determinisctic and added setting option to limit log data size

Browse Source
This commit is contained in:
Gani Georgiev
2026-08-18 22:46:51 +03:00
parent f5695a7308
commit 25081f14eb
6 changed files with 200 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+4
View File
@@ -11,6 +11,10 @@
- Added `Cross-Origin-Opener-Policy:same-origin` to the default security response headers.
_This is just an extra precaution to prevent tab-nabbing in case custom UI plugins use `target="_blank"` without `rel="noopener"`._
- Added new log settings option to limit the max `Log.Data` size (default to ~16KB).
_This is an extra precaution for the cases when logging user supplied data without validating it beforehand._
_If the resulting `Log.Data` json is above the limit, it is truncated to the last valid decoded character and an extra `"__pb_truncated__":true` log data entry will be added.`_
## v0.39.11
core/log_model.go
+61 -3
View File
@@ -1,13 +1,20 @@
package core
import "github.com/pocketbase/pocketbase/tools/types"
import (
"encoding/json/v2"
"github.com/pocketbase/pocketbase/tools/types"
)
var (
_ Model = (*Log)(nil)
_ Model = (*Log)(nil)
_ DBExporter = (*Log)(nil)
)
const LogsTableName = "_logs"
const defaultMaxLogDataSize = 16 << 10 // ~16kb
type Log struct {
BaseModel
@@ -17,6 +24,57 @@ type Log struct {
Level int `db:"level" json:"level"`
}
func (m *Log) TableName() string {
func (l *Log) TableName() string {
return LogsTableName
}
// DBExport prepares and exports the current log model for db persistence.
//
// It also truncates the log's message and data to ensure that it is
// under app.Settings().Logs.MaxDataSize.
func (l *Log) DBExport(app App) (map[string]any, error) {
result := map[string]any{
"id": l.Id,
"created": l.Created,
"level": l.Level,
}
maxDataSize := app.Settings().Logs.MaxDataSize
if maxDataSize == 0 {
maxDataSize = defaultMaxLogDataSize
}
// truncate the raw message bytes
// (this is expected to be very rare so it is ok even if multi-byte chars)
if int64(len(l.Message)) > maxDataSize {
result["message"] = l.Message[:maxDataSize]
} else {
result["message"] = l.Message
}
if len(l.Data) == 0 {
result["data"] = l.Data
} else {
rawData, err := l.Data.MarshalJSON()
if int64(len(rawData)) > maxDataSize {
truncatedData := types.JSONMap[any]{}
// ignore syntax errors in case of truncated incomplete json
//
// jsonv2 stream decodes and all "valid" attrs read up to the
// invalid part will be populated in truncatedData
_ = json.Unmarshal(rawData[:maxDataSize], &truncatedData)
truncatedData["__pb_truncated__"] = true
rawData, err = truncatedData.MarshalJSON()
if err != nil {
return nil, err
}
}
result["data"] = types.JSONRaw(rawData)
}
return result, nil
}
core/log_model_test.go
+118
View File
@@ -0,0 +1,118 @@
package core_test
import (
"bytes"
"encoding/json/v2"
"strings"
"testing"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tests"
"github.com/pocketbase/pocketbase/tools/types"
)
func TestLogTableName(t *testing.T) {
t.Parallel()
var log core.Log
if name := log.TableName(); name != core.LogsTableName {
t.Fatalf("Expected Log table name %q, got %q", core.LogsTableName, name)
}
}
func TestLogDBExport(t *testing.T) {
t.Parallel()
testApp, _ := tests.NewTestApp()
defer testApp.Cleanup()
date, err := types.ParseDateTime("2026-08-18 10:20:30.456Z")
if err != nil {
t.Fatal(err)
}
defaultLimit := 16 << 10
scenarios := []struct {
name string
log core.Log
limit int64 // 0 -> use default
expectJSON string
}{
{
"empty log",
core.Log{},
0,
`{"created":"","data":{},"id":"","level":0,"message":""}`,
},
{
"with message and data below the default limit",
core.Log{
BaseModel: core.BaseModel{Id: "test_id"},
Created: date,
Level: 123,
Message: "test_message",
Data: types.JSONMap[any]{"a": "test1", "b": "test2"},
},
0,
`{"created":"2026-08-18 10:20:30.456Z","data":{"a":"test1","b":"test2"},"id":"test_id","level":123,"message":"test_message"}`,
},
{
"with message and data exactly the default limit",
core.Log{
BaseModel: core.BaseModel{Id: "test_id"},
Created: date,
Level: 123,
Message: strings.Repeat("a", defaultLimit),
Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", defaultLimit-32)},
},
0,
`{"created":"2026-08-18 10:20:30.456Z","data":{"a":"test1","b":"test2","c":"` + strings.Repeat("a", defaultLimit-32) + `"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", defaultLimit) + `"}`,
},
{
"with message and data above the default limit",
core.Log{
BaseModel: core.BaseModel{Id: "test_id"},
Created: date,
Level: 123,
Message: strings.Repeat("a", defaultLimit) + "x", // "x" should be omitted
Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", defaultLimit-32) + "x"}, // the end will be incomplete and something like `"c":"...aaaaaax`
},
0,
`{"created":"2026-08-18 10:20:30.456Z","data":{"__pb_truncated__":true,"a":"test1","b":"test2","c":"` + strings.Repeat("a", defaultLimit-32) + `x"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", defaultLimit) + `"}`,
},
{
"with message and data above custom limit",
core.Log{
BaseModel: core.BaseModel{Id: "test_id"},
Created: date,
Level: 123,
Message: strings.Repeat("a", 2<<10) + "x", // "x" should be omitted
Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", (2<<10)-32) + "x"}, // the end will be incomplete and something like `"c":"...aaaaaax`
},
2 << 10,
`{"created":"2026-08-18 10:20:30.456Z","data":{"__pb_truncated__":true,"a":"test1","b":"test2","c":"` + strings.Repeat("a", (2<<10)-32) + `x"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", 2<<10) + `"}`,
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
testApp.Settings().Logs.MaxDataSize = s.limit
result, err := s.log.DBExport(testApp)
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(result, json.Deterministic(true))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(raw, []byte(s.expectJSON)) {
t.Fatalf("Expected export data\n%s\ngot\n%s", s.expectJSON, raw)
}
})
}
}
core/settings_model.go
+9 -1
View File
@@ -559,6 +559,12 @@ func (c MetaConfig) Validate() error {
// -------------------------------------------------------------------
type LogsConfig struct {
// MaxDataSize specifies the maximum allowed serialized log data
// size before it gets truncated (see [Log.DBExport]).
//
// If zero, fallbacks to ~16kb by default.
MaxDataSize int64 `form:"maxDataSize" json:"maxDataSize"`
MaxDays int `form:"maxDays" json:"maxDays"`
MinLevel int `form:"minLevel" json:"minLevel"`
LogIP bool `form:"logIP" json:"logIP"`
@@ -568,7 +574,9 @@ type LogsConfig struct {
// Validate makes LogsConfig validatable by implementing [validation.Validatable] interface.
func (c LogsConfig) Validate() error {
return validation.ValidateStruct(&c,
validation.Field(&c.MaxDays, validation.Min(0)),
validation.Field(&c.MaxDataSize, validation.Min(0), validation.Max(maxSafeJSONInt)),
validation.Field(&c.MaxDays, validation.Min(0), validation.Max(maxSafeJSONInt)),
validation.Field(&c.MinLevel, validation.Max(maxSafeJSONInt)),
)
}
core/settings_model_test.go
+7 -4
View File
@@ -84,7 +84,7 @@ func TestSettings_DBExport(t *testing.T) {
valueStr = string(export["value"].([]byte))
}
expected := `{"superuserIPs":[],"smtp":{"enabled":false,"port":0,"host":"smtp_host","username":"smtp_username","password":"","authMethod":"","tls":false,"localName":""},"backups":{"cron":"* * * * *","cronMaxKeep":0,"s3":{"enabled":true,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false}},"s3":{"enabled":false,"bucket":"","region":"","endpoint":"s3_endpoint","accessKey":"","secret":"s3_secret","forcePathStyle":false},"meta":{"accentColor":"","appName":"test_app_name","appURL":"","senderName":"","senderAddress":"","hideControls":false},"rateLimits":{"rules":[],"excludedIPs":[],"enabled":true},"trustedProxy":{"headers":[],"useLeftmostIP":true},"batch":{"enabled":false,"maxRequests":0,"timeout":15,"maxBodySize":0},"logs":{"maxDays":123,"minLevel":0,"logIP":false,"logAuthId":false}}`
expected := `{"superuserIPs":[],"smtp":{"enabled":false,"port":0,"host":"smtp_host","username":"smtp_username","password":"","authMethod":"","tls":false,"localName":""},"backups":{"cron":"* * * * *","cronMaxKeep":0,"s3":{"enabled":true,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false}},"s3":{"enabled":false,"bucket":"","region":"","endpoint":"s3_endpoint","accessKey":"","secret":"s3_secret","forcePathStyle":false},"meta":{"accentColor":"","appName":"test_app_name","appURL":"","senderName":"","senderAddress":"","hideControls":false},"rateLimits":{"rules":[],"excludedIPs":[],"enabled":true},"trustedProxy":{"headers":[],"useLeftmostIP":true},"batch":{"enabled":false,"maxRequests":0,"timeout":15,"maxBodySize":0},"logs":{"maxDataSize":0,"maxDays":123,"minLevel":0,"logIP":false,"logAuthId":false}}`
if valueStr != expected {
t.Fatalf("Expected exported settings\n%s\ngot\n%s", expected, valueStr)
}
@@ -180,7 +180,7 @@ func TestSettingsMarshalJSON(t *testing.T) {
}
rawStr := string(raw)
expected := `{"superuserIPs":[],"smtp":{"enabled":false,"port":0,"host":"","username":"abc","authMethod":"","tls":false,"localName":""},"backups":{"cron":"","cronMaxKeep":0,"s3":{"enabled":false,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false}},"s3":{"enabled":false,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false},"meta":{"accentColor":"","appName":"test123","appURL":"","senderName":"","senderAddress":"","hideControls":false},"rateLimits":{"rules":[],"excludedIPs":[],"enabled":false},"trustedProxy":{"headers":[],"useLeftmostIP":false},"batch":{"enabled":false,"maxRequests":0,"timeout":0,"maxBodySize":0},"logs":{"maxDays":0,"minLevel":0,"logIP":false,"logAuthId":false}}`
expected := `{"superuserIPs":[],"smtp":{"enabled":false,"port":0,"host":"","username":"abc","authMethod":"","tls":false,"localName":""},"backups":{"cron":"","cronMaxKeep":0,"s3":{"enabled":false,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false}},"s3":{"enabled":false,"bucket":"","region":"","endpoint":"","accessKey":"","forcePathStyle":false},"meta":{"accentColor":"","appName":"test123","appURL":"","senderName":"","senderAddress":"","hideControls":false},"rateLimits":{"rules":[],"excludedIPs":[],"enabled":false},"trustedProxy":{"headers":[],"useLeftmostIP":false},"batch":{"enabled":false,"maxRequests":0,"timeout":0,"maxBodySize":0},"logs":{"maxDataSize":0,"maxDays":0,"minLevel":0,"logIP":false,"logAuthId":false}}`
if rawStr != expected {
t.Fatalf("Expected\n%v\ngot\n%v", expected, rawStr)
@@ -309,8 +309,11 @@ func TestLogsConfigValidate(t *testing.T) {
},
{
"invalid data",
core.LogsConfig{MaxDays: -1},
[]string{"maxDays"},
core.LogsConfig{
MaxDays: -1,
MaxDataSize: -1,
},
[]string{"maxDays", "maxDataSize"},
},
{
"valid data",
tools/types/json_raw.go
+1 -1
View File
@@ -73,7 +73,7 @@ func (j *JSONRaw) Scan(value any) error {
data = []byte(v)
}
default:
bytes, err := json.Marshal(v)
bytes, err := json.Marshal(v, json.Deterministic(true))
if err != nil {
return err
}