use fixed message limit seperate from the log data

This commit is contained in:
Gani Georgiev
2026-08-19 10:51:06 +03:00
parent e71be7194a
commit c2cbd9707f
3 changed files with 27 additions and 22 deletions
+2 -1
View File
@@ -13,8 +13,9 @@
- Added `Cross-Origin-Opener-Policy:same-origin` to the default security response headers. - Added `Cross-Origin-Opener-Policy:same-origin` to the default security response headers.
_This is an extra precaution to prevent tab-nabbing in case custom UI plugins use `target="_blank"` without `rel="noopener"`._ _This is an extra precaution to prevent tab-nabbing in case custom UI plugins use `target="_blank"` without `rel="noopener"`._
- Added new log settings option to limit the max `Log.Data` size (default to ~16KB). - Added new log settings option to limit the max `Log.Data` size that will be saved in the database (default to ~16KB).
_This is an extra precaution for the cases when logging user supplied data without validating it beforehand._ _This is an extra precaution for the cases when logging user supplied data without validating it beforehand._
_If the resulting `Log.Data` json is above the limit, it is truncated to the last valid decoded character and an extra `"__pb_truncated__":true` log data entry will be added.`_ _If the resulting `Log.Data` json is above the limit, it is truncated to the last valid decoded character and an extra `"__pb_truncated__":true` log data entry will be added.`_
_Additionally, for just in case the log message is also truncated at max 8k characters._
- (@todo) Bumped the min Go version to 1.27.0 and migrated to the new `encoding/json/v2` package. - (@todo) Bumped the min Go version to 1.27.0 and migrated to the new `encoding/json/v2` package.
+11 -8
View File
@@ -13,7 +13,10 @@ var (
const LogsTableName = "_logs" const LogsTableName = "_logs"
const defaultMaxLogDataSize = 16 << 10 // ~16kb const (
defaultMaxLogDataSize = 16 << 10 // ~16kb
defaultMaxLogMessageSize = 8000
)
type Log struct { type Log struct {
BaseModel BaseModel
@@ -39,15 +42,10 @@ func (l *Log) DBExport(app App) (map[string]any, error) {
"level": l.Level, "level": l.Level,
} }
maxDataSize := app.Settings().Logs.MaxDataSize
if maxDataSize == 0 {
maxDataSize = defaultMaxLogDataSize
}
// truncate the raw message bytes // truncate the raw message bytes
// (this is expected to be very rare so it is ok even if multi-byte chars) // (this is expected to be very rare so it is ok even if multi-byte chars)
if int64(len(l.Message)) > maxDataSize { if int64(len(l.Message)) > defaultMaxLogMessageSize {
result["message"] = l.Message[:maxDataSize] result["message"] = l.Message[:defaultMaxLogMessageSize]
} else { } else {
result["message"] = l.Message result["message"] = l.Message
} }
@@ -55,6 +53,11 @@ func (l *Log) DBExport(app App) (map[string]any, error) {
if len(l.Data) == 0 { if len(l.Data) == 0 {
result["data"] = l.Data result["data"] = l.Data
} else { } else {
maxDataSize := app.Settings().Logs.MaxDataSize
if maxDataSize == 0 {
maxDataSize = defaultMaxLogDataSize
}
rawData, err := l.Data.MarshalJSON() rawData, err := l.Data.MarshalJSON()
if int64(len(rawData)) > maxDataSize { if int64(len(rawData)) > maxDataSize {
truncatedData := types.JSONMap[any]{} truncatedData := types.JSONMap[any]{}
+14 -13
View File
@@ -32,7 +32,8 @@ func TestLogDBExport(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defaultLimit := 16 << 10 messageLimit := 8000
dataLimit := 16 << 10
scenarios := []struct { scenarios := []struct {
name string name string
@@ -47,7 +48,7 @@ func TestLogDBExport(t *testing.T) {
`{"created":"","data":{},"id":"","level":0,"message":""}`, `{"created":"","data":{},"id":"","level":0,"message":""}`,
}, },
{ {
"with message and data below the default limit", "with message and data below the default limits",
core.Log{ core.Log{
BaseModel: core.BaseModel{Id: "test_id"}, BaseModel: core.BaseModel{Id: "test_id"},
Created: date, Created: date,
@@ -59,40 +60,40 @@ func TestLogDBExport(t *testing.T) {
`{"created":"2026-08-18 10:20:30.456Z","data":{"a":"test1","b":"test2"},"id":"test_id","level":123,"message":"test_message"}`, `{"created":"2026-08-18 10:20:30.456Z","data":{"a":"test1","b":"test2"},"id":"test_id","level":123,"message":"test_message"}`,
}, },
{ {
"with message and data exactly the default limit", "with message and data exactly the default limits",
core.Log{ core.Log{
BaseModel: core.BaseModel{Id: "test_id"}, BaseModel: core.BaseModel{Id: "test_id"},
Created: date, Created: date,
Level: 123, Level: 123,
Message: strings.Repeat("a", defaultLimit), Message: strings.Repeat("a", messageLimit),
Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", defaultLimit-32)}, Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", dataLimit-32)},
}, },
0, 0,
`{"created":"2026-08-18 10:20:30.456Z","data":{"a":"test1","b":"test2","c":"` + strings.Repeat("a", defaultLimit-32) + `"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", defaultLimit) + `"}`, `{"created":"2026-08-18 10:20:30.456Z","data":{"a":"test1","b":"test2","c":"` + strings.Repeat("a", dataLimit-32) + `"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", messageLimit) + `"}`,
}, },
{ {
"with message and data above the default limit", "with message and data above the default limits",
core.Log{ core.Log{
BaseModel: core.BaseModel{Id: "test_id"}, BaseModel: core.BaseModel{Id: "test_id"},
Created: date, Created: date,
Level: 123, Level: 123,
Message: strings.Repeat("a", defaultLimit) + "x", // "x" should be omitted Message: strings.Repeat("a", messageLimit) + "x", // "x" should be omitted
Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", defaultLimit-32) + "x"}, // the end will be incomplete and something like `"c":"...aaaaaax` Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", dataLimit-32) + "x"}, // the end will be incomplete and something like `"c":"...aaaaaax`
}, },
0, 0,
`{"created":"2026-08-18 10:20:30.456Z","data":{"__pb_truncated__":true,"a":"test1","b":"test2","c":"` + strings.Repeat("a", defaultLimit-32) + `x"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", defaultLimit) + `"}`, `{"created":"2026-08-18 10:20:30.456Z","data":{"__pb_truncated__":true,"a":"test1","b":"test2","c":"` + strings.Repeat("a", dataLimit-32) + `x"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", messageLimit) + `"}`,
}, },
{ {
"with message and data above custom limit", "with data above custom limit",
core.Log{ core.Log{
BaseModel: core.BaseModel{Id: "test_id"}, BaseModel: core.BaseModel{Id: "test_id"},
Created: date, Created: date,
Level: 123, Level: 123,
Message: strings.Repeat("a", 2<<10) + "x", // "x" should be omitted Message: "test_message",
Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", (2<<10)-32) + "x"}, // the end will be incomplete and something like `"c":"...aaaaaax` Data: types.JSONMap[any]{"a": "test1", "b": "test2", "c": strings.Repeat("a", (2<<10)-32) + "x"}, // the end will be incomplete and something like `"c":"...aaaaaax`
}, },
2 << 10, 2 << 10,
`{"created":"2026-08-18 10:20:30.456Z","data":{"__pb_truncated__":true,"a":"test1","b":"test2","c":"` + strings.Repeat("a", (2<<10)-32) + `x"},"id":"test_id","level":123,"message":"` + strings.Repeat("a", 2<<10) + `"}`, `{"created":"2026-08-18 10:20:30.456Z","data":{"__pb_truncated__":true,"a":"test1","b":"test2","c":"` + strings.Repeat("a", (2<<10)-32) + `x"},"id":"test_id","level":123,"message":"test_message"}`,
}, },
} }