diff --git a/CHANGELOG.md b/CHANGELOG.md index 4498b709..53bc0507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.40.1 + +- `encoding/json/v2` compatibility fixes: + - allow mangling invalid UTF8 characters when serializing json data instead of returning response error ([#7814](https://github.com/pocketbase/pocketbase/issues/7814)). + + ## v0.40.0 - Propagate console command errors and recovered panics to `app.Start()` so that the program can exit with non-zero code while still ensuring that `app.OnTerminate` hook was triggered _(responsible for the app graceful shutdown handling)_. diff --git a/core/log_model.go b/core/log_model.go index a85b7111..ade9232a 100644 --- a/core/log_model.go +++ b/core/log_model.go @@ -50,6 +50,8 @@ func (l *Log) DBExport(app App) (map[string]any, error) { result["message"] = l.Message } + // @todo once added in the standard library consider replacing with + // WithByteLimit and WithDepthLimit as suggested in https://github.com/golang/go/issues/56733 if len(l.Data) == 0 { result["data"] = l.Data } else { diff --git a/core/record_model.go b/core/record_model.go index c8e8bba0..b27b5203 100644 --- a/core/record_model.go +++ b/core/record_model.go @@ -3,6 +3,7 @@ package core import ( "bytes" "context" + "encoding/json/jsontext" "encoding/json/v2" "errors" "fmt" @@ -1329,7 +1330,12 @@ func (record *Record) PublicExport() map[string]any { // // Only the data exported by `PublicExport()` will be serialized. func (m Record) MarshalJSON() ([]byte, error) { - return json.Marshal(m.PublicExport(), json.Deterministic(true)) + return json.Marshal( + m.PublicExport(), + json.Deterministic(true), + // to preserve the old jsonv1 behavior in case of invalid data + jsontext.AllowInvalidUTF8(true), + ) } // UnmarshalJSON implements the [json.Unmarshaler] interface. diff --git a/core/record_model_test.go b/core/record_model_test.go index 04959e3d..75e046e2 100644 --- a/core/record_model_test.go +++ b/core/record_model_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "encoding/json/jsontext" "encoding/json/v2" "errors" "fmt" @@ -1438,7 +1439,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { false, nil, nil, - `{"collectionId":"_pbc_base_123","collectionName":"test_base","expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id"}`, + `{"collectionId":"_pbc_base_123","collectionName":"test_base","expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id"}`, }, { "[base] with email visibility", @@ -1447,7 +1448,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { false, nil, nil, - `{"collectionId":"_pbc_base_123","collectionName":"test_base","expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id"}`, + `{"collectionId":"_pbc_base_123","collectionName":"test_base","expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id"}`, }, { "[base] with custom data", @@ -1456,7 +1457,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { true, nil, nil, - `{"collectionId":"_pbc_base_123","collectionName":"test_base","email":"test_email","emailVisibility":"test_invalid","expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","password":"test_passwordHash","tokenKey":"test_tokenKey","unknown":"test_unknown","verified":true}`, + `{"collectionId":"_pbc_base_123","collectionName":"test_base","email":"test_email","emailVisibility":"test_invalid","expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","password":"test_passwordHash","tokenKey":"test_tokenKey","unknown":"test_unknown","verified":true}`, }, { "[base] with explicit hide and unhide fields", @@ -1474,7 +1475,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { true, nil, []string{"field5", "@pbInternalAbc", "email", "tokenKey", "unknown"}, - `{"collectionId":"_pbc_base_123","collectionName":"test_base","email":"test_email","emailVisibility":"test_invalid","expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"field5":"field_5","id":"test_id","password":"test_passwordHash","tokenKey":"test_tokenKey","unknown":"test_unknown","verified":true}`, + `{"collectionId":"_pbc_base_123","collectionName":"test_base","email":"test_email","emailVisibility":"test_invalid","expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"field5":"field_5","id":"test_id","password":"test_passwordHash","tokenKey":"test_tokenKey","unknown":"test_unknown","verified":true}`, }, // auth @@ -1485,7 +1486,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { false, nil, nil, - `{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","verified":true}`, + `{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","verified":true}`, }, { "[auth] with email visibility", @@ -1494,7 +1495,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { false, nil, nil, - `{"collectionId":"_pbc_auth_123","collectionName":"test_auth","email":"test_email","emailVisibility":false,"expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","verified":true}`, + `{"collectionId":"_pbc_auth_123","collectionName":"test_auth","email":"test_email","emailVisibility":false,"expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","verified":true}`, }, { "[auth] with custom data", @@ -1503,7 +1504,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { true, nil, nil, - `{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","unknown":"test_unknown","verified":true}`, + `{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","unknown":"test_unknown","verified":true}`, }, { "[auth] with explicit hide and unhide fields", @@ -1521,13 +1522,13 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { true, nil, []string{"field5", "@pbInternalAbc", "tokenKey", "unknown", "email"}, // emailVisibility:false has higher priority - `{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"field5":"field_5","id":"test_id","unknown":"test_unknown","verified":true}`, + `{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"field5":"field_5","id":"test_id","unknown":"test_unknown","verified":true}`, }, } data := map[string]any{ "id": "test_id", - "field1": "field_1", + "field1": "field_1\xc3", /* invalid utf8 suffix to test mangling */ "field2": "field_2.png", "field3": []string{"test1", "test2"}, "field4": "field_4", @@ -1554,7 +1555,11 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) { m.Unhide(s.unhideFields...) m.Hide(s.hideFields...) - exportResult, err := json.Marshal(m.PublicExport(), json.Deterministic(true)) + exportResult, err := json.Marshal( + m.PublicExport(), + json.Deterministic(true), + jsontext.AllowInvalidUTF8(true), + ) if err != nil { t.Fatal(err) } diff --git a/tools/picker/pick.go b/tools/picker/pick.go index a419a3e0..880957d3 100644 --- a/tools/picker/pick.go +++ b/tools/picker/pick.go @@ -1,6 +1,7 @@ package picker import ( + "encoding/json/jsontext" "encoding/json/v2" "strings" @@ -29,9 +30,12 @@ func Pick(data any, rawFields string) (any, error) { // implementations are invoked, and then convert it back to a plain // json value that we can further operate on. // + // Note that invalid UTF8 characters are mangled for compatibility + // with earlier versions and to prevent unnecessery causing an error. + // // @todo research other approaches to avoid the double serialization // --- - encoded, err := json.Marshal(data) + encoded, err := json.Marshal(data, jsontext.AllowInvalidUTF8(true)) if err != nil { return nil, err } diff --git a/tools/picker/pick_test.go b/tools/picker/pick_test.go index fdd438c1..4f2ea37c 100644 --- a/tools/picker/pick_test.go +++ b/tools/picker/pick_test.go @@ -62,11 +62,11 @@ func TestPickFields(t *testing.T) { "slice of maps with existing and missing fields", []any{ map[string]any{"a": 11, "b": 11, "c": "test1"}, - map[string]any{"a": 22, "b": 22, "c": "test2"}, + map[string]any{"a": 22, "b": 22, "c": "\xc3" /* test invalid utf8 mangling */}, }, "a, c ,missing", // test individual fields trim false, - `[{"a":11,"c":"test1"},{"a":22,"c":"test2"}]`, + `[{"a":11,"c":"test1"},{"a":22,"c":"�"}]`, }, { "nested fields with mixed map and any slices", diff --git a/tools/router/event.go b/tools/router/event.go index d0f30bb5..fdff8c04 100644 --- a/tools/router/event.go +++ b/tools/router/event.go @@ -185,6 +185,9 @@ const jsonFieldsParam = "fields" // It also provides a generic response data fields picker if the "fields" query parameter is set. // For example, if you are requesting `?fields=a,b` for `e.JSON(200, map[string]int{ "a":1, "b":2, "c":3 })`, // it should result in a JSON response like: `{"a":1, "b": 2}`. +// +// Note that invalid UTF8 characters are mangled for compatibility +// with earlier versions and to prevent unnecessery causing a response error. func (e *Event) JSON(status int, data any) error { e.setResponseHeaderIfEmpty(headerContentType, "application/json") e.Response.WriteHeader(status) @@ -193,7 +196,7 @@ func (e *Event) JSON(status int, data any) error { // error response or no fields to pick if rawFields == "" || status < 200 || status > 299 { - return json.MarshalWrite(e.Response, data) + return json.MarshalWrite(e.Response, data, jsontext.AllowInvalidUTF8(true)) } // pick only the requested fields @@ -202,7 +205,7 @@ func (e *Event) JSON(status int, data any) error { return err } - return json.MarshalWrite(e.Response, modified) + return json.MarshalWrite(e.Response, modified, jsontext.AllowInvalidUTF8(true)) } // XML writes an XML response. diff --git a/tools/router/event_test.go b/tools/router/event_test.go index 1a76cd93..28f11381 100644 --- a/tools/router/event_test.go +++ b/tools/router/event_test.go @@ -409,9 +409,14 @@ func TestEventHTML(t *testing.T) { } func TestEventJSON(t *testing.T) { - body := map[string]any{"a": 123, "b": 456, "c": "test"} + body := map[string]any{ + "a": 123, + "b": true, + "c": "test", + "d": "\xc3", /* invalid utf8 char to test mangling */ + } expectedPickedBody := `{"a":123,"c":"test"}` - expectedFullBody := `{"a":123,"b":456,"c":"test"}` + expectedFullBody := `{"a":123,"b":true,"c":"test","d":"�"}` scenarios := []testResponseWriteScenario[any]{ { diff --git a/tools/router/router.go b/tools/router/router.go index 77c935e8..06cc50e6 100644 --- a/tools/router/router.go +++ b/tools/router/router.go @@ -2,6 +2,7 @@ package router import ( "bufio" + "encoding/json/jsontext" "encoding/json/v2" "errors" "io" @@ -176,8 +177,13 @@ func ErrorHandler(resp http.ResponseWriter, req *http.Request, err error) { resp.WriteHeader(apiErr.Status) if req.Method != http.MethodHead { - // note: deterministic because some logs may depend on the exact serialized error - jsonErr := json.MarshalWrite(resp, apiErr, json.Deterministic(true)) + jsonErr := json.MarshalWrite( + resp, + apiErr, + // note: deterministic because some logs may depend on the exact serialized error + json.Deterministic(true), + jsontext.AllowInvalidUTF8(false), + ) if jsonErr != nil { log.Println(jsonErr) // truly rare case, log to stderr only for dev purposes }