allow invalid utf8 characters when marshalizing json responses

CHANGELOG.md

@@ -1,3 +1,9 @@
+## v0.40.1
+
+- `encoding/json/v2` compatibility fixes:
+    - allow mangling invalid UTF8 characters when serializing json data instead of returning response error ([#7814](https://github.com/pocketbase/pocketbase/issues/7814)).
+
+
 ## v0.40.0
 
 - Propagate console command errors and recovered panics to `app.Start()` so that the program can exit with non-zero code while still ensuring that `app.OnTerminate` hook was triggered _(responsible for the app graceful shutdown handling)_.

core/log_model.go

@@ -50,6 +50,8 @@ func (l *Log) DBExport(app App) (map[string]any, error) {
 		result["message"] = l.Message
 	}
 
+	// @todo once added in the standard library consider replacing with
+	// WithByteLimit and WithDepthLimit as suggested in https://github.com/golang/go/issues/56733
 	if len(l.Data) == 0 {
 		result["data"] = l.Data
 	} else {

core/record_model.go

@@ -3,6 +3,7 @@ package core
 import (
 	"bytes"
 	"context"
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"errors"
 	"fmt"
@@ -1329,7 +1330,12 @@ func (record *Record) PublicExport() map[string]any {
 //
 // Only the data exported by `PublicExport()` will be serialized.
 func (m Record) MarshalJSON() ([]byte, error) {
-	return json.Marshal(m.PublicExport(), json.Deterministic(true))
+	return json.Marshal(
+		m.PublicExport(),
+		json.Deterministic(true),
+		// to preserve the old jsonv1 behavior in case of invalid data
+		jsontext.AllowInvalidUTF8(true),
+	)
 }
 
 // UnmarshalJSON implements the [json.Unmarshaler] interface.

core/record_model_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"database/sql"
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"errors"
 	"fmt"
@@ -1438,7 +1439,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			false,
 			nil,
 			nil,
-			`{"collectionId":"_pbc_base_123","collectionName":"test_base","expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id"}`,
+			`{"collectionId":"_pbc_base_123","collectionName":"test_base","expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id"}`,
 		},
 		{
 			"[base] with email visibility",
@@ -1447,7 +1448,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			false,
 			nil,
 			nil,
-			`{"collectionId":"_pbc_base_123","collectionName":"test_base","expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id"}`,
+			`{"collectionId":"_pbc_base_123","collectionName":"test_base","expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id"}`,
 		},
 		{
 			"[base] with custom data",
@@ -1456,7 +1457,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			true,
 			nil,
 			nil,
-			`{"collectionId":"_pbc_base_123","collectionName":"test_base","email":"test_email","emailVisibility":"test_invalid","expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","password":"test_passwordHash","tokenKey":"test_tokenKey","unknown":"test_unknown","verified":true}`,
+			`{"collectionId":"_pbc_base_123","collectionName":"test_base","email":"test_email","emailVisibility":"test_invalid","expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","password":"test_passwordHash","tokenKey":"test_tokenKey","unknown":"test_unknown","verified":true}`,
 		},
 		{
 			"[base] with explicit hide and unhide fields",
@@ -1474,7 +1475,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			true,
 			nil,
 			[]string{"field5", "@pbInternalAbc", "email", "tokenKey", "unknown"},
-			`{"collectionId":"_pbc_base_123","collectionName":"test_base","email":"test_email","emailVisibility":"test_invalid","expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"field5":"field_5","id":"test_id","password":"test_passwordHash","tokenKey":"test_tokenKey","unknown":"test_unknown","verified":true}`,
+			`{"collectionId":"_pbc_base_123","collectionName":"test_base","email":"test_email","emailVisibility":"test_invalid","expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"field5":"field_5","id":"test_id","password":"test_passwordHash","tokenKey":"test_tokenKey","unknown":"test_unknown","verified":true}`,
 		},
 
 		// auth
@@ -1485,7 +1486,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			false,
 			nil,
 			nil,
-			`{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","verified":true}`,
+			`{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","verified":true}`,
 		},
 		{
 			"[auth] with email visibility",
@@ -1494,7 +1495,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			false,
 			nil,
 			nil,
-			`{"collectionId":"_pbc_auth_123","collectionName":"test_auth","email":"test_email","emailVisibility":false,"expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","verified":true}`,
+			`{"collectionId":"_pbc_auth_123","collectionName":"test_auth","email":"test_email","emailVisibility":false,"expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","verified":true}`,
 		},
 		{
 			"[auth] with custom data",
@@ -1503,7 +1504,7 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			true,
 			nil,
 			nil,
-			`{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","unknown":"test_unknown","verified":true}`,
+			`{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"id":"test_id","unknown":"test_unknown","verified":true}`,
 		},
 		{
 			"[auth] with explicit hide and unhide fields",
@@ -1521,13 +1522,13 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			true,
 			nil,
 			[]string{"field5", "@pbInternalAbc", "tokenKey", "unknown", "email"}, // emailVisibility:false has higher priority
-			`{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1","field2":"field_2.png","field3":["test1","test2"],"field5":"field_5","id":"test_id","unknown":"test_unknown","verified":true}`,
+			`{"collectionId":"_pbc_auth_123","collectionName":"test_auth","emailVisibility":false,"expand":{"test":123},"field1":"field_1�","field2":"field_2.png","field3":["test1","test2"],"field5":"field_5","id":"test_id","unknown":"test_unknown","verified":true}`,
 		},
 	}
 
 	data := map[string]any{
 		"id":              "test_id",
-		"field1":          "field_1",
+		"field1":          "field_1\xc3", /* invalid utf8 suffix to test mangling */
 		"field2":          "field_2.png",
 		"field3":          []string{"test1", "test2"},
 		"field4":          "field_4",
@@ -1554,7 +1555,11 @@ func TestRecordPublicExportAndMarshalJSON(t *testing.T) {
 			m.Unhide(s.unhideFields...)
 			m.Hide(s.hideFields...)
 
-			exportResult, err := json.Marshal(m.PublicExport(), json.Deterministic(true))
+			exportResult, err := json.Marshal(
+				m.PublicExport(),
+				json.Deterministic(true),
+				jsontext.AllowInvalidUTF8(true),
+			)
 			if err != nil {
 				t.Fatal(err)
 			}

tools/picker/pick.go

@@ -1,6 +1,7 @@
 package picker
 
 import (
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"strings"
 
@@ -29,9 +30,12 @@ func Pick(data any, rawFields string) (any, error) {
 	// implementations are invoked, and then convert it back to a plain
 	// json value that we can further operate on.
 	//
+	// Note that invalid UTF8 characters are mangled for compatibility
+	// with earlier versions and to prevent unnecessery causing an error.
+	//
 	// @todo research other approaches to avoid the double serialization
 	// ---
-	encoded, err := json.Marshal(data)
+	encoded, err := json.Marshal(data, jsontext.AllowInvalidUTF8(true))
 	if err != nil {
 		return nil, err
 	}

tools/picker/pick_test.go

@@ -62,11 +62,11 @@ func TestPickFields(t *testing.T) {
 			"slice of maps with existing and missing fields",
 			[]any{
 				map[string]any{"a": 11, "b": 11, "c": "test1"},
-				map[string]any{"a": 22, "b": 22, "c": "test2"},
+				map[string]any{"a": 22, "b": 22, "c": "\xc3" /* test invalid utf8 mangling */},
 			},
 			"a,  c  ,missing", // test individual fields trim
 			false,
-			`[{"a":11,"c":"test1"},{"a":22,"c":"test2"}]`,
+			`[{"a":11,"c":"test1"},{"a":22,"c":"�"}]`,
 		},
 		{
 			"nested fields with mixed map and any slices",

tools/router/event.go

@@ -185,6 +185,9 @@ const jsonFieldsParam = "fields"
 // It also provides a generic response data fields picker if the "fields" query parameter is set.
 // For example, if you are requesting `?fields=a,b` for `e.JSON(200, map[string]int{ "a":1, "b":2, "c":3 })`,
 // it should result in a JSON response like: `{"a":1, "b": 2}`.
+//
+// Note that invalid UTF8 characters are mangled for compatibility
+// with earlier versions and to prevent unnecessery causing a response error.
 func (e *Event) JSON(status int, data any) error {
 	e.setResponseHeaderIfEmpty(headerContentType, "application/json")
 	e.Response.WriteHeader(status)
@@ -193,7 +196,7 @@ func (e *Event) JSON(status int, data any) error {
 
 	// error response or no fields to pick
 	if rawFields == "" || status < 200 || status > 299 {
-		return json.MarshalWrite(e.Response, data)
+		return json.MarshalWrite(e.Response, data, jsontext.AllowInvalidUTF8(true))
 	}
 
 	// pick only the requested fields
@@ -202,7 +205,7 @@ func (e *Event) JSON(status int, data any) error {
 		return err
 	}
 
-	return json.MarshalWrite(e.Response, modified)
+	return json.MarshalWrite(e.Response, modified, jsontext.AllowInvalidUTF8(true))
 }
 
 // XML writes an XML response.

tools/router/event_test.go

@@ -409,9 +409,14 @@ func TestEventHTML(t *testing.T) {
 }
 
 func TestEventJSON(t *testing.T) {
-	body := map[string]any{"a": 123, "b": 456, "c": "test"}
+	body := map[string]any{
+		"a": 123,
+		"b": true,
+		"c": "test",
+		"d": "\xc3", /* invalid utf8 char to test mangling */
+	}
 	expectedPickedBody := `{"a":123,"c":"test"}`
-	expectedFullBody := `{"a":123,"b":456,"c":"test"}`
+	expectedFullBody := `{"a":123,"b":true,"c":"test","d":"�"}`
 
 	scenarios := []testResponseWriteScenario[any]{
 		{

tools/router/router.go

@@ -2,6 +2,7 @@ package router
 
 import (
 	"bufio"
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"errors"
 	"io"
@@ -176,8 +177,13 @@ func ErrorHandler(resp http.ResponseWriter, req *http.Request, err error) {
 	resp.WriteHeader(apiErr.Status)
 
 	if req.Method != http.MethodHead {
-		// note: deterministic because some logs may depend on the exact serialized error
-		jsonErr := json.MarshalWrite(resp, apiErr, json.Deterministic(true))
+		jsonErr := json.MarshalWrite(
+			resp,
+			apiErr,
+			// note: deterministic because some logs may depend on the exact serialized error
+			json.Deterministic(true),
+			jsontext.AllowInvalidUTF8(false),
+		)
 		if jsonErr != nil {
 			log.Println(jsonErr) // truly rare case, log to stderr only for dev purposes
 		}