allow invalid utf8 characters when marshalizing json responses

tools/picker/pick.go

@@ -1,6 +1,7 @@
 package picker
 
 import (
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"strings"
 
@@ -29,9 +30,12 @@ func Pick(data any, rawFields string) (any, error) {
 	// implementations are invoked, and then convert it back to a plain
 	// json value that we can further operate on.
 	//
+	// Note that invalid UTF8 characters are mangled for compatibility
+	// with earlier versions and to prevent unnecessery causing an error.
+	//
 	// @todo research other approaches to avoid the double serialization
 	// ---
-	encoded, err := json.Marshal(data)
+	encoded, err := json.Marshal(data, jsontext.AllowInvalidUTF8(true))
 	if err != nil {
 		return nil, err
 	}

tools/picker/pick_test.go

@@ -62,11 +62,11 @@ func TestPickFields(t *testing.T) {
 			"slice of maps with existing and missing fields",
 			[]any{
 				map[string]any{"a": 11, "b": 11, "c": "test1"},
-				map[string]any{"a": 22, "b": 22, "c": "test2"},
+				map[string]any{"a": 22, "b": 22, "c": "\xc3" /* test invalid utf8 mangling */},
 			},
 			"a,  c  ,missing", // test individual fields trim
 			false,
-			`[{"a":11,"c":"test1"},{"a":22,"c":"test2"}]`,
+			`[{"a":11,"c":"test1"},{"a":22,"c":"�"}]`,
 		},
 		{
 			"nested fields with mixed map and any slices",

tools/router/event.go

@@ -185,6 +185,9 @@ const jsonFieldsParam = "fields"
 // It also provides a generic response data fields picker if the "fields" query parameter is set.
 // For example, if you are requesting `?fields=a,b` for `e.JSON(200, map[string]int{ "a":1, "b":2, "c":3 })`,
 // it should result in a JSON response like: `{"a":1, "b": 2}`.
+//
+// Note that invalid UTF8 characters are mangled for compatibility
+// with earlier versions and to prevent unnecessery causing a response error.
 func (e *Event) JSON(status int, data any) error {
 	e.setResponseHeaderIfEmpty(headerContentType, "application/json")
 	e.Response.WriteHeader(status)
@@ -193,7 +196,7 @@ func (e *Event) JSON(status int, data any) error {
 
 	// error response or no fields to pick
 	if rawFields == "" || status < 200 || status > 299 {
-		return json.MarshalWrite(e.Response, data)
+		return json.MarshalWrite(e.Response, data, jsontext.AllowInvalidUTF8(true))
 	}
 
 	// pick only the requested fields
@@ -202,7 +205,7 @@ func (e *Event) JSON(status int, data any) error {
 		return err
 	}
 
-	return json.MarshalWrite(e.Response, modified)
+	return json.MarshalWrite(e.Response, modified, jsontext.AllowInvalidUTF8(true))
 }
 
 // XML writes an XML response.

tools/router/event_test.go

@@ -409,9 +409,14 @@ func TestEventHTML(t *testing.T) {
 }
 
 func TestEventJSON(t *testing.T) {
-	body := map[string]any{"a": 123, "b": 456, "c": "test"}
+	body := map[string]any{
+		"a": 123,
+		"b": true,
+		"c": "test",
+		"d": "\xc3", /* invalid utf8 char to test mangling */
+	}
 	expectedPickedBody := `{"a":123,"c":"test"}`
-	expectedFullBody := `{"a":123,"b":456,"c":"test"}`
+	expectedFullBody := `{"a":123,"b":true,"c":"test","d":"�"}`
 
 	scenarios := []testResponseWriteScenario[any]{
 		{

tools/router/router.go

@@ -2,6 +2,7 @@ package router
 
 import (
 	"bufio"
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"errors"
 	"io"
@@ -176,8 +177,13 @@ func ErrorHandler(resp http.ResponseWriter, req *http.Request, err error) {
 	resp.WriteHeader(apiErr.Status)
 
 	if req.Method != http.MethodHead {
-		// note: deterministic because some logs may depend on the exact serialized error
-		jsonErr := json.MarshalWrite(resp, apiErr, json.Deterministic(true))
+		jsonErr := json.MarshalWrite(
+			resp,
+			apiErr,
+			// note: deterministic because some logs may depend on the exact serialized error
+			json.Deterministic(true),
+			jsontext.AllowInvalidUTF8(false),
+		)
 		if jsonErr != nil {
 			log.Println(jsonErr) // truly rare case, log to stderr only for dev purposes
 		}