allow invalid utf8 characters when marshalizing json responses

tools/picker/pick.go

@@ -1,6 +1,7 @@
 package picker
 
 import (
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"strings"
 
@@ -29,9 +30,12 @@ func Pick(data any, rawFields string) (any, error) {
 	// implementations are invoked, and then convert it back to a plain
 	// json value that we can further operate on.
 	//
+	// Note that invalid UTF8 characters are mangled for compatibility
+	// with earlier versions and to prevent unnecessery causing an error.
+	//
 	// @todo research other approaches to avoid the double serialization
 	// ---
-	encoded, err := json.Marshal(data)
+	encoded, err := json.Marshal(data, jsontext.AllowInvalidUTF8(true))
 	if err != nil {
 		return nil, err
 	}

tools/picker/pick_test.go

@@ -62,11 +62,11 @@ func TestPickFields(t *testing.T) {
 			"slice of maps with existing and missing fields",
 			[]any{
 				map[string]any{"a": 11, "b": 11, "c": "test1"},
-				map[string]any{"a": 22, "b": 22, "c": "test2"},
+				map[string]any{"a": 22, "b": 22, "c": "\xc3" /* test invalid utf8 mangling */},
 			},
 			"a,  c  ,missing", // test individual fields trim
 			false,
-			`[{"a":11,"c":"test1"},{"a":22,"c":"test2"}]`,
+			`[{"a":11,"c":"test1"},{"a":22,"c":"�"}]`,
 		},
 		{
 			"nested fields with mixed map and any slices",