updated json field validator and allow duplicated keys when serializing record models

tools/picker/pick.go

@@ -41,16 +41,19 @@ func Pick(data any, rawFields string) (any, error) {
 	// Note that invalid UTF8 characters are mangled for compatibility
 	// with earlier versions and to prevent unnecessary causing an error.
 	//
+	// Duplicated keys are also enabled for just in case of old jsonv1 data
+	// (new db entries are usually validated against the jsonv2 semantics).
+	//
 	// @todo research other approaches to avoid the double serialization
 	// ---
-	encoded, err := json.Marshal(data, jsontext.AllowInvalidUTF8(true))
+	encoded, err := json.Marshal(data, jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true))
 	if err != nil {
 		return nil, err
 	}
 
 	var decoded any
 
-	err = json.Unmarshal(encoded, &decoded)
+	err = json.Unmarshal(encoded, &decoded, jsontext.AllowDuplicateNames(true))
 	if err != nil {
 		return nil, err
 	}

tools/picker/pick_test.go

@@ -1,6 +1,7 @@
 package picker_test
 
 import (
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"errors"
 	"testing"
@@ -269,6 +270,13 @@ func TestPickFields(t *testing.T) {
 			false,
 			`{"id":"12","rel":{"title":"rel..."},"title":"lo"}`,
 		},
+		{
+			"jsonv1 values",
+			map[string]any{"a": "test\xc3", "b": jsontext.Value(`{"a":1,"a":2}`)},
+			"a, b",
+			false,
+			`{"a":"test�","b":{"a":2}}`,
+		},
 	}
 
 	for _, s := range scenarios {
@@ -284,7 +292,12 @@ func TestPickFields(t *testing.T) {
 				return
 			}
 
-			serialized, err := json.Marshal(result, json.Deterministic(true))
+			serialized, err := json.Marshal(
+				result,
+				json.Deterministic(true),
+				jsontext.AllowInvalidUTF8(true),
+				jsontext.AllowDuplicateNames(true),
+			)
 			if err != nil {
 				t.Fatal(err)
 			}

tools/router/event.go

@@ -206,7 +206,14 @@ func (e *Event) JSON(status int, data any) error {
 	e.setResponseHeaderIfEmpty(headerContentType, "application/json")
 	e.Response.WriteHeader(status)
 
-	return json.MarshalWrite(e.Response, data, jsontext.AllowInvalidUTF8(true))
+	return json.MarshalWrite(
+		e.Response,
+		data,
+		// for compliance with old data (and slightly better performance)
+		jsontext.AllowDuplicateNames(true),
+		// preserve the old jsonv1 behavior in case of invalid data
+		jsontext.AllowInvalidUTF8(true),
+	)
 }
 
 // XML writes an XML response.

tools/router/event_test.go

@@ -3,6 +3,7 @@ package router_test
 import (
 	"bytes"
 	"crypto/tls"
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"encoding/xml"
 	"errors"
@@ -21,6 +22,7 @@ import (
 	validation "github.com/pocketbase/ozzo-validation/v4"
 	"github.com/pocketbase/pocketbase/tools/picker"
 	"github.com/pocketbase/pocketbase/tools/router"
+	"github.com/pocketbase/pocketbase/tools/types"
 )
 
 type unwrapTester struct {
@@ -428,10 +430,11 @@ func TestEventJSON(t *testing.T) {
 		"a": 123,
 		"b": true,
 		"c": "test",
-		"d": "\xc3", /* invalid utf8 char to test mangling */
+		"d": "\xc3",                         // invalid utf8 char to test mangling
+		"e": types.JSONRaw(`{"a":1,"a":2}`), // duplicated keys to ensure compliance with old data
 	}
 	expectedPickedBody := `{"a":123,"c":"test"}`
-	expectedFullBody := `{"a":123,"b":true,"c":"test","d":"�"}`
+	expectedFullBody := `{"a":123,"b":true,"c":"test","d":"�","e":{"a":2}}`
 
 	scenarios := []testResponseWriteScenario[any]{
 		{
@@ -1074,9 +1077,9 @@ func testEventResponseWrite[T any](
 		// try to deserialize into a map and then serialize again in a
 		// deterministic manner in case it is json
 		var jsonBody any
-		err = json.Unmarshal(rawBody, &jsonBody)
+		err = json.Unmarshal(rawBody, &jsonBody, jsontext.AllowDuplicateNames(true))
 		if err == nil {
-			normalized, err := json.Marshal(jsonBody, json.Deterministic(true))
+			normalized, err := json.Marshal(jsonBody, json.Deterministic(true), jsontext.AllowDuplicateNames(true))
 			if err != nil {
 				t.Fatalf("Failed to deterministicly serialize json body\n%s\ngot\n%v", jsonBody, err)
 			}

tools/types/json_raw.go

@@ -7,6 +7,8 @@ import (
 	"errors"
 )
 
+// @todo consider deprecating in favour of jsontext.Value
+
 // JSONRaw defines a json value type that is safe for db read/write.
 type JSONRaw []byte