updated json field validator and allow duplicated keys when serializing record models

tools/picker/pick.go

@@ -41,16 +41,19 @@ func Pick(data any, rawFields string) (any, error) {
 	// Note that invalid UTF8 characters are mangled for compatibility
 	// with earlier versions and to prevent unnecessary causing an error.
 	//
+	// Duplicated keys are also enabled for just in case of old jsonv1 data
+	// (new db entries are usually validated against the jsonv2 semantics).
+	//
 	// @todo research other approaches to avoid the double serialization
 	// ---
-	encoded, err := json.Marshal(data, jsontext.AllowInvalidUTF8(true))
+	encoded, err := json.Marshal(data, jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true))
 	if err != nil {
 		return nil, err
 	}
 
 	var decoded any
 
-	err = json.Unmarshal(encoded, &decoded)
+	err = json.Unmarshal(encoded, &decoded, jsontext.AllowDuplicateNames(true))
 	if err != nil {
 		return nil, err
 	}

tools/picker/pick_test.go

@@ -1,6 +1,7 @@
 package picker_test
 
 import (
+	"encoding/json/jsontext"
 	"encoding/json/v2"
 	"errors"
 	"testing"
@@ -269,6 +270,13 @@ func TestPickFields(t *testing.T) {
 			false,
 			`{"id":"12","rel":{"title":"rel..."},"title":"lo"}`,
 		},
+		{
+			"jsonv1 values",
+			map[string]any{"a": "test\xc3", "b": jsontext.Value(`{"a":1,"a":2}`)},
+			"a, b",
+			false,
+			`{"a":"test�","b":{"a":2}}`,
+		},
 	}
 
 	for _, s := range scenarios {
@@ -284,7 +292,12 @@ func TestPickFields(t *testing.T) {
 				return
 			}
 
-			serialized, err := json.Marshal(result, json.Deterministic(true))
+			serialized, err := json.Marshal(
+				result,
+				json.Deterministic(true),
+				jsontext.AllowInvalidUTF8(true),
+				jsontext.AllowDuplicateNames(true),
+			)
 			if err != nil {
 				t.Fatal(err)
 			}