Phase 2: the event core

Implements PLAN.md Phase 2: the library every later phase is built on,
as pure code under server/internal/matrix with no HTTP and no storage.

canonicaljson: the canonical JSON encoding, hand-written because
encoding/json escapes <, > and & and accepts floats. Integers only, in
[-(2^53)+1, 2^53-1]; lone surrogate escapes rejected rather than turned
into U+FFFD.

signing: the Signing JSON algorithm alongside the existing key handling,
and one lenient base64 decoder shared by the packages above it.

id: identifier grammars as they appear inside events, deliberately laxer
than userapi's account rules because rooms carry historical user IDs.

roomversion: the registry. Versions 3 to 12 are supported, each written
out in full as a Rules value with the flags the other packages read.
Versions 1 and 2 are known but refused: their event format and state
resolution v1 wait for federation. createRoom will default to 11, as the
spec asks for a while after 1.16, with 12 available.

event: PDU parsing and validation, content and reference hashes, event
IDs, the redaction algorithm for every supported version, signing and
verification, and a Builder.

auth: the authorisation rules in spec order, power levels with their
defaults, v12 creators with infinite power, the auth events selection
algorithm, rule 3's auth_events checks, and the separate test for
whether a redaction applies. Comments use the room version 12 rule
numbering, with the version 11 numbers given where they differ.

stateres: state resolution v2, and version 12's amendment (an empty
starting state and the conflicted state subgraph).

Tested against the spec's own vectors byte for byte: the canonical JSON
examples, both JSON signing vectors and both event signing vectors.
State resolution ports scenarios from Synapse's suite. canonicaljson
and event.Parse have fuzz targets.

The capabilities endpoint now advertises the registry instead of a
hard-coded "11". docs/event-core.md describes the packages and the
known gaps.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/internal/matrix/canonicaljson/canonicaljson.go

@@ -0,0 +1,263 @@
+// Package canonicaljson implements the canonical JSON encoding Matrix uses
+// wherever two servers have to agree on bytes rather than on values: event
+// content hashes, event IDs, and every signature.
+//
+// The rules (spec appendices, "Canonical JSON") are short. Sort object keys
+// by Unicode code point, emit no whitespace, allow only integers in
+// [-(2^53)+1, 2^53-1], and escape as little as possible — non-ASCII text
+// goes out as raw UTF-8 rather than \u escapes.
+//
+// encoding/json cannot produce this on its own: it escapes <, > and & so
+// that output is safe to paste into HTML, which no other Matrix
+// implementation does, and it reads and writes floats happily. So the
+// encoder below is hand-written and encoding/json is used only to parse.
+package canonicaljson
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+// The range Matrix allows for integers, which is the range a float64 holds
+// without losing precision — the limit comes from JavaScript, like so much
+// else about JSON.
+const (
+	MaxInt int64 = 1<<53 - 1
+	MinInt int64 = -(1<<53 - 1)
+)
+
+// ErrNotCanonical is the root of every error this package returns, so that
+// a caller who only wants to answer M_BAD_JSON can test for it.
+var ErrNotCanonical = errors.New("not canonical JSON")
+
+// Encode canonicalises a JSON value.
+//
+// Input need not already be canonical — keys may be in any order and
+// strings may use any legal escaping — but it must be *encodable* as
+// canonical JSON, so a float or an oversized integer is an error rather
+// than something to round off.
+func Encode(raw []byte) ([]byte, error) {
+	v, err := parse(raw)
+	if err != nil {
+		return nil, err
+	}
+	buf := bytes.NewBuffer(make([]byte, 0, len(raw)))
+	if err := write(buf, v); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// Marshal encodes v as JSON and canonicalises the result.
+func Marshal(v any) ([]byte, error) {
+	raw, err := json.Marshal(v)
+	if err != nil {
+		return nil, fmt.Errorf("marshal JSON: %w", err)
+	}
+	return Encode(raw)
+}
+
+// Valid reports whether raw can be encoded as canonical JSON, with an error
+// that says what is wrong with it if it cannot.
+func Valid(raw []byte) error {
+	_, err := Encode(raw)
+	return err
+}
+
+// parse reads raw into the generic Go representation, keeping numbers as
+// their source text so that write can decide what to do with them.
+func parse(raw []byte) (any, error) {
+	if !utf8.Valid(raw) {
+		return nil, fmt.Errorf("%w: not valid UTF-8", ErrNotCanonical)
+	}
+	if err := checkEscapes(raw); err != nil {
+		return nil, err
+	}
+
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.UseNumber()
+	var v any
+	if err := dec.Decode(&v); err != nil {
+		return nil, fmt.Errorf("%w: %w", ErrNotCanonical, err)
+	}
+	// A signature covers one value. Anything after it is either a second
+	// document or junk, and either way the bytes we would hash are not the
+	// bytes we were handed.
+	if dec.More() {
+		return nil, fmt.Errorf("%w: trailing data after the JSON value", ErrNotCanonical)
+	}
+	return v, nil
+}
+
+func write(buf *bytes.Buffer, v any) error {
+	switch v := v.(type) {
+	case nil:
+		buf.WriteString("null")
+	case bool:
+		if v {
+			buf.WriteString("true")
+		} else {
+			buf.WriteString("false")
+		}
+	case json.Number:
+		return writeNumber(buf, v)
+	case string:
+		writeString(buf, v)
+	case []any:
+		buf.WriteByte('[')
+		for i, elem := range v {
+			if i > 0 {
+				buf.WriteByte(',')
+			}
+			if err := write(buf, elem); err != nil {
+				return err
+			}
+		}
+		buf.WriteByte(']')
+	case map[string]any:
+		keys := make([]string, 0, len(v))
+		for k := range v {
+			keys = append(keys, k)
+		}
+		// Go compares strings byte by byte, and UTF-8 is ordered so that a
+		// byte-wise comparison is a code point comparison. Sorting the keys
+		// as Go strings is therefore exactly what the spec asks for.
+		slices.Sort(keys)
+
+		buf.WriteByte('{')
+		for i, k := range keys {
+			if i > 0 {
+				buf.WriteByte(',')
+			}
+			writeString(buf, k)
+			buf.WriteByte(':')
+			if err := write(buf, v[k]); err != nil {
+				return err
+			}
+		}
+		buf.WriteByte('}')
+	default:
+		return fmt.Errorf("%w: cannot encode a value of type %T", ErrNotCanonical, v)
+	}
+	return nil
+}
+
+// writeNumber re-emits a number from its source text.
+//
+// Matrix has no floats, so anything with a decimal point or an exponent is
+// rejected rather than rounded: a server that quietly turned 1.0 into 1
+// would compute a hash nobody else agrees with. The value is reformatted
+// from the parsed integer, which also normalises the one spelling the
+// grammar forbids but parsers accept, "-0".
+func writeNumber(buf *bytes.Buffer, n json.Number) error {
+	s := string(n)
+	if strings.ContainsAny(s, ".eE") {
+		return fmt.Errorf("%w: %s is not an integer; Matrix JSON has no floats", ErrNotCanonical, s)
+	}
+	i, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return fmt.Errorf("%w: %s is not an integer we can represent", ErrNotCanonical, s)
+	}
+	if i > MaxInt || i < MinInt {
+		return fmt.Errorf("%w: %s is outside the allowed range [%d, %d]", ErrNotCanonical, s, MinInt, MaxInt)
+	}
+	buf.WriteString(strconv.FormatInt(i, 10))
+	return nil
+}
+
+const hexDigits = "0123456789abcdef"
+
+// writeString writes a JSON string literal with the minimum of escaping:
+// the two characters that must always be escaped, the five control
+// characters that have a short form, and \u00XX for the remaining controls.
+// Everything else, non-ASCII included, is copied through as UTF-8.
+func writeString(buf *bytes.Buffer, s string) {
+	buf.WriteByte('"')
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; c {
+		case '"':
+			buf.WriteString(`\"`)
+		case '\\':
+			buf.WriteString(`\\`)
+		case '\b':
+			buf.WriteString(`\b`)
+		case '\f':
+			buf.WriteString(`\f`)
+		case '\n':
+			buf.WriteString(`\n`)
+		case '\r':
+			buf.WriteString(`\r`)
+		case '\t':
+			buf.WriteString(`\t`)
+		default:
+			if c < 0x20 {
+				buf.WriteString(`\u00`)
+				buf.WriteByte(hexDigits[c>>4])
+				buf.WriteByte(hexDigits[c&0xf])
+				continue
+			}
+			buf.WriteByte(c)
+		}
+	}
+	buf.WriteByte('"')
+}
+
+// checkEscapes rejects \u escapes that do not form a character.
+//
+// encoding/json turns a lone surrogate into U+FFFD instead of failing,
+// which would quietly change the bytes we are about to hash — Python's
+// json, which the spec uses as its reference encoder, raises instead.
+// Scanning the source for the escape is the only way to tell the two cases
+// apart once the parser has run.
+//
+// Walking from the first byte is safe because a backslash can only appear
+// inside a string, so every one found here opens an escape sequence.
+func checkEscapes(raw []byte) error {
+	for i := 0; i < len(raw); i++ {
+		if raw[i] != '\\' {
+			continue
+		}
+		if i+1 >= len(raw) {
+			break // a truncated escape; the parser reports it better than we can
+		}
+		if raw[i+1] != 'u' {
+			i++ // an escape of one character, including a backslash itself
+			continue
+		}
+		r, ok := hexEscape(raw, i)
+		if !ok {
+			break
+		}
+		switch {
+		case r >= 0xD800 && r <= 0xDBFF:
+			low, ok := hexEscape(raw, i+6)
+			if !ok || low < 0xDC00 || low > 0xDFFF {
+				return fmt.Errorf("%w: \\u%04X is a high surrogate with no low surrogate after it", ErrNotCanonical, r)
+			}
+			i += 11
+		case r >= 0xDC00 && r <= 0xDFFF:
+			return fmt.Errorf("%w: \\u%04X is a low surrogate with no high surrogate before it", ErrNotCanonical, r)
+		default:
+			i += 5
+		}
+	}
+	return nil
+}
+
+// hexEscape reads the \uXXXX escape starting at i, if that is what is there.
+func hexEscape(raw []byte, i int) (rune, bool) {
+	if i+6 > len(raw) || raw[i] != '\\' || raw[i+1] != 'u' {
+		return 0, false
+	}
+	v, err := strconv.ParseUint(string(raw[i+2:i+6]), 16, 32)
+	if err != nil {
+		return 0, false
+	}
+	return rune(v), true
+}

server/internal/matrix/canonicaljson/canonicaljson_test.go

@@ -0,0 +1,292 @@
+package canonicaljson
+
+import (
+	"errors"
+	"strings"
+	"testing"
+)
+
+// TestSpecExamples runs the worked examples from the spec appendices,
+// section "Canonical JSON". They are the one set of values every
+// implementation is expected to agree on.
+func TestSpecExamples(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"empty object", `{}`, `{}`},
+		{"values", "{\n  \"one\": 1,\n  \"two\": \"Two\"\n}", `{"one":1,"two":"Two"}`},
+		{"sorts keys", "{\n  \"b\": \"2\",\n  \"a\": \"1\"\n}", `{"a":"1","b":"2"}`},
+		{"sorts keys, no whitespace to remove", `{"b":"2","a":"1"}`, `{"a":"1","b":"2"}`},
+		{
+			"nested",
+			`{
+				"auth": {
+					"success": true,
+					"mxid": "@john.doe:example.com",
+					"profile": {
+						"display_name": "John Doe",
+						"three_pids": [
+							{"medium": "email", "address": "john.doe@example.org"},
+							{"medium": "msisdn", "address": "123456789"}
+						]
+					}
+				}
+			}`,
+			`{"auth":{"mxid":"@john.doe:example.com","profile":{"display_name":"John Doe",` +
+				`"three_pids":[{"address":"john.doe@example.org","medium":"email"},` +
+				`{"address":"123456789","medium":"msisdn"}]},"success":true}}`,
+		},
+		{"non-ASCII stays UTF-8", `{"a": "日本語"}`, `{"a":"日本語"}`},
+		{"non-ASCII keys sort by code point", `{"本":2,"日":1}`, `{"日":1,"本":2}`},
+		{"escapes are decoded", `{"a": "日"}`, `{"a":"日"}`},
+		{"null", `{"a": null}`, `{"a":null}`},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := Encode([]byte(tc.in))
+			if err != nil {
+				t.Fatalf("Encode: %v", err)
+			}
+			if string(got) != tc.want {
+				t.Errorf("Encode() = %s, want %s", got, tc.want)
+			}
+		})
+	}
+}
+
+// TestEncodeIsIdempotent checks that canonicalising canonical JSON changes
+// nothing, which is what lets a stored event be re-signed and re-hashed
+// without drifting.
+func TestEncodeIsIdempotent(t *testing.T) {
+	once, err := Encode([]byte(`{"b":[1,2,{"d":"日","c":null}],"a":true}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	twice, err := Encode(once)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(once) != string(twice) {
+		t.Errorf("second pass changed the output:\n %s\n %s", once, twice)
+	}
+}
+
+// TestNoHTMLEscaping is the reason this package exists rather than calling
+// encoding/json: the standard library escapes these three characters, and
+// an event containing one would hash differently on every other server.
+func TestNoHTMLEscaping(t *testing.T) {
+	got, err := Encode([]byte(`{"body":"<b> & </b>"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := `{"body":"<b> & </b>"}`; string(got) != want {
+		t.Errorf("Encode() = %s, want %s", got, want)
+	}
+}
+
+// TestStringEscaping checks the escapes the grammar allows: the short forms
+// where they exist and \u00XX for the other control characters, with
+// everything else passed through.
+func TestStringEscaping(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"quote and backslash", `{"a":"\" \\"}`, `{"a":"\" \\"}`},
+		{"short forms", `{"a":"\b\t\n\f\r"}`, `{"a":"\b\t\n\f\r"}`},
+		{"other controls", `{"a":"\u0000\u001f\u000b\u000e"}`, `{"a":"\u0000\u001f\u000b\u000e"}`},
+		{"literal newline is not legal JSON input", "{\"a\":\"\t\"}", ""},
+		{"solidus is not escaped", `{"a":"/"}`, `{"a":"/"}`},
+		{"surrogate pair", `{"a":"😀"}`, `{"a":"😀"}`},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := Encode([]byte(tc.in))
+			if tc.want == "" {
+				if err == nil {
+					t.Fatalf("Encode(%s) = %s, want an error", tc.in, got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("Encode: %v", err)
+			}
+			if string(got) != tc.want {
+				t.Errorf("Encode() = %s, want %s", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestNumbers(t *testing.T) {
+	cases := []struct {
+		name    string
+		in      string
+		want    string
+		wantErr string
+	}{
+		{name: "zero", in: `{"a":0}`, want: `{"a":0}`},
+		{name: "negative", in: `{"a":-12}`, want: `{"a":-12}`},
+		{name: "largest allowed", in: `{"a":9007199254740991}`, want: `{"a":9007199254740991}`},
+		{name: "smallest allowed", in: `{"a":-9007199254740991}`, want: `{"a":-9007199254740991}`},
+
+		// The grammar forbids -0 in the output, so it is normalised rather
+		// than rejected: every JSON parser reads it as zero anyway.
+		{name: "negative zero is normalised", in: `{"a":-0}`, want: `{"a":0}`},
+
+		{name: "float", in: `{"a":1.0}`, wantErr: "has no floats"},
+		{name: "fraction", in: `{"a":0.5}`, wantErr: "has no floats"},
+		{name: "exponent", in: `{"a":1e3}`, wantErr: "has no floats"},
+		{name: "above the range", in: `{"a":9007199254740992}`, wantErr: "outside the allowed range"},
+		{name: "below the range", in: `{"a":-9007199254740992}`, wantErr: "outside the allowed range"},
+		{name: "far above the range", in: `{"a":18446744073709551616}`, wantErr: "not an integer we can represent"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := Encode([]byte(tc.in))
+			if tc.wantErr != "" {
+				if err == nil {
+					t.Fatalf("Encode(%s) = %s, want an error", tc.in, got)
+				}
+				if !strings.Contains(err.Error(), tc.wantErr) {
+					t.Errorf("error = %q, want it to mention %q", err, tc.wantErr)
+				}
+				if !errors.Is(err, ErrNotCanonical) {
+					t.Errorf("error does not wrap ErrNotCanonical: %v", err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("Encode: %v", err)
+			}
+			if string(got) != tc.want {
+				t.Errorf("Encode() = %s, want %s", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestRejects(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"invalid UTF-8", "{\"a\":\"\xff\xfe\"}", "not valid UTF-8"},
+		{"lone high surrogate", `{"a":"\uD800"}`, "high surrogate"},
+		{"lone low surrogate", `{"a":"\uDC00"}`, "low surrogate"},
+		{"high surrogate then a plain character", `{"a":"\uD800x"}`, "high surrogate"},
+		{"trailing data", `{"a":1} {"b":2}`, "trailing data"},
+		{"not JSON at all", `nope`, "not canonical JSON"},
+		{"empty input", ``, "not canonical JSON"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := Encode([]byte(tc.in))
+			if err == nil {
+				t.Fatalf("Encode(%q) = %s, want an error", tc.in, got)
+			}
+			if !strings.Contains(err.Error(), tc.want) {
+				t.Errorf("error = %q, want it to mention %q", err, tc.want)
+			}
+			if !errors.Is(err, ErrNotCanonical) {
+				t.Errorf("error does not wrap ErrNotCanonical: %v", err)
+			}
+		})
+	}
+}
+
+// TestEscapedBackslashIsNotAnEscape guards the backslash counting in
+// checkEscapes: the sequence below is a literal backslash followed by the
+// letters uD800, not a lone surrogate.
+func TestEscapedBackslashIsNotAnEscape(t *testing.T) {
+	got, err := Encode([]byte(`{"a":"\\uD800"}`))
+	if err != nil {
+		t.Fatalf("Encode: %v", err)
+	}
+	if want := `{"a":"\\uD800"}`; string(got) != want {
+		t.Errorf("Encode() = %s, want %s", got, want)
+	}
+}
+
+// TestNonObjectValues checks that the encoder handles every JSON type at
+// the top level, not just objects. Signing only ever sees objects, but
+// hashing a bare array should not be a panic.
+func TestNonObjectValues(t *testing.T) {
+	for _, tc := range []struct{ in, want string }{
+		{`[1, 2, 3]`, `[1,2,3]`},
+		{`[]`, `[]`},
+		{` "x" `, `"x"`},
+		{`true`, `true`},
+		{`null`, `null`},
+		{`7`, `7`},
+	} {
+		got, err := Encode([]byte(tc.in))
+		if err != nil {
+			t.Fatalf("Encode(%s): %v", tc.in, err)
+		}
+		if string(got) != tc.want {
+			t.Errorf("Encode(%s) = %s, want %s", tc.in, got, tc.want)
+		}
+	}
+}
+
+func TestMarshal(t *testing.T) {
+	// A Go value with its fields in the wrong order, and a string the
+	// standard library would HTML-escape.
+	v := struct {
+		Two  string `json:"two"`
+		One  int    `json:"one"`
+		Body string `json:"body"`
+	}{Two: "Two", One: 1, Body: "a & b"}
+
+	got, err := Marshal(v)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if want := `{"body":"a & b","one":1,"two":"Two"}`; string(got) != want {
+		t.Errorf("Marshal() = %s, want %s", got, want)
+	}
+}
+
+func TestValid(t *testing.T) {
+	if err := Valid([]byte(`{"a":1}`)); err != nil {
+		t.Errorf("Valid: %v", err)
+	}
+	if err := Valid([]byte(`{"a":1.5}`)); err == nil {
+		t.Error("Valid accepted a float")
+	}
+}
+
+// FuzzEncode checks the two properties that matter for anything arriving
+// over the network: the encoder never panics, and whatever it accepts it
+// accepts again unchanged. The second half is what guarantees two servers
+// agree on the bytes they hash.
+func FuzzEncode(f *testing.F) {
+	for _, seed := range []string{
+		`{}`, `{"a":1}`, `{"a":"日本語"}`, `[1,2,3]`, `{"a":{"b":[null,true]}}`,
+		`{"a":"\u0000"}`, `{"a":-0}`, `{"a":"😀"}`, `0`, `""`,
+	} {
+		f.Add([]byte(seed))
+	}
+	f.Fuzz(func(t *testing.T, in []byte) {
+		once, err := Encode(in)
+		if err != nil {
+			return
+		}
+		twice, err := Encode(once)
+		if err != nil {
+			t.Fatalf("re-encoding canonical JSON failed: %v\ninput: %q\ncanonical: %s", err, in, once)
+		}
+		if string(once) != string(twice) {
+			t.Fatalf("not idempotent:\ninput:  %q\nonce:   %s\ntwice:  %s", in, once, twice)
+		}
+	})
+}