Phase 6: media
CI / Test (ubuntu-latest) (push) Canceled after 0s
CI / Test (windows-latest) (push) Canceled after 0s
CI / Storage tests on PostgreSQL (push) Canceled after 0s
CI / Complement (federation off) (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
CI / Smoke test the binary (push) Canceled after 0s
CI / Test (ubuntu-latest) (push) Canceled after 0s
CI / Test (windows-latest) (push) Canceled after 0s
CI / Storage tests on PostgreSQL (push) Canceled after 0s
CI / Complement (federation off) (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
CI / Smoke test the binary (push) Canceled after 0s
Implements PLAN.md Phase 6: the media repository. Clients upload files, reserve an mxc:// URI and upload to it later, download and ask for thumbnails, all through the authenticated media endpoints of Matrix 1.11; other servers download this server's media through the federation media endpoints, and media uploaded elsewhere is fetched over federation on first use and kept. internal/mediaapi is the component. Content goes into a BlobStore named by its SHA-256, so the same file uploaded twice is stored once and the hash doubles as the ETag; the metadata that maps an mxc:// URI onto a blob is in media_repository and media_thumbnails. The local-filesystem store writes to a temporary file and renames it into place once hashed, so nothing ever refers to half a file. A storage driver can bring its own store through storage.BlobProvider, which is where PocketBase's filesystem abstraction (local or S3) plugs in with Phase 10. Reserved uploads belong to the user who made them, take one upload, lapse after a day, and are capped per user; a download of one waits for its content, woken in memory the moment it arrives, for as long as timeout_ms allows. Thumbnails come from a fixed set of configured sizes, so a thousand requested sizes cannot mean a thousand resizes. They are made on first request and kept, never enlarge, honour a JPEG's EXIF orientation, and point at the original when it already fits. The pixel count is checked from the header before anything is decoded, at most four resizes run at once, and concurrent requests for the same thumbnail share one. Remote media is fetched through the signed client: the multipart/mixed answer of Matrix 1.11, with a fallback to the legacy download endpoint for servers that answer M_UNRECOGNIZED, and Location redirects followed by a dialer that refuses loopback, private and link-local addresses. Concurrent requests share one fetch, detached from the request that started it. With federation off nothing is fetched, and what was cached before is still served. Downloads carry a sandboxing CSP, nosniff, inline Content-Disposition only for the types the spec lists as safe, and private immutable caching. The unauthenticated legacy download endpoints are off by default behind media.legacy_unauthenticated_downloads. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6aaf554336
commit
92836f8dcd
40 files changed
+4729
-63
No files matched your search
@@ -95,6 +95,13 @@ type ClientOptions struct {
|
||||
|
||||
// Policy filters destinations. Nil allows any.
|
||||
Policy *Policy
|
||||
|
||||
// AllowPrivateMediaRedirects lets a media download follow another
|
||||
// server's redirect to a loopback, private or link-local address.
|
||||
// Only tests, whose servers are all on loopback, set it: anywhere
|
||||
// else it would let another server point this one at its own
|
||||
// internal network and read the answer back through a client.
|
||||
AllowPrivateMediaRedirects bool
|
||||
}
|
||||
|
||||
// Client sends signed Server-Server API requests.
|
||||
@@ -106,6 +113,11 @@ type Client struct {
|
||||
enabled func() bool
|
||||
policy *Policy
|
||||
userAgent string
|
||||
|
||||
// mediaHTTP follows the redirects a media download may answer with.
|
||||
// It is not the federation transport: the URL is an ordinary web
|
||||
// address, often a CDN, and nothing is signed.
|
||||
mediaHTTP *http.Client
|
||||
}
|
||||
|
||||
// dialTargetKey carries the resolved addresses of a request down to the
|
||||
@@ -166,6 +178,7 @@ func NewClient(opts ClientOptions) *Client {
|
||||
enabled: opts.Enabled,
|
||||
policy: opts.Policy,
|
||||
userAgent: buildinfo.UserAgent(),
|
||||
mediaHTTP: newMediaRedirectClient(tlsConfig, opts.AllowPrivateMediaRedirects),
|
||||
}
|
||||
if c.resolve == nil {
|
||||
// The well-known document is fetched with ordinary HTTPS and the
|
||||
@@ -210,12 +223,47 @@ type request struct {
|
||||
// do performs a request and decodes a JSON answer into out (which may be
|
||||
// nil).
|
||||
func (c *Client) do(ctx context.Context, req request, out any) error {
|
||||
if err := c.check(req.destination); err != nil {
|
||||
resp, cancel, err := c.send(ctx, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cancel()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
limit := req.maxBytes
|
||||
if limit == 0 {
|
||||
limit = defaultMaxResponse
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %s%s: read response: %w", req.method, req.destination, req.path, err)
|
||||
}
|
||||
if int64(len(body)) > limit {
|
||||
return fmt.Errorf("%s %s%s: response is larger than %d bytes", req.method, req.destination, req.path, limit)
|
||||
}
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return httpError(resp.StatusCode, body)
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
return fmt.Errorf("%s %s%s: decode response: %w", req.method, req.destination, req.path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// send performs a request and hands back the response with its body
|
||||
// unread, for the callers that stream it. The caller closes the body and
|
||||
// then calls cancel, which ends the request's deadline.
|
||||
func (c *Client) send(ctx context.Context, req request) (*http.Response, context.CancelFunc, error) {
|
||||
if err := c.check(req.destination); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
resolved, err := c.resolve(ctx, req.destination)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve %s: %w", req.destination, err)
|
||||
return nil, nil, fmt.Errorf("resolve %s: %w", req.destination, err)
|
||||
}
|
||||
|
||||
timeout := req.timeout
|
||||
@@ -223,13 +271,13 @@ func (c *Client) do(ctx context.Context, req request, out any) error {
|
||||
timeout = defaultRequestTimeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
ctx = context.WithValue(ctx, dialTargetKey{}, resolved.Addresses)
|
||||
|
||||
var content []byte
|
||||
if req.body != nil {
|
||||
if content, err = json.Marshal(req.body); err != nil {
|
||||
return fmt.Errorf("encode request: %w", err)
|
||||
cancel()
|
||||
return nil, nil, fmt.Errorf("encode request: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -239,7 +287,8 @@ func (c *Client) do(ctx context.Context, req request, out any) error {
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, req.method, target, bytes.NewReader(content))
|
||||
if err != nil {
|
||||
return fmt.Errorf("build request: %w", err)
|
||||
cancel()
|
||||
return nil, nil, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
if content == nil {
|
||||
httpReq.Body = http.NoBody
|
||||
@@ -256,47 +305,32 @@ func (c *Client) do(ctx context.Context, req request, out any) error {
|
||||
x, err := SignRequest(c.key, c.serverName, req.destination, req.method,
|
||||
httpReq.URL.RequestURI(), content)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sign request: %w", err)
|
||||
cancel()
|
||||
return nil, nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Authorization", x.String())
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(httpReq)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %s%s: %w", req.method, req.destination, req.path, err)
|
||||
cancel()
|
||||
return nil, nil, fmt.Errorf("%s %s%s: %w", req.method, req.destination, req.path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
return resp, cancel, nil
|
||||
}
|
||||
|
||||
limit := req.maxBytes
|
||||
if limit == 0 {
|
||||
limit = defaultMaxResponse
|
||||
// httpError turns a non-2xx answer into an *HTTPError, with the Matrix
|
||||
// error envelope if the body was one.
|
||||
func httpError(status int, body []byte) *HTTPError {
|
||||
httpErr := &HTTPError{Status: status}
|
||||
var envelope struct {
|
||||
Code string `json:"errcode"`
|
||||
Message string `json:"error"`
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %s%s: read response: %w", req.method, req.destination, req.path, err)
|
||||
if json.Unmarshal(body, &envelope) == nil {
|
||||
httpErr.Code, httpErr.Message = envelope.Code, envelope.Message
|
||||
}
|
||||
if int64(len(body)) > limit {
|
||||
return fmt.Errorf("%s %s%s: response is larger than %d bytes", req.method, req.destination, req.path, limit)
|
||||
}
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
httpErr := &HTTPError{Status: resp.StatusCode}
|
||||
var envelope struct {
|
||||
Code string `json:"errcode"`
|
||||
Message string `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(body, &envelope) == nil {
|
||||
httpErr.Code, httpErr.Message = envelope.Code, envelope.Message
|
||||
}
|
||||
return httpErr
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
return fmt.Errorf("%s %s%s: decode response: %w", req.method, req.destination, req.path, err)
|
||||
}
|
||||
return nil
|
||||
return httpErr
|
||||
}
|
||||
|
||||
// esc escapes one path segment. Room, event and user IDs carry sigils and
|
||||
|
||||
Reference in new issue
Block a user