CI / Test (ubuntu-latest) (push) Canceled after 0s
CI / Test (windows-latest) (push) Canceled after 0s
CI / Storage tests on PostgreSQL (push) Canceled after 0s
CI / Complement (federation off) (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
CI / Smoke test the binary (push) Canceled after 0s
Implements PLAN.md Phase 6: the media repository. Clients upload files, reserve an mxc:// URI and upload to it later, download and ask for thumbnails, all through the authenticated media endpoints of Matrix 1.11; other servers download this server's media through the federation media endpoints, and media uploaded elsewhere is fetched over federation on first use and kept. internal/mediaapi is the component. Content goes into a BlobStore named by its SHA-256, so the same file uploaded twice is stored once and the hash doubles as the ETag; the metadata that maps an mxc:// URI onto a blob is in media_repository and media_thumbnails. The local-filesystem store writes to a temporary file and renames it into place once hashed, so nothing ever refers to half a file. A storage driver can bring its own store through storage.BlobProvider, which is where PocketBase's filesystem abstraction (local or S3) plugs in with Phase 10. Reserved uploads belong to the user who made them, take one upload, lapse after a day, and are capped per user; a download of one waits for its content, woken in memory the moment it arrives, for as long as timeout_ms allows. Thumbnails come from a fixed set of configured sizes, so a thousand requested sizes cannot mean a thousand resizes. They are made on first request and kept, never enlarge, honour a JPEG's EXIF orientation, and point at the original when it already fits. The pixel count is checked from the header before anything is decoded, at most four resizes run at once, and concurrent requests for the same thumbnail share one. Remote media is fetched through the signed client: the multipart/mixed answer of Matrix 1.11, with a fallback to the legacy download endpoint for servers that answer M_UNRECOGNIZED, and Location redirects followed by a dialer that refuses loopback, private and link-local addresses. Concurrent requests share one fetch, detached from the request that started it. With federation off nothing is fetched, and what was cached before is still served. Downloads carry a sandboxing CSP, nosniff, inline Content-Disposition only for the types the spec lists as safe, and private immutable caching. The unauthenticated legacy download endpoints are off by default behind media.legacy_unauthenticated_downloads. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
150 lines
4.7 KiB
Go
150 lines
4.7 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
)
|
|
|
|
// Errors of the media store.
|
|
var (
|
|
// ErrNoSuchMedia means no media, or no thumbnail, is stored under the
|
|
// given name.
|
|
ErrNoSuchMedia = errors.New("no such media")
|
|
|
|
// ErrMediaExists is CreateMedia's answer for a media ID that is taken.
|
|
ErrMediaExists = errors.New("media already exists")
|
|
|
|
// ErrMediaUploaded is CompleteUpload's answer for media whose content
|
|
// has already been uploaded: content, once stored, never changes.
|
|
ErrMediaUploaded = errors.New("media content already uploaded")
|
|
)
|
|
|
|
// Media is the metadata of one piece of content: uploaded here, or fetched
|
|
// from the server it was uploaded to and kept.
|
|
type Media struct {
|
|
// Origin is the server name in the mxc:// URI: this server for an
|
|
// upload, another for cached remote media.
|
|
Origin string
|
|
MediaID string
|
|
|
|
// ContentType is the Content-Type the content was uploaded with,
|
|
// parameters included.
|
|
ContentType string
|
|
|
|
// UploadName is the file name the uploader gave, or "".
|
|
UploadName string
|
|
|
|
// Size is the content's length in bytes.
|
|
Size int64
|
|
|
|
// SHA256 names the content in the blob store: its SHA-256, in
|
|
// lower-case hex. Empty while the media is pending.
|
|
SHA256 string
|
|
|
|
// UserID is the uploader; "" for remote media.
|
|
UserID string
|
|
|
|
CreatedAtMS int64
|
|
|
|
// Pending is set for an ID handed out by /create whose content has
|
|
// not been uploaded yet. Pending media expires at ExpiresAtMS.
|
|
Pending bool
|
|
ExpiresAtMS int64
|
|
}
|
|
|
|
// MediaContent is what CompleteUpload fills in on pending media.
|
|
type MediaContent struct {
|
|
ContentType string
|
|
UploadName string
|
|
Size int64
|
|
SHA256 string
|
|
}
|
|
|
|
// Thumbnail is one generated (or, for remote media, fetched) thumbnail.
|
|
type Thumbnail struct {
|
|
Origin string
|
|
MediaID string
|
|
|
|
// Width, Height and Method are the size the thumbnail was made for —
|
|
// one of the configured sizes — not the size of the image itself,
|
|
// which for "scale" keeps the original's aspect ratio.
|
|
Width int
|
|
Height int
|
|
Method string
|
|
|
|
ContentType string
|
|
Size int64
|
|
SHA256 string
|
|
CreatedAtMS int64
|
|
}
|
|
|
|
// MediaStore is the mediaapi store: the metadata of local and cached
|
|
// remote media and of their thumbnails. The content itself is in a
|
|
// BlobStore.
|
|
type MediaStore interface {
|
|
// CreateMedia records media: an upload, a pending upload or a remote
|
|
// fetch. A taken (origin, media ID) is ErrMediaExists.
|
|
CreateMedia(ctx context.Context, m Media) error
|
|
|
|
// Media reads one media record, pending or not.
|
|
Media(ctx context.Context, origin, mediaID string) (Media, error)
|
|
|
|
// CompleteUpload stores the content of pending media and clears its
|
|
// pending flag. It is ErrNoSuchMedia for an unknown ID and
|
|
// ErrMediaUploaded for media that is not pending.
|
|
CompleteUpload(ctx context.Context, origin, mediaID string, content MediaContent) error
|
|
|
|
// PendingUploads counts a user's pending media that has not expired
|
|
// by nowMS.
|
|
PendingUploads(ctx context.Context, userID string, nowMS int64) (int, error)
|
|
|
|
// DeleteExpiredUploads removes pending media that expired before
|
|
// nowMS, reporting how many.
|
|
DeleteExpiredUploads(ctx context.Context, nowMS int64) (int, error)
|
|
|
|
// Thumbnail reads the thumbnail made for one size.
|
|
Thumbnail(ctx context.Context, origin, mediaID string, width, height int, method string) (Thumbnail, error)
|
|
|
|
// StoreThumbnail records a thumbnail, replacing one stored for the
|
|
// same size.
|
|
StoreThumbnail(ctx context.Context, t Thumbnail) error
|
|
}
|
|
|
|
// ErrNoSuchBlob means the blob store holds nothing under a hash.
|
|
var ErrNoSuchBlob = errors.New("no such blob")
|
|
|
|
// ErrBlobTooLarge means content passed the limit it was stored with.
|
|
var ErrBlobTooLarge = errors.New("content is larger than allowed")
|
|
|
|
// BlobInfo describes stored content.
|
|
type BlobInfo struct {
|
|
// SHA256 is the lower-case hex SHA-256 of the content, and its name in
|
|
// the store.
|
|
SHA256 string
|
|
Size int64
|
|
}
|
|
|
|
// BlobStore keeps media content, addressed by its SHA-256. The same bytes
|
|
// uploaded twice are stored once.
|
|
type BlobStore interface {
|
|
// Put stores what r yields, up to limit bytes; more is
|
|
// ErrBlobTooLarge and stores nothing.
|
|
Put(ctx context.Context, r io.Reader, limit int64) (BlobInfo, error)
|
|
|
|
// Open reads stored content.
|
|
Open(ctx context.Context, sha256 string) (io.ReadSeekCloser, error)
|
|
|
|
// Delete removes stored content. Deleting what is not there is not an
|
|
// error.
|
|
Delete(ctx context.Context, sha256 string) error
|
|
}
|
|
|
|
// BlobProvider is implemented by a storage driver that brings its own
|
|
// BlobStore. PocketBase will (PLAN.md section 4: its filesystem
|
|
// abstraction puts media on local disk or S3); a driver that does not is
|
|
// given the local filesystem store.
|
|
type BlobProvider interface {
|
|
Blobs() BlobStore
|
|
}
|