Support MSC3916 (without MSC3911)

.github/workflows/main.yml

@@ -44,5 +44,5 @@ jobs:
       - name: "Run: compile assets"
         run: "$PWD/bin/compile_assets"
       - name: "Run: tests"
-        run: "go test -c -v ./test && ./test.test '-test.v'" # cheat and work around working directory issues
+        run: "go test -c -v ./test && ./test.test '-test.v' -test.parallel 1" # cheat and work around working directory issues
         timeout-minutes: 30

CHANGELOG.md

@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * S3 datastores can now specify a `prefixLength` to improve S3 performance on some providers. See `config.sample.yaml` for details.
 * Add `multipartUploads` flag for running MMR against unsupported S3 providers. See `config.sample.yaml` for details. 
 * A new "leaky bucket" rate limit algorithm has been applied to downloads. See `rateLimit.buckets` in the config for details.
+* Add support for [MSC3916: Authentication for media](https://github.com/matrix-org/matrix-spec-proposals/pull/3916). 
+  * To enable full support, use `signingKeyPath` in your config. See sample config for details. 
 
 ### Changed
 

api/_apimeta/auth.go

@@ -16,6 +16,10 @@ type UserInfo struct {
 	IsShared    bool
 }
 
+type ServerInfo struct {
+	ServerName string
+}
+
 func GetRequestUserAdminStatus(r *http.Request, rctx rcontext.RequestContext, user UserInfo) (bool, bool) {
 	isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
 	isLocalAdmin, err := matrix.IsUserAdmin(rctx, r.Host, user.AccessToken, r.RemoteAddr)

api/_auth_cache/auth_cache.go

@@ -12,7 +12,7 @@ import (
 	"github.com/t2bot/matrix-media-repo/matrix"
 )
 
-var tokenCache = cache.New(0*time.Second, 30*time.Second)
+var tokenCache = cache.New(cache.NoExpiration, 30*time.Second)
 var rwLock = &sync.RWMutex{}
 var regexCache = make(map[string]*regexp.Regexp)
 

api/_routers/97-require-server-auth.go

@@ -0,0 +1,45 @@
+package _routers
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/t2bot/matrix-media-repo/api/_apimeta"
+	"github.com/t2bot/matrix-media-repo/api/_responses"
+	"github.com/t2bot/matrix-media-repo/common"
+	"github.com/t2bot/matrix-media-repo/common/rcontext"
+	"github.com/t2bot/matrix-media-repo/matrix"
+)
+
+type GeneratorWithServerFn = func(r *http.Request, ctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{}
+
+func RequireServerAuth(generator GeneratorWithServerFn) GeneratorFn {
+	return func(r *http.Request, ctx rcontext.RequestContext) interface{} {
+		serverName, err := matrix.ValidateXMatrixAuth(r, true)
+		if err != nil {
+			ctx.Log.Debug("Error with X-Matrix auth: ", err)
+			if errors.Is(err, matrix.ErrNoXMatrixAuth) {
+				return &_responses.ErrorResponse{
+					Code:         common.ErrCodeUnauthorized,
+					Message:      "no auth provided (required)",
+					InternalCode: common.ErrCodeMissingToken,
+				}
+			}
+			if errors.Is(err, matrix.ErrWrongDestination) {
+				return &_responses.ErrorResponse{
+					Code:         common.ErrCodeUnauthorized,
+					Message:      "no auth provided for this destination (required)",
+					InternalCode: common.ErrCodeBadRequest,
+				}
+			}
+			return &_responses.ErrorResponse{
+				Code:         common.ErrCodeForbidden,
+				Message:      "invalid auth provided (required)",
+				InternalCode: common.ErrCodeBadRequest,
+			}
+		}
+		return generator(r, ctx, _apimeta.ServerInfo{
+			ServerName: serverName,
+		})
+	}
+}

api/_routers/98-use-rcontext.go

@@ -101,20 +101,24 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 beforeParseDownload:
 	log.Infof("Replying with result: %T %+v", res, res)
 	if downloadRes, isDownload := res.(*_responses.DownloadResponse); isDownload {
-		ranges, err := http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes)
-		if errors.Is(err, http_range.ErrInvalid) {
-			proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
-			res = _responses.BadRequest("invalid range header")
-			goto beforeParseDownload // reprocess `res`
-		} else if errors.Is(err, http_range.ErrNoOverlap) {
-			proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
-			res = _responses.BadRequest("out of range")
-			goto beforeParseDownload // reprocess `res`
-		}
-		if len(ranges) > 1 {
-			proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
-			res = _responses.BadRequest("only 1 range is supported")
-			goto beforeParseDownload // reprocess `res`
+		var ranges []http_range.Range
+		var err error
+		if downloadRes.SizeBytes > 0 {
+			ranges, err = http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes)
+			if errors.Is(err, http_range.ErrInvalid) {
+				proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
+				res = _responses.BadRequest("invalid range header")
+				goto beforeParseDownload // reprocess `res`
+			} else if errors.Is(err, http_range.ErrNoOverlap) {
+				proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
+				res = _responses.BadRequest("out of range")
+				goto beforeParseDownload // reprocess `res`
+			}
+			if len(ranges) > 1 {
+				proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
+				res = _responses.BadRequest("only 1 range is supported")
+				goto beforeParseDownload // reprocess `res`
+			}
 		}
 
 		contentType = downloadRes.ContentType

api/custom/federation.go

@@ -33,7 +33,10 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim
 	}
 
 	versionUrl := url + "/_matrix/federation/v1/version"
-	versionResponse, err := matrix.FederatedGet(versionUrl, hostname, rctx)
+	versionResponse, err := matrix.FederatedGet(rctx, versionUrl, hostname, matrix.NoSigningKey)
+	if versionResponse != nil {
+		defer versionResponse.Body.Close()
+	}
 	if err != nil {
 		rctx.Log.Error(err)
 		sentry.CaptureException(err)

api/r0/versions.go

@@ -0,0 +1,36 @@
+package r0
+
+import (
+	"net/http"
+	"slices"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/t2bot/matrix-media-repo/api/_apimeta"
+	"github.com/t2bot/matrix-media-repo/api/_responses"
+	"github.com/t2bot/matrix-media-repo/matrix"
+
+	"github.com/t2bot/matrix-media-repo/common/rcontext"
+)
+
+func ClientVersions(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+	versions, err := matrix.ClientVersions(rctx, r.Host, user.UserId, user.AccessToken, r.RemoteAddr)
+	if err != nil {
+		rctx.Log.Error(err)
+		sentry.CaptureException(err)
+		return _responses.InternalServerError("unable to get versions")
+	}
+
+	// This is where we'd add our feature/version support as needed
+	if versions.Versions == nil {
+		versions.Versions = make([]string, 1)
+	}
+
+	// We add v1.11 by force, even though we can't reliably say the rest of the server implements it. This
+	// is because server admins which point `/versions` at us are effectively opting in to whatever features
+	// we need to advertise support for. In our case, it's at least Authenticated Media (MSC3916).
+	if !slices.Contains(versions.Versions, "v1.11") {
+		versions.Versions = append(versions.Versions, "v1.11")
+	}
+
+	return versions
+}

api/routes.go

@@ -18,6 +18,7 @@ import (
 
 const PrefixMedia = "/_matrix/media"
 const PrefixClient = "/_matrix/client"
+const PrefixFederation = "/_matrix/federation"
 
 func buildRoutes() http.Handler {
 	counter := &_routers.RequestCounter{}
@@ -36,12 +37,23 @@ func buildRoutes() http.Handler {
 	register([]string{"GET", "HEAD"}, PrefixMedia, "download/:server/:mediaId/:filename", mxSpecV3Transition, router, downloadRoute)
 	register([]string{"GET", "HEAD"}, PrefixMedia, "download/:server/:mediaId", mxSpecV3Transition, router, downloadRoute)
 	register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter))
-	register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter))
+	previewUrlRoute := makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter)
+	register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, previewUrlRoute)
 	register([]string{"GET"}, PrefixMedia, "identicon/*seed", mxR0, router, makeRoute(_routers.OptionalAccessToken(r0.Identicon), "identicon", counter))
-	register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", counter))
+	configRoute := makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", counter)
+	register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, configRoute)
 	register([]string{"POST"}, PrefixClient, "logout", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.Logout), "logout", counter))
 	register([]string{"POST"}, PrefixClient, "logout/all", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.LogoutAll), "logout_all", counter))
 	register([]string{"POST"}, PrefixMedia, "create", mxV1, router, makeRoute(_routers.RequireAccessToken(v1.CreateMedia), "create", counter))
+	register([]string{"GET"}, PrefixClient, "versions", mxNoVersion, router, makeRoute(_routers.OptionalAccessToken(r0.ClientVersions), "client_versions", counter))
+	register([]string{"GET"}, PrefixClient, "media/preview_url", mxV1, router, previewUrlRoute)
+	register([]string{"GET"}, PrefixClient, "media/config", mxV1, router, configRoute)
+	authedDownloadRoute := makeRoute(_routers.RequireAccessToken(v1.ClientDownloadMedia), "download", counter)
+	register([]string{"GET"}, PrefixClient, "media/download/:server/:mediaId/:filename", mxV1, router, authedDownloadRoute)
+	register([]string{"GET"}, PrefixClient, "media/download/:server/:mediaId", mxV1, router, authedDownloadRoute)
+	register([]string{"GET"}, PrefixClient, "media/thumbnail/:server/:mediaId", mxV1, router, makeRoute(_routers.RequireAccessToken(v1.ClientThumbnailMedia), "thumbnail", counter))
+	register([]string{"GET"}, PrefixFederation, "media/download/:mediaId", mxV1, router, makeRoute(_routers.RequireServerAuth(v1.FederationDownloadMedia), "download", counter))
+	register([]string{"GET"}, PrefixFederation, "media/thumbnail/:mediaId", mxV1, router, makeRoute(_routers.RequireServerAuth(v1.FederationThumbnailMedia), "thumbnail", counter))
 
 	// Custom features
 	register([]string{"GET"}, PrefixMedia, "local_copy/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.LocalCopy), "local_copy", counter))
@@ -134,12 +146,16 @@ var (
 	mxR0                 matrixVersions = []string{"r0"}
 	mxV1                 matrixVersions = []string{"v1"}
 	mxV3                 matrixVersions = []string{"v3"}
+	mxNoVersion          matrixVersions = []string{""}
 )
 
 func register(methods []string, prefix string, postfix string, versions matrixVersions, router *httprouter.Router, handler http.Handler) {
 	for _, method := range methods {
 		for _, version := range versions {
 			path := fmt.Sprintf("%s/%s/%s", prefix, version, postfix)
+			if version == "" {
+				path = fmt.Sprintf("%s/%s", prefix, postfix)
+			}
 			router.Handler(method, path, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
 				defer func() {
 					// hopefully the body was already closed, but maybe it wasn't

api/v1/download.go

@@ -0,0 +1,54 @@
+package v1
+
+import (
+	"bytes"
+	"net/http"
+
+	"github.com/t2bot/matrix-media-repo/api/_apimeta"
+	"github.com/t2bot/matrix-media-repo/api/_responses"
+	"github.com/t2bot/matrix-media-repo/api/_routers"
+	"github.com/t2bot/matrix-media-repo/api/r0"
+	"github.com/t2bot/matrix-media-repo/common/rcontext"
+	"github.com/t2bot/matrix-media-repo/util/readers"
+)
+
+func ClientDownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+	r.URL.Query().Set("allow_remote", "true")
+	r.URL.Query().Set("allow_redirect", "true")
+	return r0.DownloadMedia(r, rctx, user)
+}
+
+func FederationDownloadMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} {
+	query := r.URL.Query()
+	query.Set("allow_remote", "false")
+	query.Set("allow_redirect", "true") // we override how redirects work in the response
+	r.URL.RawQuery = query.Encode()
+	r = _routers.ForceSetParam("server", r.Host, r)
+
+	res := r0.DownloadMedia(r, rctx, _apimeta.UserInfo{})
+	if dl, ok := res.(*_responses.DownloadResponse); ok {
+		return &_responses.DownloadResponse{
+			ContentType: "multipart/mixed",
+			Filename:    "",
+			SizeBytes:   0,
+			Data: readers.NewMultipartReader(
+				&readers.MultipartPart{ContentType: "application/json", Reader: readers.MakeCloser(bytes.NewReader([]byte("{}")))},
+				&readers.MultipartPart{ContentType: dl.ContentType, FileName: dl.Filename, Reader: dl.Data},
+			),
+			TargetDisposition: "attachment",
+		}
+	} else if rd, ok := res.(*_responses.RedirectResponse); ok {
+		return &_responses.DownloadResponse{
+			ContentType: "multipart/mixed",
+			Filename:    "",
+			SizeBytes:   0,
+			Data: readers.NewMultipartReader(
+				&readers.MultipartPart{ContentType: "application/json", Reader: readers.MakeCloser(bytes.NewReader([]byte("{}")))},
+				&readers.MultipartPart{Location: rd.ToUrl},
+			),
+			TargetDisposition: "attachment",
+		}
+	} else {
+		return res
+	}
+}

api/v1/thumbnail.go

@@ -0,0 +1,54 @@
+package v1
+
+import (
+	"bytes"
+	"net/http"
+
+	"github.com/t2bot/matrix-media-repo/api/_apimeta"
+	"github.com/t2bot/matrix-media-repo/api/_responses"
+	"github.com/t2bot/matrix-media-repo/api/_routers"
+	"github.com/t2bot/matrix-media-repo/api/r0"
+	"github.com/t2bot/matrix-media-repo/common/rcontext"
+	"github.com/t2bot/matrix-media-repo/util/readers"
+)
+
+func ClientThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+	r.URL.Query().Set("allow_remote", "true")
+	r.URL.Query().Set("allow_redirect", "true")
+	return r0.ThumbnailMedia(r, rctx, user)
+}
+
+func FederationThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} {
+	query := r.URL.Query()
+	query.Set("allow_remote", "false")
+	query.Set("allow_redirect", "true") // we override how redirects work in the response
+	r.URL.RawQuery = query.Encode()
+	r = _routers.ForceSetParam("server", r.Host, r)
+
+	res := r0.ThumbnailMedia(r, rctx, _apimeta.UserInfo{})
+	if dl, ok := res.(*_responses.DownloadResponse); ok {
+		return &_responses.DownloadResponse{
+			ContentType: "multipart/mixed",
+			Filename:    "",
+			SizeBytes:   0,
+			Data: readers.NewMultipartReader(
+				&readers.MultipartPart{ContentType: "application/json", Reader: readers.MakeCloser(bytes.NewReader([]byte("{}")))},
+				&readers.MultipartPart{ContentType: dl.ContentType, FileName: dl.Filename, Reader: dl.Data},
+			),
+			TargetDisposition: "attachment",
+		}
+	} else if rd, ok := res.(*_responses.RedirectResponse); ok {
+		return &_responses.DownloadResponse{
+			ContentType: "multipart/mixed",
+			Filename:    "",
+			SizeBytes:   0,
+			Data: readers.NewMultipartReader(
+				&readers.MultipartPart{ContentType: "application/json", Reader: readers.MakeCloser(bytes.NewReader([]byte("{}")))},
+				&readers.MultipartPart{Location: rd.ToUrl},
+			),
+			TargetDisposition: "attachment",
+		}
+	} else {
+		return res
+	}
+}

cmd/utilities/generate_signing_key/main.go

@@ -1,13 +1,8 @@
 package main
 
 import (
-	"crypto/ed25519"
-	"crypto/rand"
 	"flag"
-	"fmt"
 	"os"
-	"sort"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/t2bot/matrix-media-repo/cmd/utilities/_common"
@@ -27,16 +22,7 @@ func main() {
 	if *inputFile != "" {
 		key, err = decodeKey(*inputFile)
 	} else {
-		keyVersion := makeKeyVersion()
-
-		var priv ed25519.PrivateKey
-		_, priv, err = ed25519.GenerateKey(nil)
-		priv = priv[len(priv)-32:]
-
-		key = &homeserver_interop.SigningKey{
-			PrivateKey: priv,
-			KeyVersion: keyVersion,
-		}
+		key, err = homeserver_interop.GenerateSigningKey()
 	}
 	if err != nil {
 		logrus.Fatal(err)
@@ -47,28 +33,6 @@ func main() {
 	_common.EncodeSigningKeys([]*homeserver_interop.SigningKey{key}, *outputFormat, *outputFile)
 }
 
-func makeKeyVersion() string {
-	buf := make([]byte, 2)
-	chars := strings.Split("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", "")
-	for i := 0; i < len(chars); i++ {
-		sort.Slice(chars, func(i int, j int) bool {
-			c, err := rand.Read(buf)
-
-			// "should never happen" clauses
-			if err != nil {
-				panic(err)
-			}
-			if c != len(buf) || c != 2 {
-				panic(fmt.Sprintf("crypto rand read %d bytes, expected %d", c, len(buf)))
-			}
-
-			return buf[0] < buf[1]
-		})
-	}
-
-	return strings.Join(chars[:6], "")
-}
-
 func decodeKey(fileName string) (*homeserver_interop.SigningKey, error) {
 	f, err := os.Open(fileName)
 	if err != nil {