From 7bb1e888d7d912bff50952c8664b7f67c6a0acbc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Dec 2024 17:18:50 -0700 Subject: [PATCH 1/3] Experiment with JWT-based worker redirection 84bcb7 --- api/_responses/redirect.go | 86 +++++++++++++++++++++++++++++++++++++- api/custom/byid.go | 46 ++++++++++++++++++++ api/r0/download.go | 2 +- api/r0/thumbnail.go | 2 +- api/routes.go | 2 + go.mod | 1 + go.sum | 2 + 7 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 api/custom/byid.go diff --git a/api/_responses/redirect.go b/api/_responses/redirect.go index 1e8c66c4..763f0edf 100644 --- a/api/_responses/redirect.go +++ b/api/_responses/redirect.go @@ -1,9 +1,91 @@ package _responses +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "net/url" + "os" + "time" + + "github.com/go-jose/go-jose/v4" + "github.com/go-jose/go-jose/v4/jwt" + "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/common/rcontext" +) + +// TODO: Persist and store key (or use some other source of information) +var jwtKey, keyErr = rsa.GenerateKey(rand.Reader, 2048) +var jwtSig, jwtErr = jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: []byte("0102030405060708090A0B0C0D0E0F10")}, (&jose.SignerOptions{}).WithType("JWT")) +var jweEnc, jweErr = jose.NewEncrypter(jose.A128GCM, jose.Recipient{Algorithm: jose.RSA_OAEP, Key: &jwtKey.PublicKey}, nil) + +func init() { + // We don't expect these to happen, so just panic + if keyErr != nil { + panic(keyErr) + } + if jwtErr != nil { + panic(jwtErr) + } + if jweErr != nil { + panic(jweErr) + } + + f1, _ := os.Create("./gdpr-data/jwe.rsa") + f2, _ := os.Create("./gdpr-data/jwe.rsa.pub") + defer f1.Close() + defer f2.Close() + keyPem := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(jwtKey), + }) + pubPem := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PUBLIC KEY", + Bytes: x509.MarshalPKCS1PublicKey(&jwtKey.PublicKey), + }) + f1.Write(keyPem) + f2.Write(pubPem) +} + type RedirectResponse struct { ToUrl string } -func Redirect(url string) *RedirectResponse { - return &RedirectResponse{ToUrl: url} +func Redirect(ctx rcontext.RequestContext, toUrl string, auth _apimeta.AuthContext) *RedirectResponse { + if auth.IsAuthenticated() { + // Figure out who we're authenticating as, and add that as JWT claims + claims := jwt.Claims{ + Issuer: ctx.Request.Host, + } + moreClaims := struct { + Admin bool `json:"adm,omitempty"` + AccessToken string `json:"tok,omitempty"` + }{} + if auth.Server.ServerName != "" { + claims.Subject = auth.Server.ServerName + + // The server won't necessarily re-auth itself with the redirect, so we just put an expiration on it instead + claims.Expiry = jwt.NewNumericDate(time.Now().Add(2 * time.Minute)) + } else { + claims.Subject = auth.User.UserId + moreClaims.Admin = auth.User.IsShared + moreClaims.AccessToken = auth.User.AccessToken + } + raw, err := jwt.Encrypted(jweEnc).Claims(claims).Claims(moreClaims).Serialize() + if err != nil { + panic(err) // should never happen if we've done things correctly + } + + // Now add the query string + parsedUrl, err := url.Parse(toUrl) + if err != nil { + panic(err) // it wouldn't have worked anyways + } + qs := parsedUrl.Query() + qs.Set("org.matrix.media_auth", raw) + parsedUrl.RawQuery = qs.Encode() + toUrl = parsedUrl.String() + } + return &RedirectResponse{ToUrl: toUrl} } diff --git a/api/custom/byid.go b/api/custom/byid.go new file mode 100644 index 00000000..8b64f25f --- /dev/null +++ b/api/custom/byid.go @@ -0,0 +1,46 @@ +package custom + +import ( + "net/http" + + "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/common/rcontext" + "github.com/t2bot/matrix-media-repo/database" + "github.com/t2bot/matrix-media-repo/datastores" + "github.com/t2bot/matrix-media-repo/pipelines/_steps/download" +) + +func GetMediaById(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { + if !user.IsShared { + return _responses.AuthFailed() + } + + // TODO: This is beyond dangerous and needs proper filtering + + db := database.GetInstance().Media.Prepare(rctx) + ds, err := datastores.Pick(rctx, datastores.LocalMediaKind) + if err != nil { + panic(err) + } + objectId := _routers.GetParam("objectId", r) + medias, err := db.GetByLocation(ds.Id, objectId) + if err != nil { + panic(err) + } + + media := medias[0] + stream, err := download.OpenStream(rctx, media.Locatable) + if err != nil { + panic(err) + } + + return &_responses.DownloadResponse{ + ContentType: media.ContentType, + Filename: media.UploadName, + SizeBytes: media.SizeBytes, + Data: stream, + TargetDisposition: "infer", + } +} diff --git a/api/r0/download.go b/api/r0/download.go index a4908025..1ddf9460 100644 --- a/api/r0/download.go +++ b/api/r0/download.go @@ -112,7 +112,7 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, auth _apimeta. } else if errors.Is(err, common.ErrMediaNotYetUploaded) { return _responses.NotYetUploaded() } else if errors.As(err, &redirect) { - return _responses.Redirect(redirect.RedirectUrl) + return _responses.Redirect(rctx, redirect.RedirectUrl, auth) } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go index 322b3ea8..e343ae87 100644 --- a/api/r0/thumbnail.go +++ b/api/r0/thumbnail.go @@ -184,7 +184,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, auth _apimeta } } } else if errors.As(err, &redirect) { - return _responses.Redirect(redirect.RedirectUrl) + return _responses.Redirect(rctx, redirect.RedirectUrl, auth) } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) diff --git a/api/routes.go b/api/routes.go index c5fd4cdb..affcd8ef 100644 --- a/api/routes.go +++ b/api/routes.go @@ -19,6 +19,7 @@ import ( const PrefixMedia = "/_matrix/media" const PrefixClient = "/_matrix/client" const PrefixFederation = "/_matrix/federation" +const PrefixMMR = "/_mmr" func buildRoutes() http.Handler { counter := &_routers.RequestCounter{} @@ -61,6 +62,7 @@ func buildRoutes() http.Handler { purgeOneRoute := makeRoute(_routers.RequireAccessToken(custom.PurgeIndividualRecord, false), "purge_individual_media", counter) register([]string{"DELETE"}, PrefixMedia, "download/:server/:mediaId", mxUnstable, router, purgeOneRoute) register([]string{"GET"}, PrefixMedia, "usage", msc4034, router, makeRoute(_routers.RequireAccessToken(unstable.PublicUsage, false), "usage", counter)) + register([]string{"GET"}, PrefixMMR, "byid/:objectId", mxNoVersion, router, makeRoute(_routers.RequireRepoAdmin(custom.GetMediaById), "byid", counter)) // Custom and top-level features router.Handler("GET", fmt.Sprintf("%s/version", PrefixMedia), makeRoute(_routers.OptionalAccessToken(custom.GetVersion), "get_version", counter)) diff --git a/go.mod b/go.mod index b228f3e4..cb5a3e0b 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( require ( github.com/didip/tollbooth/v7 v7.0.2 github.com/docker/go-connections v0.5.0 + github.com/go-jose/go-jose/v4 v4.0.4 github.com/go-redsync/redsync/v4 v4.13.0 github.com/h2non/filetype v1.1.3 github.com/julienschmidt/httprouter v1.3.0 diff --git a/go.sum b/go.sum index c15be357..3ab8394c 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,8 @@ github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8b github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= +github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= From 2cf5a65f44b0f7312d39212d78e0d1ef8a379542 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Dec 2024 17:19:16 -0700 Subject: [PATCH 2/3] Switch to an HMAC instead --- api/_responses/redirect.go | 105 ++++++++++++++----------------------- go.mod | 1 - go.sum | 2 - 3 files changed, 39 insertions(+), 69 deletions(-) diff --git a/api/_responses/redirect.go b/api/_responses/redirect.go index 763f0edf..9b349350 100644 --- a/api/_responses/redirect.go +++ b/api/_responses/redirect.go @@ -1,91 +1,64 @@ package _responses import ( - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" "net/url" - "os" + "strconv" "time" - "github.com/go-jose/go-jose/v4" - "github.com/go-jose/go-jose/v4/jwt" "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" ) -// TODO: Persist and store key (or use some other source of information) -var jwtKey, keyErr = rsa.GenerateKey(rand.Reader, 2048) -var jwtSig, jwtErr = jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: []byte("0102030405060708090A0B0C0D0E0F10")}, (&jose.SignerOptions{}).WithType("JWT")) -var jweEnc, jweErr = jose.NewEncrypter(jose.A128GCM, jose.Recipient{Algorithm: jose.RSA_OAEP, Key: &jwtKey.PublicKey}, nil) - -func init() { - // We don't expect these to happen, so just panic - if keyErr != nil { - panic(keyErr) - } - if jwtErr != nil { - panic(jwtErr) - } - if jweErr != nil { - panic(jweErr) - } - - f1, _ := os.Create("./gdpr-data/jwe.rsa") - f2, _ := os.Create("./gdpr-data/jwe.rsa.pub") - defer f1.Close() - defer f2.Close() - keyPem := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(jwtKey), - }) - pubPem := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PUBLIC KEY", - Bytes: x509.MarshalPKCS1PublicKey(&jwtKey.PublicKey), - }) - f1.Write(keyPem) - f2.Write(pubPem) -} - type RedirectResponse struct { ToUrl string } func Redirect(ctx rcontext.RequestContext, toUrl string, auth _apimeta.AuthContext) *RedirectResponse { if auth.IsAuthenticated() { - // Figure out who we're authenticating as, and add that as JWT claims - claims := jwt.Claims{ - Issuer: ctx.Request.Host, - } - moreClaims := struct { - Admin bool `json:"adm,omitempty"` - AccessToken string `json:"tok,omitempty"` - }{} + // Figure out who is authenticated here, as that affects the expiration time + var expirationTime time.Time if auth.Server.ServerName != "" { - claims.Subject = auth.Server.ServerName - - // The server won't necessarily re-auth itself with the redirect, so we just put an expiration on it instead - claims.Expiry = jwt.NewNumericDate(time.Now().Add(2 * time.Minute)) + expirationTime = time.Now().Add(time.Minute) } else { - claims.Subject = auth.User.UserId - moreClaims.Admin = auth.User.IsShared - moreClaims.AccessToken = auth.User.AccessToken - } - raw, err := jwt.Encrypted(jweEnc).Claims(claims).Claims(moreClaims).Serialize() - if err != nil { - panic(err) // should never happen if we've done things correctly + expirationTime = time.Now().Add(time.Minute * 5) } - // Now add the query string - parsedUrl, err := url.Parse(toUrl) + // Append the expiration time to the URL + toUrl = appendQueryParam(toUrl, "matrix_exp", strconv.FormatInt(expirationTime.UnixMilli(), 10)) + + // Prepare our HMAC message contents as a JSON object + hmacInput := make(map[string]string) + hmacInput["url"] = toUrl + if auth.User.UserId != "" { + hmacInput["access_token"] = auth.User.AccessToken + } + hmacMessage, err := json.Marshal(hmacInput) if err != nil { - panic(err) // it wouldn't have worked anyways + panic(err) // "should never happen" } - qs := parsedUrl.Query() - qs.Set("org.matrix.media_auth", raw) - parsedUrl.RawQuery = qs.Encode() - toUrl = parsedUrl.String() + + // Actually do the HMAC + mac := hmac.New(sha256.New, []byte("THIS_IS_A_SECRET_KEY")) // TODO: @@ Actual secret key + mac.Write(hmacMessage) + verifyHmac := mac.Sum(nil) + + // Append the HMAC to the URL + toUrl = appendQueryParam(toUrl, "matrix_verify", hex.EncodeToString(verifyHmac)) } return &RedirectResponse{ToUrl: toUrl} } + +func appendQueryParam(toUrl string, key string, val string) string { + parsedUrl, err := url.Parse(toUrl) + if err != nil { + panic(err) // it wouldn't have worked anyways + } + qs := parsedUrl.Query() + qs.Set(key, val) + parsedUrl.RawQuery = qs.Encode() + return parsedUrl.String() +} diff --git a/go.mod b/go.mod index cb5a3e0b..b228f3e4 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,6 @@ require ( require ( github.com/didip/tollbooth/v7 v7.0.2 github.com/docker/go-connections v0.5.0 - github.com/go-jose/go-jose/v4 v4.0.4 github.com/go-redsync/redsync/v4 v4.13.0 github.com/h2non/filetype v1.1.3 github.com/julienschmidt/httprouter v1.3.0 diff --git a/go.sum b/go.sum index 3ab8394c..c15be357 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,6 @@ github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8b github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= -github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= From afa8e96294bbbcd56727d83ccb02b148f33fcc81 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Dec 2024 18:03:37 -0700 Subject: [PATCH 3/3] Fix HMAC to avoid canonicalization issues --- api/_responses/redirect.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/api/_responses/redirect.go b/api/_responses/redirect.go index 9b349350..13a7303d 100644 --- a/api/_responses/redirect.go +++ b/api/_responses/redirect.go @@ -4,7 +4,6 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - "encoding/json" "net/url" "strconv" "time" @@ -31,19 +30,14 @@ func Redirect(ctx rcontext.RequestContext, toUrl string, auth _apimeta.AuthConte toUrl = appendQueryParam(toUrl, "matrix_exp", strconv.FormatInt(expirationTime.UnixMilli(), 10)) // Prepare our HMAC message contents as a JSON object - hmacInput := make(map[string]string) - hmacInput["url"] = toUrl + hmacMessage := toUrl + "||" if auth.User.UserId != "" { - hmacInput["access_token"] = auth.User.AccessToken - } - hmacMessage, err := json.Marshal(hmacInput) - if err != nil { - panic(err) // "should never happen" + hmacMessage += auth.User.AccessToken } // Actually do the HMAC mac := hmac.New(sha256.New, []byte("THIS_IS_A_SECRET_KEY")) // TODO: @@ Actual secret key - mac.Write(hmacMessage) + mac.Write([]byte(hmacMessage)) verifyHmac := mac.Sum(nil) // Append the HMAC to the URL