From b483f8e05a5ed0613e6f3e4514b08cb8ad81c63b Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Thu, 10 Oct 2024 21:59:20 -0500 Subject: [PATCH] New media strategy Return ni:/// URIs (or, optionally, /.well-known/ni/ paths) for media. Provide a helper to serve the .well-known paths from a service worker. No more option to get media URIs from the persistence layer, instead each layer will provide a fit-for-platform mechanism to map ni URIs to data, file paths, whatever make sense in context. For web that's Response object or service worker interception. Avatars are now two-part: the possible-null ni URI and the never-null placeholder. Display the placeholder while loading the real avatar data, or if it is null or loading it fails for any reason. --- npm/index.ts | 3 ++ snikket/Chat.hx | 43 +++++++++---------- snikket/ChatMessage.hx | 11 ++--- snikket/Client.hx | 20 ++++----- snikket/Config.hx | 12 ++++++ snikket/Hash.hx | 70 +++++++++++++++++++++++++++++++ snikket/Participant.hx | 24 +++++++++++ snikket/Persistence.hx | 2 +- snikket/persistence/Custom.hx | 5 +++ snikket/persistence/Dummy.hx | 5 +++ snikket/persistence/Sqlite.hx | 5 +++ snikket/persistence/browser.js | 46 ++++++++++++-------- snikket/queries/HttpUploadSlot.hx | 4 +- 13 files changed, 191 insertions(+), 59 deletions(-) create mode 100644 snikket/Config.hx create mode 100644 snikket/Hash.hx create mode 100644 snikket/Participant.hx diff --git a/npm/index.ts b/npm/index.ts index a66f7d6..8600c1e 100644 --- a/npm/index.ts +++ b/npm/index.ts @@ -10,10 +10,13 @@ export import Chat = snikket.Chat; export import ChatAttachment = snikket.ChatAttachment; export import ChatMessage = snikket.ChatMessage; export import Client = snikket.Client; +export import Config = snikket.Config; export import DirectChat = snikket.DirectChat; +export import Hash = snikket.Hash; export import Identicon = snikket.Identicon; export import Identity = snikket.Identity; export import Notification = snikket.Notification; +export import Participant = snikket.Participant; export import SerializedChat = snikket.SerializedChat; export import jingle = snikket.jingle; diff --git a/snikket/Chat.hx b/snikket/Chat.hx index 58cc060..17cf8ac 100644 --- a/snikket/Chat.hx +++ b/snikket/Chat.hx @@ -1,5 +1,6 @@ package snikket; +import haxe.io.Bytes; import haxe.io.BytesData; import snikket.Chat; import snikket.ChatMessage; @@ -169,9 +170,8 @@ abstract class Chat { Get the details for one participant in this Chat @param participantId the ID of the participant to look up - @param callback takes two arguments, the display name and the photo URI **/ - abstract public function getParticipantDetails(participantId:String, callback:(String, String)->Void):Void; + abstract public function getParticipantDetails(participantId: String):Participant; /** Correct an already-send message by replacing it with a new one @@ -293,22 +293,18 @@ abstract class Chat { } /** - Get an image to represent this Chat + Get the URI image to represent this Chat, or null + **/ + public function getPhoto(): Null { + if (avatarSha1 == null || Bytes.ofData(avatarSha1).length < 1) return null; + return new Hash("sha-1", avatarSha1).toUri(); + } - @param callback takes one argument, the URI to the image + /** + Get the URI to a placeholder image to represent this Chat **/ - public function getPhoto(callback:(String)->Void) { - if (avatarSha1 != null) { - persistence.getMediaUri("sha-1", avatarSha1, (uri) -> { - if (uri != null) { - callback(uri); - } else { - callback(Color.defaultPhoto(chatId, getDisplayName().charAt(0))); - } - }); - } else { - callback(Color.defaultPhoto(chatId, getDisplayName().charAt(0))); - } + public function getPlaceholder(): String { + return Color.defaultPhoto(chatId, getDisplayName().charAt(0).toUpperCase()); } /** @@ -587,9 +583,9 @@ class DirectChat extends Chat { } @HaxeCBridge.noemit // on superclass as abstract - public function getParticipantDetails(participantId:String, callback:(String, String)->Void) { + public function getParticipantDetails(participantId:String): Participant { final chat = client.getDirectChat(participantId); - chat.getPhoto((photoUri) -> callback(chat.getDisplayName(), photoUri)); + return new Participant(chat.getDisplayName(), chat.getPhoto(), chat.getPlaceholder()); } @HaxeCBridge.noemit // on superclass as abstract @@ -993,15 +989,14 @@ class Channel extends Chat { } @HaxeCBridge.noemit // on superclass as abstract - public function getParticipantDetails(participantId:String, callback:(String, String)->Void) { + public function getParticipantDetails(participantId:String): Participant { if (participantId == getFullJid().asString()) { - client.getDirectChat(client.accountId(), false).getPhoto((photoUri) -> { - callback(client.displayName(), photoUri); - }); + final chat = client.getDirectChat(client.accountId(), false); + return new Participant(client.displayName(), chat.getPhoto(), chat.getPlaceholder()); } else { final nick = JID.parse(participantId).resource; - final photoUri = Color.defaultPhoto(participantId, nick == null ? " " : nick.charAt(0)); - callback(nick, photoUri); + final placeholderUri = Color.defaultPhoto(participantId, nick == null ? " " : nick.charAt(0)); + return new Participant(client.displayName(), null, placeholderUri); } } diff --git a/snikket/ChatMessage.hx b/snikket/ChatMessage.hx index a4015c1..be2691d 100644 --- a/snikket/ChatMessage.hx +++ b/snikket/ChatMessage.hx @@ -5,11 +5,13 @@ import haxe.io.Bytes; import haxe.io.BytesData; import haxe.Exception; using Lambda; +using StringTools; #if cpp import HaxeCBridge; #end +import snikket.Hash; import snikket.JID; import snikket.Identicon; import snikket.StringUtil; @@ -28,8 +30,7 @@ class ChatAttachment { public final mime: String; public final size: Null; public final uris: Array; - @HaxeCBridge.noemit - public final hashes: Array<{algo:String, hash:BytesData}>; + public final hashes: Array; #if cpp @:allow(snikket) @@ -37,7 +38,7 @@ class ChatAttachment { #else public #end - function new(name: Null, mime: String, size: Null, uris: Array, hashes: Array<{algo:String, hash:BytesData}>) { + function new(name: Null, mime: String, size: Null, uris: Array, hashes: Array) { this.name = name; this.mime = mime; this.size = size; @@ -160,7 +161,7 @@ class ChatMessage { var size = sims.findText("{urn:xmpp:jingle:apps:file-transfer:5}/size#"); if (size == null) size = sims.findText("{urn:xmpp:jingle:apps:file-transfer:3}/size#"); final hashes = ((sims.getChild("file", "urn:xmpp:jingle:apps:file-transfer:5") ?? sims.getChild("file", "urn:xmpp:jingle:apps:file-transfer:3")) - ?.allTags("hash", "urn:xmpp:hashes:2") ?? []).map((hash) -> { algo: hash.attr.get("algo") ?? "", hash: Base64.decode(hash.getText()).getData() }); + ?.allTags("hash", "urn:xmpp:hashes:2") ?? []).map((hash) -> new Hash(hash.attr.get("algo") ?? "", Base64.decode(hash.getText()).getData())); final sources = sims.getChild("sources"); final uris = (sources?.allTags("reference", "urn:xmpp:reference:0") ?? []).map((ref) -> ref.attr.get("uri") ?? "").filter((uri) -> uri != ""); if (uris.length > 0) attachments.push(new ChatAttachment(name, mime, size == null ? null : Std.parseInt(size), uris, hashes)); @@ -331,7 +332,7 @@ class ChatMessage { stanza.textTag("media-type", attachment.mime); if (attachment.size != null) stanza.textTag("size", Std.string(attachment.size)); for (hash in attachment.hashes) { - stanza.textTag("hash", Base64.encode(Bytes.ofData(hash.hash)), { xmlns: "urn:xmpp:hashes:2", algo: hash.algo }); + stanza.textTag("hash", Base64.encode(Bytes.ofData(hash.hash)), { xmlns: "urn:xmpp:hashes:2", algo: hash.algorithm }); } stanza.up(); diff --git a/snikket/Client.hx b/snikket/Client.hx index 41be87e..bc1362c 100644 --- a/snikket/Client.hx +++ b/snikket/Client.hx @@ -257,8 +257,10 @@ class Client extends EventEmitter { final chat = this.getDirectChat(JID.parse(pubsubEvent.getFrom()).asBare().asString(), false); chat.setAvatarSha1(avatarSha1); persistence.storeChat(accountId(), chat); - persistence.getMediaUri("sha-1", avatarSha1, (uri) -> { - if (uri == null) { + persistence.hasMedia("sha-1", avatarSha1, (has) -> { + if (has) { + this.trigger("chats/update", [chat]); + } else { final pubsubGet = new PubsubGet(pubsubEvent.getFrom(), "urn:xmpp:avatar:data", avatarSha1Hex); pubsubGet.onFinished(() -> { final item = pubsubGet.getResult()[0]; @@ -270,8 +272,6 @@ class Client extends EventEmitter { }); }); sendQuery(pubsubGet); - } else { - this.trigger("chats/update", [chat]); } }); } @@ -434,8 +434,10 @@ class Client extends EventEmitter { final avatarSha1 = Bytes.ofHex(avatarSha1Hex).getData(); chat.setAvatarSha1(avatarSha1); persistence.storeChat(accountId(), chat); - persistence.getMediaUri("sha-1", avatarSha1, (uri) -> { - if (uri == null) { + persistence.hasMedia("sha-1", avatarSha1, (has) -> { + if (has) { + if (chat.livePresence()) this.trigger("chats/update", [chat]); + } else { final vcardGet = new VcardTempGet(from); vcardGet.onFinished(() -> { final vcard = vcardGet.getResult(); @@ -445,8 +447,6 @@ class Client extends EventEmitter { }); }); sendQuery(vcardGet); - } else { - if (chat.livePresence()) this.trigger("chats/update", [chat]); } }); } @@ -648,7 +648,7 @@ class Client extends EventEmitter { return tink.streams.Stream.Handled.Resume; }).handle((o) -> switch o { case Depleted: - prepareAttachmentFor(source, services, [{ algo: "sha-256", hash: sha256.digest().getData() }], callback); + prepareAttachmentFor(source, services, [new Hash("sha-256", sha256.digest().getData())], callback); default: trace("Error computing attachment hash", o); callback(null); @@ -656,7 +656,7 @@ class Client extends EventEmitter { }); } - private function prepareAttachmentFor(source: js.html.File, services: Array<{ serviceId: String }>, hashes: Array<{algo: String, hash: BytesData}>, callback: (Null)->Void) { + private function prepareAttachmentFor(source: js.html.File, services: Array<{ serviceId: String }>, hashes: Array, callback: (Null)->Void) { if (services.length < 1) { callback(null); return; diff --git a/snikket/Config.hx b/snikket/Config.hx new file mode 100644 index 0000000..69c5fca --- /dev/null +++ b/snikket/Config.hx @@ -0,0 +1,12 @@ +package snikket; + +@:expose +class Config { + /** + Produce /.well-known/ni/ paths instead of ni:/// URIs + for referencing media by hash. + + This can be useful eg for intercepting with a Service Worker. + **/ + public static var relativeHashUri = false; +} diff --git a/snikket/Hash.hx b/snikket/Hash.hx new file mode 100644 index 0000000..5cd4537 --- /dev/null +++ b/snikket/Hash.hx @@ -0,0 +1,70 @@ +package snikket; + +import haxe.crypto.Base64; +import haxe.io.Bytes; +import haxe.io.BytesData; +using StringTools; + +import snikket.Config; + +#if cpp +import HaxeCBridge; +#end + +@:expose +@:nullSafety(Strict) +#if cpp +@:build(HaxeCBridge.expose()) +@:build(HaxeSwiftBridge.expose()) +#end +class Hash { + public final algorithm: String; + @:allow(snikket) + private final hash: BytesData; + + @:allow(snikket) + private function new(algorithm: String, hash: BytesData) { + this.algorithm = algorithm; + this.hash = hash; + } + + public static function fromHex(algorithm: String, hash: String) { + return new Hash(algorithm, Bytes.ofHex(hash).getData()); + } + + public static function fromUri(uri: String): Null { + if (uri.startsWith("cid:") && uri.endsWith("@bob.xmpp.org") && uri.contains("+")) { + final parts = uri.substr(4).split("@")[0].split("+"); + final algo = parts[0] == "sha1" ? "sha-1" : parts[0]; + return Hash.fromHex(algo, parts[1]); + } if (uri.startsWith("ni:///") && uri.contains(";")) { + final parts = uri.substring(6).split(';'); + return new Hash(parts[0], Base64.urlDecode(parts[1]).getData()); + } else if (uri.startsWith("/.well-known/ni/")) { + final parts = uri.substring(16).split('/'); + return new Hash(parts[0], Base64.urlDecode(parts[1]).getData()); + } + + return null; + } + + public function toUri() { + if (Config.relativeHashUri) { + return "/.well-known/ni/" + algorithm.urlEncode() + "/" + toBase64Url(); + } else { + return "ni:///" + algorithm.urlEncode() + ";" + toBase64Url(); + } + } + + public function toHex() { + return Bytes.ofData(hash).toHex(); + } + + public function toBase64() { + return Base64.encode(Bytes.ofData(hash)); + } + + public function toBase64Url() { + return Base64.urlEncode(Bytes.ofData(hash)); + } +} diff --git a/snikket/Participant.hx b/snikket/Participant.hx new file mode 100644 index 0000000..7ee3d5b --- /dev/null +++ b/snikket/Participant.hx @@ -0,0 +1,24 @@ +package snikket; + +#if cpp +import HaxeCBridge; +#end + +@:expose +@:nullSafety(Strict) +#if cpp +@:build(HaxeCBridge.expose()) +@:build(HaxeSwiftBridge.expose()) +#end +class Participant { + public final displayName: String; + public final photoUri: Null; + public final placeholderUri: String; + + @:allow(snikket) + private function new(displayName: String, photoUri: Null, placeholderUri: String) { + this.displayName = displayName; + this.photoUri = photoUri; + this.placeholderUri = placeholderUri; + } +} diff --git a/snikket/Persistence.hx b/snikket/Persistence.hx index 03eaddb..7340f3c 100644 --- a/snikket/Persistence.hx +++ b/snikket/Persistence.hx @@ -20,7 +20,7 @@ interface Persistence { public function getMessagesBefore(accountId: String, chatId: String, beforeId: Null, beforeTime: Null, callback: (messages:Array)->Void):Void; public function getMessagesAfter(accountId: String, chatId: String, afterId: Null, afterTime: Null, callback: (messages:Array)->Void):Void; public function getMessagesAround(accountId: String, chatId: String, aroundId: Null, aroundTime: Null, callback: (messages:Array)->Void):Void; - public function getMediaUri(hashAlgorithm:String, hash:BytesData, callback: (uri:Null)->Void):Void; + public function hasMedia(hashAlgorithm:String, hash:BytesData, callback: (has:Bool)->Void):Void; public function storeMedia(mime:String, bytes:BytesData, callback: ()->Void):Void; public function storeCaps(caps:Caps):Void; public function getCaps(ver:String, callback: (Null)->Void):Void; diff --git a/snikket/persistence/Custom.hx b/snikket/persistence/Custom.hx index df489d3..e181b95 100644 --- a/snikket/persistence/Custom.hx +++ b/snikket/persistence/Custom.hx @@ -93,6 +93,11 @@ class Custom implements Persistence { backing.getMediaUri(hashAlgorithm, hash, callback); } + @HaxeCBridge.noemit + public function hasMedia(hashAlgorithm:String, hash:BytesData, callback: (Bool)->Void) { + backing.hasMedia(hashAlgorithm, hash, callback); + } + @HaxeCBridge.noemit public function storeMedia(mime:String, bd:BytesData, callback: ()->Void) { backing.storeMedia(mime, bd, callback); diff --git a/snikket/persistence/Dummy.hx b/snikket/persistence/Dummy.hx index 28e5641..c343703 100644 --- a/snikket/persistence/Dummy.hx +++ b/snikket/persistence/Dummy.hx @@ -76,6 +76,11 @@ class Dummy implements Persistence { callback(null); } + @HaxeCBridge.noemit + public function hasMedia(hashAlgorithm:String, hash:BytesData, callback: (Bool)->Void) { + callback(false); + } + @HaxeCBridge.noemit public function storeMedia(mime:String, bd:BytesData, callback: ()->Void) { callback(); diff --git a/snikket/persistence/Sqlite.hx b/snikket/persistence/Sqlite.hx index 1312424..cf88f5a 100644 --- a/snikket/persistence/Sqlite.hx +++ b/snikket/persistence/Sqlite.hx @@ -315,6 +315,11 @@ class Sqlite implements Persistence { } } + @HaxeCBridge.noemit + public function hasMedia(hashAlgorithm:String, hash:BytesData, callback: (Bool)->Void) { + getMediaUri(hashAlgorithm, hash, (uri) -> callback(uri != null)); + } + @HaxeCBridge.noemit public function storeMedia(mime:String, bd:BytesData, callback: ()->Void) { final bytes = Bytes.ofData(bd); diff --git a/snikket/persistence/browser.js b/snikket/persistence/browser.js index e33a03d..d3dbbfa 100644 --- a/snikket/persistence/browser.js +++ b/snikket/persistence/browser.js @@ -478,27 +478,39 @@ const browser = (dbname, tokenize, stemmer) => { } }, - getMediaUri: function(hashAlgorithm, hash, callback) { - (async function() { - var niUrl; - if (hashAlgorithm == "sha-256") { - niUrl = mkNiUrl(hashAlgorithm, hash); - } else { - const tx = db.transaction(["keyvaluepairs"], "readonly"); - const store = tx.objectStore("keyvaluepairs"); - niUrl = await promisifyRequest(store.get(mkNiUrl(hashAlgorithm, hash))); - if (!niUrl) { - return null; - } + routeHashPathSW: function() { + addEventListener("fetch", (event) => { + const url = new URL(event.request.url); + if (url.pathname.startsWith("/.well-known/ni/")) { + event.respondWith(this.getMediaResponse(url.pathname).then((r) => { + if (r) return r; + return Response.error(); + })); } + }); + }, - const response = await cache.match(niUrl); - if (response) { - // NOTE: the application needs to call URL.revokeObjectURL on this when done - return URL.createObjectURL(await response.blob()); + getMediaResponse: async function(uri) { + uri = uri.replace(/^ni:\/\/\//, "/.well-known/ni/").replace(/;/, "/"); + var niUrl; + if (uri.split("/")[3] === "sha-256") { + niUrl = uri; + } else { + const tx = db.transaction(["keyvaluepairs"], "readonly"); + const store = tx.objectStore("keyvaluepairs"); + niUrl = await promisifyRequest(store.get(uri)); + if (!niUrl) { + return null; } + } - return null; + return await cache.match(niUrl); + }, + + hasMedia: function(hashAlgorithm, hash, callback) { + (async () => { + const response = await this.getMediaResponse(hashAlgorithm, hash); + return !!response; })().then(callback); }, diff --git a/snikket/queries/HttpUploadSlot.hx b/snikket/queries/HttpUploadSlot.hx index d755efa..4cc8e7e 100644 --- a/snikket/queries/HttpUploadSlot.hx +++ b/snikket/queries/HttpUploadSlot.hx @@ -19,7 +19,7 @@ class HttpUploadSlot extends GenericQuery { public var responseStanza(default, null):Stanza; private var result: { put: String, putHeaders: Array, get: String }; - public function new(to: String, filename: String, size: Int, mime: String, hashes: Array<{algo:String,hash:BytesData}>) { + public function new(to: String, filename: String, size: Int, mime: String, hashes: Array) { /* Build basic query */ queryId = ID.short(); queryStanza = new Stanza( @@ -27,7 +27,7 @@ class HttpUploadSlot extends GenericQuery { { to: to, type: "get", id: queryId } ).tag("request", { xmlns: xmlns, filename: filename, size: Std.string(size), "content-type": mime }); for (hash in hashes) { - queryStanza.textTag("hash", Base64.encode(Bytes.ofData(hash.hash)), { xmlns: "urn:xmpp:hashes:2", algo: hash.algo }); + queryStanza.textTag("hash", Base64.encode(Bytes.ofData(hash.hash)), { xmlns: "urn:xmpp:hashes:2", algo: hash.algorithm }); } queryStanza.up(); }