From 5ef3301a64ed9748e3ffa651ae4eab4c3bb7b28e Mon Sep 17 00:00:00 2001 From: Matthieu Finiasz Date: Wed, 20 Mar 2024 13:42:55 +0100 Subject: [PATCH] v2.0.1 (build 234) --- CHANGELOG.md | 8 + .../io/olvid/engine/datatypes/Constants.java | 2 + .../containers/StringAndBoolean.java | 59 ++ .../java/io/olvid/engine/engine/Engine.java | 4 +- .../engine/types/RegisterApiKeyResult.java | 1 + .../engine/identity/IdentityManager.java | 1 + .../identity/databases/ContactDevice.java | 1 + ...MessagesAndListAttachmentsCoordinator.java | 2 + .../olvid/engine/networksend/SendManager.java | 3 +- .../SendAttachmentCoordinator.java | 5 + .../coordinators/SendMessageCoordinator.java | 233 +++++++- .../networksend/databases/OutboxMessage.java | 43 +- ...BatchUploadMessagesCompositeOperation.java | 139 +++++ .../BatchUploadMessagesOperation.java | 246 ++++++++ .../UploadMessageCompositeOperation.java | 4 - .../inspectionProfiles/Project_Default.xml | 1 + obv_messenger/app/build.gradle | 8 +- .../location/MapViewMapLibreFragment.java | 10 +- .../main/contacts/ContactListItem.kt | 54 +- .../messenger/openid/KeycloakManager.java | 40 +- .../ConfigurationScannedFragment.java | 1 + .../messenger/webrtc/WebrtcCallActivity.kt | 20 +- .../messenger/webrtc/WebrtcCallService.kt | 127 ++--- .../webrtc/WebrtcPeerConnectionHolder.kt | 40 +- .../messenger/webrtc/components/CallScreen.kt | 468 +++++++++------ .../webrtc/components/CustomEglRenderer.java | 533 ++++++++++++++++++ .../webrtc/components/PictureInPicture.kt | 14 +- .../components/ScaleAndOffsetControl.kt | 45 ++ .../webrtc/components/VideoRenderer.kt | 71 +-- .../components/VideoTextureViewRenderer.kt | 4 +- .../app/src/main/res/drawable/olvid_icon.xml | 21 + .../layout/activity_webrtc_incoming_call.xml | 45 +- .../app/src/main/res/values-fr/strings.xml | 11 +- .../app/src/main/res/values/strings.xml | 1 + obv_messenger/build.gradle | 4 +- 35 files changed, 1844 insertions(+), 425 deletions(-) create mode 100644 obv_engine/engine/src/main/java/io/olvid/engine/datatypes/containers/StringAndBoolean.java create mode 100644 obv_engine/engine/src/main/java/io/olvid/engine/networksend/operations/BatchUploadMessagesCompositeOperation.java create mode 100644 obv_engine/engine/src/main/java/io/olvid/engine/networksend/operations/BatchUploadMessagesOperation.java create mode 100644 obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/CustomEglRenderer.java create mode 100644 obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/ScaleAndOffsetControl.kt create mode 100644 obv_messenger/app/src/main/res/drawable/olvid_icon.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 9717ff10..d86390fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# Built 234 (2.0.1) +2024-03-20 + +- Several improvements to the video calls UI +- Fix a api key registration bug on first Keycloak binding +- Minor improvements to OpenStreetMaps integration +- Redesign of the message upload coordinator for improved performances when sending many messages + # Build 233 (2.0) 2024-03-14 diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/datatypes/Constants.java b/obv_engine/engine/src/main/java/io/olvid/engine/datatypes/Constants.java index cce70768..a0a989e3 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/datatypes/Constants.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/datatypes/Constants.java @@ -88,6 +88,7 @@ public abstract class Constants { public static final int DEFAULT_ATTACHMENT_CHUNK_LENGTH = 4*2048*1024; public static final int MAX_MESSAGE_EXTENDED_CONTENT_LENGTH = 50 * 1024; + public static final int MAX_UPLOAD_MESSAGE_BATCH_SIZE = 50; public static final UID BROADCAST_UID = new UID(new byte[]{(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff}); @@ -110,6 +111,7 @@ public abstract class Constants { // prefixes for various types of signature public static final int SIGNATURE_PADDING_LENGTH = 16; + public enum SignatureContext { SERVER_AUTHENTICATION, MUTUAL_SCAN, diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/datatypes/containers/StringAndBoolean.java b/obv_engine/engine/src/main/java/io/olvid/engine/datatypes/containers/StringAndBoolean.java new file mode 100644 index 00000000..e95dcfe4 --- /dev/null +++ b/obv_engine/engine/src/main/java/io/olvid/engine/datatypes/containers/StringAndBoolean.java @@ -0,0 +1,59 @@ +/* + * Olvid for Android + * Copyright © 2019-2024 Olvid SAS + * + * This file is part of Olvid for Android. + * + * Olvid is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * Olvid is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Olvid. If not, see . + */ + +package io.olvid.engine.datatypes.containers; + + +import java.nio.charset.StandardCharsets; + +import io.olvid.engine.crypto.Hash; +import io.olvid.engine.crypto.Suite; +import io.olvid.engine.datatypes.UID; + +public class StringAndBoolean { + public final String string; + public final boolean bool; + + public StringAndBoolean(String string, boolean bool) { + this.string = string; + this.bool = bool; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof StringAndBoolean)) { + return false; + } + StringAndBoolean other = (StringAndBoolean) o; + return string.equals(other.string) && bool == other.bool; + } + + @Override + public int hashCode() { + return string.hashCode() * 31 + Boolean.hashCode(bool); + } + + public static UID computeUniqueUid(String string, boolean bool) { + Hash sha256 = Suite.getHash(Hash.SHA256); + byte[] input = new byte[string.getBytes(StandardCharsets.UTF_8).length + 1]; + System.arraycopy(string.getBytes(StandardCharsets.UTF_8), 0, input, 0, input.length - 1); + input[input.length - 1] = bool ? (byte) 0x01 : (byte) 0x00; + return new UID(sha256.digest(input)); + } +} diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/engine/Engine.java b/obv_engine/engine/src/main/java/io/olvid/engine/engine/Engine.java index fc6a4732..7351fb85 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/engine/Engine.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/engine/Engine.java @@ -672,7 +672,7 @@ public RegisterApiKeyResult registerOwnedIdentityApiKeyOnServer(byte[] bytesOwne byte[] serverSessionToken = fetchManager.getServerAuthenticationToken(ownedIdentity); if (serverSessionToken == null) { fetchManager.createServerSession(ownedIdentity); - return RegisterApiKeyResult.FAILED; + return RegisterApiKeyResult.WAIT_FOR_SERVER_SESSION; } StandaloneServerQueryOperation standaloneServerQueryOperation = new StandaloneServerQueryOperation(new ServerQuery(null, ownedIdentity, new ServerQuery.RegisterApiKeyQuery(ownedIdentity, serverSessionToken, Logger.getUuidString(apiKey)))); @@ -693,7 +693,7 @@ public RegisterApiKeyResult registerOwnedIdentityApiKeyOnServer(byte[] bytesOwne } case StandaloneServerQueryOperation.RFC_INVALID_SERVER_SESSION: { recreateServerSession(bytesOwnedIdentity); - break; + return RegisterApiKeyResult.WAIT_FOR_SERVER_SESSION; } case StandaloneServerQueryOperation.RFC_UNSUPPORTED_SERVER_QUERY_TYPE: case StandaloneServerQueryOperation.RFC_NETWORK_ERROR: diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/engine/types/RegisterApiKeyResult.java b/obv_engine/engine/src/main/java/io/olvid/engine/engine/types/RegisterApiKeyResult.java index d9a7bfa6..fb0e67e9 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/engine/types/RegisterApiKeyResult.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/engine/types/RegisterApiKeyResult.java @@ -23,4 +23,5 @@ public enum RegisterApiKeyResult { SUCCESS, INVALID_KEY, FAILED, + WAIT_FOR_SERVER_SESSION, } diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/identity/IdentityManager.java b/obv_engine/engine/src/main/java/io/olvid/engine/identity/IdentityManager.java index 83a626cf..29a5f911 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/identity/IdentityManager.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/identity/IdentityManager.java @@ -197,6 +197,7 @@ public void initialisationComplete() { protocolStarterDelegate.startDeviceDiscoveryProtocolWithinTransaction(identityManagerSession.session, contactIdentity.getOwnedIdentity(), contactIdentity.getContactIdentity()); } } + identityManagerSession.session.commit(); } } catch (Exception e) { e.printStackTrace(); diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/identity/databases/ContactDevice.java b/obv_engine/engine/src/main/java/io/olvid/engine/identity/databases/ContactDevice.java index 7e6e5946..ad9bab6b 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/identity/databases/ContactDevice.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/identity/databases/ContactDevice.java @@ -52,6 +52,7 @@ public class ContactDevice implements ObvDatabase { static final String OWNED_IDENTITY = "owned_identity"; private byte[] serializedDeviceCapabilities; static final String SERIALIZED_DEVICE_CAPABILITIES = "serialized_device_capabilities"; + // TODO: add a last ping sent timestamp to limit useless contact discoveries public UID getUid() { return uid; diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/networkfetch/coordinators/DownloadMessagesAndListAttachmentsCoordinator.java b/obv_engine/engine/src/main/java/io/olvid/engine/networkfetch/coordinators/DownloadMessagesAndListAttachmentsCoordinator.java index f42f935a..c849d57b 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/networkfetch/coordinators/DownloadMessagesAndListAttachmentsCoordinator.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/networkfetch/coordinators/DownloadMessagesAndListAttachmentsCoordinator.java @@ -126,12 +126,14 @@ public void initialQueueing() { PendingDeleteFromServer.create(fetchManagerSession, inboxMessage.getOwnedIdentity(), inboxMessage.getUid()); } } + fetchManagerSession.session.commit(); // check all decrypted messages, with attachments, that are not yet marked as listed on the server InboxMessage[] messagesToMarkAsListedOnServer = InboxMessage.getMessagesThatCanBeMarkedAsListedOnServer(fetchManagerSession); for (InboxMessage inboxMessage : messagesToMarkAsListedOnServer) { fetchManagerSession.markAsListedOnServerListener.messageCanBeMarkedAsListedOnServer(inboxMessage.getOwnedIdentity(), inboxMessage.getUid()); } + } catch (SQLException e) { e.printStackTrace(); } diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/SendManager.java b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/SendManager.java index f0dec35b..4ed61839 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/SendManager.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/SendManager.java @@ -146,7 +146,8 @@ public void post(Session session, MessageToSend messageToSend) { messageToSend.getEncryptedContent(), messageToSend.getEncryptedExtendedContent(), messageToSend.isApplicationMessage(), - messageToSend.isVoipMessage() + messageToSend.isVoipMessage(), + messageToSend.getAttachments() != null && messageToSend.getAttachments().length != 0 ); if (messageToSend.getHeaders() != null) { diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/coordinators/SendAttachmentCoordinator.java b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/coordinators/SendAttachmentCoordinator.java index b5858c04..1ea9ddd8 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/coordinators/SendAttachmentCoordinator.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/coordinators/SendAttachmentCoordinator.java @@ -36,6 +36,7 @@ import io.olvid.engine.datatypes.PriorityOperation; import io.olvid.engine.datatypes.PriorityOperationQueue; import io.olvid.engine.datatypes.UID; +import io.olvid.engine.datatypes.containers.IdentityAndUid; import io.olvid.engine.datatypes.containers.IdentityAndUidAndNumber; import io.olvid.engine.datatypes.notifications.IdentityNotifications; import io.olvid.engine.datatypes.notifications.UploadNotifications; @@ -287,4 +288,8 @@ public long getInitialPriority() { return initialPriority; } } + + public interface MessageBatchProvider { + IdentityAndUid[] getBatchOFMessageUids(); + } } diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/coordinators/SendMessageCoordinator.java b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/coordinators/SendMessageCoordinator.java index aaf163d8..fe3e7b5d 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/coordinators/SendMessageCoordinator.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/coordinators/SendMessageCoordinator.java @@ -21,37 +21,52 @@ import java.sql.SQLException; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Queue; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import javax.net.ssl.SSLSocketFactory; import io.olvid.engine.Logger; +import io.olvid.engine.datatypes.Constants; import io.olvid.engine.datatypes.ExponentialBackoffRepeatingScheduler; import io.olvid.engine.datatypes.Identity; +import io.olvid.engine.datatypes.NoDuplicateOperationQueue; import io.olvid.engine.datatypes.Operation; import io.olvid.engine.datatypes.OperationQueue; import io.olvid.engine.datatypes.UID; import io.olvid.engine.datatypes.containers.IdentityAndUid; +import io.olvid.engine.datatypes.containers.StringAndBoolean; import io.olvid.engine.datatypes.notifications.IdentityNotifications; import io.olvid.engine.metamanager.NotificationListeningDelegate; import io.olvid.engine.networksend.databases.OutboxMessage; import io.olvid.engine.networksend.datatypes.SendManagerSession; import io.olvid.engine.networksend.datatypes.SendManagerSessionFactory; +import io.olvid.engine.networksend.operations.BatchUploadMessagesCompositeOperation; import io.olvid.engine.networksend.operations.UploadMessageCompositeOperation; -public class SendMessageCoordinator implements OutboxMessage.NewOutboxMessageListener, Operation.OnCancelCallback, Operation.OnFinishCallback { +public class SendMessageCoordinator implements OutboxMessage.NewOutboxMessageListener { + private final SendManagerSessionFactory sendManagerSessionFactory; private final SSLSocketFactory sslSocketFactory; + @SuppressWarnings("FieldCanBeLocal") private NotificationListeningDelegate notificationListeningDelegate; - private final OperationQueue sendMessageOperationQueue; + private final OperationQueue sendMessageWithAttachmentOperationQueue; private final ExponentialBackoffRepeatingScheduler scheduler; + private final HashMap> userContentMessageUidsByServer; + private final NoDuplicateOperationQueue batchSendUserContentMessageOperationQueue; + private final HashMap> protocolMessageUidsByServer; + private final NoDuplicateOperationQueue batchSendProtocolMessageOperationQueue; + private final ExponentialBackoffRepeatingScheduler batchScheduler; + private boolean initialQueueingPerformed = false; private final Object lock = new Object(); @@ -65,11 +80,23 @@ public SendMessageCoordinator(SendManagerSessionFactory sendManagerSessionFactor this.sendManagerSessionFactory = sendManagerSessionFactory; this.sslSocketFactory = sslSocketFactory; - sendMessageOperationQueue = new OperationQueue(true); - sendMessageOperationQueue.execute(1, "Engine-SendMessageCoordinator"); + sendMessageWithAttachmentOperationQueue = new OperationQueue(true); + sendMessageWithAttachmentOperationQueue.execute(1, "Engine-SendMessageCoordinator-WithAttachment"); scheduler = new ExponentialBackoffRepeatingScheduler<>(); + + userContentMessageUidsByServer = new HashMap<>(); + batchSendUserContentMessageOperationQueue = new NoDuplicateOperationQueue(); + batchSendUserContentMessageOperationQueue.execute(1, "Engine-SendMessageCoordinator-WithUserContent"); + + protocolMessageUidsByServer = new HashMap<>(); + batchSendProtocolMessageOperationQueue = new NoDuplicateOperationQueue(); + batchSendProtocolMessageOperationQueue.execute(1, "Engine-SendMessageCoordinator-Protocol"); + + batchScheduler = new ExponentialBackoffRepeatingScheduler<>(); + + awaitingIdentityReactivationOperations = new HashMap<>(); awaitingIdentityReactivationOperationsLock = new ReentrantLock(); @@ -89,7 +116,7 @@ public void initialQueueing() { try (SendManagerSession sendManagerSession = sendManagerSessionFactory.getSession()) { OutboxMessage[] outboxMessages = OutboxMessage.getAll(sendManagerSession); for (OutboxMessage outboxMessage: outboxMessages) { - queueNewSendMessageCompositeOperation(outboxMessage.getOwnedIdentity(), outboxMessage.getUid()); + queueNewSendMessageCompositeOperation(outboxMessage.getServer(), outboxMessage.getOwnedIdentity(), outboxMessage.getUid(), outboxMessage.getAttachments().length != 0, outboxMessage.isApplicationMessage()); } initialQueueingPerformed = true; } catch (SQLException e) { @@ -98,17 +125,84 @@ public void initialQueueing() { } } - private void queueNewSendMessageCompositeOperation(Identity ownedIdentity, UID messageUid) { - UploadMessageCompositeOperation op = new UploadMessageCompositeOperation(sendManagerSessionFactory, sslSocketFactory, ownedIdentity, messageUid, this, this); - sendMessageOperationQueue.queue(op); + private void queueNewSendMessageCompositeOperation(String server, Identity ownedIdentity, UID messageUid, boolean hasAttachment, boolean hasUserContent) { + if (hasAttachment || server == null) { + UploadMessageCompositeOperation op = new UploadMessageCompositeOperation(sendManagerSessionFactory, sslSocketFactory, ownedIdentity, messageUid, this::onFinishCallbackWithAttachment, this::onCancelCallbackWithAttachment); + sendMessageWithAttachmentOperationQueue.queue(op); + } else if (hasUserContent) { + if (ownedIdentity != null && messageUid != null) { + synchronized (userContentMessageUidsByServer) { + Queue queue = userContentMessageUidsByServer.get(server); + if (queue == null) { + queue = new ArrayDeque<>(); + userContentMessageUidsByServer.put(server, queue); + } + queue.add(new IdentityAndUid(ownedIdentity, messageUid)); + } + } + BatchUploadMessagesCompositeOperation op = new BatchUploadMessagesCompositeOperation(sendManagerSessionFactory, sslSocketFactory, server, true, () -> { + List messageIdentitiesAndUids = new ArrayList<>(); + synchronized (userContentMessageUidsByServer) { + Queue queue = userContentMessageUidsByServer.get(server); + if (queue != null && !queue.isEmpty()) { + do { + messageIdentitiesAndUids.add(queue.remove()); + if (messageIdentitiesAndUids.size() == Constants.MAX_UPLOAD_MESSAGE_BATCH_SIZE) { + break; + } + } while (!queue.isEmpty()); + } + } + return messageIdentitiesAndUids.toArray(new IdentityAndUid[0]); + }, this::onFinishCallbackUserContent, this::onCancelCallbackUserContent); + batchSendUserContentMessageOperationQueue.queue(op); + } else { + if (ownedIdentity != null && messageUid != null) { + synchronized (protocolMessageUidsByServer) { + Queue queue = protocolMessageUidsByServer.get(server); + if (queue == null) { + queue = new ArrayDeque<>(); + protocolMessageUidsByServer.put(server, queue); + } + queue.add(new IdentityAndUid(ownedIdentity, messageUid)); + } + } + BatchUploadMessagesCompositeOperation op = new BatchUploadMessagesCompositeOperation(sendManagerSessionFactory, sslSocketFactory, server, false, () -> { + List messageIdentitiesAndUids = new ArrayList<>(); + synchronized (protocolMessageUidsByServer) { + Queue queue = protocolMessageUidsByServer.get(server); + if (queue != null && !queue.isEmpty()) { + do { + messageIdentitiesAndUids.add(queue.remove()); + if (messageIdentitiesAndUids.size() == Constants.MAX_UPLOAD_MESSAGE_BATCH_SIZE) { + break; + } + } while (!queue.isEmpty()); + } + } + return messageIdentitiesAndUids.toArray(new IdentityAndUid[0]); + }, this::onFinishCallbackProtocol, this::onCancelCallbackProtocol); + batchSendProtocolMessageOperationQueue.queue(op); + } } private void scheduleNewSendMessageCompositeOperationQueueing(final Identity ownedIdentity, final UID messageUid) { - scheduler.schedule(new IdentityAndUid(ownedIdentity, messageUid), () -> queueNewSendMessageCompositeOperation(ownedIdentity, messageUid), "UploadMessageCompositeOperation"); + scheduler.schedule( + new IdentityAndUid(ownedIdentity, messageUid), + () -> queueNewSendMessageCompositeOperation(null, ownedIdentity, messageUid, true, true), + "UploadMessageCompositeOperation"); + } + + private void scheduleNewBatchSendMessageCompositeOperationQueueing(String server, boolean hasUserContent) { + batchScheduler.schedule( + new StringAndBoolean(server, hasUserContent), + () -> queueNewSendMessageCompositeOperation(server, null, null, false, hasUserContent), + "BatchUploadMessagesCompositeOperation"); } public void retryScheduledNetworkTasks() { scheduler.retryScheduledRunnables(); + batchScheduler.retryScheduledRunnables(); } private void waitForIdentityReactivation(Identity ownedIdentity, UID messageUid) { @@ -122,15 +216,121 @@ private void waitForIdentityReactivation(Identity ownedIdentity, UID messageUid) awaitingIdentityReactivationOperationsLock.unlock(); } - @Override - public void onFinishCallback(Operation operation) { + + + public void onFinishCallbackProtocol(Operation operation) { + String server = ((BatchUploadMessagesCompositeOperation) operation).getServer(); + List identityInactiveMessageUids = ((BatchUploadMessagesCompositeOperation) operation).getIdentityInactiveMessageUids(); + batchScheduler.clearFailedCount(new StringAndBoolean(server, false)); + + // if there are still some messages in the queue, reschedule a batch operation + synchronized (protocolMessageUidsByServer) { + Queue queue = protocolMessageUidsByServer.get(server); + if (queue != null && !queue.isEmpty()) { + queueNewSendMessageCompositeOperation(server, null, null, false, false); + } + } + + // handle message the operations couldn't because of inactive identity + for (IdentityAndUid identityAndUid : identityInactiveMessageUids) { + waitForIdentityReactivation(identityAndUid.ownedIdentity, identityAndUid.uid); + } + } + + public void onCancelCallbackProtocol(Operation operation) { + String server = ((BatchUploadMessagesCompositeOperation) operation).getServer(); + IdentityAndUid[] identityAndMessageUids = ((BatchUploadMessagesCompositeOperation) operation).getMessageIdentitiesAndUids(); + Integer rfc = operation.getReasonForCancel(); + Logger.i("BatchUploadMessagesCompositeOperation (protocol) cancelled for reason " + rfc); + if (rfc == null) { + rfc = Operation.RFC_NULL; + } + //noinspection SwitchStatementWithTooFewBranches + switch (rfc) { + case BatchUploadMessagesCompositeOperation.RFC_BATCH_TOO_LARGE: + if (identityAndMessageUids != null) { + // if the payload is too large when batching, queue each message individually + for (IdentityAndUid identityAndMessageUid : identityAndMessageUids) { + queueNewSendMessageCompositeOperation(null, identityAndMessageUid.ownedIdentity, identityAndMessageUid.uid, true, true); + } + } + break; + default: + // re-add all messageUids to the queue and schedule a new operation later + if (identityAndMessageUids != null) { + synchronized (protocolMessageUidsByServer) { + Queue queue = protocolMessageUidsByServer.get(server); + if (queue == null) { + queue = new ArrayDeque<>(); + protocolMessageUidsByServer.put(server, queue); + } + queue.addAll(Arrays.asList(identityAndMessageUids)); + } + } + scheduleNewBatchSendMessageCompositeOperationQueueing(server, false); + } + } + + public void onFinishCallbackUserContent(Operation operation) { + String server = ((BatchUploadMessagesCompositeOperation) operation).getServer(); + List identityInactiveMessageUids = ((BatchUploadMessagesCompositeOperation) operation).getIdentityInactiveMessageUids(); + batchScheduler.clearFailedCount(new StringAndBoolean(server, true)); + + // if there are still some messages in the queue, reschedule a batch operation + synchronized (userContentMessageUidsByServer) { + Queue queue = userContentMessageUidsByServer.get(server); + if (queue != null && !queue.isEmpty()) { + queueNewSendMessageCompositeOperation(server, null, null, false, true); + } + } + + // handle message the operations couldn't because of inactive identity + for (IdentityAndUid identityAndUid : identityInactiveMessageUids) { + waitForIdentityReactivation(identityAndUid.ownedIdentity, identityAndUid.uid); + } + } + + public void onCancelCallbackUserContent(Operation operation) { + String server = ((BatchUploadMessagesCompositeOperation) operation).getServer(); + IdentityAndUid[] identityAndMessageUids = ((BatchUploadMessagesCompositeOperation) operation).getMessageIdentitiesAndUids(); + Integer rfc = operation.getReasonForCancel(); + Logger.i("BatchUploadMessagesCompositeOperation (user content) cancelled for reason " + rfc); + if (rfc == null) { + rfc = Operation.RFC_NULL; + } + //noinspection SwitchStatementWithTooFewBranches + switch (rfc) { + case BatchUploadMessagesCompositeOperation.RFC_BATCH_TOO_LARGE: + if (identityAndMessageUids != null) { + // if the payload is too large when batching, queue each message individually + for (IdentityAndUid identityAndMessageUid : identityAndMessageUids) { + queueNewSendMessageCompositeOperation(null, identityAndMessageUid.ownedIdentity, identityAndMessageUid.uid, true, true); + } + } + break; + default: + // re-add all messageUids to the queue + if (identityAndMessageUids != null) { + synchronized (userContentMessageUidsByServer) { + Queue queue = userContentMessageUidsByServer.get(server); + if (queue == null) { + queue = new ArrayDeque<>(); + userContentMessageUidsByServer.put(server, queue); + } + queue.addAll(Arrays.asList(identityAndMessageUids)); + } + } + scheduleNewBatchSendMessageCompositeOperationQueueing(server, true); + } + } + + public void onFinishCallbackWithAttachment(Operation operation) { Identity ownedIdentity = ((UploadMessageCompositeOperation) operation).getOwnedIdentity(); UID messageUid = ((UploadMessageCompositeOperation) operation).getMessageUid(); scheduler.clearFailedCount(new IdentityAndUid(ownedIdentity, messageUid)); } - @Override - public void onCancelCallback(Operation operation) { + public void onCancelCallbackWithAttachment(Operation operation) { Identity ownedIdentity = ((UploadMessageCompositeOperation) operation).getOwnedIdentity(); UID messageUid = ((UploadMessageCompositeOperation) operation).getMessageUid(); Integer rfc = operation.getReasonForCancel(); @@ -152,8 +352,8 @@ public void onCancelCallback(Operation operation) { // Notification received from OutboxMessage database @Override - public void newMessageToSend(Identity ownedIdentity, UID messageUid) { - queueNewSendMessageCompositeOperation(ownedIdentity, messageUid); + public void newMessageToSend(String server, Identity ownedIdentity, UID messageUid, boolean hasAttachment, boolean hasUserContent) { + queueNewSendMessageCompositeOperation(server, ownedIdentity, messageUid, hasAttachment, hasUserContent); } class NotificationListener implements io.olvid.engine.datatypes.NotificationListener { @@ -171,7 +371,8 @@ public void callback(String notificationName, HashMap userInfo) if (messageUids != null) { awaitingIdentityReactivationOperations.remove(ownedIdentity); for (UID messageUid: messageUids) { - queueNewSendMessageCompositeOperation(ownedIdentity, messageUid); + // if unsure, queue in the traditional message upload queue, even if there is no attachment + queueNewSendMessageCompositeOperation(null, ownedIdentity, messageUid, true, true); } } awaitingIdentityReactivationOperationsLock.unlock(); diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/databases/OutboxMessage.java b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/databases/OutboxMessage.java index 70519536..b57ab48c 100644 --- a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/databases/OutboxMessage.java +++ b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/databases/OutboxMessage.java @@ -149,13 +149,16 @@ public void setUidFromServer(UID uidFromServer, byte[] nonce, long timestampFrom // region constructors - public static OutboxMessage create(SendManagerSession sendManagerSession, Identity ownedIdentity, UID uid, String server, EncryptedBytes encryptedContent, EncryptedBytes encryptedExtendedContent, boolean isApplicationMessage, boolean isVoipMessage) { + public static OutboxMessage create(SendManagerSession sendManagerSession, Identity ownedIdentity, UID uid, String server, EncryptedBytes encryptedContent, EncryptedBytes encryptedExtendedContent, boolean isApplicationMessage, boolean isVoipMessage, boolean hasAttachments) { if (ownedIdentity == null || uid == null || server == null || encryptedContent == null) { return null; } try { OutboxMessage outboxMessage = new OutboxMessage(sendManagerSession, ownedIdentity, uid, server, encryptedContent, encryptedExtendedContent, isApplicationMessage, isVoipMessage); outboxMessage.insert(); + if (hasAttachments) { + outboxMessage.commitHookBits |= HOOK_BIT_HAS_ATTACHMENTS; + } return outboxMessage; } catch (SQLException e) { e.printStackTrace(); @@ -219,6 +222,39 @@ public static OutboxMessage get(SendManagerSession sendManagerSession, Identity } } + public static OutboxMessage[] getManyWithoutUidFromServer(SendManagerSession sendManagerSession, Identity ownedIdentity, String server, UID[] uids) throws SQLException { + if (uids == null) { + return null; + } + + // build a ?,? string + int count = uids.length; + StringBuilder sb = new StringBuilder(count * 2); + while (count-- > 1) { + sb.append("?,"); + } + sb.append("?"); + + try (PreparedStatement statement = sendManagerSession.session.prepareStatement("SELECT * FROM " + TABLE_NAME + + " WHERE " + OWNED_IDENTITY + " = ? " + + " AND " + SERVER + " = ? " + + " AND " + UID_FROM_SERVER + " IS NULL " + + " AND " + UID_ + " IN (" + sb + ");")) { + statement.setBytes(1, ownedIdentity.getBytes()); + statement.setString(2, server); + for (int i = 0; i < uids.length; i++) { + statement.setBytes(i + 3, uids[i].getBytes()); + } + try (ResultSet res = statement.executeQuery()) { + List list = new ArrayList<>(); + while (res.next()) { + list.add(new OutboxMessage(sendManagerSession, res)); + } + return list.toArray(new OutboxMessage[0]); + } + } + } + public static OutboxMessage[] getAll(SendManagerSession sendManagerSession) { try (PreparedStatement statement = sendManagerSession.session.prepareStatement("SELECT * FROM " + TABLE_NAME + ";")) { try (ResultSet res = statement.executeQuery()) { @@ -411,12 +447,13 @@ public void delete() throws SQLException { public interface NewOutboxMessageListener { - void newMessageToSend(Identity ownedIdentity, UID messageUid); + void newMessageToSend(String server, Identity ownedIdentity, UID messageUid, boolean hasAttachment, boolean hasUserContent); } private long commitHookBits = 0; private static final long HOOK_BIT_INSERT = 0x1; private static final long HOOK_BIT_ACKNOWLEDGED = 0x2; + private static final long HOOK_BIT_HAS_ATTACHMENTS = 0x4; private long acknowledgedTimestampFromSever; @@ -424,7 +461,7 @@ public interface NewOutboxMessageListener { public void wasCommitted() { if ((commitHookBits & HOOK_BIT_INSERT) != 0) { if (sendManagerSession.newOutboxMessageListener != null) { - sendManagerSession.newOutboxMessageListener.newMessageToSend(ownedIdentity, uid); + sendManagerSession.newOutboxMessageListener.newMessageToSend(server, ownedIdentity, uid, (commitHookBits & HOOK_BIT_HAS_ATTACHMENTS) != 0, isApplicationMessage); } } if ((commitHookBits & HOOK_BIT_ACKNOWLEDGED) != 0) { diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/operations/BatchUploadMessagesCompositeOperation.java b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/operations/BatchUploadMessagesCompositeOperation.java new file mode 100644 index 00000000..ab133fc0 --- /dev/null +++ b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/operations/BatchUploadMessagesCompositeOperation.java @@ -0,0 +1,139 @@ +/* + * Olvid for Android + * Copyright © 2019-2024 Olvid SAS + * + * This file is part of Olvid for Android. + * + * Olvid is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * Olvid is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Olvid. If not, see . + */ + +package io.olvid.engine.networksend.operations; + +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import javax.net.ssl.SSLSocketFactory; + +import io.olvid.engine.crypto.Hash; +import io.olvid.engine.crypto.Suite; +import io.olvid.engine.datatypes.Operation; +import io.olvid.engine.datatypes.OperationQueue; +import io.olvid.engine.datatypes.UID; +import io.olvid.engine.datatypes.containers.IdentityAndUid; +import io.olvid.engine.datatypes.containers.StringAndBoolean; +import io.olvid.engine.networksend.coordinators.SendAttachmentCoordinator; +import io.olvid.engine.networksend.datatypes.SendManagerSessionFactory; + + +public class BatchUploadMessagesCompositeOperation extends Operation { + // possible reasons for cancel + public static final int RFC_BATCH_TOO_LARGE = 2; + public static final int RFC_NETWORK_ERROR = 3; + + private final SendManagerSessionFactory sendManagerSessionFactory; + private final SSLSocketFactory sslSocketFactory; + private final String server; + private final SendAttachmentCoordinator.MessageBatchProvider messageBatchProvider; + private IdentityAndUid[] messageIdentitiesAndUids; + private Operation[] suboperations; + + public BatchUploadMessagesCompositeOperation(SendManagerSessionFactory sendManagerSessionFactory, SSLSocketFactory sslSocketFactory, String server, boolean userContentMessages, SendAttachmentCoordinator.MessageBatchProvider messageBatchProvider, OnFinishCallback onFinishCallback, OnCancelCallback onCancelCallback) { + super(StringAndBoolean.computeUniqueUid(server, userContentMessages), onFinishCallback, onCancelCallback); + this.sendManagerSessionFactory = sendManagerSessionFactory; + this.sslSocketFactory = sslSocketFactory; + this.server = server; + this.messageBatchProvider = messageBatchProvider; + this.messageIdentitiesAndUids = null; + this.suboperations = null; + } + + public String getServer() { + return server; + } + + public IdentityAndUid[] getMessageIdentitiesAndUids() { + return messageIdentitiesAndUids; + } + + public List getIdentityInactiveMessageUids() { + if (suboperations != null && suboperations.length > 0) { + return ((BatchUploadMessagesOperation) suboperations[0]).getIdentityInactiveMessageUids(); + } + return Collections.emptyList(); + } + + @Override + public void doCancel() { + for (Operation op: suboperations) { + op.cancel(null); + } + } + + @Override + public void doExecute() { + boolean finished = false; + try { + // first get some messageUids from the provider + this.messageIdentitiesAndUids = messageBatchProvider.getBatchOFMessageUids(); + if (messageIdentitiesAndUids.length == 0) { + suboperations = new Operation[0]; + } else { + suboperations = new Operation[messageIdentitiesAndUids.length + 1]; + + suboperations[0] = new BatchUploadMessagesOperation(sendManagerSessionFactory, sslSocketFactory, server, messageIdentitiesAndUids); + for (int i = 0; i < messageIdentitiesAndUids.length; i++) { + suboperations[i + 1] = new TryToDeleteMessageAndAttachmentsOperation(sendManagerSessionFactory, messageIdentitiesAndUids[i].ownedIdentity, messageIdentitiesAndUids[i].uid); + suboperations[i + 1].addDependency(suboperations[0]); + } + } + + // now run the suboperations + if (suboperations.length > 0) { + OperationQueue queue = new OperationQueue(); + for (Operation op : suboperations) { + queue.queue(op); + } + queue.execute(1, "BatchUploadMessagesCompositeOperation"); + queue.join(); + + if (cancelWasRequested()) { + return; + } + + for (Operation op : suboperations) { + if (op.isCancelled()) { + cancel(op.getReasonForCancel()); + return; + } + } + } + finished = true; + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (finished) { + setFinished(); + } else { + cancel(null); + processCancel(); + } + } + } + + public static UID computeUniqueUid(String server) { + Hash sha256 = Suite.getHash(Hash.SHA256); + return new UID(sha256.digest(server.getBytes(StandardCharsets.UTF_8))); + } +} diff --git a/obv_engine/engine/src/main/java/io/olvid/engine/networksend/operations/BatchUploadMessagesOperation.java b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/operations/BatchUploadMessagesOperation.java new file mode 100644 index 00000000..0ee95ef2 --- /dev/null +++ b/obv_engine/engine/src/main/java/io/olvid/engine/networksend/operations/BatchUploadMessagesOperation.java @@ -0,0 +1,246 @@ +/* + * Olvid for Android + * Copyright © 2019-2024 Olvid SAS + * + * This file is part of Olvid for Android. + * + * Olvid is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * Olvid is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Olvid. If not, see . + */ + +package io.olvid.engine.networksend.operations; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.SSLSocketFactory; + +import io.olvid.engine.Logger; +import io.olvid.engine.datatypes.Identity; +import io.olvid.engine.datatypes.Operation; +import io.olvid.engine.datatypes.ServerMethod; +import io.olvid.engine.datatypes.UID; +import io.olvid.engine.datatypes.containers.IdentityAndUid; +import io.olvid.engine.encoder.Encoded; +import io.olvid.engine.networksend.databases.MessageHeader; +import io.olvid.engine.networksend.databases.OutboxMessage; +import io.olvid.engine.networksend.datatypes.SendManagerSession; +import io.olvid.engine.networksend.datatypes.SendManagerSessionFactory; + + +public class BatchUploadMessagesOperation extends Operation { + private final SendManagerSessionFactory sendManagerSessionFactory; + private final SSLSocketFactory sslSocketFactory; + private final String server; + private final IdentityAndUid[] messageIdentitiesAndUids; + private final List identityInactiveMessageUids; + + public BatchUploadMessagesOperation(SendManagerSessionFactory sendManagerSessionFactory, SSLSocketFactory sslSocketFactory, String server, IdentityAndUid[] messageIdentitiesAndUids) { + super(); + this.sendManagerSessionFactory = sendManagerSessionFactory; + this.sslSocketFactory = sslSocketFactory; + this.server = server; + this.messageIdentitiesAndUids = messageIdentitiesAndUids; + this.identityInactiveMessageUids = new ArrayList<>(); + } + + public List getIdentityInactiveMessageUids() { + return identityInactiveMessageUids; + } + + @Override + public void doCancel() { + // Nothing special to do on cancel + } + + @Override + public void doExecute() { + boolean finished = false; + try (SendManagerSession sendManagerSession = sendManagerSessionFactory.getSession()) { + try { + List outboxMessageAndHeaders = new ArrayList<>(); + + Logger.d("BatchUploadMessagesOperation uploading a batch of " + messageIdentitiesAndUids.length); + + HashMap> messageUidsByIdentity = new HashMap<>(); + for (IdentityAndUid identityAndUid : messageIdentitiesAndUids) { + List list = messageUidsByIdentity.get(identityAndUid.ownedIdentity); + if (list == null) { + list = new ArrayList<>(); + messageUidsByIdentity.put(identityAndUid.ownedIdentity, list); + } + list.add(identityAndUid.uid); + } + + for (Map.Entry> entry : messageUidsByIdentity.entrySet()) { + Identity ownedIdentity = entry.getKey(); + List messageUids = entry.getValue(); + // we need to block sending message for any inactive ownedIdentity, but, if the ownedIdentity was deleted, we should send the message + // this is required for the OwnedIdentityDeletion protocol, to inform your contacts + if (!sendManagerSession.identityDelegate.isActiveOwnedIdentity(sendManagerSession.session,ownedIdentity) + && sendManagerSession.identityDelegate.isOwnedIdentity(sendManagerSession.session, ownedIdentity)) { + for (UID messageUid : messageUids) { + identityInactiveMessageUids.add(new IdentityAndUid(ownedIdentity, messageUid)); + } + } else { + OutboxMessage[] outboxMessages = OutboxMessage.getManyWithoutUidFromServer(sendManagerSession, ownedIdentity, server, messageUids.toArray(new UID[0])); + for (OutboxMessage outboxMessage : outboxMessages) { + MessageHeader[] headers = outboxMessage.getHeaders(); + outboxMessageAndHeaders.add(new OutboxMessageAndHeaders(outboxMessage, headers)); + } + } + } + + if (cancelWasRequested()) { + return; + } + + BatchUploadMessagesServerMethod serverMethod = new BatchUploadMessagesServerMethod(server, outboxMessageAndHeaders.toArray(new OutboxMessageAndHeaders[0])); + serverMethod.setSslSocketFactory(sslSocketFactory); + + byte returnStatus = serverMethod.execute(true); + + sendManagerSession.session.startTransaction(); + switch (returnStatus) { + case ServerMethod.OK: + for (OutboxMessageAndHeaders outboxMessageAndHeader : serverMethod.getOutboxMessageAndHeaders()) { + outboxMessageAndHeader.outboxMessage.setUidFromServer(outboxMessageAndHeader.uidFromServer, outboxMessageAndHeader.nonce, outboxMessageAndHeader.timestampFromServer); + } + + finished = true; + return; + case ServerMethod.OK_WITH_MALFORMED_SERVER_RESPONSE: + // unable to parse server response and get message Uids --> finish the operation + for (OutboxMessageAndHeaders outboxMessageAndHeader : outboxMessageAndHeaders) { + outboxMessageAndHeader.outboxMessage.setUidFromServer(new UID(new byte[UID.UID_LENGTH]), new byte[0], 0); + } + finished = true; + return; + case ServerMethod.PAYLOAD_TOO_LARGE: { + cancel(BatchUploadMessagesCompositeOperation.RFC_BATCH_TOO_LARGE); + break; + } + case ServerMethod.GENERAL_ERROR: + default: + cancel(BatchUploadMessagesCompositeOperation.RFC_NETWORK_ERROR); + } + } catch (Exception e) { + e.printStackTrace(); + sendManagerSession.session.rollback(); + } finally { + if (finished) { + sendManagerSession.session.commit(); + setFinished(); + } else { + if (hasNoReasonForCancel()) { + cancel(null); + } + processCancel(); + } + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} + +class OutboxMessageAndHeaders { + final OutboxMessage outboxMessage; + final MessageHeader[] headers; + + UID uidFromServer = null; + byte[] nonce = null; + long timestampFromServer = 0; + + public OutboxMessageAndHeaders(OutboxMessage outboxMessage, MessageHeader[] headers) { + this.outboxMessage = outboxMessage; + this.headers = headers; + } +} + +class BatchUploadMessagesServerMethod extends ServerMethod { + private static final String SERVER_METHOD_PATH = "/batchUploadMessages"; + + private final String server; + private final OutboxMessageAndHeaders[] outboxMessageAndHeaders; + + + public OutboxMessageAndHeaders[] getOutboxMessageAndHeaders() { + return outboxMessageAndHeaders; + } + + BatchUploadMessagesServerMethod(String server, OutboxMessageAndHeaders[] outboxMessageAndHeaders) { + this.server = server; + this.outboxMessageAndHeaders = outboxMessageAndHeaders; + } + + @Override + protected String getServer() { + return server; + } + + @Override + protected String getServerMethod() { + return SERVER_METHOD_PATH; + } + + @Override + protected byte[] getDataToSend() { + Encoded[] encodeds = new Encoded[outboxMessageAndHeaders.length]; + for (int i=0; i + \ No newline at end of file diff --git a/obv_messenger/app/build.gradle b/obv_messenger/app/build.gradle index 792f71f2..fb80bbb2 100644 --- a/obv_messenger/app/build.gradle +++ b/obv_messenger/app/build.gradle @@ -16,8 +16,8 @@ android { applicationId "io.olvid.messenger" minSdkVersion 21 targetSdk 34 - versionCode 233 - versionName "2.0" + versionCode 234 + versionName "2.0.1" vectorDrawables.useSupportLibrary true multiDexEnabled true resourceConfigurations += ['en', 'fr'] @@ -197,7 +197,7 @@ dependencies { implementation 'org.bitbucket.b_c:jose4j:0.9.6' // map libre integration - implementation 'org.maplibre.gl:android-sdk:10.2.0' + implementation 'org.maplibre.gl:android-sdk:10.3.0' implementation 'org.maplibre.gl:android-plugin-annotation-v9:2.0.2' final def commonmark_version = "0.21.0" @@ -227,7 +227,7 @@ dependencies { fullImplementation('com.google.http-client:google-http-client-gson:1.44.1') { exclude group: 'org.apache.httpcomponents' } - fullImplementation('com.google.api-client:google-api-client-android:2.3.0') { + fullImplementation('com.google.api-client:google-api-client-android:2.4.0') { exclude group: 'org.apache.httpcomponents' } fullImplementation('com.google.apis:google-api-services-drive:v3-rev20221219-2.0.0') { diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/discussion/location/MapViewMapLibreFragment.java b/obv_messenger/app/src/main/java/io/olvid/messenger/discussion/location/MapViewMapLibreFragment.java index 3dbe5d29..85a2f9aa 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/discussion/location/MapViewMapLibreFragment.java +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/discussion/location/MapViewMapLibreFragment.java @@ -166,7 +166,7 @@ public View onCreateView(@NonNull LayoutInflater inflater, @Nullable ViewGroup c private void loadStyleUrls() { if (SettingsActivity.getLocationIntegration() == SettingsActivity.LocationIntegrationEnum.OSM) { List osmStyles = AppSingleton.getEngine().getOsmStyles(AppSingleton.getBytesCurrentIdentity()); - if (osmStyles == null || osmStyles.size() == 0) { + if (osmStyles == null || osmStyles.isEmpty()) { loadFallbackStyleUrl(); } else { currentStyleId = SettingsActivity.getLocationLastOsmStyleId(); @@ -225,8 +225,8 @@ public void onMapReady(@NonNull MapboxMap mapboxMap) { ((MapView) mapView).addOnDidFailLoadingMapListener((errorMessage) -> { Logger.w("OSM style not found, trying fallback style"); if (!triedStyleFallbackUrl) { - loadFallbackStyleUrl(); - mapboxMap.setStyle(new Style.Builder().fromUri(getStyleUrl()), this::onStyleLoaded); + triedStyleFallbackUrl = true; + mapboxMap.setStyle(new Style.Builder().fromUri(FALLBACK_OSM_STYLE_URL), this::onStyleLoaded); } }); } @@ -265,6 +265,9 @@ public void onStyleLoaded(Style style) { return; } + // reset this to allow reloading it when the user selects another not found style + triedStyleFallbackUrl = false; + // setup ui mapboxMap.getUiSettings().setCompassEnabled(true); Drawable compass = ContextCompat.getDrawable(activity, R.drawable.map_compass); @@ -313,6 +316,7 @@ public void onStyleLoaded(Style style) { // call parent callback if set if (onMapReadyCallback != null) { onMapReadyCallback.run(); + onMapReadyCallback = null; } } diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/main/contacts/ContactListItem.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/main/contacts/ContactListItem.kt index 53a16c7e..1189e94b 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/main/contacts/ContactListItem.kt +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/main/contacts/ContactListItem.kt @@ -148,43 +148,39 @@ fun ContactListItem( Row(verticalAlignment = CenterVertically) { - endContent?.let { - it.invoke() - } ?: kotlin.run { - AnimatedVisibility(visible = publishedDetails) { - BoxWithConstraints( + AnimatedVisibility(visible = publishedDetails) { + BoxWithConstraints( + modifier = Modifier + .padding(end = 8.dp) + .size(width = 24.dp, height = 18.dp) + ) { + Image( modifier = Modifier - .padding(end = 8.dp) - .size(width = 24.dp, height = 18.dp) - - ) { + .fillMaxWidth() + .align(Alignment.BottomStart), + painter = painterResource(id = drawable.ic_olvid_card), + contentDescription = "" + ) + if (publishedDetailsNotification) { Image( modifier = Modifier - .fillMaxWidth() - .align(Alignment.BottomStart), - painter = painterResource(id = drawable.ic_olvid_card), - contentDescription = "" - ) - if (publishedDetailsNotification) { - Image( - modifier = Modifier - .size(maxWidth.times(0.4f)) - .align(Alignment.TopEnd) - .offset(x = 4.dp, y = (-4).dp), - painter = painterResource(id = drawable.ic_dot_white_bordered), - contentDescription = stringResource( - id = string.content_description_message_status - ) + .size(maxWidth.times(0.4f)) + .align(Alignment.TopEnd) + .offset(x = 4.dp, y = (-4).dp), + painter = painterResource(id = drawable.ic_dot_white_bordered), + contentDescription = stringResource( + id = string.content_description_message_status ) - } + ) } } + } - AnimatedVisibility(visible = shouldAnimateChannel) { - EstablishingChannel() - } - + AnimatedVisibility(visible = shouldAnimateChannel) { + EstablishingChannel() } + + endContent?.invoke() } } } diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/openid/KeycloakManager.java b/obv_messenger/app/src/main/java/io/olvid/messenger/openid/KeycloakManager.java index 287a9658..c9538a4c 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/openid/KeycloakManager.java +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/openid/KeycloakManager.java @@ -589,16 +589,7 @@ public void run() { try { UUID newApiKey = UUID.fromString(keycloakUserDetailsAndStuff.apiKey); if (!Objects.equals(Logger.getUuidString(newApiKey), kms.ownApiKey)) { - App.runThread(() -> { - if (AppSingleton.getEngine().registerOwnedIdentityApiKeyOnServer(identityBytesKey.bytes, newApiKey) == RegisterApiKeyResult.SUCCESS) { - try { - AppSingleton.getEngine().saveKeycloakApiKey(identityBytesKey.bytes, keycloakUserDetailsAndStuff.apiKey); - kms.ownApiKey = keycloakUserDetailsAndStuff.apiKey; - } catch (Exception e) { - e.printStackTrace(); - } - } - }); + App.runThread(() -> registerMeApiKeyOnServer(kms, identityBytesKey, newApiKey)); } } catch (Exception e) { // do nothing @@ -673,6 +664,35 @@ public void run() { }); } + private static void registerMeApiKeyOnServer(KeycloakManagerState kms, BytesKey identityBytesKey, UUID apiKey) { + // retry at most 10 times + for (int i=0; i<10; i++) { + RegisterApiKeyResult registerApiKeyResult = AppSingleton.getEngine().registerOwnedIdentityApiKeyOnServer(identityBytesKey.bytes, apiKey); + Logger.d("Registering Keycloak api key on server: " + registerApiKeyResult); + switch (registerApiKeyResult) { + case SUCCESS: + try { + AppSingleton.getEngine().saveKeycloakApiKey(identityBytesKey.bytes, Logger.getUuidString(apiKey)); + kms.ownApiKey = Logger.getUuidString(apiKey); + return; + } catch (Exception e) { + e.printStackTrace(); + } + break; + case WAIT_FOR_SERVER_SESSION: + // wait a bit, then try again + try { + Thread.sleep((3 + i) * 1000); + } catch (InterruptedException ignored) { } + break; + case INVALID_KEY: + case FAILED: + // non-retriable failure, abort + return; + } + } + } + private static class KeycloakManagerState { @NonNull final byte[] bytesOwnedIdentity; @NonNull JsonIdentityDetails identityDetails; diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/plus_button/ConfigurationScannedFragment.java b/obv_messenger/app/src/main/java/io/olvid/messenger/plus_button/ConfigurationScannedFragment.java index 8923fabb..2dde390c 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/plus_button/ConfigurationScannedFragment.java +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/plus_button/ConfigurationScannedFragment.java @@ -516,6 +516,7 @@ public void onClick(View v) { }); break; case FAILED: + case WAIT_FOR_SERVER_SESSION: new Handler(Looper.getMainLooper()).post(() -> { activateButton.setEnabled(true); activationSpinner.setVisibility(View.GONE); diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcCallActivity.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcCallActivity.kt index b40c639f..be5cad3b 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcCallActivity.kt +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcCallActivity.kt @@ -26,7 +26,6 @@ import android.content.Context import android.content.Intent import android.content.ServiceConnection import android.content.pm.PackageManager -import android.content.res.Configuration import android.graphics.Point import android.media.projection.MediaProjectionConfig import android.media.projection.MediaProjectionManager @@ -40,6 +39,7 @@ import android.os.Handler import android.os.IBinder import android.os.Looper import android.provider.Settings +import android.view.Gravity import android.view.KeyEvent import android.view.WindowManager.LayoutParams import android.widget.Toast @@ -98,10 +98,6 @@ import io.olvid.messenger.webrtc.components.CallAction.ToggleMicrophone import io.olvid.messenger.webrtc.components.CallAction.ToggleSpeaker import io.olvid.messenger.webrtc.components.CallScreen import io.olvid.messenger.webrtc.components.enterPictureInPicture -import org.webrtc.PeerConnection -import org.webrtc.RTCStatsCollectorCallback -import org.webrtc.RTCStatsReport -import org.webrtc.VideoTrack private const val REQUEST_MEDIA_PROJECTION = 314 @@ -229,6 +225,11 @@ class WebrtcCallActivity : AppCompatActivity() { } ToggleCamera -> { + if ((webrtcCallService?.getCallParticipantsLiveData()?.value?.size ?: 0) > WebrtcPeerConnectionHolder.MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO) { + App.toast(getString(R.string.toast_message_video_calls_xxx_participants, WebrtcPeerConnectionHolder.MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO + 1), Toast.LENGTH_SHORT, Gravity.BOTTOM) + return@CallScreen + } + if ( ContextCompat.checkSelfPermission( this, @@ -255,6 +256,11 @@ class WebrtcCallActivity : AppCompatActivity() { } ShareScreen -> { + if ((webrtcCallService?.getCallParticipantsLiveData()?.value?.size ?: 0) > WebrtcPeerConnectionHolder.MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO) { + App.toast(getString(R.string.toast_message_video_calls_xxx_participants, WebrtcPeerConnectionHolder.MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO + 1), Toast.LENGTH_SHORT, Gravity.BOTTOM) + return@CallScreen + } + val width: Int val height: Int if (VERSION.SDK_INT >= VERSION_CODES.R) { @@ -303,7 +309,9 @@ class WebrtcCallActivity : AppCompatActivity() { contentColor = Color.White, onDismissRequest = onDismiss, buttons = { - Row(modifier = Modifier.fillMaxWidth().padding(start = 16.dp, bottom = 8.dp, end = 16.dp), + Row(modifier = Modifier + .fillMaxWidth() + .padding(start = 16.dp, bottom = 8.dp, end = 16.dp), horizontalArrangement = Arrangement.End) { dialog.additionalButton?.let { button -> button() diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcCallService.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcCallService.kt index 38cb4eb2..f9091d78 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcCallService.kt +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcCallService.kt @@ -139,6 +139,7 @@ import io.olvid.messenger.webrtc.WebrtcCallService.State.WAITING_FOR_AUDIO_PERMI import io.olvid.messenger.webrtc.WebrtcCallService.WakeLock.ALL import io.olvid.messenger.webrtc.WebrtcCallService.WakeLock.PROXIMITY import io.olvid.messenger.webrtc.WebrtcCallService.WakeLock.WIFI +import io.olvid.messenger.webrtc.WebrtcPeerConnectionHolder.Companion.MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO import io.olvid.messenger.webrtc.WebrtcPeerConnectionHolder.Companion.audioDeviceModule import io.olvid.messenger.webrtc.WebrtcPeerConnectionHolder.Companion.localScreenTrack import io.olvid.messenger.webrtc.WebrtcPeerConnectionHolder.Companion.localVideoTrack @@ -176,7 +177,6 @@ import org.webrtc.CameraVideoCapturer import org.webrtc.ScreenCapturerAndroid import org.webrtc.SurfaceTextureHelper import org.webrtc.VideoCapturer -import org.webrtc.VideoTrack import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.IOException @@ -298,7 +298,7 @@ class WebrtcCallService : Service() { private set private var bluetoothAutoConnect = true var wiredHeadsetConnected = false - var availableAudioOutputs = if (VERSION.SDK_INT >= VERSION_CODES.M) mutableStateListOf(PHONE, LOUDSPEAKER, MUTED) else mutableStateListOf(PHONE, LOUDSPEAKER) + var availableAudioOutputs = mutableStateListOf(PHONE, LOUDSPEAKER, MUTED) private set var selectedParticipant: ByteArray? by mutableStateOf(bytesOwnedIdentity) @@ -1141,7 +1141,7 @@ class WebrtcCallService : Service() { } callParticipant.setPeerState(RINGING) if (_state == INITIALIZING_CALL || _state == BUSY) { - outgoingCallRinger!!.ring(RING) + outgoingCallRinger?.ring(RING) setState(State.RINGING) } } @@ -1166,7 +1166,7 @@ class WebrtcCallService : Service() { createLogEntry(CallLogItem.STATUS_BUSY) } if (_state == INITIALIZING_CALL) { - outgoingCallRinger!!.ring(Type.BUSY) + outgoingCallRinger?.ring(Type.BUSY) setState(BUSY) } } @@ -1181,9 +1181,8 @@ class WebrtcCallService : Service() { if (callParticipant.peerState != START_CALL_MESSAGE_SENT && callParticipant.peerState != RINGING) { return@execute } - if (_state == State.RINGING || _state == BUSY) { - outgoingCallRinger!!.stop() - } + outgoingCallRinger?.stop() + val peerSdpDescription: String = try { gunzip(gzippedPeerSdpDescription) } catch (e: IOException) { @@ -1257,8 +1256,8 @@ class WebrtcCallService : Service() { private fun hangUpCallInternal() { // notify peer that you hung up (it's not just a connection loss) sendHangedUpMessage(callParticipants.values) - if (soundPool != null && _state != CALL_ENDED) { // do not play if state is already call ended - soundPool!!.play(disconnectSound, 1f, 1f, 0, 0, 1f) + if (_state != CALL_ENDED && selectedAudioOutput != MUTED) { // do not play if state is already call ended + soundPool?.play(disconnectSound, 1f, 1f, 0, 0, 1f) } createLogEntry(CallLogItem.STATUS_MISSED) setState(CALL_ENDED) @@ -1549,6 +1548,9 @@ class WebrtcCallService : Service() { if (_state == CALL_ENDED || _state == FAILED) { return@execute } + if (callParticipant.peerState == KICKED || callParticipant.peerState == HANGED_UP || callParticipant.peerState == PeerState.FAILED) { + return@execute + } callParticipant.setPeerState(RECONNECTING) updateStateFromPeerStates() } @@ -1581,8 +1583,8 @@ class WebrtcCallService : Service() { } acquireWakeLock(WIFI) createLogEntry(CallLogItem.STATUS_SUCCESSFUL) - if (soundPool != null) { - soundPool!!.play(connectSound, 1f, 1f, 0, 0, 1f) + if (selectedAudioOutput != MUTED) { + soundPool?.play(connectSound, 1f, 1f, 0, 0, 1f) } if (callDurationTimer != null) { callDurationTimer = null @@ -1872,8 +1874,7 @@ class WebrtcCallService : Service() { if (Arrays.equals( jsonContactBytesAndName.bytesContactIdentity, bytesOwnedIdentity - ) - ) { + )) { // the received array contains the user himself continue } @@ -1943,6 +1944,7 @@ class WebrtcCallService : Service() { } // endregion + // region Setters and Getters private fun setState(state: State) { if (this._state == FAILED) { @@ -2012,7 +2014,7 @@ class WebrtcCallService : Service() { } fun toggleCamera() { - if (cameraEnabled.not()) { + if (cameraEnabled.not() && callParticipants.size <= MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO) { try { localVideoTrack?.setEnabled(true) } catch (ignored: Exception) { @@ -2055,7 +2057,7 @@ class WebrtcCallService : Service() { } fun toggleScreenShare(intent: Intent? = null) { - if (screenShareActive.not()) { + if (screenShareActive.not() && callParticipants.size <= MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO) { if (screenCapturerAndroid == null) { intent?.let { // restart the ongoing foreground service to set the correct foreground service type @@ -2200,8 +2202,13 @@ class WebrtcCallService : Service() { bluetoothHeadsetManager!!.disconnectAudio() } - if (VERSION.SDK_INT >= VERSION_CODES.M) { - audioDeviceModule?.setSpeakerMute(audioOutput == MUTED) ?: return@execute + audioDeviceModule?.setSpeakerMute(audioOutput == MUTED) + if (audioOutput == MUTED) { + outgoingCallRinger?.stop() + reconnectingStreamId?.let { + soundPool?.stop(it) + } + soundPool } when (audioOutput) { @@ -2231,6 +2238,7 @@ class WebrtcCallService : Service() { } // endregion + // region Helper methods fun shouldISendTheOfferToCallParticipant(callParticipant: CallParticipant): Boolean { return BytesKey(bytesOwnedIdentity).compareTo(BytesKey(callParticipant.bytesContactIdentity)) > 0 @@ -2240,10 +2248,6 @@ class WebrtcCallService : Service() { executor.execute(runnable) } - fun getActiveLocalVideo(): VideoTrack? { - return if (screenShareActive) localScreenTrack else if (cameraEnabled) localVideoTrack else null - } - fun getCallParticipant(bytesContactIdentity: ByteArray?): CallParticipant? { val index = callParticipantIndexes[BytesKey(bytesContactIdentity)] ?: return null return callParticipants[index] @@ -2265,6 +2269,15 @@ class WebrtcCallService : Service() { pojos.add(CallParticipantPojo(callParticipant)) } callParticipantsLiveData.postValue(pojos) + if (callParticipants.size > MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO) { + // we disable video + if (cameraEnabled) { + toggleCamera() + } + if (screenShareActive) { + toggleScreenShare() + } + } } private fun updateStateFromPeerStates() { @@ -2287,19 +2300,19 @@ class WebrtcCallService : Service() { } } if (callParticipants.size == 1 && allReconnecting) { - if (reconnectingStreamId == null) { - reconnectingStreamId = soundPool!!.play(reconnectingSound, .5f, .5f, 0, -1, 1f) + if (reconnectingStreamId == null && selectedAudioOutput != MUTED) { + reconnectingStreamId = soundPool?.play(reconnectingSound, .5f, .5f, 0, -1, 1f) } } else { - if (reconnectingStreamId != null) { - soundPool!!.stop(reconnectingStreamId!!) + reconnectingStreamId?.let { + soundPool?.stop(it) reconnectingStreamId = null } } if (allPeersAreInFinalState) { createLogEntry(CallLogItem.STATUS_MISSED) // this only create the log if it was not yet created - if (soundPool != null && _state != CALL_ENDED) { - soundPool!!.play(disconnectSound, 1f, 1f, 0, 0, 1f) + if (_state != CALL_ENDED && selectedAudioOutput != MUTED) { + soundPool?.play(disconnectSound, 1f, 1f, 0, 0, 1f) } setState(CALL_ENDED) stopThisService() @@ -2324,9 +2337,7 @@ class WebrtcCallService : Service() { } } } - if (VERSION.SDK_INT >= VERSION_CODES.M) { - audioOutputs.add(MUTED) - } + audioOutputs.add(MUTED) if (!audioOutputs.contains(selectedAudioOutput)) { selectAudioOutput(audioOutputs[0]) } @@ -2900,7 +2911,8 @@ class WebrtcCallService : Service() { } // endregion -// region Service lifecycle + + // region Service lifecycle override fun onCreate() { super.onCreate() webrtcMessageReceivedBroadcastReceiver = WebrtcMessageReceivedBroadcastReceiver() @@ -2925,31 +2937,19 @@ class WebrtcCallService : Service() { callLogItem!!.duration = callDuration.value!! App.runThread { AppDatabase.getInstance().callLogItemDao().update(callLogItem) } } - if (outgoingCallRinger != null) { - outgoingCallRinger!!.stop() - } - if (incomingCallRinger != null) { - incomingCallRinger!!.stop() - } - if (soundPool != null) { - soundPool!!.release() - } - if (videoCapturer != null) { - try { - videoCapturer!!.stopCapture() - } catch (ignored: InterruptedException) { - } - videoCapturer!!.dispose() - videoCapturer = null - } - if (screenCapturerAndroid != null) { - try { - screenCapturerAndroid!!.stopCapture() - } catch (ignored: InterruptedException) { - } - screenCapturerAndroid!!.dispose() - screenCapturerAndroid = null - } + outgoingCallRinger?.stop() + incomingCallRinger?.stop() + soundPool?.release() + try { + videoCapturer?.stopCapture() + } catch (ignored: InterruptedException) { } + videoCapturer?.dispose() + videoCapturer = null + try { + screenCapturerAndroid?.stopCapture() + } catch (ignored: InterruptedException) { } + screenCapturerAndroid?.dispose() + screenCapturerAndroid = null unregisterScreenOffReceiver() unregisterWiredHeadsetReceiver() timeoutTimer.cancel() @@ -3191,7 +3191,8 @@ class WebrtcCallService : Service() { } // endregion -// region Send messages via Oblivious channel + + // region Send messages via Oblivious channel @Throws(IOException::class) fun sendStartCallMessage( callParticipant: CallParticipant, @@ -3600,17 +3601,16 @@ ${jsonIceCandidate.sdpMLineIndex} -> ${jsonIceCandidate.sdp}""" } // endregion -// region JsonDataChannelMessages + + // region JsonDataChannelMessages private inner class DataChannelListener(private val callParticipant: CallParticipant) : DataChannelMessageListener { override fun onConnect() { executor.execute { sendDataChannelMessage(callParticipant, JsonMutedInnerMessage(microphoneMuted)) sendDataChannelMessage(callParticipant, JsonVideoSupportedInnerMessage(true)) - sendDataChannelMessage( - callParticipant, - JsonVideoSharingInnerMessage(cameraEnabled) - ) + sendDataChannelMessage(callParticipant, JsonVideoSharingInnerMessage(cameraEnabled)) + sendDataChannelMessage(callParticipant, JsonScreenSharingInnerMessage(screenShareActive)) if (isCaller) { sendDataChannelMessage( callParticipant, @@ -3744,6 +3744,7 @@ ${jsonIceCandidate.sdpMLineIndex} -> ${jsonIceCandidate.sdp}""" } // endregion + inner class CallParticipant { internal val role: Role @@ -3945,7 +3946,7 @@ ${jsonIceCandidate.sdpMLineIndex} -> ${jsonIceCandidate.sdp}""" } // device orientation change listener for screen capture on API < 34 - val orientationChangeBroadcastReceiver = object: BroadcastReceiver() { + private val orientationChangeBroadcastReceiver = object: BroadcastReceiver() { override fun onReceive(context: Context?, intent: Intent?) { context?.resources?.configuration?.let { configuration -> if (configuration.orientation == Configuration.ORIENTATION_PORTRAIT && screenWidth > screenHeight diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcPeerConnectionHolder.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcPeerConnectionHolder.kt index 4d55f6af..a0b487ec 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcPeerConnectionHolder.kt +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/WebrtcPeerConnectionHolder.kt @@ -125,6 +125,7 @@ class WebrtcPeerConnectionHolder( companion object { + const val MAXIMUM_OTHER_PARTICIPANTS_FOR_VIDEO = 2 const val OLVID_STREAM_ID = "OlvidStreamId" const val VIDEO_STREAM_ID = "video" const val SCREENCAST_STREAM_ID = "screencast" @@ -153,10 +154,10 @@ class WebrtcPeerConnectionHolder( HashSet(mutableListOf("opus", "PCMU", "PCMA", "telephone-event", "red")) private const val ADDITIONAL_OPUS_OPTIONS = ";cbr=1" // by default send and receive are mono, no need to add "stereo=0;sprop-stereo=0" - const val FIELD_TRIAL_INTEL_VP8 = "WebRTC-IntelVP8/Enabled/" - const val FIELD_TRIAL_H264_HIGH_PROFILE = "WebRTC-H264HighProfile/Enabled/" - const val FIELD_TRIAL_H264_SIMULCAST = "WebRTC-H264Simulcast/Enabled/" - const val FIELD_TRIAL_FLEX_FEC = + private const val FIELD_TRIAL_INTEL_VP8 = "WebRTC-IntelVP8/Enabled/" + private const val FIELD_TRIAL_H264_HIGH_PROFILE = "WebRTC-H264HighProfile/Enabled/" + private const val FIELD_TRIAL_H264_SIMULCAST = "WebRTC-H264Simulcast/Enabled/" + private const val FIELD_TRIAL_FLEX_FEC = "WebRTC-FlexFEC-03/Enabled/WebRTC-FlexFEC-03-Advertised/Enabled/" var eglBase: EglBase? = null var peerConnectionFactory: PeerConnectionFactory? = null @@ -434,7 +435,7 @@ class WebrtcPeerConnectionHolder( } else { screenSender = peerConnection?.addTrack( localScreenTrack, - listOf(OLVID_STREAM_ID, SCREENCAST_STREAM_ID) + listOf(SCREENCAST_STREAM_ID) // screencast does not need to be synchronized with voice ) return true } @@ -525,33 +526,6 @@ class WebrtcPeerConnectionHolder( } } - fun restartIce() { - peerConnection?.let { peerConnection -> - if (peerConnection.signalingState() == HAVE_LOCAL_OFFER) { - // rollback to a stable set before creating the new restart offer - peerConnection.setLocalDescription( - sessionDescriptionObserver, SessionDescription( - ROLLBACK, "" - ) - ) - } else if (peerConnection.signalingState() == HAVE_REMOTE_OFFER) { - // we received a remote offer - // if we are the offer sender, rollback and send a new offer, otherwise juste wait for the answer process to finish - if (webrtcCallService.shouldISendTheOfferToCallParticipant(callParticipant)) { - peerConnection.setLocalDescription( - sessionDescriptionObserver, SessionDescription( - ROLLBACK, "" - ) - ) - } else { - return - } - } - - peerConnection.restartIce() - } - } - ////// // when reading this, have a look at // https://w3c.github.io/webrtc-pc/#rtcsignalingstate-enum @@ -1101,7 +1075,7 @@ class WebrtcPeerConnectionHolder( } override fun onSetSuccess() { - Logger.w("☎ onSetSuccess") + Logger.d("☎ onSetSuccess") // called when local or remote description are set // This automatically triggers ICE gathering or connection establishment --> nothing to do for GATHER_ONCE } diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/CallScreen.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/CallScreen.kt index 8cf2c3df..f7b0df24 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/CallScreen.kt +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/CallScreen.kt @@ -20,12 +20,15 @@ package io.olvid.messenger.webrtc.components import android.annotation.SuppressLint +import android.content.Context import androidx.compose.animation.AnimatedVisibility import androidx.compose.animation.animateColorAsState import androidx.compose.animation.core.EaseOutExpo import androidx.compose.animation.core.animateDpAsState import androidx.compose.animation.core.animateFloatAsState import androidx.compose.animation.core.tween +import androidx.compose.animation.fadeIn +import androidx.compose.animation.fadeOut import androidx.compose.animation.slideInVertically import androidx.compose.animation.slideOutVertically import androidx.compose.foundation.ExperimentalFoundationApi @@ -46,6 +49,7 @@ import androidx.compose.foundation.layout.FlowRow import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.WindowInsetsSides import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth @@ -54,15 +58,17 @@ import androidx.compose.foundation.layout.heightIn import androidx.compose.foundation.layout.imePadding import androidx.compose.foundation.layout.navigationBars import androidx.compose.foundation.layout.offset +import androidx.compose.foundation.layout.only import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.sizeIn import androidx.compose.foundation.layout.statusBars import androidx.compose.foundation.layout.statusBarsPadding +import androidx.compose.foundation.layout.systemBars import androidx.compose.foundation.layout.width import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.layout.windowInsetsPadding import androidx.compose.foundation.lazy.LazyColumn -import androidx.compose.foundation.lazy.LazyRow import androidx.compose.foundation.lazy.items import androidx.compose.foundation.shape.CircleShape import androidx.compose.foundation.shape.RoundedCornerShape @@ -89,7 +95,6 @@ import androidx.compose.runtime.State import androidx.compose.runtime.getValue import androidx.compose.runtime.livedata.observeAsState import androidx.compose.runtime.mutableFloatStateOf -import androidx.compose.runtime.mutableIntStateOf import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope @@ -126,7 +131,6 @@ import androidx.core.view.WindowInsetsControllerCompat import androidx.lifecycle.switchMap import androidx.lifecycle.viewmodel.compose.viewModel import com.google.accompanist.themeadapter.appcompat.AppCompatTheme -import io.olvid.engine.Logger import io.olvid.messenger.App import io.olvid.messenger.AppSingleton import io.olvid.messenger.R @@ -304,12 +308,13 @@ private fun BoxScope.PreCall( status: String = "", cameraEnabled: Boolean = false, videoTrack: VideoTrack?, + mirror: Boolean = true, initialViewSetup: (initialView: InitialView) -> Unit = {} ) { videoTrack?.let { if (cameraEnabled) { - VideoRenderer(modifier = Modifier.fillMaxSize(), videoTrack = it, zoomable = true, mirror = true) + VideoRenderer(modifier = Modifier.fillMaxSize(), videoTrack = it, zoomable = true, mirror = mirror) } } @@ -381,12 +386,14 @@ private fun BoxScope.PreCall( } @Composable -private fun OlvidLogo() { +private fun OlvidLogo( + large: Boolean = false +) { Box( modifier = Modifier - .size(16.dp) + .size(if (large) 48.dp else 16.dp) .background( - shape = RoundedCornerShape(4.dp), + shape = RoundedCornerShape(if (large) 12.dp else 4.dp), brush = Brush.verticalGradient( listOf( colorResource(id = R.color.olvid_gradient_light), @@ -398,7 +405,7 @@ private fun OlvidLogo() { Image( modifier = Modifier .align(Alignment.Center) - .size(14.dp), + .size(if (large) 42.dp else 14.dp), painter = painterResource(id = R.drawable.icon_olvid_no_padding), contentDescription = "Olvid" ) @@ -413,7 +420,7 @@ fun PreCallScreenPreview() { .fillMaxHeight() .background(Color.Black)) { PreCall( - "Alice Barrin", + "Alice Border", "Connexion...", false, null @@ -449,6 +456,9 @@ fun CallScreen( val iAmTheCaller = webrtcCallService?.isCaller ?: false val microphoneMuted = webrtcCallService?.getMicrophoneMuted()?.observeAsState() + val pipAspectCallback: (Context, Int, Int) -> Unit = { contextArg, width, height -> + setPictureInPictureAspectRatio(context = contextArg, width = width, height = height) + } val unfilteredContacts = AppSingleton.getCurrentIdentityLiveData().switchMap { ownedIdentity: OwnedIdentity? -> @@ -484,12 +494,12 @@ fun CallScreen( val insetsController = WindowCompat.getInsetsController(window, window.decorView) insetsController.apply { - if (fullScreenMode) { + systemBarsBehavior = if (fullScreenMode) { hide(WindowInsetsCompat.Type.navigationBars()) - systemBarsBehavior = WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE } else { show(WindowInsetsCompat.Type.navigationBars()) - systemBarsBehavior = WindowInsetsControllerCompat.BEHAVIOR_DEFAULT + WindowInsetsControllerCompat.BEHAVIOR_DEFAULT } } } @@ -540,7 +550,8 @@ fun CallScreen( webrtcCallService, peekHeight, onCallAction, - isPip = true + isPip = true, + pipAspectCallback = pipAspectCallback ) } } else { @@ -595,10 +606,11 @@ fun CallScreen( ) { if (callState?.value == CALL_IN_PROGRESS || callState?.value == CALL_ENDED) { VideoCallContent( - participants, - webrtcCallService, - peekHeight, - onCallAction, + participants = participants, + webrtcCallService = webrtcCallService, + peekHeight = peekHeight, + onCallAction = onCallAction, + pipAspectCallback = pipAspectCallback ) } else { var initialViewSetup: (InitialView) -> Unit by remember { @@ -607,6 +619,8 @@ fun CallScreen( var name by remember { mutableStateOf("") } + val selectedCamera by webrtcCallService.selectedCameraLiveData.observeAsState() + LaunchedEffect(callParticipants) { App.runThread { if (callParticipants?.value.orEmpty().size > 1 && webrtcCallService.bytesGroupOwnerAndUidOrIdentifier != null) { @@ -684,18 +698,20 @@ fun CallScreen( }, initialViewSetup = initialViewSetup, cameraEnabled = webrtcCallService.cameraEnabled, - videoTrack = localVideoTrack?.takeIf { callState?.value != CALL_ENDED } + videoTrack = localVideoTrack?.takeIf { callState?.value != CALL_ENDED }, + mirror = selectedCamera?.mirror == true ) } AnimatedVisibility( modifier = Modifier.align(Alignment.TopEnd), visible = !fullScreenMode, - enter = slideInVertically(), - exit = slideOutVertically() + enter = slideInVertically() + fadeIn(), + exit = slideOutVertically() + fadeOut() ) { SpeakerToggle( modifier = Modifier + .windowInsetsPadding(WindowInsets.systemBars.only(WindowInsetsSides.End)) .padding(top = 10.dp, end = 10.dp), audioOutputs = webrtcCallService.availableAudioOutputs, { audioOutput -> webrtcCallService.selectAudioOutput(audioOutput) @@ -719,7 +735,7 @@ private fun ColumnScope.CallBottomSheetContent( webrtcCallService: WebrtcCallService?, onCallAction: (CallAction) -> Unit, microphoneMuted: State?, - callState: State?, + @Suppress("UNUSED_PARAMETER") callState: State?, contact: Contact?, callDuration: State?, callParticipants: State?>?, @@ -954,158 +970,188 @@ private fun ColumnScope.CallBottomSheetContent( Spacer(modifier = Modifier.height(navigationBarHeight)) } +//@Composable +//fun BoxScope.MultiVideoCallContent( +// participants: List, +// webrtcCallService: WebrtcCallService, +// peekHeight: Dp +//) { +// val selectedCamera by webrtcCallService.selectedCameraLiveData.observeAsState() +// // multi +// Column { +// LazyRow( +// modifier = Modifier.padding( +// start = 16.dp, +// top = 8.dp, +// bottom = 10.dp +// ), +// horizontalArrangement = Arrangement.spacedBy(10.dp) +// ) { +// items(items = participants.filterNot { +// it.bytesContactIdentity.contentEquals( +// webrtcCallService.selectedParticipant +// ) +// }) { callParticipant -> +// val remoteVideoTrack = +// webrtcCallService.getCallParticipant(callParticipant.bytesContactIdentity)?.peerConnectionHolder?.remoteVideoTrack +// Card( +// modifier = Modifier +// .size(72.dp) +// .clickable { +// webrtcCallService.selectedParticipant = +// callParticipant.bytesContactIdentity +// }, +// shape = RoundedCornerShape(12.dp) +// ) { +// CallParticipant( +// callParticipant = callParticipant, +// videoTrack = remoteVideoTrack, +// audioLevel = webrtcCallService.getCallParticipant(callParticipant.bytesContactIdentity)?.peerConnectionHolder?.peerAudioLevel +// ) +// +// } +// +// } +// } +// if (webrtcCallService.selectedParticipant.contentEquals(webrtcCallService.bytesOwnedIdentity!!)) { +// if (webrtcCallService.screenShareActive) { +// ScreenShareOngoing { webrtcCallService.toggleScreenShare() } +// } else { +// CallParticipant( +// modifier = Modifier +// .clip( +// RoundedCornerShape( +// topStart = 20.dp, +// topEnd = 20.dp +// ) +// ) +// .fillMaxSize(), +// bytesOwnedIdentity = webrtcCallService.bytesOwnedIdentity, +// mirror = selectedCamera?.mirror == true, +// videoTrack = localVideoTrack.takeIf { webrtcCallService.cameraEnabled }, +// screenTrack = localScreenTrack.takeIf { webrtcCallService.screenShareActive }, +// zoomable = true, +// audioLevel = webrtcCallService.getAudioLevel(webrtcCallService.bytesOwnedIdentity) +// ) +// } +// } else { +// val remoteVideoTrack = +// webrtcCallService.getCallParticipant(webrtcCallService.selectedParticipant)?.peerConnectionHolder?.remoteVideoTrack +// CallParticipant( +// callParticipant = CallParticipantPojo( +// webrtcCallService.getCallParticipant( +// webrtcCallService.selectedParticipant +// )!! +// ), videoTrack = remoteVideoTrack, +// zoomable = true, +// audioLevel = webrtcCallService.getAudioLevel(webrtcCallService.selectedParticipant) +// ) +// } +// } +// if (webrtcCallService.selectedParticipant.contentEquals(webrtcCallService.bytesOwnedIdentity!!) +// .not() +// ) { +// Card( +// modifier = Modifier +// .size(120.dp) +// .align(Alignment.BottomEnd) +// .offset(y = -peekHeight) +// .clickable { +// webrtcCallService.selectedParticipant = +// webrtcCallService.bytesOwnedIdentity!! +// } +// .padding(end = 16.dp, bottom = 8.dp), +// shape = RoundedCornerShape(20.dp) +// ) { +// CallParticipant( +// bytesOwnedIdentity = webrtcCallService.bytesOwnedIdentity, +// mirror = selectedCamera?.mirror == true, +// videoTrack = localVideoTrack.takeIf { webrtcCallService.cameraEnabled }, +// screenTrack = localScreenTrack.takeIf { webrtcCallService.screenShareActive }, +// audioLevel = webrtcCallService.getAudioLevel(webrtcCallService.bytesOwnedIdentity) +// ) +// } +// } +//} + @Composable -fun BoxScope.MultiVideoCallContent( +@OptIn(ExperimentalFoundationApi::class) +private fun AudioCallContent( participants: List, webrtcCallService: WebrtcCallService, - peekHeight: Dp + onCallAction: (CallAction) -> Unit, + isPip: Boolean = false ) { - val selectedCamera by webrtcCallService.selectedCameraLiveData.observeAsState() - // multi - Column { - LazyRow( - modifier = Modifier.padding( - start = 16.dp, - top = 8.dp, - bottom = 10.dp - ), - horizontalArrangement = Arrangement.spacedBy(10.dp) + if (isPip) { + Column( + modifier = Modifier.fillMaxSize(), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.Center ) { - items(items = participants.filterNot { - it.bytesContactIdentity.contentEquals( - webrtcCallService.selectedParticipant + OlvidLogo(large = true) + Spacer(modifier = Modifier.height(32.dp)) + Row (verticalAlignment = Alignment.CenterVertically) + { + Image( + modifier = Modifier.size(32.dp), + painter = painterResource(id = R.drawable.ic_phone_outgoing), + contentDescription = stringResource(id = R.string.text_ongoing_call) ) - }) { callParticipant -> - val remoteVideoTrack = - webrtcCallService.getCallParticipant(callParticipant.bytesContactIdentity)?.peerConnectionHolder?.remoteVideoTrack - Card( - modifier = Modifier - .size(72.dp) - .clickable { - webrtcCallService.selectedParticipant = - callParticipant.bytesContactIdentity - }, - shape = RoundedCornerShape(12.dp) - ) { - CallParticipant( - callParticipant = callParticipant, - videoTrack = remoteVideoTrack, - audioLevel = webrtcCallService.getCallParticipant(callParticipant.bytesContactIdentity)?.peerConnectionHolder?.peerAudioLevel - ) - - } - - } - } - if (webrtcCallService.selectedParticipant.contentEquals(webrtcCallService.bytesOwnedIdentity!!)) { - if (webrtcCallService.screenShareActive) { - ScreenShareOngoing { webrtcCallService.toggleScreenShare() } - } else { - CallParticipant( - modifier = Modifier - .clip( - RoundedCornerShape( - topStart = 20.dp, - topEnd = 20.dp - ) - ) - .fillMaxSize(), - bytesOwnedIdentity = webrtcCallService.bytesOwnedIdentity, - mirror = selectedCamera?.mirror == true, - videoTrack = localVideoTrack.takeIf { webrtcCallService.cameraEnabled }, - screenTrack = localScreenTrack.takeIf { webrtcCallService.screenShareActive }, - zoomable = true, - audioLevel = webrtcCallService.getAudioLevel(webrtcCallService.bytesOwnedIdentity) + Spacer(modifier = Modifier.width(16.dp)) + Text( + text = participants.size.toString(), + fontSize = 32.sp, + fontWeight = FontWeight.Medium, + color = Color.White ) } - } else { - val remoteVideoTrack = - webrtcCallService.getCallParticipant(webrtcCallService.selectedParticipant)?.peerConnectionHolder?.remoteVideoTrack - CallParticipant( - callParticipant = CallParticipantPojo( - webrtcCallService.getCallParticipant( - webrtcCallService.selectedParticipant - )!! - ), videoTrack = remoteVideoTrack, - zoomable = true, - audioLevel = webrtcCallService.getAudioLevel(webrtcCallService.selectedParticipant) - ) } - } - if (webrtcCallService.selectedParticipant.contentEquals(webrtcCallService.bytesOwnedIdentity!!) - .not() - ) { - Card( - modifier = Modifier - .size(120.dp) - .align(Alignment.BottomEnd) - .offset(y = -peekHeight) - .clickable { - webrtcCallService.selectedParticipant = - webrtcCallService.bytesOwnedIdentity!! - } - .padding(end = 16.dp, bottom = 8.dp), - shape = RoundedCornerShape(20.dp) + } else { + val haptics = LocalHapticFeedback.current + LazyColumn( + modifier = Modifier.padding(top = 4.dp), + verticalArrangement = Arrangement.spacedBy(4.dp) ) { - CallParticipant( - bytesOwnedIdentity = webrtcCallService.bytesOwnedIdentity, - mirror = selectedCamera?.mirror == true, - videoTrack = localVideoTrack.takeIf { webrtcCallService.cameraEnabled }, - screenTrack = localScreenTrack.takeIf { webrtcCallService.screenShareActive }, - audioLevel = webrtcCallService.getAudioLevel(webrtcCallService.bytesOwnedIdentity) - ) - } - } -} - -@Composable -@OptIn(ExperimentalFoundationApi::class) -private fun AudioCallContent( - participants: List, - webrtcCallService: WebrtcCallService, - onCallAction: (CallAction) -> Unit -) { - val haptics = LocalHapticFeedback.current - LazyColumn(verticalArrangement = Arrangement.spacedBy(4.dp)) { - items(participants) { audioParticipant -> - var menu by remember { - mutableStateOf(false) - } - Box { - DropdownMenu( - expanded = menu, - onDismissRequest = { menu = false }) { - DropdownMenuItem(onClick = { - menu = false - webrtcCallService.callerKickParticipant( - audioParticipant.bytesContactIdentity - ) - }) { - Text(text = stringResource(id = R.string.dialog_title_webrtc_kick_participant)) - } + items(participants) { audioParticipant -> + var menu by remember { + mutableStateOf(false) } - AudioParticipant(modifier = Modifier - .padding(horizontal = 24.dp) - .combinedClickable( - onClick = { - if (audioParticipant.contact?.oneToOne == true) { - onCallAction(GoToDiscussion(audioParticipant.contact)) - } - }, - onLongClick = { - if (webrtcCallService.isCaller && participants.size > 1) { - haptics.performHapticFeedback( - HapticFeedbackType.LongPress - ) - menu = true - } + Box { + DropdownMenu( + expanded = menu, + onDismissRequest = { menu = false }) { + DropdownMenuItem(onClick = { + menu = false + webrtcCallService.callerKickParticipant( + audioParticipant.bytesContactIdentity + ) + }) { + Text(text = stringResource(id = R.string.dialog_title_webrtc_kick_participant)) } - ), - initialViewSetup = audioParticipant.initialViewSetup(), - name = audioParticipant.displayName ?: "", - isMute = audioParticipant.peerIsMuted, - state = audioParticipant.peerState, - audioLevel = webrtcCallService.getAudioLevel(audioParticipant.bytesContactIdentity)) + } + AudioParticipant(modifier = Modifier + .padding(horizontal = 4.dp) + .combinedClickable( + onClick = { + if (audioParticipant.contact?.oneToOne == true) { + onCallAction(GoToDiscussion(audioParticipant.contact)) + } + }, + onLongClick = { + if (webrtcCallService.isCaller && participants.size > 1) { + haptics.performHapticFeedback( + HapticFeedbackType.LongPress + ) + menu = true + } + } + ), + initialViewSetup = audioParticipant.initialViewSetup(), + name = audioParticipant.displayName ?: "", + isMute = audioParticipant.peerIsMuted, + state = audioParticipant.peerState, + audioLevel = webrtcCallService.getAudioLevel(audioParticipant.bytesContactIdentity)) + } } } } @@ -1118,7 +1164,8 @@ private fun VideoCallContent( webrtcCallService: WebrtcCallService, peekHeight: Dp, onCallAction: (CallAction) -> Unit, - isPip: Boolean = false + isPip: Boolean = false, + pipAspectCallback: ((Context, Int, Int) -> Unit)? = null ) { val selectedCamera by webrtcCallService.selectedCameraLiveData.observeAsState() val speakingColor = colorResource(id = R.color.olvid_gradient_light) @@ -1148,17 +1195,18 @@ private fun VideoCallContent( modifier = Modifier.fillMaxSize(), zoomable = true, audioLevel = webrtcCallService.getAudioLevel(participants.firstOrNull()?.bytesContactIdentity), - isPip = isPip + isPip = isPip, + pipAspectCallback = pipAspectCallback ) AnimatedVisibility(!isPip) { Card( modifier = Modifier - .size(120.dp) + .windowInsetsPadding(WindowInsets.systemBars.only(WindowInsetsSides.Start)) + .padding(start = 10.dp, top = 10.dp) .clickable { webrtcCallService.selectedParticipant = webrtcCallService.bytesOwnedIdentity!! } - .padding(start = 10.dp, top = 10.dp) .border( width = 2.dp, color = borderColorOwned, @@ -1167,6 +1215,7 @@ private fun VideoCallContent( shape = RoundedCornerShape(16.dp) ) { CallParticipant( + modifier = Modifier.sizeIn(maxWidth = 120.dp, maxHeight = 120.dp), bytesOwnedIdentity = webrtcCallService.bytesOwnedIdentity, mirror = selectedCamera?.mirror == true, videoTrack = localVideoTrack.takeIf { webrtcCallService.cameraEnabled }, @@ -1187,25 +1236,28 @@ private fun VideoCallContent( zoomable = true, modifier = Modifier.fillMaxSize(), audioLevel = webrtcCallService.getAudioLevel(webrtcCallService.bytesOwnedIdentity), - isPip = isPip + isPip = isPip, + fitVideo = true ) } AnimatedVisibility(!isPip) { Card( modifier = Modifier - .size(120.dp) + .windowInsetsPadding(WindowInsets.systemBars.only(WindowInsetsSides.Start)) + .padding(start = 10.dp, top = 10.dp) .clickable { webrtcCallService.selectedParticipant = participants.firstOrNull()?.bytesContactIdentity!! - } - .padding(start = 10.dp, top = 10.dp), + }, shape = RoundedCornerShape(16.dp) ) { CallParticipant( + modifier = Modifier.sizeIn(maxWidth = 120.dp, maxHeight = 120.dp), callParticipant = participants.firstOrNull(), videoTrack = remoteVideoTrack, screenTrack = remoteScreenTrack, - audioLevel = webrtcCallService.getAudioLevel(participants.firstOrNull()?.bytesContactIdentity) + audioLevel = webrtcCallService.getAudioLevel(participants.firstOrNull()?.bytesContactIdentity), + pipAspectCallback = pipAspectCallback ) } } @@ -1225,6 +1277,10 @@ private fun VideoCallContent( label = "borderColor", animationSpec = tween(durationMillis = 1000, easing = EaseOutExpo) ) + val context = LocalContext.current + LaunchedEffect(Unit) { + pipAspectCallback?.invoke(context, 9, 16) + } BoxWithConstraints( modifier = Modifier .fillMaxSize() @@ -1301,7 +1357,7 @@ private fun VideoCallContent( AnimatedVisibility(!isPip) { Card( modifier = Modifier - .size(120.dp) + .windowInsetsPadding(WindowInsets.systemBars.only(WindowInsetsSides.Start)) .padding(start = 10.dp, top = 10.dp) .border( width = 2.dp, @@ -1311,6 +1367,7 @@ private fun VideoCallContent( shape = RoundedCornerShape(16.dp) ) { CallParticipant( + modifier = Modifier.sizeIn(maxWidth = 120.dp, maxHeight = 120.dp), bytesOwnedIdentity = webrtcCallService.bytesOwnedIdentity, mirror = selectedCamera?.mirror == true, videoTrack = localVideoTrack.takeIf { webrtcCallService.cameraEnabled }, @@ -1319,10 +1376,15 @@ private fun VideoCallContent( } } } else if (participants.size > 2) { + val context = LocalContext.current + LaunchedEffect(Unit) { + pipAspectCallback?.invoke(context, 9, 16) + } AudioCallContent( participants = participants, webrtcCallService = webrtcCallService, - onCallAction = onCallAction + onCallAction = onCallAction, + isPip = isPip ) //MultiVideoCallContent(participants = participants, webrtcCallService = webrtcCallService, peekHeight = peekHeight) } @@ -1346,8 +1408,11 @@ fun CallParticipant( zoomable: Boolean = false, peekHeight: Dp = 0.dp, audioLevel: Double?, - isPip: Boolean = false + isPip: Boolean = false, + pipAspectCallback: ((Context, Int, Int) -> Unit)? = null, + fitVideo: Boolean = false ) { + BoxWithConstraints(modifier = modifier) { val largeLayout = maxWidth > 200.dp val hasVideo = callParticipant?.peerVideoSharing != false && videoTrack.isEnabledSafe() @@ -1359,37 +1424,42 @@ fun CallParticipant( modifier = Modifier.fillMaxSize(), videoTrack = screenTrack!!, zoomable = zoomable, - mirror = false + mirror = false, + pipAspectCallback = pipAspectCallback, + fitVideo = true ) if (hasVideo) { Card( modifier = Modifier .align(Alignment.BottomStart) + .windowInsetsPadding(WindowInsets.systemBars.only(WindowInsetsSides.Start)) .then( if (isPip) Modifier .offset(x = 4.dp, y = (-4).dp) - .size(60.dp) + .sizeIn(maxWidth = 60.dp, maxHeight = 60.dp) else Modifier .offset(x = 10.dp, y = -(peekHeight + 10.dp)) - .size(120.dp) + .sizeIn(maxWidth = 120.dp, maxHeight = 120.dp) ), shape = RoundedCornerShape(16.dp) ) { VideoRenderer( - modifier = Modifier.fillMaxSize(), videoTrack = videoTrack!!, - mirror = mirror + mirror = mirror, + matchVideoAspectRatio = true ) } } } else { VideoRenderer( - modifier = Modifier.fillMaxSize(), videoTrack = videoTrack!!, zoomable = zoomable, - mirror = mirror + mirror = mirror, + matchVideoAspectRatio = !largeLayout, + pipAspectCallback = pipAspectCallback, + fitVideo = fitVideo ) } callParticipant?.peerState.takeUnless { it == CONNECTED }?.let { @@ -1410,6 +1480,10 @@ fun CallParticipant( ) } } else { + val context = LocalContext.current + LaunchedEffect(Unit) { + pipAspectCallback?.invoke(context, 9, 16) + } val radius by animateFloatAsState( targetValue = audioLevel?.toFloat() ?: 0f, animationSpec = tween(durationMillis = 600), @@ -1491,8 +1565,29 @@ fun CallParticipant( ))?.let { name -> Text( modifier = Modifier - .align(Alignment.BottomStart) - .padding(start = 8.dp, bottom = peekHeight + if (hasVideo && hasScreenShare) 128.dp else 8.dp) + .then( + if (hasVideo && hasScreenShare) + Modifier + .align(Alignment.BottomEnd) + .windowInsetsPadding( + WindowInsets.systemBars.only( + WindowInsetsSides.End + ) + ) + else + Modifier + .align(Alignment.BottomStart) + .windowInsetsPadding( + WindowInsets.systemBars.only( + WindowInsetsSides.Start + ) + ) + ) + .padding( + start = 6.dp, + bottom = peekHeight + 6.dp, + end = 6.dp + ) .background( colorResource(id = R.color.blackOverlay), RoundedCornerShape(12.dp) @@ -1727,16 +1822,17 @@ fun AudioParticipant( color = Color.LightGray ) } - AnimatedVisibility(visible = isMute) { + if(isMute) { Icon( modifier = Modifier - .size(20.dp), + .size(32.dp) + .background(colorResource(id = R.color.red), CircleShape) + .padding(4.dp), painter = painterResource(id = R.drawable.ic_microphone_off), tint = Color.White, contentDescription = "muted" ) - } - AnimatedVisibility(visible = isMute.not()) { + } else { AudioLevel( modifier = Modifier.size(32.dp), audioLevel = audioLevel diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/CustomEglRenderer.java b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/CustomEglRenderer.java new file mode 100644 index 00000000..6fa738b8 --- /dev/null +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/CustomEglRenderer.java @@ -0,0 +1,533 @@ +/* + * Olvid for Android + * Copyright © 2019-2024 Olvid SAS + * + * This file is part of Olvid for Android. + * + * Olvid is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * Olvid is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Olvid. If not, see . + */ + +package io.olvid.messenger.webrtc.components; + + +import android.graphics.Matrix; +import android.graphics.SurfaceTexture; +import android.opengl.GLES20; +import android.view.Surface; + +import androidx.annotation.GuardedBy; +import androidx.annotation.Nullable; + +import org.webrtc.EglBase; +import org.webrtc.EglThread; +import org.webrtc.GlTextureFrameBuffer; +import org.webrtc.GlUtil; +import org.webrtc.RendererCommon; +import org.webrtc.ThreadUtils; +import org.webrtc.VideoFrame; +import org.webrtc.VideoFrameDrawer; +import org.webrtc.VideoSink; + +import java.util.concurrent.CountDownLatch; + +/** + * This class is an adaptation of the EglRenderer bundled in webrtc. + * Compared to the original one, it can handle pan and zoom through the ScaleAndOffsetControl + */ +public class CustomEglRenderer implements VideoSink, ScaleAndOffsetControl.ScaleAndOffsetControlListener { + private class EglSurfaceCreation implements Runnable { + private Object surface; + + public synchronized void setSurface(Object surface) { + this.surface = surface; + } + + @Override + public synchronized void run() { + if (surface != null && eglBase != null && !eglBase.hasSurface()) { + if (surface instanceof Surface) { + eglBase.createSurface((Surface) surface); + } else if (surface instanceof SurfaceTexture) { + eglBase.createSurface((SurfaceTexture) surface); + } else { + throw new IllegalStateException("Invalid surface: " + surface); + } + eglBase.makeCurrent(); + // Necessary for YUV frames with odd width. + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); + } + } + } + + protected final String name; + + // `eglThread` is used for rendering, and is synchronized on `threadLock`. + private final Object threadLock = new Object(); + @GuardedBy("threadLock") @Nullable private EglThread eglThread; + + private final Runnable eglExceptionCallback = new Runnable() { + @Override + public void run() { + synchronized (threadLock) { + eglThread = null; + } + } + }; + + + // EGL and GL resources for drawing YUV/OES textures. After initialization, these are only + // accessed from the render thread. + @Nullable private EglBase eglBase; + private final VideoFrameDrawer frameDrawer; + @Nullable private RendererCommon.GlDrawer drawer; + private boolean usePresentationTimeStamp; + private final Matrix drawMatrix = new Matrix(); + + // Pending frame to render. Serves as a queue with size 1. Synchronized on `frameLock`. + private final Object frameLock = new Object(); + @Nullable private VideoFrame pendingFrame; + + // These variables are synchronized on `layoutLock`. + private final Object layoutLock = new Object(); + private float layoutAspectRatio; + // If true, mirrors the video stream horizontally. + private boolean mirrorHorizontally; + // If true, mirrors the video stream vertically. + private boolean mirrorVertically; + + + // Used for bitmap capturing. + private final GlTextureFrameBuffer bitmapTextureFramebuffer = + new GlTextureFrameBuffer(GLES20.GL_RGBA); + + private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation(); + + /** + * Standard constructor. The name will be included when logging. In order to render something, + * you must first call init() and createEglSurface. + */ + public CustomEglRenderer(String name, @Nullable ScaleAndOffsetControl scaleAndOffsetControl) { + this.name = name; + this.frameDrawer = new VideoFrameDrawer(); + if (scaleAndOffsetControl != null) { + scaleAndOffsetControl.setListener(this); + } + } + + public void init( + EglThread eglThread, RendererCommon.GlDrawer drawer, boolean usePresentationTimeStamp) { + synchronized (threadLock) { + if (this.eglThread != null) { + throw new IllegalStateException(name + "Already initialized"); + } + + this.eglThread = eglThread; + this.drawer = drawer; + this.usePresentationTimeStamp = usePresentationTimeStamp; + + eglThread.addExceptionCallback(eglExceptionCallback); + + eglBase = eglThread.createEglBaseWithSharedConnection(); + eglThread.getHandler().post(eglSurfaceCreationRunnable); + } + } + + /** + * Initialize this class, sharing resources with `sharedContext`. The custom `drawer` will be used + * for drawing frames on the EGLSurface. This class is responsible for calling release() on + * `drawer`. It is allowed to call init() to reinitialize the renderer after a previous + * init()/release() cycle. If usePresentationTimeStamp is true, eglPresentationTimeANDROID will be + * set with the frame timestamps, which specifies desired presentation time and might be useful + * for e.g. syncing audio and video. + */ + public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes, + RendererCommon.GlDrawer drawer, boolean usePresentationTimeStamp) { + EglThread thread = + EglThread.create(/* releaseMonitor= */ null, sharedContext, configAttributes); + init(thread, drawer, usePresentationTimeStamp); + } + + /** + * Same as above with usePresentationTimeStamp set to false. + * + * @see #init(EglBase.Context, int[], RendererCommon.GlDrawer, boolean) + */ + public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes, + RendererCommon.GlDrawer drawer) { + init(sharedContext, configAttributes, drawer, /* usePresentationTimeStamp= */ false); + } + + public void createEglSurface(Surface surface) { + createEglSurfaceInternal(surface); + } + + public void createEglSurface(SurfaceTexture surfaceTexture) { + createEglSurfaceInternal(surfaceTexture); + } + + private void createEglSurfaceInternal(Object surface) { + eglSurfaceCreationRunnable.setSurface(surface); + postToRenderThread(eglSurfaceCreationRunnable); + } + + /** + * Block until any pending frame is returned and all GL resources released, even if an interrupt + * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function + * should be called before the Activity is destroyed and the EGLContext is still valid. If you + * don't call this function, the GL resources might leak. + */ + public void release() { + final CountDownLatch eglCleanupBarrier = new CountDownLatch(1); + synchronized (threadLock) { + if (eglThread == null) { + return; + } + eglThread.removeExceptionCallback(eglExceptionCallback); + + // Release EGL and GL resources on render thread. + eglThread.getHandler().postAtFrontOfQueue(() -> { + // Detach current shader program. + synchronized (EglBase.lock) { + GLES20.glUseProgram(/* program= */ 0); + } + if (drawer != null) { + drawer.release(); + drawer = null; + } + frameDrawer.release(); + bitmapTextureFramebuffer.release(); + + if (eglBase != null) { + eglBase.detachCurrent(); + eglBase.release(); + eglBase = null; + } + + eglCleanupBarrier.countDown(); + }); + + // Don't accept any more frames or messages to the render thread. + eglThread.release(); + eglThread = null; + } + // Make sure the EGL/GL cleanup posted above is executed. + ThreadUtils.awaitUninterruptibly(eglCleanupBarrier); + synchronized (frameLock) { + if (pendingFrame != null) { + pendingFrame.release(); + pendingFrame = null; + } + } + } + + + /** + * Set if the video stream should be mirrored horizontally or not. + */ + public void setMirror(final boolean mirror) { + synchronized (layoutLock) { + this.mirrorHorizontally = mirror; + } + } + + /** + * Set if the video stream should be mirrored vertically or not. + */ + public void setMirrorVertically(final boolean mirrorVertically) { + synchronized (layoutLock) { + this.mirrorVertically = mirrorVertically; + } + } + + /** + * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video. + * Set this to 0 to disable cropping. + */ + public void setLayoutAspectRatio(float layoutAspectRatio) { + synchronized (layoutLock) { + this.layoutAspectRatio = layoutAspectRatio; + } + } + + + + // VideoSink interface. + @Override + public void onFrame(VideoFrame frame) { + synchronized (threadLock) { + if (eglThread == null) { + return; + } + synchronized (frameLock) { + if (pendingFrame != null) { + pendingFrame.release(); + } + pendingFrame = frame; + pendingFrame.retain(); + eglThread.getHandler().post(this::renderFrameOnRenderThread); + } + } + } + + /** + * Release EGL surface. This function will block until the EGL surface is released. + */ + public void releaseEglSurface(final Runnable completionCallback) { + // Ensure that the render thread is no longer touching the Surface before returning from this + // function. + eglSurfaceCreationRunnable.setSurface(null /* surface */); + synchronized (threadLock) { + if (eglThread != null) { + eglThread.getHandler().removeCallbacks(eglSurfaceCreationRunnable); + eglThread.getHandler().postAtFrontOfQueue(() -> { + if (eglBase != null) { + eglBase.detachCurrent(); + eglBase.releaseSurface(); + } + completionCallback.run(); + }); + return; + } + } + completionCallback.run(); + } + + /** + * Private helper function to post tasks safely. + */ + private void postToRenderThread(Runnable runnable) { + synchronized (threadLock) { + if (eglThread != null) { + eglThread.getHandler().post(runnable); + } + } + } + + private void clearSurfaceOnRenderThread(float r, float g, float b, float a) { + if (eglBase != null && eglBase.hasSurface()) { + eglBase.makeCurrent(); + GLES20.glClearColor(r, g, b, a); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + eglBase.swapBuffers(); + } + } + + /** + * Post a task to clear the surface to a transparent uniform color. + */ + public void clearImage() { + clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); + } + + /** + * Post a task to clear the surface to a specific color. + */ + public void clearImage(final float r, final float g, final float b, final float a) { + synchronized (threadLock) { + if (eglThread == null) { + return; + } + eglThread.getHandler().postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a)); + } + } + + private void swapBuffersOnRenderThread(final VideoFrame frame) { + synchronized (threadLock) { + if (eglThread != null) { + eglThread.scheduleRenderUpdate( + runsInline -> { + if (!runsInline) { + if (eglBase == null || !eglBase.hasSurface()) { + return; + } + eglBase.makeCurrent(); + } + + if (usePresentationTimeStamp) { + eglBase.swapBuffers(frame.getTimestampNs()); + } else { + eglBase.swapBuffers(); + } + }); + } + } + } + + /** + * Renders and releases `pendingFrame`. + */ + private void renderFrameOnRenderThread() { + // Fetch and render `pendingFrame`. + final VideoFrame frame; + synchronized (frameLock) { + if (pendingFrame == null) { + return; + } + frame = pendingFrame; + pendingFrame = null; + } + if (eglBase == null || !eglBase.hasSurface()) { + frame.release(); + return; + } + eglBase.makeCurrent(); + + final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight(); + final float offsetRatioX; + final float offsetRatioY; + final float scaleX; + final float scaleY; + final int gapX; + final int gapY; + synchronized (layoutLock) { + if (lastFrameRatio - frameAspectRatio > 0.05 + || frameAspectRatio - lastFrameRatio > 0.05 + || lastEglWidth != eglBase.surfaceWidth() + || lastEglHeight != eglBase.surfaceHeight()) { + lastFrameRatio = frameAspectRatio; + lastEglWidth = eglBase.surfaceWidth(); + lastEglHeight = eglBase.surfaceHeight(); + coerceZoomAndOffset(); + } + offsetRatioX = this.offsetX/lastEglWidth; + offsetRatioY = this.offsetY/lastEglHeight; + scaleX = this.scaleX; + scaleY = this.scaleY; + gapX = (int) (this.gapRatioX*lastEglWidth); + gapY = (int) (this.gapRatioY*lastEglHeight); + } + + drawMatrix.reset(); + drawMatrix.preTranslate(0.5f, 0.5f); + drawMatrix.preScale(mirrorHorizontally ? -1f : 1f, mirrorVertically ? -1f : 1f); + drawMatrix.preScale(scaleX, scaleY); + drawMatrix.preTranslate(-0.5f - offsetRatioX, -0.5f + offsetRatioY); + + + try { + GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + frameDrawer.drawFrame(frame, drawer, drawMatrix, gapX, gapY, lastEglWidth - 2*gapX, lastEglHeight - 2*gapY); + + swapBuffersOnRenderThread(frame); + } catch (GlUtil.GlOutOfMemoryException e) { + // Attempt to free up some resources. + drawer.release(); + frameDrawer.release(); + bitmapTextureFramebuffer.release(); + // Continue here on purpose and retry again for next frame. In worst case, this is a + // continuous problem and no more frames will be drawn. + } finally { + frame.release(); + } + } + + float lastFrameRatio = 1f; + int lastEglWidth = 0; + int lastEglHeight = 0; + + /** + * Set to true to fit the whole image, false to fill, null for custom zoom and offset + */ + Boolean fit = false; + + /** + * 1 is the fill zoom + */ + float zoom = 1; + + /** + * Offsets are in pixels, and are converted to frame percent during frame render matrix computation + */ + float offsetX = 0; + float offsetY = 0; + + float gapRatioX = 0f; + float gapRatioY = 0f; + + float scaleX = 1f; + float scaleY = 1f; + + @Override + public void onFit() { + synchronized (layoutLock) { + fit = true; + zoom = (layoutAspectRatio > lastFrameRatio) ? lastFrameRatio / layoutAspectRatio : layoutAspectRatio / lastFrameRatio; + offsetX = 0; + offsetY = 0; + coerceZoomAndOffset(); + } + } + + @Override + public void onFill() { + synchronized (layoutLock) { + fit = false; + zoom = 1; + offsetX = 0; + offsetY = 0; + coerceZoomAndOffset(); + } + } + + @Override + public void onTransformation(float zoomChange, float offsetChangeX, float offsetChangeY) { + synchronized (layoutLock) { + fit = null; + zoom *= zoomChange; + offsetX += offsetChangeX; + offsetY += offsetChangeY; + coerceZoomAndOffset(); + } + } + + private void coerceZoomAndOffset() { + synchronized (layoutLock) { + if (fit != null) { + if (fit) { + zoom = (layoutAspectRatio > lastFrameRatio) ? lastFrameRatio / layoutAspectRatio : layoutAspectRatio / lastFrameRatio; + scaleX = layoutAspectRatio / lastFrameRatio / zoom; + scaleY = Math.min(1f/zoom, 1); + } else { + zoom = 1; + offsetX = 0; + offsetY = 0; + } + } + + float minZoom = (layoutAspectRatio > lastFrameRatio) ? lastFrameRatio / layoutAspectRatio : layoutAspectRatio / lastFrameRatio; + if (zoom < minZoom) { zoom = minZoom; } + if (zoom > 3f) { zoom = 3f; } + + if (lastFrameRatio > layoutAspectRatio) { + scaleX = layoutAspectRatio / lastFrameRatio / zoom; + scaleY = Math.min(1f/zoom, 1); + gapRatioX = 0; + gapRatioY = Math.max(0, .5f-.5f*zoom); + float maxOffsetX = (1 - minZoom/zoom)*lastEglWidth/2; + if (offsetX > maxOffsetX) { offsetX = maxOffsetX; } + if (offsetX < -maxOffsetX) { offsetX = -maxOffsetX; } + offsetY = 0; + } else { + scaleX = Math.min(1f/zoom, 1); + scaleY = lastFrameRatio / layoutAspectRatio / zoom; + gapRatioX = Math.max(0, .5f-.5f*zoom); + gapRatioY = 0; + offsetX = 0; + float maxOffsetY = (1 - minZoom/zoom)*lastEglHeight/2; + if (offsetY > maxOffsetY) { offsetY = maxOffsetY; } + if (offsetY < -maxOffsetY) { offsetY = -maxOffsetY; } + } + } + } +} diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/PictureInPicture.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/PictureInPicture.kt index 64109193..cd02158f 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/PictureInPicture.kt +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/PictureInPicture.kt @@ -33,9 +33,8 @@ import io.olvid.messenger.R internal fun enterPictureInPicture(context: Context) { if (context.packageManager.hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE)) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val aspect = Rational(9, 16) // TODO: use incoming video aspect ratio? val params = PictureInPictureParams.Builder() - .setAspectRatio(aspect).apply { + .setAspectRatio(pipAspect).apply { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { setAutoEnterEnabled(false) } @@ -51,6 +50,17 @@ internal fun enterPictureInPicture(context: Context) { } } +internal var pipAspect = Rational(9, 16) + +internal fun setPictureInPictureAspectRatio(context: Context, width: Int, height: Int) { + pipAspect = Rational(width, height) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + context.findActivity()?.setPictureInPictureParams(PictureInPictureParams.Builder() + .setAspectRatio(pipAspect) + .build()) + } +} + internal val Context.isInPictureInPictureMode: Boolean get() { val currentActivity = findActivity() diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/ScaleAndOffsetControl.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/ScaleAndOffsetControl.kt new file mode 100644 index 00000000..549a6c02 --- /dev/null +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/ScaleAndOffsetControl.kt @@ -0,0 +1,45 @@ +/* + * Olvid for Android + * Copyright © 2019-2024 Olvid SAS + * + * This file is part of Olvid for Android. + * + * Olvid is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * Olvid is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Olvid. If not, see . + */ + +package io.olvid.messenger.webrtc.components + +import androidx.compose.ui.geometry.Offset + + +class ScaleAndOffsetControl { + interface ScaleAndOffsetControlListener { + fun onFit() + fun onFill() + fun onTransformation(zoomChange: Float, offsetChangeX: Float, offsetChangeY: Float) + } + + var listener: ScaleAndOffsetControlListener? = null + + fun setFit() { + listener?.onFit() + } + + fun setFill() { + listener?.onFill() + } + + fun applyTransformation(zoomChange: Float, offsetChange: Offset) { + listener?.onTransformation(zoomChange, offsetChange.x, offsetChange.y) + } +} \ No newline at end of file diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/VideoRenderer.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/VideoRenderer.kt index ca03a3ef..2330ce93 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/VideoRenderer.kt +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/VideoRenderer.kt @@ -19,9 +19,11 @@ package io.olvid.messenger.webrtc.components +import android.content.Context import androidx.compose.foundation.gestures.rememberTransformableState import androidx.compose.foundation.gestures.transformable -import androidx.compose.foundation.layout.BoxWithConstraints +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.aspectRatio import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.MutableState @@ -31,11 +33,7 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier -import androidx.compose.ui.geometry.Offset -import androidx.compose.ui.graphics.graphicsLayer -import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.viewinterop.AndroidView -import io.olvid.engine.Logger import io.olvid.messenger.webrtc.WebrtcPeerConnectionHolder import org.webrtc.RendererCommon import org.webrtc.VideoTrack @@ -51,9 +49,11 @@ fun VideoRenderer( modifier: Modifier = Modifier, videoTrack: VideoTrack, zoomable: Boolean = false, - mirror: Boolean = false + mirror: Boolean = false, + matchVideoAspectRatio: Boolean = false, + pipAspectCallback: ((Context, Int, Int) -> Unit)? = null, + fitVideo: Boolean = false ) { - val density = LocalDensity.current val trackState: MutableState = remember { mutableStateOf(null) } var view: VideoTextureViewRenderer? by remember { mutableStateOf(null) } @@ -62,30 +62,18 @@ fun VideoRenderer( cleanTrack(view, trackState) } } - var scale by remember { mutableFloatStateOf(1f) } - var offset by remember { mutableStateOf(Offset.Zero) } - BoxWithConstraints(modifier = modifier) { - val width = with(density) { - maxWidth.toPx() - } - val height = with(density) { - maxHeight.toPx() - } - - val maxX = (scale - 1) * width / 2 - val maxY = (scale - 1) * height / 2 + val scaleAndOffsetControl = remember { ScaleAndOffsetControl() } + var videoAspectRatio by remember { mutableFloatStateOf(1f) } + Box(modifier = modifier) { val state = rememberTransformableState { zoomChange, offsetChange, _ -> - scale = (scale * zoomChange).coerceIn(1f, 3f) - offset = Offset( - x = (offset.x + offsetChange.x * scale).coerceIn(-maxX, maxX), - y = (offset.y + offsetChange.y * scale).coerceIn(-maxY, maxY) - ) + scaleAndOffsetControl.applyTransformation(zoomChange = zoomChange, offsetChange = offsetChange) } + AndroidView( factory = { context -> - VideoTextureViewRenderer(context).apply { + VideoTextureViewRenderer(context, scaleAndOffsetControl).apply { WebrtcPeerConnectionHolder.eglBase?.eglBaseContext?.let { init( it, @@ -93,14 +81,19 @@ fun VideoRenderer( object : RendererCommon.RendererEvents { override fun onFirstFrameRendered() = Unit - override fun onFrameResolutionChanged(p0: Int, p1: Int, p2: Int) { - // TODO: adjust min scale with this - Logger.d("onFrameResolutionChanged ${p0}x${p1} - $p2") + override fun onFrameResolutionChanged(width: Int, height: Int, rotation: Int) { + videoAspectRatio = width / height.toFloat() + pipAspectCallback?.invoke(context, width, height) } } ) } setupVideo(trackState, videoTrack, this) + if (fitVideo) { + scaleAndOffsetControl.setFit() + } else { + scaleAndOffsetControl.setFill() + } view = this } }, @@ -108,18 +101,16 @@ fun VideoRenderer( setupVideo(trackState, videoTrack, v) v.setMirror(mirror) }, - modifier = modifier.then( - if (zoomable) { - Modifier - .graphicsLayer( - scaleX = scale, - scaleY = scale, - translationX = offset.x, - translationY = offset.y - ) - .transformable(state = state) - } else Modifier - ) + modifier = modifier + .then( + if (matchVideoAspectRatio) { + Modifier.aspectRatio(videoAspectRatio) + } else Modifier + ).then( + if (zoomable) { + Modifier.transformable(state = state) + } else Modifier + ) ) } } diff --git a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/VideoTextureViewRenderer.kt b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/VideoTextureViewRenderer.kt index 2abf315b..9a57151a 100644 --- a/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/VideoTextureViewRenderer.kt +++ b/obv_messenger/app/src/main/java/io/olvid/messenger/webrtc/components/VideoTextureViewRenderer.kt @@ -28,7 +28,6 @@ import android.util.AttributeSet import android.view.TextureView import android.view.TextureView.SurfaceTextureListener import org.webrtc.EglBase -import org.webrtc.EglRenderer import org.webrtc.GlRectDrawer import org.webrtc.RendererCommon.RendererEvents import org.webrtc.ThreadUtils @@ -41,6 +40,7 @@ import java.util.concurrent.CountDownLatch */ class VideoTextureViewRenderer @JvmOverloads constructor( context: Context, + val scaleAndOffsetControl: ScaleAndOffsetControl? = null, attrs: AttributeSet? = null ) : TextureView(context, attrs), VideoSink, SurfaceTextureListener { @@ -52,7 +52,7 @@ class VideoTextureViewRenderer @JvmOverloads constructor( /** * Renderer used to render the video. */ - private val eglRenderer: EglRenderer = EglRenderer(resourceName) + private val eglRenderer: CustomEglRenderer = CustomEglRenderer(resourceName, scaleAndOffsetControl) /** * Callback used for reporting render events. diff --git a/obv_messenger/app/src/main/res/drawable/olvid_icon.xml b/obv_messenger/app/src/main/res/drawable/olvid_icon.xml new file mode 100644 index 00000000..d00083cc --- /dev/null +++ b/obv_messenger/app/src/main/res/drawable/olvid_icon.xml @@ -0,0 +1,21 @@ + + + + + + + + + + \ No newline at end of file diff --git a/obv_messenger/app/src/main/res/layout/activity_webrtc_incoming_call.xml b/obv_messenger/app/src/main/res/layout/activity_webrtc_incoming_call.xml index 55dbabd9..0742d084 100644 --- a/obv_messenger/app/src/main/res/layout/activity_webrtc_incoming_call.xml +++ b/obv_messenger/app/src/main/res/layout/activity_webrtc_incoming_call.xml @@ -4,7 +4,7 @@ xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent" android:layout_height="match_parent" - android:background="@drawable/background_gradient_olvid" + android:background="@color/black" tools:context=".webrtc.WebrtcIncomingCallActivity"> @@ -30,14 +30,14 @@ android:id="@+id/portrait_color_circle_image_view" android:layout_width="192dp" android:layout_height="192dp" - app:layout_constraintTop_toTopOf="@id/contact_initial_view" - app:layout_constraintBottom_toBottomOf="@id/contact_initial_view" - app:layout_constraintStart_toStartOf="@id/contact_initial_view" - app:layout_constraintEnd_toEndOf="@id/contact_initial_view" android:src="@drawable/initial_view_color_circle" android:visibility="gone" + app:layout_constraintBottom_toBottomOf="@id/contact_initial_view" + app:layout_constraintEnd_toEndOf="@id/contact_initial_view" + app:layout_constraintStart_toStartOf="@id/contact_initial_view" + app:layout_constraintTop_toTopOf="@id/contact_initial_view" tools:visibility="visible" - /> + android:importantForAccessibility="no" /> + tools:ignore="ImageContrastCheck" /> + + + + app:layout_constraintStart_toEndOf="@id/encrypted_call_logo" + app:layout_constraintTop_toTopOf="parent" /> \ No newline at end of file diff --git a/obv_messenger/app/src/main/res/values-fr/strings.xml b/obv_messenger/app/src/main/res/values-fr/strings.xml index 06b06aba..706b2194 100644 --- a/obv_messenger/app/src/main/res/values-fr/strings.xml +++ b/obv_messenger/app/src/main/res/values-fr/strings.xml @@ -1161,7 +1161,7 @@ Ajouter un contact Importer une configuration ou une licence Choisissez le profil qui sera géré par le fournisseur d\'identités de votre entreprise - Choisissez le profil auquel appliquer cette configuration/license + Choisissez le profil auquel appliquer cette configuration/licence Choisissez le profil à connecter au client web Olvid Choisissez le profil avec lequel inviter ce contact Veuillez d\'abord créer un profil, puis ouvrir à nouveau le lien @@ -1459,8 +1459,8 @@ Partager la position pendant 1 heure 3\u00a0heures Partager la position pendant 3 heures - Pas de limite - Partager la position indéfiniement + Indéfiniment + Partager la position indéfiniment Précis Équilibré Économie d\'énergie @@ -1611,7 +1611,7 @@ Émettre des appels sécurisés\n(disponible grâce à un autre profil) Ouvrir ce lien dans votre navigateur\u00a0? Impossible d\'ouvrir l\'URL - Choisir un service de cartographie vous permet de séléctionner précisément la position que vous souhaitez partager sur une carte. Toutefois, cela peut divulguer votre position (ou celle de vos contacts) à ce fournisseur de service. + Choisir un service de cartographie vous permet de sélectionner précisément la position que vous souhaitez partager sur une carte. Toutefois, cela peut divulguer votre position (ou celle de vos contacts) à ce fournisseur de service. Ce choix peut être modifié dans les paramètres de l\'application. Choisir un service Partage en direct @@ -1709,7 +1709,7 @@ Désactivation de votre appareil Votre appareil Olvid sera désactivé d\'ici %1$s Ajouter un appareil ? - Ajouter un appareil permet de facilement transférer votre profil vers un nouvel appareil ou, si vous avez une license multi-appareils, de recevoir vos messages simultanément sur les deux. **N\'ajoutez que des appareils auxquels vous faites confiance !**\n\nVérifiez que vous avez une **connexion internet stable** sur vos deux appareils avant de poursuivre. + Ajouter un appareil permet de facilement transférer votre profil vers un nouvel appareil ou, si vous avez une licence multi-appareils, de recevoir vos messages simultanément sur les deux. **N\'ajoutez que des appareils auxquels vous faites confiance !**\n\nVérifiez que vous avez une **connexion internet stable** sur vos deux appareils avant de poursuivre. Import de votre profil réussi Si vous avez plusieurs profils, peut-être souhaitez-vous les importer également ? Importer un autre profil @@ -1855,6 +1855,7 @@ Résolution en envoi des appels vidéo Appel en cours : %1$s Aucun son + Les appels vidéo sont limités à %1$d participants Vous êtes sur le point d\'inviter chaque membre de ce groupe avec lequel vous avez un canal (actuellement 1 membre) à une discussion privée.\nSouhaitez-vous poursuivre ? diff --git a/obv_messenger/app/src/main/res/values/strings.xml b/obv_messenger/app/src/main/res/values/strings.xml index 928883d9..7bbd2a28 100644 --- a/obv_messenger/app/src/main/res/values/strings.xml +++ b/obv_messenger/app/src/main/res/values/strings.xml @@ -1889,6 +1889,7 @@ 1080p Ongoing call: %1$s No output + Video calls are limited to %1$d participants diff --git a/obv_messenger/build.gradle b/obv_messenger/build.gradle index 1133c0e6..86e7e5de 100644 --- a/obv_messenger/build.gradle +++ b/obv_messenger/build.gradle @@ -7,7 +7,7 @@ buildscript { } dependencies { classpath 'com.android.tools.build:gradle:8.3.0' - classpath 'androidx.navigation:navigation-safe-args-gradle-plugin:2.7.6' + classpath 'androidx.navigation:navigation-safe-args-gradle-plugin:2.7.7' classpath 'com.google.android.gms:oss-licenses-plugin:0.10.6' // protobuf plugin for Web Client @@ -15,7 +15,7 @@ buildscript { // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files - classpath 'com.google.gms:google-services:4.4.0' + classpath 'com.google.gms:google-services:4.4.1' classpath 'org.jetbrains.kotlin:kotlin-gradle-plugin:1.9.20' } }