feat(frontend/backend): paginate the unified view of message lists (#…

…1415)

* wip: allow paginating the combined view of messages list

* Improve the multi-list pagination and implement it for the Flagged and Unread pages.
TODO: - Some items are duplicated across pages when paginating. They should be filtered out

* enahnacethe pagination to get full data on each page while respecting the limit date config

TODO: - The handlers imap_combined_inbox, imap_filter_by_type, and imap_folder_data share a bunch of similar code that should be refactored into one reusable function

* refactor similar code across handlers into a reusable function

modules/core/js_modules/Hm_MessagesStore.js

@@ -24,6 +24,7 @@ class Hm_MessagesStore {
         this.abortController = abortController;
         this.pages = 0;
         this.page = page;
+        this.offsets = '';
     }
 
     /**
@@ -40,24 +41,28 @@ class Hm_MessagesStore {
      */
     async load(reload = false, hideLoadingState = false, doNotFetch = false) {
         const storedMessages = this.#retrieveFromLocalStorage();
-        if (storedMessages && !reload) {
+        if (storedMessages) {
             this.rows = storedMessages.rows;
             this.pages = parseInt(storedMessages.pages);
             this.count = storedMessages.count;
             this.flagAsReadOnOpen = storedMessages.flagAsReadOnOpen;
-            return this;
+            this.offsets = storedMessages.offsets;
+            if (!reload) {
+                return this;
+            }
         }
 
         if (doNotFetch) {
             return this;
         }
 
-        const { formatted_message_list: updatedMessages, pages, folder_status, do_not_flag_as_read_on_open } = await this.#fetch(hideLoadingState);
+        const { formatted_message_list: updatedMessages, pages, folder_status, do_not_flag_as_read_on_open, offsets } = await this.#fetch(hideLoadingState);
 
         this.count = folder_status && Object.values(folder_status)[0]?.messages;
         this.pages = parseInt(pages);
         this.rows = updatedMessages;
         this.flagAsReadOnOpen = !do_not_flag_as_read_on_open;
+        this.offsets = offsets;
 
         this.#saveToLocalStorage();
 
@@ -174,10 +179,9 @@ class Hm_MessagesStore {
         } else {
             switch (this.path) {
                 case 'unread':
-                    hook = "ajax_imap_unread";
-                    break;
                 case 'flagged':
-                    hook = "ajax_imap_flagged";
+                    hook = "ajax_imap_filter_by_type";
+                    config.push({ name: "filter_type", value: this.path });
                     break;
                 case 'combined_inbox':
                     hook = "ajax_combined_message_list";
@@ -202,7 +206,7 @@ class Hm_MessagesStore {
     }
 
     #saveToLocalStorage() {
-        Hm_Utils.save_to_local_storage(this.list, JSON.stringify({ rows: this.rows, pages: this.pages, count: this.count }));
+        Hm_Utils.save_to_local_storage(this.list, JSON.stringify({ rows: this.rows, pages: this.pages, count: this.count, offsets: this.offsets }));
         Hm_Utils.save_to_local_storage('flagAsReadOnOpen', this.flagAsReadOnOpen);
     }
 

modules/core/js_modules/actions/pagination.js

@@ -1,5 +1,5 @@
 function refreshNextButton(current) {
-    const totalPages = $(".pagination .total").text();
+    const totalPages = $(".pagination .max").text();
     if (parseInt(current) >= parseInt(totalPages)) {
         $(".pagination .next").prop('disabled', true);
     } else {
@@ -20,27 +20,44 @@ async function nextPage() {
 
     const nextPage = parseInt(currentPage) + 1;
 
-    await changePage(nextPage, this);
+    const store = new Hm_MessagesStore(getListPathParam(), currentPage);
+    store.load(false, false, true);
+
+    await changePage(nextPage, this, store.offsets);
 }
 
 async function previousPage() {
     const currentPage = $(".pagination .current").text();
 
     const previousPage = parseInt(currentPage) - 1;
 
-    await changePage(previousPage, this);
+    let offsets = '';
+    if (previousPage > 1) {
+        const store = new Hm_MessagesStore(getListPathParam(), previousPage - 1);
+        store.load(false, false, true);
+        offsets = store.offsets;
+    }
+
+    await changePage(previousPage, this, offsets);
 
     if (previousPage > 1) {
         $(this).prop('disabled', false);
     }
 }
 
-async function changePage(toPage, button) {
+async function changePage(toPage, button, offsets) {
     $(button).prop('disabled', true);
     $(button).addClass('active');
 
     const url = new URL(window.location.href);
     url.searchParams.set('list_page', toPage);
+
+    if (offsets) {
+        url.searchParams.set('offsets', offsets);
+    } else {
+        url.searchParams.delete('offsets');
+    }
+
     history.pushState(history.state, "", url.toString());
     window.location.next = url.search;
 

modules/core/js_modules/markup/pagination.js

@@ -6,7 +6,7 @@ const paginationMarkup = (totalPages) => {
                 <i class="bi bi-chevron-left"></i>
             </button>
             <div>
-                <span class="current">${currentPage}</span>/<span class="total">${totalPages}</span>
+                <span class="current">${currentPage}</span>/<span class="max">${totalPages}</span>
             </div>
             <button class="btn rounded-circle btn-sm next">
                 <i class="bi bi-chevron-right"></i>

modules/imap/functions.php

@@ -1585,3 +1585,77 @@ function connect_to_imap_server($address, $name, $port, $user, $pass, $tls, $ima
         }
     }
 }
+
+function getCombinedMessagesLists($sources, $cache, $searchTerms, $listPage, $limit, $offsets = [], $defaultOffset = 0, $filter = 'ALL') {
+    $totalMessages = 0;
+    $offset = $defaultOffset;
+    $messagesLists = [];
+    $status = [];
+    foreach ($sources as $index => $dataSource) {
+
+        if ($offsets && $listPage > 1) {
+            if (isset($offsets[$index]) && (int) $offsets[$index] > 0) {
+                $offset = (int) $offsets[$index] * ($listPage - 1);
+            }
+        }
+
+        $mailbox = Hm_IMAP_List::get_connected_mailbox($dataSource['id'], $cache);
+        if ($mailbox && $mailbox->authed()) {
+            $folder = $dataSource['folder'];
+            $state = $mailbox->get_connection()->get_mailbox_status(hex2bin($folder));
+            $status['imap_'.$dataSource['id'].'_'.$folder] = $state;
+
+            $uids = $mailbox->search(hex2bin($folder), $filter, false, $searchTerms);
+            $total = count($uids);
+            // most recent messages at the top
+            $uids = array_reverse($uids);
+            $uids = array_slice($uids, $offset, $limit);
+
+            $headers = $mailbox->get_message_list(hex2bin($folder), $uids);
+            $messages = [];
+            foreach ($uids as $uid) {
+                if (isset($headers[$uid])) {
+                    $messages[] = $headers[$uid];
+                }
+            }
+
+            $messagesLists[] = array_map(function($msg) use ($dataSource, $folder) {
+                $msg['server_id'] = $dataSource['id'];
+                $msg['server_name'] = $dataSource['name'];
+                $msg['folder'] = $folder;
+                return $msg;
+            }, $messages);
+            $totalMessages += $total;
+        }
+    }
+
+    return ['lists' => $messagesLists, 'total' => $totalMessages, 'status' => $status];
+}
+
+function flattenMessagesLists($messagesLists, $listSize) {
+    $endList = [];
+    $sizesTaken = [];
+
+    $max = $listSize * count($messagesLists);
+
+    while (count($endList) < $listSize * count($messagesLists) && count(array_filter($messagesLists, fn ($list) => count($list) > 0)) > 0) {
+        foreach ($messagesLists as $index => $list) {
+            if (count($list) > 0) {
+                $part = array_slice($list, 0, $listSize);
+                $endList = array_merge($endList, $part);
+                $messagesLists[$index] = array_slice($list, $listSize);
+                $sizesTaken[$index] = $sizesTaken[$index] ? $sizesTaken[$index] + count($part) : count($part);
+                $totalTakens = array_sum(array_values($sizesTaken));
+                if ($totalTakens > $max) {
+                    $sizesTaken[$index] = $sizesTaken[$index] - ($totalTakens - $max);
+                }
+            } else {
+                $sizesTaken[$index] = $sizesTaken[$index] ? $sizesTaken[$index] : 0;
+            }
+        }
+    }
+
+    $endList = array_slice($endList, 0, $max);
+
+    return ['messages' => $endList, 'offsets' => $sizesTaken];
+}