From 199a229c2e77834e75a79686040eff41a0008261 Mon Sep 17 00:00:00 2001 From: Luka Hadzi-Djokic Date: Tue, 11 Jul 2023 15:33:07 +0200 Subject: [PATCH 1/5] Add cl-dbi as a dependency. --- .gitmodules | 3 +++ _build/cl-dbi | 1 + build-scripts/nyxt.scm | 1 + documents/SOURCES.org | 1 + nyxt.asd | 1 + 5 files changed, 7 insertions(+) create mode 160000 _build/cl-dbi diff --git a/.gitmodules b/.gitmodules index 139a394a72e..1b72e39881d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -599,4 +599,7 @@ [submodule "_build/cl-json"] path = _build/cl-json url = https://github.com/sharplispers/cl-json +[submodule "_build/cl-dbi"] + path = _build/cl-dbi + url = https://github.com/fukamachi/cl-dbi shallow = true diff --git a/_build/cl-dbi b/_build/cl-dbi new file mode 160000 index 00000000000..3d4b48c7b67 --- /dev/null +++ b/_build/cl-dbi @@ -0,0 +1 @@ +Subproject commit 3d4b48c7b6736a4257043cde60687614775714d6 diff --git a/build-scripts/nyxt.scm b/build-scripts/nyxt.scm index d841d0d9b9c..0ddb42d8637 100644 --- a/build-scripts/nyxt.scm +++ b/build-scripts/nyxt.scm @@ -143,6 +143,7 @@ cl-clss cl-cluffer cl-custom-hash-table + cl-dbi cl-dexador cl-dissect cl-trivial-custom-debugger diff --git a/documents/SOURCES.org b/documents/SOURCES.org index fdae8457f4a..38d47638f98 100644 --- a/documents/SOURCES.org +++ b/documents/SOURCES.org @@ -21,6 +21,7 @@ Experimental support. - calispel - cl-containers - cl-custom-hash-table +- cl-dbi - cl-html-diff - cl-json - cl-ppcre diff --git a/nyxt.asd b/nyxt.asd index 6b616a7e7e2..e3dabe317d6 100644 --- a/nyxt.asd +++ b/nyxt.asd @@ -67,6 +67,7 @@ The renderer is configured from NYXT_RENDERER or `*nyxt-renderer*'.")) calispel cl-base64 cl-colors2 + cl-dbi cl-gopher cl-html-diff cl-json From ea54516dd12238dd9d9bd5add2df7bfd8d1c16c4 Mon Sep 17 00:00:00 2001 From: Luka Hadzi-Djokic Date: Wed, 12 Jul 2023 15:07:55 +0200 Subject: [PATCH 2/5] Add history-migration mode for importing history. Add history-migration mode with commands for importing history from Firefox and several Chromium-based browsers. --- nyxt.asd | 1 + source/mode/history-migration.lisp | 95 ++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 source/mode/history-migration.lisp diff --git a/nyxt.asd b/nyxt.asd index e3dabe317d6..164f16c5244 100644 --- a/nyxt.asd +++ b/nyxt.asd @@ -241,6 +241,7 @@ The renderer is configured from NYXT_RENDERER or `*nyxt-renderer*'.")) (:file "emacs") (:file "expedition") (:file "force-https") + (:file "history-migration") (:file "macro-edit") (:file "no-image") (:file "no-procrastinate" :depends-on ("blocker")) diff --git a/source/mode/history-migration.lisp b/source/mode/history-migration.lisp new file mode 100644 index 00000000000..e218c2bad41 --- /dev/null +++ b/source/mode/history-migration.lisp @@ -0,0 +1,95 @@ +;;;; SPDX-FileCopyrightText: Atlas Engineer LLC +;;;; SPDX-License-Identifier: BSD-3-Clause + +(nyxt:define-package :nyxt/mode/history-migration + (:documentation "Package for `history-migration-mode', mode to import history from other +browsers.")) +(in-package :nyxt/mode/history-migration) + +(define-mode history-migration-mode () + "Mode for importing history from other browsers." + ((visible-in-status-p nil) + (rememberable-p nil)) + (:toggler-command-p nil)) + +(define-class external-browser-history-source (prompter:source) + ((prompter:name "History files") + (browser-pattern :accessor browser-pattern :initarg :browser-pattern) + (prompter:constructor + (lambda (source) + (echo "Searching for history file. This may take some time.") + (or (sera:filter #'str:non-empty-string-p + (str:split #\newline + (nth-value 0 + (uiop:run-program (list "find" (uiop:native-namestring "~/") + "-regex" (browser-pattern source)) + :output :string)))) + (echo-warning "No history files found.")))))) + +(defmacro define-history-import-command (name docstring sql-expr file-path) + "Shorthand to define a global command for importing history from a browser. +Make sure that the sql-expr is a SELECT statement that selects: + - url + - title + - lastaccess + - visits +Rename the columns in SELECT statement to match if necessary." + `(define-command-global ,name () + ,docstring + (let ((db-path ,file-path)) + (files:with-file-content (history (history-file (current-buffer)) + :default (nyxt::make-history-tree)) + (handler-bind ((dbi.error:dbi-database-error + (lambda (_) + (declare (ignore _)) + (echo-warning "Please close the browser you wish to import history from before running this command.") + (invoke-restart 'abort)))) + (dbi:with-connection (conn :sqlite3 :database-name db-path) + (let* ((query (dbi:prepare conn ,sql-expr)) + (query (dbi:execute query))) + (echo "Importing history from ~a." db-path) + (loop for row = (dbi:fetch query) + while row + do (unless (url-empty-p (getf row :|url|)) + (htree:add-entry history + (make-instance 'history-entry + :url (getf row :|url|) + :title (getf row :|title|) + :implicit-visits (getf row :|visits|)) + (local-time:unix-to-timestamp (getf row :|lastaccess|))))))))) + (echo "History import finished.")))) + +(define-history-import-command import-history-from-firefox + "Import history from Mozilla Firefox." + "SELECT url, title, last_visit_date/1000000 as lastaccess, visit_count as visits FROM moz_places WHERE last_visit_date not null" + (prompt1 :prompt "Choose Mozilla Firefox places.sqlite file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/places\.sqlite$"))) + +(define-history-import-command import-history-from-google-chrome + "Import history from Google Chrome." + "SELECT url, title, last_visit_time/1000000-11644473600 as lastaccess, visit_count as visits FROM urls" + (prompt1 :prompt "Choose Google Chrome History file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/google-chrome/Default/History$"))) + +(define-history-import-command import-history-from-chromium + "Import history from Chromium." + "SELECT url, title, last_visit_time/1000000-11644473600 as lastaccess, visit_count as visits FROM urls" + (prompt1 :prompt "Choose Chromium History file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/chromium/Default/History$"))) + +(define-history-import-command import-history-from-brave + "Import history from Brave." + "SELECT url, title, last_visit_time/1000000-11644473600 as lastaccess, visit_count as visits FROM urls" + (prompt1 :prompt "Choose Brave History file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/BraveSoftware/Brave-Browser/Default/History$"))) + +(define-history-import-command import-history-from-vivaldi + "Import history from Vivaldi." + "SELECT url, title, last_visit_time/1000000-11644473600 as lastaccess, visit_count as visits FROM urls" + (prompt1 :prompt "Choose Vivaldi History file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/vivaldi(-snapshot)?/Default/History$"))) From a702a01240a96e4017eedce63767ff501cd2eaee Mon Sep 17 00:00:00 2001 From: Luka Hadzi-Djokic Date: Thu, 27 Jul 2023 12:08:40 +0200 Subject: [PATCH 3/5] Switch from cl-dbi to cl-sqlite. Switch from cl-dbi to cl-sqlite because `mode/history-migration` only works with SQLite databases. --- .gitmodules | 7 +-- _build/cl-dbi | 1 - _build/cl-sqlite | 1 + build-scripts/nyxt.scm | 2 +- documents/SOURCES.org | 2 +- nyxt.asd | 2 +- source/mode/history-migration.lisp | 75 ++++++++++++++---------------- 7 files changed, 44 insertions(+), 46 deletions(-) delete mode 160000 _build/cl-dbi create mode 160000 _build/cl-sqlite diff --git a/.gitmodules b/.gitmodules index 1b72e39881d..83aedc535f3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -599,7 +599,8 @@ [submodule "_build/cl-json"] path = _build/cl-json url = https://github.com/sharplispers/cl-json -[submodule "_build/cl-dbi"] - path = _build/cl-dbi - url = https://github.com/fukamachi/cl-dbi + shallow = true +[submodule "_build/cl-sqlite"] + path = _build/cl-sqlite + url = https://github.com/TeMPOraL/cl-sqlite shallow = true diff --git a/_build/cl-dbi b/_build/cl-dbi deleted file mode 160000 index 3d4b48c7b67..00000000000 --- a/_build/cl-dbi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3d4b48c7b6736a4257043cde60687614775714d6 diff --git a/_build/cl-sqlite b/_build/cl-sqlite new file mode 160000 index 00000000000..be2fcc193f9 --- /dev/null +++ b/_build/cl-sqlite @@ -0,0 +1 @@ +Subproject commit be2fcc193f98e3d5bdc85958a806d612cc48740c diff --git a/build-scripts/nyxt.scm b/build-scripts/nyxt.scm index 0ddb42d8637..57ca4714c29 100644 --- a/build-scripts/nyxt.scm +++ b/build-scripts/nyxt.scm @@ -143,7 +143,6 @@ cl-clss cl-cluffer cl-custom-hash-table - cl-dbi cl-dexador cl-dissect cl-trivial-custom-debugger @@ -185,6 +184,7 @@ cl-str cl-slime-swank cl-slynk + cl-sqlite cl-tld cl-trivia cl-trivial-clipboard diff --git a/documents/SOURCES.org b/documents/SOURCES.org index 38d47638f98..c2740cf6489 100644 --- a/documents/SOURCES.org +++ b/documents/SOURCES.org @@ -21,13 +21,13 @@ Experimental support. - calispel - cl-containers - cl-custom-hash-table -- cl-dbi - cl-html-diff - cl-json - cl-ppcre - cl-ppcre-unicode - cl-prevalence - cl-qrencode +- cl-sqlite - cl-tld - closer-mop - cluffer diff --git a/nyxt.asd b/nyxt.asd index 164f16c5244..8d5d708e260 100644 --- a/nyxt.asd +++ b/nyxt.asd @@ -67,7 +67,6 @@ The renderer is configured from NYXT_RENDERER or `*nyxt-renderer*'.")) calispel cl-base64 cl-colors2 - cl-dbi cl-gopher cl-html-diff cl-json @@ -112,6 +111,7 @@ The renderer is configured from NYXT_RENDERER or `*nyxt-renderer*'.")) clss spinneret slynk + sqlite swank trivia trivial-clipboard diff --git a/source/mode/history-migration.lisp b/source/mode/history-migration.lisp index e218c2bad41..276d7ae3eb9 100644 --- a/source/mode/history-migration.lisp +++ b/source/mode/history-migration.lisp @@ -26,70 +26,67 @@ browsers.")) :output :string)))) (echo-warning "No history files found.")))))) -(defmacro define-history-import-command (name docstring sql-expr file-path) +(defmacro define-history-import-command (name docstring &key sql-query file-path) "Shorthand to define a global command for importing history from a browser. -Make sure that the sql-expr is a SELECT statement that selects: +Make sure that the sql-query is a SELECT statement that selects: - url - title - - lastaccess + - last-access (unix time in seconds) - visits -Rename the columns in SELECT statement to match if necessary." +Or the equivalent columns for the browser in question." `(define-command-global ,name () ,docstring (let ((db-path ,file-path)) (files:with-file-content (history (history-file (current-buffer)) :default (nyxt::make-history-tree)) - (handler-bind ((dbi.error:dbi-database-error + (handler-bind ((sqlite:sqlite-error (lambda (_) (declare (ignore _)) (echo-warning "Please close the browser you wish to import history from before running this command.") (invoke-restart 'abort)))) - (dbi:with-connection (conn :sqlite3 :database-name db-path) - (let* ((query (dbi:prepare conn ,sql-expr)) - (query (dbi:execute query))) - (echo "Importing history from ~a." db-path) - (loop for row = (dbi:fetch query) - while row - do (unless (url-empty-p (getf row :|url|)) - (htree:add-entry history - (make-instance 'history-entry - :url (getf row :|url|) - :title (getf row :|title|) - :implicit-visits (getf row :|visits|)) - (local-time:unix-to-timestamp (getf row :|lastaccess|))))))))) - (echo "History import finished.")))) + (sqlite:with-open-database (db db-path) + (echo "Importing history from ~a." db-path) + (loop for (url title last-access visits) in (sqlite:execute-to-list db ,sql-query) + do (unless (url-empty-p url) + (htree:add-entry history + (make-instance 'history-entry + :url url + :title title + :implicit-visits visits) + (local-time:unix-to-timestamp last-access)))) + (echo "History import finished."))))))) (define-history-import-command import-history-from-firefox "Import history from Mozilla Firefox." - "SELECT url, title, last_visit_date/1000000 as lastaccess, visit_count as visits FROM moz_places WHERE last_visit_date not null" - (prompt1 :prompt "Choose Mozilla Firefox places.sqlite file" - :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/places\.sqlite$"))) + :sql-query "SELECT url, title, last_visit_date/1000000, visit_count FROM moz_places WHERE last_visit_date not null" + :file-path (prompt1 :prompt "Choose Mozilla Firefox places.sqlite file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/places\.sqlite$"))) (define-history-import-command import-history-from-google-chrome "Import history from Google Chrome." - "SELECT url, title, last_visit_time/1000000-11644473600 as lastaccess, visit_count as visits FROM urls" - (prompt1 :prompt "Choose Google Chrome History file" - :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/google-chrome/Default/History$"))) + :sql-query "SELECT url, title, last_visit_time/1000000-11644473600, visit_count FROM urls" + :file-path (prompt1 :prompt "Choose Google Chrome History file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/google-chrome/Default/History$"))) (define-history-import-command import-history-from-chromium "Import history from Chromium." - "SELECT url, title, last_visit_time/1000000-11644473600 as lastaccess, visit_count as visits FROM urls" - (prompt1 :prompt "Choose Chromium History file" - :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/chromium/Default/History$"))) + :sql-query "SELECT url, title, last_visit_time/1000000-11644473600, visit_count FROM urls" + :file-path (prompt1 :prompt "Choose Chromium History file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/chromium/Default/History$"))) (define-history-import-command import-history-from-brave "Import history from Brave." - "SELECT url, title, last_visit_time/1000000-11644473600 as lastaccess, visit_count as visits FROM urls" - (prompt1 :prompt "Choose Brave History file" - :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/BraveSoftware/Brave-Browser/Default/History$"))) + :sql-query "SELECT url, title, last_visit_time/1000000-11644473600, visit_count FROM urls" + :file-path (prompt1 :prompt "Choose Brave History file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/BraveSoftware/Brave-Browser/Default/History$"))) (define-history-import-command import-history-from-vivaldi "Import history from Vivaldi." - "SELECT url, title, last_visit_time/1000000-11644473600 as lastaccess, visit_count as visits FROM urls" - (prompt1 :prompt "Choose Vivaldi History file" - :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/vivaldi(-snapshot)?/Default/History$"))) + :sql-query "SELECT url, title, last_visit_time/1000000-11644473600, visit_count FROM urls" + :file-path (prompt1 :prompt "Choose Vivaldi History file" + :sources (make-instance 'external-browser-history-source + :browser-pattern ".*/vivaldi(-snapshot)?/Default/History$"))) From 1a5ddc7faf7bc055c94f5969d233e957569e8b73 Mon Sep 17 00:00:00 2001 From: Luka Hadzi-Djokic Date: Tue, 1 Aug 2023 13:01:47 +0200 Subject: [PATCH 4/5] mode/history-migration: Use uiop to search for files. Use `uiop:collect-sub*directories` to search for files, instead of GNU find. --- source/mode/history-migration.lisp | 47 ++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/source/mode/history-migration.lisp b/source/mode/history-migration.lisp index 276d7ae3eb9..5267367d41f 100644 --- a/source/mode/history-migration.lisp +++ b/source/mode/history-migration.lisp @@ -14,16 +14,21 @@ browsers.")) (define-class external-browser-history-source (prompter:source) ((prompter:name "History files") - (browser-pattern :accessor browser-pattern :initarg :browser-pattern) + (browser-lambda :accessor browser-lambda :initarg :browser-lambda) (prompter:constructor (lambda (source) (echo "Searching for history file. This may take some time.") - (or (sera:filter #'str:non-empty-string-p - (str:split #\newline - (nth-value 0 - (uiop:run-program (list "find" (uiop:native-namestring "~/") - "-regex" (browser-pattern source)) - :output :string)))) + (or (let ((results '())) + (uiop:collect-sub*directories + (user-homedir-pathname) + (constantly t) + (lambda (subdir) + (equal (iolib/os:file-kind subdir) :directory)) + (lambda (subdir) + (let ((browser-history-file (funcall* (browser-lambda source) subdir))) + (when browser-history-file + (push browser-history-file results))))) + results) (echo-warning "No history files found.")))))) (defmacro define-history-import-command (name docstring &key sql-query file-path) @@ -61,32 +66,50 @@ Or the equivalent columns for the browser in question." :sql-query "SELECT url, title, last_visit_date/1000000, visit_count FROM moz_places WHERE last_visit_date not null" :file-path (prompt1 :prompt "Choose Mozilla Firefox places.sqlite file" :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/places\.sqlite$"))) + :browser-lambda (lambda (subdir) + (when (uiop:file-exists-p (uiop:merge-pathnames* "places.sqlite" subdir)) + (uiop:merge-pathnames* "places.sqlite" subdir)))))) (define-history-import-command import-history-from-google-chrome "Import history from Google Chrome." :sql-query "SELECT url, title, last_visit_time/1000000-11644473600, visit_count FROM urls" :file-path (prompt1 :prompt "Choose Google Chrome History file" :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/google-chrome/Default/History$"))) + :browser-lambda (lambda (subdir) + (when (and (string= "google-chrome" + (nfiles:basename (nfiles:parent subdir))) + (uiop:file-exists-p (uiop:merge-pathnames* "History" subdir))) + (uiop:merge-pathnames* "History" subdir)))))) (define-history-import-command import-history-from-chromium "Import history from Chromium." :sql-query "SELECT url, title, last_visit_time/1000000-11644473600, visit_count FROM urls" :file-path (prompt1 :prompt "Choose Chromium History file" :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/chromium/Default/History$"))) + :browser-lambda (lambda (subdir) + (when (and (string= "chromium" + (nfiles:basename (nfiles:parent subdir))) + (uiop:file-exists-p (uiop:merge-pathnames* "History" subdir))) + (uiop:merge-pathnames* "History" subdir)))))) (define-history-import-command import-history-from-brave "Import history from Brave." :sql-query "SELECT url, title, last_visit_time/1000000-11644473600, visit_count FROM urls" :file-path (prompt1 :prompt "Choose Brave History file" :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/BraveSoftware/Brave-Browser/Default/History$"))) + :browser-lambda (lambda (subdir) + (when (and (string= "Brave-Browser" + (nfiles:basename (nfiles:parent subdir))) + (uiop:file-exists-p (uiop:merge-pathnames* "History" subdir))) + (uiop:merge-pathnames* "History" subdir)))))) (define-history-import-command import-history-from-vivaldi "Import history from Vivaldi." :sql-query "SELECT url, title, last_visit_time/1000000-11644473600, visit_count FROM urls" :file-path (prompt1 :prompt "Choose Vivaldi History file" :sources (make-instance 'external-browser-history-source - :browser-pattern ".*/vivaldi(-snapshot)?/Default/History$"))) + :browser-lambda (lambda (subdir) + (when (and (str:starts-with-p "vivaldi" + (nfiles:basename (nfiles:parent subdir))) + (uiop:file-exists-p (uiop:merge-pathnames* "History" subdir))) + (uiop:merge-pathnames* "History" subdir)))))) From 20a3fdaa758201140c6b758e6cb91b0a8a39c46a Mon Sep 17 00:00:00 2001 From: Luka Hadzi-Djokic Date: Wed, 2 Aug 2023 13:49:25 +0200 Subject: [PATCH 5/5] changelog: Mention new history import feature. --- source/changelog.lisp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/changelog.lisp b/source/changelog.lisp index b20c10b983c..6b527dda8d0 100644 --- a/source/changelog.lisp +++ b/source/changelog.lisp @@ -809,6 +809,12 @@ buffers.") (:li "Fix styling of progress bar.") (:li "Fix styling of prompt buffer's input area.")))) +(define-version "3.X.Y" + (:ul + (:li "Add commands for importing history from other browsers. +Currently, the supported browsers are Firefox, Google Chrome, +Chromium, Brave and Vivaldi."))) + (define-version "4-pre-release-1" (:li "When on pre-release, push " (:code "X-pre-release") " feature in addition to " (:code "X-pre-release-N") "one."))