diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh index 9ed64fd8..7d40120e 100644 --- a/.github/helper/install_dependencies.sh +++ b/.github/helper/install_dependencies.sh @@ -9,10 +9,10 @@ echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://pack sudo apt update sudo apt remove mysql-server mysql-client -sudo apt install libcups2-dev redis mariadb-client-10.6 +sudo apt install libcups2-dev redis mariadb-client install_wkhtmltopdf() { - wget -q https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb - sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb + wget -q https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6.1-2.jammy_amd64.deb + sudo apt install ./wkhtmltox_0.12.6.1-2.jammy_amd64.deb } install_wkhtmltopdf & diff --git a/wiki/public/build.json b/wiki/public/build.json deleted file mode 100644 index ab471142..00000000 --- a/wiki/public/build.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "wiki/css/wiki.css": ["public/scss/wiki.scss"], - "wiki/js/wiki.min.js": ["www/editu.js"] -} \ No newline at end of file diff --git a/wiki/public/js/wiki.js b/wiki/public/js/wiki.js index eb48b1f5..e059bfa6 100644 --- a/wiki/public/js/wiki.js +++ b/wiki/public/js/wiki.js @@ -32,7 +32,7 @@ window.Wiki = class Wiki { $(".doc-sidebar,.web-sidebar").on( "click", ".collapsible", - this.toggle_sidebar + this.toggle_sidebar, ); $(".sidebar-item.active") @@ -46,7 +46,7 @@ window.Wiki = class Wiki { set_last_updated_date() { const lastUpdatedDate = frappe.datetime.prettyDate( - $(".user-contributions").data("date") + $(".user-contributions").data("date"), ); $(".user-contributions").append(`last updated ${lastUpdatedDate}`); } @@ -57,7 +57,7 @@ window.Wiki = class Wiki { const src = $(".navbar-brand img").attr("src"); if ( !["{{ light_mode_logo }}", "{{ dark_mode_logo }}", "None", ""].includes( - altSrc + altSrc, ) ) { $(".navbar-brand img").attr("src", altSrc); @@ -117,7 +117,7 @@ window.Wiki = class Wiki { $("pre code") .parent("pre") .prepend( - `` + ``, ); $(".copy-btn").on("click", function () { diff --git a/wiki/wiki/doctype/wiki_space/wiki_space.json b/wiki/wiki/doctype/wiki_space/wiki_space.json index 33d78ec2..4715eb57 100644 --- a/wiki/wiki/doctype/wiki_space/wiki_space.json +++ b/wiki/wiki/doctype/wiki_space/wiki_space.json @@ -71,7 +71,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2024-04-05 21:21:29.535486", + "modified": "2024-12-11 15:27:44.629602", "modified_by": "Administrator", "module": "Wiki", "name": "Wiki Space", @@ -105,5 +105,6 @@ ], "sort_field": "modified", "sort_order": "DESC", - "states": [] + "states": [], + "title_field": "route" } \ No newline at end of file diff --git a/wiki/wiki/report/__init__.py b/wiki/wiki/report/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wiki/wiki/report/wiki_broken_links/__init__.py b/wiki/wiki/report/wiki_broken_links/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wiki/wiki/report/wiki_broken_links/test_broken_link_checker.py b/wiki/wiki/report/wiki_broken_links/test_broken_link_checker.py new file mode 100644 index 00000000..9bd3bced --- /dev/null +++ b/wiki/wiki/report/wiki_broken_links/test_broken_link_checker.py @@ -0,0 +1,100 @@ +# Copyright (c) 2024, Frappe and Contributors +# See license.txt + +from unittest.mock import patch + +import frappe +from frappe.tests.utils import FrappeTestCase + +from wiki.wiki.report.wiki_broken_links.wiki_broken_links import execute, get_broken_links + +WORKING_EXTERNAL_URL = "https://frappe.io" +BROKEN_EXTERNAL_URL = "https://frappewiki.notavalidtld" +BROKEN_IMG_URL = "https://img.notavalidtld/failed.jpeg" +WORKING_INTERNAL_URL = "/api/method/ping" +BROKEN_INTERNAL_URL = "/api/method/ring" + + +def internal_to_external_urls(internal_url: str) -> str: + if internal_url == WORKING_INTERNAL_URL: + return WORKING_EXTERNAL_URL + else: + return BROKEN_EXTERNAL_URL + + +TEST_MD_WITH_BROKEN_LINK = f""" +## Hello + +This is a test for a [broken link]({BROKEN_EXTERNAL_URL}). + +This is a [valid link]({WORKING_EXTERNAL_URL}). +And [this is a correct relative link]({WORKING_INTERNAL_URL}). +And [this is an incorrect relative link]({BROKEN_INTERNAL_URL}). + +This [hash link](#hash-link) should be ignored. + +![Broken Image]({BROKEN_IMG_URL}) +""" + + +class TestWikiBrokenLinkChecker(FrappeTestCase): + def setUp(self): + frappe.db.delete("Wiki Page") + self.test_wiki_page = frappe.get_doc( + { + "doctype": "Wiki Page", + "content": TEST_MD_WITH_BROKEN_LINK, + "title": "My Wiki Page", + "route": "test-wiki-page-route", + } + ).insert() + + self.test_wiki_space = frappe.get_doc({"doctype": "Wiki Space", "route": "test-ws-route"}).insert() + + def test_returns_correct_broken_links(self): + broken_links = get_broken_links(TEST_MD_WITH_BROKEN_LINK) + self.assertEqual(len(broken_links), 2) + + def test_wiki_broken_link_report(self): + _, data = execute() + self.assertEqual(len(data), 1) + self.assertEqual(data[0]["broken_link"], BROKEN_EXTERNAL_URL) + + def test_wiki_broken_link_report_with_wiki_space_filter(self): + _, data = execute({"wiki_space": self.test_wiki_space.name}) + self.assertEqual(len(data), 0) + + self.test_wiki_space.append( + "wiki_sidebars", {"wiki_page": self.test_wiki_page, "parent_label": "Test Parent Label"} + ) + self.test_wiki_space.save() + + _, data = execute({"wiki_space": self.test_wiki_space.name}) + self.assertEqual(len(data), 1) + self.assertEqual(data[0]["wiki_page"], self.test_wiki_page.name) + self.assertEqual(data[0]["broken_link"], BROKEN_EXTERNAL_URL) + + def test_wiki_broken_link_report_with_image_filter(self): + _, data = execute({"check_images": 1}) + self.assertEqual(len(data), 2) + self.assertEqual(data[0]["wiki_page"], self.test_wiki_page.name) + self.assertEqual(data[0]["broken_link"], BROKEN_EXTERNAL_URL) + + self.assertEqual(data[1]["wiki_page"], self.test_wiki_page.name) + self.assertEqual(data[1]["broken_link"], BROKEN_IMG_URL) + + @patch.object(frappe.utils.data, "get_url", side_effect=internal_to_external_urls) + def test_wiki_broken_link_report_with_internal_links(self, _get_url): + # patch the get_url to return valid/invalid external links instead + # of internal links in test + _, data = execute({"check_internal_links": 1}) + + self.assertEqual(len(data), 2) + self.assertEqual(data[0]["wiki_page"], self.test_wiki_page.name) + self.assertEqual(data[0]["broken_link"], BROKEN_EXTERNAL_URL) + + self.assertEqual(data[1]["wiki_page"], self.test_wiki_page.name) + self.assertEqual(data[1]["broken_link"], BROKEN_INTERNAL_URL) + + def tearDown(self): + frappe.db.rollback() diff --git a/wiki/wiki/report/wiki_broken_links/wiki_broken_links.js b/wiki/wiki/report/wiki_broken_links/wiki_broken_links.js new file mode 100644 index 00000000..bfe4ab40 --- /dev/null +++ b/wiki/wiki/report/wiki_broken_links/wiki_broken_links.js @@ -0,0 +1,25 @@ +// Copyright (c) 2024, Frappe and contributors +// For license information, please see license.txt + +frappe.query_reports["Wiki Broken Links"] = { + filters: [ + { + fieldname: "wiki_space", + label: __("Wiki Space"), + fieldtype: "Link", + options: "Wiki Space", + }, + { + fieldname: "check_images", + label: __("Include images?"), + fieldtype: "Check", + default: 1, + }, + { + fieldname: "check_internal_links", + label: __("Include internal links?"), + fieldtype: "Check", + default: 0, + }, + ], +}; diff --git a/wiki/wiki/report/wiki_broken_links/wiki_broken_links.json b/wiki/wiki/report/wiki_broken_links/wiki_broken_links.json new file mode 100644 index 00000000..6f2aae28 --- /dev/null +++ b/wiki/wiki/report/wiki_broken_links/wiki_broken_links.json @@ -0,0 +1,30 @@ +{ + "add_total_row": 0, + "columns": [], + "creation": "2024-12-11 14:43:18.799835", + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "filters": [], + "idx": 0, + "is_standard": "Yes", + "letterhead": null, + "modified": "2024-12-11 18:58:14.479423", + "modified_by": "Administrator", + "module": "Wiki", + "name": "Wiki Broken Links", + "owner": "Administrator", + "prepared_report": 1, + "ref_doctype": "Wiki Page", + "report_name": "Wiki Broken Links", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + }, + { + "role": "Wiki Approver" + } + ], + "timeout": 0 +} \ No newline at end of file diff --git a/wiki/wiki/report/wiki_broken_links/wiki_broken_links.py b/wiki/wiki/report/wiki_broken_links/wiki_broken_links.py new file mode 100644 index 00000000..f8dc1833 --- /dev/null +++ b/wiki/wiki/report/wiki_broken_links/wiki_broken_links.py @@ -0,0 +1,132 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +import frappe +import requests +from bs4 import BeautifulSoup +from frappe import _ + + +def execute(filters: dict | None = None): + """Return columns and data for the report. + + This is the main entry point for the report. It accepts the filters as a + dictionary and should return columns and data. It is called by the framework + every time the report is refreshed or a filter is updated. + """ + columns = get_columns() + data = get_data(filters) + + return columns, data + + +def get_columns() -> list[dict]: + """Return columns for the report. + + One field definition per column, just like a DocType field definition. + """ + return [ + { + "label": _("Wiki Page"), + "fieldname": "wiki_page", + "fieldtype": "Link", + "options": "Wiki Page", + "width": 200, + }, + { + "label": _("Broken Link"), + "fieldname": "broken_link", + "fieldtype": "Data", + "options": "URL", + "width": 400, + }, + ] + + +def get_data(filters: dict | None = None) -> list[list]: + """Return data for the report. + + The report data is a list of rows, with each row being a list of cell values. + """ + data = [] + + wiki_pages = frappe.db.get_all("Wiki Page", fields=["name", "content"]) + + if filters and filters.get("wiki_space"): + wiki_space = filters.get("wiki_space") + wiki_pages = frappe.db.get_all( + "Wiki Group Item", + fields=["wiki_page as name", "wiki_page.content as content"], + filters={"parent": wiki_space, "parenttype": "Wiki Space"}, + ) + + include_images = filters and bool(filters.get("check_images")) + check_internal_links = filters and bool(filters.get("check_internal_links")) + + for page in wiki_pages: + broken_links_for_page = get_broken_links(page.content, include_images, check_internal_links) + rows = [{"broken_link": link, "wiki_page": page["name"]} for link in broken_links_for_page] + data.extend(rows) + + return data + + +def get_broken_links( + md_content: str, include_images: bool = True, include_relative_urls: bool = False +) -> list[str]: + html = frappe.utils.md_to_html(md_content) + soup = BeautifulSoup(html, "html.parser") + + links = soup.find_all("a") + if include_images: + links += soup.find_all("img") + + broken_links = [] + for el in links: + url = el.attrs.get("href") or el.attrs.get("src") + + if is_hash_link(url): + continue + + is_relative = is_relative_url(url) + relative_url = None + + if is_relative and not include_relative_urls: + continue + + if is_relative: + relative_url = url + url = frappe.utils.data.get_url(url) # absolute URL + + is_broken = is_broken_link(url) + if is_broken: + if is_relative: + broken_links.append(relative_url) # original URL + else: + broken_links.append(url) + + return broken_links + + +def is_relative_url(url: str) -> bool: + return url.startswith("/") + + +def is_hash_link(url: str) -> bool: + return url.startswith("#") + + +def is_broken_link(url: str) -> bool: + try: + status_code = get_request_status_code(url) + if status_code >= 400: + return True + except Exception: + return True + + return False + + +def get_request_status_code(url: str) -> int: + response = requests.head(url, verify=False, timeout=5) + return response.status_code diff --git a/wiki/wiki/workspace/wiki/wiki.json b/wiki/wiki/workspace/wiki/wiki.json index 571f08bb..1cf0dd40 100644 --- a/wiki/wiki/workspace/wiki/wiki.json +++ b/wiki/wiki/workspace/wiki/wiki.json @@ -1,6 +1,7 @@ { + "app": "wiki", "charts": [], - "content": "[{\"id\":\"f6laZQUa0x\",\"type\":\"header\",\"data\":{\"text\":\"Wiki\",\"col\":12}},{\"id\":\"ir8Llemis5\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Pages\",\"col\":4}},{\"id\":\"tZQ_AtqABm\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Space\",\"col\":4}},{\"id\":\"z4qT3yMggL\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Settings\",\"col\":4}},{\"id\":\"cTIBC0weUT\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Page Patches\",\"col\":4}},{\"id\":\"IfrRKY62Tc\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Feedback\",\"col\":4}},{\"id\":\"BsC6YwujPn\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Page Revisions\",\"col\":4}}]", + "content": "[{\"id\":\"f6laZQUa0x\",\"type\":\"header\",\"data\":{\"text\":\"Wiki\",\"col\":12}},{\"id\":\"ir8Llemis5\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Pages\",\"col\":4}},{\"id\":\"tZQ_AtqABm\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Space\",\"col\":4}},{\"id\":\"z4qT3yMggL\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Settings\",\"col\":4}},{\"id\":\"cTIBC0weUT\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Page Patches\",\"col\":4}},{\"id\":\"IfrRKY62Tc\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Feedback\",\"col\":4}},{\"id\":\"BsC6YwujPn\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Wiki Page Revisions\",\"col\":4}},{\"id\":\"3gsyKHzfOC\",\"type\":\"header\",\"data\":{\"text\":\"Reports\",\"col\":12}},{\"id\":\"SI4uvLzSVb\",\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Broken Links Report\",\"col\":4}}]", "creation": "2022-09-25 16:45:20.547072", "custom_blocks": [], "docstatus": 0, @@ -12,7 +13,7 @@ "is_hidden": 0, "label": "Wiki", "links": [], - "modified": "2024-06-03 16:10:59.630080", + "modified": "2024-12-17 16:49:43.372512", "modified_by": "Administrator", "module": "Wiki", "name": "Wiki", @@ -33,6 +34,13 @@ "stats_filter": "{\"published\":[\"=\",1]}", "type": "DocType" }, + { + "color": "Grey", + "doc_view": "List", + "label": "Broken Links Report", + "link_to": "Wiki Broken Links", + "type": "Report" + }, { "color": "Grey", "doc_view": "List", @@ -72,5 +80,6 @@ "type": "DocType" } ], - "title": "Wiki" + "title": "Wiki", + "type": "Workspace" } \ No newline at end of file