[wpmadara] add support

docs/supportedsites.md

@@ -535,12 +535,6 @@ Consider all listed sites to potentially be NSFW.
     <td>Chapters, Manga</td>
     <td></td>
 </tr>
-<tr>
-    <td>MangaRead</td>
-    <td>https://mangaread.org/</td>
-    <td>Chapters, Manga</td>
-    <td></td>
-</tr>
 <tr>
     <td>MangaSee</td>
     <td>https://mangasee123.com/</td>
@@ -1723,5 +1717,27 @@ Consider all listed sites to potentially be NSFW.
     <td>Albums</td>
     <td></td>
 </tr>
+
+<tr>
+    <td colspan="4"><strong>WordPressMadara based websites</strong></td>
+</tr>
+<tr>
+    <td>MangaRead</td>
+    <td>https://mangaread.org/</td>
+    <td>Chapters, Manga</td>
+    <td></td>
+</tr>
+<tr>
+    <td>Toonily</td>
+    <td>https://toonily.com/</td>
+    <td>Chapters, Manga</td>
+    <td></td>
+</tr>
+<tr>
+    <td>WebtoonXYZ</td>
+    <td>https://www.webtoon.xyz/</td>
+    <td>Chapters, Manga</td>
+    <td></td>
+</tr>
 </tbody>
 </table>

gallery_dl/extractor/__init__.py

@@ -95,7 +95,7 @@
     "mangakakalot",
     "manganelo",
     "mangapark",
-    "mangaread",
+    "wpmadara",
     "mangasee",
     "mangoxo",
     "misskey",

gallery_dl/extractor/mangaread.py → gallery_dl/extractor/wpmadara.py

@@ -4,16 +4,16 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-"""Extractors for https://mangaread.org/"""
+"""Extractors for WordPressMadara based websites."""
 
-from .common import ChapterExtractor, MangaExtractor
+from .common import BaseExtractor, ChapterExtractor, MangaExtractor
 from .. import text, exception
 import re
 
 
-class MangareadBase():
-    """Base class for Mangaread extractors"""
-    category = "mangaread"
+class WPMadaraBase(BaseExtractor):
+    """Base class for WordPressMadara based extractors"""
+    basecategory = "wpmadara"
     root = "https://www.mangaread.org"
 
     @staticmethod
@@ -31,36 +31,69 @@ def parse_chapter_string(chapter_string, data):
         data["language"] = "English"
 
 
-class MangareadChapterExtractor(MangareadBase, ChapterExtractor):
-    """Extractor for manga-chapters from mangaread.org"""
-    pattern = (r"(?:https?://)?(?:www\.)?mangaread\.org"
-               r"(/manga/[^/?#]+/[^/?#]+)")
+BASE_PATTERN = WPMadaraBase.update({
+    "mangaread": {
+        "root": "https://www.mangaread.org",
+        "pattern": r"(?:https?://)?(?:www\.)?mangaread\.org",
+    },
+    "toonily": {
+        "root": "https://www.toonily.com",
+        "pattern": r"(?:https?://)?(?:www\.)?toonily\.com",
+    },
+    "webtoonxyz": {
+        "root": "https://www.webtoon.xyz",
+        "pattern": r"(?:https?://)?(?:www\.)?webtoon\.xyz",
+    },
+})
+
+
+class WPMadaraChapterExtractor(WPMadaraBase, ChapterExtractor):
+    """Extractor for manga-chapters from WordPressMadara based websites."""
+    subcategory = "chapter"
+    pattern = BASE_PATTERN + r"(/(manga|webtoon|read)/[^/?#]+/[^/?#]+)"
     example = "https://www.mangaread.org/manga/MANGA/chapter-01/"
 
+    def __init__(self, match, url=None):
+        WPMadaraBase.__init__(self, match)
+        self.chapter = match.group(match.lastindex)
+        self.log.debug("chapter: %s", self.chapter)
+        self.gallery_url = self.root + match.group(match.lastindex)
+
+        if self.config("chapter-reverse", False):
+            self.reverse = not self.reverse
+
     def metadata(self, page):
         tags = text.extr(page, 'class="wp-manga-tags-list">', '</div>')
         data = {"tags": list(text.split_html(tags)[::2])}
         info = text.extr(page, '<h1 id="chapter-heading">', "</h1>")
         if not info:
             raise exception.NotFoundError("chapter")
         self.parse_chapter_string(info, data)
+        self.log.debug("data: %s", data)
         return data
 
     def images(self, page):
         page = text.extr(
             page, '<div class="reading-content">', '<div class="entry-header')
+        self.log.debug("page: %s", page)
         return [
             (text.extr(img, 'src="', '"').strip(), None)
             for img in text.extract_iter(page, '<img id="image-', '>')
         ]
 
 
-class MangareadMangaExtractor(MangareadBase, MangaExtractor):
-    """Extractor for manga from mangaread.org"""
-    chapterclass = MangareadChapterExtractor
-    pattern = r"(?:https?://)?(?:www\.)?mangaread\.org(/manga/[^/?#]+)/?$"
+class WPMadaraMangaExtractor(WPMadaraBase, MangaExtractor):
+    """Extractor for manga from WordPressMadara based websites."""
+    chapterclass = WPMadaraChapterExtractor
+    subcategory = "manga"
+    pattern = BASE_PATTERN + r"(/(manga|webtoon|read)/[^/?#]+)/?$"
     example = "https://www.mangaread.org/manga/MANGA"
 
+    def __init__(self, match, url=None):
+        WPMadaraBase.__init__(self, match)
+        self.manga = match.group(match.lastindex)
+        self.manga_url = url or self.root + match.group(match.lastindex)
+
     def chapters(self, page):
         if 'class="error404' in page:
             raise exception.NotFoundError("manga")
@@ -77,12 +110,24 @@ def chapters(self, page):
     def metadata(self, page):
         extr = text.extract_from(text.extr(
             page, 'class="summary_content">', 'class="manga-action"'))
+        # rating = 0.0
+        if len(text.extr(page, 'total_votes">', "</span>").strip()) > 0:
+            rating = text.parse_float(text.extr(
+                page, 'total_votes">', "</span>").strip())
+        elif len(text.extr(page, 'property="ratingValue" id="averagerate">',
+                           "</span>").strip()) > 0:
+            rating = text.parse_float(
+                text.extr(page,
+                          'property="ratingValue" id="averagerate">',
+                          "</span>").strip())
+        else:
+            rating = 0.0
+
         return {
             "manga"      : text.extr(page, "<h1>", "</h1>").strip(),
             "description": text.unescape(text.remove_html(text.extract(
                 page, ">", "</div>", page.index("summary__content"))[0])),
-            "rating"     : text.parse_float(
-                extr('total_votes">', "</span>").strip()),
+            "rating"     : rating,
             "manga_alt"  : text.remove_html(
                 extr("Alternative </h5>\n</div>", "</div>")).split("; "),
             "author"     : list(text.extract_iter(

test/results/mangaread.py

@@ -4,15 +4,15 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-from gallery_dl.extractor import mangaread
+from gallery_dl.extractor import wpmadara
 from gallery_dl import exception
 
 
 __tests__ = (
 {
     "#url"     : "https://www.mangaread.org/manga/one-piece/chapter-1053-3/",
-    "#category": ("", "mangaread", "chapter"),
-    "#class"   : mangaread.MangareadChapterExtractor,
+    "#category": ("wpmadara", "mangaread", "chapter"),
+    "#class"   : wpmadara.WPMadaraChapterExtractor,
     "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+",
     "#count"   : 11,
 
@@ -27,15 +27,15 @@
 
 {
     "#url"     : "https://www.mangaread.org/manga/one-piece/chapter-1000000/",
-    "#category": ("", "mangaread", "chapter"),
-    "#class"   : mangaread.MangareadChapterExtractor,
+    "#category": ("wpmadara", "mangaread", "chapter"),
+    "#class"   : wpmadara.WPMadaraChapterExtractor,
     "#exception": exception.NotFoundError,
 },
 
 {
     "#url"     : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/",
-    "#category": ("", "mangaread", "chapter"),
-    "#class"   : mangaread.MangareadChapterExtractor,
+    "#category": ("wpmadara", "mangaread", "chapter"),
+    "#class"   : wpmadara.WPMadaraChapterExtractor,
     "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+",
     "#count"   : 9,
 
@@ -51,8 +51,8 @@
 {
     "#url"     : "https://www.mangaread.org/manga/above-all-gods/chapter146-5/",
     "#comment" : "^^ no whitespace",
-    "#category": ("", "mangaread", "chapter"),
-    "#class"   : mangaread.MangareadChapterExtractor,
+    "#category": ("wpmadara", "mangaread", "chapter"),
+    "#class"   : wpmadara.WPMadaraChapterExtractor,
     "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+",
     "#count"   : 6,
 
@@ -67,8 +67,8 @@
 
 {
     "#url"     : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi",
-    "#category": ("", "mangaread", "manga"),
-    "#class"   : mangaread.MangareadMangaExtractor,
+    "#category": ("wpmadara", "mangaread", "manga"),
+    "#class"   : wpmadara.WPMadaraMangaExtractor,
     "#pattern" : r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+([_-].+)?/",
     "#count"   : ">= 13",
 
@@ -93,8 +93,8 @@
 
 {
     "#url"     : "https://www.mangaread.org/manga/one-piece",
-    "#category": ("", "mangaread", "manga"),
-    "#class"   : mangaread.MangareadMangaExtractor,
+    "#category": ("wpmadara", "mangaread", "manga"),
+    "#class"   : wpmadara.WPMadaraMangaExtractor,
     "#pattern" : r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/",
     "#count"   : ">= 1066",
 
@@ -114,8 +114,8 @@
 
 {
     "#url"     : "https://www.mangaread.org/manga/doesnotexist",
-    "#category": ("", "mangaread", "manga"),
-    "#class"   : mangaread.MangareadMangaExtractor,
+    "#category": ("wpmadara", "mangaread", "manga"),
+    "#class"   : wpmadara.WPMadaraMangaExtractor,
     "#exception": exception.HttpError,
 },
 

test/results/toonily.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import wpmadara
+from gallery_dl import exception
+
+
+__tests__ = (
+{
+    "#url"     : "https://toonily.com/webtoon/such-a-cute-spy/chapter-36/",
+    "#category": ("wpmadara", "toonily", "chapter"),
+    "#class"   : wpmadara.WPMadaraChapterExtractor,
+    "#pattern" : r"https://toonily\.com/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+",
+    "#count"   : 11,
+
+    "manga"        : "Jinxed",
+    "title"        : "",
+    "chapter"      : 36,
+    "tags"         : ["harem"],
+    "lang"         : "en",
+    "language"     : "English",
+},
+
+{
+    "#url"     : "https://toonily.com/webtoon/such-a-cute-spy/chapter-1000000/",
+    "#category": ("wpmadara", "toonily", "chapter"),
+    "#class"   : wpmadara.WPMadaraChapterExtractor,
+    "#exception": exception.NotFoundError,
+},
+
+{
+    "#url"     : "https://toonily.com/webtoon/such-a-cute-spy",
+    "#category": ("wpmadara", "toonily", "manga"),
+    "#class"   : wpmadara.WPMadaraMangaExtractor,
+    "#pattern" : r"https://toonily\.com/webtoon/such-a-cute-spy/chapter-\d+([_-].+)?/",
+    "#count"   : ">= 13",
+
+    "manga"      : "Such a Cute Spy",
+    "author"     : ["Life of Ruin"],
+    "artist"     : ["Ganghyeon Yeo"],
+    "genres"     : [
+        "Action",
+        "Comedy",
+        "Romance",
+        "School Life",
+    ],
+    "rating"     : float,
+    "status"     : "End",
+    "lang"       : "en",
+    "language"   : "English",
+    "manga_alt"  : list,
+},
+
+{
+    "#url"     : "https://toonily.com/webtoon/doesnotexist",
+    "#category": ("wpmadara", "toonily", "manga"),
+    "#class"   : wpmadara.WPMadaraMangaExtractor,
+    "#exception": exception.HttpError,
+},
+
+)

test/results/webtoonxyz.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import wpmadara
+from gallery_dl import exception
+
+
+__tests__ = (
+{
+    "#url"     : "https://www.webtoon.xyz/read/the-world-after-the-end/chapter-105/",
+    "#category": ("wpmadara", "webtoonxyz", "chapter"),
+    "#class"   : wpmadara.WPMadaraChapterExtractor,
+    "#pattern" : r"https://www\.webtoon\.xyz/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+",
+    "#count"   : 11,
+
+    "manga"        : "The World After The End",
+    "title"        : "",
+    "chapter"      : 105,
+    "lang"         : "en",
+    "language"     : "English",
+},
+
+{
+    "#url"     : "https://www.webtoon.xyz/read/the-world-after-the-end/chapter-1000000/",
+    "#category": ("wpmadara", "webtoonxyz", "chapter"),
+    "#class"   : wpmadara.WPMadaraChapterExtractor,
+    "#exception": exception.NotFoundError,
+},
+
+{
+    "#url"     : "https://www.webtoon.xyz/read/the-world-after-the-end/",
+    "#category": ("wpmadara", "webtoonxyz", "manga"),
+    "#class"   : wpmadara.WPMadaraMangaExtractor,
+    "#pattern" : r"https://www\.webtoon\.xyz/read/such-a-cute-spy/chapter-\d+([_-].+)?/",
+    "#count"   : ">= 13",
+
+    "manga"      : "The World After The End",
+    "author"     : ["S-Cynaan", "Sing Shong"],
+    "artist"     : ["Undead Potato"],
+    "genres"     : [
+        "Action",
+        "Adventure",
+        "Fantasy",
+    ],
+    "rating"     : float,
+    "status"     : "OnGoing",
+    "lang"       : "en",
+    "language"   : "English",
+    "manga_alt"  : list,
+},
+
+{
+    "#url"     : "https://www.webtoon.xyz/read/doesnotexist",
+    "#category": ("wpmadara", "webtoonxyz", "manga"),
+    "#class"   : wpmadara.WPMadaraMangaExtractor,
+    "#exception": exception.HttpError,
+},
+
+)