From 56c57a0b93a8a28b7cf286b6b1de2c5a89f5fc90 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Fri, 1 Dec 2023 17:08:40 +0000 Subject: [PATCH] Init --- .github/workflows/build.yaml | 35 ++ .github/workflows/release.yaml | 55 +++ .gitignore | 85 ++++ .pre-commit-config.yaml | 45 ++ LICENSE | 202 +++++++++ README.md | 14 + mangaupdates_talker/__init__.py | 1 + mangaupdates_talker/mangaupdates.py | 658 ++++++++++++++++++++++++++++ pyproject.toml | 19 + requirements-dev.txt | 11 + setup.cfg | 133 ++++++ setup.py | 6 + 12 files changed, 1264 insertions(+) create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 mangaupdates_talker/__init__.py create mode 100644 mangaupdates_talker/mangaupdates.py create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..8ccb306 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,35 @@ +name: CI + +on: + pull_request: + push: + branches: + - '**' + +jobs: + build-and-publish: + runs-on: ${{ matrix.os }} + strategy: + matrix: + python_version: ['3.9'] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + + + - name: Install build dependencies + run: | + python -m pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt + + - name: Build and install wheel + run: | + tox run -m build + python -m pip install dist/*.whl diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..a48b5e1 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,55 @@ +name: CI + +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+*" + +jobs: + build-and-publish: + runs-on: ubuntu-latest + # Specifying a GitHub environment is optional, but strongly encouraged + environment: release + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write + contents: write + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install build dependencies + run: | + python -m pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt + + - name: Build and install wheel + run: | + tox run -m build + python -m pip install dist/*.whl + + - name: "Publish distribution 📦 to PyPI" + if: startsWith(github.ref, 'refs/tags/') + uses: pypa/gh-action-pypi-publish@release/v1 + + - name: Get release name + if: startsWith(github.ref, 'refs/tags/') + shell: bash + run: | + git fetch --depth=1 origin +refs/tags/*:refs/tags/* # github is dumb + echo "release_name=$(git tag -l --format "%(refname:strip=2): %(contents:lines=1)" ${{ github.ref_name }})" >> $GITHUB_ENV + + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + name: "${{ env.release_name }}" + draft: false + files: | + dist/*.whl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d6bed2f --- /dev/null +++ b/.gitignore @@ -0,0 +1,85 @@ +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion + +*.iml + +## Directory-based project format: +.idea/ + +### Other editors +.*.swp +nbproject/ +.vscode + +*.exe +*.zip + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# for testing +temp/ +tmp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7d1a1e0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: debug-statements + - id: name-tests-test + - id: requirements-txt-fixer +- repo: https://github.com/asottile/setup-cfg-fmt + rev: v2.4.0 + hooks: + - id: setup-cfg-fmt +- repo: https://github.com/PyCQA/autoflake + rev: v2.2.1 + hooks: + - id: autoflake + args: [-i, --remove-all-unused-imports, --ignore-init-module-imports] +- repo: https://github.com/asottile/pyupgrade + rev: v3.10.1 + hooks: + - id: pyupgrade + args: [--py39-plus] +- repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: [--af,--add-import, 'from __future__ import annotations'] +- repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black +- repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + additional_dependencies: [flake8-encodings, flake8-builtins, flake8-length, flake8-print] +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.5.1 + hooks: + - id: mypy + additional_dependencies: [types-setuptools, types-requests, settngs>=0.7.1] +ci: + skip: [mypy] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..05b6a0c --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +# MangaUpdates plugin for Comic Tagger + +A plugin for [Comic Tagger](https://github.com/comictagger/comictagger/releases) to allow the use of the metadata from [MangaUpdates](https://mangaupdates.com). + +## Work in progress + +The 'master' branch attempts to be current with comictagger 'develop' branch + +When using the GUI the issues window list will always be empty as MangaUpdates does not store issue level information. Clicking `OK` will tag using the available series data. + +## Install + +`pip install .` +If ComicTagger is not installed in your Python environment already `pip install comictagger` (plus any other optional dependancies required `[GUI]` etc.) diff --git a/mangaupdates_talker/__init__.py b/mangaupdates_talker/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/mangaupdates_talker/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/mangaupdates_talker/mangaupdates.py b/mangaupdates_talker/mangaupdates.py new file mode 100644 index 0000000..b339eab --- /dev/null +++ b/mangaupdates_talker/mangaupdates.py @@ -0,0 +1,658 @@ +""" +MangaUpdates information source +""" +# Copyright comictagger team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import argparse +import json +import logging +import pathlib +import re +import time +from typing import Any, Callable, cast +from urllib.parse import urljoin + +import requests +import settngs +from comicapi import utils +from comicapi.genericmetadata import ComicSeries, GenericMetadata, TagOrigin +from comictalker import talker_utils +from comictalker.comiccacher import ComicCacher +from comictalker.comiccacher import Series as CCSeries +from comictalker.comictalker import ComicTalker, TalkerDataError, TalkerNetworkError +from pyrate_limiter import Duration, Limiter, RequestRate +from typing_extensions import TypedDict + +logger = logging.getLogger(f"comictalker.{__name__}") + + +class MUGenre(TypedDict, total=False): + genre: str + color: str + + +class MUImageURL(TypedDict): + original: str + thumb: str + + +class MUImage(TypedDict): + url: MUImageURL + height: int + width: int + + +class MULastUpdated(TypedDict): + timestamp: int + as_rfc3339: str + as_string: str + + +class MURecord(TypedDict, total=False): + series_id: int + title: str + url: str + description: str + image: MUImage + type: str + year: str + bayesian_rating: float + rating_votes: int + genres: list[MUGenre] + last_updated: MULastUpdated + + +class MUStatus(TypedDict): + volume: int + chapter: int + + +class MUUserList(TypedDict): + list_type: str + list_icon: str + status: MUStatus + + +class MUMetadata(TypedDict): + user_list: MUUserList + + +class MUResult(TypedDict): + record: MURecord + hit_title: str + metadata: MUMetadata + user_genre_highlights: list[MUGenre] + + +class MUResponse(TypedDict): + total_hits: int + page: int + per_page: int + results: list[MUResult] + + +class MUVolumeReply(TypedDict): + reason: str + status: str + context: dict[Any, Any] + total_hits: int + page: int + per_page: int + results: list[MUResult] | MUResult + + +class MUAssTitle(TypedDict): + title: str + + +class MUCategories(TypedDict): + series_id: int + category: str + votes: int + votes_plus: int + votes_minus: int + added_by: int + + +class MUAnime(TypedDict): + start: str + end: str + + +class MURelatedSeries(TypedDict): + relation_id: int + relation_type: str + related_series_id: int + related_series_name: str + triggered_by_relation_id: int + + +class MUAuthor(TypedDict): + name: str + author_id: int + type: str + + +class MUPublisher(TypedDict): + publisher_name: str + publisher_id: int + type: str + notes: str + + +class MUPublication(TypedDict): + publication_name: str + publisher_name: str + publisher_id: int + + +class MURecommendations(TypedDict): + series_name: str + series_id: int + weight: int + + +class MUPosition(TypedDict): + week: int + month: int + three_months: int + six_months: int + year: int + + +class MULists(TypedDict): + reading: int + wish: int + complete: int + unfinished: int + custom: int + + +class MURank(TypedDict): + position: MUPosition + old_position: MUPosition + + +class MUSeries(TypedDict, total=False): + series_id: int + title: str + url: str + associated: list[MUAssTitle] + description: str + image: MUImage + type: str + year: str + bayesian_rating: float + rating_votes: int + genres: list[MUGenre] + categories: list[MUCategories] + latest_chapter: int + forum_id: int + status: str + licensed: bool + completed: bool + anime: MUAnime + related_series: list[MURelatedSeries] + authors: list[MUAuthor] + publishers: list[MUPublisher] + publications: list[MUPublication] + recommendations: list[MURecommendations] + category_recommendations: list[MURecommendations] + rank: MURank + last_updated: MULastUpdated + + +# MangaUpdates states: You will use reasonable spacing between requests so as not to overwhelm the MangaUpdates servers +limiter = Limiter(RequestRate(5, Duration.SECOND)) + + +class MangaUpdatesTalker(ComicTalker): + name: str = "MangaUpdates" + id: str = "mangaupdates" + logo_url: str = "https://www.mangaupdates.com/images/mascot.gif" + website: str = "https://mangaupdates.com/" + attribution: str = f"Metadata provided by {name}" + about: str = ( + f"{name} (also known as Baka-Updates Manga) is a site dedicated to bringing the manga " + f"community (and by extension the manhwa, manhua, etc, communities) the latest scanlation and series " + f"information. We were founded in July 2004 and are the sister site of Baka-Updates." + ) + + def __init__(self, version: str, cache_folder: pathlib.Path): + super().__init__(version, cache_folder) + # Settings + self.default_api_url = self.api_url = "https://api.mangaupdates.com/v1/" + self.use_series_start_as_volume: bool = False + self.use_search_title: bool = False + self.use_original_publisher: bool = False + self.use_ongoing_issue_count: bool = False + self.filter_nsfw: bool = False + self.add_nsfw_rating: bool = False + self.filter_dojin: bool = True + + def register_settings(self, parser: settngs.Manager) -> None: + parser.add_setting( + "--mu_use-series-start-as-volume", + default=False, + action=argparse.BooleanOptionalAction, + display_name="Use series start as volume", + ) + parser.add_setting( + "--mu-use-search-title", + default=False, + action=argparse.BooleanOptionalAction, + display_name="Use search title", + help="Use search title result instead of the English title", + ) + parser.add_setting( + "--mu-use-ongoing", + default=False, + action=argparse.BooleanOptionalAction, + display_name="Use the ongoing issue count", + help='If a series is labelled as "ongoing", use the current issue count (otherwise empty)', + ) + parser.add_setting( + "--mu-use-original-publisher", + default=False, + action=argparse.BooleanOptionalAction, + display_name="Use the original publisher", + help="Use the original publisher instead of English language publisher", + ) + parser.add_setting( + "--mu-filter-nsfw", + default=False, + action=argparse.BooleanOptionalAction, + display_name="Filter out NSFW results", + help="Filter out NSFW from the search results (Genre: Adult and Hentai)", + ) + parser.add_setting( + "--mu-add-nsfw-rating", + default=False, + action=argparse.BooleanOptionalAction, + display_name="Add 'Adult' maturity rating if 'Adult' or 'Hentai' genre", + help="Add a maturity rating of 'Adult' if the genre is 'Adult' or 'Hentai'", + ) + parser.add_setting( + "--mu-filter-dojin", + default=True, + action=argparse.BooleanOptionalAction, + display_name="Filter out dojin results", + help="Filter out dojin from the search results (Genre: Doujinshi)", + ) + parser.add_setting( + f"--{self.id}-url", + display_name="API URL", + help=f"Use the given Manga Updates URL. (default: {self.default_api_url})", + ) + parser.add_setting(f"--{self.id}-key", file=False, cmdline=False) + + def parse_settings(self, settings: dict[str, Any]) -> dict[str, Any]: + settings = super().parse_settings(settings) + + self.use_series_start_as_volume = settings["mu_use_series_start_as_volume"] + self.use_search_title = settings["mu_use_search_title"] + self.use_ongoing_issue_count = settings["mu_use_ongoing"] + self.use_original_publisher = settings["mu_use_original_publisher"] + self.filter_nsfw = settings["mu_filter_nsfw"] + self.add_nsfw_rating = settings["mu_add_nsfw_rating"] + self.filter_dojin = settings["mu_filter_dojin"] + + return settings + + def check_status(self, settings: dict[str, Any]) -> tuple[str, bool]: + url = talker_utils.fix_url(settings[f"{self.id}_url"]) + if not url: + url = self.default_api_url + try: + mu_response = requests.get( + url, + headers={"user-agent": "comictagger/" + self.version}, + ).json() + + if mu_response["status"] == "success": + return "The URL is valid", True + else: + return "The URL is INVALID!", False + except Exception: + return "Failed to connect to the URL!", False + + def search_for_series( + self, + series_name: str, + callback: Callable[[int, int], None] | None = None, + refresh_cache: bool = False, + literal: bool = False, + series_match_thresh: int = 90, + ) -> list[ComicSeries]: + search_series_name = utils.sanitize_title(series_name, literal) + logger.info(f"{self.name} searching: {search_series_name}") + + # Before we search online, look in our cache, since we might have done this same search recently + # For literal searches always retrieve from online + cvc = ComicCacher(self.cache_folder, self.version) + if not refresh_cache and not literal: + cached_search_results = cvc.get_search_results(self.id, series_name) + if len(cached_search_results) > 0: + # Unpack to apply any filters + json_cache: list[MUSeries] = [json.loads(x[0].data) for x in cached_search_results] + if self.filter_nsfw: + json_cache = self._filter_nsfw(json_cache) + if self.filter_dojin: + json_cache = self._filter_dojin(json_cache) + + return self._format_search_results(json_cache) + + params: dict[str, Any] = { + "search": search_series_name, + "page": 1, + "perpage": 100, + } + + mu_response = self._get_mu_content(urljoin(self.api_url, "series/search"), params) + + search_results: list[MUSeries] = [] + + total_result_count = mu_response["total_hits"] + + # 1. Don't fetch more than some sane amount of pages. + # 2. Halt when any result on the current page is less than or equal to a set ratio using thefuzz + max_results = 500 # 5 pages + + current_result_count = mu_response["per_page"] * mu_response["page"] + total_result_count = min(total_result_count, max_results) + + if callback is None: + logger.debug( + f"Found {mu_response['per_page'] * mu_response['page']} of {mu_response['total_hits']} results" + ) + search_results.extend(s["record"] for s in mu_response["results"]) + page = 1 + + if callback is not None: + callback(current_result_count, total_result_count) + + # see if we need to keep asking for more pages... + while current_result_count < total_result_count: + if not literal: + # Stop searching once any entry falls below the threshold + stop_searching = any( + not utils.titles_match(search_series_name, volume["record"]["title"], series_match_thresh) + for volume in cast(list[MUResult], mu_response["results"]) + ) + + if stop_searching: + break + + if callback is None: + logger.debug(f"getting another page of results {current_result_count} of {total_result_count}...") + page += 1 + + params["page"] = page + mu_response = self._get_mu_content(urljoin(self.api_url, "series/search"), params) + + search_results.extend(s["record"] for s in mu_response["results"]) + # search_results.extend(cast(list[MUResult], mu_response["results"])) + # current_result_count += mu_response["number_of_page_results"] + + if callback is not None: + callback(current_result_count, total_result_count) + + # Cache raw data + cvc.add_search_results( + self.id, + series_name, + [CCSeries(id=x["series_id"], data=json.dumps(x).encode("utf-8")) for x in search_results], + False, + ) + + # Filter any tags AFTER adding to cache + if self.filter_nsfw: + search_results = self._filter_nsfw(search_results) + if self.filter_dojin: + search_results = self._filter_dojin(search_results) + + formatted_search_results = self._format_search_results(search_results) + + return formatted_search_results + + def fetch_comic_data( + self, issue_id: str | None = None, series_id: str | None = None, issue_number: str = "" + ) -> GenericMetadata: + comic_data = GenericMetadata() + # Could be sent "issue_id" only which is actually series_id + if issue_id and series_id is None: + series_id = issue_id + + if series_id is not None: + return self._map_comic_issue_to_metadata(self._fetch_series(int(series_id))) + + return comic_data + + def fetch_issues_in_series(self, series_id: str) -> list[GenericMetadata]: + # Manga Updates has no issue level data + return [GenericMetadata()] + + @limiter.ratelimit("default", delay=True) + def _get_mu_content(self, url: str, params: dict[str, Any]) -> MUResponse | MUSeries | MUPublisher: + while True: + mu_response = self._get_url_content(url, params) + if mu_response.get("status") == "exception": + logger.debug(f"{self.name} query failed with error {mu_response['reason']}.") + raise TalkerNetworkError(self.name, 0, f"{mu_response['reason']}") + + break + return mu_response + + def _get_url_content(self, url: str, params: dict[str, Any]) -> Any: + for tries in range(3): + try: + if not params: + resp = requests.get(url, headers={"user-agent": "comictagger/" + self.version}) + else: + resp = requests.post(url, json=params, headers={"user-agent": "comictagger/" + self.version}) + + if resp.status_code == requests.status_codes.codes.ok: + return resp.json() + if resp.status_code == requests.status_codes.codes.server_error: + logger.debug(f"Try #{tries + 1}: ") + time.sleep(1) + logger.debug(str(resp.status_code)) + if resp.status_code == requests.status_codes.codes.bad_request: + logger.debug(f"Bad request: {resp.json()}") + raise TalkerNetworkError(self.name, 2, f"Bad request: {resp.json()}") + if resp.status_code == requests.status_codes.codes.forbidden: + logger.debug(f"Forbidden: {resp.json()}") + raise TalkerNetworkError(self.name, 2, f"Forbidden: {resp.json()}") + if resp.status_code == requests.status_codes.codes.not_found: + logger.debug(f"Manga not found: {resp.json()}") + raise TalkerNetworkError(self.name, 2, f"Manga not found: {resp.json()}") + if resp.status_code == requests.status_codes.codes.too_many_requests: + logger.debug(f"Rate limit reached: {resp.json()}") + # If given a time to wait before re-trying, use that time + 1 sec + if resp.headers.get("x-ratelimit-retry-after", None): + wait_time = int(resp.headers["x-ratelimit-retry-after"]) - int(time.time()) + if wait_time > 0: + time.sleep(wait_time + 1) + else: + time.sleep(5) + else: + break + + except requests.exceptions.Timeout: + logger.debug(f"Connection to {self.name} timed out.") + raise TalkerNetworkError(self.name, 4) + except requests.exceptions.RequestException as e: + logger.debug(f"Request exception: {e}") + raise TalkerNetworkError(self.name, 0, str(e)) from e + except json.JSONDecodeError as e: + logger.debug(f"JSON decode error: {e}") + raise TalkerDataError(self.name, 2, f"{self.name} did not provide json") + + raise TalkerNetworkError(self.name, 5) + + def _format_search_results(self, search_results: list[MUSeries]) -> list[ComicSeries]: + formatted_results = [] + for record in search_results: + formatted_results.append(self._format_series(record)) + + return formatted_results + + def _format_series(self, series: MUSeries) -> ComicSeries: + aliases = set() + for alias in series.get("associated", []): + aliases.add(alias["title"]) + + start_year: int | None = None + if series.get("year"): + start_year = utils.xlate_int(series["year"]) + + publisher = None + if series.get("publishers"): + publisher_list = [] + for pub in series["publishers"]: + if self.use_original_publisher and pub["type"] == "Original": + publisher_list.append(pub["publisher_name"]) + elif pub["type"] == "English": + publisher_list.append(pub["publisher_name"]) + publisher = ", ".join(publisher_list) + + count_of_issues = None + if series.get("completed"): + count_of_issues = series["latest_chapter"] + + return ComicSeries( + aliases=aliases, + count_of_issues=count_of_issues, + description=series.get("description", ""), + id=str(series["series_id"]), + image_url=series["image"]["url"].get("original", ""), + name=series.get("title", ""), + publisher=publisher, + start_year=start_year, + count_of_volumes=None, + format=None, + ) + + def _fetch_publisher(self, pub_id: int) -> MUPublisher: + mu_response = self._get_mu_content(urljoin(self.api_url, f"publishers/{pub_id}"), {}) + + return cast(MUPublisher, mu_response) + + def _filter_nsfw(self, search_results: list[MUSeries]) -> list[MUSeries]: + filtered_list = [] + for series in search_results: + if not any(genre in ["Adult", "Hentai"] for genre in series.get("genres", [])): + filtered_list.append(series) + + return filtered_list + + def _filter_dojin(self, search_results: list[MUSeries]) -> list[MUSeries]: + filtered_list = [] + for series in search_results: + if "Doujinshi" not in series.get("genres", []): + filtered_list.append(series) + + return filtered_list + + def fetch_series(self, series_id: str) -> ComicSeries: + return self._format_series(self._fetch_series(int(series_id))) + + def _fetch_series(self, series_id: int) -> MUSeries: + cvc = ComicCacher(self.cache_folder, self.version) + cached_series = cvc.get_series_info(str(series_id), self.id) + + if cached_series is not None and cached_series[1]: + return json.loads(cached_series[0].data) + + issue_url = urljoin(self.api_url, f"series/{series_id}") + mu_response: MUSeries = self._get_mu_content(issue_url, {}) + + # Series will now have publisher so update cache record + # Cache raw data + cvc.add_series_info( + self.id, + CCSeries(id=str(series_id), data=json.dumps(mu_response).encode("utf-8")), + True, + ) + + return mu_response + + def fetch_issues_by_series_issue_num_and_year( + self, series_id_list: list[str], issue_number: str, year: str | int | None + ) -> list[GenericMetadata]: + series_list = [] + for series_id in series_id_list: + series_list.append(self._map_comic_issue_to_metadata(self._fetch_series(int(series_id)))) + + return series_list + + def _map_comic_issue_to_metadata(self, series: MUSeries) -> GenericMetadata: + md = GenericMetadata( + tag_origin=TagOrigin(self.id, self.name), + series_id=utils.xlate(series["series_id"]), + issue_id=utils.xlate(series["series_id"]), + ) + md.cover_image = series["image"]["url"].get("original", "") + + for alias in series["associated"]: + md.series_aliases.add(alias["title"]) + + publisher_list = [] + for pub in series["publishers"]: + if not self.use_original_publisher and pub["type"] == "English": + publisher_list.append(pub["publisher_name"]) + else: + publisher_list.append(pub["publisher_name"]) + + md.publisher = ", ".join(publisher_list) + + for person in series["authors"]: + md.add_credit(person["name"], person["type"]) + + # Types: Artbook, Doujinshi, Drama CD, Filipino, Indonesian, Manga, Manhwa, Manhua, Novel, OEL, Thai, + # Vietnamese, Malaysian, Nordic, French, Spanish + if series["type"] in ["Manga", "Doujinshi"]: + md.manga = "Yes" + + for genre in series["genres"]: + md.genres.add(genre["genre"]) + if genre in ["Adult", "Hentai"]: + md.mature_rating = "Adult" + + for cat in series["categories"]: + md.tags.add(cat["category"]) + + count_of_volumes: int | None = None # TODO parse from publisher notes depending on lang? + reg = re.compile(r"((\d+).*volume.).*(complete)(.*)", re.IGNORECASE) + reg_match = reg.search(series["status"]) + if reg_match is not None: + count_of_volumes = utils.xlate_int(reg_match.group(2)) + + # Marked as complete so latest_chapter can be confirmed as number of chapters + if series["completed"] or self.use_ongoing_issue_count: + md.count_of_issues = series["latest_chapter"] + md.count_of_volumes = count_of_volumes + + md.year = utils.xlate_int(series.get("year")) + + md.description = series.get("description") + + md.web_link = series["url"] + + if self.use_series_start_as_volume and md.year: + md.volume = md.year + + return md diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ec24fd4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[tool.black] +line-length = 120 + +[tool.isort] +line_length = 120 +profile = "black" + +[build-system] +requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +local_scheme = "no-local-version" + +[tool.pylint.messages_control] +disable = "C0330, C0326, C0115, C0116, C0103" + +[tool.pylint.format] +max-line-length=120 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..d4a5c16 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,11 @@ +black>=22 +flake8==4.* +flake8-black +flake8-encodings +flake8-isort +invoke +isort>=5.10 +pytest==7.* +setuptools>=42 +setuptools_scm[toml]>=3.4 +wheel diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..4a6de6b --- /dev/null +++ b/setup.cfg @@ -0,0 +1,133 @@ +[metadata] +name = mangaupdates_talker +version = 0.0.1 +description = A MangaUpdates API talker for ComicTagger a cross-platform GUI/CLI app for writing metadata to comic archives +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/mizaki/mangaupdates-talker +author = ComicTagger team +author_email = comictagger@gmail.com +license = Apache-2.0 +license_files = LICENSE +classifiers = + Development Status :: 3 - Alpha + Environment :: Console + Environment :: MacOS X + Environment :: Plugins + Environment :: Win32 (MS Windows) + Environment :: X11 Applications :: Qt + Intended Audience :: End Users/Desktop + License :: OSI Approved :: Apache Software License + Natural Language :: English + Operating System :: OS Independent + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Topic :: Multimedia :: Graphics + Topic :: Other/Nonlisted Topic + Topic :: Utilities +keywords = + comictagger + mangaupdates + comics + comic + metadata + tagging + tagger + +[options] +packages = find: +python_requires = >=3.9 + +[options.packages.find] +exclude = tests; testing +src = mangaupdates_talker + +[options.entry_points] +comictagger.talker = + mangaupdates = mangaupdates_talker.mangaupdates:MangaUpdatesTalker + +[options.extras_require] +dev = + black>=22 + flake8==4.* + flake8-black + flake8-encodings + flake8-isort + invoke + isort>=5.10 + pytest==7.* + setuptools>=42 + setuptools-scm[toml]>=3.4 + wheel + +[tox:tox] +envlist = py3.9 + +[testenv] +deps = -requirements-dev.txt +commands = + coverage erase + coverage run -m pytest {posargs:tests} + coverage report + +[testenv:wheel] +description = Generate wheel and tar.gz +labels = + release + build +skip_install = true +deps = + build +commands_pre = + -python -c 'import shutil,pathlib; \ + shutil.rmtree("./build/", ignore_errors=True); \ + shutil.rmtree("./dist/", ignore_errors=True)' +commands = + python -m build + +[testenv:pypi-upload] +description = Upload wheel to PyPi +platform = Linux +labels = + release +skip_install = true +depends = wheel +deps = + twine +passenv = + TWINE_* +setenv = + TWINE_NON_INTERACTIVE=true +commands = + python -m twine upload dist/*.whl dist/*.tar.gz + +[pep8] +ignore = E265,E501 +max_line_length = 120 + +[flake8] +extend-ignore = E501, A003 +max_line_length = 120 +per-file-ignores = + *_test.py: LN001 + +[coverage:run] +plugins = covdefaults + +[coverage:report] +fail_under = 95 + +[mypy] +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true + +[mypy-testing.*] +disallow_untyped_defs = false + +[mypy-tests.*] +disallow_untyped_defs = false diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8b9cbd3 --- /dev/null +++ b/setup.py @@ -0,0 +1,6 @@ +# Setup file for comictagger Metron talker python source +from __future__ import annotations + +from setuptools import setup + +setup()