feat: split client and remake boilerplate (#3)

* old prototype

* redesign

* update

* split client and utils to complier

* pre release

README.md

@@ -56,6 +56,7 @@ Jigoku apply connection retry on failed, every your network changes or something
 - Pure scraping, does not hit the API
 - Download with ease
 - Interactive prompt
+- ~~Multithreaded~~ too nasty, low coverage
 
 ## Site support
 
@@ -85,10 +86,6 @@ Jigoku supports the following imageboards:
 	<td><b>NOTE:</b> Python 3.7 or above</td>
 </table>
 
-Jigoku depends on
-- [requests](https://pypi.org/project/requests/) Python HTTP Client
-- [beautifulsoup4](https://pypi.org/project/beautifulsoup4/) HTML Parser
-
 ## Installation
 It's fairly simple to use jigoku
 

jigoku/__init__.py

@@ -1,6 +1,6 @@
-__version__ = "2.2.2"
+__version__ = "2.2.3"
 from .app import main
 from .utils.log import log_data, log_time, get_hostname
 from .utils.disk import get_size, clean_html
-from .utils.scrape_posts import download_from_multiple_posts
-from .utils.scrape_pages import download_from_multiple_pages
+from .client.scrape_posts import download_from_multiple_posts
+from .client.scrape_pages import download_from_multiple_pages

jigoku/app.py

@@ -3,28 +3,23 @@
 import time
 from inputimeout import inputimeout, TimeoutOccurred
 from jigoku.utils.log import log_time
-from jigoku.utils.scrape_posts import download_from_multiple_posts
-from jigoku.utils.scrape_pages import download_from_multiple_pages
+from jigoku.client.scrape_posts import download_from_multiple_posts
+from jigoku.client.scrape_pages import download_from_multiple_pages
 
 
-# from jigoku.utils.log import log_time
-## which is takes all galleries
-
-## from utils.log import log_time
-
 try:
     file = inputimeout(
-        prompt="[+] Please enter file you want to bulk download (e.g. file.txt): ", 
+        prompt="[+] Please enter file you want to bulk download (e.g. file.txt): ",
         timeout=30
     )
-    
+
     download_by = inputimeout(
         prompt="[+] Please choose (1) Multiple posts or (2) Multiple pages: ",
         timeout=30,
     )
-    
+
     select_type = inputimeout(
-        prompt="[+] Select type image results (1) Original size or (2) Smaller size: ", 
+        prompt="[+] Select type image results (1) Original size or (2) Smaller size: ",
         timeout=30
     )
 

jigoku/utils/scrape_pages.py → jigoku/client/scrape_pages.py

@@ -2,12 +2,10 @@
 import os
 import time
 import re
-import pkg_resources
 from bs4 import BeautifulSoup
 from jigoku.utils.log import log_data, get_hostname
 from jigoku.utils.disk import get_size, clean_html
-from jigoku.utils.constant import Jigoku
-version = pkg_resources.require("jigoku")[0].version
+from jigoku.constant.base import Jigoku
 
 jgx = Jigoku()
 

jigoku/utils/scrape_posts.py → jigoku/client/scrape_posts.py

@@ -2,14 +2,10 @@
 import os
 import time
 import re
-import pkg_resources
 from bs4 import BeautifulSoup
 from jigoku.utils.log import log_data
 from jigoku.utils.disk import get_size
-from jigoku.utils.constant import Jigoku
-version = pkg_resources.require("jigoku")[0].version
-
-# from jigoku.utils.log import log_data, get_hostname
+from jigoku.constant.base import Jigoku
 
 jgx = Jigoku()
 
@@ -155,7 +151,7 @@ def download_from_multiple_posts(file: str, select_type: str) -> None:
                             if not os.path.exists(final_name):
                                 img_count += 1
                                 ##current_progress = f"{img_count} / {len(f.readlines()) + 1}"
-                                image = requests.get(final_img, stream=True)
+                                image = requests.get(final_img, stream=True, headers=jgx.with_headers)
                                 with open(final_name, "wb") as f:
                                     f.write(image.content)
 

setup.py

@@ -33,7 +33,7 @@
         "Issue tracker": "https://github.com/sinkaroid/jigoku/issues/new/choose",
         "Documentation": "https://github.com/sinkaroid/jigoku/wiki",
     },
-    packages=['jigoku', 'jigoku.utils'],
+    packages=['jigoku', 'jigoku.constant', 'jigoku.utils', 'jigoku.client'],
     license='MIT',
     classifiers=[
         'Development Status :: 3 - Alpha',