From fe5415476fccfd11caac2c350d7d55ad4b040247 Mon Sep 17 00:00:00 2001 From: chatinEu Date: Wed, 3 Aug 2022 23:57:50 +0200 Subject: [PATCH] update callbacks workflow --- archive_updater.py | 3 +- src/Chapters.py | 21 ++++++++++- src/Downloaders.py | 84 +++++++++++++++++++++---------------------- src/main_functions.py | 4 ++- 4 files changed, 66 insertions(+), 46 deletions(-) diff --git a/archive_updater.py b/archive_updater.py index 1d37398..3dcecd3 100644 --- a/archive_updater.py +++ b/archive_updater.py @@ -21,7 +21,8 @@ def dev_tests(): x = Novel('n6912eh', 'My Skills Are Too Strong to Be a Heroine') - + x = x.updateObject() + print(type(x)) def check_env(): try: os.listdir('novel_list') diff --git a/src/Chapters.py b/src/Chapters.py index 0ff4a04..eb0aba2 100644 --- a/src/Chapters.py +++ b/src/Chapters.py @@ -1,5 +1,6 @@ import re - +from pandas import array +import requests @@ -37,6 +38,21 @@ def setUrl(self) -> str: def getUrl(self): return self.url + def processChapter(self,headers): + chapter_rep = requests.get(self.getUrl(), headers=headers) + chapter_rep.encoding = 'utf-8' + chapter_html = chapter_rep.text + self.setTitle(self.parseTitle(chapter_html)) + self.setContent(self.parseContent(chapter_html)) + + + def parseTitle(self,html) -> str: + pass + + def parseContent(self,html): + pass + + def validateTitle(self,title): rstr = r"[\/\\\:\*\?\"\<\>\|]" new_title = re.sub(rstr, "_", title) @@ -57,6 +73,9 @@ def cleanText(self,chapter_content): return chapter_content + def save(self,dir): + pass + def createFile(self,dir): print("titre"+self.title) print(self.num) diff --git a/src/Downloaders.py b/src/Downloaders.py index 5a049b7..dc1d851 100644 --- a/src/Downloaders.py +++ b/src/Downloaders.py @@ -1,6 +1,5 @@ # coding: utf-8 from abc import ABC -from gc import callbacks import requests import re from bs4 import BeautifulSoup @@ -8,28 +7,30 @@ from src.Chapters import * # packages for callbacks -import callback from inspect import signature from enum import Enum, auto +class Callbacks(Enum): + chapterListFetched = auto(), + ChapterBeginUpdate = auto(), + NovelBeginUpdate = auto(), class NovelCallbacks(): - # TODO: replace current callback list to 2DList {'enum',[funcs]} - class callbacks(Enum): - chapterListFetched = auto(), - ChapterBeginUpdate = auto(), - NovelBeginUpdate = auto(), - - def __init__(self): - self.callbacks_chapterListFetched = [] - self.callbacks_ChapterBeginUpdate = [] - self.callbacks_NovelBeginUpdate = [] - - def registerCallback(self, callback_List, callback): - callback_List.append(callback) - - def exec_callbacks(self, callback_list, args=None): - for method in callback_list: + """gives to subclass a""" + def __init__(self, enum: Enum = Callbacks): + self.enum = enum + self.callbacks_dict = dict() + self.init_callback_list() + + def init_callback_list(self): + for enum in self.enum: + self.callbacks_dict[enum] = [] + + def registerCallback(self, hook: Enum, callback): + self.callbacks_dict.get(hook).append(callback) + + def exec_callbacks(self,hook: Enum ,args=None): + for method in self.callbacks_dict[hook]: sig = signature(method) params = sig.parameters if (len(params) <= 1): @@ -37,15 +38,6 @@ def exec_callbacks(self, callback_list, args=None): else: method(args) - def onChapterListFetched(self, args=None): - self.exec_callbacks(self.callbacks_chapterListFetched, args) - - def onChapterBeginUpdate(self, args=None): - self.exec_callbacks(self.callbacks_ChapterBeginUpdate, args) - - def onBeginNovelUpdate(self, args=None): - self.exec_callbacks(self.callbacks_NovelBeginUpdate, args) - class Novel(NovelCallbacks): def __init__(self, codeNovel, titreNovel, keep_text_format=False): @@ -53,8 +45,17 @@ def __init__(self, codeNovel, titreNovel, keep_text_format=False): self.code = codeNovel self.titre = titreNovel self.keep_text_format = keep_text_format - - def tempFunc(self, args): + self.init_callbacks() + # should be used to return a + if(type(self)==Novel): + self.updateObject() + + + def init_callbacks(self): + """create a Novel basic callback""" + self.registerCallback(Callbacks.ChapterBeginUpdate,self.tempFunc) + + def tempFunc(self): print("callback works") def downloadNovel(self, chapter) -> str: @@ -187,11 +188,11 @@ def processNovel(self): class SyosetuNovel(Novel): def __init__(self, Novel): + super(SyosetuNovel, self).__init__(Novel.code, Novel.titre, Novel.keep_text_format) self.site = 'https://ncode.syosetu.com/' self.setUrl(self.site + self.code + "/") self.headers = { "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36"} - super(SyosetuNovel, self).__init__(Novel.code, Novel.titre, Novel.keep_text_format) def updatePerDate(self, html): """check every local file is the same version as online """ @@ -240,14 +241,15 @@ def parseOnlineChapterList(self, html='') -> list: return online_chap_list def fetchTOCPage(self): - if html == '': - url = self.url - headers = { - "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36" - } - rep = requests.get(url, headers=headers) - rep.encoding = 'utf-8' - html = rep.text + url = self.url + headers = { + "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36" + } + rep = requests.get(url, headers=headers) + rep.encoding = 'utf-8' + html = rep.text + self.html = html + return html def parseTocResume(self, html=''): soup = BeautifulSoup(html, 'html.parser') @@ -263,11 +265,7 @@ def parseTocResume(self, html=''): def processChapter(self, chapter_num): chapter = SyosetuChapter(self.code, chapter_num) - chapter_rep = requests.get(chapter.getUrl(), headers=self.headers) - chapter_rep.encoding = 'utf-8' - chapter_html = chapter_rep.text - chapter.getTitle(chapter_html) - chapter.getContent(chapter_html) + chapter.processChapter(self.headers) return chapter # self.createFile(i,chapter_title,chapter_content) # self.setLastChapter(i) diff --git a/src/main_functions.py b/src/main_functions.py index 38e40a8..d3675cd 100644 --- a/src/main_functions.py +++ b/src/main_functions.py @@ -1,5 +1,7 @@ import os +from typing import List + from src.Downloaders import * def archiveUpdate(dirList=[],keep_text_format=False): @@ -91,7 +93,7 @@ def archiveFullUpdate(dirList=[],force=False): -def getInputFile() -> list[str]: +def getInputFile() -> List[str]: """return code and novel name from input.txt""" inputfile=open('input.txt','r+', encoding='utf-8') line=inputfile.readline()