diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09bd36b69..19c6db621 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,9 +8,9 @@ After forking to your own github org or account, do the following steps to get s ```bash # prerequisites -node --version <= 16.13.1 (for version management, get nvm [recommended]) -java sdk --version <= 11 (for version management, get jenv [optional]) -android sdk (https://developer.android.com/studio) +node --version <= 16.13.1 (for version management, get nvm [recommended]) +java sdk --version <= 17 (for version management, get jenv [optional]) +android sdk (https://developer.android.com/studio) # clone your fork to your local machine git clone https://github.com//lnreader.git diff --git a/src/sources/ch/linovelib.js b/src/sources/ch/linovelib.js index 2b7289b9c..52110450f 100644 --- a/src/sources/ch/linovelib.js +++ b/src/sources/ch/linovelib.js @@ -1,4 +1,6 @@ -import { fetchHtml } from '@utils/fetch/fetch'; +import { showToast } from '@hooks/showToast'; +import { fetchApi, fetchHtml } from '@utils/fetch/fetch'; + import * as cheerio from 'cheerio'; const sourceId = 165; @@ -130,175 +132,186 @@ const parseNovelAndChapters = async novelUrl => { }; const parseChapter = async (novelUrl, chapterUrl) => { - let chapterName, chapterText, hasNextPage; + let chapterName, chapterText, hasNextPage, pageHasNextPage, pageText, urlNext; let pageNumber = 1; + const delay = ms => new Promise(res => setTimeout(res, ms)); /* * TODO: Maybe there are other ways to get the translation table * It is embed and encrypted inside readtool.js + * UPDATE: Decrypted, by skillgg */ + const mapping_dict = { - '“': '「', - '’': '』', - '': '是', - '': '不', - '': '他', - '': '个', - '': '来', - '': '大', - '': '子', - '': '说', - '': '年', - '': '那', - '': '她', - '': '得', - '': '自', - '': '家', - '': '而', - '': '去', - '': '小', - '': '于', - '': '么', - '': '好', - '': '发', - '': '成', - '': '事', - '': '用', - '': '道', - '': '种', - '': '乳', - '': '茎', - '': '肉', - '': '胸', - '': '淫', - '': '射', - '': '骚', - '”': '」', - '': '的', - '': '了', - '': '人', - '': '有', - '': '上', - '': '到', - '': '地', - '': '中', - '': '生', - '': '着', - '': '和', - '': '出', - '': '里', - '': '以', - '': '可', - '': '过', - '': '能', - '': '多', - '': '心', - '': '之', - '': '看', - '': '当', - '': '只', - '': '把', - '': '第', - '': '想', - '': '开', - '': '阴', - '': '欲', - '': '交', - '': '私', - '': '臀', - '': '脱', - '': '唇', - '‘': '『', - '': '一', - '': '我', - '': '在', - '': '这', - '': '们', - '': '时', - '': '为', - '': '你', - '': '国', - '': '就', - '': '要', - '': '也', - '': '后', - '': '会', - '': '下', - '': '天', - '': '对', - '': '然', - '': '学', - '': '都', - '': '起', - '': '没', - '': '如', - '': '还', - '': '样', - '': '作', - '': '美', - '': '液', - '': '呻', - '': '性', - '': '穴', - '': '舔', - '': '裸', + '\u201c': '\u300c', + '\u201d': '\u300d', + '\u2018': '\u300e', + '\u2019': '\u300f', + '\ue82c': '\u7684', + '\ue836': '\u4e00', + '\ue852': '\u662f', + '\ue850': '\u4e86', + '\ue832': '\u6211', + '\ue812': '\u4e0d', + '\ue833': '\u4eba', + '\ue849': '\u5728', + '\ue821': '\u4ed6', + '\ue810': '\u6709', + '\ue84c': '\u8fd9', + '\ue815': '\u4e2a', + '\ue842': '\u4e0a', + '\ue82e': '\u4eec', + '\ue817': '\u6765', + '\ue835': '\u5230', + '\ue837': '\u65f6', + '\ue82d': '\u5927', + '\ue859': '\u5730', + '\ue85c': '\u4e3a', + '\ue82f': '\u5b50', + '\ue84d': '\u4e2d', + '\ue854': '\u4f60', + '\ue81e': '\u8bf4', + '\ue853': '\u751f', + '\ue80f': '\u56fd', + '\ue80e': '\u5e74', + '\ue813': '\u7740', + '\ue802': '\u5c31', + '\ue81a': '\u90a3', + '\ue83b': '\u548c', + '\ue851': '\u8981', + '\ue82a': '\u5979', + '\ue838': '\u51fa', + '\ue808': '\u4e5f', + '\ue83a': '\u5f97', + '\ue814': '\u91cc', + '\ue857': '\u540e', + '\ue855': '\u81ea', + '\ue800': '\u4ee5', + '\ue81b': '\u4f1a', + '\ue85f': '\u5bb6', + '\ue816': '\u53ef', + '\ue83e': '\u4e0b', + '\ue84f': '\u800c', + '\ue80b': '\u8fc7', + '\ue828': '\u5929', + '\ue843': '\u53bb', + '\ue806': '\u80fd', + '\ue81f': '\u5bf9', + '\ue834': '\u5c0f', + '\ue81c': '\u591a', + '\ue848': '\u7136', + '\ue830': '\u4e8e', + '\ue84b': '\u5fc3', + '\ue84a': '\u5b66', + '\ue85d': '\u4e48', + '\ue861': '\u4e4b', + '\ue809': '\u90fd', + '\ue80c': '\u597d', + '\ue84e': '\u770b', + '\ue858': '\u8d77', + '\ue840': '\u53d1', + '\ue85b': '\u5f53', + '\ue863': '\u6ca1', + '\ue839': '\u6210', + '\ue827': '\u53ea', + '\ue841': '\u5982', + '\ue805': '\u4e8b', + '\ue845': '\u628a', + '\ue820': '\u8fd8', + '\ue83c': '\u7528', + '\ue847': '\u7b2c', + '\ue819': '\u6837', + '\ue82b': '\u9053', + '\ue80a': '\u60f3', + '\ue822': '\u4f5c', + '\ue85e': '\u79cd', + '\ue801': '\u5f00', + '\ue856': '\u7f8e', + '\ue811': '\u4e73', + '\ue860': '\u9634', + '\ue80d': '\u6db2', + '\ue83f': '\u830e', + '\ue803': '\u6b32', + '\ue804': '\u547b', + '\ue825': '\u8089', + '\ue846': '\u4ea4', + '\ue85a': '\u6027', + '\ue831': '\u80f8', + '\ue81d': '\u79c1', + '\ue826': '\u7a74', + '\ue818': '\u6deb', + '\ue823': '\u81c0', + '\ue829': '\u8214', + '\ue807': '\u5c04', + '\ue862': '\u8131', + '\ue83d': '\u88f8', + '\ue824': '\u9a9a', + '\ue844': '\u5507', }; const addPage = async pageCheerio => { - let pageText; - const formatPage = async () => { // Remove JS - pageCheerio('#acontent .cgo').remove(); + let style = pageCheerio('style:first').prop('innerHTML'); + style = style.replace(/(.*?)\{.*?\}/g, '$1,').split(','); + style.push(style.pop().replace(/\}/, '.cgo')); + style.map(tag => pageCheerio(tag).remove()); // Load lazyloaded images - pageCheerio('#acontent img.imagecontent').each(function () { - // Sometimes images are either in data-src or src - const imgSrc = - pageCheerio(this).attr('data-src') || pageCheerio(this).attr('src'); - if (imgSrc) { - // The original CDN URL is locked behind a CF-like challenge, switch the URL to bypass that - // There are no react-native-url-polyfill lib, can't use URL API - const regex = /\/\/.+\.com\//; - const imgUrl = imgSrc.replace(regex, '//img.linovelib.com/'); - // Clean up img element - pageCheerio(this) - .attr('src', imgUrl) - .removeAttr('data-src') - .removeClass('lazyload'); - } - }); + pageCheerio('.atitle') + .next() + .find('img.imagecontent') + .each(function () { + // Sometimes images are either in data-src or src + const imgSrc = + pageCheerio(this).attr('data-src') || pageCheerio(this).attr('src'); + if (imgSrc) { + // The original CDN URL is locked behind a CF-like challenge, switch the URL to bypass that + // There are no react-native-url-polyfill lib, can't use URL API + const regex = /\/\/.+\.com\//; + const imgUrl = imgSrc.replace(regex, '//img.linovelib.com/'); + // Clean up img element + pageCheerio(this) + .attr('src', imgUrl) + .removeAttr('data-src') + .removeClass('lazyload') + .addClass('delayed-src'); + } + }); // Recover the original character - pageText = pageCheerio('#acontent').html(); - pageText = pageText.replace(/./g, char => mapping_dict[char] || char); + pageText = pageCheerio('.atitle').next().html(); + pageText = pageText?.replace(/./g, char => mapping_dict[char] || char); return Promise.resolve(); }; await formatPage(); - chapterName ??= - pageCheerio('#atitle + h3').text() + - ' — ' + - pageCheerio('#atitle').text(); - chapterText += pageText; + chapterName = + pageCheerio('.atitle h3').text() + ' — ' + pageCheerio('#atitle').text(); + if (chapterText === undefined) { + chapterText = '

' + chapterName + '

'; + } + chapterText += + pageText || + 'Chapter not found, Report at lnreader github/discord if you see this message'; }; const loadPage = async url => { const body = await fetchHtml({ url, sourceId }); const pageCheerio = cheerio.load(body); - addPage(pageCheerio); - pageHasNextPage = - pageCheerio('#footlink > a[onclick$=ReadParams.url_next;]').text() === - '下一页' - ? true - : false; + await addPage(pageCheerio); + urlNext = pageCheerio('#aread script:first') + .prop('innerHTML') + .replace(/.*next:'(.*?)'.*/g, '$1'); + pageHasNextPage = urlNext.match(/_/) ? true : false; return { pageCheerio, pageHasNextPage }; }; let url = chapterUrl; do { const page = await loadPage(url); + await delay(1000); hasNextPage = page.pageHasNextPage; if (hasNextPage === true) { pageNumber++; @@ -318,51 +331,91 @@ const parseChapter = async (novelUrl, chapterUrl) => { }; const searchNovels = async searchTerm => { - const url = `${baseUrl}/search.html?searchkey=` + encodeURI(searchTerm); - const body = await fetchHtml({ url, sourceId }); + const searchUrl = `${baseUrl}/search/`; + const Term = encodeURI(searchTerm); + let nextPage, noNextPage, deadEnd; + let pageNumber = 1; + const novels = []; - const loadedCheerio = cheerio.load(body); + const addPage = async (pageCheerio, redirect) => { + const loadSearchResults = function () { + pageCheerio('.book-ol .book-layout').each(function () { + let novelUrl = pageCheerio(this).attr('href'); - const novels = []; + if (novelUrl) { + const novelName = pageCheerio(this).find('.book-title').text(); + const novelCover = pageCheerio(this) + .find('img.book-cover') + .attr('data-src'); + novelUrl = baseUrl + novelUrl; - const loadSearchResults = function () { - loadedCheerio('.book-ol .book-layout').each(function () { - let novelUrl = loadedCheerio(this).attr('href'); + const novel = { sourceId, novelUrl, novelName, novelCover }; - if (novelUrl) { - const novelName = loadedCheerio(this).find('.book-title').text(); - const novelCover = loadedCheerio(this) - .find('img.book-cover') - .attr('data-src'); - novelUrl = baseUrl + novelUrl; + novels.push(novel); + } + }); + }; - const novel = { - url: novelUrl, - name: novelName, - cover: novelCover, - }; + const novelResults = pageCheerio('.book-ol a.book-layout'); + if (novelResults.length === 0) { + showToast('Bypass check by searching in Webview'); + } else { + loadSearchResults(); + } - novels.push(novel); - } - }); + if (redirect.length) { + novels.length = 0; + const novelName = pageCheerio('#bookDetailWrapper .book-title').text(); + + const novelCover = pageCheerio('#bookDetailWrapper img.book-cover').attr( + 'src', + ); + const novelUrl = + baseUrl + + pageCheerio('#btnReadBook').attr('href').slice(0, -8) + + '.html'; + const novel = { sourceId, novelUrl, novelName, novelCover }; + novels.push(novel); + } }; - const novelResults = loadedCheerio('.book-ol .book-layout'); + const loadPage = async url => { + const body = await fetchHtml({ url, sourceId }); + const pageCheerio = cheerio.load(body); + const redirect = pageCheerio('div.book-layout'); + await addPage(pageCheerio, redirect); + nextPage = pageCheerio('.next').attr('href'); + if (!nextPage) { + noNextPage === true; + } else { + noNextPage = nextPage === '#' ? true : false; + } + return { pageCheerio, noNextPage }; + }; - if (novelResults.length === 0) { - // console.log('Challenge'); - } else { - loadSearchResults(); - } + let url = `${searchUrl}${Term}_${pageNumber}.html`; + do { + const page = await loadPage(url); + deadEnd = page.noNextPage; + if (deadEnd === false) { + pageNumber++; + url = `${searchUrl}${Term}_${pageNumber}.html`; + } + } while (deadEnd === false); return novels; }; +const headers = { + Referer: 'https://w.linovelib.com', +}; + const LinovelibScraper = { popularNovels, parseNovelAndChapters, parseChapter, searchNovels, + headers, }; export default LinovelibScraper; diff --git a/src/sources/en/allnovelfull.ts b/src/sources/en/allnovelfull.ts index 09fed86b6..05fee33a9 100644 --- a/src/sources/en/allnovelfull.ts +++ b/src/sources/en/allnovelfull.ts @@ -127,6 +127,7 @@ const parseChapter = async (novelUrl: string, chapterUrl: string) => { const body = await result.text(); const loadedCheerio = cheerio.load(body); + loadedCheerio('#chapter-content div').remove(); const chapterName = loadedCheerio('.chapter-title').attr('title'); const chapterText = loadedCheerio('#chapter-content').html() || ''; diff --git a/src/sources/en/icantreadjapanese.js b/src/sources/en/icantreadjapanese.js new file mode 100644 index 000000000..ab1192f37 --- /dev/null +++ b/src/sources/en/icantreadjapanese.js @@ -0,0 +1,137 @@ +import { fetchHtml } from '@utils/fetch/fetch'; +import * as cheerio from 'cheerio'; +import { FilterInputs } from '../types/filterTypes'; + +const sourceId = 170; +const sourceName = 'I cant read japanese tl'; + +const baseUrl = 'https://icantreadjapanese.wordpress.com/'; + +const popularNovels = async (page, { filters }) => { + let link = `${baseUrl}`; + + if (page === 1) { + link += (filters?.storyStatus ? filters.storyStatus : 'projects') + '/'; + } + + const body = await fetchHtml({ url: link, sourceId }); + + const loadedCheerio = cheerio.load(body); + + let novels = []; + + loadedCheerio('figure.wp-block-image').each(function () { + const novelName = loadedCheerio(this).find('a').text(); + const novelCover = loadedCheerio(this).find('img').attr('src'); + const novelUrl = loadedCheerio(this).find('a').attr('href'); + + const novel = { sourceId, novelName, novelCover, novelUrl }; + + novels.push(novel); + }); + + return { novels }; +}; + +const parseNovelAndChapters = async novelUrl => { + const body = await fetchHtml({ url: novelUrl, sourceId: sourceId }); + + let loadedCheerio = cheerio.load(body); + + let novel = { + sourceId, + sourceName, + url: novelUrl, + novelUrl, + }; + + novel.novelName = loadedCheerio('h1.entry-title').text().trim(); + + novel.novelCover = loadedCheerio('figure.wp-block-image > img').attr('src'); + + novel.author = loadedCheerio('.has-text-align-left') + .text() + .replace(/.*Author: (.*) \|.*/g, '$1'); + + novel.summary = loadedCheerio('.entry-content > p:first') + .nextUntil('hr:first') + .text(); + + let chapters = []; + + loadedCheerio('.wp-block-column li a').each(function () { + let chapterName = loadedCheerio(this).text(); + const releaseDate = null; + const chapterUrl = loadedCheerio(this).attr('href'); + + const volumeName = loadedCheerio(this) + .parent() + .prop('innerHTML') + .replace(//g, '') + .replace(/
/g, ' ') + .trim(); + if (volumeName.length) { + chapterName = volumeName + ' - ' + chapterName; + } + + chapters.push({ chapterName, releaseDate, chapterUrl }); + }); + + novel.chapters = chapters; + return novel; +}; + +const parseChapter = async (novelUrl, chapterUrl) => { + const body = await fetchHtml({ + url: chapterUrl, + sourceId: sourceId, + }); + + let loadedCheerio = cheerio.load(body); + + const chapterName = loadedCheerio('h1.entry-title').text(); + + const chapterText = loadedCheerio('.entry-content').html(); + + const chapter = { + sourceId, + novelUrl, + chapterUrl, + chapterName, + chapterText, + }; + + return chapter; +}; + +const searchNovels = async () => { + showToast('Search is not implemented for this source'); + return; +}; + +const filters = [ + { + key: 'storyStatus', + label: 'Translation Status', + values: [ + { label: 'Ongoing Projects', value: 'projects' }, + { + label: 'Maybe Will Read Again, Caught Up', + value: 'maybe-will-read-again-caught-up', + }, + { label: 'Dropped', value: 'dropped' }, + { label: 'Finished', value: 'finished' }, + ], + inputType: FilterInputs.Picker, + }, +]; + +const ICantReadJPTLScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, + filters, +}; + +export default ICantReadJPTLScraper; diff --git a/src/sources/en/lightnovelpub.js b/src/sources/en/lightnovelpub.js index 72906eaed..90c747650 100644 --- a/src/sources/en/lightnovelpub.js +++ b/src/sources/en/lightnovelpub.js @@ -87,15 +87,17 @@ const parseNovelAndChapters = async novelUrl => { const chaptersHtml = await fetchHtml({ url: chaptersUrl, init: { headers }, + sourceId, }); loadedCheerio = cheerio.load(chaptersHtml); loadedCheerio('.chapter-list li').each(function () { - const chapterName = loadedCheerio(this) - .find('.chapter-title') - .text() - .trim(); + const chapterName = + 'Chapter ' + + loadedCheerio(this).find('.chapter-no').text().trim() + + ' - '; + loadedCheerio(this).find('.chapter-title').text().trim(); const releaseDate = loadedCheerio(this) .find('.chapter-update') diff --git a/src/sources/en/lnmtl.js b/src/sources/en/lnmtl.js index 50411368d..1438e3d5d 100644 --- a/src/sources/en/lnmtl.js +++ b/src/sources/en/lnmtl.js @@ -1,6 +1,5 @@ import * as cheerio from 'cheerio'; import { showToast } from '../../hooks/showToast'; -import { htmlToText } from '../helpers/htmlToText'; const baseUrl = 'https://lnmtl.com/'; @@ -56,23 +55,26 @@ const parseNovelAndChapters = async novelUrl => { novel.novelName = loadedCheerio('.novel-name').text(); - novel.novelCover = loadedCheerio('div.novel').find('img').attr('src'); + novel.novelCover = loadedCheerio('.novel').find('img').attr('src'); - novel.summary = loadedCheerio('div.description').text().trim(); + novel.summary = loadedCheerio('.description').text().trim(); - novel.author = loadedCheerio( - 'main > div:nth-child(3) > div > div.col-lg-3.col-md-4 > div:nth-child(2) > div.panel-body > dl:nth-child(1) > dd > a > span', - ).text(); + loadedCheerio('.panel-body > dl').each(function () { + let detailName = loadedCheerio(this).find('dt').text().trim(); + let detail = loadedCheerio(this).find('dd').text().trim(); - novel.status = loadedCheerio( - 'main > div:nth-child(3) > div > div.col-lg-3.col-md-4 > div:nth-child(2) > div.panel-body > dl:last-child > dd', - ) - .text() - .trim(); + switch (detailName) { + case 'Authors': + novel.author = detail; + break; + case 'Current status': + novel.status = detail; + break; + } + }); - novel.genre = loadedCheerio( - 'main > div.container > div > div.col-lg-3.col-md-4 > div:nth-child(4) > div.panel-body > ul', - ) + novel.genre = loadedCheerio('.panel-heading:contains(" Genres ")') + .next() .text() .trim() .replace(/\s\s/g, ','); @@ -128,17 +130,15 @@ const parseChapter = async (novelUrl, chapterUrl) => { let chapterName = loadedCheerio('h3 > span.chapter-title').text().trim(); - loadedCheerio('.original').remove(); + loadedCheerio('.original, script').remove(); + loadedCheerio('sentence.translated').wrap('

'); - let chapterText = loadedCheerio('.chapter-body').html(); + let chapterText = loadedCheerio('.chapter-body').html().replace(/„/g, '“'); if (!chapterText) { chapterText = loadedCheerio('.alert.alert-warning').text(); } - chapterText = - chapterName + '\n\n' + htmlToText(chapterText, { removeLineBreaks: false }); - const chapter = { sourceId: 37, novelUrl, @@ -151,32 +151,35 @@ const parseChapter = async (novelUrl, chapterUrl) => { }; const searchNovels = async searchTerm => { - const url = 'https://lnmtl.com/term'; - - const result = await fetch(url); + const result = await fetch(baseUrl); const body = await result.text(); const loadedCheerio = cheerio.load(body); - let novels = loadedCheerio('footer') + const list = loadedCheerio('footer') .next() .next() .html() - .match(/local: \[(.*?)\]/)[0] - .replace('local: ', ''); + .match(/prefetch: '\/(.*json)/)[1]; - novels = JSON.parse(novels); + const search = await fetch(`${baseUrl}${list}`); + const data = await search.json(); - novels = novels.filter(novel => - novel.name.toLowerCase().includes(searchTerm.toLowerCase()), + let nov = data.filter(res => + res.name.toLowerCase().includes(searchTerm.toLowerCase()), ); - novels = novels.map(novel => ({ - sourceId: 37, - novelName: novel.name, - novelUrl: novel.slug, - novelCover: novel.image, - })); + const novels = []; + + nov.map(res => { + const novelName = res.name; + const novelUrl = res.slug; + const novelCover = res.image; + + const novel = { sourceId: 37, novelName, novelUrl, novelCover }; + + novels.push(novel); + }); return novels; }; diff --git a/src/sources/en/fastnovel.js b/src/sources/en/novelbin.js similarity index 95% rename from src/sources/en/fastnovel.js rename to src/sources/en/novelbin.js index 057ae9e01..9bb0959ba 100644 --- a/src/sources/en/fastnovel.js +++ b/src/sources/en/novelbin.js @@ -2,8 +2,8 @@ import { fetchHtml } from '@utils/fetch/fetch'; import * as cheerio from 'cheerio'; const sourceId = 3; -const baseUrl = 'https://fastnovel.org'; -const searchUrl = 'https://fastnovel.org/search/'; +const baseUrl = 'https://novelbin.org'; +const searchUrl = 'https://novelbin.org/search/'; const popularNovels = async page => { const url = `${baseUrl}/sort/p/?page=${page}`; @@ -42,7 +42,7 @@ const parseNovelAndChapters = async novelUrl => { let novel = { sourceId, - sourceName: 'FastNovel', + sourceName: 'NovelBin', url, novelUrl, }; @@ -158,11 +158,11 @@ const searchNovels = async searchTerm => { return novels; }; -const fastNovelScraper = { +const NovelBinScraper = { popularNovels, parseNovelAndChapters, parseChapter, searchNovels, }; -export default fastNovelScraper; +export default NovelBinScraper; diff --git a/src/sources/en/novelpub.js b/src/sources/en/novelpub.js index fb600f6e9..992e38e24 100644 --- a/src/sources/en/novelpub.js +++ b/src/sources/en/novelpub.js @@ -1,5 +1,5 @@ import * as cheerio from 'cheerio'; -import { fetchHtml } from '@utils/fetch/fetch'; +import { fetchApi, fetchHtml } from '@utils/fetch/fetch'; const baseUrl = 'https://www.novelpub.com/'; @@ -9,18 +9,12 @@ const sourceId = 94; const headers = new Headers({ Accept: 'application/json', 'Content-Type': 'application/json', - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36', }); const popularNovels = async page => { let url = baseUrl + 'browse/all/popular/all/' + page; - const body = await fetchHtml({ - url, - sourceId, - init: { headers: { 'User-Agent': userAgent } }, - }); + const body = await fetchHtml({ url, sourceId }); const loadedCheerio = cheerio.load(body); @@ -46,48 +40,39 @@ const popularNovels = async page => { const parseNovelAndChapters = async novelUrl => { const url = novelUrl; - const body = await fetchHtml({ - url, - sourceId, - init: { headers: { 'User-Agent': userAgent } }, - }); + const body = await fetchHtml({ url, sourceId }); let loadedCheerio = cheerio.load(body); - let novel = { url, novelUrl, sourceId, sourceName, genre: '' }; + let novel = { + sourceId, + url: novelUrl, + novelUrl, + sourceName, + }; novel.novelName = loadedCheerio('h1.novel-title').text().trim(); novel.novelCover = loadedCheerio('figure.cover > img').attr('data-src'); - loadedCheerio('div.categories > ul > li').each(function () { - novel.genre += - loadedCheerio(this) - .text() - .replace(/[\t\n]/g, '') + ','; - }); + novel.genre = loadedCheerio('.categories li') + .find('a') + .map((i, el) => loadedCheerio(el).text()) + .toArray() + .join(','); - loadedCheerio('div.header-stats > span').each(function () { - if (loadedCheerio(this).find('small').text() === 'Status') { - novel.status = loadedCheerio(this).find('strong').text(); - } - }); - - novel.genre = novel.genre.slice(0, -1); + novel.status = loadedCheerio('small:contains("Status")').prev().text().trim(); novel.author = loadedCheerio('.author > a > span').text(); + loadedCheerio('.expand').remove(); novel.summary = loadedCheerio('.summary > .content').text().trim(); const delay = ms => new Promise(res => setTimeout(res, ms)); let lastPage = 1; - lastPage = loadedCheerio( - '#novel > header > div.header-body.container > div.novel-info > div.header-stats > span:nth-child(1) > strong', - ) - .text() - ?.trim(); + lastPage = loadedCheerio('small:contains("Chapters")').prev().text().trim(); lastPage = Math.ceil(lastPage / 100); @@ -97,8 +82,11 @@ const parseNovelAndChapters = async novelUrl => { for (let i = 1; i <= lastPage; i++) { const chaptersUrl = `${novelUrl}/chapters/page-${i}`; - const chaptersRequest = await fetch(chaptersUrl, { headers }); - const chaptersHtml = await chaptersRequest.text(); + const chaptersHtml = await fetchHtml({ + url: chaptersUrl, + init: { headers }, + sourceId, + }); loadedCheerio = cheerio.load(chaptersHtml); @@ -133,11 +121,7 @@ const parseNovelAndChapters = async novelUrl => { const parseChapter = async (novelUrl, chapterUrl) => { const url = chapterUrl; - const body = await fetchHtml({ - url, - sourceId, - init: { headers: { 'User-Agent': userAgent } }, - }); + const body = await fetchHtml({ url, sourceId }); const loadedCheerio = cheerio.load(body); @@ -150,12 +134,23 @@ const parseChapter = async (novelUrl, chapterUrl) => { }; const searchNovels = async searchTerm => { - const url = `${baseUrl}lnwsearchlive?inputContent=${searchTerm}`; + const url = `${baseUrl}lnsearchlive`; + const link = `${baseUrl}search`; + const response = await fetchApi({ url: link, sourceId }).then(r => r.text()); + const token = cheerio.load(response); + let verifytoken = token('#novelSearchForm > input').attr('value'); + + let formData = new FormData(); + formData.append('inputContent', searchTerm); const body = await fetchHtml({ url, + init: { + method: 'POST', + headers: { 'LNRequestVerifyToken': verifytoken }, + body: formData, + }, sourceId, - init: { headers: { 'User-Agent': userAgent } }, }); let loadedCheerio = cheerio.load(body); diff --git a/src/sources/en/novelupdates.js b/src/sources/en/novelupdates.js index 63b092bc5..ca65c8ec6 100644 --- a/src/sources/en/novelupdates.js +++ b/src/sources/en/novelupdates.js @@ -204,6 +204,10 @@ const parseChapter = async (novelUrl, chapterUrl) => { let isWattpad = result.url.toLowerCase().includes('wattpad'); + let isBlossomTranslation = result.url + .toLowerCase() + .includes('blossomtranslation'); + let isLightNovelsTls = result.url .toLowerCase() .includes('lightnovelstranslations'); @@ -241,7 +245,7 @@ const parseChapter = async (novelUrl, chapterUrl) => { if (isWuxiaWorld) { chapterText = loadedCheerio('#chapter-content').html(); } else if (isRainOfSnow) { - chapterText = loadedCheerio('div.content').html(); + chapterText = loadedCheerio('.content').html(); } else if (isTumblr) { chapterText = loadedCheerio('.post').html(); } else if (isBlogspot) { @@ -250,13 +254,15 @@ const parseChapter = async (novelUrl, chapterUrl) => { } else if (isHostedNovel) { chapterText = loadedCheerio('.chapter').html(); } else if (isScribbleHub) { - chapterText = loadedCheerio('div.chp_raw').html(); + chapterText = loadedCheerio('.chp_raw').html(); } else if (isWattpad) { chapterText = loadedCheerio('.container pre').html(); } else if (isTravisTranslation) { chapterText = loadedCheerio('.reader-content').html(); } else if (isLightNovelsTls) { chapterText = loadedCheerio('.text_story').html(); + } else if (isBlossomTranslation) { + chapterText = loadedCheerio('.manga-child-content').html(); } else if (isiNovelTranslation) { const link = 'https://api.' + result.url.slice(8); const json = await fetchApi({ diff --git a/src/sources/en/royalroad.js b/src/sources/en/royalroad.js index 42b71d0de..5abc72846 100644 --- a/src/sources/en/royalroad.js +++ b/src/sources/en/royalroad.js @@ -127,7 +127,7 @@ const parseChapter = async (novelUrl, chapterUrl) => { const loadedCheerio = cheerio.load(body); - let chapterName = loadedCheerio('div.chapter-content').find('strong').text(); + let chapterName = loadedCheerio('h1.font-white').text(); let chapterText = loadedCheerio('div.chapter-content').html(); diff --git a/src/sources/en/woopread.js b/src/sources/en/woopread.js index ea3cabc87..05486c595 100644 --- a/src/sources/en/woopread.js +++ b/src/sources/en/woopread.js @@ -112,15 +112,15 @@ const parseNovelAndChapters = async novelUrl => { }; const parseChapter = async (novelUrl, chapterUrl) => { - const url = `${baseUrl}series/${novelUrl}/${chapterUrl}/`; + const url = `${baseUrl}series/${novelUrl}${chapterUrl}`; const result = await fetch(url); const body = await result.text(); const loadedCheerio = cheerio.load(body); - const chapterName = loadedCheerio('h1#chapter-heading').text(); - + const chapterName = loadedCheerio('.reading-content b:first').text(); + loadedCheerio('input, .reading-content b:first').remove(); let chapterText = loadedCheerio('.reading-content').html(); const chapter = { @@ -144,11 +144,11 @@ const searchNovels = async searchTerm => { let novels = []; - loadedCheerio('.c-tabs-item__content').each(function () { - const novelName = loadedCheerio(this).find('.h4 > a').text(); - const novelCover = loadedCheerio(this).find('img').attr('src'); + loadedCheerio('.tab-thumb').each(function () { + const novelName = loadedCheerio(this).find('a').attr('title'); + const novelCover = loadedCheerio(this).find('img').attr('data-src'); - let novelUrl = loadedCheerio(this).find('.h4 > a').attr('href'); + let novelUrl = loadedCheerio(this).find('a').attr('href'); novelUrl = novelUrl.replace(`${baseUrl}series/`, ''); const novel = { diff --git a/src/sources/en/wuxiablog.js b/src/sources/en/wuxiablog.js index 4f85eb52c..562a573f9 100644 --- a/src/sources/en/wuxiablog.js +++ b/src/sources/en/wuxiablog.js @@ -23,12 +23,12 @@ const popularNovels = async page => { if (novelName) { const novelId = loadedCheerio(this).find('td:nth-child(1)').text(); - const novelCover = baseUrl + '/data/image/' + novelId + '.jpg'; + const novelCover = baseUrl + 'data/image/' + novelId + '.jpg'; let novelUrl = loadedCheerio(this) .find('td.novel > a') .attr('href') - .replace('/novel/', ''); + .replace('/book/', ''); const novel = { sourceId, @@ -45,7 +45,7 @@ const popularNovels = async page => { }; const parseNovelAndChapters = async novelUrl => { - const url = `${baseUrl}novel/${novelUrl}/`; + const url = `${baseUrl}book/${novelUrl}/`; const result = await fetch(url); const body = await result.text(); @@ -59,25 +59,19 @@ const parseNovelAndChapters = async novelUrl => { novelUrl, }; - novel.novelName = loadedCheerio( - 'body > div.container-fluid.text-center > div.row.content > div.col-sm-8.text-left > div:nth-child(6) > div.panel-heading.clearfix > h4', - ).text(); + novel.novelName = loadedCheerio('h4[itemprop="name>"]:first').text(); - novel.novelCover = loadedCheerio('img[itemprop="image"]').attr('src'); + novel.novelCover = loadedCheerio('.img-thumbnail').attr('src'); - novel.author = loadedCheerio( - 'div.col-md-6 > div > div:nth-child(2) > div > div:nth-child(2) > a', - ).text(); + novel.author = loadedCheerio('h4:contains("Author:")').next().text(); - novel.genre = loadedCheerio( - 'div.row > div.col-md-6 > div > a:nth-child(4)', - ).text(); - - novel.summary = loadedCheerio('div[itemprop="description"]') - .find('p') - .text() - .trim(); + novel.genre = loadedCheerio('h4:contains("Genre:")') + .nextUntil('h4') + .map((i, el) => loadedCheerio(el).text()) + .toArray() + .join(','); + novel.summary = loadedCheerio('h4:contains("Description:")').next().text(); let novelChapters = []; loadedCheerio('table#chplist > #chapters > tr').each(function () { @@ -137,20 +131,18 @@ const parseNovelAndChapters = async novelUrl => { }; const parseChapter = async (novelUrl, chapterUrl) => { - const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`; + const url = `${baseUrl}book/${novelUrl}/${chapterUrl}`; const result = await fetch(url); const body = await result.text(); const loadedCheerio = cheerio.load(body); - let chapterName = loadedCheerio('div.panel-body.article').find('h4').text(); + let chapterName = loadedCheerio('.panel-body.article').find('h4').text(); - loadedCheerio('ul.pager').remove(); + loadedCheerio('ul.pager, .panel-body.article div,button,span').remove(); - let chapterText = loadedCheerio('div.panel-body.article') - .html() - .replace(/(.*?)<\/span>/g, ''); + let chapterText = loadedCheerio('.panel-body.article').html(); const chapter = { sourceId, @@ -184,7 +176,7 @@ const searchNovels = async searchTerm => { let novelUrl = loadedCheerio(this) .find('a') .attr('href') - .replace(baseUrl + 'novel/', ''); + .replace(baseUrl + 'book/', ''); const novel = { sourceId, diff --git a/src/sources/es/skynovels.js b/src/sources/es/skynovels.js index 1830533b1..8f96cfd4b 100644 --- a/src/sources/es/skynovels.js +++ b/src/sources/es/skynovels.js @@ -91,7 +91,7 @@ const parseChapter = async (novUrl, chapUrl) => { let chapterName = item.chp_index_title; - let chapterText = item.chp_content; + let chapterText = item.chp_content.replace(/\n/g, '
'); novelUrl = novelId + '/' + novelUrl + '/'; chapterUrl = item.id + '/' + item.chp_name; diff --git a/src/sources/es/tunovelaligera.js b/src/sources/es/tunovelaligera.js index 19d425a3d..8d3263a99 100644 --- a/src/sources/es/tunovelaligera.js +++ b/src/sources/es/tunovelaligera.js @@ -1,6 +1,7 @@ import * as cheerio from 'cheerio'; import { defaultCoverUri, Status } from '../helpers/constants'; import { fetchHtml } from '@utils/fetch/fetch'; +import { showToast } from '@hooks/showToast'; const sourceId = 23; const sourceName = 'TuNovelaLigera'; @@ -94,58 +95,125 @@ const parseNovelAndChapters = async novelUrl => { let novelChapters = []; - const novelId = - loadedCheerio('.rating-post-id').attr('value') || - loadedCheerio('#manga-chapters-holder').attr('data-id'); + const delay = ms => new Promise(res => setTimeout(res, ms)); + let lastPage = 1; + lastPage = loadedCheerio('.lcp_paginator li:last').prev().text().trim(); + + const getChapters = async () => { + const chaptersAjax = `${baseUrl}wp-admin/admin-ajax.php`; + showToast('Cargando desde Archivo...'); + + let formData = new FormData(); + formData.append('action', 'madara_load_more'); + formData.append('page', '0'); + formData.append('template', 'html/loop/content'); + formData.append('vars[category_name]', novelUrl.slice(0, -1)); + formData.append('vars[posts_per_page]', '10000'); + + const formBody = await fetchHtml({ + url: chaptersAjax, + init: { + method: 'POST', + body: formData, + }, + sourceId, + }); + + const loadedCheerio = cheerio.load(formBody); + + loadedCheerio('.heading').each((i, el) => { + const chapterName = loadedCheerio(el) + .text() + .replace(/[\t\n]/g, '') + .trim(); + const releaseDate = null; + let chapterUrl = loadedCheerio(el).find('a').attr('href'); + chapterUrl = chapterUrl.replace(`${baseUrl}${novelUrl}`, ''); + + novelChapters.push({ chapterName, releaseDate, chapterUrl }); + }); + return novelChapters.reverse(); + }; - let formData = new FormData(); - formData.append('action', 'manga_get_chapters'); - formData.append('manga', novelId); + const getPageChapters = async () => { + for (let i = 1; i <= lastPage; i++) { + const chaptersUrl = `${baseUrl}novelas/${novelUrl}?lcp_page0=${i}`; + showToast(`Cargando desde la página ${i}/${lastPage}...`); + const chaptersHtml = await fetchHtml({ + url: chaptersUrl, + sourceId, + }); + + loadedCheerio = cheerio.load(chaptersHtml); + loadedCheerio('h2:contains("Resumen")') + .closest('div') + .next() + .find('ul:first li') + .each((i, el) => { + const chapterName = loadedCheerio(el) + .find('a') + .text() + .replace(/[\t\n]/g, '') + .trim(); + + const releaseDate = loadedCheerio(el).find('span').text().trim(); + + const chapterUrl = loadedCheerio(el).find('a').attr('href'); + + novelChapters.push({ chapterName, releaseDate, chapterUrl }); + }); + await delay(2000); + } + return novelChapters.reverse(); + }; - const text = await fetchHtml({ - url: 'https://tunovelaligera.com/wp-admin/admin-ajax.php', - init: { - method: 'POST', - body: formData, - }, - }); + // novel.chapters = await getChapters(); + // if (!novel.chapters.length) { + // showToast('¡Archivo no encontrado!'); + // await delay(1000); + // + // } - loadedCheerio = cheerio.load(text); + novel.chapters = await getPageChapters(); - loadedCheerio('.wp-manga-chapter').each(function () { - const chapterName = loadedCheerio(this) - .find('a') - .text() - .replace(/[\t\n]/g, '') - .trim(); + return novel; +}; - const releaseDate = loadedCheerio(this).find('span').text().trim(); +const parseChapter = async (novelUrl, chapterUrl) => { + const url = `${baseUrl}${novelUrl}${chapterUrl}`; - let chapterUrl = loadedCheerio(this).find('a').attr('href').split('/'); + const body = await fetchHtml({ url, sourceId }); - chapterUrl[6] - ? (chapterUrl = chapterUrl[5] + '/' + chapterUrl[6]) - : (chapterUrl = chapterUrl[5]); + const loadedCheerio = cheerio.load(body); - novelChapters.push({ chapterName, releaseDate, chapterUrl }); - }); + const chapterName = loadedCheerio('h1#chapter-heading').text(); - novel.chapters = novelChapters.reverse(); + let pageText = loadedCheerio('li:contains("A")').closest('div').next(); - return novel; -}; - -const parseChapter = async (novelUrl, chapterUrl) => { - const url = `${baseUrl}novelas/${novelUrl}/${chapterUrl}`; - - const body = await fetchHtml({ url }); + let cleanup = []; + pageText.find('div').each((i, el) => { + let hb = loadedCheerio(el).attr('id')?.match(/hb.*/); + if (!hb) { + return; + } + let idAttr = `div[id="${hb}"]`; + cleanup.push(idAttr); + }); - let loadedCheerio = cheerio.load(body); + cleanup.push( + 'center', + '.clear', + '.code-block', + '.ai-viewport-2', + '.cbxwpbkmarkwrap', + '.flagcontent-form-container', + 'strong:last', + ); - let chapterName = loadedCheerio('h1#chapter-heading').text(); + cleanup.map(tag => pageText.find(tag).remove()); + pageText.find('a, span').removeAttr(); - let chapterText = loadedCheerio('.text-left').html(); - novelUrl = novelUrl + '/'; + const chapterText = pageText.html(); const chapter = { sourceId, @@ -161,11 +229,11 @@ const parseChapter = async (novelUrl, chapterUrl) => { const searchNovels = async searchTerm => { const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`; - const body = await fetchHtml({ url }); + const body = await fetchHtml({ url, sourceId }); - let loadedCheerio = cheerio.load(body); + const loadedCheerio = cheerio.load(body); - let novels = []; + const novels = []; loadedCheerio('.c-tabs-item__content').each(function () { const novelName = loadedCheerio(this).find('.h4 > a').text(); diff --git a/src/sources/id/indowebnovel.js b/src/sources/id/indowebnovel.js index 26f8780e1..040f2c92c 100644 --- a/src/sources/id/indowebnovel.js +++ b/src/sources/id/indowebnovel.js @@ -106,7 +106,7 @@ const parseChapter = async (novelUrl, chapterUrl) => { let loadedCheerio = cheerio.load(body); const chapterName = loadedCheerio('.title-chapter').text(); - const chapterText = loadedCheerio('.reader').html(); + const chapterText = loadedCheerio('.entry-pagination').next().html(); const chapter = { sourceId, diff --git a/src/sources/id/sakuranovel.js b/src/sources/id/sakuranovel.js index fb6becf18..f2064b2df 100644 --- a/src/sources/id/sakuranovel.js +++ b/src/sources/id/sakuranovel.js @@ -101,7 +101,7 @@ const parseChapter = async (novelUrl, chapterUrl) => { let loadedCheerio = cheerio.load(body); const chapterName = loadedCheerio('.title-chapter').text(); - const chapterText = loadedCheerio('.readers').html(); + const chapterText = loadedCheerio('.reader-setting').next().next().html(); const chapter = { sourceId, diff --git a/src/sources/multisrc/madara/MadaraGenerator.ts b/src/sources/multisrc/madara/MadaraGenerator.ts index fbaca553f..5901aeb35 100644 --- a/src/sources/multisrc/madara/MadaraGenerator.ts +++ b/src/sources/multisrc/madara/MadaraGenerator.ts @@ -350,3 +350,10 @@ export const WebNovelOkuScraper = new MadaraScraper( 'WebNovelOku ', { 'lang': 'Turkish' }, ); + +export const AsuraLightNovelScraper = new MadaraScraper( + 169, + 'https://asuralightnovel.com/', + 'Asura Light Novel', + { 'useNewChapterEndpoint': true, 'lang': 'English' }, +); diff --git a/src/sources/multisrc/madara/MadaraSources.json b/src/sources/multisrc/madara/MadaraSources.json index c8461468d..2fa0a6925 100644 --- a/src/sources/multisrc/madara/MadaraSources.json +++ b/src/sources/multisrc/madara/MadaraSources.json @@ -361,5 +361,14 @@ "options": { "lang": "Turkish" } + }, + { + "sourceId": 169, + "baseUrl": "https://asuralightnovel.com/", + "sourceName": "Asura Light Novel", + "options": { + "useNewChapterEndpoint": true, + "lang": "English" + } } ] diff --git a/src/sources/sourceManager.ts b/src/sources/sourceManager.ts index 65bfa091d..61e106873 100644 --- a/src/sources/sourceManager.ts +++ b/src/sources/sourceManager.ts @@ -1,7 +1,7 @@ import EPubSource from './local/epubSource'; import ComradeMaoScraper from './en/comrademao'; import ReadLightNovelScraper from './en/readlightnovel'; -import fastNovelScraper from './en/fastnovel'; +import NovelBinScraper from './en/novelbin'; import readNovelFullScraper from './en/readnovelfull'; import mtlNovelScraper from './en/mtlnovel'; import novelhallScraper from './en/novelhall'; @@ -29,7 +29,9 @@ import ScribbleHubScraper from './en/scribblehub'; import SyosetuScraper from './jp/syosetu'; import LNMTLScraper from './en/lnmtl'; import LightNovelFullScraper from './en/lightnovelfull'; +import ICantReadJPTLScraper from './en/icantreadjapanese'; import { + AsuraLightNovelScraper, ArMTLScraper, BoxNovelScraper, ClickNovelScraper, @@ -191,7 +193,7 @@ export const sourceManager = (sourceId: number): Scraper => { 0: EPubSource, // @ts-ignore 1: BoxNovelScraper, // @ts-ignore 2: ReadLightNovelScraper, // @ts-ignore - 3: fastNovelScraper, // @ts-ignore + 3: NovelBinScraper, // @ts-ignore 4: readNovelFullScraper, // @ts-ignore 5: mtlNovelScraper, // @ts-ignore 6: novelhallScraper, // @ts-ignore @@ -336,6 +338,8 @@ export const sourceManager = (sourceId: number): Scraper => { 166: NOVAScraper, // @ts-ignore 167: SmakolykyTlScraper, // @ts-ignore 168: LitSpaceScraper, // @ts-ignore + 169: AsuraLightNovelScraper, // @ts-ignore + 170: ICantReadJPTLScraper, // @ts-ignore }; return scrapers[sourceId]; diff --git a/src/sources/sources.json b/src/sources/sources.json index 0f43e13a1..5ecd2c1bc 100644 --- a/src/sources/sources.json +++ b/src/sources/sources.json @@ -15,8 +15,8 @@ }, { "sourceId": 3, - "url": "https://fastnovel.org/", - "sourceName": "FastNovel", + "url": "https://novelbin.org/", + "sourceName": "NovelBin", "icon": "https://github.com/LNReader/lnreader-sources/blob/main/icons/src/en/fastnovel/icon.png?raw=true", "lang": "English" }, @@ -1027,5 +1027,19 @@ "url": "https://freedlit.space/", "lang": "Russian", "icon": "https://freedlit.space/images/fav/apple-touch-icon.png" + }, + { + "sourceId": 169, + "url": "https://asuralightnovel.com/", + "sourceName": "Asura Light Novel", + "icon": "https://asuralightnovel.com/wp-content/uploads/2021/06/favicon-asura-2.png", + "lang": "English" + }, + { + "sourceId": 170, + "url": "https://icantreadjapanese.wordpress.com/", + "sourceName": "I Cant Read Japanese TL", + "icon": "https://icantreadjapanese.files.wordpress.com/2021/03/cropped-cropped-site-icon-1.png?w=16", + "lang": "English" } ]