From cd660fb8cf3b53e19d4eeb0c37cf7cc449f40491 Mon Sep 17 00:00:00 2001 From: Rider21 <58046032+Rider21@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:28:24 +0300 Subject: [PATCH] Fix ReadLightNovel (#962) * Update readlightnovel.ts * fix date * ops * test --- src/sources/en/readlightnovel.ts | 45 +++++---------- .../multisrc/ifreedom/IfreedomScraper.js | 55 +++++++++++++++---- src/sources/multisrc/rulate/RulateScraper.js | 35 +++++++++++- src/sources/ru/bookriver.js | 2 +- src/sources/ru/freedlit.ts | 32 ++++++++++- src/sources/ru/jaomix.js | 29 +++++++++- 6 files changed, 150 insertions(+), 48 deletions(-) diff --git a/src/sources/en/readlightnovel.ts b/src/sources/en/readlightnovel.ts index a6d66182d..76ed81c2d 100644 --- a/src/sources/en/readlightnovel.ts +++ b/src/sources/en/readlightnovel.ts @@ -6,8 +6,8 @@ import { FilterInputs, SourceFilter } from '../types/filterTypes'; const sourceId = 2; const sourceName = 'ReadLightNovel'; -const baseUrl = 'https://www.readlightnovel.me'; -const searchUrl = 'https://www.readlightnovel.me/detailed-search-210922'; +const baseUrl = 'https://www.readlightnovel.today'; +const searchUrl = 'https://www.readlightnovel.today/detailed-search-210922'; const popularNovels = async (page: number, options?: SourceOptions) => { const url = `${baseUrl}/${ @@ -91,37 +91,22 @@ const parseNovelAndChapters = async (novelUrl: string) => { } }); - let chapters: SourceChapterItem[] = []; + const chapters: SourceChapterItem[] = []; - loadedCheerio('.panel').each(function () { - let volumeName = loadedCheerio(this).find('h4.panel-title').text(); - - loadedCheerio(this) - .find('ul.chapter-chs > li') - .each(function () { - const chapterUrl = loadedCheerio(this) - .find('a') - .attr('href') - ?.replace(`${baseUrl}/${novelUrl}/`, ''); - - if (chapterUrl) { - let chapterName = loadedCheerio(this).find('a').text(); - - const releaseDate = null; - - if (volumeName.includes('Volume')) { - chapterName = volumeName + ' ' + chapterName; - } + loadedCheerio('ul.chapter-chs > li > a').each(function () { + const chapterUrl = loadedCheerio(this) + .attr('href') + ?.replace?.(`${baseUrl}/${novelUrl}/`, ''); - const chapter = { - chapterName, - releaseDate, - chapterUrl, - }; + if (chapterUrl) { + const chapterName = loadedCheerio(this).text(); - chapters.push(chapter); - } + chapters.push({ + chapterName, + releaseDate: null, + chapterUrl, }); + } }); novel = { @@ -172,7 +157,7 @@ const parseChapter = async (novelUrl: string, chapterUrl: string) => { const chapterText = loadedCheerio('.desc').html() || ''; const chapter: SourceChapter = { - sourceId: 2, + sourceId, novelUrl, chapterUrl, chapterName, diff --git a/src/sources/multisrc/ifreedom/IfreedomScraper.js b/src/sources/multisrc/ifreedom/IfreedomScraper.js index ba4cb94b3..5e3e094e9 100644 --- a/src/sources/multisrc/ifreedom/IfreedomScraper.js +++ b/src/sources/multisrc/ifreedom/IfreedomScraper.js @@ -1,5 +1,6 @@ import { Status } from '../../helpers/constants'; import * as cheerio from 'cheerio'; +import dayjs from 'dayjs'; class IfreedomScraper { constructor(sourceId, baseUrl, sourceName, filters) { @@ -17,15 +18,12 @@ class IfreedomScraper { ? 'По дате обновления' : filters?.sort || 'По рейтингу'); - if (filters?.status?.length) { - url += filters.status.map(i => '&status[]=' + i).join(''); - } - if (filters?.lang?.length) { - url += filters.lang.map(i => '&lang[]=' + i).join(''); - } - if (filters?.genre?.length) { - url += filters.genre.map(i => '&genre[]=' + i).join(''); - } + Object.entries(filters || {}).forEach(([type, value]) => { + if (value instanceof Array && value.length) { + url += '&' + type + '[]=' + value.join('&' + type + '[]='); + } + }); + url += '&bpage=' + page; const body = await fetch(url).then(res => res.text()); @@ -95,12 +93,16 @@ class IfreedomScraper { chapterName && chapterUrl ) { - const releaseDate = loadedCheerio(this) + const releaseTime = loadedCheerio(this) .find('div.li-col2-ranobe') .text() .trim(); - chapters.push({ chapterName, releaseDate, chapterUrl }); + chapters.push({ + chapterName, + releaseDate: parseDate(releaseTime), + chapterUrl, + }); } }); @@ -154,4 +156,35 @@ class IfreedomScraper { } } +function parseDate(dateString = '') { + const months = { + января: 1, + февраля: 2, + марта: 3, + апреля: 4, + мая: 5, + июня: 6, + июля: 7, + августа: 8, + сентября: 9, + октября: 10, + ноября: 11, + декабря: 12, + }; + + if (dateString.includes('.')) { + const [day, month, year] = dateString.split('.'); + if (day && month && year) { + return dayjs(year + '-' + month + '-' + day).format('LL'); + } + } else if (dateString.includes(' ')) { + const [day, month] = dateString.split(' '); + if (day && months[month]) { + const year = new Date().getFullYear(); + return dayjs(year + '-' + months[month] + '-' + day).format('LL'); + } + } + return dateString; +} + export default IfreedomScraper; diff --git a/src/sources/multisrc/rulate/RulateScraper.js b/src/sources/multisrc/rulate/RulateScraper.js index e24fa1422..3c2405fa4 100644 --- a/src/sources/multisrc/rulate/RulateScraper.js +++ b/src/sources/multisrc/rulate/RulateScraper.js @@ -2,6 +2,7 @@ import { FilterInputs } from '../../types/filterTypes'; import { htmlToText } from '../../helpers/htmlToText'; import { Status } from '../../helpers/constants'; import * as cheerio from 'cheerio'; +import dayjs from 'dayjs'; class RulateScraper { constructor(sourceId, baseUrl, sourceName, filter) { @@ -204,7 +205,7 @@ class RulateScraper { .find('td[class="t"] > a') .text() .trim(); - const releaseDate = loadedCheerio(this) + const releaseTime = loadedCheerio(this) .find('td > span') .attr('title') ?.trim(); @@ -214,9 +215,13 @@ class RulateScraper { if ( !loadedCheerio(this).find('td > span[class="disabled"]').length && - releaseDate + releaseTime ) { - chapters.push({ chapterName, releaseDate, chapterUrl }); + chapters.push({ + chapterName, + releaseDate: parseDate(releaseTime), + chapterUrl, + }); } }); @@ -287,4 +292,28 @@ class RulateScraper { } } +function parseDate(dateString = '') { + const months = { + 'янв.': 1, + 'февр.': 2, + 'мар.': 3, + 'апр.': 4, + мая: 5, + 'июн.': 6, + 'июл.': 7, + 'авг.': 8, + 'сент.': 9, + 'окт.': 10, + 'нояб.': 11, + 'дек.': 12, + }; + const [day, month, year, , time] = dateString.split(' '); + if (day && months[month] && year && time) { + return dayjs(year + '-' + months[month] + '-' + day + ' ' + time).format( + 'LLL', + ); + } + return dateString; +} + export default RulateScraper; diff --git a/src/sources/ru/bookriver.js b/src/sources/ru/bookriver.js index fdf084115..2f443f2db 100644 --- a/src/sources/ru/bookriver.js +++ b/src/sources/ru/bookriver.js @@ -64,7 +64,7 @@ const parseNovelAndChapters = async novelUrl => { chapters.push({ chapterName: chapter.name, releaseDate: dayjs( - chapter?.firstPublishedAt || chapter.createdAt, + chapter.firstPublishedAt || chapter.createdAt || undefined, ).format('LLL'), chapterUrl: baseUrl + '/reader/' + book.slug + '/' + chapter.chapterId, }); diff --git a/src/sources/ru/freedlit.ts b/src/sources/ru/freedlit.ts index 177d2748f..a2f2769f5 100644 --- a/src/sources/ru/freedlit.ts +++ b/src/sources/ru/freedlit.ts @@ -1,3 +1,4 @@ +import dayjs from 'dayjs'; import * as cheerio from 'cheerio'; import { FilterInputs } from '../types/filterTypes'; import { @@ -67,10 +68,14 @@ const parseNovelAndChapters = async novelUrl => { loadedCheerio('a.chapter-line').each(function () { const chapterName = loadedCheerio(this).find('h6').text(); - const releaseDate = loadedCheerio(this).find('span[class="date"]').text(); const chapterUrl = loadedCheerio(this).attr('href'); if (chapterName && chapterUrl) { - chapters.push({ chapterName, releaseDate, chapterUrl }); + const releaseTime = loadedCheerio(this).find('span[class="date"]').text(); + chapters.push({ + chapterName, + releaseDate: parseDate(releaseTime), + chapterUrl, + }); } }); @@ -249,4 +254,27 @@ const LitSpaceScraper = { filters, }; +function parseDate(dateString = '') { + const months = { + января: 1, + февраля: 2, + марта: 3, + апреля: 4, + мая: 5, + июня: 6, + июля: 7, + августа: 8, + сентября: 9, + октября: 10, + ноября: 11, + декабря: 12, + }; + + const [day, month, year] = dateString.split(' '); + if (day && months[month] && year) { + return dayjs(year + '-' + months[month] + '-' + day).format('LL'); + } + return dateString; +} + export default LitSpaceScraper; diff --git a/src/sources/ru/jaomix.js b/src/sources/ru/jaomix.js index 9359081e2..e4b2f9129 100644 --- a/src/sources/ru/jaomix.js +++ b/src/sources/ru/jaomix.js @@ -1,3 +1,4 @@ +import dayjs from 'dayjs'; import * as cheerio from 'cheerio'; import { Status } from '../helpers/constants'; import { FilterInputs } from '../types/filterTypes'; @@ -88,7 +89,7 @@ const parseNovelAndChapters = async novelUrl => { loadedCheerio('.download-chapter div.title').each(function () { chapters.push({ chapterName: loadedCheerio(this).find('a').attr('title'), - releaseDate: loadedCheerio(this).find('time').text(), + releaseDate: parseDate(loadedCheerio(this).find('time').text()), chapterUrl: loadedCheerio(this).find('a').attr('href'), }); }); @@ -314,4 +315,30 @@ const JaomixScraper = { filters, }; +function parseDate(dateString = '') { + const months = { + Янв: 1, + Фев: 2, + Мар: 3, + Апр: 4, + Май: 5, + Июн: 6, + Июл: 7, + Авг: 8, + Сен: 9, + Окт: 10, + Ноя: 11, + Дек: 12, + }; + + const [time, day, month, year] = dateString.split(' '); + if (time && day && months[month] && year) { + return dayjs(year + '-' + months[month] + '-' + day + ' ' + time).format( + 'LLL', + ); + } + + return dateString; +} + export default JaomixScraper;