Skip to content

Commit

Permalink
v2.0.5 progress saving
Browse files Browse the repository at this point in the history
  • Loading branch information
@dr-nyt
dr-nyt committed Jun 9, 2021
1 parent 1ea376c commit 649150f
Show file tree
Hide file tree
Showing 6 changed files with 149 additions and 99 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "NovelScraper",
"version": "2.0.4",
"version": "2.0.5",
"description": "App for downloading novels from pirate sites.",
"homepage": "https://github.com/HanaDigital/NovelScraper",
"author": {
Expand Down
11 changes: 7 additions & 4 deletions src/app/services/novel-factory.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export class NovelFactoryService {
"NovelScraper-Library"
);

constructor(public database: DatabaseService) {}
constructor(public database: DatabaseService) { }

async generateEpub(
novel: novelObj,
Expand All @@ -33,7 +33,7 @@ export class NovelFactoryService {
await shellJS.mkdir("-p", novelFolder);

// Save chapters in a json file and set the novel as download
await this.saveChapters(novel, chapters, novelFolder);
await this.saveChapters(novel, chapters);

// Download the cover and get its path
const coverPath = await this.downloadCover(novel.cover, novelFolder);
Expand Down Expand Up @@ -72,9 +72,12 @@ export class NovelFactoryService {
// Save chapters to a json file so we don't re-download when updating
async saveChapters(
novel: novelObj,
chapters: chapterObj[],
novelFolder: string
chapters: chapterObj[]
): Promise<void> {
// Create the folder for storing the novel files
const novelFolder = novel.folderPath;
await shellJS.mkdir("-p", novelFolder);

const chaptersFile = path.join(novelFolder, "chapters.json");
const json = JSON.stringify(chapters, null, 4);
writeFile(chaptersFile, json, (err) => {
Expand Down
78 changes: 45 additions & 33 deletions src/app/services/sources/boxnovel.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,50 +272,62 @@ export class BoxnovelService extends sourceService {
chapterNames = chapterNames.slice(update.startIndex);
}

// Download each chapter at a time
for (let i = 0; i < chapterLinks.length; i++) {
if (this.database.isCanceled(downloadID)) {
this.database.updateDownloading(novel.link, false);
console.log("Download canceled!");
return;
}
const totalLength = downloadedChapters.length + chapterLinks.length;
let canceled = false;

const html = await this.getHtml(chapterLinks[i]);
const chapterHtml = html.getElementsByClassName(
"entry-content"
)[0];
try {
chapterHtml.getElementsByClassName("cha-tit")[0].remove(); // Remove h3 tag from chapter
} catch (error) {
console.log(
"Missing 'cha-tit' class at chapter index " +
try {
// Download each chapter at a time
for (let i = 0; i < chapterLinks.length; i++) {
if (this.database.isCanceled(downloadID)) {
this.database.updateDownloading(novel.link, false);
console.log("Download canceled!");
canceled = true;
return;
}

const html = await this.getHtml(chapterLinks[i]);
const chapterHtml = html.getElementsByClassName(
"entry-content"
)[0];
try {
chapterHtml.getElementsByClassName("cha-tit")[0].remove(); // Remove h3 tag from chapter
} catch (error) {
console.log(
"Missing 'cha-tit' class at chapter index " +
i +
"and chapter name " +
chapterNames[i]
);
}
);
}

const chapterTitle = chapterNames[i];
const chapterTitle = chapterNames[i];

let chapterBody = "<h3>" + chapterTitle + "</h3>";
chapterBody += chapterHtml.outerHTML;
let chapterBody = "<h3>" + chapterTitle + "</h3>";
chapterBody += chapterHtml.outerHTML;

const chapter = this.prepChapter(
const chapter = this.prepChapter(novel, downloadID, chapterTitle, chapterBody, downloadedChapters.length, totalLength);
downloadedChapters.push(chapter);
}

if (!canceled) this.novelFactory.generateEpub(novel, downloadedChapters, downloadID);
} catch (error) {
canceled = true;
console.error("Error downloading the complete novel. Retry.");
console.error(error);
}

if (canceled) {
this.novelFactory.saveChapters(
novel,
downloadID,
chapterTitle,
chapterBody,
i,
chapterLinks.length
downloadedChapters
);
downloadedChapters.push(chapter);
this.database.cancelDownload(downloadID);
this.database.updateDownloading(novel.link, false);
this.database.updateDownloadedChapters(novel.link, downloadedChapters.length);
this.database.updateDownloaded(novel.link, true);
this.database.updateIsUpdated(novel.link, false);
}

this.novelFactory.generateEpub(
novel,
downloadedChapters,
downloadID
);
} catch (error) {
this.database.cancelDownload(downloadID);
this.database.updateDownloading(novel.link, false);
Expand Down
88 changes: 50 additions & 38 deletions src/app/services/sources/novelfull.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,9 @@ export class NovelfullService extends sourceService {
for (let x = 0; x < chapters.length; x++) {
chapterLinks.push(
"https://novelfull.com" +
chapters[x]
.getElementsByTagName("a")[0]
.getAttribute("href")
chapters[x]
.getElementsByTagName("a")[0]
.getAttribute("href")
);
chapterNames.push(
chapters[x].getElementsByTagName("a")[0].title
Expand All @@ -280,53 +280,65 @@ export class NovelfullService extends sourceService {
chapterNames = chapterNames.slice(update.startIndex);
}

// Download each chapter at a time
for (let i = 0; i < chapterLinks.length; i++) {
if (this.database.isCanceled(downloadID)) {
this.database.updateDownloading(novel.link, false);
console.log("Download canceled!");
return;
}
const totalLength = downloadedChapters.length + chapterLinks.length;
let canceled = false;

const html = await this.getHtml(chapterLinks[i]);
try {
// Download each chapter at a time
for (let i = 0; i < chapterLinks.length; i++) {
if (this.database.isCanceled(downloadID)) {
this.database.updateDownloading(novel.link, false);
console.log("Download canceled!");
canceled = true;
return;
}

//////////////////////// [2] YOUR CODE STARTS HERE ///////////////////////////////
const html = await this.getHtml(chapterLinks[i]);

// FIXME: you have the html of the chapter page
// Get the element that wraps all the paragraphs of the chapter
const chapterHtml = html.getElementsByClassName("chapter-c")[0];
//////////////////////// [2] YOUR CODE STARTS HERE ///////////////////////////////

//////////////////////// YOUR CODE ENDS HERE /////////////////////////////////
// FIXME: you have the html of the chapter page
// Get the element that wraps all the paragraphs of the chapter
const chapterHtml = html.getElementsByClassName("chapter-c")[0];

const chapterTitle = chapterNames[i];
//////////////////////// YOUR CODE ENDS HERE /////////////////////////////////

let chapterBody = "<h3>" + chapterTitle + "</h3>";
chapterBody += chapterHtml.outerHTML;
const chapterTitle = chapterNames[i];

chapterBody = chapterBody.replace(
/\(adsbygoogle = window.adsbygoogle \|\| \[\]\).push\({}\);/g,
""
);
chapterBody = chapterBody.replace(/<script.*><\/script>/g, "");
chapterBody = chapterBody.replace(/<ins.*<\/ins>/g, "");
console.log(chapterBody);
let chapterBody = "<h3>" + chapterTitle + "</h3>";
chapterBody += chapterHtml.outerHTML;

chapterBody = chapterBody.replace(
/\(adsbygoogle = window.adsbygoogle \|\| \[\]\).push\({}\);/g,
""
);
chapterBody = chapterBody.replace(/<script.*><\/script>/g, "");
chapterBody = chapterBody.replace(/<ins.*<\/ins>/g, "");
console.log(chapterBody);

const chapter = this.prepChapter(novel, downloadID, chapterTitle, chapterBody, downloadedChapters.length, totalLength);
downloadedChapters.push(chapter);
}

if (!canceled) this.novelFactory.generateEpub(novel, downloadedChapters, downloadID);
} catch (error) {
canceled = true;
console.error("Error downloading the complete novel. Retry.");
console.error(error);
}

const chapter = this.prepChapter(
if (canceled) {
this.novelFactory.saveChapters(
novel,
downloadID,
chapterTitle,
chapterBody,
i,
chapterLinks.length
downloadedChapters
);
downloadedChapters.push(chapter);
this.database.cancelDownload(downloadID);
this.database.updateDownloading(novel.link, false);
this.database.updateDownloadedChapters(novel.link, downloadedChapters.length);
this.database.updateDownloaded(novel.link, true);
this.database.updateIsUpdated(novel.link, false);
}

this.novelFactory.generateEpub(
novel,
downloadedChapters,
downloadID
);
} catch (error) {
this.database.cancelDownload(downloadID);
this.database.updateDownloading(novel.link, false);
Expand Down
60 changes: 41 additions & 19 deletions src/app/services/sources/readlightnovel-service.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,35 +163,57 @@ export class ReadlightnovelService extends sourceService {
chapterNames = chapterNames.slice(update.startIndex);
}

// Download each chapter at a time
for (let i = 0; i < chapterLinks.length; i++) {
if (this.database.isCanceled(downloadID)) {
this.database.updateDownloading(novel.link, false);
console.log('Download canceled!')
return;
}
const totalLength = downloadedChapters.length + chapterLinks.length;
let canceled = false;

try {
// Download each chapter at a time
for (let i = 0; i < chapterLinks.length; i++) {
if (this.database.isCanceled(downloadID)) {
this.database.updateDownloading(novel.link, false);
console.log('Download canceled!');
canceled = true;
break;
}

const html = await this.getHtml(chapterLinks[i]);

const html = await this.getHtml(chapterLinks[i]);
//////////////////////// YOUR CODE STARTS HERE ///////////////////////////////

//////////////////////// YOUR CODE STARTS HERE ///////////////////////////////
// You have the html of the chapter page
// Get the element that wraps all the paragraphs of the chapter
const chapterHtml = html.getElementsByClassName('desc')[0].getElementsByClassName("hidden")[0];

// You have the html of the chapter page
// Get the element that wraps all the paragraphs of the chapter
const chapterHtml = html.getElementsByClassName('desc')[0].getElementsByClassName("hidden")[0];
//////////////////////// YOUR CODE ENDS HERE /////////////////////////////////

//////////////////////// YOUR CODE ENDS HERE /////////////////////////////////
const chapterTitle = chapterNames[i];

const chapterTitle = chapterNames[i];
let chapterBody = "<h3>" + chapterTitle + "</h3>";
chapterBody += chapterHtml.outerHTML;

let chapterBody = "<h3>" + chapterTitle + "</h3>";
chapterBody += chapterHtml.outerHTML;

const chapter = this.prepChapter(novel, downloadID, chapterTitle, chapterBody, downloadedChapters.length, totalLength);
downloadedChapters.push(chapter);
}

const chapter = this.prepChapter(novel, downloadID, chapterTitle, chapterBody, i, chapterLinks.length);
downloadedChapters.push(chapter);
if (!canceled) this.novelFactory.generateEpub(novel, downloadedChapters, downloadID);
} catch (error) {
canceled = true;
console.error("Error downloading the complete novel. Retry.");
console.error(error);
}

this.novelFactory.generateEpub(novel, downloadedChapters, downloadID);
if (canceled) {
this.novelFactory.saveChapters(
novel,
downloadedChapters
);
this.database.cancelDownload(downloadID);
this.database.updateDownloading(novel.link, false);
this.database.updateDownloadedChapters(novel.link, downloadedChapters.length);
this.database.updateDownloaded(novel.link, true);
this.database.updateIsUpdated(novel.link, false);
}

} catch (error) {
this.database.cancelDownload(downloadID);
Expand Down
9 changes: 5 additions & 4 deletions src/app/services/sources/sourceService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export class sourceService {

scrollPos: number;

constructor(public database: DatabaseService) {}
constructor(public database: DatabaseService) { }

async searchWIthLink(
link: string,
Expand All @@ -23,9 +23,9 @@ export class sourceService {
return {};
}

async searchWithName(name: string, source: string): Promise<void> {}
async searchWithName(name: string, source: string): Promise<void> { }

async download(novel: novelObj, downloadID: number): Promise<void> {}
async download(novel: novelObj, downloadID: number): Promise<void> { }

update(novel: novelObj, numOfChapters: number): update {
if (numOfChapters > novel.totalChapters) {
Expand Down Expand Up @@ -90,7 +90,8 @@ export class sourceService {

const percentage = ((currentPos / destPos) * 100).toFixed(2);
this.database.updateDownloadTracker(downloadID, percentage);
console.log(percentage);

console.log(`Downloaded chapter ${currentPos}`)

return {
title: title,
Expand Down

0 comments on commit 649150f

Please sign in to comment.