From ba531ebf69739cea860f26aec2f776813ae8da0d Mon Sep 17 00:00:00 2001 From: mp3butcher Date: Mon, 18 Nov 2024 16:10:20 +0100 Subject: [PATCH] fix testing ,playlist and refactorr queries --- main.js | 9 ++---- modules/QueryManager.js | 8 +++-- modules/Utils.js | 3 +- modules/download/DownloadQuery.js | 10 +++--- modules/download/DownloadQueryList.js | 2 +- modules/info/InfoQuery.js | 13 ++++---- modules/info/InfoQueryList.js | 8 +++-- modules/size/SizeQuery.js | 2 +- modules/types/Query.js | 19 +++++++---- renderer/renderer.css | 2 +- tests/InfoQuery.test.js | 5 ++- tests/Query.test.js | 46 ++++++++++++++++----------- tests/Utils.test.js | 6 ++-- 13 files changed, 76 insertions(+), 57 deletions(-) diff --git a/main.js b/main.js index d94d65d9..58dcfdbb 100644 --- a/main.js +++ b/main.js @@ -351,7 +351,7 @@ function scan(msg) { console.warn(" [x] contentype video!!!!!!!!!!!" + data); } //Large Content-Range - if (sizeok || contentlength > 100000) { + if(!contentype.startsWith('image') && (sizeok || contentlength > 1000000)) { toscandeeply = true; } let res = atob(data.response) @@ -360,8 +360,7 @@ function scan(msg) { if (res[0] == "#") { //HLS? console.log(res); toscandeeply = true; - } - else if (res.startsWith(' { console.log(env.settings.paths.mitmproxy); if (scannerIsOn) { - if (mitmproxyclient) mitmproxyclient.destroy(); console.log(__dirname); let script = path.join(path.dirname(__dirname), 'binaries', 'send_traffic_to_videodownloader.py'); @@ -421,8 +419,7 @@ ipcMain.handle("setScannerEnabled", (event, args) => { mitmwebprocess.on('close', (code) => { console.log(`child process exited with code ${code}`); }); - let msg = 'Proxy running on localhost on port 15930: Configure proxy in your browser to scan network'; - dialog.showMessageBox({ message: msg }); + dialog.showMessageBox({ message: 'Proxy running on localhost on port 15930: Configure proxy in your browser to scan network' }); createConnection(); } else { diff --git a/modules/QueryManager.js b/modules/QueryManager.js index d36a70c8..44664bdd 100644 --- a/modules/QueryManager.js +++ b/modules/QueryManager.js @@ -25,10 +25,11 @@ class QueryManager { async manage(url, headers) { let metadataVideo = new Video(url, headers, "metadata", this.environment); this.addVideo(metadataVideo); - const initialQuery = await new InfoQuery(url, headers,metadataVideo.identifier, this.environment).connect(); + const initialQuery = await new InfoQuery(metadataVideo, metadataVideo.identifier).connect(); if(metadataVideo.error) return; if(Utils.isYouTubeChannel(url)) { - const actualQuery = await new InfoQuery(initialQuery.entries[0].url, metadataVideo.headers, metadataVideo.identifier, this.environment).connect(); + let metadataVideo2 = new Video(initialQuery.entries[0].url, metadataVideo.headers, metadataVideo.identifier, this.environment); + const actualQuery = await new InfoQuery(metadataVideo2, metadataVideo.identifier).connect(); if(metadataVideo.error) return; this.removeVideo(metadataVideo); if(actualQuery.entries == null || actualQuery.entries.length === 0) this.managePlaylist(initialQuery, url); @@ -127,6 +128,7 @@ class QueryManager { type: video.type, identifier: video.identifier, url: video.url, + headers: video.headers, title: video.title, duration: video.duration, audioOnly: video.audioOnly, @@ -157,7 +159,7 @@ class QueryManager { } downloadVideo.audioQuality = (downloadVideo.audioQuality != null) ? downloadVideo.audioQuality : "best"; let progressBar = new ProgressBar(this, downloadVideo); - downloadVideo.setQuery(new DownloadQuery(downloadVideo.url, downloadVideo.headers, downloadVideo, this.environment, progressBar, this.playlistMetadata)); + downloadVideo.setQuery(new DownloadQuery(downloadVideo, progressBar, this.playlistMetadata)); downloadVideo.query.connect().then(() => { //Backup done call, sometimes it does not trigger automatically from within the downloadQuery. if(downloadVideo.error) return; diff --git a/modules/Utils.js b/modules/Utils.js index 79d54090..1cc4f9af 100644 --- a/modules/Utils.js +++ b/modules/Utils.js @@ -60,6 +60,7 @@ class Utils { } for(const entry of infoQueryResult.entries) { let url; + entry.headers=[]; if (entry.url == null) url = entry.webpage_url; else url = entry.url; if(entry.formats != null && entry.formats.length > 0) { @@ -67,7 +68,7 @@ class Utils { alreadyDone.push(entry); continue; } - urls.push({'url':url,'headers':entry.headers}); + urls.push({'url':url,'headers':[]}); } return [urls, alreadyDone] } diff --git a/modules/download/DownloadQuery.js b/modules/download/DownloadQuery.js index b24f2d8f..b82d4fcc 100644 --- a/modules/download/DownloadQuery.js +++ b/modules/download/DownloadQuery.js @@ -5,14 +5,12 @@ const Utils = require("../Utils") const console = require("console"); class DownloadQuery extends Query { - constructor(url, headers, video, environment, progressBar, playlistMeta) { - super(environment, video.identifier); + constructor(video, progressBar, playlistMeta) { + super(video.environment, video.identifier); this.playlistMeta = playlistMeta; - this.url = url; this.video = video; this.progressBar = progressBar; this.format = video.formats[video.selected_format_index]; - this.headers = headers; } cancel() { @@ -171,7 +169,7 @@ class DownloadQuery extends Query { let result = null; const regexliverec = /size=\s*(\d+\w)iB\s*time=(\d+:\d+:\d+.\d+)\s*bitrate=\s*(\d+.\d+\w)bits\/s\s*speed=\d+.\d+x/; try { - result = await this.environment.downloadLimiter.schedule(() => this.start(this.url, args, (liveData) => { + result = await this.environment.downloadLimiter.schedule(() => this.start(this.video, args, (liveData) => { this.environment.logger.log(this.video.identifier, liveData); this.video.setFilename(liveData); if(this.video.is_live) { @@ -183,7 +181,7 @@ class DownloadQuery extends Query { try{ const livrec = liveData.match(regexliverec); if (typeof (`${livrec[1]}`) == "undefined") return; - this.progressBar.updateDownload('livestream', `${livrec[2]}`, `${livrec[3]}bits\/s`, this.video.audioOnly || this.video.downloadingAudio); + this.progressBar.updateDownload('livestream', `${livrec[2]}`, `${livrec[3]}bits/s`, this.video.audioOnly || this.video.downloadingAudio); } catch(e) { return; } diff --git a/modules/download/DownloadQueryList.js b/modules/download/DownloadQueryList.js index d48bdd3f..602514a0 100644 --- a/modules/download/DownloadQueryList.js +++ b/modules/download/DownloadQueryList.js @@ -27,7 +27,7 @@ class DownloadQueryList { return await new Promise(((resolve) => { for(let video of this.videos) { let progressBar = new ProgressBar(this.manager, video); - let task = new DownloadQuery(video.webpage_url, video.headers, video, this.environment, progressBar, Utils.getVideoInPlaylistMetadata(video.url, null, this.playlistMetadata)); + let task = new DownloadQuery(video, progressBar, Utils.getVideoInPlaylistMetadata(video.url, null, this.playlistMetadata)); if(video.parentID != null && !this.parentProgress.some(e => e.id === video.parentID)) { const bar = new ProgressBar(this.manager, video.parentID); this.parentProgress.push({ diff --git a/modules/info/InfoQuery.js b/modules/info/InfoQuery.js index 461e1765..15ddeb55 100644 --- a/modules/info/InfoQuery.js +++ b/modules/info/InfoQuery.js @@ -1,11 +1,10 @@ const Query = require("../types/Query"); class InfoQuery extends Query { - constructor(url, headers, identifier, environment) { - super(environment, identifier); - this.url = url; - this.headers = headers; - this.environment = environment; + constructor(video, identifier) { + super(video.environment, identifier); + this.video = video; + this.environment = video.environment; this.identifier = identifier; } @@ -16,8 +15,8 @@ class InfoQuery extends Query { args.push('--file-access-retries'); args.push(this.environment.settings.fileAccessRetries); } - this.headers.forEach((h) => args.push("--add-headers", h.k + ": " + h.v)); - let data = await this.environment.metadataLimiter.schedule(() => this.start(this.url, args)); + this.video.headers.forEach((h) => args.push("--add-headers", h.k + ": " + h.v)); + let data = await this.environment.metadataLimiter.schedule(() => this.start(this.video, args)); return JSON.parse(data); } catch (e) { this.environment.errorHandler.checkError(e.stderr, this.identifier) diff --git a/modules/info/InfoQueryList.js b/modules/info/InfoQueryList.js index c6869431..327d4fff 100644 --- a/modules/info/InfoQueryList.js +++ b/modules/info/InfoQueryList.js @@ -3,13 +3,14 @@ const Video = require('../types/Video'); const Utils = require("../Utils"); class InfoQueryList { - constructor(query, environment, progressBar) { + constructor(query, headers, environment, progressBar) { this.query = query; this.environment = environment; this.progressBar = progressBar; this.urls = null; this.length = null; this.done = 0; + this.headers = headers; } async start() { @@ -29,10 +30,11 @@ class InfoQueryList { resolve(null); } for (const url of this.urls) { - let task = new InfoQuery(url,this.progressBar.video.headers, this.progressBar.video.identifier, this.environment); + let metadataVideo = new Video(url.url, url.headers, "", this.environment); + let task = new InfoQuery(metadataVideo, ""); task.connect().then((data) => { if (data.formats != null) { - let video = this.createVideo(data, url, data.headers); + let video = this.createVideo(data, url.url, url.headers); totalMetadata.push(video); } this.done++; diff --git a/modules/size/SizeQuery.js b/modules/size/SizeQuery.js index 96ce3a55..af3166a8 100644 --- a/modules/size/SizeQuery.js +++ b/modules/size/SizeQuery.js @@ -30,7 +30,7 @@ class SizeQuery extends Query { formatArgument = `bestvideo+${this.format}audio/bestvideo+bestaudio/best`; } - let output = await this.environment.metadataLimiter.schedule(() => this.start(this.video.url, ["-J", "--flat-playlist", "-f", formatArgument])); + let output = await this.environment.metadataLimiter.schedule(() => this.start(this.video, ["-J", "--flat-playlist", "-f", formatArgument])); let data = JSON.parse(output); let totalSize = 0; if(data.requested_formats != null) { diff --git a/modules/types/Query.js b/modules/types/Query.js index bb4273a6..c8614604 100644 --- a/modules/types/Query.js +++ b/modules/types/Query.js @@ -8,18 +8,20 @@ class Query { this.identifier = identifier this.process = null; this.stopped = false; + this.video = null; } stop() { this.stopped = true; if(this.process != null) { - if (this.progressBar.video.is_live) - process.kill(this.process.pid, 'SIGINT');///Only way to stop ffmpeg lives - else this.process.kill(); + if (this.video.is_live) process.kill(this.process.pid, 'SIGINT');///Only way to stop ffmpeg lives + else this.process.cancel(); } } - async start(url, args, cb) { + async start(video, args, cb) { + this.video = video; + let url = video.url; if(this.stopped) return "killed"; args.push("--no-cache-dir"); args.push("--ignore-config"); @@ -82,7 +84,9 @@ class Query { //Return data while the query is running (live) //Return "done" when the query has finished return await new Promise((resolve) => { - this.process = child_process.spawn(command, args); + try { + if(video.is_live) this.process = child_process.spawn(command, args); + else this.process = execa(command, args); this.process.stdout.setEncoding('utf8'); this.process.stdout.on('data', (data) => { const lines = data @@ -108,7 +112,10 @@ class Query { resolve("killed"); } console.error(data.toString()) - }) + }); + } catch(e) { + console.log(e); + } }); } } diff --git a/renderer/renderer.css b/renderer/renderer.css index 5de02d43..e3ba13b4 100644 --- a/renderer/renderer.css +++ b/renderer/renderer.css @@ -291,7 +291,7 @@ select { .bi-card-text-strike { display: block; - background: url(img/card-text-strike.svg); + background: url(img/card-text-strike.svg) center no-repeat; height: 1em; width: 1em; margin-top: 0.2em ; diff --git a/tests/InfoQuery.test.js b/tests/InfoQuery.test.js index 02d1087f..10e9e389 100644 --- a/tests/InfoQuery.test.js +++ b/tests/InfoQuery.test.js @@ -1,4 +1,5 @@ const InfoQuery = require("../modules/info/InfoQuery"); +const Video = require("../modules/types/Video"); describe('Connect the InfoQuery', () => { beforeEach(() => { @@ -44,6 +45,8 @@ function instanceBuilder() { }, settings: {} }; - return [env, new InfoQuery("http://url.link", [], "test__id", env)]; + let metadataVideo = new Video("http://url.link", [], "test__id", env); + + return [env, new InfoQuery(metadataVideo, "test__id")]; } diff --git a/tests/Query.test.js b/tests/Query.test.js index 21113b0b..888c9296 100644 --- a/tests/Query.test.js +++ b/tests/Query.test.js @@ -1,11 +1,21 @@ const UserAgent = require('user-agents'); const Query = require("../modules/types/Query"); +const Video = require("../modules/types/Video"); const execa = require('execa'); const { PassThrough } = require('stream'); jest.mock('user-agents'); jest.mock('execa'); - +//jest.mock('child_process'); +const env = { + metadataLimiter: { + schedule: jest.fn() + }, + errorHandler: { + checkError: jest.fn() + }, + settings: {} +}; beforeEach(() => { jest.clearAllMocks(); jest.doMock('execa', () => { @@ -26,7 +36,7 @@ describe('ytdl Query', () => { UserAgent.prototype.toString = jest.fn().mockReturnValue("agent"); const errorHandlerMock = jest.fn(); const instance = instanceBuilder("spoof", null, errorHandlerMock, "python"); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(UserAgent.prototype.toString).toBeCalledTimes(1); expect(execa.mock.calls[0][1]).toContain("--user-agent"); expect(execa.mock.calls[0][1]).toContain("agent"); @@ -36,7 +46,7 @@ describe('ytdl Query', () => { UserAgent.prototype.toString = jest.fn().mockReturnValue("agent"); const errorHandlerMock = jest.fn(); const instance = instanceBuilder("empty", null, errorHandlerMock, "python"); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(UserAgent.prototype.toString).toBeCalledTimes(0); expect(execa.mock.calls[0][1]).toContain("--user-agent"); expect(execa.mock.calls[0][1]).toContain("''"); @@ -45,7 +55,7 @@ describe('ytdl Query', () => { it('adds the proxy when one is set', () => { const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", "a/path/to/cookies.txt", errorHandlerMock, "python", "https://iama.proxy"); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(execa.mock.calls[0][1]).toContain("--proxy"); expect(execa.mock.calls[0][1]).toContain("https://iama.proxy"); }); @@ -53,14 +63,14 @@ describe('ytdl Query', () => { it('does not add a proxy when none are set', () => { const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", "a/path/to/cookies.txt", errorHandlerMock, "python", ""); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(execa.mock.calls[0][1]).not.toContain("--proxy"); }); }) it('adds the cookies argument when specified in settings', () => { const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", "a/path/to/cookies.txt", errorHandlerMock, "python"); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(execa.mock.calls[0][1]).toContain("--cookies"); expect(execa.mock.calls[0][1]).toContain("a/path/to/cookies.txt"); }); @@ -68,7 +78,7 @@ describe('ytdl Query', () => { it('uses the detected python command', () => { const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", null, errorHandlerMock, "python3"); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(execa.mock.calls[0][0]).toEqual("python3"); expect(execa.mock.calls[0][1][0]).toEqual("a/path/to/ytdl"); }); @@ -76,14 +86,14 @@ describe('ytdl Query', () => { it('adds the url as final argument', () => { const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", null, errorHandlerMock, "python"); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(execa.mock.calls[0][1][execa.mock.calls[0][1].length - 1]).toContain("https://url.link"); }); }) it('adds the no-cache-dir as argument', () => { const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", null, errorHandlerMock, "python"); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(execa.mock.calls[0][1]).toContain("--no-cache-dir"); }); }); @@ -96,10 +106,10 @@ describe('Query with live callback', () => { const errorHandlerMock = jest.fn(); const callbackMock = jest.fn(); const instance = instanceBuilder("default", null, errorHandlerMock, "python"); - const result = instance.start("https://url.link", [], callbackMock); + const result = instance.start(new Video("https://url.link", [], "",env), [], callbackMock); setTimeout(() => { instance.stop(); - }, 100); + }, 10); await expect(result).resolves.toEqual("killed"); expect(callbackMock).toBeCalledWith("killed"); }); @@ -110,7 +120,7 @@ describe('Query with live callback', () => { const errorHandlerMock = jest.fn(); const callbackMock = jest.fn(); const instance = instanceBuilder("default", null, errorHandlerMock, "python"); - const result = instance.start("https://url.link", [], callbackMock); + const result = instance.start(new Video("https://url.link", [], "test__id",env), [], callbackMock); setTimeout(() => { stderr.emit("data", "test-error"); }, 100); @@ -125,7 +135,7 @@ describe('Query with live callback', () => { execa.mockReturnValue(mock) const callbackMock = jest.fn(); const instance = instanceBuilder("default", null, jest.fn(), "python"); - const result = instance.start("https://url.link", [], callbackMock); + const result = instance.start(new Video("https://url.link", [], "test__id",env), [], callbackMock); setTimeout(() => { stdout.emit("close"); }, 100); @@ -137,7 +147,7 @@ describe('Query with live callback', () => { execa.mockReturnValue(mock); const callbackMock = jest.fn(); const instance = instanceBuilder("default", null, jest.fn(), "python"); - const result = instance.start("https://url.link", [], callbackMock); + const result = instance.start(new Video("https://url.link", [], "test__id",env), [], callbackMock); setTimeout(() => { stdout.emit("data", "test-data"); stdout.emit("data", "more-test-data"); @@ -154,21 +164,21 @@ describe('Query without callback', () => { execa.mockResolvedValue({stdout: "fake-data"}); const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", null, errorHandlerMock, "python"); - const result = instance.start("https://url.link", [], null) + const result = instance.start(new Video("https://url.link", [], "test__id",env), [], null) await expect(result).resolves.toEqual("fake-data"); }); it('Returns a stringified empty object on error', async () => { execa.mockResolvedValue(null); const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", null, errorHandlerMock, "python"); - const result = instance.start("https://url.link", [], null) + const result = instance.start(new Video("https://url.link", [], "test__id",env), [], null) await expect(result).resolves.toEqual("{}"); }); it('Checks the error on error', () => { execa.mockResolvedValue(null); const errorHandlerMock = jest.fn(); const instance = instanceBuilder("default", null, errorHandlerMock, "python"); - return instance.start("https://url.link", [], null).then(() => { + return instance.start(new Video("https://url.link", [], "test__id",env), [], null).then(() => { expect(errorHandlerMock).toBeCalled(); }); }); @@ -183,4 +193,4 @@ function execaMockBuilder(killed) { function instanceBuilder(userAgent, cookiePath, errorHandlerMock, pythonCommand, proxy) { return new Query({pythonCommand: pythonCommand, errorHandler: {checkError: errorHandlerMock, raiseUnhandledError: errorHandlerMock}, paths: {ytdl: "a/path/to/ytdl"}, settings: {cookiePath: cookiePath, userAgent, proxy: proxy}}, "test__id"); -} +} \ No newline at end of file diff --git a/tests/Utils.test.js b/tests/Utils.test.js index 2893e451..0663a8b0 100644 --- a/tests/Utils.test.js +++ b/tests/Utils.test.js @@ -98,13 +98,13 @@ describe('extractPlaylistUrls', () => { expect(console.error).toBeCalledTimes(1); }); it('returns playlist urls',() => { - expect(Utils.extractPlaylistUrls({entries: [{url: "1"}, {url: "2"}]})).toContainEqual([{headers: undefined, url: "1"}, {headers: undefined, url: "2"}]) + expect(Utils.extractPlaylistUrls({entries: [{url: "1"}, {url: "2"}]})).toContainEqual([{headers: [], url: "1"}, {headers: [], url: "2"}]) }); it('returns already done (queried) urls', () => { - expect(Utils.extractPlaylistUrls({entries: [{url: "1", formats: ["format"]}, {url: "2"}]})).toContainEqual([{url: "1", headers: undefined, formats: ["format"]}]); + expect(Utils.extractPlaylistUrls({entries: [{url: "1", formats: ["format"]}, {url: "2"}]})).toContainEqual([{url: "1", headers: [], formats: ["format"]}]); }); it('uses the webpage_url when url is null', () => { - expect(Utils.extractPlaylistUrls({entries: [{webpage_url: "url"}]})).toContainEqual([{url:"url", "headers": undefined}]); + expect(Utils.extractPlaylistUrls({entries: [{webpage_url: "url"}]})).toContainEqual([{url:"url", "headers": []}]); }); });