",
"license": "AGPL-3.0-only",
"devDependencies": {
"electron": "^26.2.2",
- "electron-builder": "^24.6.4",
+ "electron-builder": "^25.1.7",
"eslint": "^7.32.0",
"jest": "^27.3.0"
},
"dependencies": {
"adm-zip": "^0.5.10",
- "axios": "^0.21.4",
- "bootstrap": "^4.5.3",
+ "amqplib": "^0.10.4",
+ "axios": "^1.7.7",
+ "bootstrap": "^5.3.3",
"bootstrap-icons": "^1.3.0",
+ "bootstrap4-toggle": "^3.6.1",
"bottleneck": "^2.19.5",
"dotenv": "^16.3.1",
"electron-updater": "^6.1.4",
@@ -50,9 +55,12 @@
},
"repository": {
"type": "git",
- "url": "https://github.com/StefanLobbenmeier/youtube-dl-gui.git"
+ "url": "https://github.com/mp3butcher/youtube-dl-gui.git"
},
"build": {
+ "extraResources": [
+ "binaries/send_traffic_to_videodownloader.py"
+ ],
"afterPack": "./build/appimage-fix.js",
"appId": "com.jelleglebbeek.youtube-dl-gui",
"asarUnpack": "**/binaries/*",
diff --git a/preload.js b/preload.js
index 5d14c467..7b84281f 100644
--- a/preload.js
+++ b/preload.js
@@ -23,7 +23,8 @@ contextBridge.exposeInMainWorld(
"getSubtitles",
"getSelectedSubtitles",
"getLog",
- "saveLog"
+ "saveLog",
+ "setScannerEnabled"
];
if (validChannels.includes(channel)) {
return await ipcRenderer.invoke(channel, data);
diff --git a/renderer/renderer.css b/renderer/renderer.css
index 5318ba12..5de02d43 100644
--- a/renderer/renderer.css
+++ b/renderer/renderer.css
@@ -128,6 +128,7 @@ select {
/* URL FORM */
.url-input {
+ display: inline-block;
margin-top: calc(40px + 1.5rem); /* Compensate the fixed navbar */
margin-bottom: 1.5rem;
}
@@ -147,6 +148,43 @@ select {
transform: scale(1.5);
}
+/* scanner toggle button*/
+.btns-input {
+ display: inline;
+ padding-left: 0px;
+ margin-top: calc(40px + 1.5rem); /* Compensate the fixed navbar */
+ margin-bottom: 1.5rem;
+}
+
+.btns-input .toggle-handle{
+ background-color:var(--font-color) !important;
+ border-color: var(--font-color) !important;
+}
+
+.btns-input .toggle-on {
+ background-color: var(--secondary-highlight-color);
+ border-color: var(--secondary-highlight-color);
+}
+
+.btns-input .toggle-off {
+ background-color: var(--secondary-bg-color);
+ border-color: var(--secondary-bg-color);
+}
+
+.btns-input .toggle {
+ background-color: var(--secondary-bg-color) !important;
+}
+
+.btns-input .btn {
+ border-color: transparent !important;
+ color: var(--font-color) !important;
+ font-size: 24px;
+ font-weight: 600;
+ line-height: 0.5;
+ border-top-right-radius: 3px !important;
+ border-bottom-right-radius: 3px !important;
+}
+
#settingsBtn {
margin-left: 1rem;
}
@@ -216,6 +254,7 @@ select {
}
.video-card .handle {
+ z-index: -1;
position: absolute;
left: 0.25rem;
top: calc(50% + 2rem);
@@ -415,6 +454,7 @@ option.audio {
.controls .buttons * {
margin: 0 5px;
+ width: 140px;
}
.controls .buttons i:before {
diff --git a/renderer/renderer.html b/renderer/renderer.html
index 17809640..514ba309 100644
--- a/renderer/renderer.html
+++ b/renderer/renderer.html
@@ -5,6 +5,7 @@
Open Video Downloader
+
@@ -12,6 +13,7 @@
+
@@ -22,9 +24,16 @@
-
+
+
+
+
+
+
+
+
@@ -177,6 +186,7 @@
Video metadata
+
@@ -460,6 +470,11 @@ Advanced
+
+
+
+
+
-
-
diff --git a/renderer/renderer.js b/renderer/renderer.js
index d08f911c..b0652d53 100644
--- a/renderer/renderer.js
+++ b/renderer/renderer.js
@@ -145,7 +145,7 @@ async function init() {
$('#download-type').on('change', async () => {
updateAllVideoSettings();
await getSettings();
- sendSettings();
+ await sendSettings();
});
$('#infoModal .img-overlay, #infoModal .info-img').on('click', () => {
@@ -186,8 +186,8 @@ async function init() {
});
$('#settingsModal .apply').on('click', () => {
- $('#settingsModal').modal("hide");
sendSettings();
+ $('#settingsModal').modal("hide");
});
$('#maxConcurrent').on('input', () => {
@@ -208,6 +208,15 @@ async function init() {
$('#settingsModal').modal("show");
});
+ $('#scannerBtn').on('change', async () => {
+ try{
+ await window.main.invoke('setScannerEnabled',{value: $('#scannerBtn').prop('checked')});
+ } catch (e) {
+ //Catch mainly proxu AbortError
+ console.error(e);
+ }
+ });
+
$('#defaultConcurrent').on('click', () => {
window.main.invoke("settingsAction", {action: "get"}).then((settings) => {
$('#concurrentLabel').html(`Max concurrent jobs (${settings.defaultConcurrent})`);
@@ -1003,6 +1012,7 @@ async function getSettings() {
$('#downloadThumbnail').prop('checked', settings.downloadThumbnail);
$('#keepUnmerged').prop('checked', settings.keepUnmerged);
$('#avoidFailingToSaveDuplicateFileName').prop('checked', settings.avoidFailingToSaveDuplicateFileName);
+ $('#allowUnsafeFileExtensions').prop('checked', settings.allowUnsafeFileExtensions);
$('#calculateTotalSize').prop('checked', settings.calculateTotalSize);
$('#maxConcurrent').val(settings.maxConcurrent);
$('#settingsModal #retries').val(settings.retries);
@@ -1015,7 +1025,7 @@ async function getSettings() {
window.settings = settings;
}
-function sendSettings() {
+async function sendSettings() {
let settings = {
updateBinary: $('#updateBinary').prop('checked'),
updateApplication: $('#updateApplication').prop('checked'),
@@ -1039,6 +1049,7 @@ function sendSettings() {
downloadThumbnail: $('#downloadThumbnail').prop('checked'),
keepUnmerged: $('#keepUnmerged').prop('checked'),
avoidFailingToSaveDuplicateFileName: $('#avoidFailingToSaveDuplicateFileName').prop('checked'),
+ allowUnsafeFileExtensions: $('#allowUnsafeFileExtensions').prop('checked'),
calculateTotalSize: $('#calculateTotalSize').prop('checked'),
sizeMode: $('#sizeSetting').val(),
splitMode: $('#splitMode').val(),
@@ -1050,7 +1061,7 @@ function sendSettings() {
theme: $('#theme').val()
}
window.settings = settings;
- window.main.invoke("settingsAction", {action: "save", settings});
+ await window.main.invoke("settingsAction", {action: "save", setting:settings});
updateEncodingDropdown(settings.enableEncoding);
toggleWhiteMode(settings.theme);
}
@@ -1138,7 +1149,10 @@ function showInfoModal(info, identifier) {
}
$(modal).find('img').prop("src", data.thumbnail);
$(modal).find('.modal-title').html(data.title);
- $(modal).find('#info-description').html(data.description == null ? "No description was found." : data.description);
+ let hs = "";
+ data.headers.forEach((h) => { hs = hs + h.k + ": " + h.v + "\n" } );
+ $(modal).find('#headers-string').html(hs);
+ $(modal).find('#info-description').html(data.description == null ? 'no description': data.description);
$(modal).find('.uploader').html('Uploader: ' + (data.uploader == null ? "Unknown" : data.uploader));
$(modal).find('.extractor').html('Extractor: ' + (data.extractor == null ? "Unknown" : data.extractor));
$(modal).find('.url').html('URL: ' + '' + data.url + '');
diff --git a/tests/InfoQuery.test.js b/tests/InfoQuery.test.js
index 2c186570..02d1087f 100644
--- a/tests/InfoQuery.test.js
+++ b/tests/InfoQuery.test.js
@@ -44,6 +44,6 @@ function instanceBuilder() {
},
settings: {}
};
- return [env, new InfoQuery("http://url.link", "test__id", env)];
+ return [env, new InfoQuery("http://url.link", [], "test__id", env)];
}
diff --git a/tests/InfoQueryList.test.js b/tests/InfoQueryList.test.js
index 93036914..e908606b 100644
--- a/tests/InfoQueryList.test.js
+++ b/tests/InfoQueryList.test.js
@@ -8,19 +8,19 @@ describe("create video", () => {
it('returns a video', () => {
const instance = instanceBuilder();
jest.spyOn(video.prototype, 'setMetadata').mockImplementation(() => {});
- const result = instance.createVideo({test: "data"}, "https://test.url");
+ const result = instance.createVideo({test: "data"}, "https://test.url",[]);
expect(result).toBeInstanceOf(video);
});
it('sets the metadata', () => {
const instance = instanceBuilder();
const metadataSpy = jest.spyOn(video.prototype, 'setMetadata').mockImplementation(() => {});
- instance.createVideo({test: "data"}, "https://test.url");
+ instance.createVideo({test: "data"}, "https://test.url",[]);
expect(metadataSpy).toBeCalledTimes(1);
});
it('uses data.entries[0] when appropriate', () => {
const instance = instanceBuilder();
const metadataSpy = jest.spyOn(video.prototype, 'setMetadata').mockImplementation(() => {});
- instance.createVideo({entries: [{formats: "insert_formats_here"}]}, "https://test.url");
+ instance.createVideo({entries: [{formats: "insert_formats_here"}]}, "https://test.url",[]);
expect(metadataSpy).toBeCalledWith({formats: "insert_formats_here"});
});
});
diff --git a/tests/MitmproxyUpdater.test.js b/tests/MitmproxyUpdater.test.js
new file mode 100644
index 00000000..07ad855f
--- /dev/null
+++ b/tests/MitmproxyUpdater.test.js
@@ -0,0 +1,129 @@
+const MitmproxyUpdater = require("../modules/MitmProxyUpdater");
+const fs = require("fs");
+const axios = require("axios");
+const { PassThrough } = require('stream');
+const os = require('os');
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ console.error = jest.fn().mockImplementation(() => {});
+ console.log = jest.fn().mockImplementation(() => {});
+})
+
+describe("writeVersionInfo", () => {
+ it('writes the version to a file', () => {
+ jest.spyOn(fs.promises, 'writeFile').mockResolvedValue("");
+ const instance = new MitmproxyUpdater({ mitmproxyVersion: "a/test/path" });
+ instance.writeVersionInfo("v2.0.0-test1");
+ expect(fs.promises.writeFile).toBeCalledTimes(1);
+ expect(fs.promises.writeFile).toBeCalledWith("a/test/path", "{\"version\":\"v2.0.0-test1\"}");
+ });
+});
+
+describe("getLocalVersion", () => {
+ it('returns null when when the file does not exist', () => {
+ jest.spyOn(fs.promises, 'readFile').mockRejectedValue("ENOTFOUND");
+ const instance = new MitmproxyUpdater({ ytdlVersion: "a/test/path" });
+ return instance.getLocalVersion().then((data) => {
+ expect(data).toBe(null);
+ });
+ });
+ it('returns the version property from the json file', () => {
+ jest.spyOn(fs.promises, 'readFile').mockResolvedValue("{\"version\": \"v2.0.0-test1\"}")
+ jest.spyOn(fs.promises, 'access').mockResolvedValue("");
+ const instance = new MitmproxyUpdater({ mitmproxyVersion: "a/test/path", mitmproxy: "Mitmproxy/path" });
+ return instance.getLocalVersion().then((data) => {
+ expect(data).toBe("v2.0.0-test1");
+ });
+ });
+ it('returns null when Mitmproxy is unset or false', () => {
+ jest.spyOn(fs.promises, 'readFile').mockResolvedValue("{\"version\": \"v2.0.0-test1\"}")
+ jest.spyOn(fs.promises, 'access').mockResolvedValue("");
+ const instance = new MitmproxyUpdater({ mitmproxyVersion: "a/test/path" });
+ return instance.getLocalVersion().then((data) => {
+ expect(data).toBe(null);
+ });
+ });
+});
+
+describe('getRemoteVersion', () => {
+ it('returns null on error', () => {
+ const axiosGetSpy = jest.spyOn(axios, 'get').mockRejectedValue({response: null});
+ jest.spyOn(os, 'arch').mockReturnValue('x64');
+ const instance = new MitmproxyUpdater({platform: "darwin"});
+ return instance.getRemoteVersion().then((data) => {
+ expect(data).toEqual({
+ remoteVersion: "10.4.2",
+ remoteMitmProxyUrl: "https://github.com/mp3butcher/mitmproxy/releases/download/zipped/mitmproxy.macos-arm.zip"
+ });
+ expect(axiosGetSpy).toBeCalledTimes(0);
+ });
+ });
+ it('returns object with the links and the version', async () => {
+ const axiosGetSpy = jest.spyOn(axios, 'get').mockResolvedValue({
+ data: { version: "4.2.1", bin: { "windows-32": { mitmproxy: "Mitmproxy/link"} } },
+ });
+ jest.spyOn(os, 'arch').mockReturnValue('ia32');
+ Object.defineProperty(process, "platform", {
+ value: "win32"
+ });
+ const instance = new MitmproxyUpdater({platform: "win32"});
+ const result = await instance.getRemoteVersion();
+ expect(result).toEqual({
+ remoteVersion: "10.4.2",
+ remoteMitmProxyUrl: "https://github.com/mp3butcher/mitmproxy/releases/download/zipped/mitmproxy.windows-amd64.zip"
+ });
+ expect(axiosGetSpy).toBeCalledTimes(0);
+ });
+});
+
+describe('checkUpdate', () => {
+ it('does nothing when local and remote version are the same', () => {
+ const win = {webContents: {send: jest.fn()}};
+ const instance = new MitmproxyUpdater({platform: "win32"}, win);
+ instance.paths.setPermissions = jest.fn();
+ const downloadUpdateSpy = jest.spyOn(instance, 'downloadUpdate');
+ jest.spyOn(instance, 'getLocalVersion').mockResolvedValue("v2.0.0");
+ jest.spyOn(instance, 'getRemoteVersion').mockResolvedValue({ remoteMitmproxyUrl: "link", remoteVersion: "v2.0.0" });
+ return instance.checkUpdate().then(() => {
+ expect(downloadUpdateSpy).not.toBeCalled();
+ expect(instance.win.webContents.send).not.toBeCalled();
+ });
+ });
+ it('does nothing when remote version returned null', () => {
+ const win = {webContents: {send: jest.fn()}};
+ const instance = new MitmproxyUpdater({platform: "win32"}, win);
+ instance.paths.setPermissions = jest.fn();
+ const downloadUpdateSpy = jest.spyOn(instance, 'downloadUpdate');
+ jest.spyOn(instance, 'getLocalVersion').mockResolvedValue("v2.0.0");
+ jest.spyOn(instance, 'getRemoteVersion').mockResolvedValue({ remoteMitmproxyUrl: "link", remoteVersion: null });
+ return instance.checkUpdate().then(() => {
+ expect(downloadUpdateSpy).not.toBeCalled();
+ expect(instance.win.webContents.send).not.toBeCalled();
+ });
+ });
+ it('downloads the latest remote version when local version is null', () => {
+ const win = {webContents: {send: jest.fn()}};
+ const instance = new MitmproxyUpdater({platform: "win32"}, win);
+ instance.paths.setPermissions = jest.fn();
+ const downloadUpdateSpy = jest.spyOn(instance, 'downloadUpdate').mockResolvedValue("");
+ jest.spyOn(instance, 'getLocalVersion').mockResolvedValue(null);
+ jest.spyOn(instance, 'getRemoteVersion').mockResolvedValue({ remoteMitmproxyUrl: "link", remoteVersion: "v2.0.0" });
+ return instance.checkUpdate().then(() => {
+ expect(downloadUpdateSpy).toBeCalledTimes(1);
+ expect(instance.win.webContents.send).toBeCalledTimes(1);
+ });
+ });
+ it('downloads the latest remote version when local version is different', () => {
+ const win = {webContents: {send: jest.fn()}};
+ const instance = new MitmproxyUpdater({platform: "win32", ytdl: "a/path/to"}, win);
+ instance.paths.setPermissions = jest.fn();
+ const downloadUpdateSpy = jest.spyOn(instance, 'downloadUpdate').mockResolvedValue("");
+ jest.spyOn(instance, 'getLocalVersion').mockResolvedValue("2021.03.10");
+ jest.spyOn(instance, 'getRemoteVersion').mockResolvedValue({ remoteMitmproxyUrl: "link", remoteFfprobeUrl: "link", remoteVersion: "2021.10.10" });
+ return instance.checkUpdate().then(() => {
+ expect(downloadUpdateSpy).toBeCalledTimes(1);
+ expect(instance.win.webContents.send).toBeCalledTimes(1);
+ });
+ });
+});
diff --git a/tests/Settings.test.js b/tests/Settings.test.js
index adcd8f84..63743c1d 100644
--- a/tests/Settings.test.js
+++ b/tests/Settings.test.js
@@ -15,7 +15,7 @@ jest.mock('electron', () => ({
const Settings = require('../modules/persistence/Settings');
const env = {version: '2.0.0-test1', app: {getPath: jest.fn().mockReturnValue('test/path')}};
-const defaultSettingsInstance = new Settings({settings: 'tests/test-settings.json'}, env, 'none', 'none', 'test/path', '', '', true, false, true, 'spoof', false, false, true, '%(title).200s-(%(height)sp%(fps).0d).%(ext)s', '%(title).200s-(%(height)sp%(fps).0d).%(ext)s', 'click', '49', 8, 6, 2, true, 'video', true, 'C:\\Users\\user\\cookies.txt', false, '', '', 'https://sponsor.ajay.app', true, false, false, false, false, true, 'dark');
+const defaultSettingsInstance = new Settings({settings: 'tests/test-settings.json'}, env, 'none', 'none', 'test/path', '', '', true, false, true, 'spoof', false, false, true, '%(title).200s-(%(height)sp%(fps).0d).%(ext)s', '%(title).200s-(%(height)sp%(fps).0d).%(ext)s', 'click', '49', 8, 6, 2, true, 'video', true, 'C:\\Users\\user\\cookies.txt', false, '', '', 'https://sponsor.ajay.app', true, false, false, false, false, false, true, 'dark');
const defaultSettings = {
outputFormat: 'none',
audioOutputFormat: 'none',
@@ -49,6 +49,7 @@ const defaultSettings = {
downloadThumbnail: false,
keepUnmerged: false,
avoidFailingToSaveDuplicateFileName: false,
+ allowUnsafeFileExtensions: false,
calculateTotalSize: true,
theme: 'dark',
version: '2.0.0-test1',
diff --git a/tests/Utils.test.js b/tests/Utils.test.js
index 868d704e..2893e451 100644
--- a/tests/Utils.test.js
+++ b/tests/Utils.test.js
@@ -98,13 +98,13 @@ describe('extractPlaylistUrls', () => {
expect(console.error).toBeCalledTimes(1);
});
it('returns playlist urls',() => {
- expect(Utils.extractPlaylistUrls({entries: [{url: "1"}, {url: "2"}]})).toContainEqual(["1", "2"])
+ expect(Utils.extractPlaylistUrls({entries: [{url: "1"}, {url: "2"}]})).toContainEqual([{headers: undefined, url: "1"}, {headers: undefined, url: "2"}])
});
it('returns already done (queried) urls', () => {
- expect(Utils.extractPlaylistUrls({entries: [{url: "1", formats: ["format"]}, {url: "2"}]})).toContainEqual([{url: "1", formats: ["format"]}]);
+ expect(Utils.extractPlaylistUrls({entries: [{url: "1", formats: ["format"]}, {url: "2"}]})).toContainEqual([{url: "1", headers: undefined, formats: ["format"]}]);
});
it('uses the webpage_url when url is null', () => {
- expect(Utils.extractPlaylistUrls({entries: [{webpage_url: "url"}]})).toContainEqual(["url"]);
+ expect(Utils.extractPlaylistUrls({entries: [{webpage_url: "url"}]})).toContainEqual([{url:"url", "headers": undefined}]);
});
});
diff --git a/tests/Video.test.js b/tests/Video.test.js
index 33357d86..ce2d616c 100644
--- a/tests/Video.test.js
+++ b/tests/Video.test.js
@@ -68,7 +68,7 @@ function instanceBuilder(type) {
nameFormatMode: "%(title).200s-(%(height)sp%(fps).0d).%(ext)s"
}
};
- const video = new Video("http://test.url", type, env);
+ const video = new Video("http://test.url", [], type, env);
let formats = [];
const displayNames = ["144p12", "1080p", "480p", "480p30", "480p29", "1080p60"];
for (const name of displayNames) {
diff --git a/tests/test-settings.json b/tests/test-settings.json
index 95899223..d94b4b89 100644
--- a/tests/test-settings.json
+++ b/tests/test-settings.json
@@ -32,6 +32,7 @@
"downloadThumbnail": false,
"keepUnmerged": false,
"avoidFailingToSaveDuplicateFileName": false,
+ "allowUnsafeFileExtensions": false,
"calculateTotalSize": true,
"theme": "dark",
"version": "2.0.0-test1"
# Open Video Downloader (youtube-dl-gui)
-[](https://github.com/StefanLobbenmeier/youtube-dl-gui/releases/latest)
+[](https://github.com/mp3butcher/youtube-dl-gui/releases/latest)
[](https://github.com/StefanLobbenmeier/youtube-dl-gui/blob/master/LICENSE)
-[](https://app.codecov.io/gh/StefanLobbenmeier/youtube-dl-gui)
-[](https://github.com/StefanLobbenmeier/youtube-dl-gui/releases/latest)
-[](https://github.com/StefanLobbenmeier/youtube-dl-gui/actions)
+[](https://app.codecov.io/gh/mp3butcher/youtube-dl-gui)
+[](https://github.com/mp3butcher/youtube-dl-gui/releases/latest)
+[](https://github.com/mp3butcher/youtube-dl-gui/actions)
-https://github.com/StefanLobbenmeier/youtube-dl-gui
A cross-platform GUI for youtube-dl made in Electron and node.js
+based on https://github.com/StefanLobbenmeier/youtube-dl-gui but adding a traffic scanning feature (using mitmproxy)
+
+- I] When scanner activated the first time don't forget to install mitmproxy authority certificate (cf image)
+
+- II] I recommend to use browser proxy switcher extension such as [this extension](https://chromewebstore.google.com/detail/proxy-switcher/iejkjpdckomcjdhmkemlfdapjodcpgih?hl=fr) to easily enable/disable proxy usage while browsing (set it up proxy=localhost and port=15930)
+
+- III] Browse to a streaming content and let the scanner capture the video
### Features:
- Download from all kind of platforms: YouTube, vimeo, twitter & many more
@@ -20,11 +26,12 @@ A cross-platform GUI for youtube-dl made in Electron and node.js
- Multithreaded, up to 32 videos can be downloaded synchronously
- Shows how much size the download will use up on your system
- The app automatically keeps ytdl up-to-date
+- The app include a network scanner through a binding with mitmproxy
Be sure to check out [a demo gif of the application](#Demo-gif)!
## How to use
-1. Download the [applicable installer or executable](https://github.com/StefanLobbenmeier/youtube-dl-gui/releases/latest) for your system.
+1. Download the [applicable installer or executable](https://github.com/mp3butcher/youtube-dl-gui/releases/latest) for your system.
2. If you are on windows, make sure that the [Microsoft Visual C++ 2010 Redistributable Package (x86)](https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe) is installed.
3. Paste a link into the box up top.
4. Wait for the app to gather all required metadata.
@@ -44,7 +51,7 @@ Please see if the answer is in the [frequently answered questions](https://githu
Still haven't found your answer? [Open up an issue](https://github.com/StefanLobbenmeier/youtube-dl-gui/issues), and describe the problem you're facing.
## Building from source
-First, clone the repository using `git clone https://github.com/StefanLobbenmeier/youtube-dl-gui.git`.
+First, clone the repository using `git clone https://github.com/mp3butcher/youtube-dl-gui.git`.
Then navigate to the directory and install the npm modules by executing: `npm install`.
diff --git a/binaries/send_traffic_to_videodownloader.py b/binaries/send_traffic_to_videodownloader.py
new file mode 100755
index 00000000..dc03932d
--- /dev/null
+++ b/binaries/send_traffic_to_videodownloader.py
@@ -0,0 +1,115 @@
+import asyncio
+import base64
+import socket, threading
+import queue
+import time
+
+from mitmproxy import ctx
+from mitmproxy import http
+LISTENING_PORT = 12000
+# Queue to hold shared data
+data_queue = queue.Queue(maxsize=15)
+
+def startserver(selfs):
+ data=''
+ try:
+ while data=='':
+ try:
+ data=selfs.socket_connection.recv(1)
+ print(data)
+ selfs.socket_instance.close();
+ selfs.socket_connection.close();
+ ctx.master.shutdown()
+ except:
+ pass
+ if data_queue.qsize()>0:
+ msg = data_queue.get()
+ if msg is not None:
+ selfs.socket_connection.send(msg.encode())
+ #else:
+ # if selfs.socket_connection.recv(data)>0:selfs.socket_connection .send('testconnection'.encode()) #dirty connection test require to exit properly
+ #time.sleep(1)
+ except:
+ selfs.socket_instance.close();
+ selfs.socket_connection.close();
+ ctx.master.shutdown()
+ selfs.socket_instance.close();
+ selfs.socket_connection.close();
+ ctx.master.shutdown()
+class TrafficLogger:
+ def __init__(self):
+
+ # Create server and specifying that it can only handle 4 connections by time!
+ self.socket_instance = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ #self.socket_instance.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ self.socket_instance.bind(('127.0.0.1', LISTENING_PORT))
+ self.socket_instance.listen(1) #limit to one client
+ print('Server running!')
+ # Accept client connection
+
+
+ self.socket_connection, self.address = self.socket_instance.accept()
+ self.socket_connection.setblocking(False)
+ print('new listener!')
+
+ # Create a thread
+ thread = threading.Thread(target=startserver, args=([self]))
+
+ # Start the thread
+ thread.start()
+
+ def __del__(self):
+ print("Object destroyed")
+ #self.socket_instance.close();
+
+
+ def request(self, flow):
+ self.checkflow(flow, False)
+ def response(self, flow):
+ self.checkflow(flow, True)
+ return
+
+ def checkflow(self,flow,isrep):
+
+ #print("FOR: " + flow.request.url)
+ url='"'+flow.request.method + " " + flow.request.url+ " " + flow.request.http_version+'"'
+ url='"' + flow.request.url+ '"'
+
+
+
+ headers="["
+ #print("-"*50 + "request headers:")
+ for k, v in flow.request.headers.items():
+ #print("%-20s: %s" % (k.upper(), v))
+ if k!='If-Range':
+ headers=headers+'{"k":"'+k+'"'+',"v":'+'"'+v.replace("\"","\\\"")+'"'+'},'
+ headers=headers[:-1]+"]"
+ rheaders="[]"
+ #print("-"*50 + "response headers:")
+ if isrep:
+ rheaders="["
+ for k, v in flow.response.headers.items():
+ #print("%-20s: %s" % (k.upper(), v))
+ #print("-"*50 + "request headers:")
+ rheaders=rheaders+'{"k":"'+k+'"'+',"v":'+'"'+v.replace("\"","\\\"")+'"'+'},'
+ rheaders=rheaders[:-1]+"]"
+ #don't send large response
+ data=""
+ if isrep:
+ if len(flow.response.text)<50000000:
+ #print(flow.response.text);
+
+ try:
+ #data=flow.response.text.replace("\"","\\\"") #.encode("utf-8")# str(flow.response.text, encoding='utf-8') #flow.response.text.decode("utf-8");
+ data=base64.b64encode(flow.response.content).decode("ascii")
+ except:
+ data=""
+ msg='{"url":'+url+',"headers":'+headers+',"rheaders":'+rheaders+',"response":"'+data+'"}'
+ #self.socket_instance.send(len(msg))
+ data_queue.put(msg)
+ #if self.socket_connection.send(msg.encode()) ==0:
+ # self.socket_connection.close();
+ # self.socket_instance.close();
+
+
+addons = [TrafficLogger()]
diff --git a/main.js b/main.js
index b999f435..e78c4b41 100644
--- a/main.js
+++ b/main.js
@@ -1,6 +1,7 @@
const { app, BrowserWindow, ipcMain, dialog, Menu, shell, clipboard } = require('electron');
const Environment = require('./modules/Environment');
const path = require('path');
+const net = require('net');
const QueryManager = require("./modules/QueryManager");
const ErrorHandler = require("./modules/exceptions/ErrorHandler");
const BinaryUpdater = require("./modules/BinaryUpdater");
@@ -9,6 +10,7 @@ const TaskList = require("./modules/persistence/TaskList");
const DoneAction = require("./modules/DoneAction");
const ClipboardWatcher = require("./modules/ClipboardWatcher");
const FfmpegUpdater = require('./modules/FfmpegUpdater');
+const MitmProxyUpdater = require('./modules/MitmProxyUpdater');
let win
let env
@@ -16,14 +18,17 @@ let queryManager
let clipboardWatcher
let taskList
let appStarting = true;
+let scannerIsOn = false;
+let mitmwebprocess;
+let mitmproxyclient;
function sendLogToRenderer(log, isErr) {
- if(win == null) return;
- win.webContents.send("log", {log: log, isErr: isErr});
+ if (win == null) return;
+ win.webContents.send("log", { log: log, isErr: isErr });
}
function startCriticalHandlers(env) {
- env.win = win;
+ env.win = win;
win.on('maximize', () => {
win.webContents.send("maximized", true)
@@ -45,28 +50,31 @@ function startCriticalHandlers(env) {
taskList = new TaskList(env.paths, queryManager)
- if(env.settings.updateBinary) {
+ if (env.settings.updateBinary) {
const binaryUpdater = new BinaryUpdater(env.paths, win);
const ffmpegUpdater = new FfmpegUpdater(env.paths, win);
- win.webContents.send("binaryLock", {lock: true, placeholder: `Checking for a new version of ffmpeg...`})
+ const mitmproxyUpdater = new MitmProxyUpdater(env.paths, win);
+ win.webContents.send("binaryLock", { lock: true, placeholder: `Checking for a new version of ffmpeg...` })
ffmpegUpdater.checkUpdate().finally(() => {
- win.webContents.send("binaryLock", {lock: true, placeholder: `Checking for a new version of yt-dlp...`})
+ win.webContents.send("binaryLock", { lock: true, placeholder: `Checking for a new version of yt-dlp...` })
binaryUpdater.checkUpdate().finally(() => {
- win.webContents.send("binaryLock", {lock: false});
- taskList.load();
- clipboardWatcher.startPolling();
+ win.webContents.send("binaryLock", { lock: false });
+ mitmproxyUpdater.checkUpdate().finally(() => {
+ taskList.load();
+ clipboardWatcher.startPolling();
+ });
});
});
- } else if(env.settings.taskList) {
+ } else if (env.settings.taskList) {
taskList.load();
}
//Send the saved download type to the renderer
- win.webContents.send("videoAction", {action: "setDownloadType", type: env.settings.downloadType});
+ win.webContents.send("videoAction", { action: "setDownloadType", type: env.settings.downloadType });
env.errorHandler = new ErrorHandler(win, queryManager, env);
- if(appStarting) {
+ if (appStarting) {
appStarting = false;
//Restore the videos from last session
@@ -98,8 +106,8 @@ function startCriticalHandlers(env) {
ipcMain.handle('iconProgress', (event, args) => {
win.setProgressBar(args);
- if(args === 1) {
- if(process.platform === "darwin") app.dock.bounce();
+ if (args === 1) {
+ if (process.platform === "darwin") app.dock.bounce();
else win.flashFrame(true);
win.setProgressBar(-1);
}
@@ -110,14 +118,14 @@ function startCriticalHandlers(env) {
case "get":
return env.settings.serialize();
case "save":
- env.settings.update(args.settings);
+ env.settings.update(args.setting);
break;
}
})
let appUpdater = new AppUpdater(env, win);
env.appUpdater = appUpdater;
- if(!env.paths.appPath.includes("\\AppData\\Local\\Temp\\") && !env.paths.appPath.includes("WindowsApps")) {
+ if (!env.paths.appPath.includes("\\AppData\\Local\\Temp\\") && !env.paths.appPath.includes("WindowsApps")) {
//Don't check the app when it is in portable mode
appUpdater.checkUpdate();
}
@@ -148,11 +156,11 @@ function startCriticalHandlers(env) {
break;
case "download":
if (args.downloadType === "all") queryManager.downloadAllVideos(args)
- else if(args.downloadType === "unified") queryManager.downloadUnifiedPlaylist(args);
- else if(args.downloadType === "single") queryManager.downloadVideo(args);
+ else if (args.downloadType === "unified") queryManager.downloadUnifiedPlaylist(args);
+ else if (args.downloadType === "single") queryManager.downloadVideo(args);
break;
case "entry":
- queryManager.manage(args.url);
+ queryManager.manage(args.url, args.headers ? args.headers : []);
break;
case "info":
queryManager.showInfo(args.identifier);
@@ -180,6 +188,7 @@ function startCriticalHandlers(env) {
//Create the window for the renderer process
function createWindow(env) {
+
win = new BrowserWindow({
show: false,
minWidth: 840,
@@ -199,22 +208,31 @@ function createWindow(env) {
contextIsolation: true
}
})
- if(process.argv[2] === '--dev') {
+ if (process.argv[2] === '--dev') {
win.webContents.openDevTools()
}
win.loadFile(path.join(__dirname, "renderer/renderer.html"))
win.on('closed', () => {
+ if (scannerIsOn) {
+ try {
+ mitmproxyclient.write('exit');
+ } catch (e) {
+ console.error(e);
+ }
+ }
win = null
- })
- win.once('focus', () => win.flashFrame(false))
+ });
+ win.once('focus', () => {
+ win.flashFrame(false)
+ });
win.webContents.on('did-finish-load', () => {
win.show();
- startCriticalHandlers(env)
+ startCriticalHandlers(env);
});
}
app.on('ready', async () => {
- app.setAppUserModelId("com.jelleglebbeek.youtube-dl-gui");
+ app.setAppUserModelId("com.mp3butcher.youtube-dl-gui");
env = new Environment(app);
await env.initialize();
createWindow(env);
@@ -283,6 +301,133 @@ ipcMain.handle("platform", () => {
return process.platform;
})
+function scan(msg) {
+ if (scannerIsOn) {
+ let data;
+ try {
+ data = JSON.parse(msg);
+ } catch (e) {
+ return console.error(e); //Error in the above string (in this case, yes)!
+ }
+ //Basic scanner
+ let sizeok = false; //Check if range is not prohibitively small
+ let contentype = "";
+ let contentlength = 0;
+ let headerstr = '';
+ data.headers.forEach(h => {
+ headerstr = headerstr + h.k + ": " + h.v + '$';
+ if (h.k.toLowerCase() == "range") {
+ const regexprange = /bytes=(\d+)-(\d+)?\/?(\d+)?/g;
+ const ranges = [...h.v.matchAll(regexprange)];
+ console.log(ranges[0]);
+ if (typeof (ranges[1]) == "undefined") contentlength = 20000000;
+ else contentlength = parseInt(ranges[1], 10) - parseInt(ranges[0], 10);
+ if (contentlength > 1000000) sizeok = true;
+ }
+ });
+ console.log(" scan url " + data.url + " headers:" + headerstr);
+
+ data.rheaders.forEach(h => {
+ if (h.k.toLowerCase() == "content-type") contentype = h.v;
+ if (h.k.toLowerCase() == "content-length") {
+ contentlength = parseInt(h.v, 10);
+ }
+ if (h.k.toLowerCase() == "content-range") {
+
+ const regexprange = /bytes (\d+)-(\d+)?\/?(\d+)?/g;
+ const ranges = [...h.v.matchAll(regexprange)];
+ if (typeof (ranges[1]) == "undefined") contentlength = 20000000;
+ else contentlength = parseInt(ranges[1], 10) - parseInt(ranges[0], 10);
+ if (contentlength > 1000000) sizeok = true;
+ }
+ });
+ let toscandeeply = false;
+ //Large Content-type video
+ if (contentype.startsWith('video') && contentlength > 1000000) {
+ toscandeeply = true;
+ console.warn(" [x] contentype video!!!!!!!!!!!" + data);
+ }
+ //Large Content-Range
+ if (sizeok || contentlength > 100000) {
+ toscandeeply = true;
+ }
+ let res = atob(data.response)
+ console.log(" [x] response " + res.substring(0, 100));
+ if (res.length > 1) {
+ if (res[0] == "#") { //HLS?
+ console.log(res);
+ toscandeeply = true;
+ }
+ }
+ if (toscandeeply) queryManager.manage(data.url, data.headers);
+ }
+}
+
+//Recursive connection try for non blocking connection
+function createConnection() {
+ //Connection to mitmproxy on tcp
+ mitmproxyclient = new net.Socket();
+ mitmproxyclient.connect(12000, '127.0.0.1', () => {
+ console.log('Connected to mitmproxy');
+ });
+ mitmproxyclient.on('data', scan);
+ mitmproxyclient.on('error', () => {
+ if (scannerIsOn) createConnection(); //Retry to connect until it's accepted
+ });
+ mitmproxyclient.on('close', () => {
+ console.log('Connection to mitmproxy closed');
+ });
+}
+
+//Set Traffic Scanner On/off
+ipcMain.handle("setScannerEnabled", (event, args) => {
+ console.log("set Network Scanning =" + args);
+
+ scannerIsOn = args.value;
+ const spawn = require('child_process').spawn;
+
+ console.log(env.settings.paths.mitmproxy);
+ if (scannerIsOn) {
+
+ if (mitmproxyclient) mitmproxyclient.destroy();
+ console.log(__dirname);
+ let script = path.join(path.dirname(__dirname), 'binaries', 'send_traffic_to_videodownloader.py');
+ console.log(script);
+ mitmwebprocess = spawn(
+ path.join(env.settings.paths.mitmproxy, 'mitmweb'),
+ ['-q', '--anticache', '--anticomp', '-s', script, '--listen-port', '15930']
+ );
+ mitmwebprocess.on('message', (message) => {
+ if (message.type === 'error') {
+ console.error('Child process error:', message.error);
+ }
+ });
+ mitmwebprocess.on('uncaughtException', (err) => {
+ console.error('SubProcessProxy uncaughtException', err);
+ });
+ mitmwebprocess.stdout.on('data', (data) => {
+ console.log(`stdout: ${data}`);
+ });
+ mitmwebprocess.stderr.on('data', (data) => {
+ console.error(`stderr: ${data}`);
+ });
+ mitmwebprocess.on('close', (code) => {
+ console.log(`child process exited with code ${code}`);
+ });
+ let msg = 'Proxy running on localhost on port 15930: Configure proxy in your browser to scan network';
+ dialog.showMessageBox({ message: msg });
+ createConnection();
+
+ } else {
+ try {
+ //Sending something will terminate both connection and server process
+ mitmproxyclient.write('exit');
+ mitmproxyclient = null;
+ } catch (e) {
+ console.log(e);
+ }
+ }
+})
//Return the available actions to the renderer process
ipcMain.handle('getDoneActions', () => {
@@ -297,12 +442,12 @@ ipcMain.handle("theme", () => {
//Handle titlebar click events from the renderer process
ipcMain.handle('titlebarClick', (event, arg) => {
- if(arg === 'close') {
+ if (arg === 'close') {
win.close()
- } else if(arg === "minimize") {
+ } else if (arg === "minimize") {
win.minimize()
- } else if(arg === "maximize") {
- if(win.isMaximized()) win.unmaximize();
+ } else if (arg === "maximize") {
+ if (win.isMaximized()) win.unmaximize();
else win.maximize();
}
})
@@ -310,14 +455,14 @@ ipcMain.handle('titlebarClick', (event, arg) => {
//Show a dialog to select a folder, and return the selected value.
ipcMain.handle('downloadFolder', async () => {
await dialog.showOpenDialog(win, {
- defaultPath: env.settings.downloadPath,
+ defaultPath: env.settings.downloadPath,
buttonLabel: "Set download location",
properties: [
'openDirectory',
'createDirectory'
]
}).then(result => {
- if(result.filePaths[0] != null) {
+ if (result.filePaths[0] != null) {
env.settings.downloadPath = result.filePaths[0];
env.settings.save();
}
@@ -325,12 +470,12 @@ ipcMain.handle('downloadFolder', async () => {
});
//Show a dialog to select a file, and return the selected value.
-ipcMain.handle('cookieFile', async (event,clear) => {
- if(clear === true) {
+ipcMain.handle('cookieFile', async (event, clear) => {
+ if (clear === true) {
env.settings.cookiePath = null;
env.settings.save();
return;
- } else if(clear === "get") {
+ } else if (clear === "get") {
return env.settings.cookiePath;
}
let result = await dialog.showOpenDialog(win, {
@@ -345,7 +490,7 @@ ipcMain.handle('cookieFile', async (event,clear) => {
{ name: "All Files", extensions: ["*"] },
],
});
- if(result.filePaths[0] != null) {
+ if (result.filePaths[0] != null) {
env.settings.cookiePath = result.filePaths[0];
env.settings.save();
}
diff --git a/modules/AppUpdater.js b/modules/AppUpdater.js
index e0a35554..fdbb5800 100644
--- a/modules/AppUpdater.js
+++ b/modules/AppUpdater.js
@@ -33,7 +33,7 @@ class AppUpdater {
this.win.webContents.send('toast', {
type: "update",
title: "An update has been found",
- body: `Update ${result.updateInfo.releaseName} is out now!