feat: Access to the various getInfo miniget requests. (#901)

* Add files via upload

* Add files via upload

* Update info.js

* Update info.js

* Add files via upload

* Update README.md

* Update info.js

* Update README.md

* Add files via upload

* Add files via upload

* rework callback

* linting

* Update index.d.ts

Co-authored-by: TimeForANinja <[email protected]>
Co-authored-by: TimeForANinja <[email protected]>

Author: gatecrasher777

README.md

@@ -69,6 +69,7 @@ Gets metainfo from a video. Includes additional formats, and ready to download d
 `options` can have the following
 
 * `requestOptions` - Anything to merge into the request options which [miniget](https://github.com/fent/node-miniget) is called with, such as `headers`.
+* `requestCallback` - Provide a callback function that receives miniget request stream objects used while fetching metainfo.
 * `lang` - The 2 character symbol of a language. Default is `en`.
 
 ### ytdl.downloadFromInfo(info, options)

lib/info.js

@@ -120,14 +120,14 @@ const isNotYetBroadcasted = player_response => {
 const getWatchHTMLURL = (id, options) => `${BASE_URL + id}&hl=${options.lang || 'en'}`;
 const getWatchHTMLPageBody = (id, options) => {
   const url = getWatchHTMLURL(id, options);
-  return exports.watchPageCache.getOrSet(url, () => miniget(url, options.requestOptions).text());
+  return exports.watchPageCache.getOrSet(url, () => utils.exposedMiniget(url, options).text());
 };
 
 
 const EMBED_URL = 'https://www.youtube.com/embed/';
 const getEmbedPageBody = (id, options) => {
   const embedUrl = `${EMBED_URL + id}?hl=${options.lang || 'en'}`;
-  return miniget(embedUrl, options.requestOptions).text();
+  return utils.exposedMiniget(embedUrl, options).text();
 };
 
 
@@ -288,7 +288,7 @@ const getWatchJSONPage = async(id, options) => {
   }
 
   const jsonUrl = getWatchJSONURL(id, options);
-  let body = await miniget(jsonUrl, reqOptions).text();
+  const body = await utils.exposedMiniget(jsonUrl, options, reqOptions).text();
   let parsedBody = parseJSON('watch.json', 'body', body);
   if (parsedBody.reload === 'now') {
     await setIdentityToken('browser', false);
@@ -330,7 +330,7 @@ const getVideoInfoPage = async(id, options) => {
   url.searchParams.set('ps', 'default');
   url.searchParams.set('gl', 'US');
   url.searchParams.set('hl', options.lang || 'en');
-  let body = await miniget(url.toString(), options.requestOptions).text();
+  const body = await utils.exposedMiniget(url.toString(), options).text();
   let info = querystring.parse(body);
   info.player_response = findPlayerResponse('get_video_info', info);
   return info;
@@ -427,7 +427,7 @@ const getDashManifest = (url, options) => new Promise((resolve, reject) => {
     }
   };
   parser.onend = () => { resolve(formats); };
-  const req = miniget(new URL(url, BASE_URL).toString(), options.requestOptions);
+  const req = utils.exposedMiniget(new URL(url, BASE_URL).toString(), options);
   req.setEncoding('utf8');
   req.on('error', reject);
   req.on('data', chunk => { parser.write(chunk); });
@@ -444,7 +444,7 @@ const getDashManifest = (url, options) => new Promise((resolve, reject) => {
  */
 const getM3U8 = async(url, options) => {
   url = new URL(url, BASE_URL);
-  let body = await miniget(url.toString(), options.requestOptions).text();
+  const body = await utils.exposedMiniget(url.toString(), options).text();
   let formats = {};
   body
     .split('\n')

lib/sig.js

@@ -1,7 +1,7 @@
 const { URL } = require('url');
-const miniget = require('miniget');
 const querystring = require('querystring');
 const Cache = require('./cache');
+const utils = require('./utils');
 
 
 // A shared cache to keep track of html5player.js tokens.
@@ -16,7 +16,7 @@ exports.cache = new Cache();
  * @returns {Promise<Array.<string>>}
  */
 exports.getTokens = (html5playerfile, options) => exports.cache.getOrSet(html5playerfile, async() => {
-  let body = await miniget(html5playerfile, options.requestOptions).text();
+  const body = await utils.exposedMiniget(html5playerfile, options).text();
   const tokens = exports.extractActions(body);
   if (!tokens || !tokens.length) {
     throw Error('Could not extract signature deciphering actions');

lib/utils.js

@@ -126,6 +126,19 @@ exports.playError = (player_response, statuses, ErrorType = Error) => {
   return null;
 };
 
+/**
+ * Does a miniget request and calls options.requestCallback if present
+ *
+ * @param {string} url the request url
+ * @param {Object} options an object with optional requestOptions and requestCallback parameters
+ * @param {Object} requestOptionsOverwrite overwrite of options.requestOptions
+ * @returns {miniget.Stream}
+ */
+exports.exposedMiniget = (url, options = {}, requestOptionsOverwrite) => {
+  const req = miniget(url, requestOptionsOverwrite || options.requestOptions);
+  if (typeof options.requestCallback === 'function') options.requestCallback(req);
+  return req;
+};
 
 /**
  * Temporary helper to help deprecating a few properties.

test/utils-test.js

@@ -180,3 +180,54 @@ describe('utils.checkForUpdates', () => {
     });
   });
 });
+
+describe('utils.exposedMiniget', () => {
+  it('does not error with undefined requestOptionsOverwrite', async() => {
+    const scope = nock('https://test.com').get('/').reply(200, 'nice');
+    const req = utils.exposedMiniget('https://test.com/', {});
+    await req.text();
+    scope.done();
+  });
+
+  it('does not error without options', async() => {
+    const scope = nock('https://test.com').get('/').reply(200, 'nice');
+    const req = utils.exposedMiniget('https://test.com/');
+    await req.text();
+    scope.done();
+  });
+
+  it('does not error without options', async() => {
+    const scope = nock('https://test.com').get('/').reply(200, 'nice');
+    const req = utils.exposedMiniget('https://test.com/');
+    assert.equal(await req.text(), 'nice');
+    scope.done();
+  });
+
+  it('calls a provided callback with the req object', async() => {
+    const scope = nock('https://test.com').get('/').reply(200, 'nice');
+    let cbReq;
+    const requestCallback = r => cbReq = r;
+    const req = utils.exposedMiniget('https://test.com/', { requestCallback });
+    await req.text();
+    assert.equal(cbReq, req);
+    scope.done();
+  });
+
+  it('it uses requestOptions', async() => {
+    const scope = nock('https://test.com', { reqheaders: { auth: 'a' } }).get('/').reply(200, 'nice');
+    const req = utils.exposedMiniget('https://test.com/', { requestOptions: { headers: { auth: 'a' } } });
+    await req.text();
+    scope.done();
+  });
+
+  it('it prefers requestOptionsOverwrite over requestOptions', async() => {
+    const scope = nock('https://test.com', { reqheaders: { auth: 'b' } }).get('/').reply(200, 'nice');
+    const req = utils.exposedMiniget(
+      'https://test.com/',
+      { requestOptions: { headers: { auth: 'a' } } },
+      { headers: { auth: 'b' } },
+    );
+    await req.text();
+    scope.done();
+  });
+});

typings/index.d.ts

@@ -7,6 +7,7 @@ declare module 'ytdl-core' {
 
     interface getInfoOptions {
       lang?: string;
+      requestCallback?: () => {};
       requestOptions?: {};
     }