Skip to content

Commit

Permalink
Merge branch 'dev'
Browse files Browse the repository at this point in the history
  • Loading branch information
@nleush
nleush committed Nov 15, 2024
2 parents b2ca2f7 + a9607f2 commit 14ed765
Show file tree
Hide file tree
Showing 11 changed files with 764 additions and 813 deletions.
2 changes: 1 addition & 1 deletion config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@
"iframely"
],

KNOWN_VIDEO_SOURCES: /(youtube|youtu|youtube\-nocookie|vimeo|dailymotion|theplatform|jwplatform|jwplayer|cnevids|newsinc|podbean|simplecast|libsyn|wistia|podiant|art19|kaltura|mtvnservices|brightcove|bcove|soundcloud|giphy|viddler|flowplayer|vidible|bandzoogle|podigee|smugmug|facebook|vid|ultimedia|mixcloud|vidyard|youplay|streamable|captivate|mdstrm|mediadelivery|hearstapps|rudo)\.\w+\//i,
KNOWN_VIDEO_SOURCES: /(youtube|youtu|youtube\-nocookie|vimeo|dailymotion|theplatform|jwplatform|jwplayer|cnevids|newsinc|podbean|simplecast|libsyn|wistia|podiant|art19|kaltura|mtvnservices|brightcove|bcove|soundcloud|giphy|viddler|flowplayer|vidible|bandzoogle|podigee|smugmug|facebook|vid|ultimedia|mixcloud|vidyard|youplay|streamable|captivate|mdstrm|mediadelivery|hearstapps|rudo|acast|screen9)\.\w+\//i,

OEMBED_RELS_PRIORITY: ["app", "player", "survey", "image", "reader"],
OEMBED_RELS_MEDIA_PRIORITY: ["player", "survey", "image", "reader", "app"],
Expand Down
11 changes: 3 additions & 8 deletions docs/WRITE-A-PLUGIN.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,14 +190,9 @@ Example:
},
{
geturls: function(cb) {
var request = require('request');
request({
url: 'https://api.domain.com/items',
json: true
}, function(error, body, data) {
if (error) {
return cb(error);
}
got('https://api.domain.com/items', { responseType: 'json' })
.then(response => {
const data = response.body;
if (!data || !data.urls) {
return cb('No urls in API data');
}
Expand Down
51 changes: 25 additions & 26 deletions lib/whitelist.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import * as _ from 'underscore';
import * as utils from './utils.js';
import log from '../logging.js';
import request from 'request';
import got from 'got';

import { fileURLToPath } from 'url';
import { dirname } from 'path';
Expand Down Expand Up @@ -356,42 +356,41 @@
log("Loading domains list from " + CONFIG.WHITELIST_URL);

var options = {
uri: CONFIG.WHITELIST_URL,
json: true,
qs: {
url: CONFIG.WHITELIST_URL,
responseType: 'json',
searchParams: {
domain: CONFIG.baseAppUrl && CONFIG.baseAppUrl.replace(/.+\/\//, ''),
v: CONFIG.VERSION
},
headers: {
// Prevent caching.
'Cache-Control': 'no-cache'
},
// TODO: remove in helix-fetch
gzip: true
};

if (whitelistLastModified) {
options.headers['If-Modified-Since'] = whitelistLastModified;
}

request(utils.prepareRequestOptions(options, {skipPlugins: true}), function(error, r, newWhitelist) {

if (error) {
console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ' : ' + error);
} else if (r.statusCode === 500) {
console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ' : ' + newWhitelist);
} else if (r.statusCode === 304) {
log('Whitelist respond: 304 (not modified)');
} else if (!newWhitelist || typeof newWhitelist === 'string') {
console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ' : incorrect data: ' + newWhitelist);
} else {

whitelistLastModified = r.headers['last-modified'];

applyParsedWhitelist(newWhitelist);
}
got(options)
.then(response => {
const newWhitelist = response.body;

if (response.statusCode === 500) {
console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ' : ' + newWhitelist);
} else if (response.statusCode === 304) {
log('Whitelist respond: 304 (not modified)');
} else if (!newWhitelist || typeof newWhitelist === 'string') {
console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ' : incorrect data: ' + newWhitelist);
} else {
whitelistLastModified = response.headers['last-modified'];
applyParsedWhitelist(newWhitelist);
}

setTimeout(loadWhitelistUrl, CONFIG.WHITELIST_URL_RELOAD_PERIOD);
});
setTimeout(loadWhitelistUrl, CONFIG.WHITELIST_URL_RELOAD_PERIOD);
})
.catch(error => {
console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ':', error.message);
});
}
}
}
95 changes: 46 additions & 49 deletions modules/tests-ui/utils.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as _ from 'underscore';
import FeedParser from 'feedparser';
import request from 'request';
import got from 'got';
import * as async from 'async';
import * as url from 'url';
import { PageTestLog, TestUrlsSet, PluginTest } from './models.js';
Expand Down Expand Up @@ -50,35 +50,32 @@ export function sendQANotification(logEntry, data) {
message += " - " + errors;
}

request({
uri: CONFIG.SLACK_WEBHOOK_FOR_QA,
method: 'POST',
json: true,
body: {
"parse": "none",
"channel": CONFIG.SLACK_CHANNEL_FOR_QA,
"username": SLACK_USERNAME,
"text": previewMessage,
"blocks": [
got.post(CONFIG.SLACK_WEBHOOK_FOR_QA, {
json: {
parse: "none",
channel: CONFIG.SLACK_CHANNEL_FOR_QA,
username: SLACK_USERNAME,
text: previewMessage,
blocks: [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": message // Message: [domain.com] Failed - errors.
type: "section",
text: {
type: "mrkdwn",
text: message
}
}
],
"attachments": [
attachments: [
{
"blocks": [{
"type": "section",
"text": {
"type": "mrkdwn",
"verbatim": true,
"text": "`<" + (CONFIG.QA_BASE_URL || baseAppUrl) + "/debug?uri=" + encodeURIComponent(logEntry.url) + "|debug>` " + logEntry.url.replace(/^https?:\/\//, '') // Debug link.
blocks: [{
type: "section",
text: {
type: "mrkdwn",
verbatim: true,
text: "`<" + (CONFIG.QA_BASE_URL || baseAppUrl) + "/debug?uri=" + encodeURIComponent(logEntry.url) + "|debug>` " + logEntry.url.replace(/^https?:\/\//, '')
}
}],
"color": COLORS[data.color]
color: COLORS[data.color]
}
]
}
Expand Down Expand Up @@ -272,43 +269,43 @@ export function fetchFeedUrls(feedUrl, options, cb) {
cb(error, urls);
};

request({
uri: feedUrl,
agentOptions: {

got.stream(feedUrl, {
https: {
rejectUnauthorized: false
}
})
.pipe(new FeedParser({addmeta: false}))
.on('error', function(error) {
_cb(error);
})
.on('readable', function () {
var stream = this, item;
while (item = stream.read()) {
.pipe(new FeedParser({ addmeta: false }))
.on('error', function(error) {
_cb(error);
})
.on('readable', function () {
var stream = this, item;
while (item = stream.read()) {

if (urls.length < MAX_FEED_URLS) {
if (urls.length < MAX_FEED_URLS) {

var url = item.origlink || item.link;
var url = item.origlink || item.link;

if (options.getUrl) {
url = options.getUrl(url);
}
if (options.getUrl) {
url = options.getUrl(url);
}

if (!url) {
return;
}
if (!url) {
return;
}

urls.push(url);
urls.push(url);

if (MAX_FEED_URLS == urls.length) {
_cb();
}
if (MAX_FEED_URLS === urls.length) {
_cb();
}
}
})
.on('end', function() {
_cb();
});
}
})
.on('end', function() {
_cb();
});
};

export function fetchUrlsByPageOnFeed(pageWithFeed, otpions, cb) {
Expand Down
4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"ejs": "3.1.10",
"entities": "1.1.2",
"express": "^4.19.0",
"got": "^14.4.4",
"graceful-cluster": "^0.0.5",
"htmlparser2": "^7.2.0",
"iconv-lite": "^0.6.3",
Expand All @@ -44,7 +45,6 @@
"readabilitySAX": "1.6.1",
"redis": "^4.6.14",
"redis-clustr": "1.7.0",
"request": "^2.88.2",
"sax": "^1.2.4",
"send": "^0.18.0",
"underscore": "^1.13.6"
Expand All @@ -66,6 +66,6 @@
"custom_plugins": "NODE_ENV=test_with_custom_plugins PORT=8080 mocha --exit test/custom_plugins.js"
},
"engines": {
"node": ">=14.0"
"node": ">=18.0"
}
}
16 changes: 6 additions & 10 deletions plugins/domains/dailymotion.com/dailymotion.com.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import * as querystring from 'querystring';
import request from 'request';
import got from 'got';

export default {

Expand Down Expand Up @@ -53,21 +53,17 @@ export default {

tests: [{
getUrls: function(cb) {
request({
url: 'https://api.dailymotion.com/videos',
json: true
}, function(error, body, data) {
if (error) {
return cb(error);
}
got('https://api.dailymotion.com/videos', { responseType: 'json' })
.then(response => {
const data = response.body;
if (!data || !data.list) {
return cb('No videos list in API data');
}
cb(null, data.list.slice(0, 10).map(function(item) {
return 'https://www.dailymotion.com/video/' + item.id;
}));
});

})
.catch(error => cb(error));
}
}, {
skipMixins: ["video", "og-description", "og-image", "canonical"],
Expand Down
2 changes: 1 addition & 1 deletion plugins/domains/dailymotion.com/dailymotion.playlist.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export default {
} : null);
},

tests: [{
tests: [{skipMethods: ['getData']}, {
noFeeds: true
},
// "https://www.dailymotion.com/playlist/x6hynp",
Expand Down
2 changes: 1 addition & 1 deletion plugins/domains/twitch.tv/twitch-live-fallback.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
export default {

re: [
/^https?:\/\/(?:www\.|go\.)?twitch\.tv\/([a-zA-Z0-9_]+)(?:\?parent=.*)?$/i,
/^https?:\/\/(?:www\.|go\.)?twitch\.tv\/([a-zA-Z0-9_]+)(?:\?_?parent=.*)?$/i,
/^https?:\/\/(?:www\.|go\.)?twitch\.tv\/(?:[a-zA-Z0-9_]+\/)?videos?\/(\d+)/i,
],

Expand Down
2 changes: 1 addition & 1 deletion plugins/domains/twitch.tv/twitch.tv.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export default {
re: [
/^https?:\/\/(?:www\.|go\.)?twitch\.tv\/[a-zA-Z0-9_]+\/v\/(\d+)/i,
/^https?:\/\/(?:www\.|go\.)?twitch\.tv\/(?:[a-zA-Z0-9_]+\/)?videos?\/(\d+)/i,
/^https?:\/\/(?:www\.|go\.)?twitch\.tv\/([a-zA-Z0-9_]+)(?:\?parent=.*)?$/i,
/^https?:\/\/(?:www\.|go\.)?twitch\.tv\/([a-zA-Z0-9_]+)(?:\?_?parent=.*)?$/i,
/^https?:\/\/clips.twitch\.tv\/([^\?\/]+)(?:\?[^\?]+)?$/i,
/^https?:\/\/www\.twitch\.tv\/\w+\/clip\/([^\?\/]+)/i
],
Expand Down
2 changes: 1 addition & 1 deletion pnpm-lock.yaml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 14ed765

Please sign in to comment.