From 9908116034f44123d3289989e9ed92b3c6a8708c Mon Sep 17 00:00:00 2001 From: Drewry Pope Date: Mon, 21 Sep 2020 06:52:58 -0500 Subject: [PATCH 1/3] feat(memes): implemented exploded words format, cleanup/improvements feat(memes): collapse exploded words feat(logging): bug report detailed and prelim fix(setup): create data/log dir if missing feat(cleanup): normalize lines feat(cleanup): space around user tweet --- lib/index.js | 134 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 15 deletions(-) diff --git a/lib/index.js b/lib/index.js index adf5b52..b79465c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -53,7 +53,7 @@ const // there are more const function variables to follow, // until near eof where there is the 'export' section { Autohook } = autohook, getNewPort = async () => { - // Will use any port from 49152 to 65535, otherwise fall back to a random sport; + // Will use any port from 49152 to 65535, otherwise fall back to a random port; return getPort({port: getPort.makeRange(49152, 65535),}); }, removeWebhooks = async () => {const _port = await getNewPort(); return new Autohook({port: _port, }).removeWebhooks();}, @@ -106,9 +106,10 @@ const // there are more const function variables to follow, .replaceAll('BOT_SCREEN_NAME', bot_screen_name || 'twetJs') .replaceAll('TWEET_ID_OR_TAG', tweet_tag_or_id || '??'); }, + trimPath = text => text.replaceAll(/[\\/]$/gu, ''), appendLog = ( log_dir, module_name, log_name, jsonObject, optional_date_time ) => { const filename = - log_dir + + trimPath(log_dir) + '/' + module_name + '.' + log_name + @@ -156,8 +157,40 @@ const // there are more const function variables to follow, .toLowerCase() .trim(); }, - getEmoji = text => { const regex = emojiRegex(); console.dir({text,regex,});return text.toString().match(regex); }, - removeEmoji = text => { const regex = emojiRegex(); console.dir({text,regex,});return text.toString().replace(regex, '').replace(/\s\n\s/gu, ' \n'); }, + getEmoji = text => { const regex = emojiRegex(); return text.toString().match(regex); }, + normalizeSpaces = text => text.replaceAll(/ +/gu, ' '), + normalizeNewlines = text => text.replaceAll(/\n\n+/gu, '\n\n').replaceAll(/\n\n+/gu, '\n\n'), + trimAllLines = text => text.split('\n').map(line => line.trim()).join('\n'), + collapseExtraSingleSpaces = text => { + let output = normalizeNewlines(normalizeSpaces(trimAllLines(text))); + const spaceMatches = output.match(/[ \n\t]\S( \S)+ \S \S[ \n\t]/gu); + // const newlineMatches = text.match(/[ \n\t]\S[ \t]*(\n\S[ \t]*)+\n/gu); + // const newlineMatches = output.match(/\n{0,1}(\s{0,1}[ ?!.]{0,2}\S*)+\n/gu); + const newlineMatches = output.match(/(^|\n)+(\S{1}[ ?!.]{0,2}(\s|$)+)+/gu); + const tabMatches = output.match(/[ \n\t]\S[ \n]*(\t\S[ \n]*)+[ \n\t]/gu); + let count = 0; + if (typeof spaceMatches !== 'undefined' && spaceMatches && spaceMatches.length > 0) { + spaceMatches.forEach(match => { + output = output.replaceAll(match, match.charAt(0).replaceAll(/\S/gu, '') + match.replaceAll(/\s/gu, '') + match.charAt(match.length - 1).replaceAll(/\S/gu, '')); + count += 1; + }); + } + if (typeof newlineMatches !== 'undefined' && newlineMatches && newlineMatches.length > 0) { + newlineMatches.forEach(match => { + const intermediate = match.split('\n\n'); + count += intermediate.length; + output = output.replaceAll(match, match.charAt(0).replaceAll(/\S/gu, '')+ intermediate.map(value => value.replaceAll(/\s/gu, '')).join('\n\n') + match.charAt(match.length - 1).replaceAll(/\S/gu, '')); + }); + } + if (typeof tabMatches !== 'undefined' && tabMatches && tabMatches.length > 0) { + tabMatches.forEach(match => { + output = output.replaceAll(match, match.charAt(0).replaceAll(/\S/gu, '') + match.replaceAll(/\s/gu, '') + match.charAt(match.length - 1).replaceAll(/\S/gu, '')); + count += 1; + }); + } + return { text: output, count, }; + }, + removeEmoji = text => { const regex = emojiRegex();return collapseExtraSingleSpaces(normalizeSpaces(text.toString().replaceAll(regex, ' ').replaceAll(/\s\n\s/gu, ' \n'))); }, emojiCount = text => (getEmoji(text.toString()) || []).length, isEmojiRidden = text => { const characterCount = (runes(text.toString()) || []).length; @@ -166,7 +199,7 @@ const // there are more const function variables to follow, const spaceCount = (text.toString().match(/ /gu) || []).length; const whitespaceCount = (text.toString().match(/[ ].*/gu) || []).length; console.dir({text, characterCount, _emojiCount, firstCharactersEmojiCount, spaceCount, whitespaceCount,}); - if (_emojiCount > spaceCount * 0.5 || _emojiCount > whitespaceCount || _emojiCount > characterCount * 0.25 || firstCharactersEmojiCount > 3) { + if (_emojiCount > 3 && (_emojiCount > spaceCount * 0.5 || _emojiCount > whitespaceCount || _emojiCount > characterCount * 0.25 || firstCharactersEmojiCount > 3)) { return _emojiCount; } else { return false; @@ -177,19 +210,25 @@ const // there are more const function variables to follow, cleanEmoji = text => { if (typeof text !== 'undefined' && text !== null && text !== '' ) { const emojiCount = isEmojiRidden(text); + const cleanedText = removeEmoji(text); if (emojiCount !== false ) { - return { 'text': removeEmoji(text), + return { 'text': cleanedText.text, 'alterations': '\n[ Removed: ' + emojiCount + ' of ' + runes(makeUnique(getEmoji(text).join(''))).length + - ' emoji, including ' + + ' Emoji, including ' + runes.substr(makeUnique(getEmoji(text).join('')), 0,runes(makeUnique(getEmoji(text).join(''))).length > 8 ? 4 : 3).split().join(' ') + - (runes(makeUnique(getEmoji(text).join(''))).length > 3 ? runes.substr(makeUnique(getEmoji(text).join('')), runes(makeUnique(getEmoji(text).join(''))).length > 4 ? -2 : -1, 0).split().join(' ') : '' )+' ] ' + + (runes(makeUnique(getEmoji(text).join(''))).length > 3 ? runes.substr(makeUnique(getEmoji(text).join('')), runes(makeUnique(getEmoji(text).join(''))).length > 4 ? -2 : -1, 0).split().join(' ') : '' )+' ]' + + (cleanedText.count > 0 ? '\n[ Collapsed: ' + cleanedText.count + ' Exploded Word' + (cleanedText.count === 1 ? '' : 's') + ' ]' : '') + '\n[ Removed: Uppercase, Fonts, Math, Diacritics ]', }; } else { - return { 'text': text, 'alterations': '\n[ Removed: Uppercase, Fonts, Math, Diacritics ]', }; + return { 'text': cleanedText.text, + 'alterations': + (cleanedText.count > -1 ? '\n[ Collapsed: ' + cleanedText.count + ' Exploded Word' + (cleanedText.count === 1 ? '' : 's') + ' ]' : '') + + '\n[ Removed: Uppercase, Fonts, Math, Diacritics ]', + }; } } else { - return { 'text': text, 'alterations': '\n[ Removed: Uppercase, Fonts, Math, Diacritics ]', }; + return { 'text': text, 'alterations': '\n[ Error: Received empty text? ]', }; } }, urlExpand = (text, urls, mediaUrls, skipUrls) => { @@ -220,6 +259,13 @@ const // there are more const function variables to follow, } return output; }, + leftTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/^\s+/gu,'')).join('\n'), + rightTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/\s+$/gu,'')).join('\n'), + replaceEmptyLines = (text, replace) => text.replaceAll(/\n\s*\n/gu, replace), + reduceEmptyLinesToSingle = text => replaceEmptyLines(text, '\n\n'), + removeEmptyLines = text => replaceEmptyLines(text, '\n'), + oneSpaceBeforeEachNewLine = text => rightTrimAllLines(text).replaceAll('\n', ' \n'), + fixAmp = text => text.replaceAll('&', '&'), start = async () => { try { // user_SCREEN_NAME is replaced with the user's name. @@ -233,7 +279,8 @@ const // there are more const function variables to follow, process.env.SIGNATURE || ' \nThanks, USER_SCREEN_NAME! \n\n@BOT_SCREEN_NAME #BOT_SCREEN_NAME \n[\'TWEET_ID_OR_TAG\']', bot_name = process.env.BOT_NAME || module_screen_name, - log_dir = process.env.LOG_DIR || './data/logs/', + data_dir = trimPath(process.env.DATA_DIR || './data'), + log_dir = data_dir + '/logs', params = { subdomain: 'api', version: '1.1', @@ -248,6 +295,9 @@ const // there are more const function variables to follow, }, port = await getNewPort(), webhook = new Autohook({port: port,}); + if (!fs.existsSync(log_dir)) { + fs.mkdirSync(log_dir); + } await removeWebhooks(); webhook.on('event', async event => { try { @@ -265,6 +315,35 @@ const // there are more const function variables to follow, tweet_create_events = event.tweet_create_events.map( async tweet => { const tweet_id = tweet.id_str; + if ( + getProperty( + getProperty( + tweet, + 'extended_tweet', + tweet + ), + 'full_text', + getProperty( + getProperty( + tweet, + 'extended_tweet', + tweet + ), + 'text', + tweet.text + ) + ) + .toString() + .toLowerCase() + .includes('bug') + ) { + appendLog( + log_dir, + bot_name, + key + '.tweetThreadReply.bugReport.prelim', + { value, event, } + ); + } if (tweet.user.id_str === bot_user_id) { appendLog( log_dir, @@ -438,6 +517,16 @@ const // there are more const function variables to follow, url_expanded_text ) ), + straightened_lines = + normalizeNewlines( + reduceEmptyLinesToSingle( + trimAllLines( + normalizeSpaces( + fixAmp(cleaned_text.text) + ) + ) + ) + ).trim(), reply_text = typeof in_reply_to_status_id === 'undefined' || @@ -446,12 +535,12 @@ const // there are more const function variables to follow, in_reply_to_status_id === '' ? '' : '@ ' + in_reply_to_screen_name + - ' SAYS: \n' + - cleaned_text.text + - ' \n' + + ' SAYS:\n\n' + + straightened_lines + + '\n\n' + 'END @ ' + in_reply_to_screen_name + - ' QUOTE \n' + + ' QUOTE\n' + cleaned_text.alterations ; appendLog( @@ -460,6 +549,21 @@ const // there are more const function variables to follow, key + '.tweetThreadReply.replyTweet', status_reply_to ); + if ( + user_tweet_text + .toString() + .toLowerCase() + .includes( + 'bug' + ) + ) { + appendLog( + log_dir, + bot_name, + key + '.tweetThreadReply.bugReport', + { status_reply_to, event, } + ); + } if ( user_tweet_text .toString() From 58c83826a2db77d5f34474b7bd24ba4d2aadf15e Mon Sep 17 00:00:00 2001 From: Drewry Pope Date: Mon, 21 Sep 2020 08:18:51 -0500 Subject: [PATCH 2/3] fix(memes): fix(emoji): fix emoji selection also refactored/cleaned up code --- lib/index.js | 724 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 464 insertions(+), 260 deletions(-) diff --git a/lib/index.js b/lib/index.js index b79465c..2d30ad4 100644 --- a/lib/index.js +++ b/lib/index.js @@ -24,25 +24,25 @@ import getPort from 'get-port'; // polyfill & 'custom method for standard object' sections if (!Object.prototype.hasOwnProperty.call(RegExp, 'escape')) { /* jshint ignore:start */ - RegExp.escape = function(string) { - // https:// developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#Escaping - // https:// github.com/benjamingr/RegExp.escape/issues/37 + RegExp.escape = function (string) { + // https:// developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#Escaping + // https:// github.com/benjamingr/RegExp.escape/issues/37 return string.replace(/[.*+\-?^${}()|[\]\\]/gu, '\\$&'); // $& means the whole matched string }; /* jshint ignore:end */ } if (!Object.prototype.hasOwnProperty.call(String, 'replaceAll')) { - String.prototype.replaceAll = function(find, replace) { // jshint ignore:line - // https:// developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replaceAll - // If you pass a RegExp to 'find', you _MUST_ include 'g' as a flag. - // TypeError: "replaceAll must be called with a global RegExp" not included, will silently cause significant errors. _MUST_ include 'g' as a flag for RegExp. - // String parameters to 'find' do not require special handling. - // Does not conform to "special replacement patterns" when "Specifying a string as a parameter" for replace - // Does not conform to "Specifying a function as a parameter" for replace + String.prototype.replaceAll = function (find, replace) { // jshint ignore:line + // https:// developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replaceAll + // If you pass a RegExp to 'find', you _MUST_ include 'g' as a flag. + // TypeError: "replaceAll must be called with a global RegExp" not included, will silently cause significant errors. _MUST_ include 'g' as a flag for RegExp. + // String parameters to 'find' do not require special handling. + // Does not conform to "special replacement patterns" when "Specifying a string as a parameter" for replace + // Does not conform to "Specifying a function as a parameter" for replace return this.replace( Object.prototype.toString.call(find) === '[object RegExp]' ? - find : - new RegExp(RegExp.escape(find), 'g'), + find + : new RegExp(RegExp.escape(find), 'g'), replace ); }; @@ -54,18 +54,24 @@ const // there are more const function variables to follow, { Autohook } = autohook, getNewPort = async () => { // Will use any port from 49152 to 65535, otherwise fall back to a random port; - return getPort({port: getPort.makeRange(49152, 65535),}); + return getPort({ port: getPort.makeRange(49152, 65535), }); }, - removeWebhooks = async () => {const _port = await getNewPort(); return new Autohook({port: _port, }).removeWebhooks();}, - processLog = reason => { + removeWebhooks = async () => { + const _port = await getNewPort(); + return new Autohook({ port: _port, }).removeWebhooks(); + }, + processLog = (reason) => { if (process.env.HEADLESS.toString().toUpperCase().trim() === 'TRUE') { console.log(JSON.stringify(reason)); } else { console.dir(reason, { depth: null, }); } }, - processError = reason => { - if ((process.env.HEADLESS || '').toString().toUpperCase().trim() !== 'TRUE') { + processError = (reason) => { + if ( + (process.env.HEADLESS || '').toString().toUpperCase().trim() !== + 'TRUE' + ) { processLog(reason); } console.error(JSON.stringify(reason)); @@ -81,11 +87,11 @@ const // there are more const function variables to follow, consumer_key: process.env.TWITTER_CONSUMER_KEY, consumer_secret: process.env.TWITTER_CONSUMER_SECRET, access_token_key: - process.env.TWITTER_ACCESS_TOKEN || - process.env.ACCESS_TOKEN, // oauth + process.env.TWITTER_ACCESS_TOKEN || + process.env.ACCESS_TOKEN, // oauth access_token_secret: - process.env.TWITTER_ACCESS_TOKEN_SECRET || - process.env.ACCESS_TOKEN_SECRET, // oauth + process.env.TWITTER_ACCESS_TOKEN_SECRET || + process.env.ACCESS_TOKEN_SECRET, // oauth }, [thread,].flat(Infinity), //.map(x => Bleetify.bleet(x * 0.02)) id, @@ -93,10 +99,15 @@ const // there are more const function variables to follow, ); } }, - stringify = json => { + stringify = (json) => { return JSON.stringify(json) + '\n'; }, - getSignature = ( user_screen_name, bot_screen_name, tweet_tag_or_id, signature ) => { + getSignature = ( + user_screen_name, + bot_screen_name, + tweet_tag_or_id, + signature + ) => { if (signature === 'none' || !signature) { return ''; } @@ -106,166 +117,286 @@ const // there are more const function variables to follow, .replaceAll('BOT_SCREEN_NAME', bot_screen_name || 'twetJs') .replaceAll('TWEET_ID_OR_TAG', tweet_tag_or_id || '??'); }, - trimPath = text => text.replaceAll(/[\\/]$/gu, ''), - appendLog = ( log_dir, module_name, log_name, jsonObject, optional_date_time ) => { + trimPath = (text) => text.replaceAll(/[\\/]$/gu, ''), + appendLog = ( + log_dir, + module_name, + log_name, + jsonObject, + optional_date_time + ) => { const filename = - trimPath(log_dir) + '/' + - module_name + - '.' + - log_name + - '.' + - (optional_date_time || new Date().toISOString().slice(0, 10)) + - '.json'; - fs.appendFile(filename, stringify(jsonObject), 'utf8', err => { - if (err) { throw err; } + trimPath(log_dir) + + '/' + + module_name + + '.' + + log_name + + '.' + + (optional_date_time || new Date().toISOString().slice(0, 10)) + + '.json'; + fs.appendFile(filename, stringify(jsonObject), 'utf8', (err) => { + if (err) { + throw err; + } }); processLog({ log_append: filename, }); }, getProperty = (obj, prop, optional_fallback) => { const fallback = - typeof optional_fallback === 'undefined' ? null : optional_fallback; + typeof optional_fallback === 'undefined' ? null : optional_fallback; if (typeof obj !== 'object' || obj === null) { return fallback; } else { if ( Object.prototype.hasOwnProperty.call(obj, prop) && - obj.length === 0 + obj.length === 0 ) { return fallback; } else { - return Object.prototype.hasOwnProperty.call(obj, prop) ? obj[prop] + return Object.prototype.hasOwnProperty.call(obj, prop) ? + obj[prop] : fallback; } } }, - unlink = text => { - return text.toString() - .replaceAll('@', '@ ') - .replaceAll('#', '# '); + unlink = (text) => { + return text.toString().replaceAll('@', '@ ').replaceAll('#', '# '); }, - normalizeCharacters = text => { + normalizeCharacters = (text) => { // translitro Bleetify return anglicize( - replaceSpecialCharacters( - weirdToNormalChars( - clean( - text.toString() - ) - ) - ) + replaceSpecialCharacters(weirdToNormalChars(clean(text.toString()))) ) .toLowerCase() .trim(); }, - getEmoji = text => { const regex = emojiRegex(); return text.toString().match(regex); }, - normalizeSpaces = text => text.replaceAll(/ +/gu, ' '), - normalizeNewlines = text => text.replaceAll(/\n\n+/gu, '\n\n').replaceAll(/\n\n+/gu, '\n\n'), - trimAllLines = text => text.split('\n').map(line => line.trim()).join('\n'), - collapseExtraSingleSpaces = text => { + getEmoji = (text) => { + const regex = emojiRegex(); + return text.toString().match(regex); + }, + normalizeSpaces = (text) => text.replaceAll(/ +/gu, ' '), + normalizeNewlines = (text) => + text.replaceAll(/\n\n+/gu, '\n\n').replaceAll(/\n\n+/gu, '\n\n'), + trimAllLines = (text) => + text + .split('\n') + .map((line) => line.trim()) + .join('\n'), + collapseExtraSingleSpaces = (text) => { let output = normalizeNewlines(normalizeSpaces(trimAllLines(text))); const spaceMatches = output.match(/[ \n\t]\S( \S)+ \S \S[ \n\t]/gu); // const newlineMatches = text.match(/[ \n\t]\S[ \t]*(\n\S[ \t]*)+\n/gu); // const newlineMatches = output.match(/\n{0,1}(\s{0,1}[ ?!.]{0,2}\S*)+\n/gu); - const newlineMatches = output.match(/(^|\n)+(\S{1}[ ?!.]{0,2}(\s|$)+)+/gu); - const tabMatches = output.match(/[ \n\t]\S[ \n]*(\t\S[ \n]*)+[ \n\t]/gu); + const newlineMatches = output.match( + /(^|\n)+(\S{1}[ ?!.]{0,2}(\s|$)+)+/gu + ); + const tabMatches = output.match( + /[ \n\t]\S[ \n]*(\t\S[ \n]*)+[ \n\t]/gu + ); let count = 0; - if (typeof spaceMatches !== 'undefined' && spaceMatches && spaceMatches.length > 0) { - spaceMatches.forEach(match => { - output = output.replaceAll(match, match.charAt(0).replaceAll(/\S/gu, '') + match.replaceAll(/\s/gu, '') + match.charAt(match.length - 1).replaceAll(/\S/gu, '')); + if ( + typeof spaceMatches !== 'undefined' && + spaceMatches && + spaceMatches.length > 0 + ) { + spaceMatches.forEach((match) => { + output = output.replaceAll( + match, + match.charAt(0).replaceAll(/\S/gu, '') + + match.replaceAll(/\s/gu, '') + + match.charAt(match.length - 1).replaceAll(/\S/gu, '') + ); count += 1; }); } - if (typeof newlineMatches !== 'undefined' && newlineMatches && newlineMatches.length > 0) { - newlineMatches.forEach(match => { + if ( + typeof newlineMatches !== 'undefined' && + newlineMatches && + newlineMatches.length > 0 + ) { + newlineMatches.forEach((match) => { const intermediate = match.split('\n\n'); count += intermediate.length; - output = output.replaceAll(match, match.charAt(0).replaceAll(/\S/gu, '')+ intermediate.map(value => value.replaceAll(/\s/gu, '')).join('\n\n') + match.charAt(match.length - 1).replaceAll(/\S/gu, '')); + output = output.replaceAll( + match, + match.charAt(0).replaceAll(/\S/gu, '') + + intermediate + .map((value) => value.replaceAll(/\s/gu, '')) + .join('\n\n') + + match.charAt(match.length - 1).replaceAll(/\S/gu, '') + ); }); } - if (typeof tabMatches !== 'undefined' && tabMatches && tabMatches.length > 0) { - tabMatches.forEach(match => { - output = output.replaceAll(match, match.charAt(0).replaceAll(/\S/gu, '') + match.replaceAll(/\s/gu, '') + match.charAt(match.length - 1).replaceAll(/\S/gu, '')); + if ( + typeof tabMatches !== 'undefined' && + tabMatches && + tabMatches.length > 0 + ) { + tabMatches.forEach((match) => { + output = output.replaceAll( + match, + match.charAt(0).replaceAll(/\S/gu, '') + + match.replaceAll(/\s/gu, '') + + match.charAt(match.length - 1).replaceAll(/\S/gu, '') + ); count += 1; }); } return { text: output, count, }; }, - removeEmoji = text => { const regex = emojiRegex();return collapseExtraSingleSpaces(normalizeSpaces(text.toString().replaceAll(regex, ' ').replaceAll(/\s\n\s/gu, ' \n'))); }, - emojiCount = text => (getEmoji(text.toString()) || []).length, - isEmojiRidden = text => { + removeEmoji = (text) => { + const regex = emojiRegex(); + return collapseExtraSingleSpaces( + normalizeSpaces( + text + .toString() + .replaceAll(regex, ' ') + .replaceAll(/\s\n\s/gu, ' \n') + ) + ); + }, + emojiCount = (text) => (getEmoji(text.toString()) || []).length, + isEmojiRidden = (text) => { const characterCount = (runes(text.toString()) || []).length; const _emojiCount = emojiCount(text.toString()); - const firstCharactersEmojiCount = emojiCount(runes.substr(text, 0, 10).toString()); + const firstCharactersEmojiCount = emojiCount( + runes.substr(text, 0, 10).toString() + ); const spaceCount = (text.toString().match(/ /gu) || []).length; const whitespaceCount = (text.toString().match(/[ ].*/gu) || []).length; - console.dir({text, characterCount, _emojiCount, firstCharactersEmojiCount, spaceCount, whitespaceCount,}); - if (_emojiCount > 3 && (_emojiCount > spaceCount * 0.5 || _emojiCount > whitespaceCount || _emojiCount > characterCount * 0.25 || firstCharactersEmojiCount > 3)) { + console.dir({ + text, + characterCount, + _emojiCount, + firstCharactersEmojiCount, + spaceCount, + whitespaceCount, + }); + if ( + _emojiCount > 3 && + (_emojiCount > spaceCount * 0.5 || + _emojiCount > whitespaceCount || + _emojiCount > characterCount * 0.25 || + firstCharactersEmojiCount > 3) + ) { return _emojiCount; } else { return false; } }, - makeUnique = text => String.prototype.concat(...new Set(text)), - cleanText = text => {console.dir({text,});return normalizeCharacters(unlink(text)); }, - cleanEmoji = text => { - if (typeof text !== 'undefined' && text !== null && text !== '' ) { + makeUnique = (text) => String.prototype.concat(...new Set(text)), + cleanText = (text) => { + console.dir({ text, }); + return normalizeCharacters(unlink(text)); + }, + cleanEmoji = (text) => { + if (typeof text !== 'undefined' && text !== null && text !== '') { const emojiCount = isEmojiRidden(text); - const cleanedText = removeEmoji(text); - if (emojiCount !== false ) { - return { 'text': cleanedText.text, - 'alterations': '\n[ Removed: ' + emojiCount + ' of ' + runes(makeUnique(getEmoji(text).join(''))).length + - ' Emoji, including ' + - runes.substr(makeUnique(getEmoji(text).join('')), 0,runes(makeUnique(getEmoji(text).join(''))).length > 8 ? 4 : 3).split().join(' ') + - (runes(makeUnique(getEmoji(text).join(''))).length > 3 ? runes.substr(makeUnique(getEmoji(text).join('')), runes(makeUnique(getEmoji(text).join(''))).length > 4 ? -2 : -1, 0).split().join(' ') : '' )+' ]' + - (cleanedText.count > 0 ? '\n[ Collapsed: ' + cleanedText.count + ' Exploded Word' + (cleanedText.count === 1 ? '' : 's') + ' ]' : '') + - '\n[ Removed: Uppercase, Fonts, Math, Diacritics ]', + const cleanedText = removeEmoji(text); + if (emojiCount !== false) { + return { + text: cleanedText.text, + alterations: + '\n[ Removed: ' + + emojiCount + + ' of ' + + runes(makeUnique(getEmoji(text).join(''))).length + + ' Emoji, including ' + + makeUnique( + runes.substr( + makeUnique(getEmoji(text).join('')), + 0, + runes(makeUnique(getEmoji(text).join(''))) + .length > 11 ? 3 : 2 + ) + + makeUnique( + runes.substr( + makeUnique(getEmoji(text).join('')), + runes( + makeUnique(getEmoji(text).join('')) + ).length > 7 ? + runes( + makeUnique( + getEmoji(text).join('') + ) + ).length - 3 + : runes( + makeUnique( + getEmoji(text).join('') + ) + ).length - 2, + runes( + makeUnique(getEmoji(text).join('')) + ).length - 1 + ) + ) + ) + + ' ]' + + (cleanedText.count > 0 ? + '\n[ Collapsed: ' + + cleanedText.count + + ' Exploded Word' + + (cleanedText.count === 1 ? '' : 's') + + ' ]' + : '') + + '\n[ Removed: Uppercase, Fonts, Math, Diacritics ]', }; } else { - return { 'text': cleanedText.text, - 'alterations': - (cleanedText.count > -1 ? '\n[ Collapsed: ' + cleanedText.count + ' Exploded Word' + (cleanedText.count === 1 ? '' : 's') + ' ]' : '') + - '\n[ Removed: Uppercase, Fonts, Math, Diacritics ]', + return { + text: collapseExtraSingleSpaces( + normalizeSpaces(text.toString()) + ).text.replaceAll(/\s\n\s/gu, ' \n'), + alterations: + (cleanedText.count > 0 ? + '\n[ Collapsed: ' + + cleanedText.count + + ' Exploded Word' + + (cleanedText.count === 1 ? '' : 's') + + ' ]' + : '') + + '\n[ Removed: Uppercase, Fonts, Math, Diacritics ]', }; } } else { - return { 'text': text, 'alterations': '\n[ Error: Received empty text? ]', }; + return { + text: text, + alterations: '\n[ Error: Received empty text? ]', + }; } }, urlExpand = (text, urls, mediaUrls, skipUrls) => { let output = text; - if (Array.isArray(urls) && - urls.length > 0) { - urls.forEach(urlValue => { - output = output - .replaceAll( - new RegExp(RegExp.escape(urlValue.url), 'ig'), - (typeof skipUrls !== 'undefined' ? [skipUrls,].flat(Infinity) : []).includes(urlValue.expanded_url) ? - ' [\'Quote-Tweet\'] ' : - urlValue.expanded_url - ); - } - ); + if (Array.isArray(urls) && urls.length > 0) { + urls.forEach((urlValue) => { + output = output.replaceAll( + new RegExp(RegExp.escape(urlValue.url), 'ig'), + (typeof skipUrls !== 'undefined' ? + [skipUrls,].flat(Infinity) + : [] + ).includes(urlValue.expanded_url) ? + ' [\'Quote-Tweet\'] ' + : urlValue.expanded_url + ); + }); } - if (Array.isArray(mediaUrls) && - mediaUrls.length > 0) { - mediaUrls.forEach(urlValue => { - output = output - .replaceAll( - new RegExp(RegExp.escape(urlValue.url), 'ig'), - urlValue.expanded_url - ); - } - ); + if (Array.isArray(mediaUrls) && mediaUrls.length > 0) { + mediaUrls.forEach((urlValue) => { + output = output.replaceAll( + new RegExp(RegExp.escape(urlValue.url), 'ig'), + urlValue.expanded_url + ); + }); } return output; }, - leftTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/^\s+/gu,'')).join('\n'), - rightTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/\s+$/gu,'')).join('\n'), - replaceEmptyLines = (text, replace) => text.replaceAll(/\n\s*\n/gu, replace), - reduceEmptyLinesToSingle = text => replaceEmptyLines(text, '\n\n'), - removeEmptyLines = text => replaceEmptyLines(text, '\n'), - oneSpaceBeforeEachNewLine = text => rightTrimAllLines(text).replaceAll('\n', ' \n'), - fixAmp = text => text.replaceAll('&', '&'), + // leftTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/^\s+/gu,'')).join('\n'), + // rightTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/\s+$/gu,'')).join('\n'), + replaceEmptyLines = (text, replace) => + text.replaceAll(/\n\s*\n/gu, replace), + reduceEmptyLinesToSingle = (text) => replaceEmptyLines(text, '\n\n'), + // removeEmptyLines = text => replaceEmptyLines(text, '\n'), + // oneSpaceBeforeEachNewLine = text => rightTrimAllLines(text).replaceAll('\n', ' \n'), + fixAmp = (text) => text.replaceAll('&', '&'), start = async () => { try { // user_SCREEN_NAME is replaced with the user's name. @@ -273,11 +404,11 @@ const // there are more const function variables to follow, // Set SIGNATURE to 'none' to actually have no signature. // 'at' @twetJs if you need 'none' as a signature and we'll see. PRs welcome! // todo parameterize module_screen_name - const - module_screen_name = process.env.MODULE_SCREEN_NAME || 'twetJs', + const module_screen_name = + process.env.MODULE_SCREEN_NAME || 'twetJs', signature = - process.env.SIGNATURE || - ' \nThanks, USER_SCREEN_NAME! \n\n@BOT_SCREEN_NAME #BOT_SCREEN_NAME \n[\'TWEET_ID_OR_TAG\']', + process.env.SIGNATURE || + ' \nThanks, USER_SCREEN_NAME! \n\n@BOT_SCREEN_NAME #BOT_SCREEN_NAME \n[\'TWEET_ID_OR_TAG\']', bot_name = process.env.BOT_NAME || module_screen_name, data_dir = trimPath(process.env.DATA_DIR || './data'), log_dir = data_dir + '/logs', @@ -287,25 +418,27 @@ const // there are more const function variables to follow, consumer_key: process.env.TWITTER_CONSUMER_KEY, consumer_secret: process.env.TWITTER_CONSUMER_SECRET, access_token_key: - process.env.TWITTER_ACCESS_TOKEN || - process.env.ACCESS_TOKEN, // oauth + process.env.TWITTER_ACCESS_TOKEN || + process.env.ACCESS_TOKEN, // oauth access_token_secret: - process.env.TWITTER_ACCESS_TOKEN_SECRET || - process.env.ACCESS_TOKEN_SECRET, // oauth + process.env.TWITTER_ACCESS_TOKEN_SECRET || + process.env.ACCESS_TOKEN_SECRET, // oauth }, port = await getNewPort(), - webhook = new Autohook({port: port,}); + webhook = new Autohook({ port: port, }); if (!fs.existsSync(log_dir)) { fs.mkdirSync(log_dir); } await removeWebhooks(); - webhook.on('event', async event => { + webhook.on('event', async (event) => { try { appendLog(log_dir, bot_name, 'event', event); processLog(event); for (const [key, value] of Object.entries(event)) { console.log( - '{ "isoString" : "' + new Date().toISOString() + '" }' + '{ "isoString" : "' + + new Date().toISOString() + + '" }' ); if (Array.isArray(value)) { appendLog(log_dir, bot_name, key, value); @@ -313,7 +446,7 @@ const // there are more const function variables to follow, if (event.tweet_create_events) { const bot_user_id = event.for_user_id, tweet_create_events = event.tweet_create_events.map( - async tweet => { + async (tweet) => { const tweet_id = tweet.id_str; if ( getProperty( @@ -340,18 +473,22 @@ const // there are more const function variables to follow, appendLog( log_dir, bot_name, - key + '.tweetThreadReply.bugReport.prelim', + key + + '.tweetThreadReply.bugReport.prelim', { value, event, } ); } - if (tweet.user.id_str === bot_user_id) { + if ( + tweet.user.id_str === + bot_user_id + ) { appendLog( log_dir, bot_name, key + '.self', value ); - return {sub_type: 'self',}; + return { sub_type: 'self', }; } else if ( tweet.text .toString() @@ -363,7 +500,7 @@ const // there are more const function variables to follow, key + '.rt', value ); - return {sub_type: 'rt',}; + return { sub_type: 'rt', }; } else { appendLog( log_dir, @@ -372,9 +509,9 @@ const // there are more const function variables to follow, value ); const in_reply_to_status_id = - tweet.in_reply_to_status_id_str, + tweet.in_reply_to_status_id_str, in_reply_to_screen_name = - tweet.in_reply_to_screen_name, + tweet.in_reply_to_screen_name, user_name = tweet.user.name.toString(), user_screen_name = tweet.user.screen_name.toString(), extended_tweet = getProperty( @@ -404,72 +541,79 @@ const // there are more const function variables to follow, 'user_mentions' ), bot_user_mentions = - user_mentions === null || - user_mentions === undefined ? null - : user_mentions.find( - obj => - obj.id_str === - bot_user_id - ), + user_mentions === + null || + user_mentions === + undefined ? + null + : user_mentions.find( + (obj) => + obj.id_str === + bot_user_id + ), bot_screen_name = getProperty( bot_user_mentions, 'screen_name', module_screen_name ), - client = new Twitter(params), + client = new Twitter( + params + ), status_reply_to = - typeof in_reply_to_status_id === - 'undefined' || - in_reply_to_status_id === - null || - in_reply_to_status_id === '' ? '' - : await (async () => { - try { - return await client.get( - 'statuses/show', - { - id: in_reply_to_status_id, - tweet_mode: - 'extended', - } - ); - } catch (e) { - if ( - 'errors' in - e - ) { - if ( + typeof in_reply_to_status_id === + 'undefined' || + in_reply_to_status_id === + null || + in_reply_to_status_id === + '' ? + '' + : await (async () => { + try { + return await client.get( + 'statuses/show', + { + id: in_reply_to_status_id, + tweet_mode: + 'extended', + } + ); + } catch (e) { + if ( + 'errors' in e - .errors[0] - .code === - 88 - ) { - // Twitter API error - processLog( - { - e, - error: { - 'x-rate-limit-reset': new Date( - e._headers.get( - 'x-rate-limit-reset' - ) * - 1000 - ), - }, - } - ); - } else { - // some other kind of error , e.g. read-only API trying to POST - } - } else { - // non-API error , e.g. network problem or invalid JSON in response - } - processError( - e - ); - return ''; - } - })(), + ) { + if ( + e + .errors[0] + .code === + 88 + ) { + // Twitter API error + processLog( + { + e, + error: { + 'x-rate-limit-reset': new Date( + e._headers.get( + 'x-rate-limit-reset' + ) * + 1000 + ), + }, + } + ); + } else { + // some other kind of error , e.g. read-only API trying to POST + } + } else { + // non-API error , e.g. network problem or invalid JSON in response + } + processError( + e + ); + return ''; + } + })(), extended_tweet_reply_to = getProperty( status_reply_to, 'extended_tweet', @@ -478,24 +622,46 @@ const // there are more const function variables to follow, reply_to_display_text_range = getProperty( extended_tweet_reply_to, 'display_text_range', - [-Infinity, Infinity,] + [-Infinity, Infinity, ] ), - reply_entities = getProperty(extended_tweet_reply_to, 'entities', null), - urls = getProperty(reply_entities, 'urls', null), + reply_entities = getProperty( + extended_tweet_reply_to, + 'entities', + null + ), + urls = getProperty( + reply_entities, + 'urls', + null + ), + /* jshint ignore:start */ urlsLog = appendLog( // eslint-disable-line no-unused-vars log_dir, bot_name, - key + '.tweetThreadReply.urls', + key + + '.tweetThreadReply.urls', urls ), - media_reply_entities = getProperty(extended_tweet_reply_to, 'extended_entities', reply_entities), - media = getProperty(media_reply_entities, 'media', null), + /* jshint ignore:end */ + media_reply_entities = getProperty( + extended_tweet_reply_to, + 'extended_entities', + reply_entities + ), + media = getProperty( + media_reply_entities, + 'media', + null + ), + /* jshint ignore:start */ mediaLog = appendLog( // eslint-disable-line no-unused-vars log_dir, bot_name, - key + '.tweetThreadReply.urls.media', + key + + '.tweetThreadReply.urls.media', media ), + /* jshint ignore:end */ quoted_status_permalink = getProperty( extended_tweet_reply_to, 'quoted_status_permalink', @@ -506,62 +672,75 @@ const // there are more const function variables to follow, 'expanded', null ), - url_expanded_text = urlExpand((getProperty( - extended_tweet_reply_to, - 'full_text', - extended_tweet_reply_to.text - ) || '').substring( - reply_to_display_text_range[0]), urls, media, quoted_status_permalink_expanded), + url_expanded_text = urlExpand( + ( + getProperty( + extended_tweet_reply_to, + 'full_text', + extended_tweet_reply_to.text + ) || '' + ).substring( + reply_to_display_text_range[0] + ), + urls, + media, + quoted_status_permalink_expanded + ), cleaned_text = cleanEmoji( cleanText( url_expanded_text ) ), - straightened_lines = - normalizeNewlines( - reduceEmptyLinesToSingle( - trimAllLines( - normalizeSpaces( - fixAmp(cleaned_text.text) + straightened_lines = normalizeNewlines( + reduceEmptyLinesToSingle( + trimAllLines( + normalizeSpaces( + fixAmp( + cleaned_text.text ) ) ) - ).trim(), + ) + ).trim(), reply_text = typeof in_reply_to_status_id === 'undefined' || in_reply_to_status_id === null || - in_reply_to_status_id === '' ? '' + in_reply_to_status_id === + '' ? + '' : '@ ' + in_reply_to_screen_name + ' SAYS:\n\n' + - straightened_lines + + straightened_lines + '\n\n' + 'END @ ' + in_reply_to_screen_name + ' QUOTE\n' + - cleaned_text.alterations - ; + cleaned_text.alterations; appendLog( log_dir, bot_name, - key + '.tweetThreadReply.replyTweet', + key + + '.tweetThreadReply.replyTweet', status_reply_to ); if ( user_tweet_text .toString() .toLowerCase() - .includes( - 'bug' - ) + .includes('bug') ) { appendLog( log_dir, bot_name, - key + '.tweetThreadReply.bugReport', - { status_reply_to, event, } + key + + '.tweetThreadReply.bugReport', + { + status_reply_to, + event, + } ); } if ( @@ -569,37 +748,46 @@ const // there are more const function variables to follow, .toString() .toLowerCase() .includes( - '@' + bot_screen_name.toLowerCase() + '@' + + bot_screen_name.toLowerCase() ) ) { appendLog( log_dir, bot_name, - key + '.tweetThreadReply.reply_text', + key + + '.tweetThreadReply.reply_text', reply_text ); const thread = [ ( '@' + - user_screen_name + - '\n' + //', here you go!\n' + - // status + '\n' + - reply_text + - '\n' + - getSignature( - user_name, - bot_screen_name, - in_reply_to_status_id || - tweet_id, - signature - ) + user_screen_name + + '\n' + //', here you go!\n' + + // status + '\n' + + reply_text + + '\n' + + getSignature( + user_name, + bot_screen_name, + in_reply_to_status_id || + tweet_id, + signature + ) ).trim(), ].flat(Infinity); appendLog( log_dir, bot_name, - key + '.tweetThreadReply', - {tweetThreadReply: {thread, tweet_id, params,},} + key + + '.tweetThreadReply', + { + tweetThreadReply: { + thread, + tweet_id, + params, + }, + } ); processLog(status_reply_to); tweetThreadReply( @@ -607,7 +795,9 @@ const // there are more const function variables to follow, tweet_id, params ).catch(processError); - return {sub_type: 'other.tweet',}; + return { + sub_type: 'other.tweet', + }; } else { const indirect = {}; indirect[key] = value; @@ -617,24 +807,34 @@ const // there are more const function variables to follow, key + '.indirect', indirect ); - return {sub_type: 'other.indirect',}; + return { + sub_type: + 'other.indirect', + }; } - return {sub_type: 'other.etc',}; // eslint-disable-line no-unreachable + return { // eslint-disable-line no-unreachable + sub_type: 'other.etc', + }; } } ), filtered_tweet_create_events = tweet_create_events.filter( - el => el && - el.then && - typeof el.then === 'function' + (el) => + el && + el.then && + typeof el.then === 'function' ); if ( - Array.isArray(filtered_tweet_create_events) && - filtered_tweet_create_events.length > 0 + Array.isArray( + filtered_tweet_create_events + ) && + filtered_tweet_create_events.length > 0 ) { Promise.all( filtered_tweet_create_events - ).then(completed => processLog(completed)); + ).then((completed) => + processLog(completed) + ); } } } else { @@ -657,13 +857,15 @@ const // there are more const function variables to follow, } catch (e) { if ('errors' in e) { if (e.errors[0].code === 88) { - // Twitter API error + // Twitter API error processLog( '{ "e": {' + - e + - '}, "error" : { "x-rate-limit-reset" : ' + - new Date(e._headers.get('x-rate-limit-reset') * 1000) + - ' } }' + e + + '}, "error" : { "x-rate-limit-reset" : ' + + new Date( + e._headers.get('x-rate-limit-reset') * 1000 + ) + + ' } }' ); } else { // some other kind of error , e.g. read-only API trying to POST @@ -680,13 +882,15 @@ const // there are more const function variables to follow, } catch (e) { if ('errors' in e) { if (e.errors[0].code === 88) { - // Twitter API error + // Twitter API error processLog( '{ "e": {' + - e + - '}, "error" : { "x-rate-limit-reset" : ' + - new Date(e._headers.get('x-rate-limit-reset') * 1000) + - ' } }' + e + + '}, "error" : { "x-rate-limit-reset" : ' + + new Date( + e._headers.get('x-rate-limit-reset') * 1000 + ) + + ' } }' ); } else { // some other kind of error , e.g. read-only API trying to POST From 407d92bf007a17f8af6774ca4772cabf9905055d Mon Sep 17 00:00:00 2001 From: Drewry Pope Date: Mon, 21 Sep 2020 08:23:50 -0500 Subject: [PATCH 3/3] refactor(cleanup): cleanup unused code --- lib/index.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/index.js b/lib/index.js index 2d30ad4..5b2013d 100644 --- a/lib/index.js +++ b/lib/index.js @@ -185,8 +185,6 @@ const // there are more const function variables to follow, collapseExtraSingleSpaces = (text) => { let output = normalizeNewlines(normalizeSpaces(trimAllLines(text))); const spaceMatches = output.match(/[ \n\t]\S( \S)+ \S \S[ \n\t]/gu); - // const newlineMatches = text.match(/[ \n\t]\S[ \t]*(\n\S[ \t]*)+\n/gu); - // const newlineMatches = output.match(/\n{0,1}(\s{0,1}[ ?!.]{0,2}\S*)+\n/gu); const newlineMatches = output.match( /(^|\n)+(\S{1}[ ?!.]{0,2}(\s|$)+)+/gu ); @@ -389,12 +387,12 @@ const // there are more const function variables to follow, } return output; }, - // leftTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/^\s+/gu,'')).join('\n'), - // rightTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/\s+$/gu,'')).join('\n'), replaceEmptyLines = (text, replace) => text.replaceAll(/\n\s*\n/gu, replace), reduceEmptyLinesToSingle = (text) => replaceEmptyLines(text, '\n\n'), // removeEmptyLines = text => replaceEmptyLines(text, '\n'), + // leftTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/^\s+/gu,'')).join('\n'), + // rightTrimAllLines = text => text.split('\n').map(line => line.replaceAll(/\s+$/gu,'')).join('\n'), // oneSpaceBeforeEachNewLine = text => rightTrimAllLines(text).replaceAll('\n', ' \n'), fixAmp = (text) => text.replaceAll('&', '&'), start = async () => {