diff --git a/src/appmixer/cloudflare/jobs.js b/src/appmixer/cloudflare/jobs.js index f36973ccd..73dc9ae7c 100644 --- a/src/appmixer/cloudflare/jobs.js +++ b/src/appmixer/cloudflare/jobs.js @@ -1,5 +1,5 @@ -const ZoneCloudflareClient = require('./lists/CloudflareListClient'); -const wafJobs = require('./waf/jobs.waf'); +const wafJobs = require('./jobs.waf'); +const listsJobs = require('./jobs.lists'); module.exports = async (context) => { @@ -15,7 +15,7 @@ module.exports = async (context) => { await context.log('trace', '[CloudFlare] rule delete job started.'); try { - await deleteExpireIpsFromList(context); + await listsJobs.deleteExpireIpsFromList(context); await wafJobs.deleteExpireIps(context); } finally { lock.unlock(); @@ -32,70 +32,6 @@ module.exports = async (context) => { } }); - const deleteExpireIpsFromList = async function(context) { - - const expired = await getExpiredItems(context); - - const groups = Object.values(expired); - - const promises = groups.map(chunk => { - - const { email, apiKey, account, list } = chunk.auth; - - context.log('info', { stage: '[CloudFlare] removing ', ips: chunk.ips, list, account }); - - const client = new ZoneCloudflareClient({ email, apiKey }); - // https://developers.cloudflare.com/api/operations/lists-delete-list-items - return client.callEndpoint(context, { - method: 'DELETE', - action: `/accounts/${account}/rules/lists/${list}/items`, - data: { items: chunk.ips } - }); - }); - - const itemsToDelete = { ids: [], lists: [] }; - - (await Promise.allSettled(promises)).forEach(async (result, i) => { - if (result.status === 'fulfilled') { - itemsToDelete.ids = itemsToDelete.ids.concat(groups[i].ips); - itemsToDelete.lists.push(groups[i]?.auth?.list); - } else { - const operations = groups[i].ips.map(item => ({ - updateOne: { - filter: { id: item.id }, update: { $set: { mtime: new Date } } - } - })); - await (context.db.collection(IPListModel.collection)).bulkWrite(operations); - } - }); - - if (itemsToDelete.ids.length) { - const deleted = await context.db.collection(IPListModel.collection) - .deleteMany({ id: { $in: itemsToDelete.ids.map(item => item.id) } }); - await context.log('info', { - stage: `[CloudFlare] Deleted total ${deleted.deletedCount} ips.`, - lists: itemsToDelete.lists, - itemIds: itemsToDelete.ids - }); - } - }; - - const getExpiredItems = async function(context) { - - const expired = await context.db.collection(IPListModel.collection) - .find({ removeAfter: { $lt: Date.now() } }) - .toArray(); - - return expired.reduce((res, item) => { - const key = item.auth.list; // listId - res[key] = res[key] || { ips: [] }; - res[key].auth = item.auth; - res[key].ips.push({ id: item.id }); - - return res; - }, {}); - }; - // Self-healing job to remove rules that have created>mtime. These rules are stuck in the system and should be removed. await context.scheduleJob('cloud-flare-lists-ips-cleanup-job', config.cleanup.schedule, async () => { diff --git a/src/appmixer/cloudflare/jobs.lists.js b/src/appmixer/cloudflare/jobs.lists.js new file mode 100644 index 000000000..41b47f00b --- /dev/null +++ b/src/appmixer/cloudflare/jobs.lists.js @@ -0,0 +1,67 @@ +const CloudflareListClient = require('./lists/CloudflareListClient'); + +const deleteExpireIpsFromList = async function(context) { + + const expired = await getExpiredItems(context); + + const groups = Object.values(expired); + + const promises = groups.map(chunk => { + + const { email, apiKey, account, list } = chunk.auth; + + context.log('info', { stage: '[CloudFlare] removing ', ips: chunk.ips, list, account }); + + const client = new CloudflareListClient({ email, apiKey }); + // https://developers.cloudflare.com/api/operations/lists-delete-list-items + return client.callEndpoint(context, { + method: 'DELETE', action: `/accounts/${account}/rules/lists/${list}/items`, data: { items: chunk.ips } + }); + }); + + const itemsToDelete = { ids: [], lists: [] }; + + (await Promise.allSettled(promises)).forEach(async (result, i) => { + if (result.status === 'fulfilled') { + itemsToDelete.ids = itemsToDelete.ids.concat(groups[i].ips); + itemsToDelete.lists.push(groups[i]?.auth?.list); + } else { + const operations = groups[i].ips.map(item => ({ + updateOne: { + filter: { id: item.id }, update: { $set: { mtime: new Date } } + } + })); + await (context.db.collection(IPListModel.collection)).bulkWrite(operations); + } + }); + + if (itemsToDelete.ids.length) { + const deleted = await context.db.collection(IPListModel.collection) + .deleteMany({ id: { $in: itemsToDelete.ids.map(item => item.id) } }); + await context.log('info', { + stage: `[CloudFlare] Deleted total ${deleted.deletedCount} ips.`, + lists: itemsToDelete.lists, + itemIds: itemsToDelete.ids + }); + } +}; + +const getExpiredItems = async function(context) { + + const expired = await context.db.collection(IPListModel.collection) + .find({ removeAfter: { $lt: Date.now() } }) + .toArray(); + + return expired.reduce((res, item) => { + const key = item.auth.list; // listId + res[key] = res[key] || { ips: [] }; + res[key].auth = item.auth; + res[key].ips.push({ id: item.id }); + + return res; + }, {}); +}; + +module.exports = { + deleteExpireIpsFromList +}; diff --git a/src/appmixer/cloudflare/waf/jobs.waf.js b/src/appmixer/cloudflare/jobs.waf.js similarity index 97% rename from src/appmixer/cloudflare/waf/jobs.waf.js rename to src/appmixer/cloudflare/jobs.waf.js index c349bf757..4e0f9e8c8 100644 --- a/src/appmixer/cloudflare/waf/jobs.waf.js +++ b/src/appmixer/cloudflare/jobs.waf.js @@ -1,5 +1,5 @@ -const CloudflareWAFClient = require('./CloudflareWAFClient'); -const lib = require('./lib'); +const CloudflareWAFClient = require('./waf/CloudflareWAFClient'); +const lib = require('./waf/lib'); const deleteExpireIps = async function(context) { diff --git a/src/appmixer/cloudflare/lists/CloudflareListClient.js b/src/appmixer/cloudflare/lists/CloudflareListClient.js index b6d141489..8690bf02e 100644 --- a/src/appmixer/cloudflare/lists/CloudflareListClient.js +++ b/src/appmixer/cloudflare/lists/CloudflareListClient.js @@ -1,4 +1,4 @@ -module.exports = class ZoneCloudflareClient { +module.exports = class CloudflareListClient { constructor({ email, apiKey, zoneId, token }) { this.email = email;