Skip to content

Commit

Permalink
Concurrent requests working + simpler code structure
Browse files Browse the repository at this point in the history
  • Loading branch information
@hdoro
hdoro committed Nov 21, 2018
1 parent 3155b17 commit 9e4b833
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 123 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/node_modules
configValidation.js
gatsby-node.js
fetchAllCampaigns.js
fetchCampaigns.js
fetchContent.js
helpers.js
yarn-error.log
Expand Down
114 changes: 114 additions & 0 deletions src/fetchCampaigns.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import axios from 'axios';
import { IAuthParams } from './gatsby-node';
import { colorizeLog, consoleColors } from './helpers';

export interface IFetchCampaigns {
concurrReq: number;
offset: number;
authParams: IAuthParams;
rootURL: string;
campaignFields: string[];
contentFields: string[];
cache: any;
createContentDigest: any;
actions: any;
nodeType: string;
}

export const fetchCampaigns = async ({
concurrReq,
offset,
authParams,
rootURL,
campaignFields,
contentFields,
cache,
createContentDigest,
actions,
nodeType,
}: IFetchCampaigns) => {
const { createNode, touchNode } = actions;
const campaignsURL = `${rootURL}/campaigns`;

const campaignsData = await axios.get(campaignsURL, {
...authParams,
params: {
count: concurrReq,
offset,
fields: campaignFields.join(','),
sort_field: 'send_time',
sort_dir: 'DESC',
},
});

// Non-cached campaigns that should have their content fetched
let campaignRequests: any[] = [];
let campaignsMetadata: any[] = [];
for (const c of campaignsData.data.campaigns) {
if (c.id === undefined) {
console.log(
`${colorizeLog("A campaign couldn't be fetched", consoleColors.BgRed)}${
c.settings && c.settings.subject_line
? `: ${c.settings.subject_line}`
: ''
}`
);
continue;
}

const internalId = `mailchimp-campaign-${c.id}`;
const cacheableContent = JSON.stringify(c);
const cachedCampaign = await cache.get(internalId);

// Make sure the campaign metadata is the same as the one just
// fetch from Mailchimp. If so, touch the node and don't mind about
// fetching the content again in order to save some build time
if (cachedCampaign && cachedCampaign.content === cacheableContent) {
touchNode({ nodeId: internalId });
continue;
}

// Fetch the campaign's content
const contentURL = `${campaignsURL}/${c.id}/content`;
campaignRequests = [
...campaignRequests,
axios.get(contentURL, {
...authParams,
params: {
fields: contentFields.join(','),
},
}),
];
campaignsMetadata = [...campaignsMetadata, c];
}

const campaignsContent = await Promise.all(campaignRequests);

for (let i = 0; i < campaignsContent.length; i++) {
const meta = campaignsMetadata[i];
const content = campaignsContent[i];

const internalId = `mailchimp-campaign-${meta.id}`;
const cacheableContent = JSON.stringify(meta);
await cache.set(internalId, { content: cacheableContent });

const campaignNode = {
...meta,
...content.data,
campaignId: meta.id,
// meta information for the node
id: internalId,
parent: null,
children: [],
internal: {
type: nodeType,
mediaType: 'text/html',
content: cacheableContent,
contentDigest: createContentDigest(cacheableContent),
},
};
createNode(campaignNode);
}

return campaignsData.data.total_items;
};
173 changes: 51 additions & 122 deletions src/gatsby-node.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import axios, { AxiosPromise } from 'axios';
import { validateConfig } from './configValidation';
import { colorizeLog, consoleColors } from './helpers';
import { colorizeLog } from './helpers';
import { fetchCampaigns } from './fetchCampaigns';

export interface IPluginOptions {
plugins: any[];
Expand All @@ -25,12 +25,20 @@ const defaultCampaignsFields = [
const defaultContentFields = ['html'];
const defaultNodeType = 'MailchimpCampaign';
const defaultCount = 30;
const concurrReq = 9;

export interface IAuthParams {
withCredentials: boolean;
auth: {
username: string;
password: string;
};
}

export const sourceNodes = async (
{ actions, cache, createContentDigest }: any,
configOptions: IPluginOptions
) => {
const { createNode, touchNode } = actions;
const {
rootURL,
key,
Expand All @@ -57,137 +65,58 @@ export const sourceNodes = async (
...(configOptions.campaignFields || defaultCampaignsFields),
'campaigns.id',
'total_items',
].join(',');
];

const campaignsURL = `${rootURL}/campaigns`;
const authParams = {
const authParams: IAuthParams = {
withCredentials: true,
auth: {
username: authUsername,
password: key,
},
};
const baseFetchArgs = {
// In case the users passess a number bigger than 0 and lower than
// concurrReq, we want to use that
concurrReq: count && count < concurrReq && count !== 0 ? count : concurrReq,
rootURL,
authParams,
nodeType,
campaignFields,
contentFields,
actions,
cache,
createContentDigest,
};

console.time(colorizeLog('\nMailchimp campaigns fetched in'));
let campaigns = [];
const campaignsFirstBatch = await axios.get(campaignsURL, {
...authParams,
params: {
fields: campaignFields,
count: count || defaultCount,
sort_field: 'send_time',
sort_dir: 'DESC',
},
});

const { data } = campaignsFirstBatch;

if (data.campaigns) {
campaigns = [...campaignsFirstBatch.data.campaigns];
}

if (data.total_items > campaigns.length && count === 0) {
const reqLength = campaigns.length;
const extraTimesToFetch = Math.ceil(data.total_items / reqLength);
const reqArray = Array.from(
{ length: extraTimesToFetch },
(v, i) => i * reqLength
);
for (const t of reqArray) {
const newBatch = await axios.get(campaignsURL, {
...authParams,
params: {
status: 'sent',
offset: t,
fields: campaignFields,
count: count || defaultCount,
sort_field: 'send_time',
sort_dir: 'DESC',
},
});
campaigns = [...campaigns, ...newBatch.data.campaigns];
}
}
console.timeEnd(colorizeLog('\nMailchimp campaigns fetched in'));

let campaignRequests: any[] = [];
let campaignsMetadata: any[] = [];
console.time(colorizeLog('\nMailchimp campaign content fetched in'));
for (const c of campaigns) {
if (c.id === undefined) {
console.log(
`${colorizeLog("A campaign couldn't be fetched", consoleColors.BgRed)}${
c.settings && c.settings.subject_line
? `: ${c.settings.subject_line}`
: ''
}`
);
continue;
}

const internalId = `mailchimp-campaign-${c.id}`;
const cacheableContent = JSON.stringify(c);
const cachedCampaign = await cache.get(internalId);

// Make sure the campaign metadata is the same as the one just
// fetch from Mailchimp. If so, touch the node and don't mind about
// fetching the content again in order to save some build time
if (cachedCampaign && cachedCampaign.content === cacheableContent) {
touchNode({ nodeId: internalId });
continue;
}

// Define the campaign's content request
const contentURL = `${campaignsURL}/${c.id}/content`;
campaignRequests = [
...campaignRequests,
axios.get(contentURL, {
...authParams,
params: {
fields: contentFields.join(','),
},
}),
];
campaignsMetadata = [...campaignsMetadata, c];
// Save the first batch and the total number of campaigns
const totalItems = await fetchCampaigns({
...baseFetchArgs,
offset: 0,
});
// The actual number of campaigns we'll fetch is equivalent to the total
// number of items if user specifies count = 0 or (count || default)
const actualCount = count === 0 ? totalItems : count || defaultCount;

// If we have more to fetch than what we already did, then we'll want
// to fetch more and re-run the fetchCampaigns again
if (actualCount < concurrReq) {
console.time(colorizeLog('\nMailchimp campaigns fetched in'));
return;
}

let campaignsContent: any = [];

const concurrReq = 3;
const reqSegments = Array.from(
{ length: Math.ceil(campaignRequests.length / concurrReq) },
(v, i) => i * concurrReq
// Create an array with a length equivalent to the number of times we
// still have to run the fetchCampaigns function
const iterable = Array.from(
{ length: Math.ceil(actualCount / concurrReq) },
(v, i) => i + 1
);
for (const t of reqSegments) {
const requests = campaignRequests.slice(t, t + concurrReq);
const newContent = await Promise.all(requests);
campaignsContent = [...campaignsContent, newContent];
}

for (let i = 0; i < campaignsContent.length; i++) {
const meta = campaignsMetadata[i];
const content = campaignsContent[i];

const internalId = `mailchimp-campaign-${meta.id}`;
const cacheableContent = JSON.stringify(meta);
await cache.set(internalId, { content: cacheableContent });

const campaignNode = {
...meta,
...content.data,
campaignId: meta.id,
// meta information for the node
id: internalId,
parent: null,
children: [],
internal: {
type: nodeType,
mediaType: 'text/html',
content: cacheableContent,
contentDigest: createContentDigest(cacheableContent),
},
};
createNode(campaignNode);
for (const i of iterable) {
await fetchCampaigns({
...baseFetchArgs,
offset: i * concurrReq,
});
}
console.timeEnd(colorizeLog('\nMailchimp campaign content fetched in'));
console.timeEnd(colorizeLog('\nMailchimp campaigns fetched in'));
};

0 comments on commit 9e4b833

Please sign in to comment.