Skip to content

Commit

Permalink
Merge pull request #75 from einaregilsson/exponential-backoff
Browse files Browse the repository at this point in the history 
Exponential backoff for all AWS API calls!
  • Loading branch information
Einar Egilsson authored Dec 16, 2021
2 parents 2fcd415 + 7a235e2 commit 6f594de
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 166 deletions.
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
run: zip -r deploy.zip . -x '*.git*'

- name: Deploy to EB
uses: einaregilsson/beanstalk-deploy@v19
uses: einaregilsson/beanstalk-deploy@v20
with:
aws_access_key: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws_secret_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
Expand All @@ -48,7 +48,7 @@ attempt to deploy that. In the example below the action would attempt to deploy

```yaml
- name: Deploy to EB
uses: einaregilsson/beanstalk-deploy@v19
uses: einaregilsson/beanstalk-deploy@v20
with:
aws_access_key: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws_secret_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
Expand Down
111 changes: 72 additions & 39 deletions aws-api-request.js
View file
Original file line number Diff line number Diff line change
@@ -1,55 +1,54 @@
const crypto = require('crypto'),
https = require('https'),
zlib = require('zlib');
const { encode } = require('punycode');

function awsApiRequest(options) {
return new Promise((resolve, reject) => {
function awsApiRequest(options, retryAttempt = 0) {
return new Promise((resolve, reject) => {
let region = options.region || awsApiRequest.region || process.env.AWS_DEFAULT_REGION,
service = options.service,
accessKey = options.accessKey || awsApiRequest.accessKey || process.env.AWS_ACCESS_KEY_ID,
secretKey = options.secretKey || awsApiRequest.secretKey || process.env.AWS_SECRET_ACCESS_KEY,
sessionToken = options.sessionToken || awsApiRequest.sessionToken || process.env.AWS_SESSION_TOKEN,
method = options.method || 'GET',
path = options.path || '/',
querystring = options.querystring || {},
querystring = options.querystring || {},
payload = options.payload || '',
host = options.host || `${service}.${region}.amazonaws.com`,
headers = options.headers || {};

if (region.match(/^cn-/)) {
host += '.cn'; //Special case for AWS China...
}
if (region.match(/^cn-/)) {
host += '.cn'; //Special case for AWS China...
}

function hmacSha256(data, key, hex=false) {
function hmacSha256(data, key, hex = false) {
return crypto.createHmac('sha256', key).update(data).digest(hex ? 'hex' : undefined);
}

function sha256(data) {
return crypto.createHash('sha256').update(data).digest('hex');
}

//Thanks to https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-javascript
function createSigningKey(secretKey, dateStamp, region, serviceName) {
function createSigningKey(secretKey, dateStamp, region, serviceName) {
let kDate = hmacSha256(dateStamp, 'AWS4' + secretKey);
let kRegion = hmacSha256(region, kDate);
let kService = hmacSha256(serviceName, kRegion);
let kSigning = hmacSha256('aws4_request', kService);
return kSigning;
}

function createSignedHeaders(headers) {
return Object.keys(headers).sort().map(h => h.toLowerCase()).join(';');
}

function createStringToSign(timestamp, region, service, canonicalRequest) {
let stringToSign = 'AWS4-HMAC-SHA256\n';
stringToSign += timestamp + '\n';
stringToSign += timestamp.substr(0,8) + '/' + region + '/' + service + '/aws4_request\n';
stringToSign += timestamp.substr(0, 8) + '/' + region + '/' + service + '/aws4_request\n';
stringToSign += sha256(canonicalRequest);
return stringToSign;
}

function createCanonicalRequest(method, path, querystring, headers, payload) {
let canonical = method + '\n';

Expand All @@ -60,46 +59,46 @@ function awsApiRequest(options) {
//Unencoded parentheses in the path is valid. However, they must be encoded in the canonical path to pass signature verification even if
//the actual path has them unencoded.
canonical += encodeURI(path).replace(/\(/g, '%28').replace(/\)/g, '%29') + '\n';

let qsKeys = Object.keys(querystring);
qsKeys.sort();

//encodeURIComponent does NOT encode ', but we need it to be encoded. escape() is considered deprecated, so encode '
//manually. Also, using escape fails for some reason.
function encodeValue(v) {
return encodeURIComponent(v).replace(/'/g,'%27').replace(/:/g, '%3A').replace(/\(/g, '%28').replace(/\)/g, '%29').replace(/!/g, '%21').replace(/\*/g, '%2A');
return encodeURIComponent(v).replace(/'/g, '%27').replace(/:/g, '%3A').replace(/\(/g, '%28').replace(/\)/g, '%29').replace(/!/g, '%21').replace(/\*/g, '%2A');
}

let qsEntries = qsKeys.map(k => `${k}=${encodeValue(querystring[k])}`);
canonical += qsEntries.join('&') + '\n';

let headerKeys = Object.keys(headers).sort();
let headerEntries = headerKeys.map(h => h.toLowerCase() + ':' + headers[h].replace(/^\s*|\s*$/g, '').replace(' +', ' '));
canonical += headerEntries.join('\n') + '\n\n';

canonical += createSignedHeaders(headers) + '\n';
canonical += sha256(payload);

return canonical;
}

function createAuthHeader(accessKey, timestamp, region, service, headers, signature) {
let date = timestamp.substr(0,8);
let date = timestamp.substr(0, 8);
let signedHeaders = createSignedHeaders(headers);
return `AWS4-HMAC-SHA256 Credential=${accessKey}/${date}/${region}/${service}/aws4_request, SignedHeaders=${signedHeaders}, Signature=${signature}`;
}

let timestamp = new Date().toISOString().replace(/(-|:|\.\d\d\d)/g, ''); // YYYYMMDD'T'HHmmSS'Z'
let datestamp = timestamp.substr(0,8);
let datestamp = timestamp.substr(0, 8);

let sessionTokenHeader = sessionToken ? {'x-amz-security-token': sessionToken} : {};
let sessionTokenHeader = sessionToken ? { 'x-amz-security-token': sessionToken } : {};

let reqHeaders = Object.assign({
Accept : 'application/json',
Host : host,
'Content-Type' : 'application/json',
'x-amz-date' : timestamp,
'x-amz-content-sha256' : sha256(payload)
Accept: 'application/json',
Host: host,
'Content-Type': 'application/json',
'x-amz-date': timestamp,
'x-amz-content-sha256': sha256(payload)
}, sessionTokenHeader, headers); // Passed in headers override these...

let canonicalRequest = createCanonicalRequest(method, path, querystring, reqHeaders, payload);
Expand All @@ -110,8 +109,10 @@ function awsApiRequest(options) {

reqHeaders.Authorization = authHeader;

const MAX_RETRY_COUNT = 10;

//Now, lets finally do a HTTP REQUEST!!!
request(method, encodeURI(path), reqHeaders, querystring, payload, (err, result) => {
request(method, encodeURI(path), reqHeaders, querystring, payload, retryAttempt, (err, result) => {
if (err) {
reject(err);
} else {
Expand All @@ -122,6 +123,33 @@ function awsApiRequest(options) {
...options,
host: url.hostname
}));
} else if (wasThrottled(result)) {
//Exponential backoff with a 500ms jitter
let timeout = Math.pow(2, retryAttempt) * 100 + Math.floor(Math.random() * 500);
//Exponential backoff...
//2~0 * 100 = 100
//2~1 * 100 = 200
//2~2 * 100 = 400
//2~3 * 100 = 800
//2~4 * 100 = 1600
//2~5 * 100 = 3200
//2~6 * 100 = 6400
//2~7 * 100 = 12800
//2~8 * 100 = 25600
//2~9 * 100 = 51200

if (retryAttempt > MAX_RETRY_COUNT) {
//Give them the error result, the caller can then deal with it...
console.warn(`Retry attempt exceeded max retry count (${MAX_RETRY_COUNT})... Giving up...`);
resolve(result);
return;
}
if (querystring.Operation) {
console.warn(`Request for ${querystring.Operation} in ${options.service} was throttled (retry attempt ${retryAttempt}). Retrying in ${timeout}ms...`);
} else {
console.warn(`Request for service "${options.service}, path "${options.path}", method "${options.method}" was throttled (retry attempt ${retryAttempt}). Retrying in ${timeout}ms...`);
}
setTimeout(() => resolve(awsApiRequest(options, retryAttempt + 1)), timeout);
} else {
resolve(result);
}
Expand All @@ -130,29 +158,34 @@ function awsApiRequest(options) {
});
}

function createResult(data, res) {
function wasThrottled(result) {
return result.statusCode === 400 && result.data && result.data.Error && result.data.Error.Code === 'Throttling';
}

function createResult(data, res) {
if (!data || data.length === 0) {
return { statusCode: res.statusCode, headers: res.headers, data:''};
return { statusCode: res.statusCode, headers: res.headers, data: '' };
}
if (data && data.length > 0 && res.headers['content-type'] === 'application/json') {
return { statusCode : res.statusCode, headers: res.headers, data : JSON.parse(data)};
return { statusCode: res.statusCode, headers: res.headers, data: JSON.parse(data) };
} else {
return { statusCode : res.statusCode, headers: res.headers, data};
return { statusCode: res.statusCode, headers: res.headers, data };
}
}

function request(method, path, headers, querystring, data, callback) {
function request(method, path, headers, querystring, data, retryAttempt, callback) {

let qs = Object.keys(querystring).map(k => `${k}=${encodeURIComponent(querystring[k])}`).join('&');
path += '?' + qs;
let hostname = headers.Host;
delete headers.Host;
headers['Content-Length'] = data.length;
const port = 443;

try {
const options = { hostname, port, path, method, headers };
const req = https.request(options, res => {

let chunks = [];
res.on('data', d => chunks.push(d));
res.on('end', () => {
Expand All @@ -169,15 +202,15 @@ function request(method, path, headers, querystring, data, callback) {
callback(null, createResult(buffer, res));
}
});

});
req.on('error', err => callback(err));

if (data) {
req.write(data);
}
req.end();
} catch(err) {
} catch (err) {
callback(err);
}
}
Expand Down
Loading

0 comments on commit 6f594de

Please sign in to comment.