diff --git a/README.md b/README.md index 046e27d..94ba234 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ jobs: run: zip -r deploy.zip . -x '*.git*' - name: Deploy to EB - uses: einaregilsson/beanstalk-deploy@v19 + uses: einaregilsson/beanstalk-deploy@v20 with: aws_access_key: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -48,7 +48,7 @@ attempt to deploy that. In the example below the action would attempt to deploy ```yaml - name: Deploy to EB - uses: einaregilsson/beanstalk-deploy@v19 + uses: einaregilsson/beanstalk-deploy@v20 with: aws_access_key: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/aws-api-request.js b/aws-api-request.js index 1deeced..44ad140 100644 --- a/aws-api-request.js +++ b/aws-api-request.js @@ -1,10 +1,9 @@ const crypto = require('crypto'), https = require('https'), zlib = require('zlib'); -const { encode } = require('punycode'); -function awsApiRequest(options) { - return new Promise((resolve, reject) => { +function awsApiRequest(options, retryAttempt = 0) { + return new Promise((resolve, reject) => { let region = options.region || awsApiRequest.region || process.env.AWS_DEFAULT_REGION, service = options.service, accessKey = options.accessKey || awsApiRequest.accessKey || process.env.AWS_ACCESS_KEY_ID, @@ -12,44 +11,44 @@ function awsApiRequest(options) { sessionToken = options.sessionToken || awsApiRequest.sessionToken || process.env.AWS_SESSION_TOKEN, method = options.method || 'GET', path = options.path || '/', - querystring = options.querystring || {}, + querystring = options.querystring || {}, payload = options.payload || '', host = options.host || `${service}.${region}.amazonaws.com`, headers = options.headers || {}; - if (region.match(/^cn-/)) { - host += '.cn'; //Special case for AWS China... - } + if (region.match(/^cn-/)) { + host += '.cn'; //Special case for AWS China... + } - function hmacSha256(data, key, hex=false) { + function hmacSha256(data, key, hex = false) { return crypto.createHmac('sha256', key).update(data).digest(hex ? 'hex' : undefined); } - + function sha256(data) { return crypto.createHash('sha256').update(data).digest('hex'); } - + //Thanks to https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-javascript - function createSigningKey(secretKey, dateStamp, region, serviceName) { + function createSigningKey(secretKey, dateStamp, region, serviceName) { let kDate = hmacSha256(dateStamp, 'AWS4' + secretKey); let kRegion = hmacSha256(region, kDate); let kService = hmacSha256(serviceName, kRegion); let kSigning = hmacSha256('aws4_request', kService); return kSigning; } - + function createSignedHeaders(headers) { return Object.keys(headers).sort().map(h => h.toLowerCase()).join(';'); } - + function createStringToSign(timestamp, region, service, canonicalRequest) { let stringToSign = 'AWS4-HMAC-SHA256\n'; stringToSign += timestamp + '\n'; - stringToSign += timestamp.substr(0,8) + '/' + region + '/' + service + '/aws4_request\n'; + stringToSign += timestamp.substr(0, 8) + '/' + region + '/' + service + '/aws4_request\n'; stringToSign += sha256(canonicalRequest); return stringToSign; } - + function createCanonicalRequest(method, path, querystring, headers, payload) { let canonical = method + '\n'; @@ -60,46 +59,46 @@ function awsApiRequest(options) { //Unencoded parentheses in the path is valid. However, they must be encoded in the canonical path to pass signature verification even if //the actual path has them unencoded. canonical += encodeURI(path).replace(/\(/g, '%28').replace(/\)/g, '%29') + '\n'; - + let qsKeys = Object.keys(querystring); qsKeys.sort(); //encodeURIComponent does NOT encode ', but we need it to be encoded. escape() is considered deprecated, so encode ' //manually. Also, using escape fails for some reason. function encodeValue(v) { - return encodeURIComponent(v).replace(/'/g,'%27').replace(/:/g, '%3A').replace(/\(/g, '%28').replace(/\)/g, '%29').replace(/!/g, '%21').replace(/\*/g, '%2A'); + return encodeURIComponent(v).replace(/'/g, '%27').replace(/:/g, '%3A').replace(/\(/g, '%28').replace(/\)/g, '%29').replace(/!/g, '%21').replace(/\*/g, '%2A'); } let qsEntries = qsKeys.map(k => `${k}=${encodeValue(querystring[k])}`); canonical += qsEntries.join('&') + '\n'; - + let headerKeys = Object.keys(headers).sort(); let headerEntries = headerKeys.map(h => h.toLowerCase() + ':' + headers[h].replace(/^\s*|\s*$/g, '').replace(' +', ' ')); canonical += headerEntries.join('\n') + '\n\n'; - + canonical += createSignedHeaders(headers) + '\n'; canonical += sha256(payload); - + return canonical; } - + function createAuthHeader(accessKey, timestamp, region, service, headers, signature) { - let date = timestamp.substr(0,8); + let date = timestamp.substr(0, 8); let signedHeaders = createSignedHeaders(headers); return `AWS4-HMAC-SHA256 Credential=${accessKey}/${date}/${region}/${service}/aws4_request, SignedHeaders=${signedHeaders}, Signature=${signature}`; } let timestamp = new Date().toISOString().replace(/(-|:|\.\d\d\d)/g, ''); // YYYYMMDD'T'HHmmSS'Z' - let datestamp = timestamp.substr(0,8); + let datestamp = timestamp.substr(0, 8); - let sessionTokenHeader = sessionToken ? {'x-amz-security-token': sessionToken} : {}; + let sessionTokenHeader = sessionToken ? { 'x-amz-security-token': sessionToken } : {}; let reqHeaders = Object.assign({ - Accept : 'application/json', - Host : host, - 'Content-Type' : 'application/json', - 'x-amz-date' : timestamp, - 'x-amz-content-sha256' : sha256(payload) + Accept: 'application/json', + Host: host, + 'Content-Type': 'application/json', + 'x-amz-date': timestamp, + 'x-amz-content-sha256': sha256(payload) }, sessionTokenHeader, headers); // Passed in headers override these... let canonicalRequest = createCanonicalRequest(method, path, querystring, reqHeaders, payload); @@ -110,8 +109,10 @@ function awsApiRequest(options) { reqHeaders.Authorization = authHeader; + const MAX_RETRY_COUNT = 10; + //Now, lets finally do a HTTP REQUEST!!! - request(method, encodeURI(path), reqHeaders, querystring, payload, (err, result) => { + request(method, encodeURI(path), reqHeaders, querystring, payload, retryAttempt, (err, result) => { if (err) { reject(err); } else { @@ -122,6 +123,33 @@ function awsApiRequest(options) { ...options, host: url.hostname })); + } else if (wasThrottled(result)) { + //Exponential backoff with a 500ms jitter + let timeout = Math.pow(2, retryAttempt) * 100 + Math.floor(Math.random() * 500); + //Exponential backoff... + //2~0 * 100 = 100 + //2~1 * 100 = 200 + //2~2 * 100 = 400 + //2~3 * 100 = 800 + //2~4 * 100 = 1600 + //2~5 * 100 = 3200 + //2~6 * 100 = 6400 + //2~7 * 100 = 12800 + //2~8 * 100 = 25600 + //2~9 * 100 = 51200 + + if (retryAttempt > MAX_RETRY_COUNT) { + //Give them the error result, the caller can then deal with it... + console.warn(`Retry attempt exceeded max retry count (${MAX_RETRY_COUNT})... Giving up...`); + resolve(result); + return; + } + if (querystring.Operation) { + console.warn(`Request for ${querystring.Operation} in ${options.service} was throttled (retry attempt ${retryAttempt}). Retrying in ${timeout}ms...`); + } else { + console.warn(`Request for service "${options.service}, path "${options.path}", method "${options.method}" was throttled (retry attempt ${retryAttempt}). Retrying in ${timeout}ms...`); + } + setTimeout(() => resolve(awsApiRequest(options, retryAttempt + 1)), timeout); } else { resolve(result); } @@ -130,29 +158,34 @@ function awsApiRequest(options) { }); } -function createResult(data, res) { +function wasThrottled(result) { + return result.statusCode === 400 && result.data && result.data.Error && result.data.Error.Code === 'Throttling'; +} + +function createResult(data, res) { if (!data || data.length === 0) { - return { statusCode: res.statusCode, headers: res.headers, data:''}; + return { statusCode: res.statusCode, headers: res.headers, data: '' }; } if (data && data.length > 0 && res.headers['content-type'] === 'application/json') { - return { statusCode : res.statusCode, headers: res.headers, data : JSON.parse(data)}; + return { statusCode: res.statusCode, headers: res.headers, data: JSON.parse(data) }; } else { - return { statusCode : res.statusCode, headers: res.headers, data}; + return { statusCode: res.statusCode, headers: res.headers, data }; } } -function request(method, path, headers, querystring, data, callback) { - +function request(method, path, headers, querystring, data, retryAttempt, callback) { + let qs = Object.keys(querystring).map(k => `${k}=${encodeURIComponent(querystring[k])}`).join('&'); path += '?' + qs; let hostname = headers.Host; delete headers.Host; headers['Content-Length'] = data.length; const port = 443; + try { const options = { hostname, port, path, method, headers }; const req = https.request(options, res => { - + let chunks = []; res.on('data', d => chunks.push(d)); res.on('end', () => { @@ -169,7 +202,7 @@ function request(method, path, headers, querystring, data, callback) { callback(null, createResult(buffer, res)); } }); - + }); req.on('error', err => callback(err)); @@ -177,7 +210,7 @@ function request(method, path, headers, querystring, data, callback) { req.write(data); } req.end(); - } catch(err) { + } catch (err) { callback(err); } } diff --git a/beanstalk-deploy.js b/beanstalk-deploy.js index 1017348..fae9a91 100755 --- a/beanstalk-deploy.js +++ b/beanstalk-deploy.js @@ -11,19 +11,19 @@ if (IS_GITHUB_ACTION) { console.warn = msg => console.log(`::warning::${msg}`); } -function createStorageLocation() { +function createStorageLocation() { return awsApiRequest({ service: 'elasticbeanstalk', - querystring: {Operation: 'CreateStorageLocation', Version: '2010-12-01'} + querystring: { Operation: 'CreateStorageLocation', Version: '2010-12-01' } }); } function checkIfFileExistsInS3(bucket, s3Key) { return awsApiRequest({ - service : 's3', + service: 's3', host: `${bucket}.s3.${awsApiRequest.region}.amazonaws.com`, - path : s3Key, + path: s3Key, method: 'HEAD' }); } @@ -31,7 +31,7 @@ function checkIfFileExistsInS3(bucket, s3Key) { function readFile(path) { return new Promise((resolve, reject) => { fs.readFile(path, (err, data) => { - if (err) { + if (err) { reject(err); } resolve(data); @@ -41,11 +41,11 @@ function readFile(path) { function uploadFileToS3(bucket, s3Key, filebuffer) { return awsApiRequest({ - service : 's3', + service: 's3', host: `${bucket}.s3.${awsApiRequest.region}.amazonaws.com`, - path : s3Key, + path: s3Key, method: 'PUT', - headers: { 'Content-Type' : 'application/octet-stream'}, + headers: { 'Content-Type': 'application/octet-stream' }, payload: filebuffer }); } @@ -54,13 +54,13 @@ function createBeanstalkVersion(application, bucket, s3Key, versionLabel, versio return awsApiRequest({ service: 'elasticbeanstalk', querystring: { - Operation: 'CreateApplicationVersion', + Operation: 'CreateApplicationVersion', Version: '2010-12-01', - ApplicationName : application, - VersionLabel : versionLabel, - Description : versionDescription, - 'SourceBundle.S3Bucket' : bucket, - 'SourceBundle.S3Key' : s3Key.substr(1) //Don't want leading / here + ApplicationName: application, + VersionLabel: versionLabel, + Description: versionDescription, + 'SourceBundle.S3Bucket': bucket, + 'SourceBundle.S3Key': s3Key.substr(1) //Don't want leading / here } }); } @@ -69,11 +69,11 @@ function deployBeanstalkVersion(application, environmentName, versionLabel) { return awsApiRequest({ service: 'elasticbeanstalk', querystring: { - Operation: 'UpdateEnvironment', + Operation: 'UpdateEnvironment', Version: '2010-12-01', - ApplicationName : application, - EnvironmentName : environmentName, - VersionLabel : versionLabel + ApplicationName: application, + EnvironmentName: environmentName, + VersionLabel: versionLabel } }); } @@ -82,12 +82,12 @@ function describeEvents(application, environmentName, startTime) { return awsApiRequest({ service: 'elasticbeanstalk', querystring: { - Operation: 'DescribeEvents', + Operation: 'DescribeEvents', Version: '2010-12-01', - ApplicationName : application, - Severity : 'TRACE', - EnvironmentName : environmentName, - StartTime : startTime.toISOString().replace(/(-|:|\.\d\d\d)/g, '') + ApplicationName: application, + Severity: 'TRACE', + EnvironmentName: environmentName, + StartTime: startTime.toISOString().replace(/(-|:|\.\d\d\d)/g, '') } }); } @@ -96,10 +96,10 @@ function describeEnvironments(application, environmentName) { return awsApiRequest({ service: 'elasticbeanstalk', querystring: { - Operation: 'DescribeEnvironments', + Operation: 'DescribeEnvironments', Version: '2010-12-01', - ApplicationName : application, - 'EnvironmentNames.members.1' : environmentName //Yes, that's the horrible way to pass an array... + ApplicationName: application, + 'EnvironmentNames.members.1': environmentName //Yes, that's the horrible way to pass an array... } }); } @@ -108,16 +108,24 @@ function getApplicationVersion(application, versionLabel) { return awsApiRequest({ service: 'elasticbeanstalk', querystring: { - Operation: 'DescribeApplicationVersions', + Operation: 'DescribeApplicationVersions', Version: '2010-12-01', - ApplicationName : application, - 'VersionLabels.members.1' : versionLabel //Yes, that's the horrible way to pass an array... + ApplicationName: application, + 'VersionLabels.members.1': versionLabel //Yes, that's the horrible way to pass an array... } }); } function expect(status, result, extraErrorMessage) { - if (status !== result.statusCode) {  + if (!result) { + throw new Error(`Null result received when expecting ${status}`); + } + + if (!result.statusCode) { + throw new Error(`Provided result ${result} is missing a status code when expecting ${status}`); + } + + if (status !== result.statusCode) { if (extraErrorMessage) { console.log(extraErrorMessage); } @@ -158,13 +166,13 @@ function deployNewVersion(application, environmentName, versionLabel, versionDes if (result.statusCode === 200) { throw new Error(`Version ${versionLabel} already exists in S3!`); } - expect(404, result); + expect(404, result); return uploadFileToS3(bucket, s3Key, fileBuffer); }).then(result => { expect(200, result); console.log(`New build successfully uploaded to S3, bucket=${bucket}, key=${s3Key}`); return createBeanstalkVersion(application, bucket, s3Key, versionLabel, versionDescription); - }).then(result => { + }).then(result => { expect(200, result); console.log(`Created new application version ${versionLabel} in Beanstalk.`); if (!environmentName) { @@ -197,39 +205,17 @@ function deployNewVersion(application, environmentName, versionLabel, versionDes }).catch(err => { console.error(`Deployment failed: ${err}`); process.exit(2); - }); -} - -function wasThrottled(result) { - return result.statusCode === 400 && result.data && result.data.Error && result.data.Error.Code === 'Throttling'; + }); } -var deployVersionConsecutiveThrottlingErrors = 0; //Deploys existing version in EB function deployExistingVersion(application, environmentName, versionLabel, waitUntilDeploymentIsFinished, waitForRecoverySeconds) { let deployStart = new Date(); console.log(`Deploying existing version ${versionLabel}`); - deployBeanstalkVersion(application, environmentName, versionLabel).then(result => { - if (result.statusCode !== 200) {  - if (result.headers['content-type'] !== 'application/json') { //Not something we know how to handle ... - throw new Error(`Status: ${result.statusCode}. Message: ${result.data}`); - } else if (wasThrottled(result)) { - deployVersionConsecutiveThrottlingErrors++; - - if (deployVersionConsecutiveThrottlingErrors >= 5) { - throw new Error(`Deployment failed, got ${deployVersionConsecutiveThrottlingErrors} throttling errors in a row while deploying existing version.`); - } else { - return new Promise((resolve, reject) => { - reject({Code: 'Throttled'}); - }); - } - } else { - throw new Error(`Status: ${result.statusCode}. Code: ${result.data.Error.Code}, Message: ${result.data.Error.Message}`); - } - } + expect(200, result, "Failed to deploy an existing version"); if (waitUntilDeploymentIsFinished) { console.log('Deployment started, "wait_for_deployment" was true...\n'); @@ -248,15 +234,9 @@ function deployExistingVersion(application, environmentName, versionLabel, waitU process.exit(1); } }).catch(err => { - - if (err.Code === 'Throttled') { - console.log(`Call to deploy version was throttled. Waiting for 10 seconds before trying again ...`); - setTimeout(() => deployExistingVersion(application, environmentName, versionLabel, waitUntilDeploymentIsFinished, waitForRecoverySeconds), 10 * 1000); - } else { - console.error(`Deployment failed: ${err}`); - process.exit(2); - } - }); + console.error(`Deployment failed: ${err}`); + process.exit(2); + }); } @@ -267,15 +247,15 @@ function strip(val) { function main() { - let application, - environmentName, + let application, + environmentName, versionLabel, versionDescription, - region, + region, file, existingBucketName = null, - useExistingVersionIfAvailable, - waitForRecoverySeconds = 30, + useExistingVersionIfAvailable, + waitForRecoverySeconds = 30, waitUntilDeploymentIsFinished = true; //Whether or not to wait for the deployment to complete... if (IS_GITHUB_ACTION) { //Running in GitHub Actions @@ -385,29 +365,21 @@ function main() { console.log(`Deploying existing version ${versionLabel}, version info:`); console.log(JSON.stringify(versionsList[0], null, 2)); deployExistingVersion(application, environmentName, versionLabel, waitUntilDeploymentIsFinished, waitForRecoverySeconds); - } + } } else { if (file) { deployNewVersion(application, environmentName, versionLabel, versionDescription, file, existingBucketName, waitUntilDeploymentIsFinished, waitForRecoverySeconds); - } else { + } else { console.error(`Deployment failed: No deployment package given but version ${versionLabel} doesn't exist, so nothing to deploy!`); process.exit(2); - } - } + } + } }).catch(err => { console.error(`Deployment failed: ${err}`); process.exit(2); }); } -function formatTimespan(since) { - let elapsed = new Date().getTime() - since; - let seconds = Math.floor(elapsed / 1000); - let minutes = Math.floor(seconds / 60); - seconds -= (minutes * 60); - return `${minutes}m${seconds}s`; -} - //Wait until the new version is deployed, printing any events happening during the wait... function waitForDeployment(application, environmentName, versionLabel, start, waitForRecoverySeconds) { let counter = 0; @@ -421,15 +393,11 @@ function waitForDeployment(application, environmentName, versionLabel, start, wa let waitPeriod = 10 * SECOND; //Start at ten seconds, increase slowly, long deployments have been erroring with too many requests. let waitStart = new Date().getTime(); - let eventCalls = 0, environmentCalls = 0; // Getting throttled on these print out how many we're doing... - - let consecutiveThrottleErrors = 0; - return new Promise((resolve, reject) => { function update() { let elapsed = new Date().getTime() - waitStart; - + //Limit update requests for really long deploys if (elapsed > (10 * MINUTE)) { waitPeriod = 30 * SECOND; @@ -438,59 +406,33 @@ function waitForDeployment(application, environmentName, versionLabel, start, wa } describeEvents(application, environmentName, start).then(result => { - eventCalls++; - - - //Allow a few throttling failures... - if (wasThrottled(result)) { - consecutiveThrottleErrors++; - console.log(`Request to DescribeEvents was throttled, that's ${consecutiveThrottleErrors} throttle errors in a row...`); - return; - } - consecutiveThrottleErrors = 0; //Reset the throttling count - - expect(200, result, `Failed in call to describeEvents, have done ${eventCalls} calls to describeEvents, ${environmentCalls} calls to describeEnvironments in ${formatTimespan(waitStart)}`); + expect(200, result); let events = result.data.DescribeEventsResponse.DescribeEventsResult.Events.reverse(); //They show up in desc, we want asc for logging... for (let ev of events) { let date = new Date(ev.EventDate * 1000); //Seconds to milliseconds, - console.log(`${date.toISOString().substr(11,8)} ${ev.Severity}: ${ev.Message}`); - if (ev.Message.match(/Failed to deploy application/)) { + console.log(`${date.toISOString().substr(11, 8)} ${ev.Severity}: ${ev.Message}`); + if (ev.Message.match(/Failed to deploy application/)) { deploymentFailed = true; //wait until next iteration to finish, to get the final messages... } } if (events.length > 0) { - start = new Date(events[events.length-1].EventDate * 1000 + 1000); //Add extra second so we don't get the same message next time... + start = new Date(events[events.length - 1].EventDate * 1000 + 1000); //Add extra second so we don't get the same message next time... } }).catch(reject); - - describeEnvironments(application, environmentName).then(result => { - environmentCalls++; - - //Allow a few throttling failures... - if (wasThrottled(result)) { - consecutiveThrottleErrors++; - console.log(`Request to DescribeEnvironments was throttled, that's ${consecutiveThrottleErrors} throttle errors in a row...`); - if (consecutiveThrottleErrors >= 5) { - throw new Error(`Deployment failed, got ${consecutiveThrottleErrors} throttling errors in a row while waiting for deployment`); - } - - setTimeout(update, waitPeriod); - return; - } - expect(200, result, `Failed in call to describeEnvironments, have done ${eventCalls} calls to describeEvents, ${environmentCalls} calls to describeEnvironments in ${formatTimespan(waitStart)}`); + describeEnvironments(application, environmentName).then(result => { - consecutiveThrottleErrors = 0; + expect(200, result, `Failed in call to describeEnvironments`); counter++; let env = result.data.DescribeEnvironmentsResponse.DescribeEnvironmentsResult.Environments[0]; if (env.VersionLabel === versionLabel && env.Status === 'Ready') { if (!degraded) { console.log(`Deployment finished. Version updated to ${env.VersionLabel}`); console.log(`Status for ${application}-${environmentName} is ${env.Status}, Health: ${env.Health}, HealthStatus: ${env.HealthStatus}`); - + if (env.Health === 'Green') { - resolve(env); + resolve(env); } else { console.warn(`Environment update finished, but health is ${env.Health} and health status is ${env.HealthStatus}. Giving it ${waitForRecoverySeconds} seconds to recover...`); degraded = true; @@ -513,17 +455,17 @@ function waitForDeployment(application, environmentName, versionLabel, start, wa } } else if (deploymentFailed) { let msg = `Deployment failed! Current State: Version: ${env.VersionLabel}, Health: ${env.Health}, Health Status: ${env.HealthStatus}`; - console.log(`${new Date().toISOString().substr(11,8)} ERROR: ${msg}`); + console.log(`${new Date().toISOString().substr(11, 8)} ERROR: ${msg}`); reject(new Error(msg)); } else { if (counter % 6 === 0 && !deploymentFailed) { - console.log(`${new Date().toISOString().substr(11,8)} INFO: Still updating, status is "${env.Status}", health is "${env.Health}", health status is "${env.HealthStatus}"`); + console.log(`${new Date().toISOString().substr(11, 8)} INFO: Still updating, status is "${env.Status}", health is "${env.Health}", health status is "${env.HealthStatus}"`); } setTimeout(update, waitPeriod); } }).catch(reject); } - + update(); }); } diff --git a/package.json b/package.json index 85c8614..805827a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "beanstalk-deploy", - "version": "19.0.0", + "version": "20.0.0", "description": "GitHub Action + command line tool to deploy to AWS Elastic Beanstalk.", "main": "beanstalk-deploy.js", "scripts": {