Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade to nodejs v12 #47

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .babelrc
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"presets": [
["env", {
"targets": {
"node": "6.10"
"node": "8.10"
}
}]
]
Expand Down
35 changes: 0 additions & 35 deletions chrome/buildChrome.sh

This file was deleted.

Binary file removed chrome/headless_shell-67.0.3361.0.tar.gz
Binary file not shown.
1 change: 0 additions & 1 deletion chrome/headless_shell.tar.gz

This file was deleted.

9 changes: 5 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,24 @@
"version": "1.1.2",
"description": "Starter Kit for running Headless-Chrome by Puppeteer on AWS Lambda",
"scripts": {
"package": "npm run package-prepare && cp chrome/headless_shell.tar.gz dist && cd dist && zip -rq ../package.zip .",
"package-nochrome": "npm run package-prepare && cd dist && zip -rq ../package.zip .",
"package": "npm run package-prepare && cd dist && rm -f ../package.zip && zip -rq ../package.zip .",
"package-nochrome": "npm run package-prepare && cd dist && rm -f ../package.zip && zip -rq ../package.zip .",
"package-prepare": "npm run lint && npm run babel && cp -r package.json dist && cd dist && PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 npm install --production",
"babel": "rm -rf dist && mkdir dist && ./node_modules/.bin/babel src --out-dir dist",
"local": "npm run babel && cp -r node_modules dist && node dist/starter-kit/local.js",
"lint": "./node_modules/.bin/eslint src"
},
"dependencies": {
"puppeteer": "^1.1.1",
"tar": "^4.0.1"
"chrome-aws-lambda": "~2.0.2",
"puppeteer-core": "~2.0.0"
},
"devDependencies": {
"aws-sdk": "^2.111.0",
"babel-cli": "^6.26.0",
"babel-preset-env": "^1.6.0",
"eslint": "^4.6.1",
"eslint-config-google": "^0.9.1",
"puppeteer": "~2.0.0",
"serverless-hooks-plugin": "^1.1.0"
}
}
26 changes: 15 additions & 11 deletions src/index.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,10 @@
const setup = require('./starter-kit/setup');

exports.handler = async (event, context, callback) => {
// For keeping the browser launch
context.callbackWaitsForEmptyEventLoop = false;
// this handler signature requires AWS Lambda Nodejs v8.1
exports.handler = async (event, context) => {
const browser = await setup.getBrowser();
try {
const result = await exports.run(browser);
callback(null, result);
} catch (e) {
callback(e);
}

return await exports.run(browser);
};

exports.run = async (browser) => {
Expand All @@ -21,15 +16,24 @@ exports.run = async (browser) => {
);
console.log((await page.content()).slice(0, 500));

await page.type('#lst-ib', 'aaaaa');
await page.type('input[name=q]', 'aaaaa');
// avoid to timeout waitForNavigation() after click()
await Promise.all([
// avoid to
// 'Cannot find context with specified id undefined' for localStorage
page.waitForNavigation(),
page.click('[name=btnK]'),

// puppeteer v1.3.0 seems to have a problem with SVG buttons
// when using page.click() it errors with ""
// see: https://github.com/GoogleChrome/puppeteer/issues/2977
// a workaround is to execute the click in the browser
//
// page.click("input[name=btnK]"),
page.evaluate(() => document.querySelector('input[name=btnK]').click()),
]);

console.log(`page url after search: ${page.url()}`);

/* screenshot
await page.screenshot({path: '/tmp/screenshot.png'});
const aws = require('aws-sdk');
Expand Down
15 changes: 11 additions & 4 deletions src/starter-kit/local.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
const index = require('../index');
const config = require('./config');
const puppeteer = require('puppeteer');
const chromium = require('chrome-aws-lambda');

(async () => {
const browser = await puppeteer.launch({
headless: false,
// when running locally, chrome-aws-lambda will use the full puppeteer
// that is included in the devDependencies
// see: https://github.com/alixaxel/chrome-aws-lambda/wiki/HOWTO:-Local-Development

const browser = await chromium.puppeteer.launch({
args: chromium.args,
defaultViewport: chromium.defaultViewport,
executablePath: await chromium.executablePath,
headless: chromium.headless,

slowMo: process.env.SLOWMO_MS,
dumpio: !!config.DEBUG,
// use chrome installed by puppeteer
});
await index.run(browser)
.then((result) => console.log(result))
Expand Down
77 changes: 8 additions & 69 deletions src/starter-kit/setup.js
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
const aws = require('aws-sdk');
const s3 = new aws.S3({apiVersion: '2006-03-01'});
const fs = require('fs');
const tar = require('tar');
const puppeteer = require('puppeteer');
const chromium = require('chrome-aws-lambda');
const config = require('./config');

exports.getBrowser = (() => {
let browser;
return async () => {
if (typeof browser === 'undefined' || !await isBrowserAvailable(browser)) {
await setupChrome();
browser = await puppeteer.launch({
headless: true,
executablePath: config.executablePath,
args: config.launchOptionForLambda,
dumpio: !!exports.DEBUG,
browser = await chromium.puppeteer.launch({
args: chromium.args,
defaultViewport: chromium.defaultViewport,
executablePath: await chromium.executablePath,
headless: chromium.headless,
slowMo: process.env.SLOWMO_MS,
dumpio: !!config.DEBUG,
});
debugLog(async (b) => `launch done: ${await browser.version()}`);
}
Expand All @@ -32,64 +29,6 @@ const isBrowserAvailable = async (browser) => {
return true;
};

const setupChrome = async () => {
if (!await existsExecutableChrome()) {
if (await existsLocalChrome()) {
debugLog('setup local chrome');
await setupLocalChrome();
} else {
debugLog('setup s3 chrome');
await setupS3Chrome();
}
debugLog('setup done');
}
};

const existsLocalChrome = () => {
return new Promise((resolve, reject) => {
fs.exists(config.localChromePath, (exists) => {
resolve(exists);
});
});
};

const existsExecutableChrome = () => {
return new Promise((resolve, reject) => {
fs.exists(config.executablePath, (exists) => {
resolve(exists);
});
});
};

const setupLocalChrome = () => {
return new Promise((resolve, reject) => {
fs.createReadStream(config.localChromePath)
.on('error', (err) => reject(err))
.pipe(tar.x({
C: config.setupChromePath,
}))
.on('error', (err) => reject(err))
.on('end', () => resolve());
});
};

const setupS3Chrome = () => {
return new Promise((resolve, reject) => {
const params = {
Bucket: config.remoteChromeS3Bucket,
Key: config.remoteChromeS3Key,
};
s3.getObject(params)
.createReadStream()
.on('error', (err) => reject(err))
.pipe(tar.x({
C: config.setupChromePath,
}))
.on('error', (err) => reject(err))
.on('end', () => resolve());
});
};

const debugLog = (log) => {
if (config.DEBUG) {
let message = log;
Expand Down