diff --git a/.babelrc b/.babelrc index 2f7f20c..20fb9dd 100644 --- a/.babelrc +++ b/.babelrc @@ -2,7 +2,7 @@ "presets": [ ["env", { "targets": { - "node": "6.10" + "node": "8.10" } }] ] diff --git a/chrome/buildChrome.sh b/chrome/buildChrome.sh deleted file mode 100644 index e637565..0000000 --- a/chrome/buildChrome.sh +++ /dev/null @@ -1,35 +0,0 @@ -# build headless chrome on EC2 -# https://github.com/adieuadieu/serverless-chrome/blob/master/chrome/README.md - -# sudo su - -yum install -y git redhat-lsb python bzip2 tar pkgconfig atk-devel alsa-lib-devel bison binutils brlapi-devel bluez-libs-devel bzip2-devel cairo-devel cups-devel dbus-devel dbus-glib-devel expat-devel fontconfig-devel freetype-devel gcc-c++ GConf2-devel glib2-devel glibc.i686 gperf glib2-devel gtk2-devel gtk3-devel java-1.*.0-openjdk-devel libatomic libcap-devel libffi-devel libgcc.i686 libgnome-keyring-devel libjpeg-devel libstdc++.i686 libX11-devel libXScrnSaver-devel libXtst-devel libxkbcommon-x11-devel ncurses-compat-libs nspr-devel nss-devel pam-devel pango-devel pciutils-devel pulseaudio-libs-devel zlib.i686 httpd mod_ssl php php-cli python-psutil wdiff --enablerepo=epel - -cd ~ -git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git -echo "export PATH=$PATH:$HOME/depot_tools" >> ~/.bash_profile -source ~/.bash_profile - -mkdir Chromium -cd Chromium -fetch --no-history chromium -cd src - -# use /tmp instead of /dev/shm -# https://groups.google.com/a/chromium.org/forum/#!msg/headless-dev/qqbZVZ2IwEw/CPInd55OBgAJ -sed -i -e "s/use_dev_shm = true;/use_dev_shm = false;/g" base/files/file_util_posix.cc - -mkdir -p out/Headless -echo 'import("//build/args/headless.gn")' > out/Headless/args.gn -echo 'is_debug = false' >> out/Headless/args.gn -echo 'symbol_level = 0' >> out/Headless/args.gn -echo 'is_component_build = false' >> out/Headless/args.gn -echo 'remove_webcore_debug_symbols = true' >> out/Headless/args.gn -echo 'enable_nacl = false' >> out/Headless/args.gn -gn gen out/Headless -ninja -C out/Headless headless_shell - -cd out/Headless -tar -zcvf /home/ec2-user/headless_shell.tar.gz headless_shell - -# scp ec2-user@xxx.amazonaws.com:~/headless_shell.tar.gz . diff --git a/chrome/headless_shell-67.0.3361.0.tar.gz b/chrome/headless_shell-67.0.3361.0.tar.gz deleted file mode 100644 index 28bfe62..0000000 Binary files a/chrome/headless_shell-67.0.3361.0.tar.gz and /dev/null differ diff --git a/chrome/headless_shell.tar.gz b/chrome/headless_shell.tar.gz deleted file mode 120000 index 30cad4b..0000000 --- a/chrome/headless_shell.tar.gz +++ /dev/null @@ -1 +0,0 @@ -headless_shell-67.0.3361.0.tar.gz \ No newline at end of file diff --git a/package.json b/package.json index 41ab805..61f3421 100644 --- a/package.json +++ b/package.json @@ -3,16 +3,16 @@ "version": "1.1.2", "description": "Starter Kit for running Headless-Chrome by Puppeteer on AWS Lambda", "scripts": { - "package": "npm run package-prepare && cp chrome/headless_shell.tar.gz dist && cd dist && zip -rq ../package.zip .", - "package-nochrome": "npm run package-prepare && cd dist && zip -rq ../package.zip .", + "package": "npm run package-prepare && cd dist && rm -f ../package.zip && zip -rq ../package.zip .", + "package-nochrome": "npm run package-prepare && cd dist && rm -f ../package.zip && zip -rq ../package.zip .", "package-prepare": "npm run lint && npm run babel && cp -r package.json dist && cd dist && PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 npm install --production", "babel": "rm -rf dist && mkdir dist && ./node_modules/.bin/babel src --out-dir dist", "local": "npm run babel && cp -r node_modules dist && node dist/starter-kit/local.js", "lint": "./node_modules/.bin/eslint src" }, "dependencies": { - "puppeteer": "^1.1.1", - "tar": "^4.0.1" + "chrome-aws-lambda": "~2.0.2", + "puppeteer-core": "~2.0.0" }, "devDependencies": { "aws-sdk": "^2.111.0", @@ -20,6 +20,7 @@ "babel-preset-env": "^1.6.0", "eslint": "^4.6.1", "eslint-config-google": "^0.9.1", + "puppeteer": "~2.0.0", "serverless-hooks-plugin": "^1.1.0" } } diff --git a/src/index.js b/src/index.js index d77ebe1..10522ef 100755 --- a/src/index.js +++ b/src/index.js @@ -1,15 +1,10 @@ const setup = require('./starter-kit/setup'); -exports.handler = async (event, context, callback) => { - // For keeping the browser launch - context.callbackWaitsForEmptyEventLoop = false; +// this handler signature requires AWS Lambda Nodejs v8.1 +exports.handler = async (event, context) => { const browser = await setup.getBrowser(); - try { - const result = await exports.run(browser); - callback(null, result); - } catch (e) { - callback(e); - } + + return await exports.run(browser); }; exports.run = async (browser) => { @@ -21,15 +16,24 @@ exports.run = async (browser) => { ); console.log((await page.content()).slice(0, 500)); - await page.type('#lst-ib', 'aaaaa'); + await page.type('input[name=q]', 'aaaaa'); // avoid to timeout waitForNavigation() after click() await Promise.all([ // avoid to // 'Cannot find context with specified id undefined' for localStorage page.waitForNavigation(), - page.click('[name=btnK]'), + + // puppeteer v1.3.0 seems to have a problem with SVG buttons + // when using page.click() it errors with "" + // see: https://github.com/GoogleChrome/puppeteer/issues/2977 + // a workaround is to execute the click in the browser + // + // page.click("input[name=btnK]"), + page.evaluate(() => document.querySelector('input[name=btnK]').click()), ]); + console.log(`page url after search: ${page.url()}`); + /* screenshot await page.screenshot({path: '/tmp/screenshot.png'}); const aws = require('aws-sdk'); diff --git a/src/starter-kit/local.js b/src/starter-kit/local.js index 50dae65..5c23b34 100644 --- a/src/starter-kit/local.js +++ b/src/starter-kit/local.js @@ -1,13 +1,20 @@ const index = require('../index'); const config = require('./config'); -const puppeteer = require('puppeteer'); +const chromium = require('chrome-aws-lambda'); (async () => { - const browser = await puppeteer.launch({ - headless: false, + // when running locally, chrome-aws-lambda will use the full puppeteer + // that is included in the devDependencies + // see: https://github.com/alixaxel/chrome-aws-lambda/wiki/HOWTO:-Local-Development + + const browser = await chromium.puppeteer.launch({ + args: chromium.args, + defaultViewport: chromium.defaultViewport, + executablePath: await chromium.executablePath, + headless: chromium.headless, + slowMo: process.env.SLOWMO_MS, dumpio: !!config.DEBUG, - // use chrome installed by puppeteer }); await index.run(browser) .then((result) => console.log(result)) diff --git a/src/starter-kit/setup.js b/src/starter-kit/setup.js index 35db790..148c86a 100644 --- a/src/starter-kit/setup.js +++ b/src/starter-kit/setup.js @@ -1,20 +1,17 @@ -const aws = require('aws-sdk'); -const s3 = new aws.S3({apiVersion: '2006-03-01'}); -const fs = require('fs'); -const tar = require('tar'); -const puppeteer = require('puppeteer'); +const chromium = require('chrome-aws-lambda'); const config = require('./config'); exports.getBrowser = (() => { let browser; return async () => { if (typeof browser === 'undefined' || !await isBrowserAvailable(browser)) { - await setupChrome(); - browser = await puppeteer.launch({ - headless: true, - executablePath: config.executablePath, - args: config.launchOptionForLambda, - dumpio: !!exports.DEBUG, + browser = await chromium.puppeteer.launch({ + args: chromium.args, + defaultViewport: chromium.defaultViewport, + executablePath: await chromium.executablePath, + headless: chromium.headless, + slowMo: process.env.SLOWMO_MS, + dumpio: !!config.DEBUG, }); debugLog(async (b) => `launch done: ${await browser.version()}`); } @@ -32,64 +29,6 @@ const isBrowserAvailable = async (browser) => { return true; }; -const setupChrome = async () => { - if (!await existsExecutableChrome()) { - if (await existsLocalChrome()) { - debugLog('setup local chrome'); - await setupLocalChrome(); - } else { - debugLog('setup s3 chrome'); - await setupS3Chrome(); - } - debugLog('setup done'); - } -}; - -const existsLocalChrome = () => { - return new Promise((resolve, reject) => { - fs.exists(config.localChromePath, (exists) => { - resolve(exists); - }); - }); -}; - -const existsExecutableChrome = () => { - return new Promise((resolve, reject) => { - fs.exists(config.executablePath, (exists) => { - resolve(exists); - }); - }); -}; - -const setupLocalChrome = () => { - return new Promise((resolve, reject) => { - fs.createReadStream(config.localChromePath) - .on('error', (err) => reject(err)) - .pipe(tar.x({ - C: config.setupChromePath, - })) - .on('error', (err) => reject(err)) - .on('end', () => resolve()); - }); -}; - -const setupS3Chrome = () => { - return new Promise((resolve, reject) => { - const params = { - Bucket: config.remoteChromeS3Bucket, - Key: config.remoteChromeS3Key, - }; - s3.getObject(params) - .createReadStream() - .on('error', (err) => reject(err)) - .pipe(tar.x({ - C: config.setupChromePath, - })) - .on('error', (err) => reject(err)) - .on('end', () => resolve()); - }); -}; - const debugLog = (log) => { if (config.DEBUG) { let message = log;