From 0fc810a7b1c8e2fa13a607f13fc53678bf92902a Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 7 Nov 2017 16:21:09 +0100 Subject: [PATCH] e2e v2 --- nightwatch-browserstack.conf.js | 1 + nightwatch.conf.js | 2 + package.json | 4 +- scripts/e2e-browserstack.js | 51 -------- scripts/e2e-ci.sh | 87 ++------------ scripts/e2e.js | 201 ++++++++++++++++++++++++++++++++ test/e2e/globals.js | 2 - 7 files changed, 212 insertions(+), 136 deletions(-) delete mode 100755 scripts/e2e-browserstack.js create mode 100755 scripts/e2e.js diff --git a/nightwatch-browserstack.conf.js b/nightwatch-browserstack.conf.js index d79f6917a..09900c321 100644 --- a/nightwatch-browserstack.conf.js +++ b/nightwatch-browserstack.conf.js @@ -22,6 +22,7 @@ const nightwatch_config = { 'browserstack.user': process.env.BROWSERSTACK_USER || 'coralproject2', 'browserstack.key': process.env.BROWSERSTACK_KEY, 'browserstack.local': true, + 'browserstack.localIdentifier': process.env.BROWSERSTACK_LOCAL_IDENTIFIER ? process.env.BROWSERSTACK_LOCAL_IDENTIFIER : undefined, 'browserstack.debug': true, // Disable this, as it makes bs slow and brittle. diff --git a/nightwatch.conf.js b/nightwatch.conf.js index 97eee0e81..e04cc04a8 100644 --- a/nightwatch.conf.js +++ b/nightwatch.conf.js @@ -40,6 +40,8 @@ module.exports = { path: process.env.REPORTS_FOLDER || './test/e2e/reports', }, }, + 'chrome': { + }, 'chrome-headless': { desiredCapabilities: { chromeOptions : { diff --git a/package.json b/package.json index 897efc819..69093c0c5 100644 --- a/package.json +++ b/package.json @@ -16,9 +16,7 @@ "lint-fix": "yarn lint --fix", "jest-watch": "TEST_MODE=unit NODE_ENV=test jest --watch", "e2e-ci": "./scripts/e2e-ci.sh", - "e2e-browserstack": "NODE_ENV=test ./scripts/e2e-browserstack.js --config nightwatch-browserstack.conf.js", - "pree2e": "selenium-standalone install", - "e2e": "NODE_ENV=test nightwatch", + "e2e": "./scripts/e2e.js", "test": "TEST_MODE=unit NODE_ENV=test jest && TEST_MODE=unit NODE_ENV=test mocha -R ${MOCHA_REPORTER:-spec}", "test-cover": "TEST_MODE=unit NODE_ENV=test istanbul cover _mocha --report text --check-coverage -- -R spec", "heroku-postbuild": "./bin/cli plugins reconcile && yarn build", diff --git a/scripts/e2e-browserstack.js b/scripts/e2e-browserstack.js deleted file mode 100755 index 103cd08fe..000000000 --- a/scripts/e2e-browserstack.js +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env node - -const Nightwatch = require('nightwatch'); -const browserstack = require('browserstack-local'); -const {onshutdown} = require('../bin/util'); - -async function start() { - try { - const bs_local = new browserstack.Local(); - process.mainModule.filename = './node_modules/.bin/nightwatch'; - - // Code to start browserstack local before start of test - console.log('Connecting local'); - Nightwatch.bs_local = bs_local; - - bs_local.start({ - 'key': process.env.BROWSERSTACK_KEY, - 'logFile': './test/e2e/bslocal.log' - }, function(error) { - if (error) { - console.error(error); - throw error; - } - - console.log('Connected. Now testing...'); - Nightwatch.cli(function(argv) { - Nightwatch.CliRunner(argv) - .setup(null, function(){ - - // Code to stop browserstack local after end of parallel test - bs_local.stop(function(){}); - }) - .runTests(function(){ - - // Code to stop browserstack local after end of single test - bs_local.stop(function(){}); - }); - }); - }); - - onshutdown([ - () => bs_local.stop(function(){}), - ]); - } catch (ex) { - console.log('There was an error while starting the test runner:\n\n'); - process.stderr.write(`${ex.stack}\n`); - process.exit(2); - } -} - -start(); diff --git a/scripts/e2e-ci.sh b/scripts/e2e-ci.sh index e72d1a6ce..5923ff682 100755 --- a/scripts/e2e-ci.sh +++ b/scripts/e2e-ci.sh @@ -1,92 +1,19 @@ #!/bin/bash -CIRCLE_TEST_REPORTS=${CIRCLE_TEST_REPORTS:-./test/e2e/reports} +REPORTS_FOLDER=${CIRCLE_TEST_REPORTS:-./test/e2e/reports} CIRCLE_BRANCH=${CIRCLE_BRANCH:-master} # Amount of retries before failure. E2E_MAX_RETRIES=${E2E_MAX_RETRIES:-1} -# Amount of seconds between tests. -E2E_SLEEP_BETWEEN_TESTS=${E2E_SLEEP_BETWEEN_TESTS:-1} - # Safari >= 8 has issues connecting to browserstack-local. Safari < 8 is too old. -BROWSERS="chrome firefox ie edge" #safari +BROWSERS="chrome,firefox,ie,edge" #safari -if [[ "${CIRCLE_BRANCH}" == "master" ]]; then - - # List of failed browsers. - failedBrowsers= - - # List of succeeded browsers. - succeededBrowsers= - - exitCode=0 - - browserstack() { - - # Current number of tries. - try=${2:-0} - - echo "-- Start e2e for $1 #$try --" - - REPORTS_FOLDER="$CIRCLE_TEST_REPORTS/$1" yarn e2e-browserstack --env "$1" - - # Determine exit code. - result=$? - if [ "$result" -ne "0" ]; then - echo "-- Failed e2e for $1 #$try --" - - # Try again until E2E_MAX_RETRIES is reached. - if [ "$try" -lt "$E2E_MAX_RETRIES" ]; then - let try=try+1 - - # Sleep a bit to let browserstack-local close properly. - sleep "$E2E_SLEEP_BETWEEN_TESTS" - - browserstack "$1" "$try" - return - fi - - # Failed, add to list of failed browsers. - failedBrowsers="$failedBrowsers $1" - - # Remember exit code. - exitCode=$result - else - echo "-- Success e2e for $1 #$try --" - - # Succeeded, add to list of succeeded browsers. - succeededBrowsers="$succeededBrowsers $1" - eval "browser_${1}_succeeded_at=$try" - fi - - # Sleep a bit to let browserstack-local close properly. - sleep "$E2E_SLEEP_BETWEEN_TESTS" - } - - # Test using browserstack. - for browser in $BROWSERS - do - browserstack "$browser" - done - - - # Print information about succeeded browsers. - for x in $succeededBrowsers - do - echo "Succeeded $x at try #$(eval "echo \$browser_${x}_succeeded_at")" - done - - # Print information about failed browsers. - for x in $failedBrowsers - do - echo "Failed $x" - done - exit $exitCode +if [[ "${CIRCLE_BRANCH}" == "master" && -n "$BROWSERSTACK_KEY" ]]; then + echo Testing on browserstack + yarn e2e --reports-folder "$REPORTS_FOLDER" --bs-key "$BROWSERSTACK_KEY" --retries "$E2E_MAX_RETRIES" --browsers $BROWSERS else # When browserstack is not available test locally using chrome headless. - REPORTS_FOLDER="$CIRCLE_TEST_REPORTS/chrome" yarn e2e -- --env chrome-headless - - # Will exit with status of last command. - exit $? + echo Testing locally + yarn e2e --reports-folder "$REPORTS_FOLDER" --retries "$E2E_MAX_RETRIES" --headless fi diff --git a/scripts/e2e.js b/scripts/e2e.js new file mode 100755 index 000000000..6cf97b2a6 --- /dev/null +++ b/scripts/e2e.js @@ -0,0 +1,201 @@ +#!/usr/bin/env node + +process.env['NODE_ENV'] = 'test'; + +const browserstack = require('browserstack-local'); +const {onshutdown, shutdown} = require('../bin/util'); +const program = require('commander'); +const Table = require('cli-table'); +const serve = require('../serve'); +const childProcess = require('child_process'); +const uuid = require('uuid').v4; + +// Make things colorful! +require('colors'); + +function startTunnel(key, localIdentifier) { + const bs_local = new browserstack.Local(); + + // Code to start browserstack local before start of test + console.log('Connecting local'); + + return new Promise((resolve, reject) => { + bs_local.start({ + key, + logFile: './test/e2e/bslocal.log', + verbose: 'true', + force: 'true', + onlyAutomate: 'true', + localIdentifier, + }, (error) => { + if (error) { + reject(error); + } + resolve(); + }); + onshutdown([ + () => bs_local.stop(function(){}), + ]); + }); +} + +function seleniumInstall() { + return new Promise((resolve, reject) => { + try { + const nw = childProcess.spawn( + './node_modules/.bin/selenium-standalone', + ['install'], + { + stdio: 'inherit', + }); + nw.on('close', (code) => { + code === 0 ? resolve() : reject(); + }); + } + catch (ex) { + reject(ex); + } + }); +} + +function nightwatch(env, config, reportsFolder, browserstack) { + return new Promise((resolve, reject) => { + try { + const nw = childProcess.spawn( + './node_modules/.bin/nightwatch', + ['--config', config, '--env', env], + { + env: Object.assign({}, process.env, { + 'BROWSERSTACK_LOCAL_IDENTIFIER': browserstack.localIdentifier, + 'BROWSERSTACK_KEY': browserstack.key, + 'BROWSERSTACK_USER': browserstack.user, + 'REPORTS_FOLDER': `${reportsFolder}/${env}`, + }), + stdio: 'inherit', + }); + nw.on('close', (code) => { + code === 0 ? resolve() : reject(); + }); + } + catch (ex) { + reject(ex); + } + }); +} + +function printResults(browsers, succeeded, retries) { + let table = new Table({ + head: [ + 'Browser'.cyan, + 'Status'.cyan, + 'Retries'.cyan, + ] + }); + + for (let browser of browsers) { + const wasSuccessful = browser in succeeded; + table.push([ + browser, + wasSuccessful ? 'success'.green : 'failed'.red, + wasSuccessful ? succeeded[browser] : retries, + ]); + } + + console.log(table.toString()); +} + +function printSection(txt) { + console.log('*****************************'.magenta); + console.log(`${'*'.magenta} ${txt.cyan}`); + console.log('*****************************'.magenta); +} + +async function runBrowserTests(browsers, config, retries = 1, reportsFolder, browserstack) { + const succeeded = {}; + for (let browser of browsers) { + for (let t = 0; t < retries + 1; t++) { + try { + printSection(`e2e test for ${browser} #${t}`); + await nightwatch(browser, config, reportsFolder, browserstack); + succeeded[browser] = t; + console.log(`==> Succeeded e2e for ${browser} #${t}\n`.green); + break; + } + catch (ex) { + if (ex) { + console.log('There was an error while starting the test runner:\n\n'); + process.stderr.write(`${ex.stack}\n`); + } + console.log(`==> Failed e2e for ${browser} #${t}\n`.red); + } + } + } + printResults(browsers, succeeded, retries); + return Object.keys(succeeded).length === browsers.length; +} + +async function start(program) { + const localIdentifier = uuid(); + let browsers = program.browsers.split(','); + const retries = Number.parseInt(program.retries); + const browserstack = {}; + const date = new Date().toISOString() + .replace(/[T.]/g, '-') + .replace(/:/g, ''); + const reportsFolder = `${program.reportsFolder}/${date}`; + let exitCode = 0; + let config = 'nightwatch.conf.js'; + try { + if (program.tunnel && program.bsKey) { + await startTunnel(program.bsKey, localIdentifier); + } + + await serve(); + + if (program.bsKey) { + config = 'nightwatch-browserstack.conf.js'; + browserstack.localIdentifier = localIdentifier; + browserstack.key = program.bsKey; + browserstack.user = program.bsUser; + } else { + await seleniumInstall(); + if (program.headless) { + browsers = browsers.map((b) => `${b}-headless`); + } + } + + const succeeded = await runBrowserTests( + browsers, + config, + retries, + reportsFolder, + browserstack, + ); + if (!succeeded) { + exitCode = 1; + } + } + catch (ex) { + console.log('There was an error:\n\n'); + process.stderr.write(`${ex.stack}\n`); + process.exit(2); + } + finally { + console.log('Shutting down'); + shutdown(); + } + process.exit(exitCode); +} + +program + .version('0.1.0') + .option('-u, --bs-user [user]', 'Browserstack user', 'coralproject2') + .option('-k, --bs-key [key]', 'Browserstack api key') + .option('--no-tunnel', 'Dont start browserstack-local') + .option('-b, --browsers [list of browsers]', 'Browsers to test', 'chrome,firefox,ie,edge') + .option('-r, --retries [number]', 'Number of retries before failing', '1') + .option('--headless', 'Start in headless mode for local e2e') + .option('--reports-folder [folder]', 'Reports folder', './test/e2e/reports') + .parse(process.argv); + +start(program); diff --git a/test/e2e/globals.js b/test/e2e/globals.js index a828804b4..5bb0a27fe 100644 --- a/test/e2e/globals.js +++ b/test/e2e/globals.js @@ -1,11 +1,9 @@ -const serve = require('../../serve'); const mongoose = require('../../services/mongoose'); const {shutdown} = require('../../bin/util'); module.exports = { before: async (done) => { await mongoose.connection.dropDatabase(); - await serve(); done(); }, after: (done) => {