From e78f17460f997fdef389ab9b4e65d51004efe6ba Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 14:27:34 +0100 Subject: [PATCH 1/9] Fix wrong spec --- test/e2e/specs/04_userStatus.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/e2e/specs/04_userStatus.js b/test/e2e/specs/04_userStatus.js index 5a7593e53..41540db65 100644 --- a/test/e2e/specs/04_userStatus.js +++ b/test/e2e/specs/04_userStatus.js @@ -94,6 +94,14 @@ module.exports = { comments .waitForElementVisible('@restrictedMessageBox'); }, + 'user should not be able to comment': (client) => { + const embedStream = client.page.embedStream(); + const comments = embedStream.section.comments; + + comments + .waitForElementNotPresent('@commentBoxTextarea') + .waitForElementNotPresent('@commentBoxPostButton'); + }, 'user picks another username': (client) => { const embedStream = client.page.embedStream(); const comments = embedStream.section.comments; @@ -106,12 +114,12 @@ module.exports = { .click('@suspendedAccountSubmitButton') .waitForElementNotPresent('@suspendedAccountInput'); }, - 'user should not be able to comment': (client) => { + 'user should be able to comment': (client) => { const embedStream = client.page.embedStream(); const comments = embedStream.section.comments; comments - .waitForElementNotPresent('@commentBoxTextarea') - .waitForElementNotPresent('@commentBoxPostButton'); + .waitForElementVisible('@commentBoxTextarea') + .waitForElementVisible('@commentBoxPostButton'); }, }; From 4116e206a37296175f9c06c7ceaee9edad887fe6 Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 14:41:51 +0100 Subject: [PATCH 2/9] Configurable timeout --- scripts/e2e-ci.sh | 5 ++++- scripts/e2e.js | 10 +++++++--- test/e2e/globals.js | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/e2e-ci.sh b/scripts/e2e-ci.sh index b86edb319..5e0bf00b3 100755 --- a/scripts/e2e-ci.sh +++ b/scripts/e2e-ci.sh @@ -6,6 +6,9 @@ CIRCLE_BRANCH=${CIRCLE_BRANCH:-master} # Amount of retries before failure. E2E_MAX_RETRIES=${E2E_MAX_RETRIES:-1} +# Timeout for WaitForConditions. +E2E_WAIT_FOR_TIMEOUT=${E2E_WAIT_FOR_TIMEOUT:-5000} + # Safari >= 8 has issues connecting to browserstack-local. Safari < 8 is too old. # IE 64bit has issues with receiving keyboard input. Let's wait for them to fix it. @@ -15,7 +18,7 @@ BROWSERS="chrome" if [[ "${CIRCLE_BRANCH}" == "master" && -n "$BROWSERSTACK_KEY" ]]; then echo Testing on browserstack - yarn e2e --reports-folder "$REPORTS_FOLDER" --bs-key "$BROWSERSTACK_KEY" --retries "$E2E_MAX_RETRIES" --browsers $BROWSERS + yarn e2e --reports-folder "$REPORTS_FOLDER" --bs-key "$BROWSERSTACK_KEY" --retries "$E2E_MAX_RETRIES" --timeout "$E2E_WAIT_FOR_TIMEOUT" --browsers $BROWSERS else # When browserstack is not available test locally using chrome headless. echo Testing locally diff --git a/scripts/e2e.js b/scripts/e2e.js index 97e321310..2f99bc5d9 100755 --- a/scripts/e2e.js +++ b/scripts/e2e.js @@ -59,7 +59,7 @@ function seleniumInstall() { }); } -function nightwatch(env, config, reportsFolder, browserstack) { +function nightwatch(env, config, reportsFolder, browserstack, timeout) { return new Promise((resolve, reject) => { try { const nw = childProcess.spawn( @@ -71,6 +71,7 @@ function nightwatch(env, config, reportsFolder, browserstack) { 'BROWSERSTACK_KEY': browserstack.key, 'BROWSERSTACK_USER': browserstack.user, 'REPORTS_FOLDER': `${reportsFolder}/${env}`, + 'WAIT_FOR_TIMEOUT': timeout, }), stdio: 'inherit', }); @@ -111,13 +112,13 @@ function printSection(txt) { console.log('*****************************'.magenta); } -async function runBrowserTests(browsers, config, retries = 1, reportsFolder, browserstack) { +async function runBrowserTests(browsers, config, retries = 1, reportsFolder, browserstack, timeout) { const succeeded = {}; for (let browser of browsers) { for (let t = 0; t < retries + 1; t++) { try { printSection(`e2e test for ${browser} #${t}`); - await nightwatch(browser, config, reportsFolder, browserstack); + await nightwatch(browser, config, reportsFolder, browserstack, timeout); succeeded[browser] = t; console.log(`\n==> Succeeded e2e for ${browser} #${t}\n`.green); break; @@ -143,6 +144,7 @@ async function start(program) { const date = new Date().toISOString() .replace(/[T.]/g, '-') .replace(/:/g, ''); + const timeout = program.timeout; const reportsFolder = `${program.reportsFolder}/${date}`; let exitCode = 0; let config = 'nightwatch.conf.js'; @@ -175,6 +177,7 @@ async function start(program) { retries, reportsFolder, browserstack, + timeout, ); if (!succeeded) { exitCode = 1; @@ -202,6 +205,7 @@ program .option('-r, --retries [number]', 'Number of retries before failing', '1') .option('--headless', 'Start in headless mode for local e2e') .option('--reports-folder [folder]', 'Reports folder', './test/e2e/reports') + .option('--timeout [number]', 'Timeout for WaitForConditions', '5000') .parse(process.argv); start(program); diff --git a/test/e2e/globals.js b/test/e2e/globals.js index 4d2740404..10549e2e7 100644 --- a/test/e2e/globals.js +++ b/test/e2e/globals.js @@ -11,7 +11,7 @@ module.exports = { shutdown(); done(); }, - waitForConditionTimeout: 5000, + waitForConditionTimeout: parseInt(process.env.WAIT_FOR_TIMEOUT) || 5000, testData: { admin: { email: 'admin@test.com', From 71f0887a820dfe6356d6d40b419ef14ad6d75791 Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 14:42:29 +0100 Subject: [PATCH 3/9] Make browsers configurable through ENV E2E_BROWSERS --- scripts/e2e-ci.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/e2e-ci.sh b/scripts/e2e-ci.sh index 5e0bf00b3..8a90ed3bd 100755 --- a/scripts/e2e-ci.sh +++ b/scripts/e2e-ci.sh @@ -13,12 +13,12 @@ E2E_WAIT_FOR_TIMEOUT=${E2E_WAIT_FOR_TIMEOUT:-5000} # IE 64bit has issues with receiving keyboard input. Let's wait for them to fix it. # FIXME: disabled firefox,edge until fixing pass is done -# BROWSERS="chrome,firefox,edge" #ie safari -BROWSERS="chrome" +# E2E_BROWSERS=${E2E_BROWSERS:-chrome,firefox,edge} #ie safari +E2E_BROWSERS=${E2E_BROWSERS:-chrome} if [[ "${CIRCLE_BRANCH}" == "master" && -n "$BROWSERSTACK_KEY" ]]; then echo Testing on browserstack - yarn e2e --reports-folder "$REPORTS_FOLDER" --bs-key "$BROWSERSTACK_KEY" --retries "$E2E_MAX_RETRIES" --timeout "$E2E_WAIT_FOR_TIMEOUT" --browsers $BROWSERS + yarn e2e --reports-folder "$REPORTS_FOLDER" --bs-key "$BROWSERSTACK_KEY" --retries "$E2E_MAX_RETRIES" --timeout "$E2E_WAIT_FOR_TIMEOUT" --browsers "$E2E_BROWSERS" else # When browserstack is not available test locally using chrome headless. echo Testing locally From 7a4b87f7438665f0e88ece6e04a414c640527ea6 Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 14:42:54 +0100 Subject: [PATCH 4/9] Fix popup issues --- test/e2e/page_objects/embedStream.js | 21 +++++------------- test/e2e/utils/SortedWindowHandler.js | 32 +++++++++++---------------- 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/test/e2e/page_objects/embedStream.js b/test/e2e/page_objects/embedStream.js index 1cecad2a3..0a31745e4 100644 --- a/test/e2e/page_objects/embedStream.js +++ b/test/e2e/page_objects/embedStream.js @@ -62,24 +62,12 @@ module.exports = { // Focusing on the Login PopUp windowHandler.windowHandles((handles) => { this.api.switchWindow(handles[1]); - }); - const popup = this.api.page.popup().ready(); - callback(popup); + const popup = this.api.page.popup().ready(); + callback(popup); - // Give a tiny bit of time to let popup close. - this.api.pause(50); - - if (this.api.capabilities.browserName === 'MicrosoftEdge') { - - // More time for edge. - // https://www.browserstack.com/automate/builds/1ceccf4efb4683b7feb890f45a32b5922b40ed3f/sessions/7393dbfda8387e43b6d5851f359b0c07db414973 - this.api.pause(1000); - } - - // Focusing on the Embed Window - windowHandler.windowHandles((handles) => { - this.api.switchWindow(handles[0]); + // Focus on the Embed Window. + windowHandler.pop(); // For some reasons firefox does not automatically load auth after login. // https://www.browserstack.com/automate/builds/37650cb4e66c6edce0ba0800a1c1b7e7f74bf991/sessions/7a4e9da69b0f9ecdf8b7fa9150639e47b1532cb0#automate_button @@ -89,6 +77,7 @@ module.exports = { this.parent.switchToIframe(); } }); + return this; }, logout() { diff --git a/test/e2e/utils/SortedWindowHandler.js b/test/e2e/utils/SortedWindowHandler.js index cb8195978..a23f71506 100644 --- a/test/e2e/utils/SortedWindowHandler.js +++ b/test/e2e/utils/SortedWindowHandler.js @@ -25,28 +25,14 @@ class SortedWindowHandler { */ windowHandles(callback) { this.client.windowHandles((result) => { - if (Array.isArray(result.value)) { - this.handles = this.handles.filter((handle) => { - for (let i = 0; i < result.value.length; i++) { - if (result.value[i] === handle) { - return true; - } - } - return false; - }); - } else { - this.handles = []; + if (result.value.message) { + throw new Error(result.value.message); } - const remaining = result.value.filter((handle) => { - for (let i = 0; i < this.handles.length; i++) { - if (this.handles[i] === handle) { - return false; - } - } - return true; - }); + this.handles = this.handles.filter((handle) => result.value.includes(handle)); + const remaining = result.value.filter((handle) => !this.handles.includes(handle)); + if (remaining.length === 1) { this.handles.push(remaining[0]); } @@ -56,6 +42,14 @@ class SortedWindowHandler { callback(this.handles); }); } + + /** + * Switch to main window handle and remove latest handle. + */ + pop() { + this.client.switchWindow(this.handles[0]); + this.handles.splice(-1, 1); + } } module.exports = SortedWindowHandler; From 82e617b19b1deab14188fa366c1c1aa019e1f42c Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 14:48:58 +0100 Subject: [PATCH 5/9] Enable firefox and edge --- scripts/e2e-ci.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/e2e-ci.sh b/scripts/e2e-ci.sh index 8a90ed3bd..64f749196 100755 --- a/scripts/e2e-ci.sh +++ b/scripts/e2e-ci.sh @@ -12,9 +12,7 @@ E2E_WAIT_FOR_TIMEOUT=${E2E_WAIT_FOR_TIMEOUT:-5000} # Safari >= 8 has issues connecting to browserstack-local. Safari < 8 is too old. # IE 64bit has issues with receiving keyboard input. Let's wait for them to fix it. -# FIXME: disabled firefox,edge until fixing pass is done -# E2E_BROWSERS=${E2E_BROWSERS:-chrome,firefox,edge} #ie safari -E2E_BROWSERS=${E2E_BROWSERS:-chrome} +E2E_BROWSERS=${E2E_BROWSERS:-chrome,firefox,edge} #ie safari if [[ "${CIRCLE_BRANCH}" == "master" && -n "$BROWSERSTACK_KEY" ]]; then echo Testing on browserstack From d06ff8c93c7c736b1fe93584567a613fa96f37e1 Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 15:56:54 +0100 Subject: [PATCH 6/9] Double default WaitForTimeout --- scripts/e2e.js | 2 +- test/e2e/globals.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/e2e.js b/scripts/e2e.js index 2f99bc5d9..d1d06e99a 100755 --- a/scripts/e2e.js +++ b/scripts/e2e.js @@ -205,7 +205,7 @@ program .option('-r, --retries [number]', 'Number of retries before failing', '1') .option('--headless', 'Start in headless mode for local e2e') .option('--reports-folder [folder]', 'Reports folder', './test/e2e/reports') - .option('--timeout [number]', 'Timeout for WaitForConditions', '5000') + .option('--timeout [number]', 'Timeout for WaitForConditions', '10000') .parse(process.argv); start(program); diff --git a/test/e2e/globals.js b/test/e2e/globals.js index 10549e2e7..09b297a24 100644 --- a/test/e2e/globals.js +++ b/test/e2e/globals.js @@ -11,7 +11,7 @@ module.exports = { shutdown(); done(); }, - waitForConditionTimeout: parseInt(process.env.WAIT_FOR_TIMEOUT) || 5000, + waitForConditionTimeout: parseInt(process.env.WAIT_FOR_TIMEOUT) || 10000, testData: { admin: { email: 'admin@test.com', From e4f1f5d08b58b2a1f80567625d246d80ad779dd9 Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 16:14:59 +0100 Subject: [PATCH 7/9] Allow disabling e2e in ci --- scripts/e2e-ci.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/e2e-ci.sh b/scripts/e2e-ci.sh index 64f749196..c9ee437e8 100755 --- a/scripts/e2e-ci.sh +++ b/scripts/e2e-ci.sh @@ -2,6 +2,7 @@ REPORTS_FOLDER=${CIRCLE_TEST_REPORTS:-./test/e2e/reports} CIRCLE_BRANCH=${CIRCLE_BRANCH:-master} +E2E_DISABLE=${E2E_DISABLE:-false} # Amount of retries before failure. E2E_MAX_RETRIES=${E2E_MAX_RETRIES:-1} @@ -14,6 +15,12 @@ E2E_WAIT_FOR_TIMEOUT=${E2E_WAIT_FOR_TIMEOUT:-5000} E2E_BROWSERS=${E2E_BROWSERS:-chrome,firefox,edge} #ie safari + +if [[ "${E2E_DISABLE}" == "true" ]]; then + echo E2E is disabled. + exit +fi + if [[ "${CIRCLE_BRANCH}" == "master" && -n "$BROWSERSTACK_KEY" ]]; then echo Testing on browserstack yarn e2e --reports-folder "$REPORTS_FOLDER" --bs-key "$BROWSERSTACK_KEY" --retries "$E2E_MAX_RETRIES" --timeout "$E2E_WAIT_FOR_TIMEOUT" --browsers "$E2E_BROWSERS" From 1fce2451c71a0afaf0ec0ec774d634fc94e7a012 Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 16:16:59 +0100 Subject: [PATCH 8/9] Remove some blank lines --- scripts/e2e-ci.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/e2e-ci.sh b/scripts/e2e-ci.sh index c9ee437e8..95672c5d7 100755 --- a/scripts/e2e-ci.sh +++ b/scripts/e2e-ci.sh @@ -12,10 +12,8 @@ E2E_WAIT_FOR_TIMEOUT=${E2E_WAIT_FOR_TIMEOUT:-5000} # Safari >= 8 has issues connecting to browserstack-local. Safari < 8 is too old. # IE 64bit has issues with receiving keyboard input. Let's wait for them to fix it. - E2E_BROWSERS=${E2E_BROWSERS:-chrome,firefox,edge} #ie safari - if [[ "${E2E_DISABLE}" == "true" ]]; then echo E2E is disabled. exit From f12abbf39ed51cf22f3d477e356aa9d0c6293303 Mon Sep 17 00:00:00 2001 From: Chi Vinh Le Date: Tue, 19 Dec 2017 16:26:55 +0100 Subject: [PATCH 9/9] Default timeout to 10000 for ci --- scripts/e2e-ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/e2e-ci.sh b/scripts/e2e-ci.sh index 95672c5d7..a509ccee7 100755 --- a/scripts/e2e-ci.sh +++ b/scripts/e2e-ci.sh @@ -8,7 +8,7 @@ E2E_DISABLE=${E2E_DISABLE:-false} E2E_MAX_RETRIES=${E2E_MAX_RETRIES:-1} # Timeout for WaitForConditions. -E2E_WAIT_FOR_TIMEOUT=${E2E_WAIT_FOR_TIMEOUT:-5000} +E2E_WAIT_FOR_TIMEOUT=${E2E_WAIT_FOR_TIMEOUT:-10000} # Safari >= 8 has issues connecting to browserstack-local. Safari < 8 is too old. # IE 64bit has issues with receiving keyboard input. Let's wait for them to fix it.