From 0544bc2cf9b8d037bf0ff623849b31253bd63998 Mon Sep 17 00:00:00 2001 From: Saturnino Abril Date: Sat, 24 Oct 2020 08:02:35 +0800 Subject: [PATCH] MM-29918 Cypress/E2E: Add ability to run tests for EE and TE (#6883) * add ability to run e2e for EE and TE * Update e2e/run_tests.js Co-authored-by: Joseph Baylon Co-authored-by: Joseph Baylon --- .../group_mentions_permissions_spec.js | 2 +- .../group_mentions_posts_spec.js | 2 +- .../group_mentions_system_messages_spec.js | 2 +- .../guest_experience_ui_spec.js | 2 +- .../guest_identification_ui_spec.js | 2 +- .../guest_invitation_ui_spec.js | 2 +- .../guest_accounts/guest_removal_ui_spec.js | 2 +- .../member_invitation_ui_spec.js | 2 +- .../system_console_guest_access_ui_spec.js | 2 +- .../system_console_manage_guest_spec.js | 2 +- .../profile_popover/profile_popover_spec.js | 8 ++-- .../system_console/channel_members_spec.js | 2 + .../system_console/team_members_spec.js | 1 + .../visual_regression/helpers.js | 2 +- .../system_console/about_section_spec.js | 2 +- .../authentication_section_spec.js | 2 +- .../system_console/compliance_section_spec.js | 2 +- .../environment_section_spec.js | 2 +- e2e/run_tests.js | 34 ++++++++++---- e2e/utils/file.js | 44 ++++++++++++------- 20 files changed, 75 insertions(+), 44 deletions(-) rename e2e/cypress/integration/{ => enterprise}/profile_popover/profile_popover_spec.js (98%) rename e2e/cypress/integration/{ => enterprise}/visual_regression/helpers.js (80%) rename e2e/cypress/integration/{ => enterprise}/visual_regression/system_console/about_section_spec.js (97%) rename e2e/cypress/integration/{ => enterprise}/visual_regression/system_console/authentication_section_spec.js (98%) rename e2e/cypress/integration/{ => enterprise}/visual_regression/system_console/compliance_section_spec.js (98%) rename e2e/cypress/integration/{ => enterprise}/visual_regression/system_console/environment_section_spec.js (98%) diff --git a/e2e/cypress/integration/enterprise/group_mentions/group_mentions_permissions_spec.js b/e2e/cypress/integration/enterprise/group_mentions/group_mentions_permissions_spec.js index cd7e8225a515..e897e939f0d4 100644 --- a/e2e/cypress/integration/enterprise/group_mentions/group_mentions_permissions_spec.js +++ b/e2e/cypress/integration/enterprise/group_mentions/group_mentions_permissions_spec.js @@ -6,7 +6,7 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** -// Group: @system_console @group_mentions +// Group: @enterprise @system_console @group_mentions import ldapUsers from '../../../fixtures/ldap_users.json'; import * as TIMEOUTS from '../../../fixtures/timeouts'; diff --git a/e2e/cypress/integration/enterprise/group_mentions/group_mentions_posts_spec.js b/e2e/cypress/integration/enterprise/group_mentions/group_mentions_posts_spec.js index 2f27fca9f3ec..8d485237c6ef 100644 --- a/e2e/cypress/integration/enterprise/group_mentions/group_mentions_posts_spec.js +++ b/e2e/cypress/integration/enterprise/group_mentions/group_mentions_posts_spec.js @@ -6,7 +6,7 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** -// Group: @system_console @group_mentions +// Group: @enterprise @system_console @group_mentions import ldapUsers from '../../../fixtures/ldap_users.json'; import * as TIMEOUTS from '../../../fixtures/timeouts'; diff --git a/e2e/cypress/integration/enterprise/group_mentions/group_mentions_system_messages_spec.js b/e2e/cypress/integration/enterprise/group_mentions/group_mentions_system_messages_spec.js index ba7d92d85f21..f4d05a76a96d 100644 --- a/e2e/cypress/integration/enterprise/group_mentions/group_mentions_system_messages_spec.js +++ b/e2e/cypress/integration/enterprise/group_mentions/group_mentions_system_messages_spec.js @@ -6,7 +6,7 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** -// Group: @system_console @group_mentions +// Group: @enterprise @system_console @group_mentions import ldapUsers from '../../../fixtures/ldap_users.json'; import * as TIMEOUTS from '../../../fixtures/timeouts'; diff --git a/e2e/cypress/integration/enterprise/guest_accounts/guest_experience_ui_spec.js b/e2e/cypress/integration/enterprise/guest_accounts/guest_experience_ui_spec.js index de7eeacacde9..329ec81c4387 100644 --- a/e2e/cypress/integration/enterprise/guest_accounts/guest_experience_ui_spec.js +++ b/e2e/cypress/integration/enterprise/guest_accounts/guest_experience_ui_spec.js @@ -8,7 +8,7 @@ // *************************************************************** // Stage: @prod -// Group: @guest_account +// Group: @enterprise @guest_account /** * Note: This test requires Enterprise license to be uploaded diff --git a/e2e/cypress/integration/enterprise/guest_accounts/guest_identification_ui_spec.js b/e2e/cypress/integration/enterprise/guest_accounts/guest_identification_ui_spec.js index 5f4f958bdd39..bcb208e35a58 100644 --- a/e2e/cypress/integration/enterprise/guest_accounts/guest_identification_ui_spec.js +++ b/e2e/cypress/integration/enterprise/guest_accounts/guest_identification_ui_spec.js @@ -8,7 +8,7 @@ // *************************************************************** // Stage: @prod -// Group: @guest_account +// Group: @enterprise @guest_account /** * Note: This test requires Enterprise license to be uploaded diff --git a/e2e/cypress/integration/enterprise/guest_accounts/guest_invitation_ui_spec.js b/e2e/cypress/integration/enterprise/guest_accounts/guest_invitation_ui_spec.js index e5e4e899b718..055e6ef3c07c 100644 --- a/e2e/cypress/integration/enterprise/guest_accounts/guest_invitation_ui_spec.js +++ b/e2e/cypress/integration/enterprise/guest_accounts/guest_invitation_ui_spec.js @@ -8,7 +8,7 @@ // *************************************************************** // Stage: @prod -// Group: @guest_account +// Group: @enterprise @guest_account /** * Note: This test requires Enterprise license to be uploaded diff --git a/e2e/cypress/integration/enterprise/guest_accounts/guest_removal_ui_spec.js b/e2e/cypress/integration/enterprise/guest_accounts/guest_removal_ui_spec.js index e69f6be210ed..4d4abe76f144 100644 --- a/e2e/cypress/integration/enterprise/guest_accounts/guest_removal_ui_spec.js +++ b/e2e/cypress/integration/enterprise/guest_accounts/guest_removal_ui_spec.js @@ -8,7 +8,7 @@ // *************************************************************** // Stage: @prod -// Group: @guest_account +// Group: @enterprise @guest_account /** * Note: This test requires Enterprise license to be uploaded diff --git a/e2e/cypress/integration/enterprise/guest_accounts/member_invitation_ui_spec.js b/e2e/cypress/integration/enterprise/guest_accounts/member_invitation_ui_spec.js index 6754a4e28323..215924a93e03 100644 --- a/e2e/cypress/integration/enterprise/guest_accounts/member_invitation_ui_spec.js +++ b/e2e/cypress/integration/enterprise/guest_accounts/member_invitation_ui_spec.js @@ -8,7 +8,7 @@ // *************************************************************** // Stage: @prod -// Group: @guest_account +// Group: @enterprise @guest_account /** * Note: This test requires Enterprise license to be uploaded diff --git a/e2e/cypress/integration/enterprise/guest_accounts/system_console_guest_access_ui_spec.js b/e2e/cypress/integration/enterprise/guest_accounts/system_console_guest_access_ui_spec.js index 9eb4d2b2ea5f..1236e25d16b9 100644 --- a/e2e/cypress/integration/enterprise/guest_accounts/system_console_guest_access_ui_spec.js +++ b/e2e/cypress/integration/enterprise/guest_accounts/system_console_guest_access_ui_spec.js @@ -8,7 +8,7 @@ // *************************************************************** // Stage: @prod -// Group: @guest_account +// Group: @enterprise @guest_account /** * Note: This test requires Enterprise license to be uploaded diff --git a/e2e/cypress/integration/enterprise/guest_accounts/system_console_manage_guest_spec.js b/e2e/cypress/integration/enterprise/guest_accounts/system_console_manage_guest_spec.js index fba94f3776c2..e1739f111ae8 100644 --- a/e2e/cypress/integration/enterprise/guest_accounts/system_console_manage_guest_spec.js +++ b/e2e/cypress/integration/enterprise/guest_accounts/system_console_manage_guest_spec.js @@ -8,7 +8,7 @@ // *************************************************************** // Stage: @prod -// Group: @guest_account +// Group: @enterprise @guest_account /** * Note: This test requires Enterprise license to be uploaded diff --git a/e2e/cypress/integration/profile_popover/profile_popover_spec.js b/e2e/cypress/integration/enterprise/profile_popover/profile_popover_spec.js similarity index 98% rename from e2e/cypress/integration/profile_popover/profile_popover_spec.js rename to e2e/cypress/integration/enterprise/profile_popover/profile_popover_spec.js index ab6150590903..3c5868dd0241 100644 --- a/e2e/cypress/integration/profile_popover/profile_popover_spec.js +++ b/e2e/cypress/integration/enterprise/profile_popover/profile_popover_spec.js @@ -7,11 +7,11 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** -// Group: @profile_popover +// Group: @enterprise @profile_popover -import * as TIMEOUTS from '../../fixtures/timeouts'; -import {createPrivateChannel} from '../enterprise/elasticsearch_autocomplete/helpers'; -import {getAdminAccount} from '../../support/env'; +import * as TIMEOUTS from '../../../fixtures/timeouts'; +import {createPrivateChannel} from '../elasticsearch_autocomplete/helpers'; +import {getAdminAccount} from '../../../support/env'; describe('Profile popover', () => { let testTeam; diff --git a/e2e/cypress/integration/enterprise/system_console/channel_members_spec.js b/e2e/cypress/integration/enterprise/system_console/channel_members_spec.js index ae72573f9ea4..cea1c303cbc3 100644 --- a/e2e/cypress/integration/enterprise/system_console/channel_members_spec.js +++ b/e2e/cypress/integration/enterprise/system_console/channel_members_spec.js @@ -7,6 +7,8 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** +// Group: @enterprise + import * as TIMEOUTS from '../../../fixtures/timeouts'; describe('Channel members test', () => { diff --git a/e2e/cypress/integration/enterprise/system_console/team_members_spec.js b/e2e/cypress/integration/enterprise/system_console/team_members_spec.js index a62efd53f839..a403bea234f3 100644 --- a/e2e/cypress/integration/enterprise/system_console/team_members_spec.js +++ b/e2e/cypress/integration/enterprise/system_console/team_members_spec.js @@ -8,6 +8,7 @@ // *************************************************************** // Stage: @prod +// Group: @enterprise import * as TIMEOUTS from '../../../fixtures/timeouts'; diff --git a/e2e/cypress/integration/visual_regression/helpers.js b/e2e/cypress/integration/enterprise/visual_regression/helpers.js similarity index 80% rename from e2e/cypress/integration/visual_regression/helpers.js rename to e2e/cypress/integration/enterprise/visual_regression/helpers.js index 620996f7fd5f..6d889b4d9107 100644 --- a/e2e/cypress/integration/visual_regression/helpers.js +++ b/e2e/cypress/integration/enterprise/visual_regression/helpers.js @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -const applitoolsConfig = require('../../../applitools.config'); +const applitoolsConfig = require('../../../../applitools.config'); export function getBatchName(suffix) { const batchName = Cypress.env('batchName') || applitoolsConfig.batchName; diff --git a/e2e/cypress/integration/visual_regression/system_console/about_section_spec.js b/e2e/cypress/integration/enterprise/visual_regression/system_console/about_section_spec.js similarity index 97% rename from e2e/cypress/integration/visual_regression/system_console/about_section_spec.js rename to e2e/cypress/integration/enterprise/visual_regression/system_console/about_section_spec.js index 935cc553288f..137823275248 100644 --- a/e2e/cypress/integration/visual_regression/system_console/about_section_spec.js +++ b/e2e/cypress/integration/enterprise/visual_regression/system_console/about_section_spec.js @@ -10,7 +10,7 @@ // Stage: @prod // Group: @enterprise @system_console @visual_regression -import * as TIMEOUTS from '../../../fixtures/timeouts'; +import * as TIMEOUTS from '../../../../fixtures/timeouts'; import {getBatchName} from '../helpers'; diff --git a/e2e/cypress/integration/visual_regression/system_console/authentication_section_spec.js b/e2e/cypress/integration/enterprise/visual_regression/system_console/authentication_section_spec.js similarity index 98% rename from e2e/cypress/integration/visual_regression/system_console/authentication_section_spec.js rename to e2e/cypress/integration/enterprise/visual_regression/system_console/authentication_section_spec.js index bb943c9368b9..123c1d478360 100644 --- a/e2e/cypress/integration/visual_regression/system_console/authentication_section_spec.js +++ b/e2e/cypress/integration/enterprise/visual_regression/system_console/authentication_section_spec.js @@ -10,7 +10,7 @@ // Stage: @prod // Group: @enterprise @system_console @visual_regression -import * as TIMEOUTS from '../../../fixtures/timeouts'; +import * as TIMEOUTS from '../../../../fixtures/timeouts'; import {getBatchName} from '../helpers'; diff --git a/e2e/cypress/integration/visual_regression/system_console/compliance_section_spec.js b/e2e/cypress/integration/enterprise/visual_regression/system_console/compliance_section_spec.js similarity index 98% rename from e2e/cypress/integration/visual_regression/system_console/compliance_section_spec.js rename to e2e/cypress/integration/enterprise/visual_regression/system_console/compliance_section_spec.js index 38f3fcb0279d..38a06fe378f0 100644 --- a/e2e/cypress/integration/visual_regression/system_console/compliance_section_spec.js +++ b/e2e/cypress/integration/enterprise/visual_regression/system_console/compliance_section_spec.js @@ -10,7 +10,7 @@ // Stage: @prod // Group: @enterprise @system_console @visual_regression -import * as TIMEOUTS from '../../../fixtures/timeouts'; +import * as TIMEOUTS from '../../../../fixtures/timeouts'; import {getBatchName} from '../helpers'; diff --git a/e2e/cypress/integration/visual_regression/system_console/environment_section_spec.js b/e2e/cypress/integration/enterprise/visual_regression/system_console/environment_section_spec.js similarity index 98% rename from e2e/cypress/integration/visual_regression/system_console/environment_section_spec.js rename to e2e/cypress/integration/enterprise/visual_regression/system_console/environment_section_spec.js index 683582d98d45..6d46a1e96999 100644 --- a/e2e/cypress/integration/visual_regression/system_console/environment_section_spec.js +++ b/e2e/cypress/integration/enterprise/visual_regression/system_console/environment_section_spec.js @@ -10,7 +10,7 @@ // Stage: @prod // Group: @enterprise @system_console @visual_regression -import * as TIMEOUTS from '../../../fixtures/timeouts'; +import * as TIMEOUTS from '../../../../fixtures/timeouts'; describe('System Console - Enterprise', () => { const testCases = [ diff --git a/e2e/run_tests.js b/e2e/run_tests.js index 8b5d5059750f..e9175aaa1d5a 100644 --- a/e2e/run_tests.js +++ b/e2e/run_tests.js @@ -16,6 +16,9 @@ * --group=[group] * Selects spec files with matching group. It can be of multiple values separated by comma. * E.g. "--group='@channel,@messaging'" will select files with either @channel or @messaging. + * --exclude-group=[group] + * Exclude spec files with matching group. It can be of multiple values separated by comma. + * E.g. "--exclude-group='@enterprise'" will select files except @enterprise. * --invert * Selected files are those not matching any of the specified stage or group. * @@ -35,6 +38,12 @@ * - will run all non-production tests * 4. "BROWSER='chrome' HEADLESS='false' node run_tests.js --stage='@prod' --group='@channel,@messaging'" * - will run spec files matching stage and group values in Chrome (headed) + * 5. "CYPRESS_runWithEELicense=true node run_tests.js --stage='@prod'" + * - will run all production tests + * - typical test run for Enterprise Edition, given license file can be found in `cypress/fixtures` folder + * 6. "node run_tests.js --stage='@prod' --exclude-group='@enterprise'" + * - will run all production tests except @enterprise group + * - typical test run for Team Edition */ const os = require('os'); @@ -72,17 +81,11 @@ async function runTests() { return; } - const {invert, group, stage} = argv; - let hasFailed = false; for (let i = 0; i < finalTestFiles.length; i++) { - const testFile = finalTestFiles[i]; - const testStage = stage ? `Stage: "${stage}" ` : ''; - const testGroup = group ? `Group: "${group}" ` : ''; + printMessage(finalTestFiles, i); - // Log which files were being tested - console.log(chalk.magenta.bold(`${invert ? 'All Except --> ' : ''}${testStage}${stage && group ? '| ' : ''}${testGroup}`)); - console.log(chalk.magenta(`(Testing ${i + 1} of ${finalTestFiles.length}) - `, testFile)); + const testFile = finalTestFiles[i]; const result = await cypress.run({ browser, @@ -146,4 +149,19 @@ async function runTests() { chai.expect(hasFailed, FAILURE_MESSAGE).to.be.false; } +function printMessage(testFiles, index) { + const {invert, excludeGroup, group, stage} = argv; + + const testFile = testFiles[index]; + const testStage = stage ? `Stage: "${stage}" ` : ''; + const withGroup = group || excludeGroup; + const groupMessage = group ? `"${group}"` : 'All'; + const excludeGroupMessage = excludeGroup ? `except "${excludeGroup}"` : ''; + const testGroup = withGroup ? `Group: ${groupMessage} ${excludeGroupMessage}` : ''; + + // Log which files were being tested + console.log(chalk.magenta.bold(`${invert ? 'All Except --> ' : ''}${testStage}${stage && withGroup ? '| ' : ''}${testGroup}`)); + console.log(chalk.magenta(`(Testing ${index + 1} of ${testFiles.length}) - `, testFile)); +} + runTests(); diff --git a/e2e/utils/file.js b/e2e/utils/file.js index e7f6d764e112..94c9acbfbe09 100644 --- a/e2e/utils/file.js +++ b/e2e/utils/file.js @@ -26,58 +26,68 @@ const grepFiles = (command) => { }; function getTestFiles() { - const {invert, group, stage} = argv; + const {invert, excludeGroup, group, stage} = argv; const allFiles = grepFiles(grepCommand()); - const stageFiles = []; + let stageFiles = allFiles; if (stage) { const sc = grepCommand(stage.split(',').join('\\|')); - stageFiles.push(...grepFiles(sc)); + stageFiles = grepFiles(sc); } - const groupFiles = []; + + let groupFiles = [...stageFiles]; if (group) { const gc = grepCommand(group.split(',').join('\\|')); - groupFiles.push(...grepFiles(gc)); + groupFiles = grepFiles(gc); + } + + const excludeGroupFiles = []; + if (excludeGroup) { + const egc = grepCommand(excludeGroup.split(',').join('\\|')); + excludeGroupFiles.push(...grepFiles(egc)); } + const finalGroupFiles = without(groupFiles, ...excludeGroupFiles); + const withGroup = group || excludeGroup; + if (invert) { - // Return no test file if no stage and group, but inverted - if (!stage && !group) { + // Return no test file if no stage and withGroup, but inverted + if (!stage && !withGroup) { return []; } // Return all excluding stage files - if (stage && !group) { + if (stage && !withGroup) { return without(allFiles, ...stageFiles); } // Return all excluding group files - if (!stage && group) { - return without(allFiles, ...groupFiles); + if (!stage && withGroup) { + return without(allFiles, ...finalGroupFiles); } // Return all excluding group and stage files - return without(allFiles, ...intersection(stageFiles, groupFiles)); + return without(allFiles, ...intersection(stageFiles, finalGroupFiles)); } // Return all files if no stage and group flags - if (!stage && !group) { + if (!stage && !withGroup) { return allFiles; } // Return stage files if no group flag - if (stage && !group) { + if (stage && !withGroup) { return stageFiles; } // Return group files if no stage flag - if (!stage && group) { - return groupFiles; + if (!stage && withGroup) { + return finalGroupFiles; } - // Return files if both in stage and group - return intersection(stageFiles, groupFiles); + // Return files if both in stage and withGroup + return intersection(stageFiles, finalGroupFiles); } function getSkippedFiles(initialTestFiles, platform, browser, headless) {