Skip to content

Commit

Permalink
chore(tests): Add regression test optimizations
Browse files Browse the repository at this point in the history
  • Loading branch information
Cliff Pyles committed Oct 18, 2017
1 parent 451b90b commit 4c808da
Show file tree
Hide file tree
Showing 303 changed files with 368 additions and 287 deletions.
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,7 @@ patternfly.spec

#Ignore backstop regression test images and reports
/backstop_data/bitmaps_test
/backstop_data/bitmaps_reference
/backstop_data/html_report
/backstop_data/pdf_report
/backstop_data/ci_report
regression-report.pdf
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
language: node_js
node_js:
- '4'
- '6'

cache:
directories:
Expand Down
3 changes: 2 additions & 1 deletion Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@ GEM

PLATFORMS
ruby
x86_64-darwin-16

DEPENDENCIES
jekyll

BUNDLED WITH
1.13.3
1.15.4
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,23 @@ or
```
grunt karma
```
### Visual Regression Testing

Visual regression tests provide a way to detect if unintended visual changes have
occured as a result of changes in the code. They work by taking screenshots of
what components or pages should look like in a browser (known as references), and then comparing the references to screenshots of those components or pages with your code changes applied. Once the tests are complete, you will be a shown the test results in a browser.

You can run all of the test scenarios with `npm run regressions`.

You can run specific test scenarios with `npm run regressions <scenario-name>`. This
will probably be most useful while you are doing active development and only want
to check a few scenarios without running the entire suite.
(Ex. `npm run regressions alerts buttons`)

To approve conflicts run: `npm run approve-conflicts`. This is the command you want to run
when the tests find conflicts, but the conflicts are intended. This command
will replace the base image, so if you run the regression tests again, the
tests should pass.

## Documentation

Expand Down
29 changes: 29 additions & 0 deletions backstop/approve.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
const backstop = require('backstopjs');
const config = require('./config');

// check for arguments passed at the command line
const cliArguments = process.argv.filter((current, index) => {
return index > 1;
});

// if arguments are passed only run the given scenarios
if (cliArguments.length) {
let scenarios = [];

cliArguments.forEach((currentScenario) => {
let scenarioPath = path.join(__dirname, 'config', 'scenarios', `${currentScenario}.js`);
let scenario = require(scenarioPath);

scenarios.push(...scenario);
});

config.scenarios = scenarios;
}

backstop('approve', { config })
.then(() => {
console.log('approved changes');
})
.catch((err) => {
throw err;
});
54 changes: 45 additions & 9 deletions backstop/config/index.js
Original file line number Diff line number Diff line change
@@ -1,29 +1,65 @@
const scenarios = require('./scenarios');
const requireAll = require('require-all');
const reportTypes = (process.env.NODE_ENV === 'test') ? ['CI'] : ['browser'];

let scenarioConfigs = requireAll(`${__dirname}/scenarios`);
let scenariosToLoad = [];

Object.keys(scenarioConfigs).forEach((scenarioConfig) => {
scenarioConfigs[scenarioConfig].forEach((scenario) => {
if (!scenario.disabled) {
scenariosToLoad.push(scenario);
}
})
});

module.exports = {
"id": "backstop_default",
"viewports": [
{
"label": "desktop",
"width": 1440,
"height": 900
"label": "large-device",
"width": 1280,
"height": 720
},
{
"label": "medium-device",
"width": 1024,
"height": 768
},
{
"label": "small-device",
"width": 768,
"height": 1024
},
{
"label": "extra-small-device",
"width": 480,
"height": 320
}
],
"onBeforeScript": "chromy/onBefore.js",
"onReadyScript": "chromy/onReady.js",
"scenarios": scenarios,
"scenarios": scenariosToLoad,
"paths": {
"bitmaps_reference": "backstop_data/bitmaps_reference",
"bitmaps_test": "backstop_data/bitmaps_test",
"engine_scripts": "backstop_data/engine_scripts",
"html_report": "backstop_data/html_report",
"ci_report": "backstop_data/ci_report"
},
"report": ["browser"],
"report": reportTypes,
"engine": "chrome",
"engineFlags": [],
"asyncCaptureLimit": 5,
"asyncCompareLimit": 25,
"asyncCaptureLimit": 20,
"asyncCompareLimit": 100,
"debug": false,
"debugWindow": false
"debugWindow": false,
"resembleOutputOptions": {
"errorColor": {
"red": 204,
"green": 0,
"blue": 0
},
"errorType": "movement",
"transparency": 0.15
}
}
3 changes: 2 additions & 1 deletion backstop/config/scenarios/about-modal.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'about-modal',
url: 'http://localhost:9000/about-modal.html'
url: 'http://localhost:4200/about-modal.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/accordions.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'accordions',
url: 'http://localhost:9000/accordions.html'
url: 'http://localhost:4200/accordions.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/alerts.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'alerts',
url: 'http://localhost:9000/alerts.html'
url: 'http://localhost:4200/alerts.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/application-launcher.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'application-launcher',
url: 'http://localhost:9000/application-launcher.html'
url: 'http://localhost:4200/application-launcher.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/area-charts.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'area-charts',
url: 'http://localhost:9000/area-charts.html'
url: 'http://localhost:4200/area-charts.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/badges.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'badges',
url: 'http://localhost:9000/badges.html'
url: 'http://localhost:4200/badges.html'
}]
10 changes: 5 additions & 5 deletions backstop/config/scenarios/bar-charts.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// module.exports = [{
// label: 'bar-charts',
// url: 'http://localhost:9000/bar-charts.html'
// }]
module.exports = []
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bar-charts',
url: 'http://localhost:4200/bar-charts.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/basic.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'basic',
url: 'http://localhost:9000/basic.html'
url: 'http://localhost:4200/basic.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/blank-slate.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'blank-slate',
url: 'http://localhost:9000/blank-slate.html'
url: 'http://localhost:4200/blank-slate.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-combobox.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-combobox',
url: 'http://localhost:9000/bootstrap-combobox.html'
url: 'http://localhost:4200/bootstrap-combobox.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-datepicker.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-datepicker',
url: 'http://localhost:9000/bootstrap-datepicker.html'
url: 'http://localhost:4200/bootstrap-datepicker.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-select.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-select',
url: 'http://localhost:9000/bootstrap-select.html'
url: 'http://localhost:4200/bootstrap-select.html'
}]
4 changes: 3 additions & 1 deletion backstop/config/scenarios/bootstrap-switch.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-switch',
url: 'http://localhost:9000/bootstrap-switch.html'
url: 'http://localhost:4200/bootstrap-switch.html',
disabled: true
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-touchspin.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-touchspin',
url: 'http://localhost:9000/bootstrap-touchspin.html'
url: 'http://localhost:4200/bootstrap-touchspin.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-treeview.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-treeview',
url: 'http://localhost:9000/bootstrap-treeview.html'
url: 'http://localhost:4200/bootstrap-treeview.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/breadcrumbs.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'breadcrumbs',
url: 'http://localhost:9000/breadcrumbs.html'
url: 'http://localhost:4200/breadcrumbs.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/buttons.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'buttons',
url: 'http://localhost:9000/buttons.html'
url: 'http://localhost:4200/buttons.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/card-view-card-variatons.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'card-view-card-variatons',
url: 'http://localhost:9000/card-view-card-variatons.html'
url: 'http://localhost:4200/card-view-card-variatons.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/card-view-multi-select.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'card-view-multi-select',
url: 'http://localhost:9000/card-view-multi-select.html'
url: 'http://localhost:4200/card-view-multi-select.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/card-view-single-select.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'card-view-single-select',
url: 'http://localhost:9000/card-view-single-select.html'
url: 'http://localhost:4200/card-view-single-select.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/cards.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'cards',
url: 'http://localhost:9000/cards.html'
url: 'http://localhost:4200/cards.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/code.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'code',
url: 'http://localhost:9000/code.html'
url: 'http://localhost:4200/code.html'
}]
7 changes: 6 additions & 1 deletion backstop/config/scenarios/dashboard.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
module.exports = [{
label: 'dashboard',
url: 'http://localhost:9000/dashboard.html'
url: 'http://localhost:4200/dashboard.html',
removeSelectors: [
'.page-header + .alert',
'.toast-notifications-list-pf'
],
disabled: true
}]
11 changes: 6 additions & 5 deletions backstop/config/scenarios/donut-charts.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// module.exports = [{
// label: 'donut-charts',
// url: 'http://localhost:9000/donut-charts.html'
// }]
module.exports = []
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'donut-charts',
url: 'http://localhost:4200/donut-charts.html',
disabled: true
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/dropdowns.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'dropdowns',
url: 'http://localhost:9000/dropdowns.html'
url: 'http://localhost:4200/dropdowns.html'
}]
4 changes: 3 additions & 1 deletion backstop/config/scenarios/form.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'form',
url: 'http://localhost:9000/form.html'
url: 'http://localhost:4200/form.html',
disabled: true
}]
4 changes: 3 additions & 1 deletion backstop/config/scenarios/forms.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'forms',
url: 'http://localhost:9000/forms.html'
url: 'http://localhost:4200/forms.html',
disabled: true
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/icons.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'icons',
url: 'http://localhost:9000/icons.html'
url: 'http://localhost:4200/icons.html'
}]
Loading

0 comments on commit 4c808da

Please sign in to comment.