Skip to content

Commit

Permalink
Merge pull request patternfly#780 from cliffpyles/feature/regression-…
Browse files Browse the repository at this point in the history
…test-optimizations

chore(tests): Add regression test optimizations
  • Loading branch information
jeff-phillips-18 authored Oct 18, 2017
2 parents 118bfc5 + 4c808da commit 93e9bbb
Show file tree
Hide file tree
Showing 303 changed files with 368 additions and 287 deletions.
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,7 @@ patternfly.spec

#Ignore backstop regression test images and reports
/backstop_data/bitmaps_test
/backstop_data/bitmaps_reference
/backstop_data/html_report
/backstop_data/pdf_report
/backstop_data/ci_report
regression-report.pdf
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
language: node_js
node_js:
- '4'
- '6'

cache:
directories:
Expand Down
3 changes: 2 additions & 1 deletion Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@ GEM

PLATFORMS
ruby
x86_64-darwin-16

DEPENDENCIES
jekyll

BUNDLED WITH
1.13.3
1.15.4
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,23 @@ or
```
grunt karma
```
### Visual Regression Testing

Visual regression tests provide a way to detect if unintended visual changes have
occured as a result of changes in the code. They work by taking screenshots of
what components or pages should look like in a browser (known as references), and then comparing the references to screenshots of those components or pages with your code changes applied. Once the tests are complete, you will be a shown the test results in a browser.

You can run all of the test scenarios with `npm run regressions`.

You can run specific test scenarios with `npm run regressions <scenario-name>`. This
will probably be most useful while you are doing active development and only want
to check a few scenarios without running the entire suite.
(Ex. `npm run regressions alerts buttons`)

To approve conflicts run: `npm run approve-conflicts`. This is the command you want to run
when the tests find conflicts, but the conflicts are intended. This command
will replace the base image, so if you run the regression tests again, the
tests should pass.

### Visual Regression Testing

Expand Down
29 changes: 29 additions & 0 deletions backstop/approve.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
const backstop = require('backstopjs');
const config = require('./config');

// check for arguments passed at the command line
const cliArguments = process.argv.filter((current, index) => {
return index > 1;
});

// if arguments are passed only run the given scenarios
if (cliArguments.length) {
let scenarios = [];

cliArguments.forEach((currentScenario) => {
let scenarioPath = path.join(__dirname, 'config', 'scenarios', `${currentScenario}.js`);
let scenario = require(scenarioPath);

scenarios.push(...scenario);
});

config.scenarios = scenarios;
}

backstop('approve', { config })
.then(() => {
console.log('approved changes');
})
.catch((err) => {
throw err;
});
54 changes: 45 additions & 9 deletions backstop/config/index.js
Original file line number Diff line number Diff line change
@@ -1,29 +1,65 @@
const scenarios = require('./scenarios');
const requireAll = require('require-all');
const reportTypes = (process.env.NODE_ENV === 'test') ? ['CI'] : ['browser'];

let scenarioConfigs = requireAll(`${__dirname}/scenarios`);
let scenariosToLoad = [];

Object.keys(scenarioConfigs).forEach((scenarioConfig) => {
scenarioConfigs[scenarioConfig].forEach((scenario) => {
if (!scenario.disabled) {
scenariosToLoad.push(scenario);
}
})
});

module.exports = {
"id": "backstop_default",
"viewports": [
{
"label": "desktop",
"width": 1440,
"height": 900
"label": "large-device",
"width": 1280,
"height": 720
},
{
"label": "medium-device",
"width": 1024,
"height": 768
},
{
"label": "small-device",
"width": 768,
"height": 1024
},
{
"label": "extra-small-device",
"width": 480,
"height": 320
}
],
"onBeforeScript": "chromy/onBefore.js",
"onReadyScript": "chromy/onReady.js",
"scenarios": scenarios,
"scenarios": scenariosToLoad,
"paths": {
"bitmaps_reference": "backstop_data/bitmaps_reference",
"bitmaps_test": "backstop_data/bitmaps_test",
"engine_scripts": "backstop_data/engine_scripts",
"html_report": "backstop_data/html_report",
"ci_report": "backstop_data/ci_report"
},
"report": ["browser"],
"report": reportTypes,
"engine": "chrome",
"engineFlags": [],
"asyncCaptureLimit": 5,
"asyncCompareLimit": 25,
"asyncCaptureLimit": 20,
"asyncCompareLimit": 100,
"debug": false,
"debugWindow": false
"debugWindow": false,
"resembleOutputOptions": {
"errorColor": {
"red": 204,
"green": 0,
"blue": 0
},
"errorType": "movement",
"transparency": 0.15
}
}
3 changes: 2 additions & 1 deletion backstop/config/scenarios/about-modal.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'about-modal',
url: 'http://localhost:9000/about-modal.html'
url: 'http://localhost:4200/about-modal.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/accordions.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'accordions',
url: 'http://localhost:9000/accordions.html'
url: 'http://localhost:4200/accordions.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/alerts.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'alerts',
url: 'http://localhost:9000/alerts.html'
url: 'http://localhost:4200/alerts.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/application-launcher.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'application-launcher',
url: 'http://localhost:9000/application-launcher.html'
url: 'http://localhost:4200/application-launcher.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/area-charts.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'area-charts',
url: 'http://localhost:9000/area-charts.html'
url: 'http://localhost:4200/area-charts.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/badges.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'badges',
url: 'http://localhost:9000/badges.html'
url: 'http://localhost:4200/badges.html'
}]
10 changes: 5 additions & 5 deletions backstop/config/scenarios/bar-charts.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// module.exports = [{
// label: 'bar-charts',
// url: 'http://localhost:9000/bar-charts.html'
// }]
module.exports = []
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bar-charts',
url: 'http://localhost:4200/bar-charts.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/basic.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'basic',
url: 'http://localhost:9000/basic.html'
url: 'http://localhost:4200/basic.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/blank-slate.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'blank-slate',
url: 'http://localhost:9000/blank-slate.html'
url: 'http://localhost:4200/blank-slate.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-combobox.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-combobox',
url: 'http://localhost:9000/bootstrap-combobox.html'
url: 'http://localhost:4200/bootstrap-combobox.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-datepicker.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-datepicker',
url: 'http://localhost:9000/bootstrap-datepicker.html'
url: 'http://localhost:4200/bootstrap-datepicker.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-select.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-select',
url: 'http://localhost:9000/bootstrap-select.html'
url: 'http://localhost:4200/bootstrap-select.html'
}]
4 changes: 3 additions & 1 deletion backstop/config/scenarios/bootstrap-switch.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-switch',
url: 'http://localhost:9000/bootstrap-switch.html'
url: 'http://localhost:4200/bootstrap-switch.html',
disabled: true
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-touchspin.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-touchspin',
url: 'http://localhost:9000/bootstrap-touchspin.html'
url: 'http://localhost:4200/bootstrap-touchspin.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/bootstrap-treeview.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'bootstrap-treeview',
url: 'http://localhost:9000/bootstrap-treeview.html'
url: 'http://localhost:4200/bootstrap-treeview.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/breadcrumbs.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'breadcrumbs',
url: 'http://localhost:9000/breadcrumbs.html'
url: 'http://localhost:4200/breadcrumbs.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/buttons.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'buttons',
url: 'http://localhost:9000/buttons.html'
url: 'http://localhost:4200/buttons.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/card-view-card-variatons.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'card-view-card-variatons',
url: 'http://localhost:9000/card-view-card-variatons.html'
url: 'http://localhost:4200/card-view-card-variatons.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/card-view-multi-select.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'card-view-multi-select',
url: 'http://localhost:9000/card-view-multi-select.html'
url: 'http://localhost:4200/card-view-multi-select.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/card-view-single-select.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'card-view-single-select',
url: 'http://localhost:9000/card-view-single-select.html'
url: 'http://localhost:4200/card-view-single-select.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/cards.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'cards',
url: 'http://localhost:9000/cards.html'
url: 'http://localhost:4200/cards.html'
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/code.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'code',
url: 'http://localhost:9000/code.html'
url: 'http://localhost:4200/code.html'
}]
7 changes: 6 additions & 1 deletion backstop/config/scenarios/dashboard.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
module.exports = [{
label: 'dashboard',
url: 'http://localhost:9000/dashboard.html'
url: 'http://localhost:4200/dashboard.html',
removeSelectors: [
'.page-header + .alert',
'.toast-notifications-list-pf'
],
disabled: true
}]
11 changes: 6 additions & 5 deletions backstop/config/scenarios/donut-charts.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// module.exports = [{
// label: 'donut-charts',
// url: 'http://localhost:9000/donut-charts.html'
// }]
module.exports = []
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'donut-charts',
url: 'http://localhost:4200/donut-charts.html',
disabled: true
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/dropdowns.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'dropdowns',
url: 'http://localhost:9000/dropdowns.html'
url: 'http://localhost:4200/dropdowns.html'
}]
4 changes: 3 additions & 1 deletion backstop/config/scenarios/form.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'form',
url: 'http://localhost:9000/form.html'
url: 'http://localhost:4200/form.html',
disabled: true
}]
4 changes: 3 additions & 1 deletion backstop/config/scenarios/forms.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'forms',
url: 'http://localhost:9000/forms.html'
url: 'http://localhost:4200/forms.html',
disabled: true
}]
3 changes: 2 additions & 1 deletion backstop/config/scenarios/icons.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
module.exports = [{
removeSelectors: ['.page-header + .alert'],
label: 'icons',
url: 'http://localhost:9000/icons.html'
url: 'http://localhost:4200/icons.html'
}]
Loading

0 comments on commit 93e9bbb

Please sign in to comment.