From b10edd3de909cf8774ffe0e1e82754f68c365703 Mon Sep 17 00:00:00 2001
From: Vaivaswat <vaivaswat2244@gmail.com>
Date: Fri, 21 Mar 2025 19:35:46 +0530
Subject: [PATCH 1/3] Adding Visual Test Report in Github Actions
---
.github/workflows/ci-test.yml | 10 +
.gitignore | 4 +-
test/unit/visual/visual-report.js | 427 ++++++++++++++++++++++++++++++
test/unit/visual/visualTest.js | 3 +
4 files changed, 443 insertions(+), 1 deletion(-)
create mode 100644 test/unit/visual/visual-report.js
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index bd3ca55ba0..d38a09b64f 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -27,6 +27,16 @@ jobs:
run: npm test
env:
CI: true
+ - name: Generate Visual Test Report
+ run: node test/unit/visual/generate-visual-report.js
+ env:
+ CI: true
+ - name: Upload Visual Test Report
+ uses: actions/upload-artifact@v3
+ with:
+ name: visual-test-report
+ path: test/unit/visual/visual-report.html
+ retention-days: 14
- name: report test coverage
run: bash <(curl -s https://codecov.io/bash) -f coverage/coverage-final.json
env:
diff --git a/.gitignore b/.gitignore
index 6d6bb7e175..1504e8b5a4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,4 +23,6 @@ yarn.lock
docs/data.json
analyzer/
preview/
-__screenshots__/
\ No newline at end of file
+__screenshots__/
+actual-screenshots/
+visual-report.html
\ No newline at end of file
diff --git a/test/unit/visual/visual-report.js b/test/unit/visual/visual-report.js
new file mode 100644
index 0000000000..1989c0b72a
--- /dev/null
+++ b/test/unit/visual/visual-report.js
@@ -0,0 +1,427 @@
+const fs = require('fs');
+const path = require('path');
+
+async function generateVisualReport() {
+ const expectedDir = path.join(process.cwd(), 'test/unit/visual/screenshots');
+ const actualDir = path.join(process.cwd(), 'test/unit/visual/actual-screenshots');
+ const outputFile = path.join(process.cwd(), 'test/unit/visual/visual-report.html');
+
+ // Make sure the output directory exists
+ const outputDir = path.dirname(outputFile);
+ if (!fs.existsSync(outputDir)) {
+ fs.mkdirSync(outputDir, { recursive: true });
+ }
+
+ // Function to read image file and convert to data URL
+ function imageToDataURL(filePath) {
+ try {
+ const data = fs.readFileSync(filePath);
+ const base64 = data.toString('base64');
+ return `data:image/png;base64,${base64}`;
+ } catch (error) {
+ console.error(`Failed to read image: ${filePath}`, error);
+ return null;
+ }
+ }
+
+ // Create a lookup map for actual screenshots
+ function createActualScreenshotMap() {
+ const actualMap = new Map();
+ if (!fs.existsSync(actualDir)) {
+ console.warn(`Actual screenshots directory does not exist: ${actualDir}`);
+ return actualMap;
+ }
+
+ const files = fs.readdirSync(actualDir);
+ for (const file of files) {
+ if (file.endsWith('.png') && !file.endsWith('-diff.png')) {
+ actualMap.set(file, path.join(actualDir, file));
+ }
+ }
+
+ return actualMap;
+ }
+
+ const actualScreenshotMap = createActualScreenshotMap();
+
+ // Recursively find all test cases
+ function findTestCases(dir, prefix = '') {
+ const testCases = [];
+
+ if (!fs.existsSync(path.join(dir, prefix))) {
+ console.warn(`Directory does not exist: ${path.join(dir, prefix)}`);
+ return testCases;
+ }
+
+ const entries = fs.readdirSync(path.join(dir, prefix), { withFileTypes: true });
+
+ for (const entry of entries) {
+ const fullPath = path.join(prefix, entry.name);
+
+ if (entry.isDirectory()) {
+ // Recursively search subdirectories
+ testCases.push(...findTestCases(dir, fullPath));
+ } else if (entry.name === 'metadata.json') {
+ // Found a test case
+ const metadataPath = path.join(dir, fullPath);
+ let metadata;
+
+ try {
+ metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
+ } catch (error) {
+ console.error(`Failed to read metadata: ${metadataPath}`, error);
+ continue;
+ }
+
+ const testDir = path.dirname(fullPath);
+
+ const test = {
+ name: testDir,
+ numScreenshots: metadata.numScreenshots || 0,
+ screenshots: []
+ };
+
+ // Create flattened name for lookup
+ const flattenedName = testDir.replace(/\//g, '-');
+
+ // Collect all screenshots for this test
+ for (let i = 0; i < test.numScreenshots; i++) {
+ const screenshotName = i.toString().padStart(3, '0') + '.png';
+ const expectedPath = path.join(dir, testDir, screenshotName);
+
+ // Use flattened name for actual screenshots
+ const actualScreenshotName = `${flattenedName}-${i.toString().padStart(3, '0')}.png`;
+ const actualPath = actualScreenshotMap.get(actualScreenshotName) || null;
+
+ // Use flattened name for diff image
+ const diffScreenshotName = `${flattenedName}-${i.toString().padStart(3, '0')}-diff.png`;
+ const diffPath = path.join(actualDir, diffScreenshotName);
+
+ const hasExpected = fs.existsSync(expectedPath);
+ const hasActual = actualPath && fs.existsSync(actualPath);
+ const hasDiff = fs.existsSync(diffPath);
+
+ const screenshot = {
+ index: i,
+ expectedImage: hasExpected ? imageToDataURL(expectedPath) : null,
+ actualImage: hasActual ? imageToDataURL(actualPath) : null,
+ diffImage: hasDiff ? imageToDataURL(diffPath) : null,
+ passed: hasExpected && hasActual && !hasDiff
+ };
+
+ test.screenshots.push(screenshot);
+ }
+
+ // Don't add tests with no screenshots
+ if (test.screenshots.length > 0) {
+ testCases.push(test);
+ }
+ }
+ }
+
+ return testCases;
+ }
+
+ // Find all test cases from the expected directory
+ const testCases = findTestCases(expectedDir);
+
+ if (testCases.length === 0) {
+ console.warn('No test cases found. Check if the expected directory is correct.');
+ }
+
+ // Count passed/failed tests and screenshots
+ const totalTests = testCases.length;
+ let passedTests = 0;
+ let totalScreenshots = 0;
+ let passedScreenshots = 0;
+
+ for (const test of testCases) {
+ const testPassed = test.screenshots.every(screenshot => screenshot.passed);
+ if (testPassed) passedTests++;
+
+ totalScreenshots += test.screenshots.length;
+ passedScreenshots += test.screenshots.filter(s => s.passed).length;
+ }
+
+ // Generate HTML
+ const html = `
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>p5.js Visual Test Results</title>
+ <style>
+ body {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
+ line-height: 1.6;
+ color: #333;
+ max-width: 1200px;
+ margin: 0 auto;
+ padding: 20px;
+ }
+
+ header {
+ margin-bottom: 30px;
+ }
+
+ .summary {
+ background-color: #f5f5f5;
+ padding: 15px;
+ border-radius: 5px;
+ margin-bottom: 30px;
+ }
+
+ .summary h2 {
+ margin-top: 0;
+ }
+
+ .test-group {
+ border: 1px solid #ddd;
+ border-radius: 5px;
+ margin-bottom: 30px;
+ overflow: hidden;
+ }
+
+ .test-header {
+ background-color: #f5f5f5;
+ padding: 10px 15px;
+ border-bottom: 1px solid #ddd;
+ font-weight: bold;
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ }
+
+ .test-status {
+ display: inline-block;
+ padding: 3px 8px;
+ border-radius: 3px;
+ font-size: 14px;
+ font-weight: normal;
+ }
+
+ .status-pass {
+ background-color: #dff0d8;
+ color: #3c763d;
+ }
+
+ .status-fail {
+ background-color: #f2dede;
+ color: #a94442;
+ }
+
+ .screenshots {
+ padding: 20px;
+ }
+
+ .screenshot-set {
+ margin-bottom: 30px;
+ border-bottom: 1px solid #eee;
+ padding-bottom: 20px;
+ position: relative;
+ }
+
+ .screenshot-set:last-child {
+ margin-bottom: 0;
+ border-bottom: none;
+ padding-bottom: 0;
+ }
+
+ .screenshot-header {
+ margin-bottom: 15px;
+ font-weight: 500;
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ }
+
+ .screenshot-status {
+ display: inline-block;
+ padding: 3px 8px;
+ border-radius: 3px;
+ font-size: 14px;
+ }
+
+ .screenshot-images {
+ display: flex;
+ flex-wrap: wrap;
+ gap: 20px;
+ }
+
+ .image-container {
+ flex: 1;
+ min-width: 300px;
+ }
+
+ .image-header {
+ margin-bottom: 5px;
+ font-weight: 500;
+ }
+
+ img {
+ max-width: 100%;
+ border: 1px solid #ddd;
+ background-color: #f8f8f8;
+ }
+
+ .toggle-btn {
+ background-color: #f8f9fa;
+ border: 1px solid #ddd;
+ padding: 5px 10px;
+ border-radius: 3px;
+ cursor: pointer;
+ margin-right: 5px;
+ }
+
+ .toggle-btn.active {
+ background-color: #e9ecef;
+ font-weight: bold;
+ }
+
+ .hidden {
+ display: none;
+ }
+
+ .filters {
+ margin-bottom: 20px;
+ }
+
+ .missing-notice {
+ padding: 10px;
+ background-color: #fff3cd;
+ color: #856404;
+ border-radius: 4px;
+ margin-top: 5px;
+ }
+ </style>
+</head>
+<body>
+ <header>
+ <h1>p5.js Visual Test Results</h1>
+ <div class="filters">
+ <button id="show-all" class="toggle-btn active">Show All</button>
+ <button id="show-failed" class="toggle-btn">Show Only Failed</button>
+ <button id="show-passed" class="toggle-btn">Show Only Passed</button>
+ </div>
+ </header>
+
+ <div class="summary">
+ <h2>Summary</h2>
+ <p>
+ <strong>Total Tests:</strong> ${totalTests}<br>
+ <strong>Passed Tests:</strong> ${passedTests} (${totalTests > 0 ? Math.round(passedTests/totalTests*100) : 0}%)<br>
+ <strong>Failed Tests:</strong> ${totalTests - passedTests} (${totalTests > 0 ? Math.round((totalTests-passedTests)/totalTests*100) : 0}%)<br>
+ <strong>Total Screenshots:</strong> ${totalScreenshots}<br>
+ <strong>Passed Screenshots:</strong> ${passedScreenshots} (${totalScreenshots > 0 ? Math.round(passedScreenshots/totalScreenshots*100) : 0}%)<br>
+ <strong>Report Generated:</strong> ${new Date().toLocaleString()}
+ </p>
+ </div>
+
+ <div id="test-results">
+ ${testCases.map(test => {
+ const passed = test.screenshots.every(s => s.passed);
+ return `
+ <div class="test-group ${passed ? 'test-passed' : 'test-failed'}">
+ <div class="test-header">
+ <span>${test.name}</span>
+ <span class="test-status ${passed ? 'status-pass' : 'status-fail'}">${passed ? 'PASS' : 'FAIL'}</span>
+ </div>
+ <div class="screenshots">
+ ${test.screenshots.map(screenshot => `
+ <div class="screenshot-set">
+ <div class="screenshot-header">
+ <span>Screenshot #${screenshot.index + 1}</span>
+ <span class="screenshot-status ${screenshot.passed ? 'status-pass' : 'status-fail'}">
+ ${screenshot.passed ? 'PASS' : 'FAIL'}
+ </span>
+ </div>
+ <div class="screenshot-images">
+ <div class="image-container">
+ <div class="image-header">Expected</div>
+ ${screenshot.expectedImage ?
+ `<img src="${screenshot.expectedImage}" alt="Expected Result">` :
+ `<div class="missing-notice">No expected image found</div>`}
+ </div>
+ <div class="image-container">
+ <div class="image-header">Actual</div>
+ ${screenshot.actualImage ?
+ `<img src="${screenshot.actualImage}" alt="Actual Result">` :
+ `<div class="missing-notice">No actual image found</div>`}
+ </div>
+ ${screenshot.diffImage ? `
+ <div class="image-container">
+ <div class="image-header">Diff</div>
+ <img src="${screenshot.diffImage}" alt="Difference">
+ </div>
+ ` : ''}
+ </div>
+ </div>
+ `).join('')}
+ </div>
+ </div>
+ `;
+ }).join('')}
+ </div>
+
+ <script>
+ // Filter functionality
+ const buttons = document.querySelectorAll('.toggle-btn');
+ const testGroups = document.querySelectorAll('.test-group');
+
+ document.getElementById('show-all').addEventListener('click', function() {
+ testGroups.forEach(el => {
+ el.style.display = 'block';
+ });
+ setActiveButton(this);
+ });
+
+ document.getElementById('show-failed').addEventListener('click', function() {
+ testGroups.forEach(el => {
+ el.style.display = el.classList.contains('test-failed') ? 'block' : 'none';
+ });
+ setActiveButton(this);
+ });
+
+ document.getElementById('show-passed').addEventListener('click', function() {
+ testGroups.forEach(el => {
+ el.style.display = el.classList.contains('test-passed') ? 'block' : 'none';
+ });
+ setActiveButton(this);
+ });
+
+ function setActiveButton(activeButton) {
+ buttons.forEach(button => {
+ button.classList.remove('active');
+ });
+ activeButton.classList.add('active');
+ }
+ </script>
+</body>
+</html>
+ `;
+
+ // Write HTML to file
+ fs.writeFileSync(outputFile, html);
+ console.log(`Visual test report generated: ${outputFile}`);
+
+ return {
+ totalTests,
+ passedTests,
+ failedTests: totalTests - passedTests,
+ totalScreenshots,
+ passedScreenshots,
+ failedScreenshots: totalScreenshots - passedScreenshots,
+ reportPath: outputFile
+ };
+}
+
+// Run the function if this script is executed directly
+if (require.main === module) {
+ generateVisualReport().catch(error => {
+ console.error('Failed to generate report:', error);
+ process.exit(1);
+ });
+}
+
+module.exports = { generateVisualReport };
\ No newline at end of file
diff --git a/test/unit/visual/visualTest.js b/test/unit/visual/visualTest.js
index 120ce79565..236110f20c 100644
--- a/test/unit/visual/visualTest.js
+++ b/test/unit/visual/visualTest.js
@@ -441,6 +441,8 @@ export function visualTest(
: [];
for (let i = 0; i < actual.length; i++) {
+ const flatName = name.replace(/\//g, '-');
+ const actualFilename = `../actual-screenshots/${flatName}-${i.toString().padStart(3, '0')}.png`;
if (expected[i]) {
const result = await checkMatch(actual[i], expected[i], myp5);
if (!result.ok) {
@@ -453,6 +455,7 @@ export function visualTest(
} else {
writeImageFile(expectedFilenames[i], toBase64(actual[i]));
}
+ writeImageFile(actualFilename, toBase64(actual[i]));
}
});
});
From 8679a7aff4deb75bdeff12733db2c735fa7c213d Mon Sep 17 00:00:00 2001
From: Vaivaswat <vaivaswat2244@gmail.com>
Date: Fri, 21 Mar 2025 20:01:51 +0530
Subject: [PATCH 2/3] Updated file name and upload-artifact version
---
.github/workflows/ci-test.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index d38a09b64f..dee848d10a 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -28,11 +28,11 @@ jobs:
env:
CI: true
- name: Generate Visual Test Report
- run: node test/unit/visual/generate-visual-report.js
+ run: node test/unit/visual/visual-report.js
env:
CI: true
- name: Upload Visual Test Report
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: visual-test-report
path: test/unit/visual/visual-report.html
From 4a674e3fad1f232a159f8f2bb45749e90c5218a4 Mon Sep 17 00:00:00 2001
From: Vaivaswat <vaivaswat2244@gmail.com>
Date: Fri, 21 Mar 2025 20:30:43 +0530
Subject: [PATCH 3/3] moved the script from test folder to root
---
.github/workflows/ci-test.yml | 2 +-
test/unit/visual/visual-report.js => visual-report.js | 0
2 files changed, 1 insertion(+), 1 deletion(-)
rename test/unit/visual/visual-report.js => visual-report.js (100%)
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index dee848d10a..58c9740882 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -28,7 +28,7 @@ jobs:
env:
CI: true
- name: Generate Visual Test Report
- run: node test/unit/visual/visual-report.js
+ run: node visual-report.js
env:
CI: true
- name: Upload Visual Test Report
diff --git a/test/unit/visual/visual-report.js b/visual-report.js
similarity index 100%
rename from test/unit/visual/visual-report.js
rename to visual-report.js