diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index bd3ca55ba0..58c9740882 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -27,6 +27,16 @@ jobs:
run: npm test
env:
CI: true
+ - name: Generate Visual Test Report
+ run: node visual-report.js
+ env:
+ CI: true
+ - name: Upload Visual Test Report
+ uses: actions/upload-artifact@v4
+ with:
+ name: visual-test-report
+ path: test/unit/visual/visual-report.html
+ retention-days: 14
- name: report test coverage
run: bash <(curl -s https://codecov.io/bash) -f coverage/coverage-final.json
env:
diff --git a/.gitignore b/.gitignore
index 6d6bb7e175..1504e8b5a4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,4 +23,6 @@ yarn.lock
docs/data.json
analyzer/
preview/
-__screenshots__/
\ No newline at end of file
+__screenshots__/
+actual-screenshots/
+visual-report.html
\ No newline at end of file
diff --git a/test/unit/visual/visualTest.js b/test/unit/visual/visualTest.js
index 120ce79565..236110f20c 100644
--- a/test/unit/visual/visualTest.js
+++ b/test/unit/visual/visualTest.js
@@ -441,6 +441,8 @@ export function visualTest(
: [];
for (let i = 0; i < actual.length; i++) {
+ const flatName = name.replace(/\//g, '-');
+ const actualFilename = `../actual-screenshots/${flatName}-${i.toString().padStart(3, '0')}.png`;
if (expected[i]) {
const result = await checkMatch(actual[i], expected[i], myp5);
if (!result.ok) {
@@ -453,6 +455,7 @@ export function visualTest(
} else {
writeImageFile(expectedFilenames[i], toBase64(actual[i]));
}
+ writeImageFile(actualFilename, toBase64(actual[i]));
}
});
});
diff --git a/visual-report.js b/visual-report.js
new file mode 100644
index 0000000000..1989c0b72a
--- /dev/null
+++ b/visual-report.js
@@ -0,0 +1,427 @@
+const fs = require('fs');
+const path = require('path');
+
+async function generateVisualReport() {
+ const expectedDir = path.join(process.cwd(), 'test/unit/visual/screenshots');
+ const actualDir = path.join(process.cwd(), 'test/unit/visual/actual-screenshots');
+ const outputFile = path.join(process.cwd(), 'test/unit/visual/visual-report.html');
+
+ // Make sure the output directory exists
+ const outputDir = path.dirname(outputFile);
+ if (!fs.existsSync(outputDir)) {
+ fs.mkdirSync(outputDir, { recursive: true });
+ }
+
+ // Function to read image file and convert to data URL
+ function imageToDataURL(filePath) {
+ try {
+ const data = fs.readFileSync(filePath);
+ const base64 = data.toString('base64');
+ return `data:image/png;base64,${base64}`;
+ } catch (error) {
+ console.error(`Failed to read image: ${filePath}`, error);
+ return null;
+ }
+ }
+
+ // Create a lookup map for actual screenshots
+ function createActualScreenshotMap() {
+ const actualMap = new Map();
+ if (!fs.existsSync(actualDir)) {
+ console.warn(`Actual screenshots directory does not exist: ${actualDir}`);
+ return actualMap;
+ }
+
+ const files = fs.readdirSync(actualDir);
+ for (const file of files) {
+ if (file.endsWith('.png') && !file.endsWith('-diff.png')) {
+ actualMap.set(file, path.join(actualDir, file));
+ }
+ }
+
+ return actualMap;
+ }
+
+ const actualScreenshotMap = createActualScreenshotMap();
+
+ // Recursively find all test cases
+ function findTestCases(dir, prefix = '') {
+ const testCases = [];
+
+ if (!fs.existsSync(path.join(dir, prefix))) {
+ console.warn(`Directory does not exist: ${path.join(dir, prefix)}`);
+ return testCases;
+ }
+
+ const entries = fs.readdirSync(path.join(dir, prefix), { withFileTypes: true });
+
+ for (const entry of entries) {
+ const fullPath = path.join(prefix, entry.name);
+
+ if (entry.isDirectory()) {
+ // Recursively search subdirectories
+ testCases.push(...findTestCases(dir, fullPath));
+ } else if (entry.name === 'metadata.json') {
+ // Found a test case
+ const metadataPath = path.join(dir, fullPath);
+ let metadata;
+
+ try {
+ metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
+ } catch (error) {
+ console.error(`Failed to read metadata: ${metadataPath}`, error);
+ continue;
+ }
+
+ const testDir = path.dirname(fullPath);
+
+ const test = {
+ name: testDir,
+ numScreenshots: metadata.numScreenshots || 0,
+ screenshots: []
+ };
+
+ // Create flattened name for lookup
+ const flattenedName = testDir.replace(/\//g, '-');
+
+ // Collect all screenshots for this test
+ for (let i = 0; i < test.numScreenshots; i++) {
+ const screenshotName = i.toString().padStart(3, '0') + '.png';
+ const expectedPath = path.join(dir, testDir, screenshotName);
+
+ // Use flattened name for actual screenshots
+ const actualScreenshotName = `${flattenedName}-${i.toString().padStart(3, '0')}.png`;
+ const actualPath = actualScreenshotMap.get(actualScreenshotName) || null;
+
+ // Use flattened name for diff image
+ const diffScreenshotName = `${flattenedName}-${i.toString().padStart(3, '0')}-diff.png`;
+ const diffPath = path.join(actualDir, diffScreenshotName);
+
+ const hasExpected = fs.existsSync(expectedPath);
+ const hasActual = actualPath && fs.existsSync(actualPath);
+ const hasDiff = fs.existsSync(diffPath);
+
+ const screenshot = {
+ index: i,
+ expectedImage: hasExpected ? imageToDataURL(expectedPath) : null,
+ actualImage: hasActual ? imageToDataURL(actualPath) : null,
+ diffImage: hasDiff ? imageToDataURL(diffPath) : null,
+ passed: hasExpected && hasActual && !hasDiff
+ };
+
+ test.screenshots.push(screenshot);
+ }
+
+ // Don't add tests with no screenshots
+ if (test.screenshots.length > 0) {
+ testCases.push(test);
+ }
+ }
+ }
+
+ return testCases;
+ }
+
+ // Find all test cases from the expected directory
+ const testCases = findTestCases(expectedDir);
+
+ if (testCases.length === 0) {
+ console.warn('No test cases found. Check if the expected directory is correct.');
+ }
+
+ // Count passed/failed tests and screenshots
+ const totalTests = testCases.length;
+ let passedTests = 0;
+ let totalScreenshots = 0;
+ let passedScreenshots = 0;
+
+ for (const test of testCases) {
+ const testPassed = test.screenshots.every(screenshot => screenshot.passed);
+ if (testPassed) passedTests++;
+
+ totalScreenshots += test.screenshots.length;
+ passedScreenshots += test.screenshots.filter(s => s.passed).length;
+ }
+
+ // Generate HTML
+ const html = `
+
+
+
+
+
+ p5.js Visual Test Results
+
+
+
+
+
+
+
Summary
+
+ Total Tests: ${totalTests}
+ Passed Tests: ${passedTests} (${totalTests > 0 ? Math.round(passedTests/totalTests*100) : 0}%)
+ Failed Tests: ${totalTests - passedTests} (${totalTests > 0 ? Math.round((totalTests-passedTests)/totalTests*100) : 0}%)
+ Total Screenshots: ${totalScreenshots}
+ Passed Screenshots: ${passedScreenshots} (${totalScreenshots > 0 ? Math.round(passedScreenshots/totalScreenshots*100) : 0}%)
+ Report Generated: ${new Date().toLocaleString()}
+
+
+ ${testCases.map(test => {
+ const passed = test.screenshots.every(s => s.passed);
+ return `
+
+
+
+ ${test.screenshots.map(screenshot => `
+
+
+
+
+
+ ${screenshot.expectedImage ?
+ `
` :
+ `
No expected image found
`}
+
+
+
+ ${screenshot.actualImage ?
+ `
` :
+ `
No actual image found
`}
+
+ ${screenshot.diffImage ? `
+
+
+
+
+ ` : ''}
+
+
+ `).join('')}
+
+
+ `;
+ }).join('')}
+