Skip to content

Commit dae131d

Browse files
sserratasserrata
authored
Codex/create visual and html diff report (#1191)
* chore: raise diff tolerance * Limit details expansion to container * Fix TypeScript error when expanding details * Increase default visual diff tolerance * Capture only container when screenshotting * Pad mismatched screenshots
1 parent 0dfe328 commit dae131d

File tree

3 files changed

+184
-11
lines changed

3 files changed

+184
-11
lines changed

‎.github/workflows/deploy-preview.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ jobs:
179179
- name: Run visual diff
180180
run: yarn ts-node scripts/sitemap-visual-diff.ts --preview-url ${{ needs.deploy.outputs.preview_url }} --summary-file visual_diffs/results.json --concurrency 4 --paths "/tests/"
181181

182+
- name: Generate report and summary
183+
run: yarn ts-node scripts/generate-visual-diff-report.ts visual_diffs/results.json visual_diffs/index.html
184+
182185
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
183186
if: always()
184187
with:

‎scripts/generate-visual-diff-report.ts

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/usr/bin/env ts-node
2+
/* ============================================================================
3+
* Copyright (c) Palo Alto Networks
4+
*
5+
* This source code is licensed under the MIT license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
* ========================================================================== */
8+
9+
import fs from "fs";
10+
import path from "path";
11+
12+
interface Summary {
13+
total: number;
14+
matches: number;
15+
mismatches: number;
16+
skipped: number;
17+
}
18+
19+
interface Page {
20+
path: string;
21+
status: string;
22+
}
23+
24+
function clean(p: string): string {
25+
return p.replace(/^\//, "").replace(/\/$/, "") || "root";
26+
}
27+
28+
function generateHTML(results: { summary: Summary; pages: Page[] }): string {
29+
const pages = results.pages.map((p) => ({ ...p, clean: clean(p.path) }));
30+
const listItems = pages
31+
.map((p, i) => {
32+
return `<li><a href="#" data-index="${i}" class="status-${p.status}">${p.path}</a></li>`;
33+
})
34+
.join("\n");
35+
return `<!doctype html>
36+
<html lang="en">
37+
<head>
38+
<meta charset="UTF-8" />
39+
<title>Visual Diff Report</title>
40+
<style>
41+
body { margin:0; display:flex; height:100vh; font-family: Arial, sans-serif; }
42+
#sidebar { width:250px; background:#f7f7f7; overflow-y:auto; border-right:1px solid #ddd; padding:10px; }
43+
#content { flex:1; overflow:auto; padding:10px; }
44+
#diffImages { display:flex; gap:10px; margin-top:1em; }
45+
#diffImages img { border:1px solid #ccc; max-width:100%; background:#000; }
46+
.status-diff { color:#d33; }
47+
.status-match { color:#090; }
48+
.status-skip { color:#999; }
49+
</style>
50+
</head>
51+
<body>
52+
<div id="sidebar">
53+
<h2>Pages</h2>
54+
<ul>${listItems}</ul>
55+
</div>
56+
<div id="content">
57+
<h2 id="title">Select a page</h2>
58+
<div id="diffImages" style="display:none">
59+
<div><div>Prod</div><img id="img-prod" /></div>
60+
<div><div>Preview</div><img id="img-prev" /></div>
61+
<div><div>Diff</div><img id="img-diff" /></div>
62+
</div>
63+
</div>
64+
<script>
65+
const pages = ${JSON.stringify(pages)};
66+
function show(i){
67+
const p = pages[i];
68+
if(!p) return;
69+
document.getElementById('title').textContent = p.path + ' (' + p.status + ')';
70+
document.getElementById('img-prod').src = 'prod/' + p.clean + '.png';
71+
document.getElementById('img-prev').src = 'preview/' + p.clean + '.png';
72+
document.getElementById('img-diff').src = 'diff/' + p.clean + '.png';
73+
document.getElementById('diffImages').style.display = 'flex';
74+
}
75+
document.querySelectorAll('#sidebar a').forEach((a) => {
76+
a.addEventListener('click', function(e){
77+
e.preventDefault();
78+
show(this.getAttribute('data-index'));
79+
});
80+
});
81+
if(pages.length) show(0);
82+
</script>
83+
</body>
84+
</html>`;
85+
}
86+
87+
function encodeImage(filePath: string): string {
88+
try {
89+
const data = fs.readFileSync(filePath);
90+
return `data:image/png;base64,${data.toString("base64")}`;
91+
} catch {
92+
return "";
93+
}
94+
}
95+
96+
function generateMarkdown(
97+
results: { summary: Summary; pages: Page[] },
98+
baseDir: string
99+
): string {
100+
const lines: string[] = [];
101+
lines.push("### Visual Diff Summary\n");
102+
lines.push(
103+
`Total: ${results.summary.total}, Matches: ${results.summary.matches}, Diffs: ${results.summary.mismatches}, Skipped: ${results.summary.skipped}\n`
104+
);
105+
if (results.pages.length) {
106+
lines.push("| Page | Status | Prod | Preview | Diff |");
107+
lines.push("| --- | --- | --- | --- | --- |");
108+
for (const p of results.pages) {
109+
const cleanPath = clean(p.path);
110+
if (p.status === "diff") {
111+
const prod = encodeImage(
112+
path.join(baseDir, "prod", `${cleanPath}.png`)
113+
);
114+
const prev = encodeImage(
115+
path.join(baseDir, "preview", `${cleanPath}.png`)
116+
);
117+
const diff = encodeImage(
118+
path.join(baseDir, "diff", `${cleanPath}.png`)
119+
);
120+
lines.push(
121+
`| ${p.path} | <span style="color:#d33">diff</span> | ![](${prod}) | ![](${prev}) | ![](${diff}) |`
122+
);
123+
} else {
124+
const color = p.status === "match" ? "#090" : "#999";
125+
lines.push(
126+
`| ${p.path} | <span style="color:${color}">${p.status}</span> | | | |`
127+
);
128+
}
129+
}
130+
}
131+
lines.push("");
132+
lines.push("[Download full report](./visual_diffs/index.html)\n");
133+
return lines.join("\n");
134+
}
135+
136+
function main() {
137+
const input = process.argv[2] || path.join("visual_diffs", "results.json");
138+
const output = process.argv[3] || path.join("visual_diffs", "index.html");
139+
const results = JSON.parse(fs.readFileSync(input, "utf8"));
140+
const html = generateHTML(results);
141+
fs.writeFileSync(output, html);
142+
143+
const summaryPath = process.env.GITHUB_STEP_SUMMARY;
144+
if (summaryPath) {
145+
const markdown = generateMarkdown(results, path.dirname(output));
146+
fs.appendFileSync(summaryPath, `${markdown}\n`);
147+
}
148+
}
149+
150+
main();

‎scripts/sitemap-visual-diff.ts

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ function parseArgs(): Options {
3131
const opts: Options = {
3232
previewUrl: "",
3333
outputDir: "visual_diffs",
34-
tolerance: 0,
34+
tolerance: 0.3,
3535
width: 1280,
3636
viewHeight: 1024,
3737
concurrency: 4,
@@ -100,16 +100,31 @@ function parseUrlsFromSitemap(xml: string): string[] {
100100
async function screenshotFullPage(page: any, url: string, outputPath: string) {
101101
await page.goto(url, { waitUntil: "networkidle" });
102102
await page.evaluate(() => {
103-
document.querySelectorAll("details").forEach((d) => {
104-
const summary = d.querySelector("summary");
105-
if (!d.open && summary) (summary as HTMLElement).click();
106-
(d as HTMLDetailsElement).open = true;
107-
d.setAttribute("data-collapsed", "false");
103+
document.querySelectorAll("div.container details").forEach((el) => {
104+
const detail = el as HTMLDetailsElement;
105+
const summary = detail.querySelector("summary");
106+
if (!detail.open && summary) (summary as HTMLElement).click();
107+
detail.open = true;
108+
detail.setAttribute("data-collapsed", "false");
108109
});
109110
});
110111
await page.waitForTimeout(500);
111112
await fs.promises.mkdir(path.dirname(outputPath), { recursive: true });
112-
await page.screenshot({ path: outputPath, fullPage: true });
113+
const container = await page.$("div.container");
114+
if (container) {
115+
await container.screenshot({ path: outputPath });
116+
} else {
117+
await page.screenshot({ path: outputPath, fullPage: true });
118+
}
119+
}
120+
121+
function padImage(img: PNG, width: number, height: number): PNG {
122+
if (img.width === width && img.height === height) {
123+
return img;
124+
}
125+
const out = new PNG({ width, height });
126+
PNG.bitblt(img, out, 0, 0, img.width, img.height, 0, 0);
127+
return out;
113128
}
114129

115130
function compareImages(
@@ -119,11 +134,16 @@ function compareImages(
119134
tolerance: number,
120135
diffAlpha: number
121136
): boolean {
122-
const prod = PNG.sync.read(fs.readFileSync(prodPath));
123-
const prev = PNG.sync.read(fs.readFileSync(prevPath));
137+
let prod = PNG.sync.read(fs.readFileSync(prodPath));
138+
let prev = PNG.sync.read(fs.readFileSync(prevPath));
124139
if (prod.width !== prev.width || prod.height !== prev.height) {
125-
console.warn(`Size mismatch for ${prevPath}`);
126-
return false;
140+
const width = Math.max(prod.width, prev.width);
141+
const height = Math.max(prod.height, prev.height);
142+
console.warn(
143+
`Size mismatch for ${prevPath}, padding images to ${width}x${height}`
144+
);
145+
prod = padImage(prod, width, height);
146+
prev = padImage(prev, width, height);
127147
}
128148
const diff = new PNG({ width: prod.width, height: prod.height });
129149
const numDiff = pixelmatch(

0 commit comments

Comments
 (0)