|
| 1 | +'use strict' |
| 2 | + |
| 3 | +// A/B of two checkouts of this repo against the same database: the pg of `--head` over the pg of |
| 4 | +// `--base`, scenario by scenario. Both arms run on the same machine in the same run, so what the |
| 5 | +// machine does moves them together and the ratio holds still; that is the only number worth |
| 6 | +// reading on a hosted runner, where the absolute queries per second are never the same twice. |
| 7 | +// |
| 8 | +// Each arm runs twice, in two processes, so every round also measures base against base and head |
| 9 | +// against head: the same code on both sides, so whatever those ratios do is the noise of this |
| 10 | +// run on this machine, and a head/base ratio is marked only when it moved further than that. |
| 11 | +// All four stay up but only one measures at a time, the others wait: the load alternates between |
| 12 | +// them one scenario at a time, in an order that changes every round, so the measurements behind |
| 13 | +// a ratio are seconds apart and a drift of the machine lands on all of them. The reported |
| 14 | +// speedup is the median of the per-round ratios. |
| 15 | + |
| 16 | +const fs = require('fs') |
| 17 | +const os = require('os') |
| 18 | +const path = require('path') |
| 19 | +const { execFileSync, fork } = require('child_process') |
| 20 | + |
| 21 | +// below this nothing is marked whatever the noise said: a same-code band can come out very |
| 22 | +// narrow by luck on a handful of rounds, and a change this small is not worth a look anyway |
| 23 | +const FLOOR = 0.02 |
| 24 | + |
| 25 | +const parseArgs = (argv) => { |
| 26 | + const args = {} |
| 27 | + for (let i = 0; i < argv.length; i++) { |
| 28 | + if (argv[i].startsWith('--')) args[argv[i].slice(2)] = argv[++i] |
| 29 | + } |
| 30 | + return args |
| 31 | +} |
| 32 | + |
| 33 | +const median = (values) => { |
| 34 | + const sorted = [...values].sort((a, b) => a - b) |
| 35 | + const mid = Math.floor(sorted.length / 2) |
| 36 | + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2 |
| 37 | +} |
| 38 | + |
| 39 | +const label = (dir) => { |
| 40 | + try { |
| 41 | + return execFileSync('git', ['-C', dir, 'rev-parse', '--short', 'HEAD'], { encoding: 'utf8' }).trim() |
| 42 | + } catch { |
| 43 | + return path.basename(path.resolve(dir)) |
| 44 | + } |
| 45 | +} |
| 46 | + |
| 47 | +// one worker per arm, driven by messages: each send resolves with the worker's reply |
| 48 | +const startArm = (dir, index) => { |
| 49 | + const child = fork(path.join(__dirname, 'worker.js'), [], { |
| 50 | + execArgv: ['--expose-gc'], |
| 51 | + env: { ...process.env, PG_BENCH_MODULE: path.resolve(dir), PG_BENCH_ARM: String(index) }, |
| 52 | + }) |
| 53 | + const waiting = [] |
| 54 | + child.on('message', (msg) => { |
| 55 | + const { resolve, reject } = waiting.shift() |
| 56 | + msg.ok ? resolve(msg) : reject(new Error(msg.error)) |
| 57 | + }) |
| 58 | + child.on('exit', (code) => { |
| 59 | + for (const { reject } of waiting.splice(0)) reject(new Error(`worker for ${dir} exited with ${code}`)) |
| 60 | + }) |
| 61 | + const ready = new Promise((resolve, reject) => waiting.push({ resolve, reject })) |
| 62 | + return { |
| 63 | + ready, |
| 64 | + send: (msg) => |
| 65 | + new Promise((resolve, reject) => { |
| 66 | + waiting.push({ resolve, reject }) |
| 67 | + child.send(msg) |
| 68 | + }), |
| 69 | + } |
| 70 | +} |
| 71 | + |
| 72 | +const percent = (ratio) => `${ratio >= 1 ? '+' : ''}${((ratio - 1) * 100).toFixed(1)}%` |
| 73 | + |
| 74 | +const markdown = (labels, rows, rounds, duration) => { |
| 75 | + const lines = ['<!-- benchmark-comment -->', '', `## Benchmark: \`${labels.head}\` against \`${labels.base}\``, ''] |
| 76 | + lines.push('| Scenario | Base q/s | Head q/s | Head / base | Rounds | Noise |') |
| 77 | + lines.push('| --- | ---: | ---: | ---: | ---: | ---: |') |
| 78 | + for (const row of rows) { |
| 79 | + const mark = row.notable ? (row.speedup < 1 ? ' :eyes:' : ' :trophy:') : '' |
| 80 | + const change = row.notable ? `**${percent(row.speedup)}**` : percent(row.speedup) |
| 81 | + lines.push( |
| 82 | + `| ${row.name}${mark} | ${row.base.toFixed(0)} | ${row.head.toFixed(0)} | ` + |
| 83 | + `${row.speedup.toFixed(3)}x (${change}) | ${row.min.toFixed(2)} to ${row.max.toFixed(2)} | ` + |
| 84 | + `±${(row.noise * 100).toFixed(1)}% |` |
| 85 | + ) |
| 86 | + } |
| 87 | + lines.push('') |
| 88 | + lines.push( |
| 89 | + `${rounds} rounds of ${duration}s per scenario, each round measuring base, head and a second process of ` + |
| 90 | + `each one after the other, in an order that changes every round. "Head / base" is the median of the ` + |
| 91 | + `per-round ratios and "Rounds" their range. "Noise" is how far base/base and head/head, the same code on ` + |
| 92 | + `both sides, got from 1 in this same run: that is what the machine did, so a row is marked only when the ` + |
| 93 | + `median moved further than that, at least ${Math.round(FLOOR * 100)}%, and every round moved the same ` + |
| 94 | + `way: :eyes: slower, :trophy: faster. Only the ratio is comparable across runs, the absolute q/s depend ` + |
| 95 | + `on the runner.` |
| 96 | + ) |
| 97 | + lines.push('') |
| 98 | + lines.push(`Node ${process.version}, ${os.cpus()[0]?.model || 'unknown cpu'}, ${os.cpus().length} cores.`) |
| 99 | + return lines.join('\n') + '\n' |
| 100 | +} |
| 101 | + |
| 102 | +const main = async () => { |
| 103 | + const args = parseArgs(process.argv.slice(2)) |
| 104 | + if (!args.base || !args.head) { |
| 105 | + throw new Error( |
| 106 | + 'usage: node benchmark/compare.js --base <dir> --head <dir> [--rounds 4] [--duration 3] [--output file]' |
| 107 | + ) |
| 108 | + } |
| 109 | + const rounds = Number(args.rounds || 4) |
| 110 | + const duration = Number(args.duration || 3) |
| 111 | + const warmup = Number(args.warmup || 1) |
| 112 | + const labels = { base: label(args.base), head: label(args.head) } |
| 113 | + |
| 114 | + // two processes per arm: base2 and head2 are the same code as base and head, measured |
| 115 | + // alongside them so the run can tell how much two identical arms differ on this machine |
| 116 | + const arms = { |
| 117 | + base: startArm(args.base, 0), |
| 118 | + head: startArm(args.head, 1), |
| 119 | + base2: startArm(args.base, 2), |
| 120 | + head2: startArm(args.head, 3), |
| 121 | + } |
| 122 | + const names = Object.keys(arms) |
| 123 | + const { scenarios } = await arms.base.ready |
| 124 | + for (const name of names) await arms[name].ready |
| 125 | + await arms.base.send({ type: 'setup' }) |
| 126 | + |
| 127 | + const ratio = (qps, over, under) => qps[over].map((value, i) => value / qps[under][i]) |
| 128 | + const rows = [] |
| 129 | + for (const scenario of scenarios) { |
| 130 | + process.stderr.write(`${scenario}\n`) |
| 131 | + const qps = Object.fromEntries(names.map((name) => [name, []])) |
| 132 | + for (let i = -1; i < rounds; i++) { |
| 133 | + // the first round is a warmup on cold code and is thrown away |
| 134 | + const ms = (i < 0 ? warmup : duration) * 1000 |
| 135 | + // rotates one place per round and flips on odd rounds, so each arm sees every position |
| 136 | + const rotated = names.map((_, k) => names[(k + i + 1) % names.length]) |
| 137 | + const order = i % 2 ? rotated.reverse() : rotated |
| 138 | + for (const arm of order) { |
| 139 | + const reply = await arms[arm].send({ type: 'run', scenario, ms }) |
| 140 | + if (i >= 0) qps[arm].push(reply.qps) |
| 141 | + } |
| 142 | + } |
| 143 | + const ratios = ratio(qps, 'head', 'base') |
| 144 | + const same = [...ratio(qps, 'base2', 'base'), ...ratio(qps, 'head2', 'head')] |
| 145 | + const speedup = median(ratios) |
| 146 | + const min = Math.min(...ratios) |
| 147 | + const max = Math.max(...ratios) |
| 148 | + const noise = Math.max(...same.map((value) => Math.abs(value - 1))) |
| 149 | + process.stderr.write( |
| 150 | + ` ${speedup.toFixed(3)}x (${min.toFixed(2)} to ${max.toFixed(2)}), noise ±${(noise * 100).toFixed(1)}%\n` |
| 151 | + ) |
| 152 | + rows.push({ |
| 153 | + name: scenario, |
| 154 | + base: median(qps.base), |
| 155 | + head: median(qps.head), |
| 156 | + speedup, |
| 157 | + min, |
| 158 | + max, |
| 159 | + noise, |
| 160 | + notable: Math.abs(speedup - 1) >= Math.max(noise, FLOOR) && (min > 1 || max < 1), |
| 161 | + }) |
| 162 | + } |
| 163 | + await Promise.all(names.map((name) => arms[name].send({ type: 'end' }))) |
| 164 | + |
| 165 | + const summary = markdown(labels, rows, rounds, duration) |
| 166 | + process.stdout.write(summary) |
| 167 | + if (args.output) fs.writeFileSync(args.output, summary) |
| 168 | +} |
| 169 | + |
| 170 | +main().catch((err) => { |
| 171 | + process.stderr.write(`${err.stack || err}\n`) |
| 172 | + process.exit(1) |
| 173 | +}) |
0 commit comments