1import { spawnSync } from "node:child_process"
2import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"
3import os from "node:os"
4import { dirname, join } from "node:path"
5import process from "node:process"
6import { materializeFixture } from "./materialize.mts"
7import { analyzePairs } from "./stats.mts"
8import {
9 aggregateMeasurements,
10 calibrateFixture,
11 comparePath,
12 configPath,
13 coverageSummary,
14 effectDir,
15 formatNs,
16 hashFile,
17 libraryVersions,
18 loadRegistry,
19 makeRunId,
20 materializePath,
21 measureFixture,
22 parseArgs,
23 printTable,
24 relativeToRepo,
25 repoRoot,
26 reportPath,
27 resolveDefaults,
28 selectFixtures,
29 sha256,
30 statsPath,
31 utilsPath,
32 workerPath,
33 writeJson
34} from "./utils.mts"
35import { applyWorktreeDiff, readWorktreeDiff } from "./worktree-diff.mts"
36
37const usage = `Usage: pnpm runtimeperf-compare [suite[/fixture]|scenario] [options]
38
39Options:
40 --base <ref> Defaults to HEAD
41 --head <ref> Defaults to worktree
42 --rounds <n>
43 --time <ms>
44 --warmup-time <ms>
45 --tier <0-3>
46 --family <name>
47 --fail-on-regression
48`
49
50const run = (command, args, options = {}) =>
51 spawnSync(command, args, {
52 encoding: "utf8",
53 maxBuffer: 50 * 1024 * 1024,
54 ...options
55 })
56
57const runGit = (args) => {
58 const result = run("git", args, { cwd: repoRoot })
59 if (result.error) throw result.error
60 if (result.status !== 0) {
61 throw new Error(`${result.stdout}${result.stderr}`.trim())
62 }
63 return result.stdout.trim()
64}
65
66const resolveRef = (ref) => runGit(["rev-parse", "--verify", `${ref}^{commit}`])
67
68const linkDirectory = (source, target) => {
69 if (!existsSync(target)) {
70 symlinkSync(source, target, process.platform === "win32" ? "junction" : "dir")
71 }
72}
73
74const removeWorktree = (path) => {
75 const result = run("git", ["worktree", "remove", "--force", path], { cwd: repoRoot })
76 if (result.status !== 0) {
77 process.stderr.write(`${result.stdout}${result.stderr}`)
78 }
79}
80
81const createWorktree = (runRoot, name, sha) => {
82 const path = join(runRoot, name)
83 const result = run("git", ["worktree", "add", "--detach", path, sha], { cwd: repoRoot })
84 if (result.error) throw result.error
85 if (result.status !== 0) {
86 throw new Error(`${result.stdout}${result.stderr}`.trim())
87 }
88 try {
89 linkDirectory(join(repoRoot, "node_modules"), join(path, "node_modules"))
90 linkDirectory(join(effectDir, "node_modules"), join(path, "packages", "effect", "node_modules"))
91 } catch (error) {
92 removeWorktree(path)
93 throw error
94 }
95 return path
96}
97
98const applyWorktreeChanges = (path, untracked) => {
99 applyWorktreeDiff(path, readWorktreeDiff(repoRoot))
100 for (const file of untracked) {
101 const target = join(path, file.path)
102 mkdirSync(dirname(target), { recursive: true })
103 cpSync(join(repoRoot, file.path), target, { recursive: true })
104 }
105}
106
107const worktreeState = () => {
108 const diff = readWorktreeDiff(repoRoot)
109 const untrackedOutput = runGit([
110 "ls-files",
111 "--others",
112 "--exclude-standard",
113 "--",
114 "package.json",
115 "packages/effect"
116 ])
117 const untracked = untrackedOutput === ""
118 ? []
119 : untrackedOutput.split("\n").map((path) => ({
120 path,
121 hash: hashFile(join(repoRoot, path))
122 }))
123 return {
124 dirty: diff !== "" || untracked.length > 0,
125 diffHash: sha256(diff),
126 untracked
127 }
128}
129
130const main = () => {
131 const options = parseArgs(process.argv.slice(2), { compare: true })
132 if (options.help) {
133 process.stdout.write(usage)
134 return
135 }
136 const { config, fixtures } = loadRegistry()
137 const selected = selectFixtures(fixtures, options, { effectOnly: true })
138 const defaults = resolveDefaults(config, options)
139 const baseSha = resolveRef(options.base)
140 const headSha = options.head === "worktree" ? resolveRef("HEAD") : resolveRef(options.head)
141 const state = worktreeState()
142 const runId = makeRunId()
143 const tmpRoot = join(repoRoot, "tmp", "runtimeperf", "worktrees")
144 mkdirSync(tmpRoot, { recursive: true })
145 const runRoot = mkdtempSync(join(tmpRoot, "run-"))
146 const worktrees = []
147 const results = []
148 const executionOrder = []
149
150 try {
151 const baseRoot = createWorktree(runRoot, "base", baseSha)
152 worktrees.push(baseRoot)
153 const headRoot = createWorktree(runRoot, "head", headSha)
154 worktrees.push(headRoot)
155 if (options.head === "worktree") applyWorktreeChanges(headRoot, state.untracked)
156
157 for (const fixture of selected) {
158 const baseFixturePath = materializeFixture(baseRoot, fixture)
159 const headFixturePath = materializeFixture(headRoot, fixture)
160 const baseCalibration = calibrateFixture(fixture, defaults, baseFixturePath)
161 const headCalibration = calibrateFixture(fixture, defaults, headFixturePath)
162 const batchSize = Math.max(baseCalibration.batchSize, headCalibration.batchSize)
163 const baseMeasurements = []
164 const headMeasurements = []
165
166 for (let round = 0; round < defaults.rounds; round++) {
167 const order = round % 2 === 0 ? ["base", "head"] : ["head", "base"]
168 for (const side of order) {
169 const measurement = side === "base"
170 ? measureFixture(fixture, defaults, batchSize, baseFixturePath)
171 : measureFixture(fixture, defaults, batchSize, headFixturePath)
172 ;(side === "base" ? baseMeasurements : headMeasurements).push(measurement)
173 executionOrder.push({ target: fixture.target, round: round + 1, side })
174 }
175 }
176
177 results.push({
178 fixture,
179 batchSize,
180 calibration: {
181 base: baseCalibration,
182 head: headCalibration
183 },
184 base: {
185 measurements: baseMeasurements,
186 aggregate: aggregateMeasurements(baseMeasurements)
187 },
188 head: {
189 measurements: headMeasurements,
190 aggregate: aggregateMeasurements(headMeasurements)
191 },
192 comparison: analyzePairs(
193 baseMeasurements.map((item) => item.nsPerOp),
194 headMeasurements.map((item) => item.nsPerOp),
195 {
196 iterations: defaults.bootstrapIterations,
197 seed: defaults.bootstrapSeed,
198 minImprovementPercent: defaults.minImprovementPercent,
199 maxRegressionPercent: defaults.maxRegressionPercent
200 }
201 )
202 })
203 }
204 } finally {
205 for (const worktree of worktrees.reverse()) {
206 removeWorktree(worktree)
207 }
208 rmSync(runRoot, { recursive: true, force: true })
209 }
210
211 const report = {
212 schemaVersion: 1,
213 kind: "comparison",
214 runId,
215 target: options.target ?? null,
216 filters: {
217 tier: options.tier ?? null,
218 family: options.family ?? null
219 },
220 config: defaults,
221 base: {
222 ref: options.base,
223 sha: baseSha
224 },
225 head: {
226 ref: options.head,
227 sha: headSha,
228 worktree: options.head === "worktree" ? state : undefined
229 },
230 environment: {
231 node: process.version,
232 v8: process.versions.v8,
233 platform: process.platform,
234 arch: process.arch,
235 cpu: os.cpus()[0]?.model ?? "unknown"
236 },
237 libraries: libraryVersions(),
238 artifactMode: "repository",
239 coverage: coverageSummary(selected),
240 hashes: {
241 compare: hashFile(comparePath),
242 config: hashFile(configPath),
243 materialize: hashFile(materializePath),
244 stats: hashFile(statsPath),
245 utils: hashFile(utilsPath),
246 worker: hashFile(workerPath),
247 fixtures: Object.fromEntries(
248 [...new Set(selected.map((fixture) => fixture.fixturePath))]
249 .map((path) => [relativeToRepo(path), hashFile(path)])
250 )
251 },
252 executionOrder,
253 results
254 }
255 const path = reportPath(runId, options.target, "compare")
256 writeJson(path, report)
257 printTable(
258 ["fixture", "base", "head", "delta", "ci low", "ci high", "status"],
259 results.map((result) => [
260 result.fixture.target,
261 formatNs(result.base.aggregate.median),
262 formatNs(result.head.aggregate.median),
263 `${result.comparison.deltaPercent.toFixed(2)}%`,
264 `${result.comparison.lowPercent.toFixed(2)}%`,
265 `${result.comparison.highPercent.toFixed(2)}%`,
266 result.comparison.status
267 ])
268 )
269 process.stdout.write(`\nReport: ${relativeToRepo(path)}\n`)
270 if (options.failOnRegression && results.some((result) => result.comparison.status === "regression")) {
271 process.exitCode = 1
272 }
273}
274
275try {
276 main()
277} catch (error) {
278 process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : error}\n`)
279 process.exitCode = 1
280}