1import { spawnSync } from "node:child_process"
2import { createHash, randomBytes } from "node:crypto"
3import { mkdirSync, readFileSync, writeFileSync } from "node:fs"
4import { dirname, join, relative, resolve } from "node:path"
5import { fileURLToPath } from "node:url"
6import { aggregate } from "./stats.mts"
7
8export const runtimeperfDir = dirname(fileURLToPath(import.meta.url))
9export const effectDir = resolve(runtimeperfDir, "..")
10export const repoRoot = resolve(effectDir, "../..")
11export const runPath = join(runtimeperfDir, "run.mts")
12export const comparePath = join(runtimeperfDir, "compare.mts")
13export const materializePath = join(runtimeperfDir, "materialize.mts")
14export const statsPath = join(runtimeperfDir, "stats.mts")
15export const utilsPath = fileURLToPath(import.meta.url)
16export const workerPath = join(runtimeperfDir, "worker.mts")
17export const configPath = join(runtimeperfDir, "config.json")
18export const resultsRoot = join(repoRoot, "tmp", "runtimeperf", "results")
19
20export const readJson = (path) => JSON.parse(readFileSync(path, "utf8"))
21
22export const writeJson = (path, value) => {
23 mkdirSync(dirname(path), { recursive: true })
24 writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`)
25}
26
27export const sha256 = (value) => createHash("sha256").update(value).digest("hex")
28
29export const hashFile = (path) => sha256(readFileSync(path))
30
31export const libraryVersions = () => ({
32 effect: readJson(join(effectDir, "package.json")).version,
33 fastCheck: readJson(join(repoRoot, "node_modules", "fast-check", "package.json")).version,
34 tinybench: readJson(join(effectDir, "node_modules", "tinybench", "package.json")).version,
35 valibot: readJson(join(effectDir, "node_modules", "valibot", "package.json")).version,
36 zod: readJson(join(repoRoot, "node_modules", "zod", "package.json")).version,
37 zodExport: "zod/v4"
38})
39
40export const currentGitState = () => {
41 const git = (args) => {
42 const result = spawnSync("git", args, { cwd: repoRoot, encoding: "utf8" })
43 if (result.error) throw result.error
44 if (result.status !== 0) {
45 throw new Error(`${result.stdout}${result.stderr}`.trim())
46 }
47 return result.stdout.trim()
48 }
49 const status = git(["status", "--short", "--untracked-files=all"])
50 const diff = git(["diff", "--binary", "HEAD", "--"])
51 return {
52 sha: git(["rev-parse", "HEAD"]),
53 dirty: status !== "",
54 status: status === "" ? [] : status.split("\n"),
55 diffHash: sha256(diff)
56 }
57}
58
59export const makeRunId = () =>
60 `${new Date().toISOString().replace(/[:.]/g, "-")}-${process.pid}-${randomBytes(3).toString("hex")}`
61
62export const sanitize = (name) => name.replace(/[^a-zA-Z0-9._-]/g, "-")
63
64export const parseArgs = (args, { compare = false } = {}) => {
65 const options = {
66 target: undefined,
67 rounds: undefined,
68 timeMs: undefined,
69 warmupTimeMs: undefined,
70 tier: undefined,
71 family: undefined,
72 implementation: undefined,
73 base: "HEAD",
74 head: "worktree",
75 failOnRegression: false,
76 help: false
77 }
78 const valueOptions = new Map([
79 ["--rounds", "rounds"],
80 ["--time", "timeMs"],
81 ["--warmup-time", "warmupTimeMs"],
82 ["--tier", "tier"],
83 ["--family", "family"],
84 ["--implementation", "implementation"],
85 ["--base", "base"],
86 ["--head", "head"]
87 ])
88 for (let index = 0; index < args.length; index++) {
89 const arg = args[index]
90 if (arg === "--help" || arg === "-h") {
91 options.help = true
92 } else if (arg === "--fail-on-regression" && compare) {
93 options.failOnRegression = true
94 } else if (valueOptions.has(arg)) {
95 const value = args[++index]
96 if (value === undefined || value.startsWith("-")) {
97 throw new Error(`Missing value for ${arg}`)
98 }
99 options[valueOptions.get(arg)] = value
100 } else if (arg.startsWith("-")) {
101 throw new Error(`Unknown option: ${arg}`)
102 } else if (options.target === undefined) {
103 options.target = arg
104 } else {
105 throw new Error(`Expected at most one target, got ${options.target} and ${arg}`)
106 }
107 }
108 for (const key of ["rounds", "timeMs", "warmupTimeMs", "tier"]) {
109 if (options[key] !== undefined) {
110 const value = Number(options[key])
111 if (!Number.isInteger(value) || value < 0 || (key !== "tier" && value === 0)) {
112 throw new Error(`--${key} must be ${key === "tier" ? "a non-negative" : "a positive"} integer`)
113 }
114 options[key] = value
115 }
116 }
117 return options
118}
119
120export const loadRegistry = () => {
121 const config = readJson(configPath)
122 const fixtures = config.suites.flatMap((suite) =>
123 suite.fixtures.flatMap((fixtureGroup) =>
124 fixtureGroup.cases.map((runtimeCase) => ({
125 ...fixtureGroup.defaults,
126 ...runtimeCase,
127 suite: suite.name,
128 target: `${suite.name}/${runtimeCase.name}`,
129 fixturePath: resolve(runtimeperfDir, fixtureGroup.file)
130 }))
131 )
132 )
133 return { config, fixtures }
134}
135
136export const selectFixtures = (fixtures, options, { effectOnly = false } = {}) => {
137 let selected = fixtures
138 if (options.target !== undefined) {
139 selected = selected.filter((fixture) =>
140 fixture.suite === options.target ||
141 fixture.target === options.target ||
142 fixture.scenario === options.target
143 )
144 }
145 if (options.tier !== undefined) {
146 selected = selected.filter((fixture) => fixture.tier === options.tier)
147 }
148 if (options.family !== undefined) {
149 selected = selected.filter((fixture) => fixture.family === options.family)
150 }
151 if (options.implementation !== undefined) {
152 selected = selected.filter((fixture) => fixture.implementation === options.implementation)
153 }
154 if (effectOnly) {
155 selected = selected.filter((fixture) => fixture.implementation.startsWith("effect"))
156 }
157 if (selected.length === 0) {
158 throw new Error("No runtimeperf fixtures matched the selection")
159 }
160 return selected
161}
162
163export const resolveDefaults = (config, options) => ({
164 rounds: options.rounds ?? config.defaults.rounds,
165 timeMs: options.timeMs ?? config.defaults.timeMs,
166 warmupTimeMs: options.warmupTimeMs ?? config.defaults.warmupTimeMs,
167 targetBatchTimeNs: config.defaults.targetBatchTimeNs,
168 maxBatchSize: config.defaults.maxBatchSize,
169 bootstrapIterations: config.defaults.bootstrapIterations,
170 bootstrapSeed: config.defaults.bootstrapSeed,
171 minImprovementPercent: config.defaults.minImprovementPercent,
172 maxRegressionPercent: config.defaults.maxRegressionPercent
173})
174
175export const runWorker = (workerArgs) => {
176 const result = spawnSync(process.execPath, [workerPath, ...workerArgs], {
177 cwd: effectDir,
178 encoding: "utf8",
179 maxBuffer: 10 * 1024 * 1024
180 })
181 if (result.error) throw result.error
182 if (result.status !== 0) {
183 throw new Error(`${result.stderr || result.stdout}`.trim())
184 }
185 try {
186 return JSON.parse(result.stdout)
187 } catch {
188 throw new Error(`Worker returned invalid JSON: ${result.stdout}`)
189 }
190}
191
192export const calibrateFixture = (fixture, defaults, fixturePath = fixture.fixturePath) =>
193 runWorker([
194 "--mode",
195 "calibrate",
196 "--fixture",
197 fixturePath,
198 "--export",
199 fixture.export,
200 "--target-batch-time-ns",
201 String(defaults.targetBatchTimeNs),
202 "--max-batch-size",
203 String(defaults.maxBatchSize)
204 ])
205
206export const measureFixture = (fixture, defaults, batchSize, fixturePath = fixture.fixturePath) =>
207 runWorker([
208 "--mode",
209 "measure",
210 "--fixture",
211 fixturePath,
212 "--export",
213 fixture.export,
214 "--batch-size",
215 String(batchSize),
216 "--time-ms",
217 String(defaults.timeMs),
218 "--warmup-time-ms",
219 String(defaults.warmupTimeMs)
220 ])
221
222export const aggregateMeasurements = (measurements) => aggregate(measurements.map((item) => item.nsPerOp))
223
224export const scenarioBatchSize = (fixtures, calibrations) => {
225 const reference = fixtures.find((fixture) => fixture.implementation === "effect") ?? fixtures[0]
226 const calibration = calibrations.get(reference)
227 if (calibration === undefined) throw new Error("Missing scenario reference calibration")
228 return calibration.batchSize
229}
230
231export const coverageSummary = (fixtures) => ({
232 tiers: [...new Set(fixtures.map((fixture) => fixture.tier))].sort(),
233 families: [...new Set(fixtures.map((fixture) => fixture.family))].sort(),
234 implementations: [...new Set(fixtures.map((fixture) => fixture.implementation))].sort(),
235 effectAstTags: [
236 ...new Set(
237 fixtures
238 .filter((fixture) => fixture.implementation.startsWith("effect"))
239 .flatMap((fixture) => fixture.astTags)
240 )
241 ].sort()
242})
243
244export const formatNs = (value) => value < 1_000
245 ? value.toFixed(1)
246 : value < 1_000_000
247 ? `${(value / 1_000).toFixed(2)}µs`
248 : `${(value / 1_000_000).toFixed(2)}ms`
249
250export const printTable = (headers, rows) => {
251 const textRows = rows.map((row) => row.map(String))
252 const table = [headers, ...textRows]
253 const widths = headers.map((_, index) => Math.max(...table.map((row) => row[index].length)))
254 table.forEach((row, index) => {
255 process.stdout.write(`${row.map((cell, cellIndex) => cell.padEnd(widths[cellIndex])).join(" ")}\n`)
256 if (index === 0) {
257 process.stdout.write(`${widths.map((width) => "-".repeat(width)).join(" ")}\n`)
258 }
259 })
260}
261
262export const reportPath = (runId, target, kind) =>
263 join(resultsRoot, `${runId}-${kind}-${sanitize(target ?? "all")}.json`)
264
265export const relativeToRepo = (path) => relative(repoRoot, path)