1import * as AWS from "alchemy/AWS";
2import * as Alchemy from "alchemy";
3import * as Cloudflare from "alchemy/Cloudflare";
4import * as Test from "alchemy/Test/Bun";
5import { expect } from "bun:test";
6import * as Effect from "effect/Effect";
7import * as FileSystem from "effect/FileSystem";
8import * as Layer from "effect/Layer";
9import * as Path from "effect/Path";
10import * as Schedule from "effect/Schedule";
11import * as HttpClient from "effect/http/HttpClient";
12import * as HttpClientRequest from "effect/http/HttpClientRequest";
13import Stack, { benchMicrovm } from "../alchemy.run.ts";
14
15/**
16 * Cold-start benchmark driver. Deploys the {@link Stack} once, drives a
17 * batched-concurrent boot/shutdown load against every variant, records each
18 * sample DIRECTLY (no stdout parsing), then writes:
19 *
20 * - `data/samples-<run>.csv` — one row per boot (raw, for plotting)
21 * - `report/summary.csv` — per-variant aggregate stats
22 * - `report/report.md` — human-readable markdown report
23 *
24 * Methodology: targets run sequentially so they never contend for shared
25 * quotas. Each round uses distinct keys → distinct instances; a round boots its
26 * instances, waits for each to reach usable service, then shuts them all down
27 * before the next round, and the round-over-round trend shows image/edge warm-up.
28 *
29 * - Containers are measured UNDER CONCURRENT LOAD: BATCHES rounds × CONCURRENCY
30 * simultaneous boots (no comparable per-instance API throttle).
31 * - MicroVMs are measured INDEPENDENTLY: MICROVM_BOOTS serial boots at
32 * concurrency 1. The Lambda MicroVM API is throttled per account/Region
33 * (RunMicrovm 5 TPS/burst 5, TerminateMicrovm 10 TPS), so booting >1 at once
34 * would measure API admission, not VM cold start.
35 *
36 * MicroVM variants (Lambda + cross-cloud Worker hosts) only run when the stack
37 * was deployed with `BENCH_MICROVM=1`. Set `NO_DESTROY=1` to keep the deploy
38 * between runs while iterating.
39 */
40const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
41 providers: Layer.mergeAll(AWS.providers(), Cloudflare.providers()),
42 state: Alchemy.localState(),
43});
44
45const HOOK_TIMEOUT = 1_500_000;
46const TEST_TIMEOUT = 3_000_000;
47
48// Containers tolerate high concurrency (no comparable per-instance API limit),
49// so they're measured under concurrent load: CONCURRENCY boots per round.
50const CONCURRENCY = Number(process.env.BENCH_CONCURRENCY ?? 10);
51const BATCHES = Number(process.env.BENCH_BATCHES ?? 10);
52
53// MicroVM cold starts MUST be measured independently. The Lambda MicroVM API is
54// throttled per account/Region — RunMicrovm is 5 TPS / burst 5, SuspendMicrovm
55// 2 TPS, TerminateMicrovm 10 TPS
56// (https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html#microvms-quotas).
57// Booting N>1 at once means boots 2..N partly measure API admission/throttling
58// instead of VM cold start. So MicroVM defaults to concurrency 1 (each boot is a
59// fully isolated launch→ready→terminate), sampled MICROVM_BOOTS times serially.
60const MICROVM_CONCURRENCY = Number(process.env.BENCH_MICROVM_CONCURRENCY ?? 1);
61const MICROVM_BOOTS = Number(process.env.BENCH_MICROVM_BOOTS ?? 25);
62
63// Optional comma-separated variant filter (e.g. `BENCH_VARIANTS=opencode`) to
64// re-run a subset without re-measuring everything.
65const VARIANT_FILTER = process.env.BENCH_VARIANTS
66 ? new Set(process.env.BENCH_VARIANTS.split(","))
67 : undefined;
68const REQUEST_TIMEOUT = "240 seconds";
69
70const DEPLOY_PLACEHOLDER = "Alchemy worker is being deployed...";
71
72// Force `Connection: close` so each request opens a fresh connection rather
73// than pinning to one edge/host over a pooled keep-alive socket.
74const freshConn = HttpClient.mapRequest(
75 HttpClientRequest.setHeader("connection", "close"),
76);
77
78interface Target {
79 /** Coarse environment, e.g. `container`, `lambda→microvm`, `worker→microvm`. */
80 readonly env: string;
81 readonly host: string;
82 readonly variant: string;
83 readonly label: string;
84 /** Query param naming the instance: containers use `name`, MicroVMs use `key`. */
85 readonly keyParam: "name" | "key";
86 /** Shutdown addresses by the boot key (`name`) or by the returned id (`id`). */
87 readonly shutdownBy: "name" | "id";
88 /** Instances booted simultaneously per round (1 = fully isolated boots). */
89 readonly concurrency: number;
90 /** Number of rounds; total samples = rounds × concurrency. */
91 readonly rounds: number;
92}
93
94interface Sample {
95 readonly env: string;
96 readonly variant: string;
97 readonly label: string;
98 readonly batch: number;
99 readonly key: string;
100 /** Inside the host: start → usable service. */
101 readonly readyMs: number | undefined;
102 /** Wall-clock latency of the boot request, measured by the client (outside). */
103 readonly outside: number;
104 readonly ok: boolean;
105 readonly error: string | undefined;
106}
107
108const waitForHost = (url: string) =>
109 Effect.gen(function* () {
110 const client = freshConn(yield* HttpClient.HttpClient);
111 yield* client.get(url).pipe(
112 Effect.flatMap((r) =>
113 r.status !== 200
114 ? Effect.fail(new Error(`host not ready: ${r.status}`))
115 : Effect.flatMap(r.text, (body) =>
116 body.includes(DEPLOY_PLACEHOLDER) || !body.includes("ok")
117 ? Effect.fail(new Error(`not ready: ${body.slice(0, 80)}`))
118 : Effect.succeed(body),
119 ),
120 ),
121 Effect.timeout("30 seconds"),
122 Effect.retry({
123 schedule: Schedule.min([Schedule.exponential("500 millis"), Schedule.spaced("3 seconds")]),
124 times: 30,
125 }),
126 );
127 });
128
129// Boot ONE fresh instance and time it from the outside. Returns the recorded
130// sample plus the instance id (when the host reports one, for id-based
131// shutdown). Does NOT shut down — the batch tears its instances down together.
132const bootOne = (t: Target, batch: number, key: string) =>
133 Effect.gen(function* () {
134 const client = freshConn(yield* HttpClient.HttpClient);
135 const q = `variant=${t.variant}&${t.keyParam}=${encodeURIComponent(key)}`;
136 const start = yield* Effect.sync(() => Date.now());
137 const result = yield* client.get(`${t.host}/boot?${q}`).pipe(
138 Effect.flatMap((r) =>
139 Effect.map(r.text, (body) => ({ status: r.status, body })),
140 ),
141 Effect.timeout(REQUEST_TIMEOUT),
142 Effect.map((res) => ({ ok: true as const, ...res })),
143 Effect.catch((err) =>
144 Effect.succeed({ ok: false as const, error: String(err) }),
145 ),
146 );
147 const outside = (yield* Effect.sync(() => Date.now())) - start;
148
149 const base = {
150 env: t.env,
151 variant: t.variant,
152 label: t.label,
153 batch,
154 key,
155 outside,
156 };
157
158 if (!result.ok) {
159 return {
160 sample: { ...base, readyMs: undefined, ok: false, error: result.error },
161 id: undefined,
162 };
163 }
164 if (result.status !== 200) {
165 return {
166 sample: {
167 ...base,
168 readyMs: undefined,
169 ok: false,
170 error: `HTTP ${result.status} ${result.body.slice(0, 160)}`,
171 },
172 id: undefined,
173 };
174 }
175 const parsed = (() => {
176 try {
177 return JSON.parse(result.body) as { id?: string; readyMs?: number };
178 } catch {
179 return {} as { id?: string; readyMs?: number };
180 }
181 })();
182 return {
183 sample: { ...base, readyMs: parsed.readyMs, ok: true, error: undefined },
184 id: parsed.id,
185 };
186 });
187
188// Stop one instance (best-effort, untimed) so the next batch is a fresh cold
189// start and we never hold more than `concurrency` instances at once.
190const shutdownOne = (t: Target, key: string, id: string | undefined) =>
191 Effect.gen(function* () {
192 const client = freshConn(yield* HttpClient.HttpClient);
193 const target = t.shutdownBy === "id" ? id : key;
194 if (target === undefined) return;
195 const q = `variant=${t.variant}&${t.shutdownBy}=${encodeURIComponent(target)}`;
196 yield* client
197 .get(`${t.host}/shutdown?${q}`)
198 .pipe(Effect.timeout("60 seconds"), Effect.ignore);
199 });
200
201const runTarget = (t: Target, nonce: string) =>
202 Effect.gen(function* () {
203 // Untimed warm-up: the FIRST pull of a freshly-pushed image onto a cold
204 // edge/host costs tens of seconds — a one-time DEPLOY artifact, not a
205 // per-cold-start cost. Boot+shutdown once before timing so the image is
206 // distributed and we measure container/VM cold start, not first-pull.
207 const warm = `${nonce}-${t.env}-${t.variant}-warm`;
208 const { id: warmId } = yield* bootOne(t, 0, warm);
209 yield* shutdownOne(t, warm, warmId);
210
211 const samples: Sample[] = [];
212 for (let b = 1; b <= t.rounds; b++) {
213 const keys = Array.from(
214 { length: t.concurrency },
215 (_, i) => `${nonce}-${t.env}-${t.variant}-b${b}-${i}`,
216 );
217 const outcomes = yield* Effect.forEach(
218 keys,
219 (key) => bootOne(t, b, key),
220 { concurrency: t.concurrency },
221 );
222 samples.push(...outcomes.map((o) => o.sample));
223 yield* Effect.forEach(
224 outcomes.map((o, i) => ({ key: keys[i], id: o.id })),
225 ({ key, id }) => shutdownOne(t, key, id),
226 { concurrency: t.concurrency },
227 );
228 }
229 return samples;
230 });
231
232const stats = (xs: ReadonlyArray<number>) => {
233 if (xs.length === 0) {
234 return { n: 0, min: 0, max: 0, mean: 0, p50: 0, p95: 0 };
235 }
236 const sorted = [...xs].sort((a, b) => a - b);
237 const pct = (p: number) =>
238 sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))];
239 return {
240 n: sorted.length,
241 min: sorted[0],
242 max: sorted[sorted.length - 1],
243 mean: Math.round(sorted.reduce((a, b) => a + b, 0) / sorted.length),
244 p50: pct(50),
245 p95: pct(95),
246 };
247};
248
249const sN = (n: number) => `${(n / 1000).toFixed(1)}s`;
250
251const csvEscape = (v: string | number | boolean | undefined) => {
252 const s = v === undefined ? "" : String(v);
253 return /[",\n]/.test(s) ? `"${s.replaceAll('"', '""')}"` : s;
254};
255
256const buildSamplesCsv = (samples: ReadonlyArray<Sample>) => {
257 const header = [
258 "env",
259 "variant",
260 "label",
261 "batch",
262 "key",
263 "readyMs",
264 "outside",
265 "ok",
266 "error",
267 ].join(",");
268 const rows = samples.map((s) =>
269 [
270 s.env,
271 s.variant,
272 s.label,
273 s.batch,
274 s.key,
275 s.readyMs ?? "",
276 s.outside,
277 s.ok,
278 s.error,
279 ]
280 .map(csvEscape)
281 .join(","),
282 );
283 return [header, ...rows].join("\n") + "\n";
284};
285
286// One aggregate row per (env, variant): ready-to-service stats + per-batch
287// mean trend, computed from the readyMs of successful samples.
288const summarize = (samples: ReadonlyArray<Sample>) => {
289 const groups = new Map<string, Sample[]>();
290 for (const s of samples) {
291 const k = `${s.env}\u0000${s.variant}`;
292 (groups.get(k) ?? groups.set(k, []).get(k)!).push(s);
293 }
294 return [...groups.values()].map((g) => {
295 const ready = stats(
296 g.map((s) => s.readyMs).filter((m): m is number => typeof m === "number"),
297 );
298 const outside = stats(g.filter((s) => s.ok).map((s) => s.outside));
299 // Variable round count per target (containers vs serial MicroVM), so derive
300 // the span from the samples themselves rather than a global batch count.
301 const maxBatch = g.reduce((m, s) => Math.max(m, s.batch), 0);
302 const byBatch: Array<number | undefined> = [];
303 for (let b = 1; b <= maxBatch; b++) {
304 const xs = g
305 .filter((s) => s.batch === b)
306 .map((s) => s.readyMs)
307 .filter((m): m is number => typeof m === "number");
308 byBatch.push(xs.length > 0 ? stats(xs).mean : undefined);
309 }
310 return {
311 env: g[0].env,
312 variant: g[0].variant,
313 label: g[0].label,
314 ok: g.filter((s) => s.ok).length,
315 total: g.length,
316 ready,
317 outside,
318 byBatch,
319 };
320 });
321};
322
323type Summary = ReturnType<typeof summarize>[number];
324
325const buildSummaryCsv = (rows: ReadonlyArray<Summary>) => {
326 const header = [
327 "env",
328 "variant",
329 "ok",
330 "total",
331 "ready_min_ms",
332 "ready_p50_ms",
333 "ready_p95_ms",
334 "ready_mean_ms",
335 "ready_max_ms",
336 // Per-round mean trend (semicolon-separated; round count varies by target).
337 "round_means_ms",
338 ].join(",");
339 const lines = rows.map((r) =>
340 [
341 r.env,
342 r.variant,
343 r.ok,
344 r.total,
345 r.ready.min,
346 r.ready.p50,
347 r.ready.p95,
348 r.ready.mean,
349 r.ready.max,
350 r.byBatch.map((b) => b ?? "").join(";"),
351 ]
352 .map(csvEscape)
353 .join(","),
354 );
355 return [header, ...lines].join("\n") + "\n";
356};
357
358const buildReport = (
359 rows: ReadonlyArray<Summary>,
360 meta: { runId: string; total: number },
361) => {
362 const head =
363 "| environment | variant | ok | ready p50 | ready p95 | ready mean | ready max | first → last round (mean) |";
364 const sep = "| --- | --- | --- | --- | --- | --- | --- | --- |";
365 const body = rows.map((r) => {
366 const first = r.byBatch[0];
367 const last = [...r.byBatch].reverse().find((b) => b !== undefined);
368 const trend =
369 first !== undefined && last !== undefined
370 ? `${sN(first)} → ${sN(last)}`
371 : "—";
372 return `| ${r.env} | ${r.variant} | ${r.ok}/${r.total} | ${sN(r.ready.p50)} | ${sN(r.ready.p95)} | ${sN(r.ready.mean)} | ${sN(r.ready.max)} | ${trend} |`;
373 });
374 return [
375 `# Cold-start benchmark`,
376 ``,
377 `Run \`${meta.runId}\` — ${meta.total} samples. Metric: **time to usable service** (host start → first successful request).`,
378 ``,
379 `Methodology: targets run sequentially so they never contend for shared quotas. Containers are measured under concurrent load (${BATCHES} rounds × ${CONCURRENCY} simultaneous boots). MicroVMs are measured **independently** — ${MICROVM_BOOTS} serial boots at concurrency ${MICROVM_CONCURRENCY} — because the Lambda MicroVM API is throttled per account/Region (RunMicrovm 5 TPS/burst 5, TerminateMicrovm 10 TPS), so concurrent boots would measure API admission rather than VM cold start. Each boot launches, waits until the service is usable, then terminates before the next; the image is pre-warmed once (untimed) so we measure cold start, not first-pull distribution.`,
380 ``,
381 head,
382 sep,
383 ...body,
384 ``,
385 `Raw per-boot samples: \`data/samples-${meta.runId}.csv\`. Aggregates: \`report/summary.csv\`.`,
386 ``,
387 ].join("\n");
388};
389
390const buildTargets = (outputs: {
391 containerWorkerUrl: string;
392 lambdaUrl?: string;
393 microvmWorkerUrl?: string;
394}): Target[] => {
395 const clean = (u: string) => u.replace(/\/+$/, "");
396 const targets: Target[] = [
397 {
398 env: "container",
399 host: clean(outputs.containerWorkerUrl),
400 variant: "effectful",
401 label: "Cloudflare container (bundled Effect image)",
402 keyParam: "name",
403 shutdownBy: "name",
404 concurrency: CONCURRENCY,
405 rounds: BATCHES,
406 },
407 {
408 env: "container",
409 host: clean(outputs.containerWorkerUrl),
410 variant: "bun",
411 label: "Cloudflare container (oven/bun, no Effect bundle)",
412 keyParam: "name",
413 shutdownBy: "name",
414 concurrency: CONCURRENCY,
415 rounds: BATCHES,
416 },
417 {
418 env: "container",
419 host: clean(outputs.containerWorkerUrl),
420 variant: "remote",
421 label: "Cloudflare container (remote echo image)",
422 keyParam: "name",
423 shutdownBy: "name",
424 concurrency: CONCURRENCY,
425 rounds: BATCHES,
426 },
427 {
428 env: "container",
429 host: clean(outputs.containerWorkerUrl),
430 variant: "opencode",
431 label: "Cloudflare container (opencode server, eager entrypoint)",
432 keyParam: "name",
433 shutdownBy: "name",
434 concurrency: CONCURRENCY,
435 rounds: BATCHES,
436 },
437 ];
438 if (benchMicrovm && outputs.lambdaUrl && outputs.microvmWorkerUrl) {
439 const lambda = clean(outputs.lambdaUrl);
440 const worker = clean(outputs.microvmWorkerUrl);
441 // Per host, the node-vs-bun matrix: effectful (alchemy/Effect bundle) on
442 // each runtime, the raw baseline on each runtime, plus the Python external.
443 const microvmVariants: ReadonlyArray<{ variant: string; label: string }> = [
444 { variant: "effectful-bun", label: "effectful (Effect bundle, bun)" },
445 { variant: "effectful-node", label: "effectful (Effect bundle, node)" },
446 { variant: "bun", label: "bun baseline (raw Bun.serve)" },
447 { variant: "node", label: "node baseline (raw http)" },
448 { variant: "external", label: "external (Python Dockerfile)" },
449 // The MicroVM image build starts opencode and snapshots the running
450 // memory, so each boot resumes with the server already up.
451 { variant: "opencode", label: "opencode (snapshotted eager server)" },
452 ];
453 for (const host of [
454 { env: "lambda→microvm", url: lambda },
455 { env: "worker→microvm", url: worker },
456 ]) {
457 for (const v of microvmVariants) {
458 targets.push({
459 env: host.env,
460 host: host.url,
461 variant: v.variant,
462 label: `${host.env} (${v.label})`,
463 keyParam: "key",
464 shutdownBy: "id",
465 // Isolated, serial boots — see the MICROVM_CONCURRENCY rationale above.
466 concurrency: MICROVM_CONCURRENCY,
467 rounds: MICROVM_BOOTS,
468 });
469 }
470 }
471 }
472 return targets;
473};
474
475const stack = beforeAll(deploy(Stack), { timeout: HOOK_TIMEOUT });
476afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(Stack), {
477 timeout: HOOK_TIMEOUT,
478});
479
480test(
481 "cold-start benchmark → data/*.csv + report/report.md",
482 Effect.gen(function* () {
483 const outputs = (yield* stack) as {
484 containerWorkerUrl: string;
485 lambdaUrl?: string;
486 microvmWorkerUrl?: string;
487 };
488 const targets = buildTargets(outputs).filter(
489 (t) => VARIANT_FILTER === undefined || VARIANT_FILTER.has(t.variant),
490 );
491
492 // Wait for each distinct host to answer before timing.
493 const hosts = [...new Set(targets.map((t) => t.host))];
494 yield* Effect.forEach(hosts, waitForHost, { concurrency: hosts.length });
495
496 const nonce = yield* Effect.sync(() => crypto.randomUUID().slice(0, 8));
497
498 const samples: Sample[] = [];
499 for (const t of targets) {
500 samples.push(...(yield* runTarget(t, nonce)));
501 }
502
503 const rows = summarize(samples);
504
505 const fs = yield* FileSystem.FileSystem;
506 const path = yield* Path.Path;
507 const root = path.join(import.meta.dirname, "..");
508 const dataDir = path.join(root, "data");
509 const reportDir = path.join(root, "report");
510 yield* fs.makeDirectory(dataDir, { recursive: true });
511 yield* fs.makeDirectory(reportDir, { recursive: true });
512
513 const runId = new Date().toISOString().replace(/[:.]/g, "-");
514 yield* fs.writeFileString(
515 path.join(dataDir, `samples-${runId}.csv`),
516 buildSamplesCsv(samples),
517 );
518 yield* fs.writeFileString(
519 path.join(reportDir, "summary.csv"),
520 buildSummaryCsv(rows),
521 );
522 yield* fs.writeFileString(
523 path.join(reportDir, "report.md"),
524 buildReport(rows, { runId, total: samples.length }),
525 );
526
527 yield* Effect.sync(() =>
528 console.log(
529 `\nWrote ${samples.length} samples → data/samples-${runId}.csv` +
530 `\nReport → report/report.md\n`,
531 ),
532 );
533
534 // Informational, but a run where nothing started indicates a broken deploy.
535 expect(samples.filter((s) => s.ok).length).toBeGreaterThan(0);
536 }),
537 { timeout: TEST_TIMEOUT },
538);