1/**
2 * Standalone cold-start baseline for the plain wrangler + `@cloudflare/containers`
3 * worker — no Alchemy involved. Mirrors the methodology of the Alchemy driver
4 * (`benchmark/container/test/bench.test.ts`) so the two are directly
5 * comparable, but uses zero Alchemy code — just `fetch` against the deployed
6 * worker.
7 *
8 * Usage (from this directory):
9 * bun x wrangler deploy # build + push the image, deploy the worker
10 * WORKER_URL=https://<worker>.workers.dev bun bench.ts
11 * BENCH_N=2 BENCH_CONCURRENCY=2 WORKER_URL=... bun bench.ts
12 * BENCH_CRASH=1 WORKER_URL=... bun bench.ts # crash-loop fail-fast baseline
13 * bun x wrangler delete # tear down
14 *
15 * `wrangler` and `@cloudflare/containers` resolve from the `alchemy` package's
16 * devDependencies (catalog), so no separate install is needed.
17 */
18
19const WORKER_URL = (process.env.WORKER_URL ?? process.argv[2] ?? "").replace(
20 /\/$/,
21 "",
22);
23const N = Number(process.env.BENCH_N ?? 2);
24const CONCURRENCY = Number(process.env.BENCH_CONCURRENCY ?? N);
25const REQUEST_TIMEOUT_MS = Number(
26 process.env.BENCH_REQUEST_TIMEOUT_MS ?? 240_000,
27);
28// When set, benchmark how fast native surfaces a fatal crash (the baseline
29// Alchemy's fail-fast behaviour is compared against) instead of cold start.
30const CRASH = process.env.BENCH_CRASH === "1";
31const CRASH_N = Number(process.env.BENCH_CRASH_N ?? 10);
32
33if (!WORKER_URL) {
34 console.error(
35 "Set WORKER_URL to the deployed worker, e.g. WORKER_URL=https://x.workers.dev bun bench.ts",
36 );
37 process.exit(1);
38}
39
40interface Sample {
41 outside: number;
42 inside: number | undefined;
43}
44
45const wait = async (url: string) => {
46 for (let i = 0; i < 30; i++) {
47 try {
48 const res = await fetch(url, { headers: { connection: "close" } });
49 const body = await res.text();
50 if (res.status === 200 && body.includes("ok")) return;
51 } catch {
52 // retry
53 }
54 await new Promise((r) => setTimeout(r, Math.min(3000, 500 * 2 ** i)));
55 }
56 throw new Error("worker never became ready");
57};
58
59const boot = async (
60 name: string,
61): Promise<{ sample?: Sample; failure?: string }> => {
62 const controller = new AbortController();
63 const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
64 const start = Date.now();
65 try {
66 const res = await fetch(
67 `${WORKER_URL}/start?name=${encodeURIComponent(name)}`,
68 { headers: { connection: "close" }, signal: controller.signal },
69 );
70 const body = await res.text();
71 const outside = Date.now() - start;
72 if (res.status !== 200) {
73 return { failure: `${name}: HTTP ${res.status} ${body.slice(0, 200)}` };
74 }
75 let inside: number | undefined;
76 try {
77 inside = (JSON.parse(body) as { ms?: number }).ms;
78 } catch {
79 inside = undefined;
80 }
81 return { sample: { outside, inside } };
82 } catch (err) {
83 return { failure: `${name}: ${String(err)}` };
84 } finally {
85 clearTimeout(timer);
86 }
87};
88
89// Simple bounded-concurrency pool.
90const runPool = async (
91 names: string[],
92 concurrency: number,
93): Promise<Array<{ sample?: Sample; failure?: string }>> => {
94 const results: Array<{ sample?: Sample; failure?: string }> = [];
95 let next = 0;
96 const workers = Array.from(
97 { length: Math.min(concurrency, names.length) },
98 async () => {
99 while (next < names.length) {
100 const i = next++;
101 results[i] = await boot(names[i]);
102 }
103 },
104 );
105 await Promise.all(workers);
106 return results;
107};
108
109const stats = (xs: number[]) => {
110 if (xs.length === 0)
111 return { min: 0, max: 0, mean: 0, p50: 0, p90: 0, p95: 0, p99: 0 };
112 const s = [...xs].sort((a, b) => a - b);
113 const pct = (p: number) =>
114 s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))];
115 return {
116 min: s[0],
117 max: s[s.length - 1],
118 mean: Math.round(s.reduce((a, b) => a + b, 0) / s.length),
119 p50: pct(50),
120 p90: pct(90),
121 p95: pct(95),
122 p99: pct(99),
123 };
124};
125
126// One crash-loop attempt: time how long until the failure surfaces. `ok` is
127// true only on a 2xx (never, for a crash-looping container).
128const crashAttempt = async (
129 name: string,
130): Promise<{ outside: number; inside: number | undefined; ok: boolean }> => {
131 const controller = new AbortController();
132 const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
133 const start = Date.now();
134 try {
135 const res = await fetch(
136 `${WORKER_URL}/crashloop?name=${encodeURIComponent(name)}`,
137 { headers: { connection: "close" }, signal: controller.signal },
138 );
139 const body = await res.text();
140 const outside = Date.now() - start;
141 const parsed = JSON.parse(body) as { ms?: number; ok?: boolean };
142 return { outside, inside: parsed.ms, ok: parsed.ok ?? false };
143 } catch {
144 return { outside: Date.now() - start, inside: undefined, ok: false };
145 } finally {
146 clearTimeout(timer);
147 }
148};
149
150const runCrash = async () => {
151 const nonce = crypto.randomUUID().slice(0, 8);
152 const names = Array.from(
153 { length: CRASH_N },
154 (_, i) => `crash-${nonce}-${i}`,
155 );
156 const results = await Promise.all(names.map(crashAttempt));
157 const outside = stats(results.map((r) => r.outside));
158 const inside = stats(
159 results
160 .map((r) => r.inside)
161 .filter((m): m is number => typeof m === "number"),
162 );
163 const sec = (n: number) => `${(n / 1000).toFixed(1)}s`;
164 const failed = results.filter((r) => !r.ok).length;
165 console.log(
166 [
167 "",
168 `wrangler + @cloudflare/containers crash-loop fail-fast (N=${CRASH_N})`,
169 `── fatal crash (container exits immediately) ──`,
170 ` detected-as-failed: ${failed}/${CRASH_N}`,
171 ` time-to-fail inside (worker):`,
172 ` min ${sec(inside.min)} p50 ${sec(inside.p50)} p95 ${sec(inside.p95)} max ${sec(inside.max)} mean ${sec(inside.mean)}`,
173 ` time-to-fail outside (client):`,
174 ` min ${sec(outside.min)} p50 ${sec(outside.p50)} p95 ${sec(outside.p95)} max ${sec(outside.max)} mean ${sec(outside.mean)}`,
175 "",
176 ].join("\n"),
177 );
178};
179
180const main = async () => {
181 await wait(WORKER_URL);
182
183 if (CRASH) {
184 await runCrash();
185 return;
186 }
187
188 const nonce = crypto.randomUUID().slice(0, 8);
189 const names = Array.from({ length: N }, (_, i) => `${nonce}-${i}`);
190 const outcomes = await runPool(names, CONCURRENCY);
191
192 const samples = outcomes
193 .map((o) => o.sample)
194 .filter((s): s is Sample => s !== undefined);
195 const failures = outcomes
196 .map((o) => o.failure)
197 .filter((f): f is string => f !== undefined);
198
199 const outside = stats(samples.map((s) => s.outside));
200 const inside = stats(
201 samples
202 .map((s) => s.inside)
203 .filter((m): m is number => typeof m === "number"),
204 );
205 const sec = (n: number) => `${(n / 1000).toFixed(1)}s`;
206
207 console.log(
208 [
209 "",
210 `wrangler + @cloudflare/containers cold-start benchmark (N=${N}, concurrency=${CONCURRENCY})`,
211 `── non-effectful (oven/bun:latest, plain wrangler) ──`,
212 ` ok: ${samples.length}/${N} failed: ${failures.length}`,
213 ` outside (client round-trip):`,
214 ` min ${sec(outside.min)} p50 ${sec(outside.p50)} p90 ${sec(outside.p90)} p95 ${sec(outside.p95)} p99 ${sec(outside.p99)} max ${sec(outside.max)} mean ${sec(outside.mean)}`,
215 ` inside (worker start→reachable):`,
216 ` min ${sec(inside.min)} p50 ${sec(inside.p50)} p90 ${sec(inside.p90)} p95 ${sec(inside.p95)} p99 ${sec(inside.p99)} max ${sec(inside.max)} mean ${sec(inside.mean)}`,
217 ...(failures.length > 0
218 ? [" failures:", ...failures.slice(0, 5).map((f) => ` - ${f}`)]
219 : []),
220 "",
221 ].join("\n"),
222 );
223};
224
225main().catch((err) => {
226 console.error(err);
227 process.exit(1);
228});