1/**
2 * Real-world load: push **this repository** to a deployed git-service.
3 *
4 * Every other suite pushes toy histories. This one takes the actual alchemy
5 * monorepo — ~13.6k objects, ~36 MiB, 12k+ files in one tree — and drives
6 * the whole pipeline with it: thin-pack ingest at scale, the object store,
7 * compaction into R2 packs, clone bundles, and byte-for-byte clone
8 * fidelity verified by `git fsck --strict` plus a tree comparison.
9 *
10 * The source is the repo the tests are running inside (`git rev-parse
11 * --show-toplevel`), so it works on any checkout, and the suite skips
12 * itself when that is not a git repo.
13 *
14 * Shape of the fixture: a **depth-1 clone re-committed as a single root
15 * commit**. That keeps the object count and byte volume of the real repo
16 * (which is the point) while staying a fully connected history — pushing
17 * from a shallow clone would send commits whose parents are missing, which
18 * receive-pack correctly refuses.
19 *
20 * Slow by construction; skipped under `--fast`.
21 */
22import * as Cloudflare from "@/Cloudflare";
23import * as Test from "@/Test/Alchemy";
24import { expect } from "alchemy-test";
25import * as Data from "effect/Data";
26import * as Effect from "effect/Effect";
27import * as FileSystem from "effect/FileSystem";
28import { MinimumLogLevel } from "effect/References";
29import * as Schedule from "effect/Schedule";
30import * as Stream from "effect/Stream";
31import * as HttpClient from "effect/http/HttpClient";
32import * as HttpClientRequest from "effect/http/HttpClientRequest";
33import * as HttpApiClient from "effect/http-api/HttpApiClient";
34import * as ChildProcess from "effect/process/ChildProcess";
35import { GitApi } from "@/Git/Api.ts";
36import { makeTestStack, TEST_SECRET } from "./fixtures/stack.ts";
37
38const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
39 providers: Cloudflare.providers(),
40});
41
42const logLevel = Effect.provideService(
43 MinimumLogLevel,
44 process.env.DEBUG ? "Debug" : "Info",
45);
46
47const Stack = makeTestStack("GitRealWorldStack");
48
49const skipHuge = !!process.env.FAST;
50
51// ── plumbing ────────────────────────────────────────────────────────────────
52
53const makeClient = (url: string, token: string) =>
54 HttpApiClient.make(GitApi, {
55 baseUrl: url,
56 transformClient: HttpClient.mapRequest((request) =>
57 request.pipe(HttpClientRequest.bearerToken(token)),
58 ),
59 });
60
61class ShellError extends Data.TaggedError("ShellError")<{
62 readonly script: string;
63 readonly exitCode: number;
64 readonly stdout: string;
65 readonly stderr: string;
66}> {
67 override get message(): string {
68 return `shell exited ${this.exitCode}\n--- script ---\n${this.script}\n--- stderr ---\n${this.stderr}`;
69 }
70}
71
72/** Bounded `sh -c` with hermetic git identity; 10 min covers a 36 MiB push. */
73const sh = Effect.fn(function* (cwd: string, script: string) {
74 const handle = yield* ChildProcess.make("sh", ["-ec", script], {
75 cwd,
76 env: {
77 GIT_TERMINAL_PROMPT: "0",
78 GIT_CONFIG_NOSYSTEM: "1",
79 GIT_CONFIG_GLOBAL: "/dev/null",
80 GIT_AUTHOR_NAME: "Real World Bot",
81 GIT_AUTHOR_EMAIL: "realworld@example.com",
82 GIT_COMMITTER_NAME: "Real World Bot",
83 GIT_COMMITTER_EMAIL: "realworld@example.com",
84 GIT_AUTHOR_DATE: "2026-01-01T00:00:00Z",
85 GIT_COMMITTER_DATE: "2026-01-01T00:00:00Z",
86 },
87 extendEnv: true,
88 });
89 const [exitCode, stdout, stderr] = yield* Effect.all(
90 [
91 handle.exitCode,
92 Stream.mkString(Stream.decodeText(handle.stdout)),
93 Stream.mkString(Stream.decodeText(handle.stderr)),
94 ],
95 { concurrency: 3 },
96 );
97 return { script, exitCode, stdout: stdout.trim(), stderr };
98}, Effect.timeout("600 seconds"));
99
100const mustSh = Effect.fn(function* (cwd: string, script: string) {
101 const result = yield* sh(cwd, script);
102 if (result.exitCode !== 0) return yield* new ShellError(result);
103 return result;
104});
105
106const tempDir = Effect.gen(function* () {
107 const fs = yield* FileSystem.FileSystem;
108 return yield* fs.makeTempDirectory({ prefix: "git-realworld-" });
109});
110
111const edgeRetry = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
112 effect.pipe(
113 Effect.timeout("15 seconds"),
114 Effect.retry({ schedule: Schedule.spaced("1500 millis"), times: 40 }),
115 );
116
117const freshRepo = Effect.fn(function* (
118 url: string,
119 owner: string,
120 name: string,
121) {
122 const admin = yield* makeClient(url, TEST_SECRET);
123 const created = yield* Effect.gen(function* () {
124 yield* admin.repos.delete({ params: { owner, repo: name } }).pipe(
125 Effect.catchTag("RepoNotFound", () => Effect.void),
126 edgeRetry,
127 );
128 yield* admin.repos.get({ params: { owner, repo: name } }).pipe(
129 edgeRetry,
130 Effect.as(false),
131 Effect.catchTag("RepoNotFound", () => Effect.succeed(true)),
132 Effect.repeat({
133 schedule: Schedule.spaced("1 second"),
134 until: (gone) => gone,
135 times: 60,
136 }),
137 );
138 return yield* admin.repos
139 .create({ payload: { owner, name } })
140 .pipe(edgeRetry);
141 }).pipe(
142 Effect.retry({
143 while: (error: { readonly _tag?: string }) =>
144 error._tag === "RepoAlreadyExists",
145 schedule: Schedule.spaced("1 second"),
146 times: 3,
147 }),
148 );
149 const parsed = new URL(url);
150 return {
151 admin,
152 token: TEST_SECRET,
153 remote: `${parsed.protocol}//x:${TEST_SECRET}@${parsed.host}/${owner}/${name}.git`,
154 };
155});
156
157const stack = beforeAll(
158 deploy(Stack).pipe(
159 Effect.tap(({ url }) =>
160 Effect.gen(function* () {
161 yield* Effect.logInfo(`git-service deployed at ${url}`);
162 const admin = yield* makeClient(url, TEST_SECRET);
163 yield* admin.repos.list({ query: {} }).pipe(
164 edgeRetry,
165 Effect.retry({
166 schedule: Schedule.spaced("1500 millis"),
167 times: 40,
168 }),
169 );
170 }),
171 ),
172 logLevel,
173 ),
174);
175afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(Stack));
176
177// ═══════════════════════════════════════════════════════════════════════════
178
179test.skipIf(skipHuge)(
180 "real world: push this entire repository, clone it back, compact it",
181 Effect.gen(function* () {
182 const { url } = yield* stack;
183 const tmp = yield* tempDir;
184
185 // The repo under test is the one these tests live in.
186 const root = (yield* mustSh(process.cwd(), `git rev-parse --show-toplevel`))
187 .stdout;
188 expect(root.length).toBeGreaterThan(0);
189
190 // Depth-1 clone, re-committed as a single root commit: keeps the real
191 // object count and byte volume, but fully connected (a shallow push
192 // would reference parents the server does not have).
193 yield* mustSh(
194 tmp,
195 `
196 rm -rf src
197 git clone -q --depth 1 "file://${root}" src
198 cd src
199 rm -rf .git
200 git init -q -b main
201 git add -A
202 git commit -q -m "alchemy snapshot"
203 `,
204 );
205
206 const stats = (yield* mustSh(
207 tmp,
208 `cd src && git count-objects -v | tr '\\n' ' '`,
209 )).stdout;
210 const files = Number.parseInt(
211 (yield* mustSh(tmp, `cd src && git ls-files | wc -l`)).stdout,
212 10,
213 );
214 const head = (yield* mustSh(tmp, `cd src && git rev-parse HEAD`)).stdout;
215 const tree = (yield* mustSh(tmp, `cd src && git rev-parse HEAD^{tree}`))
216 .stdout;
217 yield* Effect.logInfo(
218 `[real-world] source: ${files} files, ${stats.trim()}`,
219 );
220 expect(files).toBeGreaterThan(5_000);
221
222 const repo = yield* freshRepo(url, "real", "alchemy");
223
224 // ── the push ─────────────────────────────────────────────────────────
225 const pushStarted = yield* Effect.sync(() => performance.now());
226 yield* mustSh(
227 tmp,
228 `cd src && git remote add origin '${repo.remote}' && git push origin main`,
229 );
230 const pushMs = yield* Effect.sync(() => performance.now() - pushStarted);
231 yield* Effect.logInfo(
232 `[real-world] pushed ${files} files in ${(pushMs / 1000).toFixed(1)}s`,
233 );
234
235 // The server's view must match the client's exactly.
236 const remoteRef = yield* repo.admin.refs.get({
237 params: { owner: "real", repo: "alchemy" },
238 query: { name: "refs/heads/main" },
239 });
240 expect(remoteRef.oid).toBe(head);
241
242 const meta = yield* repo.admin.repos.get({
243 params: { owner: "real", repo: "alchemy" },
244 });
245 yield* Effect.logInfo(
246 `[real-world] stored ${meta.objects.loose + meta.objects.packed + meta.objects.r2} objects, ` +
247 `${(meta.objects.bytes / 1024 / 1024).toFixed(1)} MiB`,
248 );
249 // Server-side timing (DESIGN.md §19 phase 0): the wall clock above also
250 // covers client packing and the upload, so this is what is actually ours.
251 if (meta.lastPush !== null) {
252 const p = meta.lastPush;
253 yield* Effect.logInfo(
254 `[real-world] SERVER ingest ${p.ingestMs}ms (sql ${p.stageMs}ms / cpu ${p.ingestMs - p.stageMs}ms) + connectivity ${p.connectivityMs}ms + ` +
255 `finalize ${p.finalizeMs}ms = ${p.totalMs}ms for ${p.objects} objects ` +
256 `[phases: ${Object.entries(p.phases ?? {})
257 .sort((a, b) => b[1] - a[1])
258 .map(([k, v]) => `${k} ${v}ms`)
259 .join(", ")}] ` +
260 `(${(p.totalMs / Math.max(p.objects, 1)).toFixed(2)}ms/object); ` +
261 `client+network = ${(pushMs - p.totalMs).toFixed(0)}ms of the ${(pushMs / 1000).toFixed(1)}s wall clock`,
262 );
263 expect(p.objects).toBeGreaterThan(5_000);
264 }
265 expect(
266 meta.objects.loose + meta.objects.packed + meta.objects.r2,
267 ).toBeGreaterThan(5_000);
268
269 // ── clone it back and prove it is identical ──────────────────────────
270 const cloneStarted = yield* Effect.sync(() => performance.now());
271 yield* mustSh(tmp, `rm -rf back && git clone -q '${repo.remote}' back`);
272 const cloneMs = yield* Effect.sync(() => performance.now() - cloneStarted);
273 yield* Effect.logInfo(
274 `[real-world] cloned back in ${(cloneMs / 1000).toFixed(1)}s`,
275 );
276
277 yield* mustSh(tmp, `cd back && git fsck --strict`);
278 expect((yield* mustSh(tmp, `cd back && git rev-parse HEAD`)).stdout).toBe(
279 head,
280 );
281 // The tree oid matching means every path, mode and blob matches.
282 expect(
283 (yield* mustSh(tmp, `cd back && git rev-parse HEAD^{tree}`)).stdout,
284 ).toBe(tree);
285 expect(
286 Number.parseInt(
287 (yield* mustSh(tmp, `cd back && git ls-files | wc -l`)).stdout,
288 10,
289 ),
290 ).toBe(files);
291 // NOTE: no `diff -r` here. The tree oid equality above is the stronger
292 // and *correct* assertion — identical tree oids mean every path, mode
293 // and blob content matches. A directory diff also compares things git
294 // deliberately does not track (empty directories, e.g. the checked-out
295 // submodule mount points), which differ legitimately between a working
296 // tree and a fresh clone of it.
297
298 // ── compact the real repo into R2 packs, then clone again ────────────
299 yield* repo.admin.repos.compact({
300 params: { owner: "real", repo: "alchemy" },
301 });
302 const packed = yield* repo.admin.repos
303 .get({ params: { owner: "real", repo: "alchemy" } })
304 .pipe(
305 Effect.map((found) => found.objects),
306 Effect.repeat({
307 schedule: Schedule.spaced("2 seconds"),
308 until: (objects: { loose: number; packed: number }) =>
309 objects.packed > 0 && objects.loose === 0,
310 times: 150,
311 }),
312 );
313 yield* Effect.logInfo(
314 `[real-world] compacted: ${packed.packed} objects in R2 packs`,
315 );
316 expect(packed.packed).toBeGreaterThan(5_000);
317
318 const packedCloneStarted = yield* Effect.sync(() => performance.now());
319 yield* mustSh(tmp, `rm -rf packed && git clone -q '${repo.remote}' packed`);
320 yield* Effect.logInfo(
321 `[real-world] cloned from R2 packs in ${(
322 (yield* Effect.sync(() => performance.now())) - packedCloneStarted
323 ).toFixed(0)}ms`,
324 );
325 yield* mustSh(tmp, `cd packed && git fsck --strict`);
326 expect(
327 (yield* mustSh(tmp, `cd packed && git rev-parse HEAD^{tree}`)).stdout,
328 ).toBe(tree);
329
330 // ── an incremental push on top of the real repo still works ──────────
331 yield* mustSh(
332 tmp,
333 `
334 cd src
335 echo "real world" > REALWORLD.md
336 git add REALWORLD.md
337 git commit -q -m "incremental on top of a real repo"
338 git push origin main
339 `,
340 );
341 yield* mustSh(tmp, `cd back && git pull -q origin main`);
342 expect((yield* mustSh(tmp, `cd back && git rev-parse HEAD`)).stdout).toBe(
343 (yield* mustSh(tmp, `cd src && git rev-parse HEAD`)).stdout,
344 );
345 }).pipe(logLevel),
346 {
347 tags: [
348 "provider:cloudflare",
349 "provider:cloudflare:r2",
350 "provider:cloudflare:worker",
351 "live",
352 ],
353 timeout: 900_000,
354 },
355);