1// @effect-diagnostics nodeBuiltinImport:off - Effect has no incremental digest.
2import * as EffectNodeStream from "@effect/platform-node/NodeStream";
3import * as HostProcess from "@t3tools/shared/HostProcess";
4import * as Cause from "effect/Cause";
5import * as Clock from "effect/Clock";
6import * as Context from "effect/Context";
7import * as Deferred from "effect/Deferred";
8import type * as Duration from "effect/Duration";
9import * as Effect from "effect/Effect";
10import * as Exit from "effect/Exit";
11import * as FileSystem from "effect/FileSystem";
12import * as Layer from "effect/Layer";
13import * as Option from "effect/Option";
14import * as Path from "effect/Path";
15import * as Schema from "effect/Schema";
16import * as Semaphore from "effect/Semaphore";
17import * as Stream from "effect/Stream";
18import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
19import * as NodeCrypto from "node:crypto";
20
21import * as ServerConfig from "../config.ts";
22import { openZipArchive } from "../zipArchive.ts";
23
24// The one browser T3 installs on a host. HTML render previews and server
25// browser tabs both run this pinned Chrome for Testing headless shell, so a host
26// downloads it once; neither uses a browser the user installed. To bump the pin, pick a version
27// from https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json,
28// download each platform's chrome-headless-shell zip, and replace the version
29// and every byte count and SHA-256 below. Hosts drop the old build after the
30// new one installs.
31const VERSION = "154.0.8037.92";
32const ARCHIVES = {
33 linux64: {
34 bytes: 120_477_194,
35 sha256: "636aa5c79f2693632e9921b8bbb050038ba11672e02346c06c20f991aed096f9",
36 },
37 "linux-arm64": {
38 bytes: 121_182_296,
39 sha256: "0ed0e47d9e9f639197f508d62ada09e5c6b4c4c60edab3160a9312a733091df6",
40 },
41 "mac-arm64": {
42 bytes: 99_221_129,
43 sha256: "77da14e75d7f2568e6f7898d3df7cdc6faac74b15e903b2c9d486ebb6ca9b929",
44 },
45 "mac-x64": {
46 bytes: 104_748_425,
47 sha256: "a54292aaacbb77f76f6ef47558e7c51ab884044e0adacca315567f83c060bcc4",
48 },
49 win32: {
50 bytes: 114_295_943,
51 sha256: "56b30d2d6c35775ebf8dc3618680f6529e1c38c87f7feb28a16e9904273d51f7",
52 },
53 win64: {
54 bytes: 120_822_223,
55 sha256: "3ac2561f02d9d87aadc0399d00b9002d718a4c365624fa67db9e7bfaf6b1a568",
56 },
57} as const;
58
59const chromePlatform = (platform: NodeJS.Platform, arch: NodeJS.Architecture) => {
60 switch (platform) {
61 case "linux":
62 return arch === "x64" ? "linux64" : arch === "arm64" ? "linux-arm64" : null;
63 case "darwin":
64 return arch === "arm64" ? "mac-arm64" : arch === "x64" ? "mac-x64" : null;
65 case "win32":
66 // There is no Windows arm64 build; Windows on Arm runs the x64 one under emulation.
67 return arch === "ia32" ? "win32" : arch === "x64" || arch === "arm64" ? "win64" : null;
68 default:
69 return null;
70 }
71};
72
73export interface PreviewBrowserRelease {
74 readonly version: string;
75 /** Chrome for Testing's platform name, which also names the archive's top directory. */
76 readonly platform: string;
77 readonly url: string;
78 readonly bytes: number;
79 readonly sha256: string;
80}
81
82export const previewBrowserRelease = (
83 platform: NodeJS.Platform,
84 arch: NodeJS.Architecture,
85): PreviewBrowserRelease | null => {
86 const chrome = chromePlatform(platform, arch);
87 return chrome === null
88 ? null
89 : {
90 version: VERSION,
91 platform: chrome,
92 url: `https://storage.googleapis.com/chrome-for-testing-public/${VERSION}/${chrome}/chrome-headless-shell-${chrome}.zip`,
93 ...ARCHIVES[chrome],
94 };
95};
96
97const megabytes = (bytes: number) => Math.round(bytes / 1_000_000);
98
99export class PreviewBrowserInstallError extends Schema.TaggedError<PreviewBrowserInstallError>()(
100 "PreviewBrowserInstallError",
101 { detail: Schema.String, cause: Schema.optional(Schema.Defect()) },
102) {
103 override get message(): string {
104 return `T3 could not install its headless browser: ${this.detail} Try again.`;
105 }
106}
107const isInstallError = Schema.is(PreviewBrowserInstallError);
108
109export class PreviewBrowserInstallingError extends Schema.TaggedError<PreviewBrowserInstallingError>()(
110 "PreviewBrowserInstallingError",
111 { downloadedBytes: Schema.Number, totalBytes: Schema.Number, unpacking: Schema.Boolean },
112) {
113 override get message(): string {
114 const progress = this.unpacking
115 ? "unpacking"
116 : `${megabytes(this.downloadedBytes)} of ${megabytes(this.totalBytes)} MB downloaded`;
117 return `T3 is installing its headless browser (${progress}). Try again in a minute.`;
118 }
119}
120
121export class PreviewBrowserUnsupportedError extends Schema.TaggedError<PreviewBrowserUnsupportedError>()(
122 "PreviewBrowserUnsupportedError",
123 { platform: Schema.String, arch: Schema.String },
124) {
125 override get message(): string {
126 return `T3's headless browser is not available on ${this.platform}-${this.arch}: Chrome for Testing has no headless shell for it.`;
127 }
128}
129
130export class PreviewBrowser extends Context.Service<
131 PreviewBrowser,
132 {
133 /**
134 * Path to the installed headless shell. The first call starts the
135 * install; callers wait for it up to a bound and then get its progress
136 * instead, while the install keeps running.
137 */
138 readonly executable: Effect.Effect<
139 string,
140 PreviewBrowserInstallError | PreviewBrowserInstallingError | PreviewBrowserUnsupportedError
141 >;
142 /** The installed headless shell, if any. Never starts or waits on an install. */
143 readonly installed: Effect.Effect<Option.Option<string>>;
144 }
145>()("t3/preview/PreviewBrowser") {}
146
147export interface PreviewBrowserOptions {
148 readonly baseDir: string;
149 readonly release?: PreviewBrowserRelease | null;
150 /** How long a caller waits on an install in progress. Provider tool calls time out near 60s. */
151 readonly wait?: Duration.Input;
152}
153
154const DOWNLOAD_TIMEOUT = "15 minutes";
155// Longer than any install can run: the download times out at 15 minutes.
156const STAGING_STALE_AFTER_MS = 60 * 60_000;
157
158const wrapFailure = (detail: string) => (cause: unknown) =>
159 isInstallError(cause) ? cause : new PreviewBrowserInstallError({ detail, cause });
160
161type InstallState =
162 | { readonly _tag: "idle" }
163 | {
164 readonly _tag: "installing";
165 readonly done: Deferred.Deferred<string, PreviewBrowserInstallError>;
166 }
167 | { readonly _tag: "installed"; readonly executable: string }
168 | { readonly _tag: "failed"; readonly error: PreviewBrowserInstallError };
169
170export const makePreviewBrowser = Effect.fn("PreviewBrowser.make")(function* (
171 options: PreviewBrowserOptions,
172) {
173 const fs = yield* FileSystem.FileSystem;
174 const path = yield* Path.Path;
175 const http = yield* HttpClient.HttpClient;
176 // Installs belong to the service, so they finish even when no caller is still waiting.
177 const serviceScope = yield* Effect.scope;
178 const platform = yield* HostProcess.Platform;
179 const arch = yield* HostProcess.Architecture;
180 const release =
181 options.release === undefined ? previewBrowserRelease(platform, arch) : options.release;
182 const wait = options.wait ?? "45 seconds";
183 const installRoot = path.join(
184 options.baseDir,
185 "tools",
186 "chrome-headless-shell",
187 release?.platform ?? `${platform}-${arch}`,
188 );
189 const executableName =
190 platform === "win32" ? "chrome-headless-shell.exe" : "chrome-headless-shell";
191 const gate = yield* Semaphore.make(1);
192 let state: InstallState = { _tag: "idle" };
193 const progress = { downloadedBytes: 0, unpacking: false };
194
195 // The version directory appears by one rename of a fully unpacked tree, so a
196 // runnable binary there is the complete-install marker.
197 const installedExecutable = (release: PreviewBrowserRelease) => {
198 const executable = path.join(installRoot, release.version, executableName);
199 return fs.stat(executable).pipe(
200 Effect.map((info) =>
201 info.type === "File" && (platform === "win32" || (info.mode & 0o111) !== 0)
202 ? Option.some(executable)
203 : Option.none<string>(),
204 ),
205 Effect.orElseSucceed(() => Option.none<string>()),
206 );
207 };
208
209 const download = Effect.fn("PreviewBrowser.download")(
210 function* (release: PreviewBrowserRelease, archivePath: string) {
211 const response = yield* http
212 .execute(HttpClientRequest.get(release.url))
213 .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk));
214 const contentLength = response.headers["content-length"];
215 const contentEncoding = response.headers["content-encoding"]?.trim().toLowerCase();
216 if (
217 (contentEncoding === undefined || contentEncoding === "identity") &&
218 contentLength !== undefined &&
219 Number(contentLength) !== release.bytes
220 ) {
221 return yield* new PreviewBrowserInstallError({
222 detail: "The download size did not match the pinned release.",
223 });
224 }
225 const hash = NodeCrypto.createHash("sha256");
226 yield* response.stream.pipe(
227 Stream.tap((chunk) =>
228 Effect.suspend(() => {
229 progress.downloadedBytes += chunk.byteLength;
230 if (progress.downloadedBytes > release.bytes) {
231 return Effect.fail(
232 new PreviewBrowserInstallError({
233 detail: "The download was larger than the pinned release.",
234 }),
235 );
236 }
237 hash.update(chunk);
238 return Effect.void;
239 }),
240 ),
241 Stream.run(fs.sink(archivePath, { flag: "wx", mode: 0o600 })),
242 );
243 if (progress.downloadedBytes !== release.bytes || hash.digest("hex") !== release.sha256) {
244 return yield* new PreviewBrowserInstallError({
245 detail: "The download failed its size or SHA-256 check. Nothing was installed.",
246 });
247 }
248 },
249 Effect.timeout(DOWNLOAD_TIMEOUT),
250 Effect.mapError(
251 wrapFailure(
252 "The download from storage.googleapis.com failed. Check this machine's network access.",
253 ),
254 ),
255 );
256
257 const extract = Effect.fn("PreviewBrowser.extract")(
258 function* (release: PreviewBrowserRelease, archivePath: string, destination: string) {
259 const archive = yield* openZipArchive(
260 archivePath,
261 (detail, cause) => new PreviewBrowserInstallError({ detail, cause }),
262 );
263 const unsafe = new PreviewBrowserInstallError({
264 detail: "The archive contains an unexpected or unsafe entry.",
265 });
266 const root = `chrome-headless-shell-${release.platform}/`;
267 yield* fs.makeDirectory(destination);
268 let foundExecutable = false;
269 for (;;) {
270 const entry = yield* archive.next;
271 if (!entry) break;
272 const unixMode = entry.externalFileAttributes >>> 16;
273 const unixType = unixMode & 0o170000;
274 const isDirectory = entry.fileName.endsWith("/");
275 // Everything sits under the one top directory, which is stripped.
276 const relative = entry.fileName.startsWith(root) ? entry.fileName.slice(root.length) : null;
277 const target = relative === null ? null : path.resolve(destination, relative);
278 if (
279 relative === null ||
280 target === null ||
281 (relative !== "" && !target.startsWith(`${destination}${path.sep}`)) ||
282 relative.split("/").includes("..") ||
283 (unixType !== 0 && unixType !== (isDirectory ? 0o040000 : 0o100000)) ||
284 (entry.generalPurposeBitFlag & 1) !== 0 ||
285 ![0, 8].includes(entry.compressionMethod)
286 ) {
287 return yield* unsafe;
288 }
289 if (isDirectory) {
290 yield* fs.makeDirectory(target, { recursive: true });
291 continue;
292 }
293 yield* fs.makeDirectory(path.dirname(target), { recursive: true });
294 yield* Effect.gen(function* () {
295 const readable = yield* archive.streamEntry(entry);
296 yield* EffectNodeStream.fromReadable<Uint8Array, PreviewBrowserInstallError>({
297 evaluate: () => readable,
298 onError: wrapFailure("Could not unpack the browser."),
299 }).pipe(Stream.run(fs.sink(target, { flag: "wx" })));
300 }).pipe(Effect.scoped);
301 // The binary and its shared libraries carry exec bits in the archive.
302 if (platform !== "win32" && (unixMode & 0o777) !== 0) {
303 yield* fs.chmod(target, unixMode & 0o777);
304 }
305 if (relative === executableName) foundExecutable = true;
306 }
307 if (!foundExecutable) {
308 return yield* new PreviewBrowserInstallError({
309 detail: "The archive does not contain chrome-headless-shell.",
310 });
311 }
312 },
313 Effect.scoped,
314 Effect.mapError(
315 wrapFailure("Could not unpack the browser. Check free disk space in T3's home directory."),
316 ),
317 );
318
319 const install = Effect.fn("PreviewBrowser.install")(
320 function* (release: PreviewBrowserRelease) {
321 yield* Effect.gen(function* () {
322 yield* fs.makeDirectory(installRoot, { recursive: true });
323 // Staging shares the install root's filesystem so publishing is one rename.
324 const staging = yield* fs.makeTempDirectoryScoped({
325 directory: installRoot,
326 prefix: ".install-",
327 });
328 const archivePath = path.join(staging, "download.zip");
329 const unpacked = path.join(staging, "browser");
330 yield* download(release, archivePath);
331 progress.unpacking = true;
332 yield* extract(release, archivePath, unpacked);
333 const destination = path.join(installRoot, release.version);
334 // A directory left without a runnable binary would block the rename.
335 yield* fs.remove(destination, { recursive: true, force: true });
336 yield* fs.rename(unpacked, destination);
337 }).pipe(Effect.scoped);
338 // Older builds and abandoned staging directories. Two servers can share a
339 // home, so a staging directory touched within the last hour may be another
340 // server's install in progress and stays.
341 const entries = yield* fs.readDirectory(installRoot);
342 const now = yield* Clock.currentTimeMillis;
343 yield* Effect.forEach(
344 entries.filter((name) => name !== release.version),
345 (name) =>
346 Effect.gen(function* () {
347 const target = path.join(installRoot, name);
348 if (name.startsWith(".install-")) {
349 const info = yield* fs.stat(target);
350 const modifiedAt = Option.match(info.mtime, {
351 onNone: () => now,
352 onSome: (date) => date.getTime(),
353 });
354 if (now - modifiedAt < STAGING_STALE_AFTER_MS) return;
355 }
356 yield* fs.remove(target, { recursive: true, force: true });
357 }).pipe(
358 Effect.catch((cause) =>
359 Effect.logWarning("Could not remove an old preview browser.", { name, cause }),
360 ),
361 ),
362 { discard: true },
363 );
364 return path.join(installRoot, release.version, executableName);
365 },
366 Effect.mapError(wrapFailure("Could not save the browser in T3's home directory.")),
367 );
368
369 // Joins the current install or starts one. A failure is reported once, then cleared.
370 const join = (release: PreviewBrowserRelease) =>
371 gate.withPermit(
372 Effect.gen(function* () {
373 if (state._tag === "installing") return state;
374 if (state._tag === "failed") {
375 const failed = state;
376 state = { _tag: "idle" };
377 return failed;
378 }
379 const installed = yield* installedExecutable(release);
380 if (Option.isSome(installed)) {
381 state = { _tag: "installed", executable: installed.value };
382 return state;
383 }
384 const done = yield* Deferred.make<string, PreviewBrowserInstallError>();
385 progress.downloadedBytes = 0;
386 progress.unpacking = false;
387 const installing: InstallState = { _tag: "installing", done };
388 state = installing;
389 yield* install(release).pipe(
390 Effect.onExit((exit) =>
391 gate
392 .withPermit(
393 Effect.sync(() => {
394 state = Exit.isSuccess(exit)
395 ? { _tag: "installed", executable: exit.value }
396 : {
397 _tag: "failed",
398 error: Option.getOrElse(
399 Cause.findErrorOption(exit.cause),
400 () => new PreviewBrowserInstallError({ detail: "The install stopped." }),
401 ),
402 };
403 }),
404 )
405 .pipe(Effect.andThen(Deferred.done(done, exit))),
406 ),
407 Effect.ignoreCause,
408 // Server shutdown closes the service scope and stops an install in progress.
409 Effect.interruptible,
410 Effect.forkIn(serviceScope),
411 );
412 return installing;
413 }),
414 );
415
416 // A waiter that saw the failure has reported it; the next call retries.
417 const clearFailure = (error: PreviewBrowserInstallError) =>
418 gate.withPermit(
419 Effect.sync(() => {
420 if (state._tag === "failed" && state.error === error) state = { _tag: "idle" };
421 }),
422 );
423
424 const executable = Effect.gen(function* () {
425 if (release === null) {
426 return yield* new PreviewBrowserUnsupportedError({ platform, arch });
427 }
428 const current = yield* join(release);
429 switch (current._tag) {
430 case "installed":
431 return current.executable;
432 case "failed":
433 return yield* current.error;
434 case "installing":
435 return yield* Deferred.await(current.done).pipe(
436 Effect.tapError(clearFailure),
437 Effect.timeoutOrElse({
438 duration: wait,
439 orElse: () =>
440 Effect.fail(
441 new PreviewBrowserInstallingError({
442 downloadedBytes: progress.downloadedBytes,
443 totalBytes: release.bytes,
444 unpacking: progress.unpacking,
445 }),
446 ),
447 }),
448 );
449 }
450 }).pipe(Effect.withSpan("PreviewBrowser.executable"));
451
452 const installed = release === null ? Effect.succeedNone : installedExecutable(release);
453
454 return PreviewBrowser.of({ executable, installed });
455});
456
457export const layer = Layer.effect(
458 PreviewBrowser,
459 Effect.gen(function* () {
460 const config = yield* ServerConfig.ServerConfig;
461 return yield* makePreviewBrowser({ baseDir: config.baseDir });
462 }),
463);