1import * as Effect from "effect/Effect";
2import { createWriteStream } from "node:fs";
3import { lstat, opendir } from "node:fs/promises";
4import { posix } from "node:path";
5import { Readable, Transform } from "node:stream";
6import { pipeline } from "node:stream/promises";
7import { createGzip } from "node:zlib";
8import { verifiedFileChunks, type VerifiedFile } from "./ArtifactFile.ts";
9
10/**
11 * Node/Bun boundary for filesystem operations that Effect's portable
12 * FileSystem service cannot currently express: incremental directory handles
13 * and streaming a verified file descriptor through gzip.
14 */
15
16export interface ArchiveTarFileEntry {
17 readonly type: "file";
18 readonly name: string;
19 readonly mode: number;
20 readonly file: VerifiedFile;
21}
22
23export interface ArchiveTarSymlinkEntry {
24 readonly type: "symlink";
25 readonly name: string;
26 readonly mode: number;
27 readonly linkname: string;
28}
29
30export type ArchiveTarEntry = ArchiveTarFileEntry | ArchiveTarSymlinkEntry;
31
32export interface SecureDirectoryEntry {
33 readonly name: string;
34 readonly type: "Directory" | "File" | "SymbolicLink" | "Other";
35}
36
37interface CloseableDirectoryHandle {
38 readonly close: () => void | Promise<void>;
39}
40
41export const closeDirectoryHandle = async (
42 handle: CloseableDirectoryHandle,
43) => {
44 try {
45 await handle.close();
46 } catch {
47 // Async directory iteration closes the handle when it exits. Ignore both
48 // Node's already-closed rejection and Bun's equivalent synchronous error.
49 }
50};
51
52const entryLimitError = (maxEntries: number) =>
53 new Error(
54 `Compute artifact exceeds the ${maxEntries} entry safety limit. Narrow the artifact directory or add ignore patterns.`,
55 );
56
57export const readDirectoryEntriesSecure = (options: {
58 readonly directory: string;
59 readonly relativePrefix: string;
60 readonly ignore: readonly RegExp[];
61 readonly entriesAlreadyObserved: number;
62 readonly maxEntries: number;
63}) =>
64 Effect.tryPromise({
65 try: async () => {
66 const before = await lstat(options.directory, { bigint: true });
67 if (!before.isDirectory() || before.isSymbolicLink()) {
68 throw new Error(
69 `Compute artifact directory changed before traversal: ${options.relativePrefix || "."}`,
70 );
71 }
72 const handle = await opendir(options.directory);
73 const entries: SecureDirectoryEntry[] = [];
74 let observedEntries = 0;
75 try {
76 for await (const entry of handle) {
77 if (entry.name.includes("\\")) {
78 throw new Error(
79 `Compute artifact entry contains an unsupported backslash: ${options.relativePrefix || "."}`,
80 );
81 }
82 const name = entry.name;
83 if (name.length === 0) continue;
84 const relativeName = options.relativePrefix
85 ? `${options.relativePrefix}/${name}`
86 : name;
87 observedEntries += 1;
88 if (
89 options.entriesAlreadyObserved + observedEntries >
90 options.maxEntries
91 ) {
92 throw entryLimitError(options.maxEntries);
93 }
94 if (options.ignore.some((pattern) => pattern.test(relativeName))) {
95 continue;
96 }
97 entries.push({
98 name,
99 type: entry.isSymbolicLink()
100 ? "SymbolicLink"
101 : entry.isDirectory()
102 ? "Directory"
103 : entry.isFile()
104 ? "File"
105 : "Other",
106 });
107 }
108 } finally {
109 await closeDirectoryHandle(handle);
110 }
111 const after = await lstat(options.directory, { bigint: true });
112 if (
113 !after.isDirectory() ||
114 after.isSymbolicLink() ||
115 before.dev !== after.dev ||
116 before.ino !== after.ino ||
117 before.mtimeNs !== after.mtimeNs ||
118 before.ctimeNs !== after.ctimeNs
119 ) {
120 throw new Error(
121 `Compute artifact directory changed during traversal: ${options.relativePrefix || "."}`,
122 );
123 }
124 return {
125 entries: entries.sort((a, b) =>
126 a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
127 ),
128 observedEntries,
129 };
130 },
131 catch: (error) =>
132 error instanceof Error ? error : new Error(String(error)),
133 });
134
135export const writeCompressedArchiveSecure = (
136 archivePath: string,
137 entries: readonly ArchiveTarEntry[],
138 manifest: Uint8Array,
139 maxCompressedBytes: number,
140) =>
141 Effect.tryPromise({
142 try: (signal) => {
143 let compressedBytes = 0;
144 const compressedByteLimit = new Transform({
145 transform(chunk: Buffer, _encoding, callback) {
146 compressedBytes += chunk.byteLength;
147 if (compressedBytes > maxCompressedBytes) {
148 callback(
149 new Error(
150 `Prisma compute archive exceeds the ${maxCompressedBytes} byte compressed upload safety limit.`,
151 ),
152 );
153 return;
154 }
155 callback(null, chunk);
156 },
157 });
158 return pipeline(
159 Readable.from(tarChunks(entries, manifest), { objectMode: false }),
160 createGzip(),
161 compressedByteLimit,
162 createWriteStream(archivePath, { flags: "w", mode: 0o600 }),
163 { signal },
164 );
165 },
166 catch: (error) =>
167 error instanceof Error ? error : new Error(String(error)),
168 });
169
170export const isArchivedRegularFile = (
171 entries: readonly ArchiveTarEntry[],
172 entrypointName: string,
173) => {
174 const byName = new Map(entries.map((entry) => [entry.name, entry]));
175 const visited = new Set<string>();
176 let currentName = entrypointName;
177 while (!visited.has(currentName)) {
178 visited.add(currentName);
179 const entry = byName.get(currentName);
180 if (entry?.type === "file") return true;
181 if (entry?.type !== "symlink") return false;
182 const target = posix.normalize(
183 posix.join(posix.dirname(currentName), entry.linkname),
184 );
185 if (target === ".." || target.startsWith("../") || target.startsWith("/")) {
186 return false;
187 }
188 currentName = target;
189 }
190 return false;
191};
192
193async function* tarChunks(
194 entries: readonly ArchiveTarEntry[],
195 manifest: Uint8Array,
196): AsyncGenerator<Uint8Array> {
197 for (const entry of entries) {
198 if (entry.type === "symlink") {
199 yield* createEntryHeaders({
200 name: entry.name,
201 mode: entry.mode,
202 size: 0,
203 type: "symlink",
204 linkname: entry.linkname,
205 });
206 continue;
207 }
208
209 yield* createEntryHeaders({
210 name: entry.name,
211 mode: entry.mode,
212 size: entry.file.size,
213 type: "file",
214 });
215 let emitted = 0;
216 for await (const chunk of verifiedFileChunks(entry.file)) {
217 emitted += chunk.byteLength;
218 yield chunk;
219 }
220 if (emitted !== entry.file.size) {
221 throw new Error(
222 `Compute artifact file '${entry.name}' changed while the archive was being created.`,
223 );
224 }
225 const padding = paddingLength(entry.file.size);
226 if (padding > 0) yield new Uint8Array(padding);
227 }
228
229 yield createHeader({
230 name: "compute.manifest.json",
231 mode: 0o644,
232 size: manifest.byteLength,
233 type: "file",
234 });
235 yield manifest;
236 const manifestPadding = paddingLength(manifest.byteLength);
237 if (manifestPadding > 0) yield new Uint8Array(manifestPadding);
238 yield new Uint8Array(1024);
239}
240
241interface TarHeader {
242 readonly name: string;
243 readonly mode: number;
244 readonly size: number;
245 readonly type: "file" | "symlink" | "pax";
246 readonly linkname?: string;
247}
248
249const paxRecord = (key: string, value: string) => {
250 const body = ` ${key}=${value}\n`;
251 let length = byteLength(body) + 1;
252 while (byteLength(String(length)) + byteLength(body) !== length) {
253 length = byteLength(String(length)) + byteLength(body);
254 }
255 return `${length}${body}`;
256};
257
258function* createEntryHeaders(entry: TarHeader): Generator<Uint8Array> {
259 const longPath = splitTarName(entry.name) === undefined;
260 const longLink =
261 entry.linkname !== undefined && byteLength(entry.linkname) > 100;
262 if (longPath || longLink) {
263 const data = new TextEncoder().encode(
264 (longPath ? paxRecord("path", entry.name) : "") +
265 (longLink ? paxRecord("linkpath", entry.linkname!) : ""),
266 );
267 yield createHeader({
268 name: "PaxHeaders/entry",
269 mode: 0o644,
270 size: data.byteLength,
271 type: "pax",
272 });
273 yield data;
274 const padding = paddingLength(data.byteLength);
275 if (padding > 0) yield new Uint8Array(padding);
276 }
277 yield createHeader({
278 ...entry,
279 name: longPath ? "PaxEntry" : entry.name,
280 linkname: longLink ? "PaxLink" : entry.linkname,
281 });
282}
283
284const createHeader = (entry: TarHeader) => {
285 const header = new Uint8Array(512);
286 const { name, prefix } = splitTarName(entry.name)!;
287 if (entry.linkname && byteLength(entry.linkname) > 100) {
288 throw new Error(
289 `Archive symlink target is too long for tar header: ${entry.linkname}`,
290 );
291 }
292
293 writeString(header, 0, 100, name);
294 writeOctal(header, 100, 8, entry.mode);
295 writeOctal(header, 108, 8, 0);
296 writeOctal(header, 116, 8, 0);
297 writeOctal(header, 124, 12, entry.size);
298 writeOctal(header, 136, 12, 0);
299 header.fill(0x20, 148, 156);
300 writeString(
301 header,
302 156,
303 1,
304 entry.type === "pax" ? "x" : entry.type === "symlink" ? "2" : "0",
305 );
306 if (entry.linkname) writeString(header, 157, 100, entry.linkname);
307 writeString(header, 257, 6, "ustar");
308 writeString(header, 263, 2, "00");
309 writeString(header, 265, 32, "alchemy");
310 writeString(header, 297, 32, "alchemy");
311 if (prefix) writeString(header, 345, 155, prefix);
312
313 const checksum = header.reduce((sum, value) => sum + value, 0);
314 writeChecksum(header, checksum);
315 return header;
316};
317
318const splitTarName = (
319 name: string,
320): { name: string; prefix?: string } | undefined => {
321 if (byteLength(name) <= 100) return { name };
322 const slashIndexes = Array.from(name.matchAll(/\//g), (match) => match.index);
323 for (const index of slashIndexes.reverse()) {
324 if (index === undefined) continue;
325 const prefix = name.slice(0, index);
326 const suffix = name.slice(index + 1);
327 if (byteLength(prefix) <= 155 && byteLength(suffix) <= 100) {
328 return { name: suffix, prefix };
329 }
330 }
331 return undefined;
332};
333
334const byteLength = (value: string) => new TextEncoder().encode(value).length;
335
336const writeString = (
337 buffer: Uint8Array,
338 offset: number,
339 length: number,
340 value: string,
341) => {
342 const bytes = new TextEncoder().encode(value);
343 buffer.set(bytes.slice(0, length), offset);
344};
345
346const writeOctal = (
347 buffer: Uint8Array,
348 offset: number,
349 length: number,
350 value: number,
351) => {
352 const text = value
353 .toString(8)
354 .padStart(length - 1, "0")
355 .slice(0, length - 1);
356 writeString(buffer, offset, length, `${text}\0`);
357};
358
359const writeChecksum = (buffer: Uint8Array, checksum: number) => {
360 const text = checksum.toString(8).padStart(6, "0").slice(0, 6);
361 writeString(buffer, 148, 8, `${text}\0 `);
362};
363
364const paddingLength = (size: number) => (512 - (size % 512)) % 512;