1import { getFiletypeFromFileName } from "@pierre/diffs";
2import {
3 parse,
4 type Command,
5 type DoubleQuotedChild,
6 type HereDoc,
7 type Word,
8 type WordPart,
9} from "unbash";
10
11/**
12 * Code in another language inside a shell command: a `bash -lc "…"` script, a
13 * `python3 -c '…'` argument, or a heredoc fed to an interpreter or written to
14 * a file. `starts[i]` and `ends[i]` are the span of the original command that
15 * spells `text[i]`, so a quoted `\"` maps back to both of its characters.
16 */
17export interface EmbeddedScript {
18 readonly language: string;
19 readonly text: string;
20 readonly starts: ReadonlyArray<number>;
21 readonly ends: ReadonlyArray<number>;
22}
23
24interface MappedText {
25 readonly text: string;
26 readonly starts: ReadonlyArray<number>;
27 readonly ends: ReadonlyArray<number>;
28}
29
30interface Interpreter {
31 readonly language: string;
32 /** Flags whose next argument is the script. */
33 readonly inline: (flag: string) => boolean;
34 /** Options that consume the next argument. */
35 readonly valueOptions?: ReadonlySet<string>;
36 /** Options after which the script comes from a file or module instead. */
37 readonly scriptFileOptions?: ReadonlySet<string>;
38 /** "always" for SQL shells, which read statements from stdin next to their database argument. */
39 readonly stdin: "always" | "without-script-file" | "never";
40 /** The first positional argument holding a script, as in `sqlite3 app.db "select 1"`. */
41 readonly positionalScript?: number;
42 /** SQL shells run every `-c` and every statement argument, in order. */
43 readonly repeatable?: boolean;
44 readonly subcommand?: string;
45 /** PowerShell matches parameters case-insensitively; the lists above are lowercase. */
46 readonly ignoreCase?: boolean;
47}
48
49const MAX_DEPTH = 4;
50
51const flagIn =
52 (...flags: string[]) =>
53 (flag: string) =>
54 flags.includes(flag);
55
56const SHELL: Interpreter = {
57 language: "shellscript",
58 inline: (flag) => /^-[a-z]*c[a-z]*$/u.test(flag),
59 valueOptions: new Set(["-o", "-O", "--rcfile", "--init-file"]),
60 stdin: "without-script-file",
61};
62
63const PYTHON: Interpreter = {
64 language: "python",
65 inline: flagIn("-c"),
66 valueOptions: new Set(["-W", "-X"]),
67 scriptFileOptions: new Set(["-m"]),
68 stdin: "without-script-file",
69};
70
71// Shared by SQL clients; each names its own script flags, since `-c` and `-f`
72// mean different things to psql and mysql.
73const SQL_SHELL = {
74 language: "sql",
75 valueOptions: new Set([
76 "-d",
77 "-h",
78 "-u",
79 "-U",
80 "--dbname",
81 "--host",
82 "--user",
83 "--username",
84 "-separator",
85 "-nullvalue",
86 "-init",
87 ]),
88 stdin: "always",
89 repeatable: true,
90} satisfies Omit<Interpreter, "inline">;
91
92const INTERPRETERS: Record<string, Interpreter> = {
93 sh: SHELL,
94 bash: SHELL,
95 zsh: SHELL,
96 dash: SHELL,
97 ash: SHELL,
98 ksh: SHELL,
99 fish: { language: "fish", inline: flagIn("-c", "--command"), stdin: "without-script-file" },
100 node: {
101 language: "javascript",
102 inline: flagIn("-e", "--eval", "-p", "--print"),
103 valueOptions: new Set(["-r", "--require", "--import", "--loader", "-C", "--conditions"]),
104 stdin: "without-script-file",
105 },
106 bun: { language: "typescript", inline: flagIn("-e", "--eval", "-p", "--print"), stdin: "never" },
107 deno: {
108 language: "typescript",
109 inline: () => false,
110 // `--env-file` takes its optional path only as `--env-file=path`.
111 valueOptions: new Set(["-c", "--config", "--import-map", "--ext"]),
112 stdin: "never",
113 subcommand: "eval",
114 },
115 ruby: {
116 language: "ruby",
117 inline: flagIn("-e"),
118 valueOptions: new Set(["-r", "-I", "-C"]),
119 stdin: "without-script-file",
120 },
121 perl: { language: "perl", inline: flagIn("-e", "-E"), stdin: "without-script-file" },
122 osascript: {
123 language: "applescript",
124 inline: flagIn("-e"),
125 valueOptions: new Set(["-l", "-s"]),
126 stdin: "without-script-file",
127 },
128 pwsh: {
129 language: "powershell",
130 inline: flagIn("-c", "-command"),
131 valueOptions: new Set(["-executionpolicy", "-workingdirectory", "-windowstyle"]),
132 scriptFileOptions: new Set(["-f", "-file", "-encodedcommand"]),
133 stdin: "never",
134 ignoreCase: true,
135 },
136 psql: {
137 ...SQL_SHELL,
138 inline: flagIn("-c", "--command"),
139 scriptFileOptions: new Set(["-f", "--file"]),
140 },
141 mysql: { ...SQL_SHELL, inline: flagIn("-e", "--execute") },
142 sqlite3: { ...SQL_SHELL, inline: flagIn("-cmd"), positionalScript: 1 },
143 duckdb: {
144 ...SQL_SHELL,
145 inline: flagIn("-c", "-cmd"),
146 scriptFileOptions: new Set(["-f"]),
147 positionalScript: 1,
148 },
149};
150INTERPRETERS.powershell = INTERPRETERS.pwsh!;
151INTERPRETERS.mariadb = INTERPRETERS.mysql!;
152
153// Commands that run the next word as the program, with the options that consume a value.
154const PROGRAM_WRAPPERS: Record<string, ReadonlySet<string>> = {
155 env: new Set(["-u", "--unset", "-C", "--chdir"]),
156 sudo: new Set(["-u", "--user", "-g", "--group", "-C", "-D", "--chdir"]),
157 nice: new Set(["-n", "--adjustment"]),
158 nohup: new Set(),
159 exec: new Set(["-a"]),
160 command: new Set(),
161 builtin: new Set(),
162};
163
164// Heredoc delimiters that name their language, as in `<<'SQL'`.
165const DELIMITER_LANGUAGES: Record<string, string> = {
166 py: "python",
167 python: "python",
168 js: "javascript",
169 javascript: "javascript",
170 ts: "typescript",
171 typescript: "typescript",
172 sql: "sql",
173 json: "json",
174 yaml: "yaml",
175 yml: "yaml",
176 toml: "toml",
177 sh: "shellscript",
178 bash: "shellscript",
179 shell: "shellscript",
180 rb: "ruby",
181 ruby: "ruby",
182 html: "html",
183 css: "css",
184 md: "markdown",
185 markdown: "markdown",
186 xml: "xml",
187 diff: "diff",
188 patch: "diff",
189};
190
191/**
192 * The scripts embedded in a shell command, outermost first, so a highlighter
193 * painting them in order lets a nested script override the one containing it.
194 * Returns nothing for a command that does not parse cleanly; where a broken
195 * quote ends cannot be trusted.
196 */
197export function embeddedScripts(command: string): EmbeddedScript[] {
198 const identity = Array.from({ length: command.length }, (_, index) => index);
199 return scriptsIn(
200 { text: command, starts: identity, ends: identity.map((index) => index + 1) },
201 0,
202 );
203}
204
205function scriptsIn(source: MappedText, depth: number): EmbeddedScript[] {
206 const script = parse(source.text);
207 if (script.errors?.length) return [];
208 const found: EmbeddedScript[] = [];
209 forEachCommand(script.commands, (command) => {
210 for (const { language, mapped } of commandScripts(command, source.text)) {
211 if (mapped.text.trim() === "") continue;
212 const embedded: EmbeddedScript = {
213 language,
214 text: mapped.text,
215 starts: mapped.starts.map((start) => source.starts[start]!),
216 ends: mapped.ends.map((end) => source.ends[end - 1]!),
217 };
218 found.push(embedded);
219 if (language === "shellscript" && depth + 1 < MAX_DEPTH) {
220 found.push(...scriptsIn(embedded, depth + 1));
221 }
222 }
223 });
224 return found;
225}
226
227function isNode(value: unknown): value is { readonly type: string } {
228 return typeof value === "object" && value !== null && "type" in value;
229}
230
231/** Visits every simple command, including those inside `$(…)`, loops and subshells. */
232function forEachCommand(
233 node: unknown,
234 visit: (command: Command) => void,
235 seen = new WeakSet<object>(),
236): void {
237 if (Array.isArray(node)) {
238 for (const child of node) forEachCommand(child, visit, seen);
239 return;
240 }
241 if (!isNode(node) || seen.has(node)) return;
242 seen.add(node);
243 switch (node.type) {
244 case "Command":
245 visit(node as Command);
246 break;
247 case "CommandExpansion":
248 case "ProcessSubstitution": {
249 const { script } = node as Extract<WordPart, { type: "CommandExpansion" }>;
250 // Escaped-backtick scripts own a decoded source their positions index.
251 if (script && !script.errors?.length && script.source === undefined) {
252 forEachCommand(script.commands, visit, seen);
253 }
254 return;
255 }
256 }
257 // Word parts are lazy getters, which Object.values skips.
258 if ("parts" in node) forEachCommand(node.parts, visit, seen);
259 for (const value of Object.values(node)) {
260 if (typeof value === "object") forEachCommand(value, visit, seen);
261 }
262}
263
264function programName(word: Word): string {
265 // Bash reads `C:\Python312\python.exe` unquoted as escapes; Windows meant a path.
266 const path = /\\/u.test(word.text) ? word.text.replace(/^["']|["']$/gu, "") : word.value;
267 return (path.split(/[\\/]/u).at(-1) ?? "").toLowerCase().replace(/\.exe$/u, "");
268}
269
270function interpreterFor(name: string): Interpreter | undefined {
271 // `py` is the Windows Python launcher, as in `py -3 -c '…'`.
272 if (/^python(?:\d+(?:\.\d+)?)?$|^pypy3?$|^py$/u.test(name)) return PYTHON;
273 return Object.hasOwn(INTERPRETERS, name) ? INTERPRETERS[name] : undefined;
274}
275
276/** Skips `env FOO=1`, `sudo -u bob` and similar to the word naming the real program. */
277function programIndex(words: ReadonlyArray<Word>): number {
278 let index = 0;
279 while (index < words.length) {
280 const name = programName(words[index]!);
281 if (!Object.hasOwn(PROGRAM_WRAPPERS, name)) return index;
282 const valueOptions = PROGRAM_WRAPPERS[name]!;
283 index += 1;
284 let optionsEnded = false;
285 while (index < words.length) {
286 const value = words[index]!.value;
287 const isOption = !optionsEnded && value.startsWith("-");
288 if (isOption && value === "--") optionsEnded = true;
289 // `env -- FOO=1 cmd` still takes assignments after the options end.
290 if (isOption || /^[A-Za-z_]\w*=/u.test(value)) {
291 index += isOption && valueOptions.has(value) ? 2 : 1;
292 } else {
293 break;
294 }
295 }
296 }
297 return index;
298}
299
300function commandScripts(
301 command: Command,
302 source: string,
303): Array<{ language: string; mapped: MappedText }> {
304 if (!command.name) return [];
305 const words = [command.name, ...command.args.filter((arg): arg is Word => arg.type === "Word")];
306 const index = programIndex(words);
307 const program = words[index];
308 if (!program) return [];
309 const name = programName(program);
310 const interpreter = interpreterFor(name);
311 const args = words.slice(index + 1);
312 const scripts: Array<{ language: string; mapped: MappedText }> = [];
313
314 let readsStdinScript = false;
315 if (interpreter) {
316 const found = interpreterArgs(interpreter, args);
317 for (const script of found.scripts) {
318 const mapped = wordText(script.word, source);
319 if (!mapped) continue;
320 const start = script.inValue ? mapped.text.indexOf("=") + 1 : 0;
321 scripts.push({ language: interpreter.language, mapped: sliceMapped(mapped, start) });
322 }
323 readsStdinScript =
324 interpreter.stdin !== "never" && found.scripts.length === 0 && !found.scriptFromFile;
325 }
326
327 const stdinLanguage = readsStdinScript ? interpreter?.language : undefined;
328 const writtenLanguage = writtenFileLanguage(name, command, args);
329 for (const redirect of command.redirects) {
330 const toStdin = isDescriptor(redirect.descriptor, 0);
331 if (redirect.type === "HereString" && toStdin && stdinLanguage && redirect.target) {
332 const mapped = wordText(redirect.target, source);
333 if (mapped) scripts.push({ language: stdinLanguage, mapped });
334 }
335 if (redirect.type === "HereDoc") {
336 const language =
337 (toStdin ? (stdinLanguage ?? writtenLanguage) : undefined) ?? delimiterLanguage(redirect);
338 if (language) scripts.push({ language, mapped: hereDocText(redirect, source) });
339 }
340 }
341 return scripts;
342}
343
344/** Whether a redirection without a descriptor, or with `fd`, targets file descriptor `fd`. */
345function isDescriptor(descriptor: HereDoc["descriptor"], fd: number): boolean {
346 return (
347 descriptor === undefined || (descriptor.type === "FileDescriptor" && descriptor.value === fd)
348 );
349}
350
351interface ScriptArgument {
352 readonly word: Word;
353 /** The script follows the `=` of `--eval=…` rather than filling the word. */
354 readonly inValue: boolean;
355}
356
357/** The script arguments, and whether the script comes from a file instead. */
358function interpreterArgs(
359 interpreter: Interpreter,
360 args: ReadonlyArray<Word>,
361): { scripts: ScriptArgument[]; scriptFromFile: boolean } {
362 const positionals: Word[] = [];
363 const scripts: ScriptArgument[] = [];
364 const whole = (word: Word): ScriptArgument => ({ word, inValue: false });
365 let optionsEnded = false;
366 let scriptFromFile = false;
367 for (let index = 0; index < args.length; index += 1) {
368 const arg = args[index]!;
369 const flag = interpreter.ignoreCase ? arg.value.toLowerCase() : arg.value;
370 if (!optionsEnded && flag === "--") {
371 optionsEnded = true;
372 continue;
373 }
374 if (!optionsEnded && flag.startsWith("-") && flag !== "-") {
375 // `--eval=code` and `--file=path` carry their value in the same word.
376 const [name = flag, value] = flag.startsWith("--") ? flag.split(/=(.*)/su) : [flag];
377 if (interpreter.scriptFileOptions?.has(name)) {
378 if (!interpreter.repeatable) return { scripts, scriptFromFile: true };
379 // SQL shells run `-f` files and `-c` statements alike, in order.
380 scriptFromFile = true;
381 if (value === undefined) index += 1;
382 continue;
383 }
384 if (interpreter.inline(name)) {
385 const next = args[index + 1];
386 if (value !== undefined) scripts.push({ word: arg, inValue: true });
387 else if (next) scripts.push(whole(next));
388 if (!interpreter.repeatable) return { scripts, scriptFromFile: false };
389 if (value === undefined) index += 1;
390 continue;
391 }
392 if (value === undefined && interpreter.valueOptions?.has(flag)) index += 1;
393 continue;
394 }
395 positionals.push(arg);
396 // An interpreter stops reading its own options at the script file.
397 if (interpreter.stdin === "without-script-file" && !interpreter.subcommand) break;
398 }
399
400 if (interpreter.subcommand) {
401 if (positionals[0]?.value !== interpreter.subcommand) return { scripts, scriptFromFile: true };
402 return { scripts: positionals.slice(1, 2).map(whole), scriptFromFile: false };
403 }
404 if (interpreter.positionalScript !== undefined) {
405 scripts.push(...positionals.slice(interpreter.positionalScript).map(whole));
406 }
407 // SQL shells take databases as positional arguments, not script files.
408 if (interpreter.stdin === "always") return { scripts, scriptFromFile };
409 const first = positionals[0];
410 return { scripts, scriptFromFile: first !== undefined && first.value !== "-" };
411}
412
413function sliceMapped(mapped: MappedText, start: number): MappedText {
414 if (start === 0) return mapped;
415 return {
416 text: mapped.text.slice(start),
417 starts: mapped.starts.slice(start),
418 ends: mapped.ends.slice(start),
419 };
420}
421
422/** The language of the file `cat > file` or `tee file` writes its stdin to. */
423function writtenFileLanguage(
424 name: string,
425 command: Command,
426 args: ReadonlyArray<Word>,
427): string | undefined {
428 let path: string | undefined;
429 if (name === "cat") {
430 for (const redirect of command.redirects) {
431 if (
432 redirect.type === "Redirect" &&
433 [">", ">>", ">|", "&>", "&>>"].includes(redirect.operator) &&
434 isDescriptor(redirect.descriptor, 1)
435 ) {
436 path = redirect.target?.value;
437 }
438 }
439 } else if (name === "tee") {
440 path = args.find((arg) => !arg.value.startsWith("-"))?.value;
441 }
442 if (!path) return undefined;
443 const language = getFiletypeFromFileName(path);
444 return language === "text" ? undefined : language;
445}
446
447function delimiterLanguage(hereDoc: HereDoc): string | undefined {
448 const name = hereDoc.delimiter?.value.toLowerCase();
449 return name && Object.hasOwn(DELIMITER_LANGUAGES, name) ? DELIMITER_LANGUAGES[name] : undefined;
450}
451
452/** A heredoc body as written; the interpreter reads its characters one to one. */
453function hereDocText(hereDoc: HereDoc, source: string): MappedText {
454 const { pos, end } = hereDoc.body;
455 const starts = Array.from({ length: end - pos }, (_, index) => pos + index);
456 return {
457 text: source.slice(pos, end),
458 starts,
459 ends: starts.map((start) => start + 1),
460 };
461}
462
463/**
464 * A word's value with the source span of each character: quotes removed and
465 * escapes resolved, the way the program receives it. Expansions such as
466 * `$HOME` stay as written. Returns undefined for spellings it cannot map.
467 */
468function wordText(word: Word, source: string): MappedText | undefined {
469 const builder = new MappedTextBuilder();
470 const parts: ReadonlyArray<WordPart> = word.parts ?? [
471 { type: "Literal", pos: word.pos, end: word.end, text: word.text, value: word.value },
472 ];
473 for (const part of parts) {
474 switch (part.type) {
475 case "Literal":
476 if (!builder.addEscaped(part, false)) return undefined;
477 break;
478 case "SingleQuoted":
479 builder.addRaw(part.value, part.pos + 1);
480 break;
481 case "DoubleQuoted":
482 case "LocaleString":
483 for (const child of part.parts) {
484 if (!builder.addDoubleQuotedChild(child)) return undefined;
485 }
486 break;
487 case "AnsiCQuoted":
488 // $'…' escapes such as \x41 do not map character for character.
489 if (part.value !== part.text.slice(2, -1)) return undefined;
490 builder.addRaw(part.value, part.pos + 2);
491 break;
492 default:
493 builder.addRaw(source.slice(part.pos, part.end), part.pos);
494 }
495 }
496 return builder.build();
497}
498
499class MappedTextBuilder {
500 private text = "";
501 private readonly starts: number[] = [];
502 private readonly ends: number[] = [];
503
504 addRaw(text: string, pos: number) {
505 for (let index = 0; index < text.length; index += 1) this.push(text[index]!, pos + index, 1);
506 }
507
508 addDoubleQuotedChild(child: DoubleQuotedChild): boolean {
509 if (child.type === "Literal") return this.addEscaped(child, true);
510 this.addRaw(child.text, child.pos);
511 return true;
512 }
513
514 /** Adds a literal, resolving backslash escapes; false when the result disagrees with the parser. */
515 addEscaped(literal: { pos: number; text: string; value: string }, doubleQuoted: boolean) {
516 const before = this.text.length;
517 const { text, pos } = literal;
518 for (let index = 0; index < text.length; index += 1) {
519 const next = text[index + 1];
520 if (
521 text[index] === "\\" &&
522 next !== undefined &&
523 (!doubleQuoted || '$`"\\\n'.includes(next))
524 ) {
525 // A backslash-newline joins lines and spells nothing.
526 if (next !== "\n") this.push(next, pos + index, 2);
527 index += 1;
528 } else {
529 this.push(text[index]!, pos + index, 1);
530 }
531 }
532 return this.text.slice(before) === literal.value;
533 }
534
535 build(): MappedText {
536 return { text: this.text, starts: this.starts, ends: this.ends };
537 }
538
539 private push(character: string, start: number, length: number) {
540 this.text += character;
541 this.starts.push(start);
542 this.ends.push(start + length);
543 }
544}