diff --git a/.gitignore b/.gitignore index fd9d1ca548f88694b09655ec42567e08161475b9..6710eec8822e2529b586501bb4a5f9711025c5c8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ flake.lock result result-* +node_modules diff --git a/packages/autofmt/autofmt.js b/packages/autofmt/autofmt.js index 8986cf9974c2abaa3f9b832f73b77d961d97e0eb..7211f0b32c6e745f2c6807356e8d38b48cb79425 100755 --- a/packages/autofmt/autofmt.js +++ b/packages/autofmt/autofmt.js @@ -69,12 +69,25 @@ const formatters = { cmd: ["treefmt", "--", "$files"], stdin: (file) => ["treefmt", "--stdin", file], }, + vp: { + languages: ["javascript", "markdown", "json", "yaml", "mdx", "html", "css"], + mustMatchFile: true, + files: ["node_modules/.bin/vp"], + cmd: ["node_modules/.bin/vp", "fmt", "$files"], + stdin: (file) => [ + "node_modules/.bin/vp", + "fmt", + "--stdin-filepath", + file, + ], + }, // -- web ecosystem -- - deno: { - languages: ["javascript", "markdown", "json", "yaml", "html", "css"], - files: ["dprint.json", "dprint.jsonc"], - cmd: ["deno", "fmt", "--", "$files"], - stdin: (file) => ["deno", "fmt", "--ext", path.extname(file).slice(1), "-"], + oxfmt: { + languages: ["javascript", "markdown", "json", "yaml", "mdx", "html", "css"], + mustMatchFile: true, + files: ["node_modules/.bin/oxfmt", ".oxfmtrc.json", ".oxfmtrc.jsonc"], + cmd: ["node_modules/.bin/oxfmt", "$files"], + stdin: (file) => ["node_modules/.bin/oxfmt", `--stdin-filepath=${file}`], }, prettierd: { // keep in mind that pretterd never exits. @@ -101,6 +114,12 @@ const formatters = { `--stdin-file-path=${file}`, ], }, + deno: { + languages: ["javascript", "markdown", "json", "yaml", "html", "css"], + files: ["deno.json", "deno.jsonc"], + cmd: ["deno", "fmt", "--", "$files"], + stdin: (file) => ["deno", "fmt", "--ext", path.extname(file).slice(1), "-"], + }, "nixfmt": { languages: ["nix"], files: { diff --git a/users/clover/home.nix b/users/clover/home.nix index 7e413bb4c3b2119ee41154823fb239c50d796f50..673e3b5cb39ed98b2ce92fea51331f2069b90d03 100644 --- a/users/clover/home.nix +++ b/users/clover/home.nix @@ -10,6 +10,9 @@ in { home = { stateVersion = "23.05"; # Please read the comment before changing. + sessionPath = [ + "$HOME/.vite-plus/bin" + ]; packages = with pkgs; let @@ -195,6 +198,14 @@ in }; aliases = { "ab" = [ "abandon" ]; + "ci" = [ + "util" + "exec" + "--" + "node" + "--experimental-strip-types" + "${config.home.homeDirectory}/config/users/clover/scripts/jj-ci.ts" + ]; # "ai" = [ # "util" # "exec" diff --git a/users/clover/package.json b/users/clover/package.json new file mode 100644 index 0000000000000000000000000000000000000000..00fbc7e587fc82d92d21e075432e10f0758c86e2 --- /dev/null +++ b/users/clover/package.json @@ -0,0 +1,9 @@ +{ + "type": "module", + "packageManager": "pnpm@10.28.0", + "dependencies": { + "@clo/lib": "jsr:*", + "@lydell/node-pty": "1.2.0-beta.12", + "@types/node": "^24.12.2" + } +} diff --git a/users/clover/pnpm-lock.yaml b/users/clover/pnpm-lock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..859de6f0999948c34e8427d904d6a1ef314828c9 --- /dev/null +++ b/users/clover/pnpm-lock.yaml @@ -0,0 +1,100 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@clo/lib': + specifier: jsr:* + version: '@jsr/clo__lib@4.0.0' + '@lydell/node-pty': + specifier: 1.2.0-beta.12 + version: 1.2.0-beta.12 + '@types/node': + specifier: ^24.12.2 + version: 24.12.2 + +packages: + + '@jsr/clo__lib@4.0.0': + resolution: {integrity: sha512-uOzRjwbFn9HizCpJeqPXzuiri3hqqcePNLA7ESi1Vgwon9dOmhMSzHfmb9kz599VsKF1RLH7i3NK302K60Oe/A==, tarball: https://npm.jsr.io/~/11/@jsr/clo__lib/4.0.0.tgz} + + '@lydell/node-pty-darwin-arm64@1.2.0-beta.12': + resolution: {integrity: sha512-tqaifcY9Cr41SblO1+FLzh8oxxtkNhuW9Dhl22lKme9BreYvKvxEZcdPIXTuqkJc5tagOEC4QHShKmJjLyLXLQ==} + cpu: [arm64] + os: [darwin] + + '@lydell/node-pty-darwin-x64@1.2.0-beta.12': + resolution: {integrity: sha512-4LrS5pCJwqHKDVf1zS2gyNV0m4hKAXch+XZNhbZ6LY8uwVL8BhchzQBO40Os5anuRxRCWzHpw4Sp64Ie8q7E4Q==} + cpu: [x64] + os: [darwin] + + '@lydell/node-pty-linux-arm64@1.2.0-beta.12': + resolution: {integrity: sha512-Sx+A71x5BDGHt9ansfrtGxwq2VFVDWvJUAdlUL0Hv0qeiJUfts+hgopx+CgT4PSwahKjdEgtu0+FAfY9rICKRw==} + cpu: [arm64] + os: [linux] + + '@lydell/node-pty-linux-x64@1.2.0-beta.12': + resolution: {integrity: sha512-bJzs94njofYhGg/UDqW1nj0dtvvu+2OvxMY+RlLS1T17VgcktKoIR6PuenTwE5HJ/D6StCPADmXcT0nNsCKmIQ==} + cpu: [x64] + os: [linux] + + '@lydell/node-pty-win32-arm64@1.2.0-beta.12': + resolution: {integrity: sha512-p7POgjVEiFaBC3/y+AKuV1FzePCsJ6HmZDv2XK+jBZSfwP8+uBAw181ZiKYN1YuRa/XpmBGaWezcI8hZkbW++g==} + cpu: [arm64] + os: [win32] + + '@lydell/node-pty-win32-x64@1.2.0-beta.12': + resolution: {integrity: sha512-IDFa00g7qUDGUYgByrUBJtC+mOjYVt/8KYyWivCg5JjGOHbBUACUQZLl0jTWmnr+tld/UyTpX90a2PY6oTVtRw==} + cpu: [x64] + os: [win32] + + '@lydell/node-pty@1.2.0-beta.12': + resolution: {integrity: sha512-qIK890UwPupoj07osVvgOIa++1mxeHbcGry4PKRHhNVNs81V2SCG34eJr46GybiOmBtc8Sj5PB1/GGM5PL549g==} + + '@types/node@24.12.2': + resolution: {integrity: sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==} + + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + +snapshots: + + '@jsr/clo__lib@4.0.0': {} + + '@lydell/node-pty-darwin-arm64@1.2.0-beta.12': + optional: true + + '@lydell/node-pty-darwin-x64@1.2.0-beta.12': + optional: true + + '@lydell/node-pty-linux-arm64@1.2.0-beta.12': + optional: true + + '@lydell/node-pty-linux-x64@1.2.0-beta.12': + optional: true + + '@lydell/node-pty-win32-arm64@1.2.0-beta.12': + optional: true + + '@lydell/node-pty-win32-x64@1.2.0-beta.12': + optional: true + + '@lydell/node-pty@1.2.0-beta.12': + optionalDependencies: + '@lydell/node-pty-darwin-arm64': 1.2.0-beta.12 + '@lydell/node-pty-darwin-x64': 1.2.0-beta.12 + '@lydell/node-pty-linux-arm64': 1.2.0-beta.12 + '@lydell/node-pty-linux-x64': 1.2.0-beta.12 + '@lydell/node-pty-win32-arm64': 1.2.0-beta.12 + '@lydell/node-pty-win32-x64': 1.2.0-beta.12 + + '@types/node@24.12.2': + dependencies: + undici-types: 7.16.0 + + undici-types@7.16.0: {} diff --git a/users/clover/pnpm-workspace.yaml b/users/clover/pnpm-workspace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3116a5fe3b913a743bcc38eda05dc03c5b53ebbb --- /dev/null +++ b/users/clover/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +onlyBuiltDependencies: + - node-pty diff --git a/users/clover/scripts/jj-ci.ts b/users/clover/scripts/jj-ci.ts new file mode 100755 index 0000000000000000000000000000000000000000..7fdb2b319bf0ee57e994edad131f7dbdad99a611 --- /dev/null +++ b/users/clover/scripts/jj-ci.ts @@ -0,0 +1,1115 @@ +#!/usr/bin/env node +import * as child_process from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import * as log from "@clo/lib/log"; +import * as progress from "@clo/lib/progress"; +import * as ansi from "@clo/lib/string/ansi"; +import * as pty from "@lydell/node-pty"; +import * as ts from "@clo/lib/ts"; +import process from "node:process"; + +const progressRoot = new progress.Root(); +progressAttachToScreen(progressRoot, log); + +type CommandOptions = { + cwd?: string; + env?: NodeJS.ProcessEnv; + ignoreWorkingCopy?: boolean; + node?: progress.Node; + plainLog?: string[]; +}; + +type Plan = { + changeId: string; + dependencyChangeIds: string[]; + label: string; + logPath: string; +}; + +type ScriptResult = { + durationMs: number; + exitCode: number | null; + ok: boolean; + plainText: string; +}; + +type JobPhase = "prep" | "ci" | "skipped" | "snapshot" | "stalled"; +type JobResult = { + blockedBy?: string[]; + changeId: string; + durationMs: number; + exitCode: number | null; + label: string; + logPath?: string; + ok: boolean; + phase: JobPhase; + workspacePath?: string; +}; + +type JobState = "failed" | "ok" | "pending" | "running" | "skipped"; + +type PoolSlot = { + name: string; + workspaceName: string; + workspacePath: string; +}; + +type ActiveJob = { + slot: PoolSlot; + task: Promise; +}; + +const dimStyle = ansi.dim + ansi.fgBrightBlack; +const headingStyle = ansi.bold + ansi.fgBrightWhite; +const okStyle = ansi.bold + ansi.fgBrightGreen; +const failStyle = ansi.bold + ansi.fgBrightRed; +const skipStyle = ansi.bold + ansi.fgBrightYellow; +const infoStyle = ansi.bold + ansi.fgBrightCyan; +const workspaceCacheManifestName = ".paths"; + +let clean = false; +let dryRun = false; +let ignoreImmutable = false; +let jobs = os.availableParallelism(); +let parallelize = false; +let retry = false; +let sawJobs = false; +let warnedAboutPtyFallback = false; +const revisions: string[] = []; + +const argv = process.argv.slice(2); +while (argv.length > 0) { + const arg = argv.shift(); + if (arg == null) break; + + if (arg === "-h" || arg === "--help") usage(0); + else if (arg === "--dry-run") dryRun = true; + else if (arg === "--clean") clean = true; + else if (arg === "--ignore-immutable") ignoreImmutable = true; + else if (arg === "-p" || arg === "--parallel") parallelize = true; + else if (arg === "-r" || arg === "--retry") retry = true; + else if (arg === "-j" || arg === "--jobs" || arg === "--cores") { + if (sawJobs) { + log.error("can only pass --jobs once"); + usage(1); + } + + const value = argv.shift(); + if (value == null) { + log.error("missing value for --jobs"); + usage(1); + } + + jobs = Number(value); + sawJobs = true; + } else if (arg.startsWith("--jobs=") || arg.startsWith("--cores=")) { + if (sawJobs) { + log.error("can only pass --jobs once"); + usage(1); + } + + jobs = Number(arg.slice(arg.indexOf("=") + 1)); + sawJobs = true; + } else if (arg === "--") { + revisions.push(...argv); + break; + } else if (arg.startsWith("-")) { + log.error("unknown option", JSON.stringify(arg)); + usage(1); + } else { + revisions.push(arg); + } +} + +if (!Number.isInteger(jobs) || jobs < 1) { + log.error("--jobs must be a positive integer"); + usage(1); +} + +if (retry && revisions.length > 0) { + log.error("cannot pass revisions together with --retry"); + usage(1); +} + +if (!retry && revisions.length === 0) usage(1); + +const root = jj(["workspace", "root"]).trim(); +const cacheDir = path.join(root, ".jj", "ci"); +const lastRunPath = path.join(cacheDir, "last-run.txt"); +const logDir = path.join(cacheDir, "_logs"); +const ciScript = path.join(root, ".jj", "ci.sh"); +const gitDir = resolveGitDir(root); +const selectedRevisions = retry ? readLastRun(lastRunPath) : revisions; + +let invalid = false; + +if (!fs.existsSync(ciScript)) { + log.error("missing .jj/ci.sh"); + invalid = true; +} + +if (invalid) process.exit(1); + +if (parallelize) { + if (dryRun) log.info("would run", "jj parallelize " + selectedRevisions.join(" ")); + else jj(["parallelize", ...selectedRevisions]); +} + +const changeIds = [...new Set(selectedRevisions.flatMap(resolveChangeIds))]; +const selectedExpr = `(${changeIds.join("|")})`; +const trackedWorkspaces = new Set( + jj(["workspace", "list"], { ignoreWorkingCopy: true }) + .split("\n") + .filter(Boolean) + .map((line) => line.split(":", 1)[0]?.trim()) + .filter((name): name is string => Boolean(name)), +); +const plans: Plan[] = changeIds.map((changeId) => ({ + changeId, + dependencyChangeIds: [ + ...new Set( + jj( + [ + "log", + "--no-graph", + "-r", + `ancestors(${changeId}) & ${selectedExpr}`, + "-T", + 'self.change_id() ++ "\n"', + ], + { ignoreWorkingCopy: true }, + ) + .split("\n") + .filter((candidate) => candidate !== "" && candidate !== changeId), + ), + ], + label: changeId.slice(0, 6) + ': ' + jj( + [ + "log", + "--no-graph", + "-r", + `${changeId}`, + "-T", + 'self.description().first_line()', + ], + { ignoreWorkingCopy: true }, + ), + logPath: path.join(logDir, `ci-${changeId}.log`), +})); +const concurrency = Math.min(jobs, maxPlanConcurrency(plans)); +const pool = Array.from({ length: concurrency }, (_, index): PoolSlot => { + const name = poolName(index); + return { + name, + workspaceName: `ci-${name}`, + workspacePath: path.join(cacheDir, name), + }; +}); +const planByChangeId = new Map(plans.map((plan) => [plan.changeId, plan])); +const childrenByChangeId = new Map( + changeIds.map((changeId) => [changeId, [] as string[]]), +); + +for (const plan of plans) { + for (const dependencyChangeId of plan.dependencyChangeIds) { + childrenByChangeId.get(dependencyChangeId)?.push(plan.changeId); + } +} + +if (dryRun) { + log.writeOutput( + JSON.stringify( + { + cacheDir, + ciScript, + clean, + concurrency, + ignoreImmutable, + parallelize, + plans, + pool, + retry, + selectedRevisions, + root, + }, + null, + 2, + ) + "\n", + ); + process.exit(0); +} + +if (clean && fs.existsSync(cacheDir)) { + fs.rmSync(cacheDir, { force: true, recursive: true }); +} + +fs.mkdirSync(cacheDir, { recursive: true }); +fs.mkdirSync(logDir, { recursive: true }); + +let runError: unknown = null; + +try { + const ready = plans.filter((plan) => plan.dependencyChangeIds.length === 0); + const availableSlots = [...pool]; + const remainingDependenciesByChangeId = new Map( + plans.map((plan) => [plan.changeId, plan.dependencyChangeIds.length]), + ); + const resultsByChangeId = new Map(); + const stateByChangeId = new Map( + changeIds.map((changeId) => [changeId, "pending"]), + ); + const active = new Map(); + + { + using runNode = progressRoot.start("continuous integration", { + total: plans.length, + valueFormatter() { + let passed = 0; + let failed = 0; + for (const state of stateByChangeId.values()) { + if (state === "ok") passed += 1; + else if (state === "failed" || state === "skipped") failed += 1; + } + return `${ansi.fgGreen}${passed} pass${ansi.reset}, ${failed > 0 ? ansi.fgRed : ""}${failed} fail${ansi.reset}`; + }, + }); + + while (resultsByChangeId.size < plans.length) { + while ( + ready.length > 0 && + active.size < concurrency && + availableSlots.length > 0 + ) { + const plan = ready.shift(); + const slot = availableSlots.shift(); + if (plan == null || slot == null) break; + + stateByChangeId.set(plan.changeId, "running"); + + const task = (async (): Promise => { + const startedAt = Date.now(); + using node = runNode.start(plan.label); + if (fs.existsSync(plan.logPath)) fs.rmSync(plan.logPath); + const env = { + ...process.env, + JJ_CI_CHANGE_ID: plan.changeId, + JJ_CI_ROOT: root, + JJ_CI_WORKSPACE: slot.workspacePath, + }; + const plainLog = [ + `change: ${plan.changeId}\n`, + `dependencies: ${ + plan.dependencyChangeIds.length === 0 + ? "(selected root)" + : plan.dependencyChangeIds.join(", ") + }\n`, + `slot: ${slot.name}\n`, + `workspace: ${slot.workspacePath}\n`, + `started: ${new Date(startedAt).toISOString()}\n\n`, + ]; + let phase: JobPhase = "prep"; + + try { + plainLog.push("--- prep ---\n"); + ensureWorkspace(slot, plan.changeId, plan.label, trackedWorkspaces, node, plainLog); + plainLog.push("\n"); + + phase = "ci"; + node.text = `${plan.label} ci [slot ${slot.name}]`; + const ci = await runScript(ciScript, { + cwd: slot.workspacePath, + env, + node, + }); + plainLog.push( + `--- ci ---\nexit: ${ci.exitCode ?? "signal"}\nduration_ms: ${ci.durationMs}\n\n`, + ); + plainLog.push(ci.plainText); + if (ci.plainText !== "" && !ci.plainText.endsWith("\n")) { + plainLog.push("\n"); + } + plainLog.push("\n"); + + phase = "snapshot"; + node.text = `${plan.label} snapshot [slot ${slot.name}]`; + plainLog.push("--- snapshot ---\n"); + snapshotWorkspace(slot.workspacePath, node, plainLog); + plainLog.push("\n"); + + if (!ci.ok) { + writeFailureLog(plan.logPath, plainLog); + return { + changeId: plan.changeId, + durationMs: Date.now() - startedAt, + exitCode: ci.exitCode, + label: plan.label, + logPath: plan.logPath, + ok: false, + phase: "ci", + workspacePath: slot.workspacePath, + }; + } + + node.value += 1; + node.text = `${plan.label} ok [slot ${slot.name}]`; + if (fs.existsSync(plan.logPath)) fs.rmSync(plan.logPath); + return { + changeId: plan.changeId, + durationMs: Date.now() - startedAt, + exitCode: ci.exitCode, + label: plan.label, + ok: true, + phase: "ci", + workspacePath: slot.workspacePath, + }; + } catch (error) { + const message = error instanceof Error + ? (error.stack ?? error.message) + : String(error); + node.log.error(error); + plainLog.push(`--- ${phase} error ---\n${normalizePlain(message)}\n`); + writeFailureLog(plan.logPath, plainLog); + return { + changeId: plan.changeId, + durationMs: Date.now() - startedAt, + exitCode: null, + label: plan.label, + logPath: plan.logPath, + ok: false, + phase, + workspacePath: slot.workspacePath, + }; + } + })(); + + active.set(plan.changeId, { slot, task }); + } + + if (active.size === 0) { + let skippedAny = false; + + for (const plan of plans) { + if (stateByChangeId.get(plan.changeId) !== "pending") continue; + + const blockedBy = plan.dependencyChangeIds.filter((dependencyChangeId) => { + const dependencyState = stateByChangeId.get(dependencyChangeId); + return dependencyState === "failed" || dependencyState === "skipped"; + }); + + if (blockedBy.length === 0) continue; + + skippedAny = true; + stateByChangeId.set(plan.changeId, "skipped"); + if (fs.existsSync(plan.logPath)) fs.rmSync(plan.logPath); + resultsByChangeId.set(plan.changeId, { + blockedBy, + changeId: plan.changeId, + durationMs: 0, + exitCode: null, + label: plan.label, + ok: false, + phase: "skipped", + }); + } + + if (skippedAny) { + runNode.value = resultsByChangeId.size; + continue; + } + + log.error("dependency scheduling stalled"); + for (const plan of plans) { + if (stateByChangeId.get(plan.changeId) !== "pending") continue; + + resultsByChangeId.set(plan.changeId, { + blockedBy: plan.dependencyChangeIds.filter((dependencyChangeId) => + stateByChangeId.get(dependencyChangeId) !== "ok" + ), + changeId: plan.changeId, + durationMs: 0, + exitCode: null, + label: plan.label, + ok: false, + phase: "stalled", + }); + stateByChangeId.set(plan.changeId, "failed"); + } + runNode.value = resultsByChangeId.size; + process.exitCode = 1; + continue; + } + + const result = await Promise.race( + [...active.values()].map(({ task }) => task), + ); + const completedJob = active.get(result.changeId); + if (completedJob == null) { + throw new Error(`missing active job for ${result.changeId}`); + } + active.delete(result.changeId); + availableSlots.push(completedJob.slot); + resultsByChangeId.set(result.changeId, result); + stateByChangeId.set(result.changeId, result.ok ? "ok" : "failed"); + runNode.value = resultsByChangeId.size; + + if (!result.ok) continue; + + for (const childChangeId of childrenByChangeId.get(result.changeId) ?? []) { + if (stateByChangeId.get(childChangeId) !== "pending") continue; + + const remainingDependencies = + (remainingDependenciesByChangeId.get(childChangeId) ?? 1) - 1; + remainingDependenciesByChangeId.set(childChangeId, remainingDependencies); + + if (remainingDependencies === 0) { + const childPlan = planByChangeId.get(childChangeId); + if (childPlan != null) ready.push(childPlan); + } + } + } + } + + const results = plans.map((plan) => { + const result = resultsByChangeId.get(plan.changeId); + if (result == null) { + throw new Error(`missing result for ${plan.changeId}`); + } + return result; + }); + const failures = results.filter( + (result) => result.phase !== "skipped" && !result.ok, + ); + const skipped = results.filter((result) => result.phase === "skipped"); + const passed = results.length - failures.length - skipped.length; + const successfulChangeIds = results + .filter((result) => result.ok) + .map((result) => result.changeId.slice(0, 8)); + + if (failures.length > 0 || skipped.length > 0) { + writeLastRun(lastRunPath, changeIds); + } + + const summary: string[] = [ + "\n", + ansi.style(headingStyle, "summary"), + "\n", + ` ${ansi.style(okStyle, "pass:")} ${passed}/${results.length}`, + "\n", + ` ${ansi.style(failStyle, "fail:")} ${failures.length}/${results.length}`, + "\n", + ` ${ansi.style(skipStyle, "skip:")} ${skipped.length}/${results.length}`, + "\n", + ]; + + if (failures.length > 0) { + summary.push("\n", ansi.style(failStyle, "failures"), "\n"); + for (const result of failures) { + summary.push( + `${ansi.style(failStyle, "x")} ${result.label} (${result.phase})`, + result.blockedBy?.length + ? ` ${ansi.style(dimStyle, `[waiting on ${result.blockedBy.map((id) => id.slice(0, 12)).join(", ")}]`)}` + : "", + result.logPath + ? ` ${ansi.style(dimStyle, `[log ${result.logPath}]`)}` + : "", + "\n", + ); + } + } + + if (skipped.length > 0) { + summary.push("\n", ansi.style(skipStyle, "skipped"), "\n"); + for (const result of skipped) { + summary.push( + `${ansi.style(skipStyle, "-")} ${result.label}`, + result.blockedBy?.length + ? ` ${ansi.style(dimStyle, `[blocked by ${result.blockedBy.map((id) => id.slice(0, 12)).join(", ")}]`)}` + : "", + "\n", + ); + } + } + + if (failures.length === 0 && skipped.length === 0) { + summary.push( + "\n", + ansi.style(infoStyle, "all selected changes passed"), + "\n", + ); + } + + if (successfulChangeIds.length > 0) { + summary.push( + "\n", + `${ansi.style(infoStyle, "push successful commits:")} jj git push -c "${successfulChangeIds.join("|")}"`, + "\n", + ); + } + + log.writeOutput(summary.join("")); + process.exitCode = failures.length > 0 || skipped.length > 0 ? 1 : 0; +} catch (error) { + runError = error; + throw error; +} finally { + try { + forgetPoolWorkspaces(pool, trackedWorkspaces); + } catch (cleanupError) { + if (runError != null) { + log.error(cleanupError); + } else { + throw cleanupError; + } + } +} + +function usage(code: number): never { + log.writeOutput(`jj ci: run continuous integration on many revisions at once + +usage: jj ci [options] <...revsets> + or: jj ci [options] --retry + +requires \`./.jj/ci.sh\` to setup and run checks. for example: + + #!/bin/sh + pnpm install + pnpm format + pnpm test + +the script is run with fast failure, so if a command fails, it fails the run. + +options: + -j, --jobs=[n] maximum number of jobs to run at once + -p, --parallel rewrite the selected revisions with \`jj parallelize\` + -r, --retry re-run the set of commits that most recently failed + --ignore-immutable allow rewriting immutable commits + --dry-run print the execution plan + --clean delete the workspace cache first + +jobs will be run in separate workspaces placed in \`.jj/ci\`, and +autofixes will be propagated across your revisions. if your changes are all +separate and aren't meant to conflict, pass \`-p\` to run \`jj parallelize\` +first. parallel commits will be run in parallel + +'jj ci' reuses worktrees within a single execution, then forgets the +temporary workspaces afterward so they don't show up in \`jj log\`. +'jj ci' speeds up multiple runs by preserving ignored files between runs (for caches). +`); + process.exit(code); +} + +function resolveGitDir(cwd: string): string | null { + const result = child_process.spawnSync( + "git", + ["-C", cwd, "rev-parse", "--absolute-git-dir"], + { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }, + ); + if (result.error || result.status !== 0) return null; + + const value = (result.stdout ?? "").trim(); + if (value === "") return null; + return value; +} + + +function jj(args: string[], options: CommandOptions = {}): string { + const result = child_process.spawnSync( + "jj", + [ + ...(options.ignoreWorkingCopy ? ["--ignore-working-copy"] : []), + ...(ignoreImmutable ? ["--ignore-immutable"] : []), + ...args, + ], + { + cwd: options.cwd, + encoding: "utf8", + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + const stdout = result.stdout ?? ""; + const stderr = result.stderr ?? ""; + const text = stdout + stderr; + + if (options.node && text !== "") options.node.log.write(text); + if (options.plainLog && text !== "") options.plainLog.push(normalizePlain(text)); + + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error( + `jj ${args.join(" ")} failed with exit code ${result.status}` + + (text ? `\n\n${normalizePlain(text)}` : ""), + ); + } + + return stdout.trimEnd(); +} + +function listIgnoredWorkspacePaths(workspacePath: string): string[] | null { + if (gitDir == null) return null; + + const result = child_process.spawnSync( + "git", + [ + `--git-dir=${gitDir}`, + `--work-tree=${workspacePath}`, + "ls-files", + "--others", + "-i", + "--exclude-standard", + "--directory", + ], + { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }, + ); + if (result.error || result.status !== 0) return null; + + return (result.stdout ?? "") + .split("\n") + .map((line) => normalizeRelativeCachePath(line)) + .filter((line): line is string => line != null); +} + +function normalizeRelativeCachePath(relativePath: string): string | null { + const trimmed = relativePath.trim().replace(/\/+$/g, ""); + if (trimmed === "") return null; + + const normalized = path.normalize(trimmed); + if ( + normalized === ".git" || + normalized === ".jj" || + normalized.startsWith(`.git${path.sep}`) || + normalized.startsWith(`.jj${path.sep}`) || + normalized === ".." || + normalized.startsWith(`..${path.sep}`) || + path.isAbsolute(normalized) + ) { + return null; + } + + return normalized; +} + +function pathExists(filePath: string): boolean { + try { + fs.lstatSync(filePath); + return true; + } catch { + return false; + } +} + +function ensureWorkspace( + slot: PoolSlot, + changeId: string, + label: string, + trackedWorkspaces: Set, + node: progress.Node, + plainLog: string[], +): void { + const hasWorkspace = trackedWorkspaces.has(slot.workspaceName); + const hasWorkspacePath = fs.existsSync(slot.workspacePath); + let createdWorkspace = false; + + if (hasWorkspace && !hasWorkspacePath) { + log.warn("forgetting stale workspace", slot.workspaceName); + jj(["workspace", "forget", slot.workspaceName], { + ignoreWorkingCopy: true, + node, + plainLog, + }); + trackedWorkspaces.delete(slot.workspaceName); + } + + if (!hasWorkspace && hasWorkspacePath) { + preserveIgnoredWorkspaceCache(slot, plainLog); + log.warn("removing orphaned workspace", slot.workspacePath); + fs.rmSync(slot.workspacePath, { force: true, recursive: true }); + } + + if (!trackedWorkspaces.has(slot.workspaceName)) { + createdWorkspace = true; + jj( + [ + "workspace", + "add", + "--name", + slot.workspaceName, + "--revision", + changeId, + slot.workspacePath, + ], + { node, plainLog }, + ); + trackedWorkspaces.add(slot.workspaceName); + } else { + jj(["workspace", "update-stale"], { + cwd: slot.workspacePath, + node, + plainLog, + }); + } + + jj(["edit", changeId], { cwd: slot.workspacePath, node, plainLog }); + if (createdWorkspace) restoreIgnoredWorkspaceCache(slot, plainLog); +} + +function snapshotWorkspace( + workspacePath: string, + node: progress.Node, + plainLog: string[], +): void { + jj(["status"], { cwd: workspacePath, node, plainLog }); +} + +function workspaceCachePath(slot: PoolSlot): string { + return path.join(cacheDir, `${slot.name}.cache`); +} + +function workspaceCacheManifestPath(slot: PoolSlot): string { + return path.join(workspaceCachePath(slot), workspaceCacheManifestName); +} + +function preserveIgnoredWorkspaceCache( + slot: PoolSlot, + plainLog: string[], +): void { + const ignoredPaths = listIgnoredWorkspacePaths(slot.workspacePath); + if (ignoredPaths == null) return; + if (ignoredPaths.length === 0) return; + + const cachePath = workspaceCachePath(slot); + if (fs.existsSync(cachePath)) { + fs.rmSync(cachePath, { force: true, recursive: true }); + } + + fs.mkdirSync(cachePath, { recursive: true }); + for (const relativePath of ignoredPaths) { + const sourcePath = path.join(slot.workspacePath, relativePath); + if (!pathExists(sourcePath)) continue; + + const destinationPath = path.join(cachePath, relativePath); + fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); + fs.renameSync(sourcePath, destinationPath); + } + + fs.writeFileSync( + workspaceCacheManifestPath(slot), + ignoredPaths.join("\n") + "\n", + ); + plainLog.push( + `preserved ignored cache paths: ${ignoredPaths.join(", ")}\n`, + ); +} + +function restoreIgnoredWorkspaceCache( + slot: PoolSlot, + plainLog: string[], +): void { + const manifestPath = workspaceCacheManifestPath(slot); + if (!fs.existsSync(manifestPath)) return; + + const cachePath = workspaceCachePath(slot); + const relativePaths = fs.readFileSync(manifestPath, "utf8") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + + if (relativePaths.length === 0) { + fs.rmSync(cachePath, { force: true, recursive: true }); + return; + } + + const restoredPaths: string[] = []; + const skippedPaths: string[] = []; + + for (const relativePath of relativePaths) { + const sourcePath = path.join(cachePath, relativePath); + if (!pathExists(sourcePath)) continue; + + const destinationPath = path.join(slot.workspacePath, relativePath); + if (pathExists(destinationPath)) { + skippedPaths.push(relativePath); + continue; + } + + try { + fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); + fs.renameSync(sourcePath, destinationPath); + restoredPaths.push(relativePath); + } catch { + skippedPaths.push(relativePath); + } + } + + if (restoredPaths.length > 0) { + plainLog.push( + `restored ignored cache paths: ${restoredPaths.join(", ")}\n`, + ); + } + + if (skippedPaths.length > 0) { + plainLog.push( + `skipped ignored cache paths: ${skippedPaths.join(", ")}\n`, + ); + fs.writeFileSync(manifestPath, skippedPaths.join("\n") + "\n"); + } else { + fs.rmSync(cachePath, { force: true, recursive: true }); + } +} + +function forgetPoolWorkspaces( + pool: PoolSlot[], + trackedWorkspaces: Set, +): void { + const errors: Error[] = []; + + for (const slot of pool) { + if (!trackedWorkspaces.has(slot.workspaceName)) continue; + + try { + if (fs.existsSync(slot.workspacePath)) { + preserveIgnoredWorkspaceCache(slot, []); + } + jj(["workspace", "forget", slot.workspaceName], { + cwd: root, + ignoreWorkingCopy: true, + }); + trackedWorkspaces.delete(slot.workspaceName); + } catch (error) { + errors.push( + error instanceof Error ? error : new Error(String(error)), + ); + } + } + + if (errors.length > 0) { + throw new AggregateError(errors, "failed to forget one or more jj-ci workspaces"); + } +} + +function resolveChangeIds(revision: string): string[] { + const output = jj( + [ + "log", + "--no-graph", + "-r", + revision, + "-T", + 'self.change_id() ++ "\n"', + ], + { ignoreWorkingCopy: true }, + ); + const changeIds = output.split("\n").filter(Boolean); + + if (changeIds.length === 0) { + log.error("revision resolved to no change IDs", JSON.stringify(revision)); + process.exit(1); + } + + return changeIds; +} + +function readLastRun(lastRunPath: string): string[] { + if (!fs.existsSync(lastRunPath)) { + log.error("no failed run to retry:", lastRunPath); + process.exit(1); + } + + const changeIds = fs.readFileSync(lastRunPath, "utf8") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + + if (changeIds.length === 0) { + log.error("retry file is empty:", lastRunPath); + process.exit(1); + } + + return changeIds; +} + +function writeLastRun(lastRunPath: string, changeIds: string[]): void { + fs.mkdirSync(path.dirname(lastRunPath), { recursive: true }); + fs.writeFileSync(lastRunPath, changeIds.join("\n") + "\n"); +} + +function runScript( + scriptPath: string, + options: CommandOptions, +): Promise { + return new Promise((resolve) => { + const startedAt = Date.now(); + let plainText = ""; + const baseEnv = Object.fromEntries( + Object.entries({ + CLICOLOR_FORCE: "1", + FORCE_COLOR: "1", + TERM: process.env.TERM ?? "xterm-256color", + ...options.env, + }).filter((entry): entry is [string, string] => entry[1] != null), + ); + delete baseEnv.NO_COLOR; + const preferredShell = process.env.SHELL; + const shell = fs.existsSync("/bin/bash") + ? "/bin/bash" + : preferredShell != null && path.basename(preferredShell) === "bash" + ? preferredShell + : "bash"; + + const term = pty.spawn(shell, ["-c", 'set -o pipefail; set -eux; source "$1"', "jj-ci", scriptPath], { + cols: process.stdout.columns ?? 120, + cwd: options.cwd, + encoding: "utf8", + env: baseEnv, + name: process.env.TERM ?? "xterm-256color", + rows: process.stdout.rows ?? 30, + }); + + term.onData((chunk) => { + plainText += normalizePlain(chunk); + options.node?.log.write(chunk); + }); + + term.onExit(({ exitCode, signal }) => { + if (signal != null) { + const note = ansi.style(failStyle, `terminated by signal ${signal}`); + plainText += normalizePlain(note); + options.node?.log.info(note); + } + resolve({ + durationMs: Date.now() - startedAt, + exitCode, + ok: exitCode === 0, + plainText, + }); + }); + }); +} + +function writeFailureLog(logPath: string, plainLog: string[]): void { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.writeFileSync(logPath, plainLog.join("")); +} + +function normalizePlain(text: string): string { + return ansi.strip(text) + .replace(/\r\n/g, "\n") + .replace(/\r/g, "\n"); +} + +function poolName(index: number): string { + let value = index; + let name = ""; + + while (value >= 0) { + name = String.fromCharCode(97 + (value % 26)) + name; + value = Math.floor(value / 26) - 1; + } + + return name; +} + +function maxPlanConcurrency(plans: Plan[]): number { + if (plans.length === 0) return 0; + + const descendantsByChangeId = new Map( + plans.map((plan) => [plan.changeId, [] as string[]]), + ); + + for (const plan of plans) { + for (const dependencyChangeId of plan.dependencyChangeIds) { + descendantsByChangeId.get(dependencyChangeId)?.push(plan.changeId); + } + } + + const matchedDependencyByDescendant = new Map(); + let matchCount = 0; + + for (const plan of plans) { + if ( + tryMatchDescendant( + plan.changeId, + descendantsByChangeId, + matchedDependencyByDescendant, + new Set(), + ) + ) { + matchCount += 1; + } + } + + return plans.length - matchCount; +} + +function tryMatchDescendant( + dependencyChangeId: string, + descendantsByChangeId: Map, + matchedDependencyByDescendant: Map, + seenDescendants: Set, +): boolean { + for (const descendantChangeId of descendantsByChangeId.get(dependencyChangeId) ?? []) { + if (seenDescendants.has(descendantChangeId)) continue; + seenDescendants.add(descendantChangeId); + + const matchedDependency = matchedDependencyByDescendant.get(descendantChangeId); + if ( + matchedDependency == null || + tryMatchDescendant( + matchedDependency, + descendantsByChangeId, + matchedDependencyByDescendant, + seenDescendants, + ) + ) { + matchedDependencyByDescendant.set(descendantChangeId, dependencyChangeId); + return true; + } + } + + return false; +} + +function progressAttachToScreen( + root: progress.Root, + { writeOutput, startWidget }: Pick< + log.WidgetHost, + "writeOutput" | "startWidget" + >, +): ts.Dispose { + const stack = new DisposableStack(); + let widget: log.WidgetInstance | null = null; + + stack.use(root.on("change", (items) => { + if (items.length > 0) { + widget ??= startWidget({ + format: ({ now }) => progress.formatAnsi(now, root.active), + }) ?? null; + if (!widget)return; + widget.fps = items.some((x) => x.showTotal !== false && x.total > 0) + ? 15 + : null; + widget.redraw(); + } else { + widget?.stop(); + widget = null; + } + })); + stack.use(root.on("node-detached-log", (msg) => { + writeOutput(log.formatAnsiMessage(msg, true)); + })); + stack.use(root.on("node-end", (node) => { + let title = node.text; + let p: progress.ReadOnlyNode | null = node; + while (p = p.parent) title = p.text + " / " + title; + })); + + return ts.defer(() => stack[Symbol.dispose]); +} diff --git a/users/clover/tsconfig.json b/users/clover/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..d8d6194032ada00c79430cf01726df5d860367a6 --- /dev/null +++ b/users/clover/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "baseUrl": ".", + "lib": ["esnext", "es2024", "dom", "dom.iterable"], + "module": "nodenext", + "noEmit": true, + "rootDir": ".", + "skipLibCheck": true, + "strict": true, + "verbatimModuleSyntax": true, + "target": "esnext", + "noUncheckedIndexedAccess": true, + "resolveJsonModule": true, + "types": ["node"] + }, + "include": ["./**/*"] +}