| ... | ... | @@ -0,0 +1,1115 @@ |
| 1 | #!/usr/bin/env node |
| 2 | import * as child_process from "node:child_process"; |
| 3 | import * as fs from "node:fs"; |
| 4 | import * as os from "node:os"; |
| 5 | import * as path from "node:path"; |
| 6 | import * as log from "@clo/lib/log"; |
| 7 | import * as progress from "@clo/lib/progress"; |
| 8 | import * as ansi from "@clo/lib/string/ansi"; |
| 9 | import * as pty from "@lydell/node-pty"; |
| 10 | import * as ts from "@clo/lib/ts"; |
| 11 | import process from "node:process"; |
| 12 | |
| 13 | const progressRoot = new progress.Root(); |
| 14 | progressAttachToScreen(progressRoot, log); |
| 15 | |
| 16 | type CommandOptions = { |
| 17 | cwd?: string; |
| 18 | env?: NodeJS.ProcessEnv; |
| 19 | ignoreWorkingCopy?: boolean; |
| 20 | node?: progress.Node; |
| 21 | plainLog?: string[]; |
| 22 | }; |
| 23 | |
| 24 | type Plan = { |
| 25 | changeId: string; |
| 26 | dependencyChangeIds: string[]; |
| 27 | label: string; |
| 28 | logPath: string; |
| 29 | }; |
| 30 | |
| 31 | type ScriptResult = { |
| 32 | durationMs: number; |
| 33 | exitCode: number | null; |
| 34 | ok: boolean; |
| 35 | plainText: string; |
| 36 | }; |
| 37 | |
| 38 | type JobPhase = "prep" | "ci" | "skipped" | "snapshot" | "stalled"; |
| 39 | type JobResult = { |
| 40 | blockedBy?: string[]; |
| 41 | changeId: string; |
| 42 | durationMs: number; |
| 43 | exitCode: number | null; |
| 44 | label: string; |
| 45 | logPath?: string; |
| 46 | ok: boolean; |
| 47 | phase: JobPhase; |
| 48 | workspacePath?: string; |
| 49 | }; |
| 50 | |
| 51 | type JobState = "failed" | "ok" | "pending" | "running" | "skipped"; |
| 52 | |
| 53 | type PoolSlot = { |
| 54 | name: string; |
| 55 | workspaceName: string; |
| 56 | workspacePath: string; |
| 57 | }; |
| 58 | |
| 59 | type ActiveJob = { |
| 60 | slot: PoolSlot; |
| 61 | task: Promise<JobResult>; |
| 62 | }; |
| 63 | |
| 64 | const dimStyle = ansi.dim + ansi.fgBrightBlack; |
| 65 | const headingStyle = ansi.bold + ansi.fgBrightWhite; |
| 66 | const okStyle = ansi.bold + ansi.fgBrightGreen; |
| 67 | const failStyle = ansi.bold + ansi.fgBrightRed; |
| 68 | const skipStyle = ansi.bold + ansi.fgBrightYellow; |
| 69 | const infoStyle = ansi.bold + ansi.fgBrightCyan; |
| 70 | const workspaceCacheManifestName = ".paths"; |
| 71 | |
| 72 | let clean = false; |
| 73 | let dryRun = false; |
| 74 | let ignoreImmutable = false; |
| 75 | let jobs = os.availableParallelism(); |
| 76 | let parallelize = false; |
| 77 | let retry = false; |
| 78 | let sawJobs = false; |
| 79 | let warnedAboutPtyFallback = false; |
| 80 | const revisions: string[] = []; |
| 81 | |
| 82 | const argv = process.argv.slice(2); |
| 83 | while (argv.length > 0) { |
| 84 | const arg = argv.shift(); |
| 85 | if (arg == null) break; |
| 86 | |
| 87 | if (arg === "-h" || arg === "--help") usage(0); |
| 88 | else if (arg === "--dry-run") dryRun = true; |
| 89 | else if (arg === "--clean") clean = true; |
| 90 | else if (arg === "--ignore-immutable") ignoreImmutable = true; |
| 91 | else if (arg === "-p" || arg === "--parallel") parallelize = true; |
| 92 | else if (arg === "-r" || arg === "--retry") retry = true; |
| 93 | else if (arg === "-j" || arg === "--jobs" || arg === "--cores") { |
| 94 | if (sawJobs) { |
| 95 | log.error("can only pass --jobs once"); |
| 96 | usage(1); |
| 97 | } |
| 98 | |
| 99 | const value = argv.shift(); |
| 100 | if (value == null) { |
| 101 | log.error("missing value for --jobs"); |
| 102 | usage(1); |
| 103 | } |
| 104 | |
| 105 | jobs = Number(value); |
| 106 | sawJobs = true; |
| 107 | } else if (arg.startsWith("--jobs=") || arg.startsWith("--cores=")) { |
| 108 | if (sawJobs) { |
| 109 | log.error("can only pass --jobs once"); |
| 110 | usage(1); |
| 111 | } |
| 112 | |
| 113 | jobs = Number(arg.slice(arg.indexOf("=") + 1)); |
| 114 | sawJobs = true; |
| 115 | } else if (arg === "--") { |
| 116 | revisions.push(...argv); |
| 117 | break; |
| 118 | } else if (arg.startsWith("-")) { |
| 119 | log.error("unknown option", JSON.stringify(arg)); |
| 120 | usage(1); |
| 121 | } else { |
| 122 | revisions.push(arg); |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | if (!Number.isInteger(jobs) || jobs < 1) { |
| 127 | log.error("--jobs must be a positive integer"); |
| 128 | usage(1); |
| 129 | } |
| 130 | |
| 131 | if (retry && revisions.length > 0) { |
| 132 | log.error("cannot pass revisions together with --retry"); |
| 133 | usage(1); |
| 134 | } |
| 135 | |
| 136 | if (!retry && revisions.length === 0) usage(1); |
| 137 | |
| 138 | const root = jj(["workspace", "root"]).trim(); |
| 139 | const cacheDir = path.join(root, ".jj", "ci"); |
| 140 | const lastRunPath = path.join(cacheDir, "last-run.txt"); |
| 141 | const logDir = path.join(cacheDir, "_logs"); |
| 142 | const ciScript = path.join(root, ".jj", "ci.sh"); |
| 143 | const gitDir = resolveGitDir(root); |
| 144 | const selectedRevisions = retry ? readLastRun(lastRunPath) : revisions; |
| 145 | |
| 146 | let invalid = false; |
| 147 | |
| 148 | if (!fs.existsSync(ciScript)) { |
| 149 | log.error("missing .jj/ci.sh"); |
| 150 | invalid = true; |
| 151 | } |
| 152 | |
| 153 | if (invalid) process.exit(1); |
| 154 | |
| 155 | if (parallelize) { |
| 156 | if (dryRun) log.info("would run", "jj parallelize " + selectedRevisions.join(" ")); |
| 157 | else jj(["parallelize", ...selectedRevisions]); |
| 158 | } |
| 159 | |
| 160 | const changeIds = [...new Set(selectedRevisions.flatMap(resolveChangeIds))]; |
| 161 | const selectedExpr = `(${changeIds.join("|")})`; |
| 162 | const trackedWorkspaces = new Set( |
| 163 | jj(["workspace", "list"], { ignoreWorkingCopy: true }) |
| 164 | .split("\n") |
| 165 | .filter(Boolean) |
| 166 | .map((line) => line.split(":", 1)[0]?.trim()) |
| 167 | .filter((name): name is string => Boolean(name)), |
| 168 | ); |
| 169 | const plans: Plan[] = changeIds.map((changeId) => ({ |
| 170 | changeId, |
| 171 | dependencyChangeIds: [ |
| 172 | ...new Set( |
| 173 | jj( |
| 174 | [ |
| 175 | "log", |
| 176 | "--no-graph", |
| 177 | "-r", |
| 178 | `ancestors(${changeId}) & ${selectedExpr}`, |
| 179 | "-T", |
| 180 | 'self.change_id() ++ "\n"', |
| 181 | ], |
| 182 | { ignoreWorkingCopy: true }, |
| 183 | ) |
| 184 | .split("\n") |
| 185 | .filter((candidate) => candidate !== "" && candidate !== changeId), |
| 186 | ), |
| 187 | ], |
| 188 | label: changeId.slice(0, 6) + ': ' + jj( |
| 189 | [ |
| 190 | "log", |
| 191 | "--no-graph", |
| 192 | "-r", |
| 193 | `${changeId}`, |
| 194 | "-T", |
| 195 | 'self.description().first_line()', |
| 196 | ], |
| 197 | { ignoreWorkingCopy: true }, |
| 198 | ), |
| 199 | logPath: path.join(logDir, `ci-${changeId}.log`), |
| 200 | })); |
| 201 | const concurrency = Math.min(jobs, maxPlanConcurrency(plans)); |
| 202 | const pool = Array.from({ length: concurrency }, (_, index): PoolSlot => { |
| 203 | const name = poolName(index); |
| 204 | return { |
| 205 | name, |
| 206 | workspaceName: `ci-${name}`, |
| 207 | workspacePath: path.join(cacheDir, name), |
| 208 | }; |
| 209 | }); |
| 210 | const planByChangeId = new Map(plans.map((plan) => [plan.changeId, plan])); |
| 211 | const childrenByChangeId = new Map( |
| 212 | changeIds.map((changeId) => [changeId, [] as string[]]), |
| 213 | ); |
| 214 | |
| 215 | for (const plan of plans) { |
| 216 | for (const dependencyChangeId of plan.dependencyChangeIds) { |
| 217 | childrenByChangeId.get(dependencyChangeId)?.push(plan.changeId); |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | if (dryRun) { |
| 222 | log.writeOutput( |
| 223 | JSON.stringify( |
| 224 | { |
| 225 | cacheDir, |
| 226 | ciScript, |
| 227 | clean, |
| 228 | concurrency, |
| 229 | ignoreImmutable, |
| 230 | parallelize, |
| 231 | plans, |
| 232 | pool, |
| 233 | retry, |
| 234 | selectedRevisions, |
| 235 | root, |
| 236 | }, |
| 237 | null, |
| 238 | 2, |
| 239 | ) + "\n", |
| 240 | ); |
| 241 | process.exit(0); |
| 242 | } |
| 243 | |
| 244 | if (clean && fs.existsSync(cacheDir)) { |
| 245 | fs.rmSync(cacheDir, { force: true, recursive: true }); |
| 246 | } |
| 247 | |
| 248 | fs.mkdirSync(cacheDir, { recursive: true }); |
| 249 | fs.mkdirSync(logDir, { recursive: true }); |
| 250 | |
| 251 | let runError: unknown = null; |
| 252 | |
| 253 | try { |
| 254 | const ready = plans.filter((plan) => plan.dependencyChangeIds.length === 0); |
| 255 | const availableSlots = [...pool]; |
| 256 | const remainingDependenciesByChangeId = new Map( |
| 257 | plans.map((plan) => [plan.changeId, plan.dependencyChangeIds.length]), |
| 258 | ); |
| 259 | const resultsByChangeId = new Map<string, JobResult>(); |
| 260 | const stateByChangeId = new Map<JobResult["changeId"], JobState>( |
| 261 | changeIds.map((changeId) => [changeId, "pending"]), |
| 262 | ); |
| 263 | const active = new Map<string, ActiveJob>(); |
| 264 | |
| 265 | { |
| 266 | using runNode = progressRoot.start("continuous integration", { |
| 267 | total: plans.length, |
| 268 | valueFormatter() { |
| 269 | let passed = 0; |
| 270 | let failed = 0; |
| 271 | for (const state of stateByChangeId.values()) { |
| 272 | if (state === "ok") passed += 1; |
| 273 | else if (state === "failed" || state === "skipped") failed += 1; |
| 274 | } |
| 275 | return `${ansi.fgGreen}${passed} pass${ansi.reset}, ${failed > 0 ? ansi.fgRed : ""}${failed} fail${ansi.reset}`; |
| 276 | }, |
| 277 | }); |
| 278 | |
| 279 | while (resultsByChangeId.size < plans.length) { |
| 280 | while ( |
| 281 | ready.length > 0 && |
| 282 | active.size < concurrency && |
| 283 | availableSlots.length > 0 |
| 284 | ) { |
| 285 | const plan = ready.shift(); |
| 286 | const slot = availableSlots.shift(); |
| 287 | if (plan == null || slot == null) break; |
| 288 | |
| 289 | stateByChangeId.set(plan.changeId, "running"); |
| 290 | |
| 291 | const task = (async (): Promise<JobResult> => { |
| 292 | const startedAt = Date.now(); |
| 293 | using node = runNode.start(plan.label); |
| 294 | if (fs.existsSync(plan.logPath)) fs.rmSync(plan.logPath); |
| 295 | const env = { |
| 296 | ...process.env, |
| 297 | JJ_CI_CHANGE_ID: plan.changeId, |
| 298 | JJ_CI_ROOT: root, |
| 299 | JJ_CI_WORKSPACE: slot.workspacePath, |
| 300 | }; |
| 301 | const plainLog = [ |
| 302 | `change: ${plan.changeId}\n`, |
| 303 | `dependencies: ${ |
| 304 | plan.dependencyChangeIds.length === 0 |
| 305 | ? "(selected root)" |
| 306 | : plan.dependencyChangeIds.join(", ") |
| 307 | }\n`, |
| 308 | `slot: ${slot.name}\n`, |
| 309 | `workspace: ${slot.workspacePath}\n`, |
| 310 | `started: ${new Date(startedAt).toISOString()}\n\n`, |
| 311 | ]; |
| 312 | let phase: JobPhase = "prep"; |
| 313 | |
| 314 | try { |
| 315 | plainLog.push("--- prep ---\n"); |
| 316 | ensureWorkspace(slot, plan.changeId, plan.label, trackedWorkspaces, node, plainLog); |
| 317 | plainLog.push("\n"); |
| 318 | |
| 319 | phase = "ci"; |
| 320 | node.text = `${plan.label} ci [slot ${slot.name}]`; |
| 321 | const ci = await runScript(ciScript, { |
| 322 | cwd: slot.workspacePath, |
| 323 | env, |
| 324 | node, |
| 325 | }); |
| 326 | plainLog.push( |
| 327 | `--- ci ---\nexit: ${ci.exitCode ?? "signal"}\nduration_ms: ${ci.durationMs}\n\n`, |
| 328 | ); |
| 329 | plainLog.push(ci.plainText); |
| 330 | if (ci.plainText !== "" && !ci.plainText.endsWith("\n")) { |
| 331 | plainLog.push("\n"); |
| 332 | } |
| 333 | plainLog.push("\n"); |
| 334 | |
| 335 | phase = "snapshot"; |
| 336 | node.text = `${plan.label} snapshot [slot ${slot.name}]`; |
| 337 | plainLog.push("--- snapshot ---\n"); |
| 338 | snapshotWorkspace(slot.workspacePath, node, plainLog); |
| 339 | plainLog.push("\n"); |
| 340 | |
| 341 | if (!ci.ok) { |
| 342 | writeFailureLog(plan.logPath, plainLog); |
| 343 | return { |
| 344 | changeId: plan.changeId, |
| 345 | durationMs: Date.now() - startedAt, |
| 346 | exitCode: ci.exitCode, |
| 347 | label: plan.label, |
| 348 | logPath: plan.logPath, |
| 349 | ok: false, |
| 350 | phase: "ci", |
| 351 | workspacePath: slot.workspacePath, |
| 352 | }; |
| 353 | } |
| 354 | |
| 355 | node.value += 1; |
| 356 | node.text = `${plan.label} ok [slot ${slot.name}]`; |
| 357 | if (fs.existsSync(plan.logPath)) fs.rmSync(plan.logPath); |
| 358 | return { |
| 359 | changeId: plan.changeId, |
| 360 | durationMs: Date.now() - startedAt, |
| 361 | exitCode: ci.exitCode, |
| 362 | label: plan.label, |
| 363 | ok: true, |
| 364 | phase: "ci", |
| 365 | workspacePath: slot.workspacePath, |
| 366 | }; |
| 367 | } catch (error) { |
| 368 | const message = error instanceof Error |
| 369 | ? (error.stack ?? error.message) |
| 370 | : String(error); |
| 371 | node.log.error(error); |
| 372 | plainLog.push(`--- ${phase} error ---\n${normalizePlain(message)}\n`); |
| 373 | writeFailureLog(plan.logPath, plainLog); |
| 374 | return { |
| 375 | changeId: plan.changeId, |
| 376 | durationMs: Date.now() - startedAt, |
| 377 | exitCode: null, |
| 378 | label: plan.label, |
| 379 | logPath: plan.logPath, |
| 380 | ok: false, |
| 381 | phase, |
| 382 | workspacePath: slot.workspacePath, |
| 383 | }; |
| 384 | } |
| 385 | })(); |
| 386 | |
| 387 | active.set(plan.changeId, { slot, task }); |
| 388 | } |
| 389 | |
| 390 | if (active.size === 0) { |
| 391 | let skippedAny = false; |
| 392 | |
| 393 | for (const plan of plans) { |
| 394 | if (stateByChangeId.get(plan.changeId) !== "pending") continue; |
| 395 | |
| 396 | const blockedBy = plan.dependencyChangeIds.filter((dependencyChangeId) => { |
| 397 | const dependencyState = stateByChangeId.get(dependencyChangeId); |
| 398 | return dependencyState === "failed" || dependencyState === "skipped"; |
| 399 | }); |
| 400 | |
| 401 | if (blockedBy.length === 0) continue; |
| 402 | |
| 403 | skippedAny = true; |
| 404 | stateByChangeId.set(plan.changeId, "skipped"); |
| 405 | if (fs.existsSync(plan.logPath)) fs.rmSync(plan.logPath); |
| 406 | resultsByChangeId.set(plan.changeId, { |
| 407 | blockedBy, |
| 408 | changeId: plan.changeId, |
| 409 | durationMs: 0, |
| 410 | exitCode: null, |
| 411 | label: plan.label, |
| 412 | ok: false, |
| 413 | phase: "skipped", |
| 414 | }); |
| 415 | } |
| 416 | |
| 417 | if (skippedAny) { |
| 418 | runNode.value = resultsByChangeId.size; |
| 419 | continue; |
| 420 | } |
| 421 | |
| 422 | log.error("dependency scheduling stalled"); |
| 423 | for (const plan of plans) { |
| 424 | if (stateByChangeId.get(plan.changeId) !== "pending") continue; |
| 425 | |
| 426 | resultsByChangeId.set(plan.changeId, { |
| 427 | blockedBy: plan.dependencyChangeIds.filter((dependencyChangeId) => |
| 428 | stateByChangeId.get(dependencyChangeId) !== "ok" |
| 429 | ), |
| 430 | changeId: plan.changeId, |
| 431 | durationMs: 0, |
| 432 | exitCode: null, |
| 433 | label: plan.label, |
| 434 | ok: false, |
| 435 | phase: "stalled", |
| 436 | }); |
| 437 | stateByChangeId.set(plan.changeId, "failed"); |
| 438 | } |
| 439 | runNode.value = resultsByChangeId.size; |
| 440 | process.exitCode = 1; |
| 441 | continue; |
| 442 | } |
| 443 | |
| 444 | const result = await Promise.race( |
| 445 | [...active.values()].map(({ task }) => task), |
| 446 | ); |
| 447 | const completedJob = active.get(result.changeId); |
| 448 | if (completedJob == null) { |
| 449 | throw new Error(`missing active job for ${result.changeId}`); |
| 450 | } |
| 451 | active.delete(result.changeId); |
| 452 | availableSlots.push(completedJob.slot); |
| 453 | resultsByChangeId.set(result.changeId, result); |
| 454 | stateByChangeId.set(result.changeId, result.ok ? "ok" : "failed"); |
| 455 | runNode.value = resultsByChangeId.size; |
| 456 | |
| 457 | if (!result.ok) continue; |
| 458 | |
| 459 | for (const childChangeId of childrenByChangeId.get(result.changeId) ?? []) { |
| 460 | if (stateByChangeId.get(childChangeId) !== "pending") continue; |
| 461 | |
| 462 | const remainingDependencies = |
| 463 | (remainingDependenciesByChangeId.get(childChangeId) ?? 1) - 1; |
| 464 | remainingDependenciesByChangeId.set(childChangeId, remainingDependencies); |
| 465 | |
| 466 | if (remainingDependencies === 0) { |
| 467 | const childPlan = planByChangeId.get(childChangeId); |
| 468 | if (childPlan != null) ready.push(childPlan); |
| 469 | } |
| 470 | } |
| 471 | } |
| 472 | } |
| 473 | |
| 474 | const results = plans.map((plan) => { |
| 475 | const result = resultsByChangeId.get(plan.changeId); |
| 476 | if (result == null) { |
| 477 | throw new Error(`missing result for ${plan.changeId}`); |
| 478 | } |
| 479 | return result; |
| 480 | }); |
| 481 | const failures = results.filter( |
| 482 | (result) => result.phase !== "skipped" && !result.ok, |
| 483 | ); |
| 484 | const skipped = results.filter((result) => result.phase === "skipped"); |
| 485 | const passed = results.length - failures.length - skipped.length; |
| 486 | const successfulChangeIds = results |
| 487 | .filter((result) => result.ok) |
| 488 | .map((result) => result.changeId.slice(0, 8)); |
| 489 | |
| 490 | if (failures.length > 0 || skipped.length > 0) { |
| 491 | writeLastRun(lastRunPath, changeIds); |
| 492 | } |
| 493 | |
| 494 | const summary: string[] = [ |
| 495 | "\n", |
| 496 | ansi.style(headingStyle, "summary"), |
| 497 | "\n", |
| 498 | ` ${ansi.style(okStyle, "pass:")} ${passed}/${results.length}`, |
| 499 | "\n", |
| 500 | ` ${ansi.style(failStyle, "fail:")} ${failures.length}/${results.length}`, |
| 501 | "\n", |
| 502 | ` ${ansi.style(skipStyle, "skip:")} ${skipped.length}/${results.length}`, |
| 503 | "\n", |
| 504 | ]; |
| 505 | |
| 506 | if (failures.length > 0) { |
| 507 | summary.push("\n", ansi.style(failStyle, "failures"), "\n"); |
| 508 | for (const result of failures) { |
| 509 | summary.push( |
| 510 | `${ansi.style(failStyle, "x")} ${result.label} (${result.phase})`, |
| 511 | result.blockedBy?.length |
| 512 | ? ` ${ansi.style(dimStyle, `[waiting on ${result.blockedBy.map((id) => id.slice(0, 12)).join(", ")}]`)}` |
| 513 | : "", |
| 514 | result.logPath |
| 515 | ? ` ${ansi.style(dimStyle, `[log ${result.logPath}]`)}` |
| 516 | : "", |
| 517 | "\n", |
| 518 | ); |
| 519 | } |
| 520 | } |
| 521 | |
| 522 | if (skipped.length > 0) { |
| 523 | summary.push("\n", ansi.style(skipStyle, "skipped"), "\n"); |
| 524 | for (const result of skipped) { |
| 525 | summary.push( |
| 526 | `${ansi.style(skipStyle, "-")} ${result.label}`, |
| 527 | result.blockedBy?.length |
| 528 | ? ` ${ansi.style(dimStyle, `[blocked by ${result.blockedBy.map((id) => id.slice(0, 12)).join(", ")}]`)}` |
| 529 | : "", |
| 530 | "\n", |
| 531 | ); |
| 532 | } |
| 533 | } |
| 534 | |
| 535 | if (failures.length === 0 && skipped.length === 0) { |
| 536 | summary.push( |
| 537 | "\n", |
| 538 | ansi.style(infoStyle, "all selected changes passed"), |
| 539 | "\n", |
| 540 | ); |
| 541 | } |
| 542 | |
| 543 | if (successfulChangeIds.length > 0) { |
| 544 | summary.push( |
| 545 | "\n", |
| 546 | `${ansi.style(infoStyle, "push successful commits:")} jj git push -c "${successfulChangeIds.join("|")}"`, |
| 547 | "\n", |
| 548 | ); |
| 549 | } |
| 550 | |
| 551 | log.writeOutput(summary.join("")); |
| 552 | process.exitCode = failures.length > 0 || skipped.length > 0 ? 1 : 0; |
| 553 | } catch (error) { |
| 554 | runError = error; |
| 555 | throw error; |
| 556 | } finally { |
| 557 | try { |
| 558 | forgetPoolWorkspaces(pool, trackedWorkspaces); |
| 559 | } catch (cleanupError) { |
| 560 | if (runError != null) { |
| 561 | log.error(cleanupError); |
| 562 | } else { |
| 563 | throw cleanupError; |
| 564 | } |
| 565 | } |
| 566 | } |
| 567 | |
| 568 | function usage(code: number): never { |
| 569 | log.writeOutput(`jj ci: run continuous integration on many revisions at once |
| 570 | |
| 571 | usage: jj ci [options] <...revsets> |
| 572 | or: jj ci [options] --retry |
| 573 | |
| 574 | requires \`./.jj/ci.sh\` to setup and run checks. for example: |
| 575 | |
| 576 | #!/bin/sh |
| 577 | pnpm install |
| 578 | pnpm format |
| 579 | pnpm test |
| 580 | |
| 581 | the script is run with fast failure, so if a command fails, it fails the run. |
| 582 | |
| 583 | options: |
| 584 | -j, --jobs=[n] maximum number of jobs to run at once |
| 585 | -p, --parallel rewrite the selected revisions with \`jj parallelize\` |
| 586 | -r, --retry re-run the set of commits that most recently failed |
| 587 | --ignore-immutable allow rewriting immutable commits |
| 588 | --dry-run print the execution plan |
| 589 | --clean delete the workspace cache first |
| 590 | |
| 591 | jobs will be run in separate workspaces placed in \`.jj/ci\`, and |
| 592 | autofixes will be propagated across your revisions. if your changes are all |
| 593 | separate and aren't meant to conflict, pass \`-p\` to run \`jj parallelize\` |
| 594 | first. parallel commits will be run in parallel |
| 595 | |
| 596 | 'jj ci' reuses worktrees within a single execution, then forgets the |
| 597 | temporary workspaces afterward so they don't show up in \`jj log\`. |
| 598 | 'jj ci' speeds up multiple runs by preserving ignored files between runs (for caches). |
| 599 | `); |
| 600 | process.exit(code); |
| 601 | } |
| 602 | |
| 603 | function resolveGitDir(cwd: string): string | null { |
| 604 | const result = child_process.spawnSync( |
| 605 | "git", |
| 606 | ["-C", cwd, "rev-parse", "--absolute-git-dir"], |
| 607 | { |
| 608 | encoding: "utf8", |
| 609 | stdio: ["ignore", "pipe", "pipe"], |
| 610 | }, |
| 611 | ); |
| 612 | if (result.error || result.status !== 0) return null; |
| 613 | |
| 614 | const value = (result.stdout ?? "").trim(); |
| 615 | if (value === "") return null; |
| 616 | return value; |
| 617 | } |
| 618 | |
| 619 | |
| 620 | function jj(args: string[], options: CommandOptions = {}): string { |
| 621 | const result = child_process.spawnSync( |
| 622 | "jj", |
| 623 | [ |
| 624 | ...(options.ignoreWorkingCopy ? ["--ignore-working-copy"] : []), |
| 625 | ...(ignoreImmutable ? ["--ignore-immutable"] : []), |
| 626 | ...args, |
| 627 | ], |
| 628 | { |
| 629 | cwd: options.cwd, |
| 630 | encoding: "utf8", |
| 631 | env: options.env, |
| 632 | stdio: ["ignore", "pipe", "pipe"], |
| 633 | }, |
| 634 | ); |
| 635 | const stdout = result.stdout ?? ""; |
| 636 | const stderr = result.stderr ?? ""; |
| 637 | const text = stdout + stderr; |
| 638 | |
| 639 | if (options.node && text !== "") options.node.log.write(text); |
| 640 | if (options.plainLog && text !== "") options.plainLog.push(normalizePlain(text)); |
| 641 | |
| 642 | if (result.error) throw result.error; |
| 643 | if (result.status !== 0) { |
| 644 | throw new Error( |
| 645 | `jj ${args.join(" ")} failed with exit code ${result.status}` + |
| 646 | (text ? `\n\n${normalizePlain(text)}` : ""), |
| 647 | ); |
| 648 | } |
| 649 | |
| 650 | return stdout.trimEnd(); |
| 651 | } |
| 652 | |
| 653 | function listIgnoredWorkspacePaths(workspacePath: string): string[] | null { |
| 654 | if (gitDir == null) return null; |
| 655 | |
| 656 | const result = child_process.spawnSync( |
| 657 | "git", |
| 658 | [ |
| 659 | `--git-dir=${gitDir}`, |
| 660 | `--work-tree=${workspacePath}`, |
| 661 | "ls-files", |
| 662 | "--others", |
| 663 | "-i", |
| 664 | "--exclude-standard", |
| 665 | "--directory", |
| 666 | ], |
| 667 | { |
| 668 | encoding: "utf8", |
| 669 | stdio: ["ignore", "pipe", "pipe"], |
| 670 | }, |
| 671 | ); |
| 672 | if (result.error || result.status !== 0) return null; |
| 673 | |
| 674 | return (result.stdout ?? "") |
| 675 | .split("\n") |
| 676 | .map((line) => normalizeRelativeCachePath(line)) |
| 677 | .filter((line): line is string => line != null); |
| 678 | } |
| 679 | |
| 680 | function normalizeRelativeCachePath(relativePath: string): string | null { |
| 681 | const trimmed = relativePath.trim().replace(/\/+$/g, ""); |
| 682 | if (trimmed === "") return null; |
| 683 | |
| 684 | const normalized = path.normalize(trimmed); |
| 685 | if ( |
| 686 | normalized === ".git" || |
| 687 | normalized === ".jj" || |
| 688 | normalized.startsWith(`.git${path.sep}`) || |
| 689 | normalized.startsWith(`.jj${path.sep}`) || |
| 690 | normalized === ".." || |
| 691 | normalized.startsWith(`..${path.sep}`) || |
| 692 | path.isAbsolute(normalized) |
| 693 | ) { |
| 694 | return null; |
| 695 | } |
| 696 | |
| 697 | return normalized; |
| 698 | } |
| 699 | |
| 700 | function pathExists(filePath: string): boolean { |
| 701 | try { |
| 702 | fs.lstatSync(filePath); |
| 703 | return true; |
| 704 | } catch { |
| 705 | return false; |
| 706 | } |
| 707 | } |
| 708 | |
| 709 | function ensureWorkspace( |
| 710 | slot: PoolSlot, |
| 711 | changeId: string, |
| 712 | label: string, |
| 713 | trackedWorkspaces: Set<string>, |
| 714 | node: progress.Node, |
| 715 | plainLog: string[], |
| 716 | ): void { |
| 717 | const hasWorkspace = trackedWorkspaces.has(slot.workspaceName); |
| 718 | const hasWorkspacePath = fs.existsSync(slot.workspacePath); |
| 719 | let createdWorkspace = false; |
| 720 | |
| 721 | if (hasWorkspace && !hasWorkspacePath) { |
| 722 | log.warn("forgetting stale workspace", slot.workspaceName); |
| 723 | jj(["workspace", "forget", slot.workspaceName], { |
| 724 | ignoreWorkingCopy: true, |
| 725 | node, |
| 726 | plainLog, |
| 727 | }); |
| 728 | trackedWorkspaces.delete(slot.workspaceName); |
| 729 | } |
| 730 | |
| 731 | if (!hasWorkspace && hasWorkspacePath) { |
| 732 | preserveIgnoredWorkspaceCache(slot, plainLog); |
| 733 | log.warn("removing orphaned workspace", slot.workspacePath); |
| 734 | fs.rmSync(slot.workspacePath, { force: true, recursive: true }); |
| 735 | } |
| 736 | |
| 737 | if (!trackedWorkspaces.has(slot.workspaceName)) { |
| 738 | createdWorkspace = true; |
| 739 | jj( |
| 740 | [ |
| 741 | "workspace", |
| 742 | "add", |
| 743 | "--name", |
| 744 | slot.workspaceName, |
| 745 | "--revision", |
| 746 | changeId, |
| 747 | slot.workspacePath, |
| 748 | ], |
| 749 | { node, plainLog }, |
| 750 | ); |
| 751 | trackedWorkspaces.add(slot.workspaceName); |
| 752 | } else { |
| 753 | jj(["workspace", "update-stale"], { |
| 754 | cwd: slot.workspacePath, |
| 755 | node, |
| 756 | plainLog, |
| 757 | }); |
| 758 | } |
| 759 | |
| 760 | jj(["edit", changeId], { cwd: slot.workspacePath, node, plainLog }); |
| 761 | if (createdWorkspace) restoreIgnoredWorkspaceCache(slot, plainLog); |
| 762 | } |
| 763 | |
| 764 | function snapshotWorkspace( |
| 765 | workspacePath: string, |
| 766 | node: progress.Node, |
| 767 | plainLog: string[], |
| 768 | ): void { |
| 769 | jj(["status"], { cwd: workspacePath, node, plainLog }); |
| 770 | } |
| 771 | |
| 772 | function workspaceCachePath(slot: PoolSlot): string { |
| 773 | return path.join(cacheDir, `${slot.name}.cache`); |
| 774 | } |
| 775 | |
| 776 | function workspaceCacheManifestPath(slot: PoolSlot): string { |
| 777 | return path.join(workspaceCachePath(slot), workspaceCacheManifestName); |
| 778 | } |
| 779 | |
| 780 | function preserveIgnoredWorkspaceCache( |
| 781 | slot: PoolSlot, |
| 782 | plainLog: string[], |
| 783 | ): void { |
| 784 | const ignoredPaths = listIgnoredWorkspacePaths(slot.workspacePath); |
| 785 | if (ignoredPaths == null) return; |
| 786 | if (ignoredPaths.length === 0) return; |
| 787 | |
| 788 | const cachePath = workspaceCachePath(slot); |
| 789 | if (fs.existsSync(cachePath)) { |
| 790 | fs.rmSync(cachePath, { force: true, recursive: true }); |
| 791 | } |
| 792 | |
| 793 | fs.mkdirSync(cachePath, { recursive: true }); |
| 794 | for (const relativePath of ignoredPaths) { |
| 795 | const sourcePath = path.join(slot.workspacePath, relativePath); |
| 796 | if (!pathExists(sourcePath)) continue; |
| 797 | |
| 798 | const destinationPath = path.join(cachePath, relativePath); |
| 799 | fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); |
| 800 | fs.renameSync(sourcePath, destinationPath); |
| 801 | } |
| 802 | |
| 803 | fs.writeFileSync( |
| 804 | workspaceCacheManifestPath(slot), |
| 805 | ignoredPaths.join("\n") + "\n", |
| 806 | ); |
| 807 | plainLog.push( |
| 808 | `preserved ignored cache paths: ${ignoredPaths.join(", ")}\n`, |
| 809 | ); |
| 810 | } |
| 811 | |
| 812 | function restoreIgnoredWorkspaceCache( |
| 813 | slot: PoolSlot, |
| 814 | plainLog: string[], |
| 815 | ): void { |
| 816 | const manifestPath = workspaceCacheManifestPath(slot); |
| 817 | if (!fs.existsSync(manifestPath)) return; |
| 818 | |
| 819 | const cachePath = workspaceCachePath(slot); |
| 820 | const relativePaths = fs.readFileSync(manifestPath, "utf8") |
| 821 | .split("\n") |
| 822 | .map((line) => line.trim()) |
| 823 | .filter(Boolean); |
| 824 | |
| 825 | if (relativePaths.length === 0) { |
| 826 | fs.rmSync(cachePath, { force: true, recursive: true }); |
| 827 | return; |
| 828 | } |
| 829 | |
| 830 | const restoredPaths: string[] = []; |
| 831 | const skippedPaths: string[] = []; |
| 832 | |
| 833 | for (const relativePath of relativePaths) { |
| 834 | const sourcePath = path.join(cachePath, relativePath); |
| 835 | if (!pathExists(sourcePath)) continue; |
| 836 | |
| 837 | const destinationPath = path.join(slot.workspacePath, relativePath); |
| 838 | if (pathExists(destinationPath)) { |
| 839 | skippedPaths.push(relativePath); |
| 840 | continue; |
| 841 | } |
| 842 | |
| 843 | try { |
| 844 | fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); |
| 845 | fs.renameSync(sourcePath, destinationPath); |
| 846 | restoredPaths.push(relativePath); |
| 847 | } catch { |
| 848 | skippedPaths.push(relativePath); |
| 849 | } |
| 850 | } |
| 851 | |
| 852 | if (restoredPaths.length > 0) { |
| 853 | plainLog.push( |
| 854 | `restored ignored cache paths: ${restoredPaths.join(", ")}\n`, |
| 855 | ); |
| 856 | } |
| 857 | |
| 858 | if (skippedPaths.length > 0) { |
| 859 | plainLog.push( |
| 860 | `skipped ignored cache paths: ${skippedPaths.join(", ")}\n`, |
| 861 | ); |
| 862 | fs.writeFileSync(manifestPath, skippedPaths.join("\n") + "\n"); |
| 863 | } else { |
| 864 | fs.rmSync(cachePath, { force: true, recursive: true }); |
| 865 | } |
| 866 | } |
| 867 | |
| 868 | function forgetPoolWorkspaces( |
| 869 | pool: PoolSlot[], |
| 870 | trackedWorkspaces: Set<string>, |
| 871 | ): void { |
| 872 | const errors: Error[] = []; |
| 873 | |
| 874 | for (const slot of pool) { |
| 875 | if (!trackedWorkspaces.has(slot.workspaceName)) continue; |
| 876 | |
| 877 | try { |
| 878 | if (fs.existsSync(slot.workspacePath)) { |
| 879 | preserveIgnoredWorkspaceCache(slot, []); |
| 880 | } |
| 881 | jj(["workspace", "forget", slot.workspaceName], { |
| 882 | cwd: root, |
| 883 | ignoreWorkingCopy: true, |
| 884 | }); |
| 885 | trackedWorkspaces.delete(slot.workspaceName); |
| 886 | } catch (error) { |
| 887 | errors.push( |
| 888 | error instanceof Error ? error : new Error(String(error)), |
| 889 | ); |
| 890 | } |
| 891 | } |
| 892 | |
| 893 | if (errors.length > 0) { |
| 894 | throw new AggregateError(errors, "failed to forget one or more jj-ci workspaces"); |
| 895 | } |
| 896 | } |
| 897 | |
| 898 | function resolveChangeIds(revision: string): string[] { |
| 899 | const output = jj( |
| 900 | [ |
| 901 | "log", |
| 902 | "--no-graph", |
| 903 | "-r", |
| 904 | revision, |
| 905 | "-T", |
| 906 | 'self.change_id() ++ "\n"', |
| 907 | ], |
| 908 | { ignoreWorkingCopy: true }, |
| 909 | ); |
| 910 | const changeIds = output.split("\n").filter(Boolean); |
| 911 | |
| 912 | if (changeIds.length === 0) { |
| 913 | log.error("revision resolved to no change IDs", JSON.stringify(revision)); |
| 914 | process.exit(1); |
| 915 | } |
| 916 | |
| 917 | return changeIds; |
| 918 | } |
| 919 | |
| 920 | function readLastRun(lastRunPath: string): string[] { |
| 921 | if (!fs.existsSync(lastRunPath)) { |
| 922 | log.error("no failed run to retry:", lastRunPath); |
| 923 | process.exit(1); |
| 924 | } |
| 925 | |
| 926 | const changeIds = fs.readFileSync(lastRunPath, "utf8") |
| 927 | .split("\n") |
| 928 | .map((line) => line.trim()) |
| 929 | .filter(Boolean); |
| 930 | |
| 931 | if (changeIds.length === 0) { |
| 932 | log.error("retry file is empty:", lastRunPath); |
| 933 | process.exit(1); |
| 934 | } |
| 935 | |
| 936 | return changeIds; |
| 937 | } |
| 938 | |
| 939 | function writeLastRun(lastRunPath: string, changeIds: string[]): void { |
| 940 | fs.mkdirSync(path.dirname(lastRunPath), { recursive: true }); |
| 941 | fs.writeFileSync(lastRunPath, changeIds.join("\n") + "\n"); |
| 942 | } |
| 943 | |
| 944 | function runScript( |
| 945 | scriptPath: string, |
| 946 | options: CommandOptions, |
| 947 | ): Promise<ScriptResult> { |
| 948 | return new Promise((resolve) => { |
| 949 | const startedAt = Date.now(); |
| 950 | let plainText = ""; |
| 951 | const baseEnv = Object.fromEntries( |
| 952 | Object.entries({ |
| 953 | CLICOLOR_FORCE: "1", |
| 954 | FORCE_COLOR: "1", |
| 955 | TERM: process.env.TERM ?? "xterm-256color", |
| 956 | ...options.env, |
| 957 | }).filter((entry): entry is [string, string] => entry[1] != null), |
| 958 | ); |
| 959 | delete baseEnv.NO_COLOR; |
| 960 | const preferredShell = process.env.SHELL; |
| 961 | const shell = fs.existsSync("/bin/bash") |
| 962 | ? "/bin/bash" |
| 963 | : preferredShell != null && path.basename(preferredShell) === "bash" |
| 964 | ? preferredShell |
| 965 | : "bash"; |
| 966 | |
| 967 | const term = pty.spawn(shell, ["-c", 'set -o pipefail; set -eux; source "$1"', "jj-ci", scriptPath], { |
| 968 | cols: process.stdout.columns ?? 120, |
| 969 | cwd: options.cwd, |
| 970 | encoding: "utf8", |
| 971 | env: baseEnv, |
| 972 | name: process.env.TERM ?? "xterm-256color", |
| 973 | rows: process.stdout.rows ?? 30, |
| 974 | }); |
| 975 | |
| 976 | term.onData((chunk) => { |
| 977 | plainText += normalizePlain(chunk); |
| 978 | options.node?.log.write(chunk); |
| 979 | }); |
| 980 | |
| 981 | term.onExit(({ exitCode, signal }) => { |
| 982 | if (signal != null) { |
| 983 | const note = ansi.style(failStyle, `terminated by signal ${signal}`); |
| 984 | plainText += normalizePlain(note); |
| 985 | options.node?.log.info(note); |
| 986 | } |
| 987 | resolve({ |
| 988 | durationMs: Date.now() - startedAt, |
| 989 | exitCode, |
| 990 | ok: exitCode === 0, |
| 991 | plainText, |
| 992 | }); |
| 993 | }); |
| 994 | }); |
| 995 | } |
| 996 | |
| 997 | function writeFailureLog(logPath: string, plainLog: string[]): void { |
| 998 | fs.mkdirSync(path.dirname(logPath), { recursive: true }); |
| 999 | fs.writeFileSync(logPath, plainLog.join("")); |
| 1000 | } |
| 1001 | |
| 1002 | function normalizePlain(text: string): string { |
| 1003 | return ansi.strip(text) |
| 1004 | .replace(/\r\n/g, "\n") |
| 1005 | .replace(/\r/g, "\n"); |
| 1006 | } |
| 1007 | |
| 1008 | function poolName(index: number): string { |
| 1009 | let value = index; |
| 1010 | let name = ""; |
| 1011 | |
| 1012 | while (value >= 0) { |
| 1013 | name = String.fromCharCode(97 + (value % 26)) + name; |
| 1014 | value = Math.floor(value / 26) - 1; |
| 1015 | } |
| 1016 | |
| 1017 | return name; |
| 1018 | } |
| 1019 | |
| 1020 | function maxPlanConcurrency(plans: Plan[]): number { |
| 1021 | if (plans.length === 0) return 0; |
| 1022 | |
| 1023 | const descendantsByChangeId = new Map( |
| 1024 | plans.map((plan) => [plan.changeId, [] as string[]]), |
| 1025 | ); |
| 1026 | |
| 1027 | for (const plan of plans) { |
| 1028 | for (const dependencyChangeId of plan.dependencyChangeIds) { |
| 1029 | descendantsByChangeId.get(dependencyChangeId)?.push(plan.changeId); |
| 1030 | } |
| 1031 | } |
| 1032 | |
| 1033 | const matchedDependencyByDescendant = new Map<string, string>(); |
| 1034 | let matchCount = 0; |
| 1035 | |
| 1036 | for (const plan of plans) { |
| 1037 | if ( |
| 1038 | tryMatchDescendant( |
| 1039 | plan.changeId, |
| 1040 | descendantsByChangeId, |
| 1041 | matchedDependencyByDescendant, |
| 1042 | new Set(), |
| 1043 | ) |
| 1044 | ) { |
| 1045 | matchCount += 1; |
| 1046 | } |
| 1047 | } |
| 1048 | |
| 1049 | return plans.length - matchCount; |
| 1050 | } |
| 1051 | |
| 1052 | function tryMatchDescendant( |
| 1053 | dependencyChangeId: string, |
| 1054 | descendantsByChangeId: Map<string, string[]>, |
| 1055 | matchedDependencyByDescendant: Map<string, string>, |
| 1056 | seenDescendants: Set<string>, |
| 1057 | ): boolean { |
| 1058 | for (const descendantChangeId of descendantsByChangeId.get(dependencyChangeId) ?? []) { |
| 1059 | if (seenDescendants.has(descendantChangeId)) continue; |
| 1060 | seenDescendants.add(descendantChangeId); |
| 1061 | |
| 1062 | const matchedDependency = matchedDependencyByDescendant.get(descendantChangeId); |
| 1063 | if ( |
| 1064 | matchedDependency == null || |
| 1065 | tryMatchDescendant( |
| 1066 | matchedDependency, |
| 1067 | descendantsByChangeId, |
| 1068 | matchedDependencyByDescendant, |
| 1069 | seenDescendants, |
| 1070 | ) |
| 1071 | ) { |
| 1072 | matchedDependencyByDescendant.set(descendantChangeId, dependencyChangeId); |
| 1073 | return true; |
| 1074 | } |
| 1075 | } |
| 1076 | |
| 1077 | return false; |
| 1078 | } |
| 1079 | |
| 1080 | function progressAttachToScreen( |
| 1081 | root: progress.Root, |
| 1082 | { writeOutput, startWidget }: Pick< |
| 1083 | log.WidgetHost, |
| 1084 | "writeOutput" | "startWidget" |
| 1085 | >, |
| 1086 | ): ts.Dispose { |
| 1087 | const stack = new DisposableStack(); |
| 1088 | let widget: log.WidgetInstance | null = null; |
| 1089 | |
| 1090 | stack.use(root.on("change", (items) => { |
| 1091 | if (items.length > 0) { |
| 1092 | widget ??= startWidget({ |
| 1093 | format: ({ now }) => progress.formatAnsi(now, root.active), |
| 1094 | }) ?? null; |
| 1095 | if (!widget)return; |
| 1096 | widget.fps = items.some((x) => x.showTotal !== false && x.total > 0) |
| 1097 | ? 15 |
| 1098 | : null; |
| 1099 | widget.redraw(); |
| 1100 | } else { |
| 1101 | widget?.stop(); |
| 1102 | widget = null; |
| 1103 | } |
| 1104 | })); |
| 1105 | stack.use(root.on("node-detached-log", (msg) => { |
| 1106 | writeOutput(log.formatAnsiMessage(msg, true)); |
| 1107 | })); |
| 1108 | stack.use(root.on("node-end", (node) => { |
| 1109 | let title = node.text; |
| 1110 | let p: progress.ReadOnlyNode | null = node; |
| 1111 | while (p = p.parent) title = p.text + " / " + title; |
| 1112 | })); |
| 1113 | |
| 1114 | return ts.defer(() => stack[Symbol.dispose]); |
| 1115 | } |
Natalie Marks <nmarks413@gmail.com> 2026-04-17 00:03:20-07:00