authoravatarclover caruso<git@paperclover.net>2026-10-22 13:14:53-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
log628f936c593ab09248911fa3f03f28e819d870d4
treef22224873926bc9331889151c504a81cb410cc22
parent67a75449a270804854c5cdae4b2b6a83c1fcd234
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Accelerate shared Mac workspaces and Vite+ installs

Offload compatible pnpm/Vite+ installs and Jujutsu filesystem work to the NAS while retaining native Darwin hooks, package policies, configuration, and failure propagation. Avoid NFS build-cache writes and expose VM loopback previews through the existing SSH connection. Assisted-by: gpt-6.1-sol

20 files changed, 2604 insertions(+), 0 deletions(-)

tools/mac-agent-forward-test.py created+88
...@@ -0,0 +1,88 @@
1#!/usr/bin/env python3
2"""Check loopback forwarding against a disposable HTTP server in the live VM."""
3
4import hashlib
5from importlib import import_module
6from pathlib import Path
7import shlex
8import socket
9import subprocess
10import sys
11import time
12import urllib.error
13import urllib.request
14
15
16vm = import_module("mac-agent-vm")
17
18
19def main():
20 payload = b"forwarding fixture\n" * 40000
21 code = '''from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
22import sys, threading
23class Handler(BaseHTTPRequestHandler):
24 def do_GET(self):
25 body = b"forwarding fixture\\n" * 40000
26 self.send_response(200)
27 self.send_header("Content-Length", str(len(body)))
28 self.end_headers()
29 self.wfile.write(body)
30 def log_message(self, *args): pass
31server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
32threading.Thread(target=server.serve_forever, daemon=True).start()
33print(server.server_port, flush=True)
34sys.stdin.buffer.read()
35server.shutdown()
36server.server_close()
37'''
38 ip = vm.tart("ip", vm.NAME, capture_output=True, text=True, timeout=5).stdout.strip()
39 server = subprocess.Popen([*vm.ssh_args(), "admin@" + ip, "python3 -u -c " + shlex.quote(code)],
40 stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
41 tunnel = None
42 try:
43 port = int(server.stdout.readline())
44 with socket.socket() as available:
45 available.bind(("127.0.0.1", 0))
46 local_port = available.getsockname()[1]
47 command = [sys.executable, str(Path(__file__).with_name("mac-agent-vm.py")),
48 "forward", str(port), "--local-port", str(local_port)]
49 tunnel = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
50 url = f"http://127.0.0.1:{local_port}/fixture"
51 opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
52 deadline = time.monotonic() + 15
53 while True:
54 if tunnel.poll() is not None:
55 raise AssertionError(tunnel.communicate()[1])
56 try:
57 with opener.open(url, timeout=2) as response:
58 actual = response.read()
59 break
60 except urllib.error.URLError:
61 if time.monotonic() >= deadline:
62 raise
63 time.sleep(0.05)
64 assert hashlib.sha256(actual).digest() == hashlib.sha256(payload).digest()
65 collision = subprocess.run(command, capture_output=True, text=True, timeout=15)
66 assert collision.returncode != 0, "A second tunnel silently reused an occupied port."
67 with opener.open(url, timeout=2) as response:
68 assert response.read() == payload, "Port collision disturbed the first tunnel."
69 tunnel.terminate()
70 tunnel.communicate(timeout=5)
71 try:
72 opener.open(url, timeout=2)
73 except urllib.error.URLError:
74 pass
75 else:
76 raise AssertionError("The tunnel listener survived its process.")
77 print(f"Forwarded {len(payload)} bytes intact; collision preserved the live tunnel; shutdown closed its listener.")
78 finally:
79 if tunnel is not None and tunnel.poll() is None:
80 tunnel.terminate()
81 tunnel.communicate(timeout=5)
82 server.stdin.close()
83 server.wait(timeout=10)
84 server.stdout.close()
85
86
87if __name__ == "__main__":
88 main()
tools/mac-agent-vm.md+13
...@@ -23,6 +23,19 @@ python3 tools/mac-agent-vm.py console...@@ -23,6 +23,19 @@ python3 tools/mac-agent-vm.py console
23python3 tools/mac-agent-vm.py stop23python3 tools/mac-agent-vm.py stop
24```24```
2525
26To reach a server bound to guest loopback from this Mac:
27
28```sh
29python3 tools/mac-agent-vm.py forward 5173 --local-port 15173
30```
31
32Open `http://127.0.0.1:15173` on the Mac. The tunnel stays in the foreground;
33Ctrl-C closes it. Both ends use loopback, occupied ports fail explicitly, and
34no guest command or LAN listener is created. T3's browser separately requires
35its toolbar's port approval. Run `python3 tools/mac-agent-forward-test.py` to
36check a disposable guest server, a transfer larger than 512 KiB, port collision,
37and listener shutdown. This test uses the existing guest without changing its data.
38
26`start` resumes a suspended VM or boots a stopped VM, then waits for commands39`start` resumes a suspended VM or boots a stopped VM, then waits for commands
27to work through the logged-in guest agent. `stop` requests a graceful40to work through the logged-in guest agent. `stop` requests a graceful
28guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM41guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM
tools/mac-agent-vm.py+13
...@@ -210,10 +210,23 @@ def main():...@@ -210,10 +210,23 @@ def main():
210 for action in ["exec", "ssh"]:210 for action in ["exec", "ssh"]:
211 command = sub.add_parser(action)211 command = sub.add_parser(action)
212 command.add_argument("command", nargs=argparse.REMAINDER)212 command.add_argument("command", nargs=argparse.REMAINDER)
213 forward = sub.add_parser("forward", help="Reach a guest loopback server from this Mac")
214 forward.add_argument("port", type=int, help="Guest server port")
215 forward.add_argument("--local-port", type=int, help="Host loopback port (default: same as guest)")
213 viewer = sub.add_parser("console")216 viewer = sub.add_parser("console")
214 viewer.add_argument("--listen", default="127.0.0.1")217 viewer.add_argument("--listen", default="127.0.0.1")
215 viewer.add_argument("--port", type=int, default=17891)218 viewer.add_argument("--port", type=int, default=17891)
216 args = parser.parse_args()219 args = parser.parse_args()
220 if args.action == "forward":
221 local_port = args.local_port if args.local_port is not None else args.port
222 if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535:
223 parser.error("Ports must be between 1 and 65535.")
224 ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip()
225 print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.",
226 file=sys.stderr, flush=True)
227 os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes",
228 "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2",
229 "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", "admin@" + ip])
217 if args.action in {"start", "stop", "suspend", "reboot"}:230 if args.action in {"start", "stop", "suspend", "reboot"}:
218 lock = (PRIVATE / "lifecycle.lock").open("a")231 lock = (PRIVATE / "lifecycle.lock").open("a")
219 os.chmod(PRIVATE / "lifecycle.lock", 0o600)232 os.chmod(PRIVATE / "lifecycle.lock", 0o600)
tools/project-pnpm/pnpm-10.32.1.patch created+102
...@@ -0,0 +1,102 @@
1--- pnpm.cjs
2+++ pnpm-darwin-rebuild.cjs
3@@ -128122,7 +128122,7 @@
4 var dependency_path_1 = require_lib87();
5 var crypto_object_hasher_1 = require_lib119();
6 function calcDepState(depsGraph, cache, depPath, opts) {
7- let result2 = constants_1.ENGINE_NAME;
8+ let result2 = opts.includeDepGraphHash ? "darwin;arm64;node26" : constants_1.ENGINE_NAME;
9 if (opts.includeDepGraphHash) {
10 const depGraphHash = calcDepGraphHash(depsGraph, cache, /* @__PURE__ */ new Set(), depPath);
11 result2 += `;deps=${depGraphHash}`;
12@@ -130347,12 +130347,6 @@
13 storeController: store.ctrl,
14 unsafePerm: opts.unsafePerm || false
15 };
16- await (0, lifecycle_1.runLifecycleHooksConcurrently)(["preinstall", "install", "postinstall", "prepublish", "prepare"], Object.values(ctx.projects), opts.childConcurrency || 5, scriptsOpts);
17- for (const { id, manifest } of Object.values(ctx.projects)) {
18- if (manifest?.scripts != null && (!opts.pending || ctx.pendingBuilds.includes(id))) {
19- ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(id), 1);
20- }
21- }
22 await (0, modules_yaml_1.writeModulesManifest)(ctx.rootModulesDir, {
23 prunedAt: (/* @__PURE__ */ new Date()).toUTCString(),
24 ...ctx.modulesFile,
25@@ -130445,10 +130439,12 @@
26 } else {
27 extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts.lockfileDir));
28 }
29+ if (!allowBuild(pkgInfo.name, pkgInfo.version, depPath)) { pkgsThatWereRebuilt.add(depPath); return; }
30 const resolution = pkgSnapshot.resolution;
31 let sideEffectsCacheKey;
32 const pkgId = `${pkgInfo.name}@${pkgInfo.version}`;
33- if (opts.skipIfHasSideEffectsCache && resolution.integrity) {
34+ if (!["esbuild@0.28.0", "sharp@0.34.5", "msgpackr-extract@3.0.3", "@parcel/watcher@2.5.6"].includes(pkgId)) throw new Error("Review this package before caching its hooks: " + pkgId);
35+ if (resolution.integrity) {
36 const filesIndexFile = (0, store_cafs_1.getIndexFilePathInCafs)(opts.storeDir, resolution.integrity.toString(), pkgId);
37 let pkgFilesIndex;
38 try {
39@@ -130460,58 +130456,16 @@
40 includeDepGraphHash: true
41 });
42 if (pkgFilesIndex.sideEffects?.[sideEffectsCacheKey]) {
43+ const delta = pkgFilesIndex.sideEffects[sideEffectsCacheKey];
44+ require("/opt/project-store/restore.cjs")(pkgRoot,
45+ Object.entries(delta.added ?? {}).map(([rel, info]) => [rel, store_cafs_1.getFilePathByModeInCafs(opts.storeDir, info.integrity, info.mode), info.mode]), delta.deleted ?? []);
46+ process.stderr.write(JSON.stringify({cachePackage:pkgId,cacheKey:sideEffectsCacheKey,restored:Object.keys(delta.added ?? {}).length})+"\n");
47 pkgsThatWereRebuilt.add(depPath);
48 return;
49 }
50 }
51 }
52- let requiresBuild = true;
53- const pgkManifest = await (0, read_package_json_1.safeReadPackageJsonFromDir)(pkgRoot);
54- if (pgkManifest != null) {
55- requiresBuild = (0, exec_pkg_requires_build_1.pkgRequiresBuild)(pgkManifest, {});
56- }
57- const hasSideEffects = requiresBuild && allowBuild(pkgInfo.name, pkgInfo.version, depPath) && await (0, lifecycle_1.runPostinstallHooks)({
58- depPath,
59- extraBinPaths,
60- extraEnv: opts.extraEnv,
61- optional: pkgSnapshot.optional === true,
62- pkgRoot,
63- rawConfig: opts.rawConfig,
64- rootModulesDir: ctx.rootModulesDir,
65- scriptsPrependNodePath: opts.scriptsPrependNodePath,
66- shellEmulator: opts.shellEmulator,
67- unsafePerm: opts.unsafePerm || false
68- });
69- if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) {
70- builtDepPaths.add(depPath);
71- const filesIndexFile = (0, store_cafs_1.getIndexFilePathInCafs)(opts.storeDir, resolution.integrity.toString(), pkgId);
72- try {
73- if (!sideEffectsCacheKey) {
74- sideEffectsCacheKey = (0, calc_dep_state_1.calcDepState)(depGraph, depsStateCache, depPath, {
75- includeDepGraphHash: true
76- });
77- }
78- await opts.storeController.upload(pkgRoot, {
79- sideEffectsCacheKey,
80- filesIndexFile
81- });
82- } catch (err) {
83- (0, assert_1.default)(util_1.default.types.isNativeError(err));
84- if ("statusCode" in err && err.statusCode === 403) {
85- logger_1.logger.warn({
86- message: `The store server disabled upload requests, could not upload ${pkgRoot}`,
87- prefix: opts.lockfileDir
88- });
89- } else {
90- logger_1.logger.warn({
91- error: err,
92- message: `An error occurred while uploading ${pkgRoot}`,
93- prefix: opts.lockfileDir
94- });
95- }
96- }
97- }
98- pkgsThatWereRebuilt.add(depPath);
99+ throw new Error("Missing target cache for " + pkgId);
100 } catch (err) {
101 (0, assert_1.default)(util_1.default.types.isNativeError(err));
102 if (pkgSnapshot.optional) {
tools/project-pnpm/pnpm-11.10.0.patch created+105
...@@ -0,0 +1,105 @@
1--- pnpm.mjs
2+++ pnpm-target-cache.mjs
3@@ -155612,7 +155612,7 @@
4 }
5 function calcDepState(depsGraph, cache, depPath, opts3) {
6 const ownPin = readSnapshotRuntimePin(depsGraph[depPath]?.children);
7- let result2 = engineName(ownPin ?? opts3.nodeVersion);
8+ let result2 = opts3.includeDepGraphHash ? "darwin;arm64;node26" : engineName(ownPin ?? opts3.nodeVersion);
9 if (opts3.includeDepGraphHash) {
10 const depGraphHash = calcDepGraphHash(depsGraph, cache, /* @__PURE__ */ new Set(), depPath, opts3.supportedArchitectures);
11 result2 += `;deps=${depGraphHash}`;
12@@ -163136,12 +163136,7 @@
13 unsafePerm: opts3.unsafePerm || false,
14 userAgent: opts3.userAgent
15 };
16- await runLifecycleHooksConcurrently(["preinstall", "install", "postinstall", "prepublish", "prepare"], Object.values(ctx.projects), opts3.childConcurrency || 5, scriptsOpts);
17- for (const { id, manifest } of Object.values(ctx.projects)) {
18- if (manifest?.scripts != null && (!opts3.pending || ctx.pendingBuilds.includes(id))) {
19- ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(id), 1);
20- }
21- }
22+ process.stderr.write(JSON.stringify({guestPending:ctx.pendingBuilds})+"\n");
23 await writeModulesManifest(ctx.rootModulesDir, {
24 prunedAt: (/* @__PURE__ */ new Date()).toUTCString(),
25 ...ctx.modulesFile,
26@@ -163216,7 +163211,7 @@
27 return false;
28 };
29 const builtDepPaths = /* @__PURE__ */ new Set();
30- const storeIndex = opts3.skipIfHasSideEffectsCache ? opts3.frozenStore ? new ReadOnlyStoreIndex(opts3.storeDir) : new StoreIndex(opts3.storeDir) : void 0;
31+ const storeIndex = new ReadOnlyStoreIndex(opts3.storeDir);
32 const gvsDirByDepPath = /* @__PURE__ */ new Map();
33 if (opts3.enableGlobalVirtualStore) {
34 const globalVirtualStoreDir = opts3.globalVirtualStoreDir ?? path92.join(opts3.storeDir, "links");
35@@ -163268,10 +163263,12 @@
36 } else {
37 extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts3.lockfileDir));
38 }
39+ if (!allowBuild(depPath)) { pkgsThatWereRebuilt.add(depPath); return; }
40 const resolution = pkgSnapshot.resolution;
41 let sideEffectsCacheKey;
42 const pkgId = pkgInfo.nonSemverVersion ?? `${pkgInfo.name}@${pkgInfo.version}`;
43- if (opts3.skipIfHasSideEffectsCache && (resolution.gitHosted || resolution.integrity)) {
44+ if (!["esbuild@0.28.2", "esbuild@0.25.12", "tsc-rs@0.7.0", "node-pty@1.2.0-beta.15", "file:apps/mobile/deps/react-native-enriched-markdown-1.2.0-t3.11.tgz"].includes(pkgId)) throw new Error("Review this package before caching its hooks: " + pkgId);
45+ if (resolution.gitHosted || resolution.integrity) {
46 const filesIndexFile = pickStoreIndexKey(resolution, pkgId, { built: true });
47 const pkgFilesIndex = storeIndex.get(filesIndexFile);
48 if (pkgFilesIndex) {
49@@ -163281,6 +163278,13 @@
50 nodeVersion
51 });
52 if (pkgFilesIndex.sideEffects?.has(sideEffectsCacheKey)) {
53+ const delta = pkgFilesIndex.sideEffects.get(sideEffectsCacheKey);
54+ const restore = (await import("file:///opt/project-store/restore.cjs")).default;
55+ restore(pkgRoot, [...(delta.added ?? [])].map(([rel, info]) => {
56+ const digest = typeof info.digest === "string" ? info.digest : Buffer.from(info.digest).toString("hex");
57+ return [rel, getFilePathByModeInCafs(opts3.storeDir, digest, info.mode), info.mode];
58+ }), delta.deleted ?? []);
59+ process.stderr.write(JSON.stringify({cachePackage:pkgId,cacheKey:sideEffectsCacheKey,restored:delta.added?.size ?? 0})+"\n");
60 pkgsThatWereRebuilt.add(depPath);
61 return;
62 }
63@@ -163291,41 +163295,7 @@
64 if (pgkManifest != null) {
65 requiresBuild = pkgRequiresBuild(pgkManifest, /* @__PURE__ */ new Map());
66 }
67- const hasSideEffects = requiresBuild && allowBuild(depPath) && await runPostinstallHooks({
68- depPath,
69- extraBinPaths,
70- extraEnv: opts3.extraEnv,
71- optional: pkgSnapshot.optional === true,
72- pkgRoot,
73- rootModulesDir: ctx.rootModulesDir,
74- scriptsPrependNodePath: opts3.scriptsPrependNodePath,
75- shellEmulator: opts3.shellEmulator,
76- unsafePerm: opts3.unsafePerm || false,
77- userAgent: opts3.userAgent
78- });
79- if (hasSideEffects && (opts3.sideEffectsCacheWrite ?? true) && (resolution.gitHosted || resolution.integrity)) {
80- builtDepPaths.add(depPath);
81- const filesIndexFile = pickStoreIndexKey(resolution, pkgId, { built: true });
82- try {
83- if (!sideEffectsCacheKey) {
84- sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
85- includeDepGraphHash: true,
86- nodeVersion
87- });
88- }
89- await opts3.storeController.upload(pkgRoot, {
90- sideEffectsCacheKey,
91- filesIndexFile
92- });
93- } catch (err2) {
94- assert7(util31.types.isNativeError(err2));
95- logger2.warn({
96- error: err2,
97- message: `An error occurred while uploading ${pkgRoot}`,
98- prefix: opts3.lockfileDir
99- });
100- }
101- }
102+ if (requiresBuild) throw new Error("Missing target cache for " + pkgId);
103 pkgsThatWereRebuilt.add(depPath);
104 } catch (err2) {
105 assert7(util31.types.isNativeError(err2));
tools/project-pnpm/restore-test.cjs created+34
...@@ -0,0 +1,34 @@
1const assert = require('node:assert/strict');
2const fs = require('node:fs');
3const os = require('node:os');
4const path = require('node:path');
5const crypto = require('node:crypto');
6const restore = require('./restore.cjs');
7
8const root = fs.mkdtempSync(path.join(os.tmpdir(), 'project-restore-'));
9try {
10 const bytes = process.env.PROJECT_RESTORE_SOURCE ? fs.readFileSync(process.env.PROJECT_RESTORE_SOURCE) : Buffer.from('cached native artifact');
11 const digest = crypto.createHash('sha512').update(bytes).digest('hex');
12 const source = process.env.PROJECT_RESTORE_SOURCE || path.join(root, digest.slice(0, 2), digest.slice(2));
13 if (!process.env.PROJECT_RESTORE_SOURCE) {
14 fs.mkdirSync(path.dirname(source));
15 fs.writeFileSync(source, bytes);
16 }
17 const destination = path.join(root, 'package');
18 fs.mkdirSync(destination);
19 restore(destination, [['bin/tool', source, 0o755]], []);
20 assert.equal(fs.readFileSync(path.join(destination, 'bin/tool'), 'utf8'), bytes.toString());
21 fs.writeFileSync(path.join(destination, 'bin/tool'), 'changed');
22 assert.deepEqual(fs.readFileSync(source), bytes);
23 fs.symlinkSync(root, path.join(destination, 'escape'));
24 assert.throws(() => restore(destination, [['escape/tool', source, 0o755]], []), /symlink/);
25 assert.throws(() => restore(destination, [['../tool', source, 0o755]], []), /Invalid/);
26 const corrupt = path.join(root, digest.slice(0, 2), digest.slice(2) + '-exec');
27 fs.mkdirSync(path.dirname(corrupt), {recursive: true});
28 fs.writeFileSync(corrupt, 'corrupt');
29 assert.throws(() => restore(destination, [['bin/tool', corrupt, 0o755]], []), /integrity/);
30 assert.equal(fs.readFileSync(path.join(destination, 'bin/tool'), 'utf8'), 'changed');
31 console.log('Reflink independence, path guards, and corrupt-cache rejection passed.');
32} finally {
33 fs.rmSync(root, {recursive: true, force: true});
34}
tools/project-pnpm/restore.cjs created+45
...@@ -0,0 +1,45 @@
1const fs = require('node:fs');
2const path = require('node:path');
3const crypto = require('node:crypto');
4
5module.exports = function restore(root, added, deleted) {
6 root = fs.realpathSync(root);
7 function destination(relative) {
8 const parts = relative.split('/');
9 if (parts.some(p => !p || p === '.' || p === '..') || relative.includes('\\')) {
10 throw new Error('Invalid cached path');
11 }
12 let parent = root;
13 for (const part of parts.slice(0, -1)) {
14 parent = path.join(parent, part);
15 try { fs.mkdirSync(parent); } catch (error) { if (error.code !== 'EEXIST') throw error; }
16 if (!fs.lstatSync(parent).isDirectory() || fs.realpathSync(parent) !== parent) {
17 throw new Error('Cached path follows a symlink');
18 }
19 }
20 return path.join(parent, parts.at(-1));
21 }
22 for (const relative of deleted) fs.rmSync(destination(relative), {force: true});
23 for (const [relative, source, mode] of added) {
24 const dest = destination(relative);
25 const temporary = dest + '.project-' + crypto.randomUUID();
26 const fd = fs.openSync(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
27 try {
28 if (!fs.fstatSync(fd).isFile()) throw new Error('Cache entry is not a regular file');
29 const expected = path.basename(path.dirname(source)) + path.basename(source).replace(/-exec$/, '');
30 const hash = crypto.createHash('sha512');
31 const buffer = Buffer.alloc(65536);
32 let length;
33 while ((length = fs.readSync(fd, buffer)) > 0) hash.update(buffer.subarray(0, length));
34 if (!/^[a-f0-9]{128}$/.test(expected) || hash.digest('hex') !== expected) {
35 throw new Error('Cached file failed integrity verification');
36 }
37 fs.copyFileSync('/proc/self/fd/' + fd, temporary, fs.constants.COPYFILE_FICLONE_FORCE);
38 fs.chmodSync(temporary, mode & 0o777);
39 fs.renameSync(temporary, dest);
40 } finally {
41 fs.closeSync(fd);
42 fs.rmSync(temporary, {force: true});
43 }
44 }
45};
tools/project-store-install.py created+161
...@@ -0,0 +1,161 @@
1#!/usr/bin/env python3
2"""Stage project tools on Snow Globe; activation is a separate atomic switch."""
3
4import argparse
5import hashlib
6import io
7import json
8import os
9from pathlib import Path
10import shutil
11import socket
12import subprocess
13import tarfile
14import tempfile
15import urllib.request
16
17
18HERE = Path(__file__).resolve().parent
19BASE = Path("/opt/studio-project-store")
20PROJECTS = "/srv/mac-studio-vm/sandbox/project"
21GUEST_PROJECTS = "/Volumes/Code/project"
22
23
24def download(url, digest):
25 with urllib.request.urlopen(url, timeout=60) as response:
26 data = response.read()
27 if hashlib.sha256(data).hexdigest() != digest:
28 raise ValueError("Downloaded tool failed checksum verification.")
29 return tarfile.open(fileobj=io.BytesIO(data), mode="r:gz")
30
31
32def stage(name):
33 destination = BASE / "releases" / name
34 if destination.exists():
35 raise ValueError("Release already exists. Choose a new name.")
36 current = (BASE / "current").resolve(strict=True)
37 binaries = {}
38 for name, pattern, executable in (("python", "project-store-*-python3-*", "python3"),
39 ("node", "project-store-*-nodejs-*", "node"),
40 ("bwrap", "project-store-*-bubblewrap-*", "bwrap"),
41 ("coreutils", "project-store-coreutils", "true")):
42 roots = list(Path("/nix/var/nix/gcroots").glob(pattern))
43 if len(roots) != 1:
44 raise ValueError(f"Expected one pinned {name} runtime, found {len(roots)}.")
45 binaries[name] = str(roots[0].resolve(strict=True) / "bin" / executable)
46 with tempfile.TemporaryDirectory(prefix=".prepare-", dir=destination.parent) as temporary:
47 release = Path(temporary)
48 shutil.copytree(HERE / "project-tools", release / "project-tools",
49 ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
50 for filename in ("project-store.py", "project-store-test.py"):
51 shutil.copy2(HERE / filename, release / filename)
52 for filename in ("restore.cjs", "restore-test.cjs"):
53 shutil.copy2(HERE / "project-pnpm" / filename, release / filename)
54 profiles = json.loads((current / "profiles.json").read_text())
55 for profile in profiles.values():
56 profile.pop("workspace_config", None)
57 (release / "profiles.json").write_text(json.dumps(profiles))
58 shutil.copy2(current / "jj.toml", release / "jj.toml")
59 (release / "bin").mkdir()
60 with download("https://github.com/jj-vcs/jj/releases/download/v0.42.0/"
61 "jj-v0.42.0-x86_64-unknown-linux-musl.tar.gz",
62 "2d91e81d649e617a81608e7401ad1106029c15ece01ac928c4a351abef42be6a") as archive:
63 member, = [m for m in archive if m.isfile() and Path(m.name).name == "jj"]
64 (release / "bin/jj-real").write_bytes(archive.extractfile(member).read())
65 (release / "bin/jj-real").chmod(0o755)
66 (release / "bin/jj").symlink_to("../project-tools/jj")
67 with download("https://github.com/BurntSushi/ripgrep/releases/download/15.2.0/"
68 "ripgrep-15.2.0-x86_64-unknown-linux-musl.tar.gz",
69 "33e15bcf1624b25cdd2a55813a47a2f95dbe126268203e76aa6a585d1e7b149c") as archive:
70 member, = [m for m in archive if m.isfile() and Path(m.name).name == "rg"]
71 (release / "bin/rg").write_bytes(archive.extractfile(member).read())
72 (release / "bin/rg").chmod(0o755)
73 for name in ("jj", "pnpm", "project"):
74 path = release / "project-tools" / name
75 content = path.read_text().split("\n", 1)[1]
76 path.write_text("#!" + binaries["python"] + "\n" + content)
77 path.chmod(0o755)
78 for version, digest, directory, original, patched in (
79 ("10.32.1", "9b943b94bc8f55efb993aad8e44b538e6b091e60a9e4a944dcde869855f233e3",
80 "pnpm10", "pnpm.cjs", "pnpm-darwin-rebuild.cjs"),
81 ("11.10.0", "620b6605ea4f62fc56a6d0a98733f479071d0321e03e486b522c7e9a74617431",
82 "pnpm11", "pnpm.mjs", "pnpm-target-cache.mjs"),
83 ):
84 with download(f"https://registry.npmjs.org/pnpm/-/pnpm-{version}.tgz", digest) as archive:
85 archive.extractall(release / directory, filter="data")
86 package = release / directory / "package"
87 for path in (package / "dist").iterdir():
88 path.rename(release / directory / path.name)
89 shutil.rmtree(package)
90 shutil.copy2(release / directory / original, release / directory / patched)
91 subprocess.run(["patch", "--batch", str(release / directory / patched),
92 str(HERE / "project-pnpm" / f"pnpm-{version}.patch")], check=True)
93 (release / "pnpm12").mkdir()
94 with download("https://github.com/pnpm/pnpm/releases/download/v12.14.0/pnpm-linux-x64-musl.tar.gz",
95 "d7115e2f2791d79bf9e7e448609537595bb7787de1ef5292ce91ff3d1be2629e") as archive:
96 member, = [m for m in archive if m.isfile() and Path(m.name).name == "pnpm"]
97 (release / "pnpm12/pnpm").write_bytes(archive.extractfile(member).read())
98 (release / "pnpm12/pnpm").chmod(0o755)
99 command = [binaries["bwrap"], "--die-with-parent", "--unshare-all", "--cap-drop", "ALL",
100 "--ro-bind", "/nix/store", "/nix/store", "--proc", "/proc", "--dev", "/dev",
101 "--tmpfs", "/tmp", "--dir", "/home/clo",
102 "--ro-bind", str(destination), "/opt/project-store",
103 "--bind", PROJECTS, GUEST_PROJECTS, "--bind", PROJECTS, PROJECTS]
104 stores = "/srv/mac-studio-vm/sandbox/cache/project-pnpm"
105 for target in (stores, "/Volumes/Code/cache/project-pnpm", "/Users/admin/Library/Caches/project"):
106 command.extend(("--bind", stores, target))
107 command.extend(("--bind", "/srv/mac-studio-vm/sandbox/cache/pnpm12", "/Volumes/Code/cache/pnpm12"))
108 command.extend(("--bind", "/home/clo/.cache/pnpm", "/home/clo/.cache/pnpm",
109 "--clearenv", "--setenv", "HOME", "/home/clo", "--setenv", "PATH",
110 "/opt/project-store/bin:" + ":".join(str(Path(binaries[key]).parent)
111 for key in ("node", "python", "coreutils")),
112 "--setenv", "JJ_CONFIG", "/opt/project-store/jj.toml",
113 "--setenv", "PROJECT_STORE_SSH", "", "--setenv", "LC_ALL", "C.UTF-8",
114 "--chdir", GUEST_PROJECTS, binaries["python"],
115 "/opt/project-store/project-store.py", GUEST_PROJECTS))
116 network = ["--share-net", "--ro-bind", "/etc/resolv.conf", "/etc/resolv.conf",
117 "--ro-bind", "/etc/ssl/certs/ca-bundle.crt", "/etc/ssl/certs/ca-bundle.crt",
118 "--setenv", "SSL_CERT_FILE", "/etc/ssl/certs/ca-bundle.crt",
119 "--setenv", "NODE_EXTRA_CA_CERTS", "/etc/ssl/certs/ca-bundle.crt"]
120 (release / "ssh-command").write_text(
121 "#!" + binaries["python"] + "\nimport json, subprocess, sys\n"
122 "request = sys.stdin.buffer.readline(4 * 1024 * 1024 + 1)\n"
123 "if len(request) > 4 * 1024 * 1024: sys.exit('Project request exceeds 4 MiB.')\n"
124 "command = " + repr(command) + "\n"
125 "if json.loads(request).get('command') == 'read':\n"
126 " command = ['--ro-bind' if arg == '--bind' else arg for arg in command]\n"
127 "if json.loads(request).get('command') == 'pnpm':\n"
128 " command[command.index('--chdir'):command.index('--chdir')] = " + repr(network) + "\n"
129 "result = subprocess.run(command, input=request)\n"
130 "sys.exit(result.returncode if result.returncode >= 0 else 128 - result.returncode)\n")
131 (release / "ssh-command").chmod(0o755)
132 release.chmod(0o755)
133 release.rename(destination)
134 release.mkdir()
135 print(destination)
136
137
138def main():
139 parser = argparse.ArgumentParser(description=__doc__)
140 parser.add_argument("action", choices=("stage", "activate"))
141 parser.add_argument("release")
142 args = parser.parse_args()
143 if socket.gethostname() != "snowglobe" or os.geteuid() != 0:
144 parser.error("Run this installer as root on Snow Globe.")
145 if not args.release or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789-." for c in args.release):
146 parser.error("Use a release name containing lowercase letters, numbers, dots, or hyphens.")
147 if args.action == "stage":
148 os.umask(0o022)
149 stage(args.release)
150 else:
151 release = BASE / "releases" / args.release
152 if not (release / "ssh-command").is_file():
153 parser.error("Stage this release before activating it.")
154 link = BASE / (".current-" + args.release)
155 link.symlink_to(Path("releases") / args.release)
156 link.replace(BASE / "current")
157 print(release)
158
159
160if __name__ == "__main__":
161 main()
tools/project-store-test.py created+128
...@@ -0,0 +1,128 @@
1#!/usr/bin/env python3
2"""Disposable workspace tests; PROJECT_STORE_REAL_REFLINK=1 requires Linux/ZFS."""
3
4import importlib.util
5import os
6from pathlib import Path
7import shutil
8import subprocess
9import tempfile
10import unittest
11from unittest.mock import patch
12
13spec = importlib.util.spec_from_file_location("project_store", Path(__file__).with_name("project-store.py"))
14store = importlib.util.module_from_spec(spec)
15spec.loader.exec_module(store)
16
17
18class WorkspaceTest(unittest.TestCase):
19 def setUp(self):
20 self.tmp = tempfile.TemporaryDirectory(prefix="project-store-test-")
21 self.addCleanup(self.tmp.cleanup)
22 self.root = Path(self.tmp.name).resolve()
23 config = self.root / "jj.toml"
24 config.write_text('[user]\nname="Fixture"\nemail="fixture@example.invalid"\n[signing]\nbehavior="drop"\n')
25 self.env = patch.dict(os.environ, {"JJ_CONFIG": str(config)})
26 self.env.start()
27 self.addCleanup(self.env.stop)
28 if not os.environ.get("PROJECT_STORE_REAL_REFLINK"):
29 self.copy = patch.object(store.fcntl, "ioctl", side_effect=self.copy_bytes)
30 self.copy.start()
31 self.addCleanup(self.copy.stop)
32 self.main = self.root / "Example/main"
33 self.main.parent.mkdir()
34 self.jj("git", "init", "--no-colocate", str(self.main))
35 (self.main / ".gitignore").write_text("target/\n")
36 (self.main / "source.txt").write_text("committed\n")
37 (self.main / "unchanged.txt").write_text("same\n")
38 (self.main / "link").symlink_to("source.txt")
39 self.jj("-R", str(self.main), "describe", "-m", "fixture")
40 self.revision = self.jj("-R", str(self.main), "log", "--no-graph", "-r", "@", "-T", "commit_id").strip()
41 (self.main / "target/debug").mkdir(parents=True)
42 (self.main / "target/debug/artifact").write_bytes(b"warm artifact" * 10000)
43 (self.main / "target/debug/.cargo-lock").touch()
44 self.request = dict(project="Example", source="main", destination="agent",
45 revision=self.revision, targets=["target"])
46
47 @staticmethod
48 def copy_bytes(destination, command, source):
49 with os.fdopen(os.dup(source), "rb") as src, os.fdopen(os.dup(destination), "wb") as dst:
50 shutil.copyfileobj(src, dst)
51
52 def jj(self, *args):
53 return subprocess.run(["jj", *args], text=True, capture_output=True, check=True).stdout
54
55 def test_host_checkout_without_cache_preserves_donor(self):
56 (self.main / "source.txt").write_text("active agent edit\n")
57 request = dict(project="Example", destination="cold", revision=self.revision,
58 engine="darwin;arm64;node26", install=True)
59 self.assertEqual(store.seed(self.root, request), {"workspace": "cold", "installed": False, "pnpm": False})
60 self.assertEqual((self.main.parent / "cold/source.txt").read_text(), "committed\n")
61 self.assertEqual((self.main / "source.txt").read_text(), "active agent edit\n")
62 with self.assertRaises(ValueError):
63 store.seed(self.root, request)
64 for name in ("../escape", "main", "default"):
65 with self.assertRaises(ValueError):
66 store.seed(self.root, dict(request, destination=name))
67
68 def test_cache_inputs_track_dependency_changes_only(self):
69 (self.main / "package.json").write_text('{"packageManager":"pnpm@10.32.1"}')
70 before = store.dependency_inputs(self.main)
71 operation = self.jj("--ignore-working-copy", "-R", str(self.main), "op", "log", "--no-graph", "-n", "1", "-T", "id")
72 (self.main / "source.txt").write_text("new source\n")
73 self.assertEqual(store.dependency_inputs(self.main), before)
74 (self.main / "package.json").write_text('{"packageManager":"pnpm@12.14.0"}')
75 self.assertNotEqual(store.dependency_inputs(self.main), before)
76 self.assertEqual(self.jj("--ignore-working-copy", "-R", str(self.main), "op", "log", "--no-graph", "-n", "1", "-T", "id"), operation)
77 (self.main / "package.json").unlink()
78 (self.main / "package.json").symlink_to("source.txt")
79 with self.assertRaises(ValueError):
80 store.dependency_inputs(self.main)
81
82 def test_clone_restores_revision_and_preserves_warm_files(self):
83 (self.main / "source.txt").write_text("donor edit\n")
84 (self.main / "donor-only.txt").write_text("uncommitted\n")
85 original = self.main / "target/debug/artifact"
86 mtime = (self.main / "unchanged.txt").stat().st_mtime_ns
87 self.assertEqual(store.seed(self.root, self.request), {"workspace": "agent"})
88 agent = self.main.parent / "agent"
89 self.assertEqual((agent / "source.txt").read_text(), "committed\n")
90 self.assertFalse((agent / "donor-only.txt").exists())
91 self.assertEqual((self.main / "source.txt").read_text(), "donor edit\n")
92 self.assertEqual((agent / "unchanged.txt").stat().st_mtime_ns, mtime)
93 artifact = agent / "target/debug/artifact"
94 self.assertEqual(artifact.read_bytes(), original.read_bytes())
95 self.assertEqual(artifact.stat().st_mtime_ns, original.stat().st_mtime_ns)
96 self.assertNotEqual(artifact.stat().st_ino, original.stat().st_ino)
97 artifact.write_bytes(b"modified")
98 self.assertNotEqual(artifact.read_bytes(), original.read_bytes())
99 self.assertEqual(os.readlink(agent / "link"), "source.txt")
100 self.assertEqual(self.jj("-R", str(agent), "diff", "--summary"), "")
101 self.assertFalse((agent / ".jj/repo").is_dir())
102 self.assertFalse(Path((agent / ".jj/repo").read_text()).is_absolute())
103
104 def test_rejects_paths_and_existing_destination(self):
105 for key, value in (("source", "../outside"), ("destination", "main"),
106 ("project", "/tmp"), ("targets", ["../outside"])):
107 with self.subTest(key=key), self.assertRaises(ValueError):
108 store.seed(self.root, dict(self.request, **{key: value}))
109 (self.main.parent / "agent").mkdir()
110 with self.assertRaises(ValueError):
111 store.seed(self.root, self.request)
112
113 def test_rejects_symlink_workspace(self):
114 (self.main.parent / "linked").symlink_to(self.main)
115 with self.assertRaises(OSError):
116 store.seed(self.root, dict(self.request, source="linked"))
117
118 def test_failed_reflink_removes_workspace_registration_and_files(self):
119 with patch.object(store.fcntl, "ioctl", side_effect=OSError("reflink unavailable")):
120 with self.assertRaises(OSError):
121 store.seed(self.root, self.request)
122 self.assertFalse((self.main.parent / "agent").exists())
123 self.assertNotIn("agent:", self.jj("-R", str(self.main), "workspace", "list"))
124 self.assertEqual((self.main / "source.txt").read_text(), "committed\n")
125
126
127if __name__ == "__main__":
128 unittest.main()
tools/project-store.md created+220
...@@ -0,0 +1,220 @@
1# Shared project commands
2
3`tools/project-tools/` owns the VM's `jj`, `pnpm`, and `project` wrappers.
4The config repo's `users/clover/agents/vm` entries link here; installed wrappers
5use `~/.local/share/project-tools/current`. Keep native executables adjacent to
6the public commands as `jj-real` and `pnpm-real`.
7
8## Defaults
9
10- `project read rg ARGS...` searches on Linux beside ZFS and streams normal
11 ripgrep output and exit status. `project read rg --files` lists files.
12 `.gitignore` works in Jujutsu checkouts without a visible `.git` directory;
13 `--require-git` restores ripgrep's usual Git-only policy. Search preprocessors
14 are disabled. `project read sha256sum FILE...` hashes named shared files
15 without transferring their contents over NFS. Run these from the project
16 share; read requests have read-only mounts, no network, and no repository lock.
17- Noninteractive `jj` operations in `/Volumes/Code/project` run on Snow Globe.
18 This includes workspace creation, snapshots, rebases, restores, and reads.
19 The guest's resolved configuration, identity, ignores, and paths travel with
20 the request. Both sides must run exactly the same Jujutsu build.
21- Editors, external diff tools, signing, network Git commands, and local-disk
22 repositories retain native execution. Explicit file paths for a shared
23 repository cannot escape the project share, including through symlinks.
24- `pnpm install`, `i`, `add`, `update`, and `remove` resolve, fetch, and import
25 packages on Linux, including changed manifests and lockfiles. The runtime must
26 match the project's exact pin: 10.32.1, 11.10.0, or 12.14.0. Directory selection,
27 workspace filters, frozen/offline installs, and lockfile-only updates retain
28 their normal meanings. `--ignore-scripts` skips the Mac hook phase.
29- Activation links Vite+'s installed managed pnpm launchers to this same wrapper,
30 preserving each original as the adjacent `pnpm-real`. This covers the
31 project-local JavaScript `vp` launcher, which otherwise bypasses pnpm on PATH.
32 Vite+ keeps package-manager detection and argument handling; its notifier
33 setting does not prevent offloading. Explicit `VP_PACKAGE_MANAGER` overrides
34 retain native execution. Reactivate after Vite+ downloads another pnpm version.
35- `pnpm run`, `exec`, and script shortcuts execute natively. Their implicit
36 dependency installs use Linux too; dependency verification opt-outs and custom
37 verification policies are preserved. Inherited npm client metadata and Node
38 heap limits retain offloading; `NODE_ENV=production` reaches the pinned pnpm
39 unchanged on both hosts. Unsupported options, runtime versions, other custom
40 install environment settings, and unreviewed pnpmfile hooks retain native
41 execution. Environment fallbacks name the variables without exposing values.
42 A failed remote operation is reported once.
43- `project worktree PROJECT NAME` uses those same `jj` and `pnpm` commands.
44 `--no-install` skips dependencies. The checkout path is printed only after
45 installation and Mac hooks succeed, including with `--warm-from`.
46 New workspaces wait for NFS visibility.
47
48Linux installs with lifecycle scripts disabled, selects Darwin
49ARM64 packages, validates the guest's Node version, and requires ZFS reflinks.
50Knife's reviewed `.pnpmfile.cjs` runs in the Linux sandbox: its peer-range change
51and Windows-only architecture selection behave identically on Linux and Darwin.
52The backend checks its exact SHA-256 before offloading. Changed hooks, other
53pnpmfiles, and config dependencies retain native installation.
54Pending dependency and workspace hooks finish natively on the Mac. pnpm 10/11
55use `pnpm --recursive --include-workspace-root rebuild --pending`. pnpm 12 uses
56a frozen install with the original filters and dependency groups, preserving
57prepared patches. Its repeat-install shortcut and build-cache reads/writes are
58disabled for this phase: restoring cached generated files over NFS can take
59longer than running their hooks. Ordinary native installs retain their cache policy.
60The wrapper announces the Mac handoff.
61It also announces storage-host installation and package-store lock contention.
62The two phases share policy and registry metadata in the project's `.pnpm-cache`
63directory, outside its workspaces. This avoids repeating network verification on
64the Mac. The handoff waits until NFS exposes the server's new modules manifest.
65Repository configuration is not temporarily rewritten; manifests and
66lockfiles change only as requested by the dependency command.
67
68The root-owned `profiles.json` binds manifests, lockfiles, workspace settings,
69npmrc, pnpmfiles, patches, and local tarballs. Matching alphaXiv pnpm 10.32.1 and
70Knife pnpm 11.10.0 profiles on Node 26 can additionally restore reviewed Darwin
71side effects on Linux after SHA-512 verification using `project-pnpm/`'s patches.
72A profile miss still installs on Linux; it leaves native builds pending for the
73Mac instead of replaying an older build. Validation does not snapshot the working
74copy. Ordinary Linux side-effect cache reuse is disabled.
75
76## Locking and storage
77
78The Mac wrapper uses `flock`; Linux uses a POSIX record lock on the same
79`.jj/repo/command.lock` inode. macOS NFS translates `flock` into that server lock.
80The executing Jujutsu process inherits the descriptor. The forwarding guest
81never holds a second lock. Ctrl-C cancels a waiter; never unlink the lock inode.
82A command lock does not reserve revisions across a sequence of commands.
83
84The Code export is one ZFS dataset. `project worktree --warm-from DONOR` uses
85per-file FICLONE, preserving independent inodes and source timestamps, excluding
86root repository metadata and Mac sidecars. It restores the requested revision
87and retains ignored build outputs. The guest holds the donor's Cargo locks;
88editors and other writers must also be idle. No byte-copy fallback is allowed.
89
90The dedicated SSH key forces the unprivileged uid-3000 Bubblewrap launcher.
91Jujutsu requests have no network. Dependency requests get network access with
92DNS and the host CA bundle; lifecycle scripts remain disabled on Linux. Neither path
93exposes host credentials, forwarding, or snapshot privileges.
94The sandbox binds projects and package stores at their guest paths as well as
95host paths. Identical store paths allow repeated installs without repaving
96`node_modules`. Artifacts remain under Code. pnpm 11 uses a guest-local SQLite
97index under `~/Library/Caches/project/PROJECT/v11`; pnpm 12 uses the shared
98`/Volumes/Code/cache/pnpm12` file index. Both phases use workspace-local virtual
99stores so platform-specific build outputs belong to that workspace.
100
101## Updating the existing deployment
102
103Upload `project-store-install.py`, `project-store.py`, `project-store-test.py`,
104`project-tools/`, and `project-pnpm/` together to a private staging directory on
105Snow Globe. Run the installer there as root:
106
107```sh
108python3 project-store-install.py stage RELEASE
109```
110
111It verifies pinned upstream downloads, applies the cache patches, preserves the
112reviewed profiles, and builds a versioned runtime under
113`/opt/studio-project-store/releases/`. Existing `project-store-*` Nix GC roots
114pin Python, Node, Bubblewrap, and coreutils. `current` is unchanged by staging.
115Test the staged `ssh-command` as `clo` with disposable projects and a temporary
116restricted key. Do not run Jujutsu as root against shared repositories.
117
118Upload `project-tools/` to a guest staging directory and run its installer as
119`admin`. After the staged host and guest tests pass:
120
121```sh
122# Snow Globe
123python3 project-store-install.py activate RELEASE
124# VM, using tools/project-tools/install.py
125python3 install.py stage RELEASE
126python3 install.py activate RELEASE
127```
128
129Guest activation saves the prior tools under `project-tools/previous`, preserves
130the native executables, and changes symlinks atomically. Subsequent updates switch
131the complete wrapper directory through `current`. Keep the previous host release
132and guest tools for rollback. Roll back both `current` links; do not remove
133workspaces, locks, stores, or caches.
134
135## Verification
136
137```sh
138python3 tools/project-store-test.py
139python3 tools/project-tools/test.py
140python3 tools/project-tools/jj-test.py
141```
142
143Run `project-tools/platform-test.py` inside the development VM against the
144normal endpoint. Its disposable project checks Darwin-only optional dependencies
145immediately after the Linux import, before Mac build scripts can repair them.
146`project-tools/finish-test.py` checks native hooks, denied scripts, patch inode
147preservation, repeated installs, filtered dependency groups, and nested mutations.
148
149The backend tests create disposable repositories. `PROJECT_STORE_REAL_REFLINK=1`
150with `TMPDIR` on disposable Linux/ZFS storage exercises real clone independence,
151timestamps, donor edits, path rejection, and failure cleanup. The config repo's
152`vm/test.py` covers overlay preparation, worktrees, and shared memory links.
153`project-pnpm/restore-test.cjs` checks cache integrity and path guards on Linux/ZFS.
154
155On 2026-10-22, the endpoint created an 801-file workspace through the VM in 2.9
156seconds and rebased a two-commit fixture in 1.6 seconds. Cross-client lock
157contention and cancellation passed. Disposable pnpm 10, 11, and 12 projects
158installed changed dependencies on Linux and ran Darwin dependency and workspace
159hooks on the Mac. Frozen-lockfile failures, lockfile-only updates, filtered
160add/update/remove commands, ignored scripts, and native esbuild execution passed.
161A pnpm 12 filtered update took 12.4 seconds, including 1.4 seconds on Linux;
162a subsequent script ran in 1.2 seconds without reinstalling. Nested pnpm scripts
163and dependency-verification opt-outs passed. An existing global virtual store
164migrated successfully. After activation, ordinary commands against the real
165shared store passed: a fresh install took 70.5 seconds, a filtered add took 32.9
166seconds, and a script took 2.5 seconds. Native rebuilding left all 1,048 sampled
167lodash files' inodes and mtimes unchanged. These are fixture measurements with other agents
168active, not latency bounds or full-project build measurements.
169
170Vite+ 1.3.0 was also verified through the normal endpoint with pnpm 12.14.0
171and Knife's reviewed `.pnpmfile.cjs`: package imports ran on Linux/ZFS, while
172the workspace postinstall and native esbuild ran on Darwin ARM64. Frozen/offline
173installs, add/remove, lockfile-only, ignored scripts, script execution, and frozen
174lockfile failure propagation passed. Editing the hook selected Mac installation.
175The host and VM tools were activated as `20261022-vp-preview-2`.
176
177pnpm 12 needs `supportedArchitectures: [darwin-arm64]` for host-side package
178selection; the older `config.os`/`config.cpu` overrides do not select its optional
179dependencies. This is passed as a command setting without editing the workspace.
180The platform regression fails against the old endpoint and passes with the fix.
181The full Knife workspace imported 75 missing Darwin packages, and its Vite+,
182Vite, Vitest, Rolldown, and linter bindings loaded successfully before rebuilding.
183With Mac build-cache writes disabled, a normal frozen install in `knife-merge`
184completed successfully in 278 seconds, including the mobile dependency's native
185asset downloads and the root prepare hook. The host and guest were activated as
186`20261023-vp-platform-3`. Linux side-effect cache reads and writes are explicitly
187disabled with pnpm 12's supported boolean settings.
188After installation, `pnpm exec vp --version`, the TypeScript compiler, and the
189Vitest CLI started successfully. The node-pty addon loaded, and sharp encoded a
190small PNG using its Darwin native module.
191
192A follow-up trace in `knife-disk` found the Mac phase copying patched packages
193and restoring cached mobile assets over NFS. The frozen-install completion
194described above disables those cache transfers. Five disposable VM cases cover
195native and workspace hooks, denied scripts, patch preservation, filtered
196production/development installs, nested adds, repeated installs, and inherited
197agent environments. A deployed
198frozen install in `knife-disk` completed in 43.5 seconds with its native assets
199already present. Vite+, TypeScript, and the node-pty native binding loaded.
200The real project-local `vp i --frozen-lockfile` then completed in 28.8 seconds
201through Vite+'s managed pnpm launcher and the shared wrapper. Exporting
202`VP_PNPM_MANAGER=no` alone did not route that JavaScript launcher through PATH;
203the obsolete VM environment setting was removed.
204
205The host and guest were activated as `20261023-vp-env-4`. Staged pnpm 10, 11,
206and 12 installs with `NODE_ENV=production` preserved their native dependency
207selection; production mode is forwarded as environment, without inventing a
208`--prod` flag. The VM fixture also checked that Mac hooks receive that value
209while npm client metadata and Node memory limits remain compatible with offloading.
210A real `vp i --frozen-lockfile` with those inherited metadata and memory settings
211completed in 23.2 seconds on `knife-disk`, including Mac hooks with native assets
212already present. Vite+'s tools, TypeScript, and the node-pty Darwin binding loaded.
213
214Release `20261023-tools-6` adds the read-only storage tools and waits for NFS
215visibility before opening warm workspaces. A VM request listed 2,001 fixture
216files in 1.93 seconds and hashed a 330 MiB fixture in 0.95 seconds. Staged
217checks verified read-only mounts, network isolation, ignore rules, normal
218search exit statuses, hash equality, and preprocessor rejection. A real warm
219checkout installed a changed dependency and its Darwin hook in 7.81 seconds;
220the donor and an explicit `--no-install` copy retained the original dependency.
tools/project-store.py created+410
...@@ -0,0 +1,410 @@
1#!/usr/bin/env python3
2"""Create Jujutsu workspaces and restore dependencies using host-side ZFS reflinks.
3
4Run behind a forced SSH command with a fixed project root and an unprivileged
5account. The guest supplies one JSON request on stdin, never a shell command.
6"""
7
8import argparse
9from contextlib import ExitStack
10import fcntl
11import hashlib
12import json
13import os
14from pathlib import Path
15import re
16import shutil
17import stat
18import subprocess
19import sys
20import tempfile
21
22sys.path.insert(0, str(Path(__file__).with_name("project-tools")))
23import client
24
25
26def parts(value):
27 if not isinstance(value, str) or not value or value.startswith("/"):
28 raise ValueError("Use a relative path inside the project directory.")
29 names = value.split("/")
30 if any(n in ("", ".", "..", ".jj", ".git") or "\\" in n
31 or any(ord(c) < 32 or ord(c) == 127 for c in n) for n in names):
32 raise ValueError("Paths cannot contain traversal or repository metadata.")
33 return names
34
35
36def directory(stack, root, names):
37 for name in names:
38 root = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root)
39 stack.callback(os.close, root)
40 return root
41
42
43def clone_tree(source, destination, *, workspace=False):
44 initial = os.fstat(source)
45 for name in os.listdir(source):
46 if (workspace and name in (".jj", ".git")) or name.startswith("._") or name == ".DS_Store":
47 continue
48 before = os.stat(name, dir_fd=source, follow_symlinks=False)
49 with ExitStack() as stack:
50 if stat.S_ISDIR(before.st_mode):
51 src = directory(stack, source, [name])
52 os.mkdir(name, mode=0o700, dir_fd=destination)
53 dst = directory(stack, destination, [name])
54 clone_tree(src, dst)
55 elif stat.S_ISREG(before.st_mode):
56 src = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=source)
57 stack.callback(os.close, src)
58 if not stat.S_ISREG(os.fstat(src).st_mode):
59 raise ValueError("Target changed during cloning. Retry with an idle donor.")
60 dst = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
61 0o600, dir_fd=destination)
62 stack.callback(os.close, dst)
63 # Linux FICLONE must succeed; a byte-copy fallback defeats warm workspace creation.
64 fcntl.ioctl(dst, 0x40049409, src)
65 elif stat.S_ISLNK(before.st_mode):
66 os.symlink(os.readlink(name, dir_fd=source), name, dir_fd=destination)
67 continue
68 else:
69 raise ValueError(f"Workspace contains a special file: {name}. Remove it before cloning.")
70 current = os.fstat(src)
71 if (before.st_ino, before.st_size, before.st_mtime_ns) != (
72 current.st_ino, current.st_size, current.st_mtime_ns):
73 raise ValueError("Target changed during cloning. Retry with an idle donor.")
74 os.fchmod(dst, stat.S_IMODE(current.st_mode) & 0o777)
75 os.utime(dst, ns=(current.st_atime_ns, current.st_mtime_ns))
76 if os.fstat(source).st_mtime_ns != initial.st_mtime_ns:
77 raise ValueError("Workspace changed during cloning. Retry with an idle donor.")
78
79
80def seed(root, request):
81 if not isinstance(request, dict):
82 raise ValueError("Invalid project-store request.")
83 if request.get("command") == "jj":
84 return run_jj(root, request)
85 if request.get("command") == "read":
86 if set(request) != {"command", "cwd", "tool", "args"} or request["tool"] not in ("rg", "sha256sum"):
87 raise ValueError("Use rg or sha256sum for read-only storage commands.")
88 cwd = shared_directory(root, request["cwd"])
89 args = request["args"]
90 if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args):
91 raise ValueError("Invalid read-only command arguments.")
92 if request["tool"] == "rg":
93 if any(arg in ("--pre", "--hostname-bin") or arg.startswith(("--pre=", "--hostname-bin=")) for arg in args):
94 raise ValueError("Search preprocessors are unavailable on the storage host.")
95 args = ["--no-require-git", *args]
96 else:
97 has_file = False
98 options = True
99 for arg in args:
100 if options and arg == "--":
101 options = False
102 elif options and arg in ("-b", "--binary", "-t", "--text", "--tag", "-z", "--zero"):
103 continue
104 elif arg == "-" or (options and arg.startswith("-")):
105 raise ValueError("Hash named files; stdin and checksum verification are unavailable.")
106 else:
107 path = (cwd / arg).resolve(strict=True)
108 if not path.is_relative_to(root) or not path.is_file():
109 raise ValueError("Hash files inside the project share.")
110 has_file = True
111 if not has_file:
112 raise ValueError("Select at least one file to hash.")
113 result = subprocess.run([request["tool"], *args], cwd=cwd, stdin=subprocess.DEVNULL)
114 return result.returncode if result.returncode >= 0 else 128 - result.returncode
115 if request.get("command") == "pnpm":
116 if (set(request) - {"command", "cwd", "engine", "args", "prepare", "node_env"}
117 or not isinstance(request.get("prepare", False), bool)
118 or request.get("node_env") not in (None, "production")):
119 raise ValueError("Invalid pnpm request.")
120 cwd = shared_directory(root, request["cwd"])
121 args = request["args"]
122 if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args):
123 raise ValueError("Invalid pnpm arguments.")
124 if not args or all(arg.startswith("--") for arg in args):
125 args = ["install", *args]
126 try:
127 parsed = client.pnpm_arguments(args, cwd)
128 except StopIteration:
129 raise ValueError("A dependency command option is missing its value.") from None
130 if parsed[1] is None:
131 raise ValueError("Unsupported dependency command.")
132 cwd, args, _ = parsed
133 cwd = shared_directory(root, cwd)
134 relative = cwd.relative_to(root)
135 if len(relative.parts) < 2:
136 return {"installed": False, "reason": "run the install from the workspace root."}
137 workspace = root.joinpath(*relative.parts[:2])
138 return install(workspace, relative.parts[0], request["engine"], args, cwd=cwd,
139 prepare=request.get("prepare", False), node_env=request.get("node_env"))
140 if set(request) == {"project", "destination", "revision", "engine", "install"}:
141 return checkout(Path(root), request)
142 if set(request) != {"project", "source", "destination", "revision", "targets"}:
143 raise ValueError("Invalid project-store request.")
144 project, source, destination = (parts(request[k]) for k in ("project", "source", "destination"))
145 if (any(len(p) != 1 for p in (project, source, destination)) or source == destination
146 or destination[0] in ("main", "default")):
147 raise ValueError("Select distinct workspaces in one project.")
148 if not isinstance(request["revision"], str) or not request["revision"]:
149 raise ValueError("Select a parent revision.")
150 targets = [parts(p) for p in request["targets"]]
151 if not targets or len({tuple(p) for p in targets}) != len(targets):
152 raise ValueError("Select each target directory once.")
153 root = Path(root).absolute()
154 with ExitStack() as stack:
155 root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
156 stack.callback(os.close, root_fd)
157 project_fd = directory(stack, root_fd, project)
158 src = directory(stack, project_fd, source)
159 directory(stack, project_fd, ["main", ".jj"])
160 for path in targets:
161 directory(stack, src, path)
162 destination_path = root / project[0] / destination[0]
163 if os.path.lexists(destination_path):
164 raise ValueError("Destination exists. Choose a new workspace name.")
165 primary = root / project[0] / "main"
166 jj = ["jj", "--config", "snapshot.auto-track=" + json.dumps('~glob:"**/._*" & ~glob:"**/.DS_Store"')]
167 base = [*jj, "-R", str(primary)]
168 revision = subprocess.run(
169 [*base, "log", "--no-graph", "-r", request["revision"], "-T", 'commit_id ++ "\n"'],
170 text=True, capture_output=True, check=True).stdout.splitlines()
171 if len(revision) != 1:
172 raise ValueError("Select exactly one parent revision.")
173 subprocess.run([*base, "workspace", "add", "--sparse-patterns", "empty",
174 "--name", destination[0], "-r", revision[0], "--", str(destination_path)],
175 stdout=sys.stderr, check=True)
176 try:
177 dst = directory(stack, project_fd, destination)
178 clone_tree(src, dst, workspace=True)
179 local = [*jj, "-R", str(destination_path)]
180 subprocess.run([*local, "sparse", "set", "--add", "."], stdout=sys.stderr, check=True)
181 # Discard donor edits while retaining matching files and ignored build outputs.
182 subprocess.run([*local, "restore", "--from", revision[0]], stdout=sys.stderr, check=True)
183 except BaseException:
184 subprocess.run([*base, "workspace", "forget", destination[0]],
185 stdout=sys.stderr, check=True)
186 shutil.rmtree(destination_path)
187 raise
188 return {"workspace": destination[0]}
189
190
191def dependency_inputs(workspace, revision=None):
192 base = ["jj", "--ignore-working-copy", "-R", str(workspace), "file"]
193 revision_args = ["-r", revision] if revision else []
194 names = subprocess.check_output([*base, "list", *revision_args], cwd=workspace, text=True).splitlines()
195 if revision is None:
196 names = set(names) | {path.name for path in workspace.iterdir() if path.is_file()}
197 result = {}
198 for name in names:
199 path = Path(name)
200 if (path.name in ("package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc")
201 or path.suffix in (".patch", ".tgz") or "pnpmfile" in path.name):
202 target = workspace / path
203 if not revision and target.resolve() != target.absolute():
204 raise ValueError("Dependency input follows a symlink.")
205 if not revision and not target.exists():
206 continue
207 content = (subprocess.check_output([*base, "show", json.dumps(name), *revision_args], cwd=workspace)
208 if revision else target.read_bytes())
209 result[name] = hashlib.sha256(content).hexdigest()
210 return result
211
212
213def checkout(root, request):
214 project, name = (parts(request[k]) for k in ("project", "destination"))
215 if len(project) != 1 or len(name) != 1 or name[0] in ("main", "default"):
216 raise ValueError("Select a new workspace name.")
217 if not isinstance(request["revision"], str) or not request["revision"]:
218 raise ValueError("Select a parent revision.")
219 with ExitStack() as stack:
220 fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
221 stack.callback(os.close, fd)
222 directory(stack, fd, [*project, "main", ".jj"])
223 workspace = root / project[0] / name[0]
224 if os.path.lexists(workspace):
225 raise ValueError("Destination exists. Choose a new workspace name.")
226 subprocess.run(["jj", "-R", str(root / project[0] / "main"), "workspace", "add",
227 "--name", name[0], "-r", request["revision"], "--", str(workspace)],
228 stdout=sys.stderr, check=True)
229 manifest = workspace / "package.json"
230 package = json.loads(manifest.read_text()) if manifest.is_file() else {}
231 result = {"workspace": name[0], "installed": False,
232 "pnpm": package.get("packageManager", "").startswith("pnpm@")}
233 if request["install"] is True:
234 result["installed"] = install(workspace, project[0], request["engine"], [])["installed"]
235 return result
236
237
238def install(workspace, project, engine, args, *, cwd=None, prepare=False, node_env=None):
239 if not args:
240 args = ["install"]
241 cwd = cwd or workspace
242 environment = dict(os.environ, NODE_ENV=node_env) if node_env else None
243 manifest = workspace / "package.json"
244 package = json.loads(manifest.read_text()) if manifest.is_file() else {}
245 manager = re.fullmatch(r"pnpm@(\d+)\.(\d+)\.(\d+)(?:\+.*)?", package.get("packageManager", ""))
246 if not manager:
247 return {"installed": False, "reason": "the workspace does not pin a supported pnpm version."}
248 version = ".".join(manager.groups())
249 major = manager.group(1)
250 executable = Path(__file__).parent / ("pnpm" + major) / {
251 "10": "pnpm.cjs", "11": "pnpm.mjs", "12": "pnpm"}.get(major, "unsupported")
252 if not executable.is_file():
253 return {"installed": False, "reason": f"pnpm {version} is not installed on the storage host."}
254 pnpm = (["node"] if major in ("10", "11") else []) + [str(executable)]
255 if subprocess.check_output([*pnpm, "--version"], text=True, env=environment).strip() != version:
256 return {"installed": False, "reason": f"pnpm {version} is not installed on the storage host."}
257 target = re.fullmatch(r"darwin;arm64;node(\d+(?:\.\d+\.\d+)?)", engine)
258 if not target:
259 return {"installed": False, "reason": "the storage install targets Darwin ARM64."}
260 node_version = target.group(1)
261 profiles = Path(__file__).with_name("profiles.json")
262 profile = json.loads(profiles.read_text()).get(project) if profiles.exists() else None
263 guest_store = ("/Volumes/Code/cache/pnpm12" if major == "12" else
264 "/Users/admin/Library/Caches/project/" + project if major == "11" else
265 "/Volumes/Code/cache/project-pnpm/" + project)
266 if profile and Path(profile["pnpm"]).parent.name == "pnpm" + major:
267 guest_store = str(Path(profile["guest_store"]).parent)
268 workspace_config = workspace / "pnpm-workspace.yaml"
269 config_text = workspace_config.read_text().lower() if workspace_config.is_file() else ""
270 npmrc = workspace / ".npmrc"
271 if npmrc.is_file():
272 config_text += npmrc.read_text().lower()
273 # Knife's reviewed hook behaves identically on Linux and Darwin; edits require review.
274 if ("pnpmfile" in config_text or "configdependencies" in config_text or any(
275 path.name != ".pnpmfile.cjs" or hashlib.sha256(path.read_bytes()).hexdigest() !=
276 "6334e7e025aa23897f0b39365407691b3b65f64bf64fe167b7f1b0d6627b2eed"
277 for path in workspace.glob(".pnpmfile.*"))):
278 return {"installed": False, "store": guest_store, "reason": "workspace pnpmfile hooks require Mac execution."}
279 if prepare:
280 verify = subprocess.check_output([*pnpm, "config", "get", "verify-deps-before-run"],
281 cwd=cwd, text=True, env=environment).strip()
282 if verify == "undefined":
283 verify = "false" if major == "10" else "install"
284 if verify == "false":
285 return {"ready": True}
286 if verify != "install":
287 return {"installed": False, "reason": "the workspace selects its own dependency verification policy."}
288 cached = (not node_env and profile and Path(profile["pnpm"]).parent.name == "pnpm" + major
289 and "darwin;arm64;node" + node_version.split(".")[0] == profile["engine"]
290 and args[0] in ("install", "i")
291 and set(args[1:]) <= {"--frozen-lockfile", "--offline", "--prefer-offline", "--force", "--ignore-scripts"}
292 and dependency_inputs(workspace) == profile["inputs"])
293 if cached:
294 packages = json.loads(subprocess.check_output(
295 [*pnpm, "list", "--recursive", "--depth=-1", "--json"],
296 cwd=workspace, text=True, env=environment))
297 cached = all(str((Path(package["path"]) / "package.json").relative_to(workspace)) in profile["inputs"]
298 for package in packages)
299 Path(guest_store).mkdir(parents=True, exist_ok=True)
300 common = ["--config.store-dir=" + guest_store,
301 "--config.cache-dir=" + str(workspace.parent / ".pnpm-cache"),
302 "--config.enable-global-virtual-store=false",
303 "--config.prefer-symlinked-executables=true",
304 "--reporter=append-only"]
305 if "." in node_version:
306 common.append("--config.node-version=" + node_version)
307 if major == "12":
308 common.extend(('--config.supported-architectures=["darwin-arm64"]',
309 "--config.side-effects-cache=false", "--config.side-effects-cache-readonly=false",
310 "--config.store-index-format=files", "--config.virtual-store-type=local"))
311 else:
312 common.extend(("--config.side-effects-cache-read=false", "--config.cpu=arm64", "--config.os=darwin"))
313 modules = workspace / "node_modules/.modules.yaml"
314 if prepare and modules.is_file():
315 check = subprocess.run([*pnpm, *common, "--config.verify-deps-before-run=error", "exec", "true"],
316 cwd=cwd, text=True, capture_output=True, env=environment)
317 if check.returncode == 0:
318 return {"ready": True}
319 if "VERIFY_DEPS_BEFORE_RUN" not in check.stdout + check.stderr:
320 sys.stderr.write(check.stdout + check.stderr)
321 raise subprocess.CalledProcessError(check.returncode, pnpm)
322 included = json.loads(modules.read_text())["included"]
323 args = ["install"]
324 if included["dependencies"] and not included["devDependencies"]:
325 args.append("--prod")
326 elif included["devDependencies"] and not included["dependencies"]:
327 args.append("--dev")
328 if not included["optionalDependencies"]:
329 args.append("--no-optional")
330 with ExitStack() as stack:
331 lock = os.open(guest_store + "/.project-install.lock", os.O_CREAT | os.O_RDWR, 0o600)
332 stack.callback(os.close, lock)
333 try:
334 fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
335 except BlockingIOError:
336 print("pnpm: Waiting for another install using this package store…", file=sys.stderr, flush=True)
337 fcntl.flock(lock, fcntl.LOCK_EX)
338 subprocess.run([*pnpm, *(arg for arg in args if arg != "--ignore-scripts"),
339 "--config.ignore-scripts=true", "--config.package-import-method=clone", *common], cwd=cwd,
340 stdout=sys.stderr, check=True, env=environment)
341 if cached and "--ignore-scripts" not in args:
342 subprocess.run(["node", profile["restore"], "-r", "rebuild", "--pending", *common],
343 cwd=workspace, stdout=sys.stderr, check=True, env=environment)
344 result = {"installed": True, "workspace": str(workspace)}
345 if "--lockfile-only" not in args:
346 result["modules_hash"] = hashlib.sha256(modules.read_bytes()).hexdigest()
347 return result
348
349
350def shared_directory(root, value):
351 path = Path(value).resolve(strict=True)
352 if not path.is_relative_to(root) or not path.is_dir():
353 raise ValueError("Path is outside the project share.")
354 return path
355
356
357def run_jj(root, request):
358 if set(request) != {"command", "cwd", "args", "config", "version", "ignore"}:
359 raise ValueError("Invalid jj request.")
360 cwd = shared_directory(root, request["cwd"])
361 args = request["args"]
362 if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args):
363 raise ValueError("Invalid jj arguments.")
364 words, options, _ = client.jj_arguments(args)
365 paths = client.jj_paths(words, options)
366 paths.extend(value for name, value in options if name in ("-R", "--repository", "--config-file"))
367 for value in paths:
368 if not (cwd / value).resolve().is_relative_to(root):
369 raise ValueError("Path is outside the project share.")
370 version = subprocess.check_output(["jj-real", "--version"], text=True).strip()
371 if version != request["version"]:
372 raise ValueError("Guest and storage Jujutsu versions differ. Update the project tools together.")
373 with tempfile.TemporaryDirectory(prefix="project-jj-") as temporary:
374 config = Path(temporary) / "jj.toml"
375 config.write_text(request["config"])
376 ignore = Path(temporary) / "git/ignore"
377 ignore.parent.mkdir()
378 ignore.write_text(request["ignore"])
379 # The resolved guest config carries its identity, aliases, and environment overrides.
380 # Global flags must precede `--`, whose remaining arguments are paths or messages.
381 result = subprocess.run(["jj", "--config-file", str(config), "--no-pager", *args], cwd=cwd,
382 env=dict(os.environ, JJ_CONFIG=str(config), XDG_CONFIG_HOME=temporary),
383 stdin=subprocess.DEVNULL)
384 return result.returncode if result.returncode >= 0 else 128 - result.returncode
385
386
387def main():
388 parser = argparse.ArgumentParser(description=__doc__)
389 parser.add_argument("root", help="Fixed host-side project directory")
390 args = parser.parse_args()
391 if os.geteuid() == 0:
392 parser.error("Run project-store as the workspace owner, without root privileges.")
393 request = sys.stdin.buffer.readline(4 * 1024 * 1024 + 1)
394 if len(request) > 4 * 1024 * 1024:
395 parser.error("Project request exceeds 4 MiB.")
396 try:
397 result = seed(Path(args.root).resolve(), json.loads(request))
398 if isinstance(result, int):
399 return result
400 print(json.dumps(result))
401 except subprocess.CalledProcessError as error:
402 return error.returncode if error.returncode >= 0 else 128 - error.returncode
403 except (OSError, ValueError, TypeError, KeyError) as error:
404 print(f"project-store: {error}", file=sys.stderr)
405 return 1
406 return 0
407
408
409if __name__ == "__main__":
410 sys.exit(main())
tools/project-tools/client.py created+201
...@@ -0,0 +1,201 @@
1"""Guest transport and lifecycle hooks for the shared project store."""
2
3import errno
4import hashlib
5import json
6import os
7from pathlib import Path
8import re
9import subprocess
10import sys
11import time
12
13
14ROOT = Path(os.environ.get("PROJECT_ROOT", "/Volumes/Code/project")).resolve()
15
16
17def host():
18 value = os.environ.get("PROJECT_STORE_SSH")
19 if value is None:
20 value = "project-store" if (Path.home() / ".ssh/project-store").is_file() else ""
21 if value and not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", value):
22 raise ValueError("Use an SSH alias for PROJECT_STORE_SSH.")
23 return value
24
25
26def inside(path):
27 path = Path(path).resolve()
28 if not path.is_relative_to(ROOT):
29 raise ValueError(f"Path is outside the project share: {path}")
30 return path
31
32
33def request(payload, *, capture=True):
34 return subprocess.run(["ssh", "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10",
35 "--", host()], input=json.dumps(payload) + "\n", text=True,
36 stdout=subprocess.PIPE if capture else None)
37
38
39def engine():
40 return subprocess.check_output(["node", "-p",
41 "[process.platform,process.arch,'node'+process.versions.node].join(';')"],
42 text=True).strip()
43
44
45def wait_workspace(destination):
46 deadline = time.monotonic() + 30
47 while True:
48 try:
49 list(destination.parent.iterdir())
50 list(destination.iterdir())
51 list((destination / ".jj/working_copy").iterdir())
52 return
53 except OSError as error:
54 if error.errno not in (errno.ENOENT, errno.ESTALE):
55 raise
56 if time.monotonic() >= deadline:
57 raise ValueError("Workspace was created but is not visible over NFS yet. Wait before opening it.")
58 time.sleep(0.1)
59
60
61def pnpm_arguments(args, cwd):
62 command = None
63 forwarded = []
64 parsed = iter(args)
65 for arg in parsed:
66 if arg in ("-C", "--dir"):
67 cwd = (cwd / next(parsed)).resolve()
68 elif arg.startswith("--dir="):
69 cwd = (cwd / arg.split("=", 1)[1]).resolve()
70 elif arg in ("--filter", "--filter-prod", "-F"):
71 forwarded.extend((arg, next(parsed)))
72 elif arg.startswith(("--filter=", "--filter-prod=")) or arg in {
73 "--frozen-lockfile", "--no-frozen-lockfile", "--offline", "--prefer-offline",
74 "--force", "--ignore-scripts", "--lockfile-only", "-r", "--recursive",
75 "-w", "--workspace-root", "-D", "--save-dev", "-P", "--save-prod",
76 "-O", "--save-optional", "--save-peer", "-E", "--save-exact", "--latest", "-L",
77 "--prod", "--dev", "--no-optional", "--ignore-workspace-root-check"}:
78 forwarded.append(arg)
79 elif command is None and arg in {"install", "i", "add", "update", "up", "upgrade",
80 "remove", "rm", "uninstall", "un", "uni"}:
81 command = arg
82 forwarded.append(arg)
83 elif command is not None and not arg.startswith("-"):
84 forwarded.append(arg)
85 else:
86 return cwd, None, arg if command is None and not arg.startswith("-") else command
87 return cwd, forwarded if command else None, command
88
89
90def finish_install(workspace, native, *, ignore_scripts=False, modules_hash=None, cwd=None, args=()):
91 deadline = time.monotonic() + 30
92 while True:
93 try:
94 list(workspace.iterdir())
95 modules = workspace / "node_modules/.modules.yaml"
96 content = modules.read_bytes()
97 if modules_hash is None or hashlib.sha256(content).hexdigest() == modules_hash:
98 data = json.loads(content)
99 break
100 except OSError as error:
101 if error.errno not in (errno.ENOENT, errno.ESTALE):
102 raise
103 if time.monotonic() >= deadline:
104 raise ValueError("Installation is not visible over NFS yet. Retry after the mount catches up.")
105 time.sleep(0.1)
106 if ignore_scripts or not data.get("pendingBuilds"):
107 return
108 store = Path(data["storeDir"])
109 target = workspace
110 command = ["--recursive", "--include-workspace-root", "rebuild", "--pending"]
111 extra = []
112 if data["packageManager"].startswith("pnpm@11."):
113 store.mkdir(parents=True, exist_ok=True)
114 files = store / "files"
115 if not os.path.lexists(files):
116 relative = store.relative_to(Path.home() / "Library/Caches/project")
117 files.symlink_to(Path("/Volumes/Code/cache/project-pnpm") / relative / "files")
118 elif data["packageManager"].startswith("pnpm@12."):
119 # pnpm 12 rebuild can recopy patched packages that the storage host already prepared.
120 target = cwd or workspace
121 command = ["install", "--frozen-lockfile"]
122 filtered = False
123 parsed = iter(args)
124 for arg in parsed:
125 if arg in ("--filter", "--filter-prod", "-F"):
126 command.extend((arg, next(parsed)))
127 filtered = True
128 elif arg.startswith(("--filter=", "--filter-prod=")):
129 command.append(arg)
130 filtered = True
131 elif arg in ("-r", "--recursive", "-w", "--workspace-root", "--offline", "--prefer-offline"):
132 command.append(arg)
133 _, _, operation = pnpm_arguments(args, target)
134 if operation not in (None, "install", "i") and not filtered and not any(
135 arg in args for arg in ("-r", "--recursive")):
136 command.append("--filter=./")
137 included = data["included"]
138 if included["dependencies"] and not included["devDependencies"]:
139 command.append("--prod")
140 elif included["devDependencies"] and not included["dependencies"]:
141 command.append("--dev")
142 if not included["optionalDependencies"]:
143 command.append("--no-optional")
144 extra = ["--config.virtual-store-type=local", "--config.store-index-format=files",
145 "--config.side-effects-cache=false", "--config.side-effects-cache-readonly=false",
146 "--config.prefer-symlinked-executables=true",
147 "--config.optimistic-repeat-install=false"]
148 print("pnpm: Running Mac build hooks.", file=sys.stderr)
149 subprocess.run([str(native), "--dir", str(target), *command,
150 "--store-dir=" + str(store.parent),
151 "--config.cache-dir=" + str(workspace.parent / ".pnpm-cache"),
152 "--config.enable-global-virtual-store=false", "--config.verify-deps-before-run=false", *extra],
153 cwd=workspace, check=True)
154
155
156VALUES = {"-R", "--repository", "--config", "--config-file", "--color", "--at-operation", "--at-op",
157 "-r", "--revision", "-s", "--source", "-d", "--destination", "-b", "--branch",
158 "--from", "--to", "-A", "--insert-after", "-B", "--insert-before", "-T", "--template",
159 "-m", "--message", "--name", "--sparse-patterns", "--author", "--tool", "--editor",
160 "-n", "--limit", "--sort", "--add", "--remove"}
161GLOBALS = {"-R", "--repository", "--config", "--config-file", "--color", "--at-operation", "--at-op"}
162
163
164def jj_arguments(args):
165 words, options, config = [], [], []
166 args = iter(args)
167 for arg in args:
168 if arg == "--":
169 words.extend(args)
170 break
171 name, equal, value = arg.partition("=")
172 if name in VALUES:
173 value = value if equal else next(args)
174 options.append((name, value))
175 if name in GLOBALS:
176 config.extend((name, value))
177 elif len(arg) > 2 and arg[:2] in VALUES and not arg.startswith("--"):
178 name, value = arg[:2], arg[2:].removeprefix("=")
179 options.append((name, value))
180 if name in GLOBALS:
181 config.extend((name, value))
182 elif arg.startswith("-"):
183 options.append((arg, None))
184 else:
185 words.append(arg)
186 return words, options, config
187
188def jj_paths(words, options):
189 if not words:
190 return []
191 command = tuple(words[:2]) if words and words[0] in ("workspace", "file", "sparse") else tuple(words[:1])
192 paths = []
193 if command == ("workspace", "add"):
194 paths = words[2:]
195 elif command[0] in ("diff", "show", "restore"):
196 paths = words[1:]
197 elif command[0] == "file":
198 paths = words[2:]
199 elif command == ("sparse", "set"):
200 paths = [value for name, value in options if name in ("--add", "--remove")]
201 return paths
tools/project-tools/finish-test.py created+124
...@@ -0,0 +1,124 @@
1#!/usr/bin/env python3
2"""Exercise native completion without recopying patches or widening selection."""
3
4import io
5import json
6import os
7from pathlib import Path
8import subprocess
9import sys
10import tarfile
11import tempfile
12import unittest
13
14import client
15
16
17class FinishTest(unittest.TestCase):
18 def setUp(self):
19 temporary = tempfile.TemporaryDirectory(prefix="infra-finish-", dir=client.ROOT)
20 self.addCleanup(temporary.cleanup)
21 self.workspace = Path(temporary.name) / "main"
22 self.workspace.mkdir()
23 self.native = Path.home() / ".local/bin/pnpm-real"
24 allow = {}
25 for name in ("allowed", "denied", "patched", "dev", "optional"):
26 package = dict(name="native-finish-" + name, version="1.0.0")
27 if name != "patched":
28 package["scripts"] = {"postinstall":
29 'node -e "require(\'fs\').writeFileSync(\'built\',process.platform)"'}
30 with tarfile.open(self.workspace / (name + ".tgz"), "w:gz") as archive:
31 for filename, content in (("package.json", json.dumps(package).encode()),
32 ("index.js", b"module.exports = 'before';\n")):
33 entry = tarfile.TarInfo("package/" + filename)
34 entry.size = len(content)
35 archive.addfile(entry, io.BytesIO(content))
36 allow[f"native-finish-{name}@file:{name}.tgz"] = name != "denied"
37 (self.workspace / "patched.patch").write_text(
38 "diff --git a/index.js b/index.js\n--- a/index.js\n+++ b/index.js\n@@ -1 +1 @@\n"
39 "-module.exports = 'before';\n+module.exports = 'after';\n")
40 (self.workspace / "pnpm-workspace.yaml").write_text(json.dumps(dict(
41 packages=["child", "other"], allowBuilds=allow,
42 patchedDependencies={"native-finish-patched@1.0.0": "patched.patch"})))
43 root = dict(name="finish-root", private=True, packageManager="pnpm@12.14.0")
44 for directory, package in ((self.workspace, root),
45 (self.workspace / "child", dict(name="finish-child", private=True)),
46 (self.workspace / "other", dict(name="finish-other", private=True))):
47 directory.mkdir(exist_ok=True)
48 package["scripts"] = {"prepare":
49 'node -e "require(\'fs\').writeFileSync(\'prepared\',process.platform)"'}
50 if directory.name == "child":
51 package["dependencies"] = {"native-finish-" + name: "file:../" + name + ".tgz"
52 for name in ("allowed", "denied", "patched")}
53 package["devDependencies"] = {"native-finish-dev": "file:../dev.tgz"}
54 package["optionalDependencies"] = {"native-finish-optional": "file:../optional.tgz"}
55 (directory / "package.json").write_text(json.dumps(package))
56
57 def finish(self, args, *, nested=False):
58 cwd = self.workspace / "child" if nested else self.workspace
59 response = client.request(dict(command="pnpm", cwd=str(cwd), engine=client.engine(),
60 args=[*args, "--ignore-scripts"]))
61 self.assertEqual(response.returncode, 0)
62 data = json.loads(response.stdout)
63 self.assertIs(data.get("installed"), True, data)
64 client.finish_install(self.workspace, None, ignore_scripts=True, modules_hash=data["modules_hash"])
65 patched = self.workspace / "child/node_modules/native-finish-patched/index.js"
66 before = patched.stat() if patched.exists() else None
67 client.finish_install(self.workspace, self.native, modules_hash=data["modules_hash"], cwd=cwd, args=args)
68 if before:
69 self.assertEqual(patched.read_text(), "module.exports = 'after';\n")
70 self.assertEqual((patched.stat().st_ino, patched.stat().st_mtime_ns),
71 (before.st_ino, before.st_mtime_ns))
72 self.assertFalse((self.workspace / "child/node_modules/native-finish-denied/built").exists())
73
74 def test_pending_dependency_and_workspace_hooks_preserve_patched_files(self):
75 self.finish(["install"])
76 for name in ("allowed", "dev", "optional"):
77 self.assertEqual((self.workspace / f"child/node_modules/native-finish-{name}/built").read_text(), "darwin")
78 for directory in (self.workspace, self.workspace / "child", self.workspace / "other"):
79 self.assertEqual((directory / "prepared").read_text(), "darwin")
80 (directory / "prepared").unlink()
81 self.finish(["install", "--frozen-lockfile"])
82 for directory in (self.workspace, self.workspace / "child", self.workspace / "other"):
83 self.assertEqual((directory / "prepared").read_text(), "darwin")
84
85 def test_filtered_production_install_keeps_other_projects_and_excluded_groups_untouched(self):
86 self.finish(["--filter", "finish-child", "install", "--prod", "--no-optional"])
87 self.assertEqual((self.workspace / "child/node_modules/native-finish-allowed/built").read_text(), "darwin")
88 for path in ("prepared", "child/prepared", "other/prepared", "child/node_modules/native-finish-dev",
89 "child/node_modules/native-finish-optional"):
90 self.assertFalse((self.workspace / path).exists(), path)
91
92 def test_nested_add_only_finishes_the_selected_project(self):
93 self.finish(["add", "native-finish-allowed@file:../allowed.tgz"], nested=True)
94 self.assertEqual((self.workspace / "child/node_modules/native-finish-allowed/built").read_text(), "darwin")
95 self.assertEqual((self.workspace / "child/prepared").read_text(), "darwin")
96 self.assertFalse((self.workspace / "other/prepared").exists())
97
98 def test_development_only_install_does_not_add_production_dependencies(self):
99 self.finish(["install", "--dev", "--no-optional"])
100 self.assertEqual((self.workspace / "child/node_modules/native-finish-dev/built").read_text(), "darwin")
101 self.assertEqual((self.workspace / "child/prepared").read_text(), "darwin")
102 self.assertFalse((self.workspace / "child/node_modules/native-finish-allowed").exists())
103
104 def test_inherited_agent_environment_keeps_native_pnpm_semantics(self):
105 manifest = self.workspace / "package.json"
106 package = json.loads(manifest.read_text())
107 package["scripts"]["prepare"] += (
108 ' && node -e "require(\'fs\').writeFileSync(\'mode\',process.env.NODE_ENV)"')
109 manifest.write_text(json.dumps(package))
110 result = subprocess.run([str(Path(__file__).with_name("pnpm")), "install"], cwd=self.workspace,
111 env=dict(os.environ, NODE_ENV="production", NODE_OPTIONS="--max-old-space-size=8192",
112 npm_config_user_agent="pnpm/12.14.0 node/v26.12.0 darwin arm64"),
113 text=True, capture_output=True, timeout=120)
114 self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
115 self.assertIn("pnpm: Running Mac build hooks.", result.stderr)
116 self.assertNotIn("Using Mac installation", result.stderr)
117 self.assertEqual((self.workspace / "mode").read_text(), "production")
118 self.assertEqual((self.workspace / "child/node_modules/native-finish-dev/built").read_text(), "darwin")
119
120
121if __name__ == "__main__":
122 if sys.platform != "darwin" or not client.ROOT.is_dir():
123 sys.exit("Run this test in the development VM with Code mounted.")
124 unittest.main()
tools/project-tools/install.py created+81
...@@ -0,0 +1,81 @@
1#!/usr/bin/env python3
2"""Stage or activate project command wrappers in the development VM."""
3
4import argparse
5import os
6from pathlib import Path
7import shutil
8import sys
9import time
10
11
12def main():
13 parser = argparse.ArgumentParser(description=__doc__)
14 parser.add_argument("action", choices=("stage", "activate"))
15 parser.add_argument("release")
16 args = parser.parse_args()
17 if sys.platform != "darwin" or not os.path.ismount("/Volumes/Code") or os.geteuid() == 0:
18 parser.error("Run this installer as the development VM user with Code mounted.")
19 if not args.release or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789-." for c in args.release):
20 parser.error("Use a release name containing lowercase letters, numbers, dots, or hyphens.")
21 local = Path.home() / ".local"
22 base = local / "share/project-tools"
23 release = base / "releases" / args.release
24 names = ("client.py", "pnpm", "jj", "project")
25 if args.action == "stage":
26 release.mkdir(parents=True)
27 for name in names:
28 shutil.copy2(Path(__file__).with_name(name), release / name)
29 print(release)
30 return
31 if any(not (release / name).is_file() for name in names):
32 parser.error("Stage this release before activating it.")
33 if not (local / "bin/jj-real").is_file():
34 parser.error("Install the native Jujutsu executable as ~/.local/bin/jj-real first.")
35 native_pnpm = local / "bin/pnpm-real"
36 if not native_pnpm.exists():
37 original = (local / "bin/pnpm").resolve(strict=True)
38 with original.open("rb") as binary:
39 magic = binary.read(4)
40 if magic != b"\xcf\xfa\xed\xfe":
41 parser.error("Preserve the native pnpm executable as ~/.local/bin/pnpm-real first.")
42 native_pnpm.symlink_to(original)
43 overlay = local / "share/agents/vm"
44 current = base / "current"
45 if not current.is_symlink():
46 previous = base / "releases" / ("before-recovery-" + time.strftime("%Y%m%d%H%M%S"))
47 previous.mkdir()
48 for name in ("jj", "project"):
49 shutil.copy2(overlay / name, previous / name)
50 (previous / "pnpm").symlink_to(native_pnpm.resolve())
51 shutil.copy2(release / "client.py", previous / "client.py")
52 (base / "previous").symlink_to(previous.relative_to(base))
53 elif current.resolve() != release.resolve():
54 previous = base / ".previous-next"
55 previous.symlink_to(os.readlink(current))
56 previous.replace(base / "previous")
57 link = base / ".current-next"
58 link.symlink_to(release.relative_to(base))
59 link.replace(current)
60 for name in names:
61 link = overlay / ("." + name + "-next")
62 link.symlink_to(os.path.relpath(current / name, overlay))
63 link.replace(overlay / name)
64 link = local / "bin/.pnpm-next"
65 link.symlink_to("../share/agents/vm/pnpm")
66 link.replace(local / "bin/pnpm")
67 for directory in (local / "share/vite-plus/package_manager/pnpm").glob("*/pnpm/bin"):
68 binary = directory / "pnpm"
69 if not binary.is_file():
70 continue
71 native = directory / "pnpm-real"
72 if not native.exists():
73 os.link(binary, native)
74 link = directory / ".pnpm-next"
75 link.symlink_to(os.path.relpath(overlay / "pnpm", directory))
76 link.replace(binary)
77 print(release)
78
79
80if __name__ == "__main__":
81 main()
tools/project-tools/jj created+136
...@@ -0,0 +1,136 @@
1#!/usr/bin/env python3
2"""Serialize Jujutsu commands across workspaces and NFS clients."""
3
4import errno
5import fcntl
6import json
7import os
8from pathlib import Path
9import subprocess
10import sys
11
12sys.path.insert(0, str(Path(__file__).resolve().parent))
13import client
14
15
16def remote(executable, args, path):
17 if not client.host():
18 return False
19 words, options, config_args = client.jj_arguments(args)
20 shared = path.is_relative_to(client.ROOT)
21 if Path.cwd().is_relative_to(client.ROOT) or shared:
22 for name, value in options:
23 if name in ("-R", "--repository", "--config-file"):
24 client.inside(value)
25 if not shared:
26 return False
27 client.inside(path)
28 if not words or words[0] in ("config", "git", "util", "help"):
29 return False
30 command = tuple(words[:2]) if words[0] in ("workspace", "file", "bookmark", "op", "sparse") else (words[0],)
31 safe = {("rebase",), ("status",), ("st",), ("log",), ("diff",), ("show",), ("edit",),
32 ("new",), ("abandon",), ("restore",), ("describe",), ("desc",), ("commit",), ("squash",),
33 ("workspace", "add"), ("workspace", "list"), ("workspace", "root"),
34 ("workspace", "forget"), ("workspace", "update-stale"),
35 ("file", "list"), ("file", "show"), ("file", "track"), ("file", "untrack"),
36 ("bookmark", "list"), ("bookmark", "create"), ("bookmark", "set"),
37 ("bookmark", "move"), ("bookmark", "delete"), ("bookmark", "forget"),
38 ("bookmark", "rename"), ("op", "log"), ("sparse", "list"), ("sparse", "set")}
39 if command not in safe:
40 return False
41 flags = {name for name, _ in options}
42 if flags & {"-i", "--interactive", "--tool", "--editor", "--edit", "--stdin", "--help", "-h"}:
43 return False
44 if command[0] in ("describe", "desc", "commit", "squash") and not flags & {
45 "-m", "--message", "--no-edit", "--use-destination-message"}:
46 return False
47 paths = client.jj_paths(words, options)
48 for value in paths:
49 if any(c in value for c in ':"|&~()'):
50 return False
51 client.inside(value)
52 base = [str(executable), *config_args, "--ignore-working-copy", "--no-pager", "config", "list"]
53 rows = subprocess.check_output([*base, "-T", 'json(self) ++ "\\n"'], text=True)
54 settings = {row["name"]: row["value"] for row in map(json.loads, rows.splitlines())}
55 if settings.get("signing.behavior", "drop") != "drop" or settings.get("snapshot.fsmonitor", "none") != "none":
56 return False
57 if command[0] in ("diff", "show", "log") and not flags & {"--git", "--summary", "--stat", "--types", "--name-only"}:
58 formatter = settings.get("ui.diff-formatter", ":color-words")
59 if not isinstance(formatter, str) or not formatter.startswith(":"):
60 return False
61 if sys.stdout.isatty() and "--no-pager" not in flags and settings.get("ui.paginate", "auto") != "never":
62 return False
63 config = subprocess.check_output([*base, "-T", 'name ++ " = " ++ value ++ "\\n"'], text=True)
64 version = subprocess.check_output([str(executable), "--version"], text=True).strip()
65 ignore = Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config"))) / "git/ignore"
66 cwd = Path.cwd()
67 if not cwd.is_relative_to(client.ROOT):
68 if any(not Path(value).is_absolute() for value in paths):
69 raise ValueError("Run commands with file paths from inside the project share.")
70 cwd = path
71 forwarded = ["-R", str(path)]
72 original = iter(args)
73 for arg in original:
74 if arg == "--":
75 forwarded.extend((arg, *original))
76 break
77 if arg in ("-R", "--repository", "--config-file", "--config"):
78 next(original)
79 elif arg.startswith(("-R", "--repository=", "--config-file=", "--config=")):
80 continue
81 else:
82 forwarded.append(arg)
83 if arg in client.VALUES:
84 forwarded.append(next(original))
85 result = client.request(dict(command="jj", cwd=str(cwd), args=forwarded, config=config, version=version,
86 ignore=ignore.read_text() if ignore.is_file() else ""), capture=False)
87 if result.returncode == 0 and command == ("workspace", "add") and len(paths) == 1:
88 client.wait_workspace(Path(paths[0]).absolute())
89 sys.exit(result.returncode)
90
91
92def main():
93 executable = Path(sys.argv[0]).absolute().with_name("jj-real")
94 path = Path.cwd()
95 _, options, _ = client.jj_arguments(sys.argv[1:])
96 for name, value in options:
97 if name in ("-R", "--repository"):
98 path = Path(value)
99 path = path.resolve()
100 remote(executable, sys.argv[1:], path)
101 for parent in (path, *path.parents):
102 metadata = parent / ".jj"
103 if not metadata.is_dir():
104 continue
105 repository = metadata / "repo"
106 if repository.is_file():
107 repository = (metadata / repository.read_text()).resolve(strict=True)
108 # Never unlink this inode: existing waiters must keep using the same lock.
109 descriptor = os.open(repository / "command.lock", os.O_CREAT | os.O_RDWR, 0o600)
110 # macOS NFS drops lockf across exec; flock maps to the server's POSIX lock.
111 lock = fcntl.flock if sys.platform == "darwin" else fcntl.lockf
112 try:
113 lock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
114 except OSError as error:
115 if error.errno not in (errno.EACCES, errno.EAGAIN):
116 raise
117 print("jj: Waiting for another command in this repository…", file=sys.stderr, flush=True)
118 lock(descriptor, fcntl.LOCK_EX)
119 # jj owns the lock even if its caller dies.
120 os.set_inheritable(descriptor, True)
121 break
122 os.execv(executable, ["jj", *sys.argv[1:]])
123
124
125if __name__ == "__main__":
126 try:
127 main()
128 except StopIteration:
129 os.execv(Path(sys.argv[0]).absolute().with_name("jj-real"), ["jj", *sys.argv[1:]])
130 except KeyboardInterrupt:
131 sys.exit(130)
132 except subprocess.CalledProcessError as error:
133 sys.exit(error.returncode)
134 except (OSError, ValueError) as error:
135 print(f"jj: {error}", file=sys.stderr)
136 sys.exit(1)
tools/project-tools/jj-test.py created+137
...@@ -0,0 +1,137 @@
1#!/usr/bin/env python3
2"""Exercise command locking with disposable repositories and real jj processes."""
3
4import fcntl
5import os
6from pathlib import Path
7import shutil
8import signal
9import subprocess
10import sys
11import tempfile
12import time
13import unittest
14
15
16lock_file = fcntl.flock if sys.platform == "darwin" else fcntl.lockf
17
18
19class CommandLockTest(unittest.TestCase):
20 def setUp(self):
21 temporary = tempfile.TemporaryDirectory(prefix="jj-lock-test-")
22 self.addCleanup(temporary.cleanup)
23 self.root = Path(temporary.name).resolve()
24 self.bin = self.root / "bin"
25 self.bin.mkdir()
26 shutil.copy2(Path(__file__).with_name("jj"), self.bin / "jj")
27 shutil.copy2(Path(__file__).with_name("client.py"), self.bin / "client.py")
28 native = Path(shutil.which("jj"))
29 if native.with_name("jj-real").exists():
30 native = native.with_name("jj-real")
31 (self.bin / "jj-real").symlink_to(native)
32 config = self.root / "config.toml"
33 config.write_text('[user]\nname="Fixture"\nemail="fixture@example.invalid"\n'
34 '[signing]\nbehavior="drop"\n')
35 self.env = dict(os.environ, JJ_CONFIG=str(config), JJ_EDITOR="true", PROJECT_STORE_SSH="",
36 GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=str(self.root / "absent"))
37 self.repo = self.root / "main"
38 self.run_jj("git", "init", "--no-colocate", str(self.repo))
39 self.other = self.root / "other"
40 self.run_jj("git", "init", "--no-colocate", str(self.other))
41 self.workspace = self.root / "agent workspace"
42 self.run_jj("-R", str(self.repo), "workspace", "add", str(self.workspace))
43 self.lock = self.repo / ".jj/repo/command.lock"
44
45 def run_jj(self, *args, cwd=None, check=True):
46 return subprocess.run([str(self.bin / "jj"), *args], cwd=cwd or self.root,
47 env=self.env, capture_output=True, text=True, timeout=10, check=check)
48
49 def start(self, *args, cwd=None):
50 process = subprocess.Popen([str(self.bin / "jj"), *args], cwd=cwd or self.root,
51 env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
52 text=True, start_new_session=True)
53 def cleanup():
54 if process.poll() is None:
55 os.killpg(process.pid, signal.SIGKILL)
56 process.communicate(timeout=5)
57 self.addCleanup(cleanup)
58 return process
59
60 def holder(self):
61 marker = self.root / "ready"
62 process = self.start("-R", str(self.repo), "util", "exec", "--", sys.executable, "-c",
63 "from pathlib import Path; import time; "
64 f"Path({str(marker)!r}).touch(); time.sleep(30)")
65 deadline = time.monotonic() + 10
66 while not marker.exists():
67 if process.poll() is not None or time.monotonic() > deadline:
68 self.fail("Holder did not start")
69 time.sleep(0.02)
70 with self.lock.open("r+") as lock:
71 with self.assertRaises(BlockingIOError):
72 lock_file(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
73 return process
74
75 def test_shared_workspace_lock_and_independent_repository(self):
76 holder = self.holder()
77 waiting = self.start("-R", str(self.workspace), "status")
78 time.sleep(0.2)
79 self.assertIsNone(waiting.poll())
80 self.run_jj("-R", str(self.other), "describe", "-m", "independent")
81 os.killpg(holder.pid, signal.SIGKILL)
82 stdout, stderr = waiting.communicate(timeout=10)
83 self.assertEqual(waiting.returncode, 0, stderr)
84 self.assertIn("Waiting for another command", stderr)
85 self.assertNotIn("Waiting", stdout)
86 self.assertFalse((self.workspace / ".jj/command.lock").exists())
87
88 def test_selectors_and_snapshotting_reads_wait(self):
89 subdir = self.workspace / "subdir"
90 subdir.mkdir()
91 link = self.root / "linked"
92 link.symlink_to(subdir)
93 commands = [
94 (["status"], link),
95 (["--repository=" + str(self.workspace), "diff"], self.root),
96 (["-R" + str(self.workspace), "log", "--no-graph", "-r", "@"], self.root),
97 (["--ignore-working-copy", "-R", str(self.workspace), "describe", "-m", "guarded"], self.root),
98 (["--config", 'user.name="-Rignored"', "-R", str(self.workspace), "status"], self.root),
99 ]
100 with self.lock.open("r+") as lock:
101 lock_file(lock, fcntl.LOCK_EX)
102 processes = [self.start(*args, cwd=cwd) for args, cwd in commands]
103 time.sleep(0.3)
104 self.assertTrue(all(process.poll() is None for process in processes))
105 for process in processes:
106 _, stderr = process.communicate(timeout=10)
107 self.assertEqual(process.returncode, 0, stderr)
108
109 def test_cancelled_waiter_and_failed_command_release(self):
110 with self.lock.open("r+") as lock:
111 lock_file(lock, fcntl.LOCK_EX)
112 waiting = self.start("-R", str(self.workspace), "status")
113 time.sleep(0.2)
114 waiting.send_signal(signal.SIGINT)
115 waiting.communicate(timeout=5)
116 self.assertEqual(waiting.returncode, 130)
117 self.assertNotEqual(self.run_jj("-R", str(self.repo), "unknown-command", check=False).returncode, 0)
118 self.run_jj("-R", str(self.workspace), "status")
119 self.assertEqual(self.run_jj("--version").returncode, 0)
120
121 def test_concurrent_mutations_have_one_operation_head(self):
122 processes = [self.start("-R", str(self.repo), "new", "root()", "-m", f"fixture {i}")
123 for i in range(12)]
124 for process in processes:
125 _, stderr = process.communicate(timeout=10)
126 self.assertEqual(process.returncode, 0, stderr)
127 self.assertNotIn("Concurrent modification", stderr)
128 heads = self.repo / ".jj/repo/op_heads/heads"
129 self.assertEqual(len(list(heads.iterdir())), 1)
130 descriptions = self.run_jj("-R", str(self.repo), "log", "--no-graph", "-r", "all()",
131 "-T", 'description ++ "\n"').stdout
132 for i in range(12):
133 self.assertIn(f"fixture {i}\n", descriptions)
134
135
136if __name__ == "__main__":
137 unittest.main()
tools/project-tools/platform-test.py created+48
...@@ -0,0 +1,48 @@
1#!/usr/bin/env python3
2"""Check storage-host platform selection before any native rebuild can repair it."""
3
4import io
5import json
6from pathlib import Path
7import sys
8import tarfile
9import tempfile
10import unittest
11
12import client
13
14
15class PlatformTest(unittest.TestCase):
16 def test_only_darwin_arm64_optional_packages_are_imported(self):
17 with tempfile.TemporaryDirectory(prefix="infra-platform-", dir=client.ROOT) as temporary:
18 workspace = Path(temporary) / "main"
19 workspace.mkdir()
20 optional = {}
21 for platform, cpu in (("darwin", "arm64"), ("linux", "x64")):
22 name = f"@project-platform-test/{platform}-{cpu}"
23 package = dict(name=name, version="1.0.0", os=[platform], cpu=[cpu])
24 content = json.dumps(package).encode()
25 archive = workspace / f"{platform}-{cpu}.tgz"
26 with tarfile.open(archive, "w:gz") as output:
27 entry = tarfile.TarInfo("package/package.json")
28 entry.size = len(content)
29 output.addfile(entry, io.BytesIO(content))
30 optional[name] = "file:" + archive.name
31 (workspace / "package.json").write_text(json.dumps(dict(
32 name="platform-fixture", private=True, packageManager="pnpm@12.14.0",
33 optionalDependencies=optional)))
34 response = client.request(dict(command="pnpm", cwd=str(workspace),
35 engine=client.engine(), args=["install", "--ignore-scripts"]))
36 self.assertEqual(response.returncode, 0)
37 result = json.loads(response.stdout)
38 self.assertIs(result.get("installed"), True, result)
39 client.finish_install(workspace, None, ignore_scripts=True, modules_hash=result["modules_hash"])
40 packages = workspace / "node_modules/@project-platform-test"
41 self.assertTrue((packages / "darwin-arm64/package.json").is_file())
42 self.assertFalse((packages / "linux-x64").exists())
43
44
45if __name__ == "__main__":
46 if sys.platform != "darwin" or not client.ROOT.is_dir():
47 sys.exit("Run this test in the development VM with Code mounted.")
48 unittest.main()
tools/project-tools/pnpm created+90
...@@ -0,0 +1,90 @@
1#!/usr/bin/env python3
2"""Install dependencies on the storage host; run programs and build hooks on Mac."""
3
4import json
5import os
6from pathlib import Path
7import re
8import subprocess
9import sys
10
11sys.path.insert(0, str(Path(__file__).resolve().parent))
12import client
13
14
15def main():
16 native = Path(sys.argv[0]).absolute().with_name("pnpm-real")
17 args = sys.argv[1:]
18 cwd, forwarded, command = client.pnpm_arguments(args, Path.cwd())
19 prepare = command in ("run", "run-script", "exec", "test", "t", "tst", "start", "stop", "restart")
20 if not prepare and command and cwd.is_relative_to(client.ROOT):
21 for parent in (cwd, *cwd.parents):
22 if parent == client.ROOT:
23 break
24 manifest = parent / "package.json"
25 if manifest.is_file():
26 prepare = command in json.loads(manifest.read_text()).get("scripts", {})
27 break
28 if forwarded or prepare:
29 custom_environment = [key for key, value in os.environ.items() if value and (
30 (key.lower().startswith(("npm_config_", "pnpm_config_"))
31 and key.lower() not in ("npm_config_prefix", "npm_config_cache", "npm_config_user_agent",
32 "pnpm_config_update_notifier"))
33 or key in ("VP_PACKAGE_MANAGER", "NODE_PATH")
34 or (key == "NODE_OPTIONS" and not re.fullmatch(
35 r"(?:\s*--max[-_](?:old|semi)[-_]space[-_]size=\d+)+\s*", value)))]
36 if custom_environment and cwd.is_relative_to(client.ROOT) and client.host():
37 print("pnpm: Using Mac installation: custom environment (" +
38 ", ".join(sorted(custom_environment)) + ").", file=sys.stderr)
39 if not custom_environment and cwd.is_relative_to(client.ROOT) and client.host():
40 client.inside(cwd)
41 request = dict(command="pnpm", cwd=str(cwd), engine=client.engine(), args=forwarded or [])
42 if os.environ.get("NODE_ENV") == "production":
43 request["node_env"] = "production"
44 if prepare:
45 request["prepare"] = True
46 else:
47 print("pnpm: Installing dependencies on the storage host.", file=sys.stderr, flush=True)
48 result = client.request(request)
49 if result.returncode:
50 return result.returncode
51 response = json.loads(result.stdout)
52 if response.get("ready") is True:
53 os.execv(native, [str(native), "--config.verify-deps-before-run=false", *args])
54 if response.get("installed") is True:
55 if prepare or "--lockfile-only" not in forwarded:
56 client.finish_install(Path(response.get("workspace", cwd)), native,
57 ignore_scripts=not prepare and "--ignore-scripts" in forwarded,
58 modules_hash=response.get("modules_hash"), cwd=cwd,
59 args=forwarded or ())
60 if prepare:
61 os.execv(native, [str(native), "--config.verify-deps-before-run=false", *args])
62 return 0
63 print("pnpm: Using Mac installation: " + response["reason"], file=sys.stderr)
64 store = response.get("store")
65 modules = cwd / "node_modules/.modules.yaml"
66 if modules.is_file():
67 try:
68 current = json.loads(modules.read_text())["storeDir"]
69 except (ValueError, KeyError):
70 pass
71 else:
72 if isinstance(current, str) and Path(current).is_absolute():
73 store = str(Path(current).parent)
74 if store:
75 args = [*args, "--store-dir=" + store]
76 os.execv(native, [str(native), *args])
77
78
79if __name__ == "__main__":
80 try:
81 sys.exit(main())
82 except StopIteration:
83 os.execv(Path(sys.argv[0]).absolute().with_name("pnpm-real"), ["pnpm", *sys.argv[1:]])
84 except KeyboardInterrupt:
85 sys.exit(130)
86 except subprocess.CalledProcessError as error:
87 sys.exit(error.returncode)
88 except (OSError, ValueError, KeyError) as error:
89 print(f"pnpm: {error}", file=sys.stderr)
90 sys.exit(1)
tools/project-tools/project created+184
...@@ -0,0 +1,184 @@
1#!/usr/bin/env python3
2"""Manage shared projects and run read-only tools beside their ZFS storage."""
3
4import argparse
5from contextlib import ExitStack
6import fcntl
7import json
8import os
9from pathlib import Path
10import re
11import subprocess
12import sys
13import time
14
15sys.path.insert(0, str(Path(__file__).resolve().parent))
16import client
17
18
19def component(value):
20 if (
21 not value
22 or value in (".", "..")
23 or value != value.strip()
24 or any(c in value for c in "/\\")
25 or any(ord(c) < 32 or ord(c) == 127 for c in value)
26 ):
27 raise argparse.ArgumentTypeError("Use one folder name without slashes or control characters.")
28 return value
29
30
31def main():
32 parser = argparse.ArgumentParser(
33 description=__doc__,
34 epilog="Prints the checkout path on success; jj progress goes to stderr. "
35 "PROJECT_ROOT overrides the project directory, which must already exist.",
36 )
37 commands = parser.add_subparsers(dest="command", required=True)
38 read = commands.add_parser("read", help="Search, list, or hash shared files on the storage host")
39 read.add_argument("tool", choices=("rg", "sha256sum"))
40 read.add_argument("args", nargs=argparse.REMAINDER)
41 clone = commands.add_parser("clone", help="Clone a Git repository into PROJECT/main")
42 clone.add_argument("source", metavar="URL")
43 clone.add_argument("project", type=component, metavar="PROJECT")
44 worktree = commands.add_parser("worktree", help="Create a workspace and install pnpm dependencies")
45 worktree.add_argument("project", type=component, metavar="PROJECT")
46 worktree.add_argument("identity", type=component, metavar="NAME")
47 worktree.add_argument("--from", dest="revision", default="trunk()", metavar="REV",
48 help="Parent revision, resolved in main (default: trunk())")
49 worktree.add_argument("--warm-from", type=component, metavar="WORKSPACE",
50 help="Clone an idle workspace and its build outputs on the ZFS host")
51 worktree.add_argument("--target", action="append", metavar="PATH",
52 help="Target directory relative to each workspace; repeat as needed (default: target)")
53 worktree.add_argument("--no-install", action="store_true", help="Create the workspace without installing dependencies")
54 args = parser.parse_args()
55
56 root = Path(os.environ.get("PROJECT_ROOT", "/Volumes/Code/project")).expanduser().absolute()
57 if "PROJECT_ROOT" not in os.environ and not os.path.ismount("/Volumes/Code"):
58 parser.error("/Volumes/Code is not mounted. Restore the mount before creating a checkout.")
59 if not root.is_dir():
60 parser.error(f"Project directory is missing: {root}")
61 if args.command == "read":
62 cwd = client.inside(Path.cwd())
63 if not client.host():
64 parser.error("Storage access is unavailable. Set PROJECT_STORE_SSH to its SSH alias.")
65 return client.request(dict(command="read", cwd=str(cwd), tool=args.tool, args=args.args),
66 capture=False).returncode
67 project = root / args.project
68 if project.is_symlink():
69 parser.error(f"Project directory is a symlink: {project}")
70 destination = project / ("main" if args.command == "clone" else args.identity)
71 if destination.exists() or destination.is_symlink():
72 parser.error(f"Checkout already exists: {destination}. Choose another name.")
73 if args.command == "clone":
74 if project.exists():
75 parser.error(f"Project already exists: {project}. Use project worktree or another name.")
76 command = ["jj", "git", "clone", "--no-colocate", "--", args.source, str(destination)]
77 else:
78 if args.identity in ("main", "default"):
79 parser.error("main and default are reserved. Choose another worktree name.")
80 primary = project / "main"
81 if primary.is_symlink() or not (primary / ".jj").is_dir():
82 parser.error(f"No primary Jujutsu checkout at {primary}. Clone the project first.")
83 command = ["jj", "-R", str(primary), "workspace", "add", "--name", args.identity,
84 "-r", args.revision, "--", str(destination)]
85
86 if "PROJECT_STORE_SSH" not in os.environ and (Path.home() / ".ssh/project-store").is_file():
87 os.environ["PROJECT_STORE_SSH"] = "project-store"
88 with ExitStack() as locks:
89 warm = args.command == "worktree" and args.warm_from
90 if args.command == "worktree" and args.target and not warm:
91 parser.error("--target requires --warm-from.")
92 if warm:
93 host = os.environ.get("PROJECT_STORE_SSH", "")
94 if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", host):
95 parser.error("Set PROJECT_STORE_SSH to the SSH alias for the project-store forced command.")
96 if os.environ.get("CARGO_TARGET_DIR") or os.environ.get("CARGO_BUILD_TARGET_DIR"):
97 parser.error("Unset the Cargo target override before seeding worktree-local targets.")
98 donor = project / args.warm_from
99 if donor.is_symlink() or not (donor / ".jj").is_dir():
100 parser.error(f"No donor workspace at {donor}.")
101 targets = args.target or ["target"]
102 for target in targets:
103 path = Path(target)
104 if (path.is_absolute() or not path.parts or
105 any(p in ("..", ".jj", ".git") for p in path.parts)):
106 parser.error("Use target paths inside the workspace.")
107 source = donor / path
108 if not source.is_dir() or source.resolve() != source.absolute():
109 parser.error(f"Target is missing or follows a symlink: {source}")
110 lock_paths = sorted(set(source.glob("*/.cargo-lock")) |
111 set(source.glob("*/*/.cargo-lock")))
112 if not lock_paths:
113 parser.error(f"No Cargo build locks in {source}. Build the donor first.")
114 for lock_path in lock_paths:
115 if lock_path.resolve() != lock_path.absolute():
116 parser.error(f"Cargo build lock follows a symlink: {lock_path}")
117 fd = os.open(lock_path, os.O_RDWR | os.O_NOFOLLOW)
118 locks.callback(os.close, fd)
119 try:
120 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
121 except BlockingIOError:
122 parser.error(f"Donor is building: {source}. Retry after the build finishes.")
123 return create(parser, args, project, destination, command, warm)
124
125
126def create(parser, args, project, destination, command, warm):
127 memory = Path.home() / ".local/share/agents/memory" / args.project
128 slug = re.sub(r"[^a-zA-Z0-9_-]", "-", str(destination))
129 memory_link = Path.home() / ".claude/projects" / slug / "memory"
130 if os.path.lexists(memory_link) and memory_link.resolve() != memory.resolve():
131 parser.error(f"Another memory directory owns {memory_link}. Resolve it before creating this checkout.")
132 if args.command == "clone":
133 project.mkdir()
134 host = os.environ.get("PROJECT_STORE_SSH")
135 if warm:
136 request = dict(project=args.project, source=args.warm_from, destination=args.identity,
137 revision=args.revision, targets=args.target or ["target"])
138 started = time.monotonic()
139 result = subprocess.run(["ssh", "-T", "-o", "BatchMode=yes", "--", host],
140 input=json.dumps(request), text=True, stdout=subprocess.PIPE)
141 if result.returncode:
142 return result.returncode
143 print(f"project: Host setup finished in {time.monotonic() - started:.1f}s.", file=sys.stderr)
144 try:
145 response = json.loads(result.stdout)
146 if not isinstance(response, dict) or response.get("workspace") != args.identity:
147 raise ValueError()
148 except (ValueError, TypeError):
149 print(f"project: Invalid storage response. Inspect {destination} before building.", file=sys.stderr)
150 return 1
151 client.wait_workspace(destination)
152 else:
153 result = subprocess.run(command, stdout=sys.stderr)
154 if result.returncode:
155 if args.command == "clone":
156 try:
157 project.rmdir()
158 except OSError:
159 pass
160 return result.returncode
161
162 memory.mkdir(parents=True, exist_ok=True, mode=0o700)
163 if not (memory / "MEMORY.md").exists():
164 (memory / "MEMORY.md").touch(mode=0o600)
165 memory_link.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
166 if not os.path.lexists(memory_link):
167 memory_link.symlink_to(os.path.relpath(memory, memory_link.parent), target_is_directory=True)
168 if args.command == "worktree" and not args.no_install:
169 manifest = destination / "package.json"
170 if manifest.is_file():
171 package = json.loads(manifest.read_text())
172 if package.get("packageManager", "").startswith("pnpm@"):
173 subprocess.run(["pnpm", "install", "--frozen-lockfile"], cwd=destination,
174 stdout=sys.stderr, check=True)
175 print(destination)
176 return 0
177
178
179if __name__ == "__main__":
180 try:
181 sys.exit(main())
182 except (OSError, ValueError, subprocess.CalledProcessError) as error:
183 print(f"project: {error}", file=sys.stderr)
184 sys.exit(1)
tools/project-tools/test.py created+284
...@@ -0,0 +1,284 @@
1#!/usr/bin/env python3
2"""Exercise routing with real Jujutsu repositories and a local SSH stand-in."""
3
4import hashlib
5import fcntl
6import json
7import os
8from pathlib import Path
9import shutil
10import subprocess
11import sys
12import tempfile
13import threading
14import time
15import unittest
16import client
17
18
19HERE = Path(__file__).resolve().parent
20
21
22class RoutingTest(unittest.TestCase):
23 def setUp(self):
24 temporary = tempfile.TemporaryDirectory(prefix="project-routing-")
25 self.addCleanup(temporary.cleanup)
26 self.root = Path(temporary.name).resolve()
27 self.projects = self.root / "project"
28 self.projects.mkdir()
29 self.bin = self.root / "bin"
30 self.bin.mkdir()
31 for name in ("jj", "pnpm", "project", "client.py"):
32 shutil.copy2(HERE / name, self.bin / name)
33 native = Path(shutil.which("jj"))
34 if native.with_name("jj-real").exists():
35 native = native.with_name("jj-real")
36 (self.bin / "jj-real").symlink_to(native)
37 self.requests = self.root / "requests"
38 self.native = self.root / "native"
39 (self.bin / "pnpm-real").write_text(f'''#!{sys.executable}
40import json, pathlib, sys
41pathlib.Path({str(self.native)!r}).write_text(json.dumps(sys.argv[1:]))
42print("native output")
43sys.exit(23)
44''')
45 (self.bin / "pnpm-real").chmod(0o755)
46 (self.bin / "ssh").write_text(f'''#!{sys.executable}
47import json, os, pathlib, subprocess, sys
48request = json.load(sys.stdin)
49with open({str(self.requests)!r}, "a") as log: log.write(json.dumps(request) + "\\n")
50if request["command"] == "pnpm":
51 print(json.dumps({{"ready": True}} if request.get("prepare") else {{"installed": False, "reason": "fixture miss"}}))
52 sys.exit(0)
53sys.exit(subprocess.run([{sys.executable!r}, {str(HERE.parent / "project-store.py")!r}, {str(self.projects)!r}],
54 input=json.dumps(request), text=True, env=dict(os.environ, PROJECT_STORE_SSH="")).returncode)
55''')
56 (self.bin / "ssh").chmod(0o755)
57 config = self.root / "jj.toml"
58 config.write_text('[user]\nname="Guest Identity"\nemail="guest@example.invalid"\n'
59 '[signing]\nbehavior="drop"\n[ui]\ndiff-formatter=":git"\n')
60 self.env = dict(os.environ, PATH=str(self.bin) + os.pathsep + os.environ["PATH"],
61 JJ_CONFIG=str(config), PROJECT_ROOT=str(self.projects), PROJECT_STORE_SSH="fixture",
62 GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=str(self.root / "absent"))
63 for key in list(self.env):
64 if key.lower().startswith(("npm_config_", "pnpm_config_")) or key in (
65 "NODE_ENV", "NODE_OPTIONS", "NODE_PATH", "VP_PACKAGE_MANAGER"):
66 self.env.pop(key)
67 self.main = self.projects / "Example/main"
68 self.main.parent.mkdir()
69 self.run_tool("jj-real", "git", "init", "--no-colocate", str(self.main), cwd=self.root)
70 (self.main / "source").write_text("base\n")
71 self.run_tool("jj-real", "describe", "-m", "base")
72
73 def run_tool(self, tool, *args, cwd=None, check=True):
74 result = subprocess.run([str(self.bin / tool), *args], cwd=cwd or self.main,
75 env=self.env, text=True, capture_output=True, timeout=15)
76 if check:
77 self.assertEqual(result.returncode, 0, result.stderr)
78 return result
79
80 def test_workspace_and_mutations_keep_paths_identity_and_dirty_content(self):
81 destination = self.main.parent / "agent space"
82 result = self.run_tool("jj", "workspace", "add", "--", str(destination))
83 self.assertIn("agent space", result.stderr)
84 (destination / "dirty").write_text("unsaved agent work\n")
85 self.run_tool("jj", "new", "-m", "child", cwd=destination)
86 self.run_tool("jj", "rebase", "-s", "@-", "-d", "root()", cwd=destination)
87 self.assertEqual((destination / "dirty").read_text(), "unsaved agent work\n")
88 author = self.run_tool("jj", "log", "-r", "@", "--no-graph", "-T", "author.email()", cwd=destination)
89 self.assertEqual(author.stdout, "guest@example.invalid")
90 self.assertEqual(len(self.requests.read_text().splitlines()), 4)
91 self.assertFalse(Path((destination / ".jj/repo").read_text()).is_absolute())
92
93 def test_paths_cannot_escape_share_even_through_symlinks(self):
94 outside = self.root / "outside"
95 outside.mkdir()
96 (self.main / "escape").symlink_to(outside)
97 for args in [("workspace", "add", str(outside / "new")),
98 ("workspace", "add", "../../../../escape"),
99 ("workspace", "add", "escape/new"),
100 ("-R", str(outside), "status"),
101 ("file", "show", str(outside / "file"))]:
102 with self.subTest(args=args):
103 result = self.run_tool("jj", *args, check=False)
104 self.assertNotEqual(result.returncode, 0)
105 self.assertIn("outside the project share", result.stderr)
106 self.assertFalse(self.requests.exists())
107
108 def test_remote_errors_preserve_exit_status_without_local_retry(self):
109 result = self.run_tool("jj", "rebase", "-r", "missing-revision", "-d", "root()", check=False)
110 self.assertNotEqual(result.returncode, 0)
111 self.assertIn("missing-revision", result.stderr)
112 self.assertEqual(len(self.requests.read_text().splitlines()), 1)
113
114 def test_relative_repository_selection_from_outside_the_share(self):
115 destination = self.main.parent / "selected"
116 result = self.run_tool("jj", "-R", str(self.main.relative_to(self.root)),
117 "--config", 'user.email="override@example.invalid"',
118 "workspace", "add", "-r", "@", str(destination), cwd=self.root)
119 self.assertEqual(result.returncode, 0)
120 author = self.run_tool("jj", "log", "--no-graph", "-r", "@", "-T", "author.email()", cwd=destination)
121 self.assertEqual(author.stdout, "override@example.invalid")
122
123 def test_local_repositories_and_interactive_commands_stay_local(self):
124 local = self.root / "local"
125 self.run_tool("jj-real", "git", "init", "--no-colocate", str(local))
126 self.run_tool("jj", "status", cwd=local)
127 self.env["JJ_EDITOR"] = "true"
128 self.run_tool("jj", "describe")
129 self.assertFalse(self.requests.exists())
130
131 def test_pnpm_scripts_and_unknown_options_keep_native_argv_and_exit(self):
132 for args in [("run", "build"), ("exec", "node", "-p", "process.platform"),
133 ("build",), ("install", "--config.ignore-scripts=true"),
134 ("--version",), ("run", "install")]:
135 with self.subTest(args=args):
136 result = self.run_tool("pnpm", *args, check=False)
137 self.assertEqual(result.returncode, 23)
138 self.assertEqual(result.stdout, "native output\n")
139 prefix = ["--config.verify-deps-before-run=false"] if args[0] in ("run", "exec") else []
140 self.assertEqual(json.loads(self.native.read_text()), prefix + list(args))
141 requests = [json.loads(line) for line in self.requests.read_text().splitlines()]
142 self.assertTrue(all(request["prepare"] and request["args"] == [] for request in requests))
143
144 def test_script_shortcut_checks_dependencies_remotely_then_runs_natively(self):
145 (self.main / "package.json").write_text(json.dumps({"scripts": {"build": "never run on Linux"}}))
146 result = self.run_tool("pnpm", "build", "--flag", check=False)
147 self.assertEqual(result.returncode, 23)
148 self.assertEqual(json.loads(self.requests.read_text())["args"], [])
149 self.assertEqual(json.loads(self.native.read_text()), ["--config.verify-deps-before-run=false", "build", "--flag"])
150
151 def test_handoff_waits_for_the_servers_modules_manifest(self):
152 modules = self.main / "node_modules/.modules.yaml"
153 modules.parent.mkdir()
154 modules.write_text('{"pendingBuilds":[]}')
155 content = b'{"pendingBuilds":["new-native-package"]}'
156 def publish():
157 time.sleep(0.15)
158 temporary = modules.with_suffix(".next")
159 temporary.write_bytes(content)
160 temporary.replace(modules)
161 publisher = threading.Thread(target=publish)
162 publisher.start()
163 self.addCleanup(publisher.join)
164 client.finish_install(self.main, self.bin / "pnpm-real", ignore_scripts=True,
165 modules_hash=hashlib.sha256(content).hexdigest())
166 self.assertEqual(modules.read_bytes(), content)
167
168 def test_workspace_handoff_waits_for_metadata_visibility(self):
169 destination = self.main.parent / "new workspace"
170 def publish():
171 time.sleep(0.15)
172 (destination / ".jj/working_copy").mkdir(parents=True)
173 publisher = threading.Thread(target=publish)
174 publisher.start()
175 self.addCleanup(publisher.join)
176 client.wait_workspace(destination)
177 self.assertTrue((destination / ".jj/working_copy").is_dir())
178
179 def test_install_miss_falls_back_once_with_exact_arguments(self):
180 self.env.update(npm_config_prefix=str(self.root / "local"), npm_config_cache=str(self.root / "cache"),
181 PNPM_CONFIG_UPDATE_NOTIFIER="false")
182 args = ["--dir", str(self.main), "install", "--frozen-lockfile", "--offline"]
183 result = self.run_tool("pnpm", *args, check=False)
184 self.assertEqual(result.returncode, 23)
185 self.assertIn("fixture miss", result.stderr)
186 self.assertEqual(json.loads(self.native.read_text()), args)
187 self.assertEqual(json.loads(self.requests.read_text())["args"], ["install", "--frozen-lockfile", "--offline"])
188
189 def test_dependency_changes_and_filters_use_storage_before_native_hooks(self):
190 for args in [("add", "-D", "is-number@7.0.0"), ("update", "is-number"),
191 ("remove", "is-number"), ("--filter", "app", "install", "--lockfile-only")]:
192 with self.subTest(args=args):
193 result = self.run_tool("pnpm", *args, check=False)
194 self.assertEqual(result.returncode, 23)
195 request = json.loads(self.requests.read_text().splitlines()[-1])
196 self.assertEqual(request["args"], list(args))
197 self.assertRegex(request["engine"], r";node\d+\.\d+\.\d+$")
198
199 def test_install_miss_keeps_the_existing_store(self):
200 modules = self.main / "node_modules/.modules.yaml"
201 modules.parent.mkdir()
202 store = self.root / "cache/v10"
203 modules.write_text(json.dumps({"storeDir": str(store)}))
204 result = self.run_tool("pnpm", "install", check=False)
205 self.assertEqual(result.returncode, 23)
206 self.assertEqual(json.loads(self.native.read_text()), ["install", "--store-dir=" + str(store.parent)])
207
208 def test_custom_install_environment_stays_native(self):
209 for key, value in (("npm_config_ignore_scripts", "true"),
210 ("PNPM_CONFIG_STORE_DIR", str(self.root / "custom-store")),
211 ("VP_PACKAGE_MANAGER", "pnpm@12.15.0"),
212 ("NODE_OPTIONS", "--require=/private/agent-hook.cjs"),
213 ("NODE_PATH", "/private/agent-modules")):
214 with self.subTest(key=key):
215 self.env[key] = value
216 result = self.run_tool("pnpm", "install", check=False)
217 self.assertEqual(result.returncode, 23)
218 self.assertEqual(json.loads(self.native.read_text()), ["install"])
219 self.assertIn("Using Mac installation: custom environment (" + key + ")", result.stderr)
220 self.assertNotIn(value, result.stderr)
221 self.env.pop(key)
222 self.assertFalse(self.requests.exists())
223
224 def test_inherited_metadata_and_memory_limits_still_offload(self):
225 self.env["npm_config_user_agent"] = "pnpm/12.14.0 npm/? node/v26.12.0 darwin arm64"
226 for options in ("--max-old-space-size=8192", "--max_old_space_size=8192",
227 "--max-old-space-size=8192 --max-semi-space-size=32"):
228 with self.subTest(options=options):
229 self.env["NODE_OPTIONS"] = options
230 result = self.run_tool("pnpm", "install", "--frozen-lockfile", check=False)
231 self.assertEqual(result.returncode, 23)
232 self.assertIn("fixture miss", result.stderr)
233 self.assertNotIn("custom environment", result.stderr)
234 request = json.loads(self.requests.read_text().splitlines()[-1])
235 self.assertEqual(request["args"], ["install", "--frozen-lockfile"])
236
237 def test_production_environment_reaches_pnpm_without_changing_arguments(self):
238 self.env["NODE_ENV"] = "production"
239 for args in (("install",), ("install", "--dev"), ("add", "is-number@7.0.0"), ("exec", "node")):
240 with self.subTest(args=args):
241 result = self.run_tool("pnpm", *args, check=False)
242 self.assertEqual(result.returncode, 23)
243 request = json.loads(self.requests.read_text().splitlines()[-1])
244 self.assertEqual(request["node_env"], "production")
245 self.assertEqual(request["args"], [] if args[0] == "exec" else list(args))
246
247 def test_storage_reads_preserve_paths_output_and_ignore_files_without_snapshotting(self):
248 (self.main / "file with spaces").write_text("needle\n")
249 (self.main / ".gitignore").write_text("node_modules/\n")
250 (self.main / "node_modules").mkdir()
251 (self.main / "node_modules/ignored").write_text("needle\n")
252 with (self.main / ".jj/repo/command.lock").open("w") as lock:
253 fcntl.flock(lock, fcntl.LOCK_EX)
254 listed = self.run_tool("project", "read", "rg", "--files")
255 self.assertIn("file with spaces", listed.stdout.splitlines())
256 self.assertNotIn("node_modules/ignored", listed.stdout)
257 result = self.run_tool("project", "read", "rg", "--json", "needle", "file with spaces")
258 matches = [json.loads(line) for line in result.stdout.splitlines()]
259 self.assertEqual(next(row for row in matches if row["type"] == "match")
260 ["data"]["path"]["text"], "file with spaces")
261 for pattern, status in (("not present", 1), ("[", 2)):
262 result = self.run_tool("project", "read", "rg", pattern, check=False)
263 self.assertEqual(result.returncode, status)
264 requests = [json.loads(line) for line in self.requests.read_text().splitlines()]
265 self.assertTrue(all(request["command"] == "read" for request in requests))
266
267 def test_storage_hashes_match_content_and_reject_external_paths_and_commands(self):
268 content = b"fixture" * 1000
269 (self.main / "archive with spaces.zip").write_bytes(content)
270 result = self.run_tool("project", "read", "sha256sum", "archive with spaces.zip")
271 self.assertEqual(result.stdout, hashlib.sha256(content).hexdigest() + " archive with spaces.zip\n")
272 outside = self.root / "private"
273 outside.write_text("not shared")
274 (self.main / "escape").symlink_to(outside)
275 for args in (("sha256sum", str(outside)), ("sha256sum", "escape"),
276 ("sha256sum", "-"), ("sha256sum", "--check", "archive with spaces.zip"),
277 ("rg", "--pre=touch marker", "needle"), ("sh", "-c", "touch marker")):
278 with self.subTest(args=args):
279 self.assertNotEqual(self.run_tool("project", "read", *args, check=False).returncode, 0)
280 self.assertFalse((self.main / "marker").exists())
281
282
283if __name__ == "__main__":
284 unittest.main()