diff --git a/tools/mac-agent-forward-test.py b/tools/mac-agent-forward-test.py new file mode 100644 index 0000000000000000000000000000000000000000..6fb89ea874678bc50026ef70b0351e60d21be644 --- /dev/null +++ b/tools/mac-agent-forward-test.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Check loopback forwarding against a disposable HTTP server in the live VM.""" + +import hashlib +from importlib import import_module +from pathlib import Path +import shlex +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request + + +vm = import_module("mac-agent-vm") + + +def main(): + payload = b"forwarding fixture\n" * 40000 + code = '''from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import sys, threading +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"forwarding fixture\\n" * 40000 + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): pass +server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) +threading.Thread(target=server.serve_forever, daemon=True).start() +print(server.server_port, flush=True) +sys.stdin.buffer.read() +server.shutdown() +server.server_close() +''' + ip = vm.tart("ip", vm.NAME, capture_output=True, text=True, timeout=5).stdout.strip() + server = subprocess.Popen([*vm.ssh_args(), "admin@" + ip, "python3 -u -c " + shlex.quote(code)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + tunnel = None + try: + port = int(server.stdout.readline()) + with socket.socket() as available: + available.bind(("127.0.0.1", 0)) + local_port = available.getsockname()[1] + command = [sys.executable, str(Path(__file__).with_name("mac-agent-vm.py")), + "forward", str(port), "--local-port", str(local_port)] + tunnel = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + url = f"http://127.0.0.1:{local_port}/fixture" + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + deadline = time.monotonic() + 15 + while True: + if tunnel.poll() is not None: + raise AssertionError(tunnel.communicate()[1]) + try: + with opener.open(url, timeout=2) as response: + actual = response.read() + break + except urllib.error.URLError: + if time.monotonic() >= deadline: + raise + time.sleep(0.05) + assert hashlib.sha256(actual).digest() == hashlib.sha256(payload).digest() + collision = subprocess.run(command, capture_output=True, text=True, timeout=15) + assert collision.returncode != 0, "A second tunnel silently reused an occupied port." + with opener.open(url, timeout=2) as response: + assert response.read() == payload, "Port collision disturbed the first tunnel." + tunnel.terminate() + tunnel.communicate(timeout=5) + try: + opener.open(url, timeout=2) + except urllib.error.URLError: + pass + else: + raise AssertionError("The tunnel listener survived its process.") + print(f"Forwarded {len(payload)} bytes intact; collision preserved the live tunnel; shutdown closed its listener.") + finally: + if tunnel is not None and tunnel.poll() is None: + tunnel.terminate() + tunnel.communicate(timeout=5) + server.stdin.close() + server.wait(timeout=10) + server.stdout.close() + + +if __name__ == "__main__": + main() diff --git a/tools/mac-agent-vm.md b/tools/mac-agent-vm.md index 4e712efae1249af79b2f30819ae11681df3a878a..3f7e032e77c9a067b19846045657e7f9f8ac8fd6 100644 --- a/tools/mac-agent-vm.md +++ b/tools/mac-agent-vm.md @@ -23,6 +23,19 @@ python3 tools/mac-agent-vm.py console python3 tools/mac-agent-vm.py stop ``` +To reach a server bound to guest loopback from this Mac: + +```sh +python3 tools/mac-agent-vm.py forward 5173 --local-port 15173 +``` + +Open `http://127.0.0.1:15173` on the Mac. The tunnel stays in the foreground; +Ctrl-C closes it. Both ends use loopback, occupied ports fail explicitly, and +no guest command or LAN listener is created. T3's browser separately requires +its toolbar's port approval. Run `python3 tools/mac-agent-forward-test.py` to +check a disposable guest server, a transfer larger than 512 KiB, port collision, +and listener shutdown. This test uses the existing guest without changing its data. + `start` resumes a suspended VM or boots a stopped VM, then waits for commands to work through the logged-in guest agent. `stop` requests a graceful guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM diff --git a/tools/mac-agent-vm.py b/tools/mac-agent-vm.py index cc4cd92801f025c99cb84ee5b987404c1f8c2750..c92206e4a00d19cf3cc3ea58a688795458c0cbaf 100644 --- a/tools/mac-agent-vm.py +++ b/tools/mac-agent-vm.py @@ -210,10 +210,23 @@ def main(): for action in ["exec", "ssh"]: command = sub.add_parser(action) command.add_argument("command", nargs=argparse.REMAINDER) + forward = sub.add_parser("forward", help="Reach a guest loopback server from this Mac") + forward.add_argument("port", type=int, help="Guest server port") + forward.add_argument("--local-port", type=int, help="Host loopback port (default: same as guest)") viewer = sub.add_parser("console") viewer.add_argument("--listen", default="127.0.0.1") viewer.add_argument("--port", type=int, default=17891) args = parser.parse_args() + if args.action == "forward": + local_port = args.local_port if args.local_port is not None else args.port + if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535: + parser.error("Ports must be between 1 and 65535.") + ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip() + print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.", + file=sys.stderr, flush=True) + os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes", + "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2", + "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", "admin@" + ip]) if args.action in {"start", "stop", "suspend", "reboot"}: lock = (PRIVATE / "lifecycle.lock").open("a") os.chmod(PRIVATE / "lifecycle.lock", 0o600) diff --git a/tools/project-pnpm/pnpm-10.32.1.patch b/tools/project-pnpm/pnpm-10.32.1.patch new file mode 100644 index 0000000000000000000000000000000000000000..95ac5422cec9207d3de11f32361e3be152f9a899 --- /dev/null +++ b/tools/project-pnpm/pnpm-10.32.1.patch @@ -0,0 +1,102 @@ +--- pnpm.cjs ++++ pnpm-darwin-rebuild.cjs +@@ -128122,7 +128122,7 @@ + var dependency_path_1 = require_lib87(); + var crypto_object_hasher_1 = require_lib119(); + function calcDepState(depsGraph, cache, depPath, opts) { +- let result2 = constants_1.ENGINE_NAME; ++ let result2 = opts.includeDepGraphHash ? "darwin;arm64;node26" : constants_1.ENGINE_NAME; + if (opts.includeDepGraphHash) { + const depGraphHash = calcDepGraphHash(depsGraph, cache, /* @__PURE__ */ new Set(), depPath); + result2 += `;deps=${depGraphHash}`; +@@ -130347,12 +130347,6 @@ + storeController: store.ctrl, + unsafePerm: opts.unsafePerm || false + }; +- await (0, lifecycle_1.runLifecycleHooksConcurrently)(["preinstall", "install", "postinstall", "prepublish", "prepare"], Object.values(ctx.projects), opts.childConcurrency || 5, scriptsOpts); +- for (const { id, manifest } of Object.values(ctx.projects)) { +- if (manifest?.scripts != null && (!opts.pending || ctx.pendingBuilds.includes(id))) { +- ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(id), 1); +- } +- } + await (0, modules_yaml_1.writeModulesManifest)(ctx.rootModulesDir, { + prunedAt: (/* @__PURE__ */ new Date()).toUTCString(), + ...ctx.modulesFile, +@@ -130445,10 +130439,12 @@ + } else { + extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts.lockfileDir)); + } ++ if (!allowBuild(pkgInfo.name, pkgInfo.version, depPath)) { pkgsThatWereRebuilt.add(depPath); return; } + const resolution = pkgSnapshot.resolution; + let sideEffectsCacheKey; + const pkgId = `${pkgInfo.name}@${pkgInfo.version}`; +- if (opts.skipIfHasSideEffectsCache && resolution.integrity) { ++ if (!["esbuild@0.28.0", "sharp@0.34.5", "msgpackr-extract@3.0.3", "@parcel/watcher@2.5.6"].includes(pkgId)) throw new Error("Review this package before caching its hooks: " + pkgId); ++ if (resolution.integrity) { + const filesIndexFile = (0, store_cafs_1.getIndexFilePathInCafs)(opts.storeDir, resolution.integrity.toString(), pkgId); + let pkgFilesIndex; + try { +@@ -130460,58 +130456,16 @@ + includeDepGraphHash: true + }); + if (pkgFilesIndex.sideEffects?.[sideEffectsCacheKey]) { ++ const delta = pkgFilesIndex.sideEffects[sideEffectsCacheKey]; ++ require("/opt/project-store/restore.cjs")(pkgRoot, ++ Object.entries(delta.added ?? {}).map(([rel, info]) => [rel, store_cafs_1.getFilePathByModeInCafs(opts.storeDir, info.integrity, info.mode), info.mode]), delta.deleted ?? []); ++ process.stderr.write(JSON.stringify({cachePackage:pkgId,cacheKey:sideEffectsCacheKey,restored:Object.keys(delta.added ?? {}).length})+"\n"); + pkgsThatWereRebuilt.add(depPath); + return; + } + } + } +- let requiresBuild = true; +- const pgkManifest = await (0, read_package_json_1.safeReadPackageJsonFromDir)(pkgRoot); +- if (pgkManifest != null) { +- requiresBuild = (0, exec_pkg_requires_build_1.pkgRequiresBuild)(pgkManifest, {}); +- } +- const hasSideEffects = requiresBuild && allowBuild(pkgInfo.name, pkgInfo.version, depPath) && await (0, lifecycle_1.runPostinstallHooks)({ +- depPath, +- extraBinPaths, +- extraEnv: opts.extraEnv, +- optional: pkgSnapshot.optional === true, +- pkgRoot, +- rawConfig: opts.rawConfig, +- rootModulesDir: ctx.rootModulesDir, +- scriptsPrependNodePath: opts.scriptsPrependNodePath, +- shellEmulator: opts.shellEmulator, +- unsafePerm: opts.unsafePerm || false +- }); +- if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) { +- builtDepPaths.add(depPath); +- const filesIndexFile = (0, store_cafs_1.getIndexFilePathInCafs)(opts.storeDir, resolution.integrity.toString(), pkgId); +- try { +- if (!sideEffectsCacheKey) { +- sideEffectsCacheKey = (0, calc_dep_state_1.calcDepState)(depGraph, depsStateCache, depPath, { +- includeDepGraphHash: true +- }); +- } +- await opts.storeController.upload(pkgRoot, { +- sideEffectsCacheKey, +- filesIndexFile +- }); +- } catch (err) { +- (0, assert_1.default)(util_1.default.types.isNativeError(err)); +- if ("statusCode" in err && err.statusCode === 403) { +- logger_1.logger.warn({ +- message: `The store server disabled upload requests, could not upload ${pkgRoot}`, +- prefix: opts.lockfileDir +- }); +- } else { +- logger_1.logger.warn({ +- error: err, +- message: `An error occurred while uploading ${pkgRoot}`, +- prefix: opts.lockfileDir +- }); +- } +- } +- } +- pkgsThatWereRebuilt.add(depPath); ++ throw new Error("Missing target cache for " + pkgId); + } catch (err) { + (0, assert_1.default)(util_1.default.types.isNativeError(err)); + if (pkgSnapshot.optional) { diff --git a/tools/project-pnpm/pnpm-11.10.0.patch b/tools/project-pnpm/pnpm-11.10.0.patch new file mode 100644 index 0000000000000000000000000000000000000000..2c25d74ecb343fbd143a6be1657ceff8038ba871 --- /dev/null +++ b/tools/project-pnpm/pnpm-11.10.0.patch @@ -0,0 +1,105 @@ +--- pnpm.mjs ++++ pnpm-target-cache.mjs +@@ -155612,7 +155612,7 @@ + } + function calcDepState(depsGraph, cache, depPath, opts3) { + const ownPin = readSnapshotRuntimePin(depsGraph[depPath]?.children); +- let result2 = engineName(ownPin ?? opts3.nodeVersion); ++ let result2 = opts3.includeDepGraphHash ? "darwin;arm64;node26" : engineName(ownPin ?? opts3.nodeVersion); + if (opts3.includeDepGraphHash) { + const depGraphHash = calcDepGraphHash(depsGraph, cache, /* @__PURE__ */ new Set(), depPath, opts3.supportedArchitectures); + result2 += `;deps=${depGraphHash}`; +@@ -163136,12 +163136,7 @@ + unsafePerm: opts3.unsafePerm || false, + userAgent: opts3.userAgent + }; +- await runLifecycleHooksConcurrently(["preinstall", "install", "postinstall", "prepublish", "prepare"], Object.values(ctx.projects), opts3.childConcurrency || 5, scriptsOpts); +- for (const { id, manifest } of Object.values(ctx.projects)) { +- if (manifest?.scripts != null && (!opts3.pending || ctx.pendingBuilds.includes(id))) { +- ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(id), 1); +- } +- } ++ process.stderr.write(JSON.stringify({guestPending:ctx.pendingBuilds})+"\n"); + await writeModulesManifest(ctx.rootModulesDir, { + prunedAt: (/* @__PURE__ */ new Date()).toUTCString(), + ...ctx.modulesFile, +@@ -163216,7 +163211,7 @@ + return false; + }; + const builtDepPaths = /* @__PURE__ */ new Set(); +- const storeIndex = opts3.skipIfHasSideEffectsCache ? opts3.frozenStore ? new ReadOnlyStoreIndex(opts3.storeDir) : new StoreIndex(opts3.storeDir) : void 0; ++ const storeIndex = new ReadOnlyStoreIndex(opts3.storeDir); + const gvsDirByDepPath = /* @__PURE__ */ new Map(); + if (opts3.enableGlobalVirtualStore) { + const globalVirtualStoreDir = opts3.globalVirtualStoreDir ?? path92.join(opts3.storeDir, "links"); +@@ -163268,10 +163263,12 @@ + } else { + extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts3.lockfileDir)); + } ++ if (!allowBuild(depPath)) { pkgsThatWereRebuilt.add(depPath); return; } + const resolution = pkgSnapshot.resolution; + let sideEffectsCacheKey; + const pkgId = pkgInfo.nonSemverVersion ?? `${pkgInfo.name}@${pkgInfo.version}`; +- if (opts3.skipIfHasSideEffectsCache && (resolution.gitHosted || resolution.integrity)) { ++ if (!["esbuild@0.28.2", "esbuild@0.25.12", "tsc-rs@0.7.0", "node-pty@1.2.0-beta.15", "file:apps/mobile/deps/react-native-enriched-markdown-1.2.0-t3.11.tgz"].includes(pkgId)) throw new Error("Review this package before caching its hooks: " + pkgId); ++ if (resolution.gitHosted || resolution.integrity) { + const filesIndexFile = pickStoreIndexKey(resolution, pkgId, { built: true }); + const pkgFilesIndex = storeIndex.get(filesIndexFile); + if (pkgFilesIndex) { +@@ -163281,6 +163278,13 @@ + nodeVersion + }); + if (pkgFilesIndex.sideEffects?.has(sideEffectsCacheKey)) { ++ const delta = pkgFilesIndex.sideEffects.get(sideEffectsCacheKey); ++ const restore = (await import("file:///opt/project-store/restore.cjs")).default; ++ restore(pkgRoot, [...(delta.added ?? [])].map(([rel, info]) => { ++ const digest = typeof info.digest === "string" ? info.digest : Buffer.from(info.digest).toString("hex"); ++ return [rel, getFilePathByModeInCafs(opts3.storeDir, digest, info.mode), info.mode]; ++ }), delta.deleted ?? []); ++ process.stderr.write(JSON.stringify({cachePackage:pkgId,cacheKey:sideEffectsCacheKey,restored:delta.added?.size ?? 0})+"\n"); + pkgsThatWereRebuilt.add(depPath); + return; + } +@@ -163291,41 +163295,7 @@ + if (pgkManifest != null) { + requiresBuild = pkgRequiresBuild(pgkManifest, /* @__PURE__ */ new Map()); + } +- const hasSideEffects = requiresBuild && allowBuild(depPath) && await runPostinstallHooks({ +- depPath, +- extraBinPaths, +- extraEnv: opts3.extraEnv, +- optional: pkgSnapshot.optional === true, +- pkgRoot, +- rootModulesDir: ctx.rootModulesDir, +- scriptsPrependNodePath: opts3.scriptsPrependNodePath, +- shellEmulator: opts3.shellEmulator, +- unsafePerm: opts3.unsafePerm || false, +- userAgent: opts3.userAgent +- }); +- if (hasSideEffects && (opts3.sideEffectsCacheWrite ?? true) && (resolution.gitHosted || resolution.integrity)) { +- builtDepPaths.add(depPath); +- const filesIndexFile = pickStoreIndexKey(resolution, pkgId, { built: true }); +- try { +- if (!sideEffectsCacheKey) { +- sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, { +- includeDepGraphHash: true, +- nodeVersion +- }); +- } +- await opts3.storeController.upload(pkgRoot, { +- sideEffectsCacheKey, +- filesIndexFile +- }); +- } catch (err2) { +- assert7(util31.types.isNativeError(err2)); +- logger2.warn({ +- error: err2, +- message: `An error occurred while uploading ${pkgRoot}`, +- prefix: opts3.lockfileDir +- }); +- } +- } ++ if (requiresBuild) throw new Error("Missing target cache for " + pkgId); + pkgsThatWereRebuilt.add(depPath); + } catch (err2) { + assert7(util31.types.isNativeError(err2)); diff --git a/tools/project-pnpm/restore-test.cjs b/tools/project-pnpm/restore-test.cjs new file mode 100644 index 0000000000000000000000000000000000000000..6ffe07a66b02feb7d82572f6808298cd1d59ed04 --- /dev/null +++ b/tools/project-pnpm/restore-test.cjs @@ -0,0 +1,34 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const restore = require('./restore.cjs'); + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'project-restore-')); +try { + const bytes = process.env.PROJECT_RESTORE_SOURCE ? fs.readFileSync(process.env.PROJECT_RESTORE_SOURCE) : Buffer.from('cached native artifact'); + const digest = crypto.createHash('sha512').update(bytes).digest('hex'); + const source = process.env.PROJECT_RESTORE_SOURCE || path.join(root, digest.slice(0, 2), digest.slice(2)); + if (!process.env.PROJECT_RESTORE_SOURCE) { + fs.mkdirSync(path.dirname(source)); + fs.writeFileSync(source, bytes); + } + const destination = path.join(root, 'package'); + fs.mkdirSync(destination); + restore(destination, [['bin/tool', source, 0o755]], []); + assert.equal(fs.readFileSync(path.join(destination, 'bin/tool'), 'utf8'), bytes.toString()); + fs.writeFileSync(path.join(destination, 'bin/tool'), 'changed'); + assert.deepEqual(fs.readFileSync(source), bytes); + fs.symlinkSync(root, path.join(destination, 'escape')); + assert.throws(() => restore(destination, [['escape/tool', source, 0o755]], []), /symlink/); + assert.throws(() => restore(destination, [['../tool', source, 0o755]], []), /Invalid/); + const corrupt = path.join(root, digest.slice(0, 2), digest.slice(2) + '-exec'); + fs.mkdirSync(path.dirname(corrupt), {recursive: true}); + fs.writeFileSync(corrupt, 'corrupt'); + assert.throws(() => restore(destination, [['bin/tool', corrupt, 0o755]], []), /integrity/); + assert.equal(fs.readFileSync(path.join(destination, 'bin/tool'), 'utf8'), 'changed'); + console.log('Reflink independence, path guards, and corrupt-cache rejection passed.'); +} finally { + fs.rmSync(root, {recursive: true, force: true}); +} diff --git a/tools/project-pnpm/restore.cjs b/tools/project-pnpm/restore.cjs new file mode 100644 index 0000000000000000000000000000000000000000..4518c32cfdd0b4442ef1078b0793c2e650fc6a09 --- /dev/null +++ b/tools/project-pnpm/restore.cjs @@ -0,0 +1,45 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); + +module.exports = function restore(root, added, deleted) { + root = fs.realpathSync(root); + function destination(relative) { + const parts = relative.split('/'); + if (parts.some(p => !p || p === '.' || p === '..') || relative.includes('\\')) { + throw new Error('Invalid cached path'); + } + let parent = root; + for (const part of parts.slice(0, -1)) { + parent = path.join(parent, part); + try { fs.mkdirSync(parent); } catch (error) { if (error.code !== 'EEXIST') throw error; } + if (!fs.lstatSync(parent).isDirectory() || fs.realpathSync(parent) !== parent) { + throw new Error('Cached path follows a symlink'); + } + } + return path.join(parent, parts.at(-1)); + } + for (const relative of deleted) fs.rmSync(destination(relative), {force: true}); + for (const [relative, source, mode] of added) { + const dest = destination(relative); + const temporary = dest + '.project-' + crypto.randomUUID(); + const fd = fs.openSync(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + if (!fs.fstatSync(fd).isFile()) throw new Error('Cache entry is not a regular file'); + const expected = path.basename(path.dirname(source)) + path.basename(source).replace(/-exec$/, ''); + const hash = crypto.createHash('sha512'); + const buffer = Buffer.alloc(65536); + let length; + while ((length = fs.readSync(fd, buffer)) > 0) hash.update(buffer.subarray(0, length)); + if (!/^[a-f0-9]{128}$/.test(expected) || hash.digest('hex') !== expected) { + throw new Error('Cached file failed integrity verification'); + } + fs.copyFileSync('/proc/self/fd/' + fd, temporary, fs.constants.COPYFILE_FICLONE_FORCE); + fs.chmodSync(temporary, mode & 0o777); + fs.renameSync(temporary, dest); + } finally { + fs.closeSync(fd); + fs.rmSync(temporary, {force: true}); + } + } +}; diff --git a/tools/project-store-install.py b/tools/project-store-install.py new file mode 100644 index 0000000000000000000000000000000000000000..40c1bd569ea08caec743273df172d104501bacd3 --- /dev/null +++ b/tools/project-store-install.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Stage project tools on Snow Globe; activation is a separate atomic switch.""" + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import tarfile +import tempfile +import urllib.request + + +HERE = Path(__file__).resolve().parent +BASE = Path("/opt/studio-project-store") +PROJECTS = "/srv/mac-studio-vm/sandbox/project" +GUEST_PROJECTS = "/Volumes/Code/project" + + +def download(url, digest): + with urllib.request.urlopen(url, timeout=60) as response: + data = response.read() + if hashlib.sha256(data).hexdigest() != digest: + raise ValueError("Downloaded tool failed checksum verification.") + return tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") + + +def stage(name): + destination = BASE / "releases" / name + if destination.exists(): + raise ValueError("Release already exists. Choose a new name.") + current = (BASE / "current").resolve(strict=True) + binaries = {} + for name, pattern, executable in (("python", "project-store-*-python3-*", "python3"), + ("node", "project-store-*-nodejs-*", "node"), + ("bwrap", "project-store-*-bubblewrap-*", "bwrap"), + ("coreutils", "project-store-coreutils", "true")): + roots = list(Path("/nix/var/nix/gcroots").glob(pattern)) + if len(roots) != 1: + raise ValueError(f"Expected one pinned {name} runtime, found {len(roots)}.") + binaries[name] = str(roots[0].resolve(strict=True) / "bin" / executable) + with tempfile.TemporaryDirectory(prefix=".prepare-", dir=destination.parent) as temporary: + release = Path(temporary) + shutil.copytree(HERE / "project-tools", release / "project-tools", + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + for filename in ("project-store.py", "project-store-test.py"): + shutil.copy2(HERE / filename, release / filename) + for filename in ("restore.cjs", "restore-test.cjs"): + shutil.copy2(HERE / "project-pnpm" / filename, release / filename) + profiles = json.loads((current / "profiles.json").read_text()) + for profile in profiles.values(): + profile.pop("workspace_config", None) + (release / "profiles.json").write_text(json.dumps(profiles)) + shutil.copy2(current / "jj.toml", release / "jj.toml") + (release / "bin").mkdir() + with download("https://github.com/jj-vcs/jj/releases/download/v0.42.0/" + "jj-v0.42.0-x86_64-unknown-linux-musl.tar.gz", + "2d91e81d649e617a81608e7401ad1106029c15ece01ac928c4a351abef42be6a") as archive: + member, = [m for m in archive if m.isfile() and Path(m.name).name == "jj"] + (release / "bin/jj-real").write_bytes(archive.extractfile(member).read()) + (release / "bin/jj-real").chmod(0o755) + (release / "bin/jj").symlink_to("../project-tools/jj") + with download("https://github.com/BurntSushi/ripgrep/releases/download/15.2.0/" + "ripgrep-15.2.0-x86_64-unknown-linux-musl.tar.gz", + "33e15bcf1624b25cdd2a55813a47a2f95dbe126268203e76aa6a585d1e7b149c") as archive: + member, = [m for m in archive if m.isfile() and Path(m.name).name == "rg"] + (release / "bin/rg").write_bytes(archive.extractfile(member).read()) + (release / "bin/rg").chmod(0o755) + for name in ("jj", "pnpm", "project"): + path = release / "project-tools" / name + content = path.read_text().split("\n", 1)[1] + path.write_text("#!" + binaries["python"] + "\n" + content) + path.chmod(0o755) + for version, digest, directory, original, patched in ( + ("10.32.1", "9b943b94bc8f55efb993aad8e44b538e6b091e60a9e4a944dcde869855f233e3", + "pnpm10", "pnpm.cjs", "pnpm-darwin-rebuild.cjs"), + ("11.10.0", "620b6605ea4f62fc56a6d0a98733f479071d0321e03e486b522c7e9a74617431", + "pnpm11", "pnpm.mjs", "pnpm-target-cache.mjs"), + ): + with download(f"https://registry.npmjs.org/pnpm/-/pnpm-{version}.tgz", digest) as archive: + archive.extractall(release / directory, filter="data") + package = release / directory / "package" + for path in (package / "dist").iterdir(): + path.rename(release / directory / path.name) + shutil.rmtree(package) + shutil.copy2(release / directory / original, release / directory / patched) + subprocess.run(["patch", "--batch", str(release / directory / patched), + str(HERE / "project-pnpm" / f"pnpm-{version}.patch")], check=True) + (release / "pnpm12").mkdir() + with download("https://github.com/pnpm/pnpm/releases/download/v12.14.0/pnpm-linux-x64-musl.tar.gz", + "d7115e2f2791d79bf9e7e448609537595bb7787de1ef5292ce91ff3d1be2629e") as archive: + member, = [m for m in archive if m.isfile() and Path(m.name).name == "pnpm"] + (release / "pnpm12/pnpm").write_bytes(archive.extractfile(member).read()) + (release / "pnpm12/pnpm").chmod(0o755) + command = [binaries["bwrap"], "--die-with-parent", "--unshare-all", "--cap-drop", "ALL", + "--ro-bind", "/nix/store", "/nix/store", "--proc", "/proc", "--dev", "/dev", + "--tmpfs", "/tmp", "--dir", "/home/clo", + "--ro-bind", str(destination), "/opt/project-store", + "--bind", PROJECTS, GUEST_PROJECTS, "--bind", PROJECTS, PROJECTS] + stores = "/srv/mac-studio-vm/sandbox/cache/project-pnpm" + for target in (stores, "/Volumes/Code/cache/project-pnpm", "/Users/admin/Library/Caches/project"): + command.extend(("--bind", stores, target)) + command.extend(("--bind", "/srv/mac-studio-vm/sandbox/cache/pnpm12", "/Volumes/Code/cache/pnpm12")) + command.extend(("--bind", "/home/clo/.cache/pnpm", "/home/clo/.cache/pnpm", + "--clearenv", "--setenv", "HOME", "/home/clo", "--setenv", "PATH", + "/opt/project-store/bin:" + ":".join(str(Path(binaries[key]).parent) + for key in ("node", "python", "coreutils")), + "--setenv", "JJ_CONFIG", "/opt/project-store/jj.toml", + "--setenv", "PROJECT_STORE_SSH", "", "--setenv", "LC_ALL", "C.UTF-8", + "--chdir", GUEST_PROJECTS, binaries["python"], + "/opt/project-store/project-store.py", GUEST_PROJECTS)) + network = ["--share-net", "--ro-bind", "/etc/resolv.conf", "/etc/resolv.conf", + "--ro-bind", "/etc/ssl/certs/ca-bundle.crt", "/etc/ssl/certs/ca-bundle.crt", + "--setenv", "SSL_CERT_FILE", "/etc/ssl/certs/ca-bundle.crt", + "--setenv", "NODE_EXTRA_CA_CERTS", "/etc/ssl/certs/ca-bundle.crt"] + (release / "ssh-command").write_text( + "#!" + binaries["python"] + "\nimport json, subprocess, sys\n" + "request = sys.stdin.buffer.readline(4 * 1024 * 1024 + 1)\n" + "if len(request) > 4 * 1024 * 1024: sys.exit('Project request exceeds 4 MiB.')\n" + "command = " + repr(command) + "\n" + "if json.loads(request).get('command') == 'read':\n" + " command = ['--ro-bind' if arg == '--bind' else arg for arg in command]\n" + "if json.loads(request).get('command') == 'pnpm':\n" + " command[command.index('--chdir'):command.index('--chdir')] = " + repr(network) + "\n" + "result = subprocess.run(command, input=request)\n" + "sys.exit(result.returncode if result.returncode >= 0 else 128 - result.returncode)\n") + (release / "ssh-command").chmod(0o755) + release.chmod(0o755) + release.rename(destination) + release.mkdir() + print(destination) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("stage", "activate")) + parser.add_argument("release") + args = parser.parse_args() + if socket.gethostname() != "snowglobe" or os.geteuid() != 0: + parser.error("Run this installer as root on Snow Globe.") + if not args.release or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789-." for c in args.release): + parser.error("Use a release name containing lowercase letters, numbers, dots, or hyphens.") + if args.action == "stage": + os.umask(0o022) + stage(args.release) + else: + release = BASE / "releases" / args.release + if not (release / "ssh-command").is_file(): + parser.error("Stage this release before activating it.") + link = BASE / (".current-" + args.release) + link.symlink_to(Path("releases") / args.release) + link.replace(BASE / "current") + print(release) + + +if __name__ == "__main__": + main() diff --git a/tools/project-store-test.py b/tools/project-store-test.py new file mode 100644 index 0000000000000000000000000000000000000000..2944ed034dfb08b07bf34ca882049950c5dfd8b8 --- /dev/null +++ b/tools/project-store-test.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Disposable workspace tests; PROJECT_STORE_REAL_REFLINK=1 requires Linux/ZFS.""" + +import importlib.util +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("project_store", Path(__file__).with_name("project-store.py")) +store = importlib.util.module_from_spec(spec) +spec.loader.exec_module(store) + + +class WorkspaceTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="project-store-test-") + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name).resolve() + config = self.root / "jj.toml" + config.write_text('[user]\nname="Fixture"\nemail="fixture@example.invalid"\n[signing]\nbehavior="drop"\n') + self.env = patch.dict(os.environ, {"JJ_CONFIG": str(config)}) + self.env.start() + self.addCleanup(self.env.stop) + if not os.environ.get("PROJECT_STORE_REAL_REFLINK"): + self.copy = patch.object(store.fcntl, "ioctl", side_effect=self.copy_bytes) + self.copy.start() + self.addCleanup(self.copy.stop) + self.main = self.root / "Example/main" + self.main.parent.mkdir() + self.jj("git", "init", "--no-colocate", str(self.main)) + (self.main / ".gitignore").write_text("target/\n") + (self.main / "source.txt").write_text("committed\n") + (self.main / "unchanged.txt").write_text("same\n") + (self.main / "link").symlink_to("source.txt") + self.jj("-R", str(self.main), "describe", "-m", "fixture") + self.revision = self.jj("-R", str(self.main), "log", "--no-graph", "-r", "@", "-T", "commit_id").strip() + (self.main / "target/debug").mkdir(parents=True) + (self.main / "target/debug/artifact").write_bytes(b"warm artifact" * 10000) + (self.main / "target/debug/.cargo-lock").touch() + self.request = dict(project="Example", source="main", destination="agent", + revision=self.revision, targets=["target"]) + + @staticmethod + def copy_bytes(destination, command, source): + with os.fdopen(os.dup(source), "rb") as src, os.fdopen(os.dup(destination), "wb") as dst: + shutil.copyfileobj(src, dst) + + def jj(self, *args): + return subprocess.run(["jj", *args], text=True, capture_output=True, check=True).stdout + + def test_host_checkout_without_cache_preserves_donor(self): + (self.main / "source.txt").write_text("active agent edit\n") + request = dict(project="Example", destination="cold", revision=self.revision, + engine="darwin;arm64;node26", install=True) + self.assertEqual(store.seed(self.root, request), {"workspace": "cold", "installed": False, "pnpm": False}) + self.assertEqual((self.main.parent / "cold/source.txt").read_text(), "committed\n") + self.assertEqual((self.main / "source.txt").read_text(), "active agent edit\n") + with self.assertRaises(ValueError): + store.seed(self.root, request) + for name in ("../escape", "main", "default"): + with self.assertRaises(ValueError): + store.seed(self.root, dict(request, destination=name)) + + def test_cache_inputs_track_dependency_changes_only(self): + (self.main / "package.json").write_text('{"packageManager":"pnpm@10.32.1"}') + before = store.dependency_inputs(self.main) + operation = self.jj("--ignore-working-copy", "-R", str(self.main), "op", "log", "--no-graph", "-n", "1", "-T", "id") + (self.main / "source.txt").write_text("new source\n") + self.assertEqual(store.dependency_inputs(self.main), before) + (self.main / "package.json").write_text('{"packageManager":"pnpm@12.14.0"}') + self.assertNotEqual(store.dependency_inputs(self.main), before) + self.assertEqual(self.jj("--ignore-working-copy", "-R", str(self.main), "op", "log", "--no-graph", "-n", "1", "-T", "id"), operation) + (self.main / "package.json").unlink() + (self.main / "package.json").symlink_to("source.txt") + with self.assertRaises(ValueError): + store.dependency_inputs(self.main) + + def test_clone_restores_revision_and_preserves_warm_files(self): + (self.main / "source.txt").write_text("donor edit\n") + (self.main / "donor-only.txt").write_text("uncommitted\n") + original = self.main / "target/debug/artifact" + mtime = (self.main / "unchanged.txt").stat().st_mtime_ns + self.assertEqual(store.seed(self.root, self.request), {"workspace": "agent"}) + agent = self.main.parent / "agent" + self.assertEqual((agent / "source.txt").read_text(), "committed\n") + self.assertFalse((agent / "donor-only.txt").exists()) + self.assertEqual((self.main / "source.txt").read_text(), "donor edit\n") + self.assertEqual((agent / "unchanged.txt").stat().st_mtime_ns, mtime) + artifact = agent / "target/debug/artifact" + self.assertEqual(artifact.read_bytes(), original.read_bytes()) + self.assertEqual(artifact.stat().st_mtime_ns, original.stat().st_mtime_ns) + self.assertNotEqual(artifact.stat().st_ino, original.stat().st_ino) + artifact.write_bytes(b"modified") + self.assertNotEqual(artifact.read_bytes(), original.read_bytes()) + self.assertEqual(os.readlink(agent / "link"), "source.txt") + self.assertEqual(self.jj("-R", str(agent), "diff", "--summary"), "") + self.assertFalse((agent / ".jj/repo").is_dir()) + self.assertFalse(Path((agent / ".jj/repo").read_text()).is_absolute()) + + def test_rejects_paths_and_existing_destination(self): + for key, value in (("source", "../outside"), ("destination", "main"), + ("project", "/tmp"), ("targets", ["../outside"])): + with self.subTest(key=key), self.assertRaises(ValueError): + store.seed(self.root, dict(self.request, **{key: value})) + (self.main.parent / "agent").mkdir() + with self.assertRaises(ValueError): + store.seed(self.root, self.request) + + def test_rejects_symlink_workspace(self): + (self.main.parent / "linked").symlink_to(self.main) + with self.assertRaises(OSError): + store.seed(self.root, dict(self.request, source="linked")) + + def test_failed_reflink_removes_workspace_registration_and_files(self): + with patch.object(store.fcntl, "ioctl", side_effect=OSError("reflink unavailable")): + with self.assertRaises(OSError): + store.seed(self.root, self.request) + self.assertFalse((self.main.parent / "agent").exists()) + self.assertNotIn("agent:", self.jj("-R", str(self.main), "workspace", "list")) + self.assertEqual((self.main / "source.txt").read_text(), "committed\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/project-store.md b/tools/project-store.md new file mode 100644 index 0000000000000000000000000000000000000000..a54a7aa5295b18557d11bf12b4521840bad6ef7f --- /dev/null +++ b/tools/project-store.md @@ -0,0 +1,220 @@ +# Shared project commands + +`tools/project-tools/` owns the VM's `jj`, `pnpm`, and `project` wrappers. +The config repo's `users/clover/agents/vm` entries link here; installed wrappers +use `~/.local/share/project-tools/current`. Keep native executables adjacent to +the public commands as `jj-real` and `pnpm-real`. + +## Defaults + +- `project read rg ARGS...` searches on Linux beside ZFS and streams normal + ripgrep output and exit status. `project read rg --files` lists files. + `.gitignore` works in Jujutsu checkouts without a visible `.git` directory; + `--require-git` restores ripgrep's usual Git-only policy. Search preprocessors + are disabled. `project read sha256sum FILE...` hashes named shared files + without transferring their contents over NFS. Run these from the project + share; read requests have read-only mounts, no network, and no repository lock. +- Noninteractive `jj` operations in `/Volumes/Code/project` run on Snow Globe. + This includes workspace creation, snapshots, rebases, restores, and reads. + The guest's resolved configuration, identity, ignores, and paths travel with + the request. Both sides must run exactly the same Jujutsu build. +- Editors, external diff tools, signing, network Git commands, and local-disk + repositories retain native execution. Explicit file paths for a shared + repository cannot escape the project share, including through symlinks. +- `pnpm install`, `i`, `add`, `update`, and `remove` resolve, fetch, and import + packages on Linux, including changed manifests and lockfiles. The runtime must + match the project's exact pin: 10.32.1, 11.10.0, or 12.14.0. Directory selection, + workspace filters, frozen/offline installs, and lockfile-only updates retain + their normal meanings. `--ignore-scripts` skips the Mac hook phase. +- Activation links Vite+'s installed managed pnpm launchers to this same wrapper, + preserving each original as the adjacent `pnpm-real`. This covers the + project-local JavaScript `vp` launcher, which otherwise bypasses pnpm on PATH. + Vite+ keeps package-manager detection and argument handling; its notifier + setting does not prevent offloading. Explicit `VP_PACKAGE_MANAGER` overrides + retain native execution. Reactivate after Vite+ downloads another pnpm version. +- `pnpm run`, `exec`, and script shortcuts execute natively. Their implicit + dependency installs use Linux too; dependency verification opt-outs and custom + verification policies are preserved. Inherited npm client metadata and Node + heap limits retain offloading; `NODE_ENV=production` reaches the pinned pnpm + unchanged on both hosts. Unsupported options, runtime versions, other custom + install environment settings, and unreviewed pnpmfile hooks retain native + execution. Environment fallbacks name the variables without exposing values. + A failed remote operation is reported once. +- `project worktree PROJECT NAME` uses those same `jj` and `pnpm` commands. + `--no-install` skips dependencies. The checkout path is printed only after + installation and Mac hooks succeed, including with `--warm-from`. + New workspaces wait for NFS visibility. + +Linux installs with lifecycle scripts disabled, selects Darwin +ARM64 packages, validates the guest's Node version, and requires ZFS reflinks. +Knife's reviewed `.pnpmfile.cjs` runs in the Linux sandbox: its peer-range change +and Windows-only architecture selection behave identically on Linux and Darwin. +The backend checks its exact SHA-256 before offloading. Changed hooks, other +pnpmfiles, and config dependencies retain native installation. +Pending dependency and workspace hooks finish natively on the Mac. pnpm 10/11 +use `pnpm --recursive --include-workspace-root rebuild --pending`. pnpm 12 uses +a frozen install with the original filters and dependency groups, preserving +prepared patches. Its repeat-install shortcut and build-cache reads/writes are +disabled for this phase: restoring cached generated files over NFS can take +longer than running their hooks. Ordinary native installs retain their cache policy. +The wrapper announces the Mac handoff. +It also announces storage-host installation and package-store lock contention. +The two phases share policy and registry metadata in the project's `.pnpm-cache` +directory, outside its workspaces. This avoids repeating network verification on +the Mac. The handoff waits until NFS exposes the server's new modules manifest. +Repository configuration is not temporarily rewritten; manifests and +lockfiles change only as requested by the dependency command. + +The root-owned `profiles.json` binds manifests, lockfiles, workspace settings, +npmrc, pnpmfiles, patches, and local tarballs. Matching alphaXiv pnpm 10.32.1 and +Knife pnpm 11.10.0 profiles on Node 26 can additionally restore reviewed Darwin +side effects on Linux after SHA-512 verification using `project-pnpm/`'s patches. +A profile miss still installs on Linux; it leaves native builds pending for the +Mac instead of replaying an older build. Validation does not snapshot the working +copy. Ordinary Linux side-effect cache reuse is disabled. + +## Locking and storage + +The Mac wrapper uses `flock`; Linux uses a POSIX record lock on the same +`.jj/repo/command.lock` inode. macOS NFS translates `flock` into that server lock. +The executing Jujutsu process inherits the descriptor. The forwarding guest +never holds a second lock. Ctrl-C cancels a waiter; never unlink the lock inode. +A command lock does not reserve revisions across a sequence of commands. + +The Code export is one ZFS dataset. `project worktree --warm-from DONOR` uses +per-file FICLONE, preserving independent inodes and source timestamps, excluding +root repository metadata and Mac sidecars. It restores the requested revision +and retains ignored build outputs. The guest holds the donor's Cargo locks; +editors and other writers must also be idle. No byte-copy fallback is allowed. + +The dedicated SSH key forces the unprivileged uid-3000 Bubblewrap launcher. +Jujutsu requests have no network. Dependency requests get network access with +DNS and the host CA bundle; lifecycle scripts remain disabled on Linux. Neither path +exposes host credentials, forwarding, or snapshot privileges. +The sandbox binds projects and package stores at their guest paths as well as +host paths. Identical store paths allow repeated installs without repaving +`node_modules`. Artifacts remain under Code. pnpm 11 uses a guest-local SQLite +index under `~/Library/Caches/project/PROJECT/v11`; pnpm 12 uses the shared +`/Volumes/Code/cache/pnpm12` file index. Both phases use workspace-local virtual +stores so platform-specific build outputs belong to that workspace. + +## Updating the existing deployment + +Upload `project-store-install.py`, `project-store.py`, `project-store-test.py`, +`project-tools/`, and `project-pnpm/` together to a private staging directory on +Snow Globe. Run the installer there as root: + +```sh +python3 project-store-install.py stage RELEASE +``` + +It verifies pinned upstream downloads, applies the cache patches, preserves the +reviewed profiles, and builds a versioned runtime under +`/opt/studio-project-store/releases/`. Existing `project-store-*` Nix GC roots +pin Python, Node, Bubblewrap, and coreutils. `current` is unchanged by staging. +Test the staged `ssh-command` as `clo` with disposable projects and a temporary +restricted key. Do not run Jujutsu as root against shared repositories. + +Upload `project-tools/` to a guest staging directory and run its installer as +`admin`. After the staged host and guest tests pass: + +```sh +# Snow Globe +python3 project-store-install.py activate RELEASE +# VM, using tools/project-tools/install.py +python3 install.py stage RELEASE +python3 install.py activate RELEASE +``` + +Guest activation saves the prior tools under `project-tools/previous`, preserves +the native executables, and changes symlinks atomically. Subsequent updates switch +the complete wrapper directory through `current`. Keep the previous host release +and guest tools for rollback. Roll back both `current` links; do not remove +workspaces, locks, stores, or caches. + +## Verification + +```sh +python3 tools/project-store-test.py +python3 tools/project-tools/test.py +python3 tools/project-tools/jj-test.py +``` + +Run `project-tools/platform-test.py` inside the development VM against the +normal endpoint. Its disposable project checks Darwin-only optional dependencies +immediately after the Linux import, before Mac build scripts can repair them. +`project-tools/finish-test.py` checks native hooks, denied scripts, patch inode +preservation, repeated installs, filtered dependency groups, and nested mutations. + +The backend tests create disposable repositories. `PROJECT_STORE_REAL_REFLINK=1` +with `TMPDIR` on disposable Linux/ZFS storage exercises real clone independence, +timestamps, donor edits, path rejection, and failure cleanup. The config repo's +`vm/test.py` covers overlay preparation, worktrees, and shared memory links. +`project-pnpm/restore-test.cjs` checks cache integrity and path guards on Linux/ZFS. + +On 2026-10-22, the endpoint created an 801-file workspace through the VM in 2.9 +seconds and rebased a two-commit fixture in 1.6 seconds. Cross-client lock +contention and cancellation passed. Disposable pnpm 10, 11, and 12 projects +installed changed dependencies on Linux and ran Darwin dependency and workspace +hooks on the Mac. Frozen-lockfile failures, lockfile-only updates, filtered +add/update/remove commands, ignored scripts, and native esbuild execution passed. +A pnpm 12 filtered update took 12.4 seconds, including 1.4 seconds on Linux; +a subsequent script ran in 1.2 seconds without reinstalling. Nested pnpm scripts +and dependency-verification opt-outs passed. An existing global virtual store +migrated successfully. After activation, ordinary commands against the real +shared store passed: a fresh install took 70.5 seconds, a filtered add took 32.9 +seconds, and a script took 2.5 seconds. Native rebuilding left all 1,048 sampled +lodash files' inodes and mtimes unchanged. These are fixture measurements with other agents +active, not latency bounds or full-project build measurements. + +Vite+ 1.3.0 was also verified through the normal endpoint with pnpm 12.14.0 +and Knife's reviewed `.pnpmfile.cjs`: package imports ran on Linux/ZFS, while +the workspace postinstall and native esbuild ran on Darwin ARM64. Frozen/offline +installs, add/remove, lockfile-only, ignored scripts, script execution, and frozen +lockfile failure propagation passed. Editing the hook selected Mac installation. +The host and VM tools were activated as `20261022-vp-preview-2`. + +pnpm 12 needs `supportedArchitectures: [darwin-arm64]` for host-side package +selection; the older `config.os`/`config.cpu` overrides do not select its optional +dependencies. This is passed as a command setting without editing the workspace. +The platform regression fails against the old endpoint and passes with the fix. +The full Knife workspace imported 75 missing Darwin packages, and its Vite+, +Vite, Vitest, Rolldown, and linter bindings loaded successfully before rebuilding. +With Mac build-cache writes disabled, a normal frozen install in `knife-merge` +completed successfully in 278 seconds, including the mobile dependency's native +asset downloads and the root prepare hook. The host and guest were activated as +`20261023-vp-platform-3`. Linux side-effect cache reads and writes are explicitly +disabled with pnpm 12's supported boolean settings. +After installation, `pnpm exec vp --version`, the TypeScript compiler, and the +Vitest CLI started successfully. The node-pty addon loaded, and sharp encoded a +small PNG using its Darwin native module. + +A follow-up trace in `knife-disk` found the Mac phase copying patched packages +and restoring cached mobile assets over NFS. The frozen-install completion +described above disables those cache transfers. Five disposable VM cases cover +native and workspace hooks, denied scripts, patch preservation, filtered +production/development installs, nested adds, repeated installs, and inherited +agent environments. A deployed +frozen install in `knife-disk` completed in 43.5 seconds with its native assets +already present. Vite+, TypeScript, and the node-pty native binding loaded. +The real project-local `vp i --frozen-lockfile` then completed in 28.8 seconds +through Vite+'s managed pnpm launcher and the shared wrapper. Exporting +`VP_PNPM_MANAGER=no` alone did not route that JavaScript launcher through PATH; +the obsolete VM environment setting was removed. + +The host and guest were activated as `20261023-vp-env-4`. Staged pnpm 10, 11, +and 12 installs with `NODE_ENV=production` preserved their native dependency +selection; production mode is forwarded as environment, without inventing a +`--prod` flag. The VM fixture also checked that Mac hooks receive that value +while npm client metadata and Node memory limits remain compatible with offloading. +A real `vp i --frozen-lockfile` with those inherited metadata and memory settings +completed in 23.2 seconds on `knife-disk`, including Mac hooks with native assets +already present. Vite+'s tools, TypeScript, and the node-pty Darwin binding loaded. + +Release `20261023-tools-6` adds the read-only storage tools and waits for NFS +visibility before opening warm workspaces. A VM request listed 2,001 fixture +files in 1.93 seconds and hashed a 330 MiB fixture in 0.95 seconds. Staged +checks verified read-only mounts, network isolation, ignore rules, normal +search exit statuses, hash equality, and preprocessor rejection. A real warm +checkout installed a changed dependency and its Darwin hook in 7.81 seconds; +the donor and an explicit `--no-install` copy retained the original dependency. diff --git a/tools/project-store.py b/tools/project-store.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c766640af19d3ccbb894adbde6689c55995f74 --- /dev/null +++ b/tools/project-store.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +"""Create Jujutsu workspaces and restore dependencies using host-side ZFS reflinks. + +Run behind a forced SSH command with a fixed project root and an unprivileged +account. The guest supplies one JSON request on stdin, never a shell command. +""" + +import argparse +from contextlib import ExitStack +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import sys +import tempfile + +sys.path.insert(0, str(Path(__file__).with_name("project-tools"))) +import client + + +def parts(value): + if not isinstance(value, str) or not value or value.startswith("/"): + raise ValueError("Use a relative path inside the project directory.") + names = value.split("/") + if any(n in ("", ".", "..", ".jj", ".git") or "\\" in n + or any(ord(c) < 32 or ord(c) == 127 for c in n) for n in names): + raise ValueError("Paths cannot contain traversal or repository metadata.") + return names + + +def directory(stack, root, names): + for name in names: + root = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root) + stack.callback(os.close, root) + return root + + +def clone_tree(source, destination, *, workspace=False): + initial = os.fstat(source) + for name in os.listdir(source): + if (workspace and name in (".jj", ".git")) or name.startswith("._") or name == ".DS_Store": + continue + before = os.stat(name, dir_fd=source, follow_symlinks=False) + with ExitStack() as stack: + if stat.S_ISDIR(before.st_mode): + src = directory(stack, source, [name]) + os.mkdir(name, mode=0o700, dir_fd=destination) + dst = directory(stack, destination, [name]) + clone_tree(src, dst) + elif stat.S_ISREG(before.st_mode): + src = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=source) + stack.callback(os.close, src) + if not stat.S_ISREG(os.fstat(src).st_mode): + raise ValueError("Target changed during cloning. Retry with an idle donor.") + dst = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, dir_fd=destination) + stack.callback(os.close, dst) + # Linux FICLONE must succeed; a byte-copy fallback defeats warm workspace creation. + fcntl.ioctl(dst, 0x40049409, src) + elif stat.S_ISLNK(before.st_mode): + os.symlink(os.readlink(name, dir_fd=source), name, dir_fd=destination) + continue + else: + raise ValueError(f"Workspace contains a special file: {name}. Remove it before cloning.") + current = os.fstat(src) + if (before.st_ino, before.st_size, before.st_mtime_ns) != ( + current.st_ino, current.st_size, current.st_mtime_ns): + raise ValueError("Target changed during cloning. Retry with an idle donor.") + os.fchmod(dst, stat.S_IMODE(current.st_mode) & 0o777) + os.utime(dst, ns=(current.st_atime_ns, current.st_mtime_ns)) + if os.fstat(source).st_mtime_ns != initial.st_mtime_ns: + raise ValueError("Workspace changed during cloning. Retry with an idle donor.") + + +def seed(root, request): + if not isinstance(request, dict): + raise ValueError("Invalid project-store request.") + if request.get("command") == "jj": + return run_jj(root, request) + if request.get("command") == "read": + if set(request) != {"command", "cwd", "tool", "args"} or request["tool"] not in ("rg", "sha256sum"): + raise ValueError("Use rg or sha256sum for read-only storage commands.") + cwd = shared_directory(root, request["cwd"]) + args = request["args"] + if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args): + raise ValueError("Invalid read-only command arguments.") + if request["tool"] == "rg": + if any(arg in ("--pre", "--hostname-bin") or arg.startswith(("--pre=", "--hostname-bin=")) for arg in args): + raise ValueError("Search preprocessors are unavailable on the storage host.") + args = ["--no-require-git", *args] + else: + has_file = False + options = True + for arg in args: + if options and arg == "--": + options = False + elif options and arg in ("-b", "--binary", "-t", "--text", "--tag", "-z", "--zero"): + continue + elif arg == "-" or (options and arg.startswith("-")): + raise ValueError("Hash named files; stdin and checksum verification are unavailable.") + else: + path = (cwd / arg).resolve(strict=True) + if not path.is_relative_to(root) or not path.is_file(): + raise ValueError("Hash files inside the project share.") + has_file = True + if not has_file: + raise ValueError("Select at least one file to hash.") + result = subprocess.run([request["tool"], *args], cwd=cwd, stdin=subprocess.DEVNULL) + return result.returncode if result.returncode >= 0 else 128 - result.returncode + if request.get("command") == "pnpm": + if (set(request) - {"command", "cwd", "engine", "args", "prepare", "node_env"} + or not isinstance(request.get("prepare", False), bool) + or request.get("node_env") not in (None, "production")): + raise ValueError("Invalid pnpm request.") + cwd = shared_directory(root, request["cwd"]) + args = request["args"] + if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args): + raise ValueError("Invalid pnpm arguments.") + if not args or all(arg.startswith("--") for arg in args): + args = ["install", *args] + try: + parsed = client.pnpm_arguments(args, cwd) + except StopIteration: + raise ValueError("A dependency command option is missing its value.") from None + if parsed[1] is None: + raise ValueError("Unsupported dependency command.") + cwd, args, _ = parsed + cwd = shared_directory(root, cwd) + relative = cwd.relative_to(root) + if len(relative.parts) < 2: + return {"installed": False, "reason": "run the install from the workspace root."} + workspace = root.joinpath(*relative.parts[:2]) + return install(workspace, relative.parts[0], request["engine"], args, cwd=cwd, + prepare=request.get("prepare", False), node_env=request.get("node_env")) + if set(request) == {"project", "destination", "revision", "engine", "install"}: + return checkout(Path(root), request) + if set(request) != {"project", "source", "destination", "revision", "targets"}: + raise ValueError("Invalid project-store request.") + project, source, destination = (parts(request[k]) for k in ("project", "source", "destination")) + if (any(len(p) != 1 for p in (project, source, destination)) or source == destination + or destination[0] in ("main", "default")): + raise ValueError("Select distinct workspaces in one project.") + if not isinstance(request["revision"], str) or not request["revision"]: + raise ValueError("Select a parent revision.") + targets = [parts(p) for p in request["targets"]] + if not targets or len({tuple(p) for p in targets}) != len(targets): + raise ValueError("Select each target directory once.") + root = Path(root).absolute() + with ExitStack() as stack: + root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + stack.callback(os.close, root_fd) + project_fd = directory(stack, root_fd, project) + src = directory(stack, project_fd, source) + directory(stack, project_fd, ["main", ".jj"]) + for path in targets: + directory(stack, src, path) + destination_path = root / project[0] / destination[0] + if os.path.lexists(destination_path): + raise ValueError("Destination exists. Choose a new workspace name.") + primary = root / project[0] / "main" + jj = ["jj", "--config", "snapshot.auto-track=" + json.dumps('~glob:"**/._*" & ~glob:"**/.DS_Store"')] + base = [*jj, "-R", str(primary)] + revision = subprocess.run( + [*base, "log", "--no-graph", "-r", request["revision"], "-T", 'commit_id ++ "\n"'], + text=True, capture_output=True, check=True).stdout.splitlines() + if len(revision) != 1: + raise ValueError("Select exactly one parent revision.") + subprocess.run([*base, "workspace", "add", "--sparse-patterns", "empty", + "--name", destination[0], "-r", revision[0], "--", str(destination_path)], + stdout=sys.stderr, check=True) + try: + dst = directory(stack, project_fd, destination) + clone_tree(src, dst, workspace=True) + local = [*jj, "-R", str(destination_path)] + subprocess.run([*local, "sparse", "set", "--add", "."], stdout=sys.stderr, check=True) + # Discard donor edits while retaining matching files and ignored build outputs. + subprocess.run([*local, "restore", "--from", revision[0]], stdout=sys.stderr, check=True) + except BaseException: + subprocess.run([*base, "workspace", "forget", destination[0]], + stdout=sys.stderr, check=True) + shutil.rmtree(destination_path) + raise + return {"workspace": destination[0]} + + +def dependency_inputs(workspace, revision=None): + base = ["jj", "--ignore-working-copy", "-R", str(workspace), "file"] + revision_args = ["-r", revision] if revision else [] + names = subprocess.check_output([*base, "list", *revision_args], cwd=workspace, text=True).splitlines() + if revision is None: + names = set(names) | {path.name for path in workspace.iterdir() if path.is_file()} + result = {} + for name in names: + path = Path(name) + if (path.name in ("package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc") + or path.suffix in (".patch", ".tgz") or "pnpmfile" in path.name): + target = workspace / path + if not revision and target.resolve() != target.absolute(): + raise ValueError("Dependency input follows a symlink.") + if not revision and not target.exists(): + continue + content = (subprocess.check_output([*base, "show", json.dumps(name), *revision_args], cwd=workspace) + if revision else target.read_bytes()) + result[name] = hashlib.sha256(content).hexdigest() + return result + + +def checkout(root, request): + project, name = (parts(request[k]) for k in ("project", "destination")) + if len(project) != 1 or len(name) != 1 or name[0] in ("main", "default"): + raise ValueError("Select a new workspace name.") + if not isinstance(request["revision"], str) or not request["revision"]: + raise ValueError("Select a parent revision.") + with ExitStack() as stack: + fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + stack.callback(os.close, fd) + directory(stack, fd, [*project, "main", ".jj"]) + workspace = root / project[0] / name[0] + if os.path.lexists(workspace): + raise ValueError("Destination exists. Choose a new workspace name.") + subprocess.run(["jj", "-R", str(root / project[0] / "main"), "workspace", "add", + "--name", name[0], "-r", request["revision"], "--", str(workspace)], + stdout=sys.stderr, check=True) + manifest = workspace / "package.json" + package = json.loads(manifest.read_text()) if manifest.is_file() else {} + result = {"workspace": name[0], "installed": False, + "pnpm": package.get("packageManager", "").startswith("pnpm@")} + if request["install"] is True: + result["installed"] = install(workspace, project[0], request["engine"], [])["installed"] + return result + + +def install(workspace, project, engine, args, *, cwd=None, prepare=False, node_env=None): + if not args: + args = ["install"] + cwd = cwd or workspace + environment = dict(os.environ, NODE_ENV=node_env) if node_env else None + manifest = workspace / "package.json" + package = json.loads(manifest.read_text()) if manifest.is_file() else {} + manager = re.fullmatch(r"pnpm@(\d+)\.(\d+)\.(\d+)(?:\+.*)?", package.get("packageManager", "")) + if not manager: + return {"installed": False, "reason": "the workspace does not pin a supported pnpm version."} + version = ".".join(manager.groups()) + major = manager.group(1) + executable = Path(__file__).parent / ("pnpm" + major) / { + "10": "pnpm.cjs", "11": "pnpm.mjs", "12": "pnpm"}.get(major, "unsupported") + if not executable.is_file(): + return {"installed": False, "reason": f"pnpm {version} is not installed on the storage host."} + pnpm = (["node"] if major in ("10", "11") else []) + [str(executable)] + if subprocess.check_output([*pnpm, "--version"], text=True, env=environment).strip() != version: + return {"installed": False, "reason": f"pnpm {version} is not installed on the storage host."} + target = re.fullmatch(r"darwin;arm64;node(\d+(?:\.\d+\.\d+)?)", engine) + if not target: + return {"installed": False, "reason": "the storage install targets Darwin ARM64."} + node_version = target.group(1) + profiles = Path(__file__).with_name("profiles.json") + profile = json.loads(profiles.read_text()).get(project) if profiles.exists() else None + guest_store = ("/Volumes/Code/cache/pnpm12" if major == "12" else + "/Users/admin/Library/Caches/project/" + project if major == "11" else + "/Volumes/Code/cache/project-pnpm/" + project) + if profile and Path(profile["pnpm"]).parent.name == "pnpm" + major: + guest_store = str(Path(profile["guest_store"]).parent) + workspace_config = workspace / "pnpm-workspace.yaml" + config_text = workspace_config.read_text().lower() if workspace_config.is_file() else "" + npmrc = workspace / ".npmrc" + if npmrc.is_file(): + config_text += npmrc.read_text().lower() + # Knife's reviewed hook behaves identically on Linux and Darwin; edits require review. + if ("pnpmfile" in config_text or "configdependencies" in config_text or any( + path.name != ".pnpmfile.cjs" or hashlib.sha256(path.read_bytes()).hexdigest() != + "6334e7e025aa23897f0b39365407691b3b65f64bf64fe167b7f1b0d6627b2eed" + for path in workspace.glob(".pnpmfile.*"))): + return {"installed": False, "store": guest_store, "reason": "workspace pnpmfile hooks require Mac execution."} + if prepare: + verify = subprocess.check_output([*pnpm, "config", "get", "verify-deps-before-run"], + cwd=cwd, text=True, env=environment).strip() + if verify == "undefined": + verify = "false" if major == "10" else "install" + if verify == "false": + return {"ready": True} + if verify != "install": + return {"installed": False, "reason": "the workspace selects its own dependency verification policy."} + cached = (not node_env and profile and Path(profile["pnpm"]).parent.name == "pnpm" + major + and "darwin;arm64;node" + node_version.split(".")[0] == profile["engine"] + and args[0] in ("install", "i") + and set(args[1:]) <= {"--frozen-lockfile", "--offline", "--prefer-offline", "--force", "--ignore-scripts"} + and dependency_inputs(workspace) == profile["inputs"]) + if cached: + packages = json.loads(subprocess.check_output( + [*pnpm, "list", "--recursive", "--depth=-1", "--json"], + cwd=workspace, text=True, env=environment)) + cached = all(str((Path(package["path"]) / "package.json").relative_to(workspace)) in profile["inputs"] + for package in packages) + Path(guest_store).mkdir(parents=True, exist_ok=True) + common = ["--config.store-dir=" + guest_store, + "--config.cache-dir=" + str(workspace.parent / ".pnpm-cache"), + "--config.enable-global-virtual-store=false", + "--config.prefer-symlinked-executables=true", + "--reporter=append-only"] + if "." in node_version: + common.append("--config.node-version=" + node_version) + if major == "12": + common.extend(('--config.supported-architectures=["darwin-arm64"]', + "--config.side-effects-cache=false", "--config.side-effects-cache-readonly=false", + "--config.store-index-format=files", "--config.virtual-store-type=local")) + else: + common.extend(("--config.side-effects-cache-read=false", "--config.cpu=arm64", "--config.os=darwin")) + modules = workspace / "node_modules/.modules.yaml" + if prepare and modules.is_file(): + check = subprocess.run([*pnpm, *common, "--config.verify-deps-before-run=error", "exec", "true"], + cwd=cwd, text=True, capture_output=True, env=environment) + if check.returncode == 0: + return {"ready": True} + if "VERIFY_DEPS_BEFORE_RUN" not in check.stdout + check.stderr: + sys.stderr.write(check.stdout + check.stderr) + raise subprocess.CalledProcessError(check.returncode, pnpm) + included = json.loads(modules.read_text())["included"] + args = ["install"] + if included["dependencies"] and not included["devDependencies"]: + args.append("--prod") + elif included["devDependencies"] and not included["dependencies"]: + args.append("--dev") + if not included["optionalDependencies"]: + args.append("--no-optional") + with ExitStack() as stack: + lock = os.open(guest_store + "/.project-install.lock", os.O_CREAT | os.O_RDWR, 0o600) + stack.callback(os.close, lock) + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("pnpm: Waiting for another install using this package store…", file=sys.stderr, flush=True) + fcntl.flock(lock, fcntl.LOCK_EX) + subprocess.run([*pnpm, *(arg for arg in args if arg != "--ignore-scripts"), + "--config.ignore-scripts=true", "--config.package-import-method=clone", *common], cwd=cwd, + stdout=sys.stderr, check=True, env=environment) + if cached and "--ignore-scripts" not in args: + subprocess.run(["node", profile["restore"], "-r", "rebuild", "--pending", *common], + cwd=workspace, stdout=sys.stderr, check=True, env=environment) + result = {"installed": True, "workspace": str(workspace)} + if "--lockfile-only" not in args: + result["modules_hash"] = hashlib.sha256(modules.read_bytes()).hexdigest() + return result + + +def shared_directory(root, value): + path = Path(value).resolve(strict=True) + if not path.is_relative_to(root) or not path.is_dir(): + raise ValueError("Path is outside the project share.") + return path + + +def run_jj(root, request): + if set(request) != {"command", "cwd", "args", "config", "version", "ignore"}: + raise ValueError("Invalid jj request.") + cwd = shared_directory(root, request["cwd"]) + args = request["args"] + if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args): + raise ValueError("Invalid jj arguments.") + words, options, _ = client.jj_arguments(args) + paths = client.jj_paths(words, options) + paths.extend(value for name, value in options if name in ("-R", "--repository", "--config-file")) + for value in paths: + if not (cwd / value).resolve().is_relative_to(root): + raise ValueError("Path is outside the project share.") + version = subprocess.check_output(["jj-real", "--version"], text=True).strip() + if version != request["version"]: + raise ValueError("Guest and storage Jujutsu versions differ. Update the project tools together.") + with tempfile.TemporaryDirectory(prefix="project-jj-") as temporary: + config = Path(temporary) / "jj.toml" + config.write_text(request["config"]) + ignore = Path(temporary) / "git/ignore" + ignore.parent.mkdir() + ignore.write_text(request["ignore"]) + # The resolved guest config carries its identity, aliases, and environment overrides. + # Global flags must precede `--`, whose remaining arguments are paths or messages. + result = subprocess.run(["jj", "--config-file", str(config), "--no-pager", *args], cwd=cwd, + env=dict(os.environ, JJ_CONFIG=str(config), XDG_CONFIG_HOME=temporary), + stdin=subprocess.DEVNULL) + return result.returncode if result.returncode >= 0 else 128 - result.returncode + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", help="Fixed host-side project directory") + args = parser.parse_args() + if os.geteuid() == 0: + parser.error("Run project-store as the workspace owner, without root privileges.") + request = sys.stdin.buffer.readline(4 * 1024 * 1024 + 1) + if len(request) > 4 * 1024 * 1024: + parser.error("Project request exceeds 4 MiB.") + try: + result = seed(Path(args.root).resolve(), json.loads(request)) + if isinstance(result, int): + return result + print(json.dumps(result)) + except subprocess.CalledProcessError as error: + return error.returncode if error.returncode >= 0 else 128 - error.returncode + except (OSError, ValueError, TypeError, KeyError) as error: + print(f"project-store: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/project-tools/client.py b/tools/project-tools/client.py new file mode 100644 index 0000000000000000000000000000000000000000..d9299f4b59c834c5fde8e48b475577ac6510391e --- /dev/null +++ b/tools/project-tools/client.py @@ -0,0 +1,201 @@ +"""Guest transport and lifecycle hooks for the shared project store.""" + +import errno +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time + + +ROOT = Path(os.environ.get("PROJECT_ROOT", "/Volumes/Code/project")).resolve() + + +def host(): + value = os.environ.get("PROJECT_STORE_SSH") + if value is None: + value = "project-store" if (Path.home() / ".ssh/project-store").is_file() else "" + if value and not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", value): + raise ValueError("Use an SSH alias for PROJECT_STORE_SSH.") + return value + + +def inside(path): + path = Path(path).resolve() + if not path.is_relative_to(ROOT): + raise ValueError(f"Path is outside the project share: {path}") + return path + + +def request(payload, *, capture=True): + return subprocess.run(["ssh", "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", + "--", host()], input=json.dumps(payload) + "\n", text=True, + stdout=subprocess.PIPE if capture else None) + + +def engine(): + return subprocess.check_output(["node", "-p", + "[process.platform,process.arch,'node'+process.versions.node].join(';')"], + text=True).strip() + + +def wait_workspace(destination): + deadline = time.monotonic() + 30 + while True: + try: + list(destination.parent.iterdir()) + list(destination.iterdir()) + list((destination / ".jj/working_copy").iterdir()) + return + except OSError as error: + if error.errno not in (errno.ENOENT, errno.ESTALE): + raise + if time.monotonic() >= deadline: + raise ValueError("Workspace was created but is not visible over NFS yet. Wait before opening it.") + time.sleep(0.1) + + +def pnpm_arguments(args, cwd): + command = None + forwarded = [] + parsed = iter(args) + for arg in parsed: + if arg in ("-C", "--dir"): + cwd = (cwd / next(parsed)).resolve() + elif arg.startswith("--dir="): + cwd = (cwd / arg.split("=", 1)[1]).resolve() + elif arg in ("--filter", "--filter-prod", "-F"): + forwarded.extend((arg, next(parsed))) + elif arg.startswith(("--filter=", "--filter-prod=")) or arg in { + "--frozen-lockfile", "--no-frozen-lockfile", "--offline", "--prefer-offline", + "--force", "--ignore-scripts", "--lockfile-only", "-r", "--recursive", + "-w", "--workspace-root", "-D", "--save-dev", "-P", "--save-prod", + "-O", "--save-optional", "--save-peer", "-E", "--save-exact", "--latest", "-L", + "--prod", "--dev", "--no-optional", "--ignore-workspace-root-check"}: + forwarded.append(arg) + elif command is None and arg in {"install", "i", "add", "update", "up", "upgrade", + "remove", "rm", "uninstall", "un", "uni"}: + command = arg + forwarded.append(arg) + elif command is not None and not arg.startswith("-"): + forwarded.append(arg) + else: + return cwd, None, arg if command is None and not arg.startswith("-") else command + return cwd, forwarded if command else None, command + + +def finish_install(workspace, native, *, ignore_scripts=False, modules_hash=None, cwd=None, args=()): + deadline = time.monotonic() + 30 + while True: + try: + list(workspace.iterdir()) + modules = workspace / "node_modules/.modules.yaml" + content = modules.read_bytes() + if modules_hash is None or hashlib.sha256(content).hexdigest() == modules_hash: + data = json.loads(content) + break + except OSError as error: + if error.errno not in (errno.ENOENT, errno.ESTALE): + raise + if time.monotonic() >= deadline: + raise ValueError("Installation is not visible over NFS yet. Retry after the mount catches up.") + time.sleep(0.1) + if ignore_scripts or not data.get("pendingBuilds"): + return + store = Path(data["storeDir"]) + target = workspace + command = ["--recursive", "--include-workspace-root", "rebuild", "--pending"] + extra = [] + if data["packageManager"].startswith("pnpm@11."): + store.mkdir(parents=True, exist_ok=True) + files = store / "files" + if not os.path.lexists(files): + relative = store.relative_to(Path.home() / "Library/Caches/project") + files.symlink_to(Path("/Volumes/Code/cache/project-pnpm") / relative / "files") + elif data["packageManager"].startswith("pnpm@12."): + # pnpm 12 rebuild can recopy patched packages that the storage host already prepared. + target = cwd or workspace + command = ["install", "--frozen-lockfile"] + filtered = False + parsed = iter(args) + for arg in parsed: + if arg in ("--filter", "--filter-prod", "-F"): + command.extend((arg, next(parsed))) + filtered = True + elif arg.startswith(("--filter=", "--filter-prod=")): + command.append(arg) + filtered = True + elif arg in ("-r", "--recursive", "-w", "--workspace-root", "--offline", "--prefer-offline"): + command.append(arg) + _, _, operation = pnpm_arguments(args, target) + if operation not in (None, "install", "i") and not filtered and not any( + arg in args for arg in ("-r", "--recursive")): + command.append("--filter=./") + included = data["included"] + if included["dependencies"] and not included["devDependencies"]: + command.append("--prod") + elif included["devDependencies"] and not included["dependencies"]: + command.append("--dev") + if not included["optionalDependencies"]: + command.append("--no-optional") + extra = ["--config.virtual-store-type=local", "--config.store-index-format=files", + "--config.side-effects-cache=false", "--config.side-effects-cache-readonly=false", + "--config.prefer-symlinked-executables=true", + "--config.optimistic-repeat-install=false"] + print("pnpm: Running Mac build hooks.", file=sys.stderr) + subprocess.run([str(native), "--dir", str(target), *command, + "--store-dir=" + str(store.parent), + "--config.cache-dir=" + str(workspace.parent / ".pnpm-cache"), + "--config.enable-global-virtual-store=false", "--config.verify-deps-before-run=false", *extra], + cwd=workspace, check=True) + + +VALUES = {"-R", "--repository", "--config", "--config-file", "--color", "--at-operation", "--at-op", + "-r", "--revision", "-s", "--source", "-d", "--destination", "-b", "--branch", + "--from", "--to", "-A", "--insert-after", "-B", "--insert-before", "-T", "--template", + "-m", "--message", "--name", "--sparse-patterns", "--author", "--tool", "--editor", + "-n", "--limit", "--sort", "--add", "--remove"} +GLOBALS = {"-R", "--repository", "--config", "--config-file", "--color", "--at-operation", "--at-op"} + + +def jj_arguments(args): + words, options, config = [], [], [] + args = iter(args) + for arg in args: + if arg == "--": + words.extend(args) + break + name, equal, value = arg.partition("=") + if name in VALUES: + value = value if equal else next(args) + options.append((name, value)) + if name in GLOBALS: + config.extend((name, value)) + elif len(arg) > 2 and arg[:2] in VALUES and not arg.startswith("--"): + name, value = arg[:2], arg[2:].removeprefix("=") + options.append((name, value)) + if name in GLOBALS: + config.extend((name, value)) + elif arg.startswith("-"): + options.append((arg, None)) + else: + words.append(arg) + return words, options, config + +def jj_paths(words, options): + if not words: + return [] + command = tuple(words[:2]) if words and words[0] in ("workspace", "file", "sparse") else tuple(words[:1]) + paths = [] + if command == ("workspace", "add"): + paths = words[2:] + elif command[0] in ("diff", "show", "restore"): + paths = words[1:] + elif command[0] == "file": + paths = words[2:] + elif command == ("sparse", "set"): + paths = [value for name, value in options if name in ("--add", "--remove")] + return paths diff --git a/tools/project-tools/finish-test.py b/tools/project-tools/finish-test.py new file mode 100644 index 0000000000000000000000000000000000000000..430e322ba14550cec61ac354f2b2161fb6441b5a --- /dev/null +++ b/tools/project-tools/finish-test.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Exercise native completion without recopying patches or widening selection.""" + +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile +import unittest + +import client + + +class FinishTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="infra-finish-", dir=client.ROOT) + self.addCleanup(temporary.cleanup) + self.workspace = Path(temporary.name) / "main" + self.workspace.mkdir() + self.native = Path.home() / ".local/bin/pnpm-real" + allow = {} + for name in ("allowed", "denied", "patched", "dev", "optional"): + package = dict(name="native-finish-" + name, version="1.0.0") + if name != "patched": + package["scripts"] = {"postinstall": + 'node -e "require(\'fs\').writeFileSync(\'built\',process.platform)"'} + with tarfile.open(self.workspace / (name + ".tgz"), "w:gz") as archive: + for filename, content in (("package.json", json.dumps(package).encode()), + ("index.js", b"module.exports = 'before';\n")): + entry = tarfile.TarInfo("package/" + filename) + entry.size = len(content) + archive.addfile(entry, io.BytesIO(content)) + allow[f"native-finish-{name}@file:{name}.tgz"] = name != "denied" + (self.workspace / "patched.patch").write_text( + "diff --git a/index.js b/index.js\n--- a/index.js\n+++ b/index.js\n@@ -1 +1 @@\n" + "-module.exports = 'before';\n+module.exports = 'after';\n") + (self.workspace / "pnpm-workspace.yaml").write_text(json.dumps(dict( + packages=["child", "other"], allowBuilds=allow, + patchedDependencies={"native-finish-patched@1.0.0": "patched.patch"}))) + root = dict(name="finish-root", private=True, packageManager="pnpm@12.14.0") + for directory, package in ((self.workspace, root), + (self.workspace / "child", dict(name="finish-child", private=True)), + (self.workspace / "other", dict(name="finish-other", private=True))): + directory.mkdir(exist_ok=True) + package["scripts"] = {"prepare": + 'node -e "require(\'fs\').writeFileSync(\'prepared\',process.platform)"'} + if directory.name == "child": + package["dependencies"] = {"native-finish-" + name: "file:../" + name + ".tgz" + for name in ("allowed", "denied", "patched")} + package["devDependencies"] = {"native-finish-dev": "file:../dev.tgz"} + package["optionalDependencies"] = {"native-finish-optional": "file:../optional.tgz"} + (directory / "package.json").write_text(json.dumps(package)) + + def finish(self, args, *, nested=False): + cwd = self.workspace / "child" if nested else self.workspace + response = client.request(dict(command="pnpm", cwd=str(cwd), engine=client.engine(), + args=[*args, "--ignore-scripts"])) + self.assertEqual(response.returncode, 0) + data = json.loads(response.stdout) + self.assertIs(data.get("installed"), True, data) + client.finish_install(self.workspace, None, ignore_scripts=True, modules_hash=data["modules_hash"]) + patched = self.workspace / "child/node_modules/native-finish-patched/index.js" + before = patched.stat() if patched.exists() else None + client.finish_install(self.workspace, self.native, modules_hash=data["modules_hash"], cwd=cwd, args=args) + if before: + self.assertEqual(patched.read_text(), "module.exports = 'after';\n") + self.assertEqual((patched.stat().st_ino, patched.stat().st_mtime_ns), + (before.st_ino, before.st_mtime_ns)) + self.assertFalse((self.workspace / "child/node_modules/native-finish-denied/built").exists()) + + def test_pending_dependency_and_workspace_hooks_preserve_patched_files(self): + self.finish(["install"]) + for name in ("allowed", "dev", "optional"): + self.assertEqual((self.workspace / f"child/node_modules/native-finish-{name}/built").read_text(), "darwin") + for directory in (self.workspace, self.workspace / "child", self.workspace / "other"): + self.assertEqual((directory / "prepared").read_text(), "darwin") + (directory / "prepared").unlink() + self.finish(["install", "--frozen-lockfile"]) + for directory in (self.workspace, self.workspace / "child", self.workspace / "other"): + self.assertEqual((directory / "prepared").read_text(), "darwin") + + def test_filtered_production_install_keeps_other_projects_and_excluded_groups_untouched(self): + self.finish(["--filter", "finish-child", "install", "--prod", "--no-optional"]) + self.assertEqual((self.workspace / "child/node_modules/native-finish-allowed/built").read_text(), "darwin") + for path in ("prepared", "child/prepared", "other/prepared", "child/node_modules/native-finish-dev", + "child/node_modules/native-finish-optional"): + self.assertFalse((self.workspace / path).exists(), path) + + def test_nested_add_only_finishes_the_selected_project(self): + self.finish(["add", "native-finish-allowed@file:../allowed.tgz"], nested=True) + self.assertEqual((self.workspace / "child/node_modules/native-finish-allowed/built").read_text(), "darwin") + self.assertEqual((self.workspace / "child/prepared").read_text(), "darwin") + self.assertFalse((self.workspace / "other/prepared").exists()) + + def test_development_only_install_does_not_add_production_dependencies(self): + self.finish(["install", "--dev", "--no-optional"]) + self.assertEqual((self.workspace / "child/node_modules/native-finish-dev/built").read_text(), "darwin") + self.assertEqual((self.workspace / "child/prepared").read_text(), "darwin") + self.assertFalse((self.workspace / "child/node_modules/native-finish-allowed").exists()) + + def test_inherited_agent_environment_keeps_native_pnpm_semantics(self): + manifest = self.workspace / "package.json" + package = json.loads(manifest.read_text()) + package["scripts"]["prepare"] += ( + ' && node -e "require(\'fs\').writeFileSync(\'mode\',process.env.NODE_ENV)"') + manifest.write_text(json.dumps(package)) + result = subprocess.run([str(Path(__file__).with_name("pnpm")), "install"], cwd=self.workspace, + env=dict(os.environ, NODE_ENV="production", NODE_OPTIONS="--max-old-space-size=8192", + npm_config_user_agent="pnpm/12.14.0 node/v26.12.0 darwin arm64"), + text=True, capture_output=True, timeout=120) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("pnpm: Running Mac build hooks.", result.stderr) + self.assertNotIn("Using Mac installation", result.stderr) + self.assertEqual((self.workspace / "mode").read_text(), "production") + self.assertEqual((self.workspace / "child/node_modules/native-finish-dev/built").read_text(), "darwin") + + +if __name__ == "__main__": + if sys.platform != "darwin" or not client.ROOT.is_dir(): + sys.exit("Run this test in the development VM with Code mounted.") + unittest.main() diff --git a/tools/project-tools/install.py b/tools/project-tools/install.py new file mode 100644 index 0000000000000000000000000000000000000000..64c87fb2623325f63d408561f78eda9ba7cef144 --- /dev/null +++ b/tools/project-tools/install.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Stage or activate project command wrappers in the development VM.""" + +import argparse +import os +from pathlib import Path +import shutil +import sys +import time + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("stage", "activate")) + parser.add_argument("release") + args = parser.parse_args() + if sys.platform != "darwin" or not os.path.ismount("/Volumes/Code") or os.geteuid() == 0: + parser.error("Run this installer as the development VM user with Code mounted.") + if not args.release or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789-." for c in args.release): + parser.error("Use a release name containing lowercase letters, numbers, dots, or hyphens.") + local = Path.home() / ".local" + base = local / "share/project-tools" + release = base / "releases" / args.release + names = ("client.py", "pnpm", "jj", "project") + if args.action == "stage": + release.mkdir(parents=True) + for name in names: + shutil.copy2(Path(__file__).with_name(name), release / name) + print(release) + return + if any(not (release / name).is_file() for name in names): + parser.error("Stage this release before activating it.") + if not (local / "bin/jj-real").is_file(): + parser.error("Install the native Jujutsu executable as ~/.local/bin/jj-real first.") + native_pnpm = local / "bin/pnpm-real" + if not native_pnpm.exists(): + original = (local / "bin/pnpm").resolve(strict=True) + with original.open("rb") as binary: + magic = binary.read(4) + if magic != b"\xcf\xfa\xed\xfe": + parser.error("Preserve the native pnpm executable as ~/.local/bin/pnpm-real first.") + native_pnpm.symlink_to(original) + overlay = local / "share/agents/vm" + current = base / "current" + if not current.is_symlink(): + previous = base / "releases" / ("before-recovery-" + time.strftime("%Y%m%d%H%M%S")) + previous.mkdir() + for name in ("jj", "project"): + shutil.copy2(overlay / name, previous / name) + (previous / "pnpm").symlink_to(native_pnpm.resolve()) + shutil.copy2(release / "client.py", previous / "client.py") + (base / "previous").symlink_to(previous.relative_to(base)) + elif current.resolve() != release.resolve(): + previous = base / ".previous-next" + previous.symlink_to(os.readlink(current)) + previous.replace(base / "previous") + link = base / ".current-next" + link.symlink_to(release.relative_to(base)) + link.replace(current) + for name in names: + link = overlay / ("." + name + "-next") + link.symlink_to(os.path.relpath(current / name, overlay)) + link.replace(overlay / name) + link = local / "bin/.pnpm-next" + link.symlink_to("../share/agents/vm/pnpm") + link.replace(local / "bin/pnpm") + for directory in (local / "share/vite-plus/package_manager/pnpm").glob("*/pnpm/bin"): + binary = directory / "pnpm" + if not binary.is_file(): + continue + native = directory / "pnpm-real" + if not native.exists(): + os.link(binary, native) + link = directory / ".pnpm-next" + link.symlink_to(os.path.relpath(overlay / "pnpm", directory)) + link.replace(binary) + print(release) + + +if __name__ == "__main__": + main() diff --git a/tools/project-tools/jj b/tools/project-tools/jj new file mode 100755 index 0000000000000000000000000000000000000000..c2319e4b91ec19c6218f10731585db1d005a974f --- /dev/null +++ b/tools/project-tools/jj @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Serialize Jujutsu commands across workspaces and NFS clients.""" + +import errno +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import client + + +def remote(executable, args, path): + if not client.host(): + return False + words, options, config_args = client.jj_arguments(args) + shared = path.is_relative_to(client.ROOT) + if Path.cwd().is_relative_to(client.ROOT) or shared: + for name, value in options: + if name in ("-R", "--repository", "--config-file"): + client.inside(value) + if not shared: + return False + client.inside(path) + if not words or words[0] in ("config", "git", "util", "help"): + return False + command = tuple(words[:2]) if words[0] in ("workspace", "file", "bookmark", "op", "sparse") else (words[0],) + safe = {("rebase",), ("status",), ("st",), ("log",), ("diff",), ("show",), ("edit",), + ("new",), ("abandon",), ("restore",), ("describe",), ("desc",), ("commit",), ("squash",), + ("workspace", "add"), ("workspace", "list"), ("workspace", "root"), + ("workspace", "forget"), ("workspace", "update-stale"), + ("file", "list"), ("file", "show"), ("file", "track"), ("file", "untrack"), + ("bookmark", "list"), ("bookmark", "create"), ("bookmark", "set"), + ("bookmark", "move"), ("bookmark", "delete"), ("bookmark", "forget"), + ("bookmark", "rename"), ("op", "log"), ("sparse", "list"), ("sparse", "set")} + if command not in safe: + return False + flags = {name for name, _ in options} + if flags & {"-i", "--interactive", "--tool", "--editor", "--edit", "--stdin", "--help", "-h"}: + return False + if command[0] in ("describe", "desc", "commit", "squash") and not flags & { + "-m", "--message", "--no-edit", "--use-destination-message"}: + return False + paths = client.jj_paths(words, options) + for value in paths: + if any(c in value for c in ':"|&~()'): + return False + client.inside(value) + base = [str(executable), *config_args, "--ignore-working-copy", "--no-pager", "config", "list"] + rows = subprocess.check_output([*base, "-T", 'json(self) ++ "\\n"'], text=True) + settings = {row["name"]: row["value"] for row in map(json.loads, rows.splitlines())} + if settings.get("signing.behavior", "drop") != "drop" or settings.get("snapshot.fsmonitor", "none") != "none": + return False + if command[0] in ("diff", "show", "log") and not flags & {"--git", "--summary", "--stat", "--types", "--name-only"}: + formatter = settings.get("ui.diff-formatter", ":color-words") + if not isinstance(formatter, str) or not formatter.startswith(":"): + return False + if sys.stdout.isatty() and "--no-pager" not in flags and settings.get("ui.paginate", "auto") != "never": + return False + config = subprocess.check_output([*base, "-T", 'name ++ " = " ++ value ++ "\\n"'], text=True) + version = subprocess.check_output([str(executable), "--version"], text=True).strip() + ignore = Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config"))) / "git/ignore" + cwd = Path.cwd() + if not cwd.is_relative_to(client.ROOT): + if any(not Path(value).is_absolute() for value in paths): + raise ValueError("Run commands with file paths from inside the project share.") + cwd = path + forwarded = ["-R", str(path)] + original = iter(args) + for arg in original: + if arg == "--": + forwarded.extend((arg, *original)) + break + if arg in ("-R", "--repository", "--config-file", "--config"): + next(original) + elif arg.startswith(("-R", "--repository=", "--config-file=", "--config=")): + continue + else: + forwarded.append(arg) + if arg in client.VALUES: + forwarded.append(next(original)) + result = client.request(dict(command="jj", cwd=str(cwd), args=forwarded, config=config, version=version, + ignore=ignore.read_text() if ignore.is_file() else ""), capture=False) + if result.returncode == 0 and command == ("workspace", "add") and len(paths) == 1: + client.wait_workspace(Path(paths[0]).absolute()) + sys.exit(result.returncode) + + +def main(): + executable = Path(sys.argv[0]).absolute().with_name("jj-real") + path = Path.cwd() + _, options, _ = client.jj_arguments(sys.argv[1:]) + for name, value in options: + if name in ("-R", "--repository"): + path = Path(value) + path = path.resolve() + remote(executable, sys.argv[1:], path) + for parent in (path, *path.parents): + metadata = parent / ".jj" + if not metadata.is_dir(): + continue + repository = metadata / "repo" + if repository.is_file(): + repository = (metadata / repository.read_text()).resolve(strict=True) + # Never unlink this inode: existing waiters must keep using the same lock. + descriptor = os.open(repository / "command.lock", os.O_CREAT | os.O_RDWR, 0o600) + # macOS NFS drops lockf across exec; flock maps to the server's POSIX lock. + lock = fcntl.flock if sys.platform == "darwin" else fcntl.lockf + try: + lock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as error: + if error.errno not in (errno.EACCES, errno.EAGAIN): + raise + print("jj: Waiting for another command in this repository…", file=sys.stderr, flush=True) + lock(descriptor, fcntl.LOCK_EX) + # jj owns the lock even if its caller dies. + os.set_inheritable(descriptor, True) + break + os.execv(executable, ["jj", *sys.argv[1:]]) + + +if __name__ == "__main__": + try: + main() + except StopIteration: + os.execv(Path(sys.argv[0]).absolute().with_name("jj-real"), ["jj", *sys.argv[1:]]) + except KeyboardInterrupt: + sys.exit(130) + except subprocess.CalledProcessError as error: + sys.exit(error.returncode) + except (OSError, ValueError) as error: + print(f"jj: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tools/project-tools/jj-test.py b/tools/project-tools/jj-test.py new file mode 100644 index 0000000000000000000000000000000000000000..d780bdbeee9994667e29d19eeba7295e9ed4206a --- /dev/null +++ b/tools/project-tools/jj-test.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Exercise command locking with disposable repositories and real jj processes.""" + +import fcntl +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import unittest + + +lock_file = fcntl.flock if sys.platform == "darwin" else fcntl.lockf + + +class CommandLockTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="jj-lock-test-") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.bin = self.root / "bin" + self.bin.mkdir() + shutil.copy2(Path(__file__).with_name("jj"), self.bin / "jj") + shutil.copy2(Path(__file__).with_name("client.py"), self.bin / "client.py") + native = Path(shutil.which("jj")) + if native.with_name("jj-real").exists(): + native = native.with_name("jj-real") + (self.bin / "jj-real").symlink_to(native) + config = self.root / "config.toml" + config.write_text('[user]\nname="Fixture"\nemail="fixture@example.invalid"\n' + '[signing]\nbehavior="drop"\n') + self.env = dict(os.environ, JJ_CONFIG=str(config), JJ_EDITOR="true", PROJECT_STORE_SSH="", + GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=str(self.root / "absent")) + self.repo = self.root / "main" + self.run_jj("git", "init", "--no-colocate", str(self.repo)) + self.other = self.root / "other" + self.run_jj("git", "init", "--no-colocate", str(self.other)) + self.workspace = self.root / "agent workspace" + self.run_jj("-R", str(self.repo), "workspace", "add", str(self.workspace)) + self.lock = self.repo / ".jj/repo/command.lock" + + def run_jj(self, *args, cwd=None, check=True): + return subprocess.run([str(self.bin / "jj"), *args], cwd=cwd or self.root, + env=self.env, capture_output=True, text=True, timeout=10, check=check) + + def start(self, *args, cwd=None): + process = subprocess.Popen([str(self.bin / "jj"), *args], cwd=cwd or self.root, + env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, start_new_session=True) + def cleanup(): + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.communicate(timeout=5) + self.addCleanup(cleanup) + return process + + def holder(self): + marker = self.root / "ready" + process = self.start("-R", str(self.repo), "util", "exec", "--", sys.executable, "-c", + "from pathlib import Path; import time; " + f"Path({str(marker)!r}).touch(); time.sleep(30)") + deadline = time.monotonic() + 10 + while not marker.exists(): + if process.poll() is not None or time.monotonic() > deadline: + self.fail("Holder did not start") + time.sleep(0.02) + with self.lock.open("r+") as lock: + with self.assertRaises(BlockingIOError): + lock_file(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + return process + + def test_shared_workspace_lock_and_independent_repository(self): + holder = self.holder() + waiting = self.start("-R", str(self.workspace), "status") + time.sleep(0.2) + self.assertIsNone(waiting.poll()) + self.run_jj("-R", str(self.other), "describe", "-m", "independent") + os.killpg(holder.pid, signal.SIGKILL) + stdout, stderr = waiting.communicate(timeout=10) + self.assertEqual(waiting.returncode, 0, stderr) + self.assertIn("Waiting for another command", stderr) + self.assertNotIn("Waiting", stdout) + self.assertFalse((self.workspace / ".jj/command.lock").exists()) + + def test_selectors_and_snapshotting_reads_wait(self): + subdir = self.workspace / "subdir" + subdir.mkdir() + link = self.root / "linked" + link.symlink_to(subdir) + commands = [ + (["status"], link), + (["--repository=" + str(self.workspace), "diff"], self.root), + (["-R" + str(self.workspace), "log", "--no-graph", "-r", "@"], self.root), + (["--ignore-working-copy", "-R", str(self.workspace), "describe", "-m", "guarded"], self.root), + (["--config", 'user.name="-Rignored"', "-R", str(self.workspace), "status"], self.root), + ] + with self.lock.open("r+") as lock: + lock_file(lock, fcntl.LOCK_EX) + processes = [self.start(*args, cwd=cwd) for args, cwd in commands] + time.sleep(0.3) + self.assertTrue(all(process.poll() is None for process in processes)) + for process in processes: + _, stderr = process.communicate(timeout=10) + self.assertEqual(process.returncode, 0, stderr) + + def test_cancelled_waiter_and_failed_command_release(self): + with self.lock.open("r+") as lock: + lock_file(lock, fcntl.LOCK_EX) + waiting = self.start("-R", str(self.workspace), "status") + time.sleep(0.2) + waiting.send_signal(signal.SIGINT) + waiting.communicate(timeout=5) + self.assertEqual(waiting.returncode, 130) + self.assertNotEqual(self.run_jj("-R", str(self.repo), "unknown-command", check=False).returncode, 0) + self.run_jj("-R", str(self.workspace), "status") + self.assertEqual(self.run_jj("--version").returncode, 0) + + def test_concurrent_mutations_have_one_operation_head(self): + processes = [self.start("-R", str(self.repo), "new", "root()", "-m", f"fixture {i}") + for i in range(12)] + for process in processes: + _, stderr = process.communicate(timeout=10) + self.assertEqual(process.returncode, 0, stderr) + self.assertNotIn("Concurrent modification", stderr) + heads = self.repo / ".jj/repo/op_heads/heads" + self.assertEqual(len(list(heads.iterdir())), 1) + descriptions = self.run_jj("-R", str(self.repo), "log", "--no-graph", "-r", "all()", + "-T", 'description ++ "\n"').stdout + for i in range(12): + self.assertIn(f"fixture {i}\n", descriptions) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/project-tools/platform-test.py b/tools/project-tools/platform-test.py new file mode 100644 index 0000000000000000000000000000000000000000..7036146c9a8d6a0af91b14bb8ed89d41efd53a39 --- /dev/null +++ b/tools/project-tools/platform-test.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Check storage-host platform selection before any native rebuild can repair it.""" + +import io +import json +from pathlib import Path +import sys +import tarfile +import tempfile +import unittest + +import client + + +class PlatformTest(unittest.TestCase): + def test_only_darwin_arm64_optional_packages_are_imported(self): + with tempfile.TemporaryDirectory(prefix="infra-platform-", dir=client.ROOT) as temporary: + workspace = Path(temporary) / "main" + workspace.mkdir() + optional = {} + for platform, cpu in (("darwin", "arm64"), ("linux", "x64")): + name = f"@project-platform-test/{platform}-{cpu}" + package = dict(name=name, version="1.0.0", os=[platform], cpu=[cpu]) + content = json.dumps(package).encode() + archive = workspace / f"{platform}-{cpu}.tgz" + with tarfile.open(archive, "w:gz") as output: + entry = tarfile.TarInfo("package/package.json") + entry.size = len(content) + output.addfile(entry, io.BytesIO(content)) + optional[name] = "file:" + archive.name + (workspace / "package.json").write_text(json.dumps(dict( + name="platform-fixture", private=True, packageManager="pnpm@12.14.0", + optionalDependencies=optional))) + response = client.request(dict(command="pnpm", cwd=str(workspace), + engine=client.engine(), args=["install", "--ignore-scripts"])) + self.assertEqual(response.returncode, 0) + result = json.loads(response.stdout) + self.assertIs(result.get("installed"), True, result) + client.finish_install(workspace, None, ignore_scripts=True, modules_hash=result["modules_hash"]) + packages = workspace / "node_modules/@project-platform-test" + self.assertTrue((packages / "darwin-arm64/package.json").is_file()) + self.assertFalse((packages / "linux-x64").exists()) + + +if __name__ == "__main__": + if sys.platform != "darwin" or not client.ROOT.is_dir(): + sys.exit("Run this test in the development VM with Code mounted.") + unittest.main() diff --git a/tools/project-tools/pnpm b/tools/project-tools/pnpm new file mode 100755 index 0000000000000000000000000000000000000000..e0e872c9de105041c7a3662ea0d271c7cc5bbd3f --- /dev/null +++ b/tools/project-tools/pnpm @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Install dependencies on the storage host; run programs and build hooks on Mac.""" + +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import client + + +def main(): + native = Path(sys.argv[0]).absolute().with_name("pnpm-real") + args = sys.argv[1:] + cwd, forwarded, command = client.pnpm_arguments(args, Path.cwd()) + prepare = command in ("run", "run-script", "exec", "test", "t", "tst", "start", "stop", "restart") + if not prepare and command and cwd.is_relative_to(client.ROOT): + for parent in (cwd, *cwd.parents): + if parent == client.ROOT: + break + manifest = parent / "package.json" + if manifest.is_file(): + prepare = command in json.loads(manifest.read_text()).get("scripts", {}) + break + if forwarded or prepare: + custom_environment = [key for key, value in os.environ.items() if value and ( + (key.lower().startswith(("npm_config_", "pnpm_config_")) + and key.lower() not in ("npm_config_prefix", "npm_config_cache", "npm_config_user_agent", + "pnpm_config_update_notifier")) + or key in ("VP_PACKAGE_MANAGER", "NODE_PATH") + or (key == "NODE_OPTIONS" and not re.fullmatch( + r"(?:\s*--max[-_](?:old|semi)[-_]space[-_]size=\d+)+\s*", value)))] + if custom_environment and cwd.is_relative_to(client.ROOT) and client.host(): + print("pnpm: Using Mac installation: custom environment (" + + ", ".join(sorted(custom_environment)) + ").", file=sys.stderr) + if not custom_environment and cwd.is_relative_to(client.ROOT) and client.host(): + client.inside(cwd) + request = dict(command="pnpm", cwd=str(cwd), engine=client.engine(), args=forwarded or []) + if os.environ.get("NODE_ENV") == "production": + request["node_env"] = "production" + if prepare: + request["prepare"] = True + else: + print("pnpm: Installing dependencies on the storage host.", file=sys.stderr, flush=True) + result = client.request(request) + if result.returncode: + return result.returncode + response = json.loads(result.stdout) + if response.get("ready") is True: + os.execv(native, [str(native), "--config.verify-deps-before-run=false", *args]) + if response.get("installed") is True: + if prepare or "--lockfile-only" not in forwarded: + client.finish_install(Path(response.get("workspace", cwd)), native, + ignore_scripts=not prepare and "--ignore-scripts" in forwarded, + modules_hash=response.get("modules_hash"), cwd=cwd, + args=forwarded or ()) + if prepare: + os.execv(native, [str(native), "--config.verify-deps-before-run=false", *args]) + return 0 + print("pnpm: Using Mac installation: " + response["reason"], file=sys.stderr) + store = response.get("store") + modules = cwd / "node_modules/.modules.yaml" + if modules.is_file(): + try: + current = json.loads(modules.read_text())["storeDir"] + except (ValueError, KeyError): + pass + else: + if isinstance(current, str) and Path(current).is_absolute(): + store = str(Path(current).parent) + if store: + args = [*args, "--store-dir=" + store] + os.execv(native, [str(native), *args]) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except StopIteration: + os.execv(Path(sys.argv[0]).absolute().with_name("pnpm-real"), ["pnpm", *sys.argv[1:]]) + except KeyboardInterrupt: + sys.exit(130) + except subprocess.CalledProcessError as error: + sys.exit(error.returncode) + except (OSError, ValueError, KeyError) as error: + print(f"pnpm: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tools/project-tools/project b/tools/project-tools/project new file mode 100755 index 0000000000000000000000000000000000000000..f4c47e43d58544cca9af76a207db0e833b06c688 --- /dev/null +++ b/tools/project-tools/project @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Manage shared projects and run read-only tools beside their ZFS storage.""" + +import argparse +from contextlib import ExitStack +import fcntl +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import client + + +def component(value): + if ( + not value + or value in (".", "..") + or value != value.strip() + or any(c in value for c in "/\\") + or any(ord(c) < 32 or ord(c) == 127 for c in value) + ): + raise argparse.ArgumentTypeError("Use one folder name without slashes or control characters.") + return value + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + epilog="Prints the checkout path on success; jj progress goes to stderr. " + "PROJECT_ROOT overrides the project directory, which must already exist.", + ) + commands = parser.add_subparsers(dest="command", required=True) + read = commands.add_parser("read", help="Search, list, or hash shared files on the storage host") + read.add_argument("tool", choices=("rg", "sha256sum")) + read.add_argument("args", nargs=argparse.REMAINDER) + clone = commands.add_parser("clone", help="Clone a Git repository into PROJECT/main") + clone.add_argument("source", metavar="URL") + clone.add_argument("project", type=component, metavar="PROJECT") + worktree = commands.add_parser("worktree", help="Create a workspace and install pnpm dependencies") + worktree.add_argument("project", type=component, metavar="PROJECT") + worktree.add_argument("identity", type=component, metavar="NAME") + worktree.add_argument("--from", dest="revision", default="trunk()", metavar="REV", + help="Parent revision, resolved in main (default: trunk())") + worktree.add_argument("--warm-from", type=component, metavar="WORKSPACE", + help="Clone an idle workspace and its build outputs on the ZFS host") + worktree.add_argument("--target", action="append", metavar="PATH", + help="Target directory relative to each workspace; repeat as needed (default: target)") + worktree.add_argument("--no-install", action="store_true", help="Create the workspace without installing dependencies") + args = parser.parse_args() + + root = Path(os.environ.get("PROJECT_ROOT", "/Volumes/Code/project")).expanduser().absolute() + if "PROJECT_ROOT" not in os.environ and not os.path.ismount("/Volumes/Code"): + parser.error("/Volumes/Code is not mounted. Restore the mount before creating a checkout.") + if not root.is_dir(): + parser.error(f"Project directory is missing: {root}") + if args.command == "read": + cwd = client.inside(Path.cwd()) + if not client.host(): + parser.error("Storage access is unavailable. Set PROJECT_STORE_SSH to its SSH alias.") + return client.request(dict(command="read", cwd=str(cwd), tool=args.tool, args=args.args), + capture=False).returncode + project = root / args.project + if project.is_symlink(): + parser.error(f"Project directory is a symlink: {project}") + destination = project / ("main" if args.command == "clone" else args.identity) + if destination.exists() or destination.is_symlink(): + parser.error(f"Checkout already exists: {destination}. Choose another name.") + if args.command == "clone": + if project.exists(): + parser.error(f"Project already exists: {project}. Use project worktree or another name.") + command = ["jj", "git", "clone", "--no-colocate", "--", args.source, str(destination)] + else: + if args.identity in ("main", "default"): + parser.error("main and default are reserved. Choose another worktree name.") + primary = project / "main" + if primary.is_symlink() or not (primary / ".jj").is_dir(): + parser.error(f"No primary Jujutsu checkout at {primary}. Clone the project first.") + command = ["jj", "-R", str(primary), "workspace", "add", "--name", args.identity, + "-r", args.revision, "--", str(destination)] + + if "PROJECT_STORE_SSH" not in os.environ and (Path.home() / ".ssh/project-store").is_file(): + os.environ["PROJECT_STORE_SSH"] = "project-store" + with ExitStack() as locks: + warm = args.command == "worktree" and args.warm_from + if args.command == "worktree" and args.target and not warm: + parser.error("--target requires --warm-from.") + if warm: + host = os.environ.get("PROJECT_STORE_SSH", "") + if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", host): + parser.error("Set PROJECT_STORE_SSH to the SSH alias for the project-store forced command.") + if os.environ.get("CARGO_TARGET_DIR") or os.environ.get("CARGO_BUILD_TARGET_DIR"): + parser.error("Unset the Cargo target override before seeding worktree-local targets.") + donor = project / args.warm_from + if donor.is_symlink() or not (donor / ".jj").is_dir(): + parser.error(f"No donor workspace at {donor}.") + targets = args.target or ["target"] + for target in targets: + path = Path(target) + if (path.is_absolute() or not path.parts or + any(p in ("..", ".jj", ".git") for p in path.parts)): + parser.error("Use target paths inside the workspace.") + source = donor / path + if not source.is_dir() or source.resolve() != source.absolute(): + parser.error(f"Target is missing or follows a symlink: {source}") + lock_paths = sorted(set(source.glob("*/.cargo-lock")) | + set(source.glob("*/*/.cargo-lock"))) + if not lock_paths: + parser.error(f"No Cargo build locks in {source}. Build the donor first.") + for lock_path in lock_paths: + if lock_path.resolve() != lock_path.absolute(): + parser.error(f"Cargo build lock follows a symlink: {lock_path}") + fd = os.open(lock_path, os.O_RDWR | os.O_NOFOLLOW) + locks.callback(os.close, fd) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + parser.error(f"Donor is building: {source}. Retry after the build finishes.") + return create(parser, args, project, destination, command, warm) + + +def create(parser, args, project, destination, command, warm): + memory = Path.home() / ".local/share/agents/memory" / args.project + slug = re.sub(r"[^a-zA-Z0-9_-]", "-", str(destination)) + memory_link = Path.home() / ".claude/projects" / slug / "memory" + if os.path.lexists(memory_link) and memory_link.resolve() != memory.resolve(): + parser.error(f"Another memory directory owns {memory_link}. Resolve it before creating this checkout.") + if args.command == "clone": + project.mkdir() + host = os.environ.get("PROJECT_STORE_SSH") + if warm: + request = dict(project=args.project, source=args.warm_from, destination=args.identity, + revision=args.revision, targets=args.target or ["target"]) + started = time.monotonic() + result = subprocess.run(["ssh", "-T", "-o", "BatchMode=yes", "--", host], + input=json.dumps(request), text=True, stdout=subprocess.PIPE) + if result.returncode: + return result.returncode + print(f"project: Host setup finished in {time.monotonic() - started:.1f}s.", file=sys.stderr) + try: + response = json.loads(result.stdout) + if not isinstance(response, dict) or response.get("workspace") != args.identity: + raise ValueError() + except (ValueError, TypeError): + print(f"project: Invalid storage response. Inspect {destination} before building.", file=sys.stderr) + return 1 + client.wait_workspace(destination) + else: + result = subprocess.run(command, stdout=sys.stderr) + if result.returncode: + if args.command == "clone": + try: + project.rmdir() + except OSError: + pass + return result.returncode + + memory.mkdir(parents=True, exist_ok=True, mode=0o700) + if not (memory / "MEMORY.md").exists(): + (memory / "MEMORY.md").touch(mode=0o600) + memory_link.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if not os.path.lexists(memory_link): + memory_link.symlink_to(os.path.relpath(memory, memory_link.parent), target_is_directory=True) + if args.command == "worktree" and not args.no_install: + manifest = destination / "package.json" + if manifest.is_file(): + package = json.loads(manifest.read_text()) + if package.get("packageManager", "").startswith("pnpm@"): + subprocess.run(["pnpm", "install", "--frozen-lockfile"], cwd=destination, + stdout=sys.stderr, check=True) + print(destination) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + print(f"project: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tools/project-tools/test.py b/tools/project-tools/test.py new file mode 100644 index 0000000000000000000000000000000000000000..08f9ab35361e1b310c31cffa8bb336793a6c0cd5 --- /dev/null +++ b/tools/project-tools/test.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Exercise routing with real Jujutsu repositories and a local SSH stand-in.""" + +import hashlib +import fcntl +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +import client + + +HERE = Path(__file__).resolve().parent + + +class RoutingTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="project-routing-") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.projects = self.root / "project" + self.projects.mkdir() + self.bin = self.root / "bin" + self.bin.mkdir() + for name in ("jj", "pnpm", "project", "client.py"): + shutil.copy2(HERE / name, self.bin / name) + native = Path(shutil.which("jj")) + if native.with_name("jj-real").exists(): + native = native.with_name("jj-real") + (self.bin / "jj-real").symlink_to(native) + self.requests = self.root / "requests" + self.native = self.root / "native" + (self.bin / "pnpm-real").write_text(f'''#!{sys.executable} +import json, pathlib, sys +pathlib.Path({str(self.native)!r}).write_text(json.dumps(sys.argv[1:])) +print("native output") +sys.exit(23) +''') + (self.bin / "pnpm-real").chmod(0o755) + (self.bin / "ssh").write_text(f'''#!{sys.executable} +import json, os, pathlib, subprocess, sys +request = json.load(sys.stdin) +with open({str(self.requests)!r}, "a") as log: log.write(json.dumps(request) + "\\n") +if request["command"] == "pnpm": + print(json.dumps({{"ready": True}} if request.get("prepare") else {{"installed": False, "reason": "fixture miss"}})) + sys.exit(0) +sys.exit(subprocess.run([{sys.executable!r}, {str(HERE.parent / "project-store.py")!r}, {str(self.projects)!r}], + input=json.dumps(request), text=True, env=dict(os.environ, PROJECT_STORE_SSH="")).returncode) +''') + (self.bin / "ssh").chmod(0o755) + config = self.root / "jj.toml" + config.write_text('[user]\nname="Guest Identity"\nemail="guest@example.invalid"\n' + '[signing]\nbehavior="drop"\n[ui]\ndiff-formatter=":git"\n') + self.env = dict(os.environ, PATH=str(self.bin) + os.pathsep + os.environ["PATH"], + JJ_CONFIG=str(config), PROJECT_ROOT=str(self.projects), PROJECT_STORE_SSH="fixture", + GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=str(self.root / "absent")) + for key in list(self.env): + if key.lower().startswith(("npm_config_", "pnpm_config_")) or key in ( + "NODE_ENV", "NODE_OPTIONS", "NODE_PATH", "VP_PACKAGE_MANAGER"): + self.env.pop(key) + self.main = self.projects / "Example/main" + self.main.parent.mkdir() + self.run_tool("jj-real", "git", "init", "--no-colocate", str(self.main), cwd=self.root) + (self.main / "source").write_text("base\n") + self.run_tool("jj-real", "describe", "-m", "base") + + def run_tool(self, tool, *args, cwd=None, check=True): + result = subprocess.run([str(self.bin / tool), *args], cwd=cwd or self.main, + env=self.env, text=True, capture_output=True, timeout=15) + if check: + self.assertEqual(result.returncode, 0, result.stderr) + return result + + def test_workspace_and_mutations_keep_paths_identity_and_dirty_content(self): + destination = self.main.parent / "agent space" + result = self.run_tool("jj", "workspace", "add", "--", str(destination)) + self.assertIn("agent space", result.stderr) + (destination / "dirty").write_text("unsaved agent work\n") + self.run_tool("jj", "new", "-m", "child", cwd=destination) + self.run_tool("jj", "rebase", "-s", "@-", "-d", "root()", cwd=destination) + self.assertEqual((destination / "dirty").read_text(), "unsaved agent work\n") + author = self.run_tool("jj", "log", "-r", "@", "--no-graph", "-T", "author.email()", cwd=destination) + self.assertEqual(author.stdout, "guest@example.invalid") + self.assertEqual(len(self.requests.read_text().splitlines()), 4) + self.assertFalse(Path((destination / ".jj/repo").read_text()).is_absolute()) + + def test_paths_cannot_escape_share_even_through_symlinks(self): + outside = self.root / "outside" + outside.mkdir() + (self.main / "escape").symlink_to(outside) + for args in [("workspace", "add", str(outside / "new")), + ("workspace", "add", "../../../../escape"), + ("workspace", "add", "escape/new"), + ("-R", str(outside), "status"), + ("file", "show", str(outside / "file"))]: + with self.subTest(args=args): + result = self.run_tool("jj", *args, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("outside the project share", result.stderr) + self.assertFalse(self.requests.exists()) + + def test_remote_errors_preserve_exit_status_without_local_retry(self): + result = self.run_tool("jj", "rebase", "-r", "missing-revision", "-d", "root()", check=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("missing-revision", result.stderr) + self.assertEqual(len(self.requests.read_text().splitlines()), 1) + + def test_relative_repository_selection_from_outside_the_share(self): + destination = self.main.parent / "selected" + result = self.run_tool("jj", "-R", str(self.main.relative_to(self.root)), + "--config", 'user.email="override@example.invalid"', + "workspace", "add", "-r", "@", str(destination), cwd=self.root) + self.assertEqual(result.returncode, 0) + author = self.run_tool("jj", "log", "--no-graph", "-r", "@", "-T", "author.email()", cwd=destination) + self.assertEqual(author.stdout, "override@example.invalid") + + def test_local_repositories_and_interactive_commands_stay_local(self): + local = self.root / "local" + self.run_tool("jj-real", "git", "init", "--no-colocate", str(local)) + self.run_tool("jj", "status", cwd=local) + self.env["JJ_EDITOR"] = "true" + self.run_tool("jj", "describe") + self.assertFalse(self.requests.exists()) + + def test_pnpm_scripts_and_unknown_options_keep_native_argv_and_exit(self): + for args in [("run", "build"), ("exec", "node", "-p", "process.platform"), + ("build",), ("install", "--config.ignore-scripts=true"), + ("--version",), ("run", "install")]: + with self.subTest(args=args): + result = self.run_tool("pnpm", *args, check=False) + self.assertEqual(result.returncode, 23) + self.assertEqual(result.stdout, "native output\n") + prefix = ["--config.verify-deps-before-run=false"] if args[0] in ("run", "exec") else [] + self.assertEqual(json.loads(self.native.read_text()), prefix + list(args)) + requests = [json.loads(line) for line in self.requests.read_text().splitlines()] + self.assertTrue(all(request["prepare"] and request["args"] == [] for request in requests)) + + def test_script_shortcut_checks_dependencies_remotely_then_runs_natively(self): + (self.main / "package.json").write_text(json.dumps({"scripts": {"build": "never run on Linux"}})) + result = self.run_tool("pnpm", "build", "--flag", check=False) + self.assertEqual(result.returncode, 23) + self.assertEqual(json.loads(self.requests.read_text())["args"], []) + self.assertEqual(json.loads(self.native.read_text()), ["--config.verify-deps-before-run=false", "build", "--flag"]) + + def test_handoff_waits_for_the_servers_modules_manifest(self): + modules = self.main / "node_modules/.modules.yaml" + modules.parent.mkdir() + modules.write_text('{"pendingBuilds":[]}') + content = b'{"pendingBuilds":["new-native-package"]}' + def publish(): + time.sleep(0.15) + temporary = modules.with_suffix(".next") + temporary.write_bytes(content) + temporary.replace(modules) + publisher = threading.Thread(target=publish) + publisher.start() + self.addCleanup(publisher.join) + client.finish_install(self.main, self.bin / "pnpm-real", ignore_scripts=True, + modules_hash=hashlib.sha256(content).hexdigest()) + self.assertEqual(modules.read_bytes(), content) + + def test_workspace_handoff_waits_for_metadata_visibility(self): + destination = self.main.parent / "new workspace" + def publish(): + time.sleep(0.15) + (destination / ".jj/working_copy").mkdir(parents=True) + publisher = threading.Thread(target=publish) + publisher.start() + self.addCleanup(publisher.join) + client.wait_workspace(destination) + self.assertTrue((destination / ".jj/working_copy").is_dir()) + + def test_install_miss_falls_back_once_with_exact_arguments(self): + self.env.update(npm_config_prefix=str(self.root / "local"), npm_config_cache=str(self.root / "cache"), + PNPM_CONFIG_UPDATE_NOTIFIER="false") + args = ["--dir", str(self.main), "install", "--frozen-lockfile", "--offline"] + result = self.run_tool("pnpm", *args, check=False) + self.assertEqual(result.returncode, 23) + self.assertIn("fixture miss", result.stderr) + self.assertEqual(json.loads(self.native.read_text()), args) + self.assertEqual(json.loads(self.requests.read_text())["args"], ["install", "--frozen-lockfile", "--offline"]) + + def test_dependency_changes_and_filters_use_storage_before_native_hooks(self): + for args in [("add", "-D", "is-number@7.0.0"), ("update", "is-number"), + ("remove", "is-number"), ("--filter", "app", "install", "--lockfile-only")]: + with self.subTest(args=args): + result = self.run_tool("pnpm", *args, check=False) + self.assertEqual(result.returncode, 23) + request = json.loads(self.requests.read_text().splitlines()[-1]) + self.assertEqual(request["args"], list(args)) + self.assertRegex(request["engine"], r";node\d+\.\d+\.\d+$") + + def test_install_miss_keeps_the_existing_store(self): + modules = self.main / "node_modules/.modules.yaml" + modules.parent.mkdir() + store = self.root / "cache/v10" + modules.write_text(json.dumps({"storeDir": str(store)})) + result = self.run_tool("pnpm", "install", check=False) + self.assertEqual(result.returncode, 23) + self.assertEqual(json.loads(self.native.read_text()), ["install", "--store-dir=" + str(store.parent)]) + + def test_custom_install_environment_stays_native(self): + for key, value in (("npm_config_ignore_scripts", "true"), + ("PNPM_CONFIG_STORE_DIR", str(self.root / "custom-store")), + ("VP_PACKAGE_MANAGER", "pnpm@12.15.0"), + ("NODE_OPTIONS", "--require=/private/agent-hook.cjs"), + ("NODE_PATH", "/private/agent-modules")): + with self.subTest(key=key): + self.env[key] = value + result = self.run_tool("pnpm", "install", check=False) + self.assertEqual(result.returncode, 23) + self.assertEqual(json.loads(self.native.read_text()), ["install"]) + self.assertIn("Using Mac installation: custom environment (" + key + ")", result.stderr) + self.assertNotIn(value, result.stderr) + self.env.pop(key) + self.assertFalse(self.requests.exists()) + + def test_inherited_metadata_and_memory_limits_still_offload(self): + self.env["npm_config_user_agent"] = "pnpm/12.14.0 npm/? node/v26.12.0 darwin arm64" + for options in ("--max-old-space-size=8192", "--max_old_space_size=8192", + "--max-old-space-size=8192 --max-semi-space-size=32"): + with self.subTest(options=options): + self.env["NODE_OPTIONS"] = options + result = self.run_tool("pnpm", "install", "--frozen-lockfile", check=False) + self.assertEqual(result.returncode, 23) + self.assertIn("fixture miss", result.stderr) + self.assertNotIn("custom environment", result.stderr) + request = json.loads(self.requests.read_text().splitlines()[-1]) + self.assertEqual(request["args"], ["install", "--frozen-lockfile"]) + + def test_production_environment_reaches_pnpm_without_changing_arguments(self): + self.env["NODE_ENV"] = "production" + for args in (("install",), ("install", "--dev"), ("add", "is-number@7.0.0"), ("exec", "node")): + with self.subTest(args=args): + result = self.run_tool("pnpm", *args, check=False) + self.assertEqual(result.returncode, 23) + request = json.loads(self.requests.read_text().splitlines()[-1]) + self.assertEqual(request["node_env"], "production") + self.assertEqual(request["args"], [] if args[0] == "exec" else list(args)) + + def test_storage_reads_preserve_paths_output_and_ignore_files_without_snapshotting(self): + (self.main / "file with spaces").write_text("needle\n") + (self.main / ".gitignore").write_text("node_modules/\n") + (self.main / "node_modules").mkdir() + (self.main / "node_modules/ignored").write_text("needle\n") + with (self.main / ".jj/repo/command.lock").open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + listed = self.run_tool("project", "read", "rg", "--files") + self.assertIn("file with spaces", listed.stdout.splitlines()) + self.assertNotIn("node_modules/ignored", listed.stdout) + result = self.run_tool("project", "read", "rg", "--json", "needle", "file with spaces") + matches = [json.loads(line) for line in result.stdout.splitlines()] + self.assertEqual(next(row for row in matches if row["type"] == "match") + ["data"]["path"]["text"], "file with spaces") + for pattern, status in (("not present", 1), ("[", 2)): + result = self.run_tool("project", "read", "rg", pattern, check=False) + self.assertEqual(result.returncode, status) + requests = [json.loads(line) for line in self.requests.read_text().splitlines()] + self.assertTrue(all(request["command"] == "read" for request in requests)) + + def test_storage_hashes_match_content_and_reject_external_paths_and_commands(self): + content = b"fixture" * 1000 + (self.main / "archive with spaces.zip").write_bytes(content) + result = self.run_tool("project", "read", "sha256sum", "archive with spaces.zip") + self.assertEqual(result.stdout, hashlib.sha256(content).hexdigest() + " archive with spaces.zip\n") + outside = self.root / "private" + outside.write_text("not shared") + (self.main / "escape").symlink_to(outside) + for args in (("sha256sum", str(outside)), ("sha256sum", "escape"), + ("sha256sum", "-"), ("sha256sum", "--check", "archive with spaces.zip"), + ("rg", "--pre=touch marker", "needle"), ("sh", "-c", "touch marker")): + with self.subTest(args=args): + self.assertNotEqual(self.run_tool("project", "read", *args, check=False).returncode, 0) + self.assertFalse((self.main / "marker").exists()) + + +if __name__ == "__main__": + unittest.main()