diff --git a/nixos/configuration.nix b/nixos/configuration.nix index fcdbf2c7aa0127aedd95d99e7d4492812531ad80..2045d44dcc3a31aeeb29c2d9a12ca8d5406abb90 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -6,7 +6,7 @@ let proxyToken = "/var/lib/snowglobe/dashboard-proxy.token"; hostTools = lib.fileset.toSource { root = ../.; - fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-windows.cs ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; + fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-windows.cs ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; }; computerAhk = pkgs.callPackage ./computer-ahk.nix { }; nativePkl = pkgs.callPackage ./pkl.nix { }; diff --git a/nixos/snowglobe.nix b/nixos/snowglobe.nix index 9eea6412ae8b0ca0c4d0e643d3b903c471dc36ea..8aa274d8c8b6082e5e883b6dad5ebf7249e99063 100644 --- a/nixos/snowglobe.nix +++ b/nixos/snowglobe.nix @@ -15,7 +15,7 @@ let }; in { - imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ./gpu-unlock.nix ]; + imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./vm-disks.nix ./zfs-read-cache.nix ./gpu-unlock.nix ]; networking.hostName = "snowglobe"; environment.etc."machine-info".text = '' diff --git a/nixos/tests/vm-disks.nix b/nixos/tests/vm-disks.nix new file mode 100644 index 0000000000000000000000000000000000000000..2d4d62453f4fc94737c97a9eb3b5e26f59762e38 --- /dev/null +++ b/nixos/tests/vm-disks.nix @@ -0,0 +1,42 @@ +{ pkgs }: +pkgs.testers.runNixOSTest { + name = "vm-disks"; + nodes.machine = { lib, ... }: { + imports = [ ../vm-disks.nix ]; + virtualisation = { memorySize = 2048; cores = 2; emptyDiskImages = [ 2048 ]; }; + networking.hostId = "c3d4e5f6"; + boot.supportedFilesystems = [ "zfs" ]; + boot.zfs.forceImportRoot = false; + environment.systemPackages = [ pkgs.zfs pkgs.util-linux ]; + systemd.services.snowglobe-host.enable = false; + systemd.services.snowglobe-vm-expiry.enable = false; + systemd.services.snowglobe-vm-storage = { + requires = lib.mkForce [ ]; + after = lib.mkForce [ ]; + serviceConfig = { + LoadCredentialEncrypted = lib.mkForce [ ]; + LoadCredential = "zfs-clover:/run/test-key"; + }; + }; + }; + testScript = '' + start_all() + machine.succeed("head -c 32 /dev/urandom | od -An -v -tx1 | tr -d ' \\n' > /run/test-key; chmod 600 /run/test-key") + machine.succeed("zpool create -O mountpoint=none work /dev/vdb") + machine.succeed("systemctl start snowglobe-vm-storage.service") + assert machine.succeed("findmnt -n -o SOURCE --mountpoint /srv/vm-disks").strip() == "work/vm-disks" + assert int(machine.succeed("zfs get -Hp -o value quota work/vm-disks").strip()) > 0 + assert machine.succeed("zfs get -Hp -o value reservation work/vm-disks").strip() == "0" + assert machine.succeed("zfs get -H -o value encryptionroot work/vm-disks").strip() == "work/vm-disks" + assert machine.succeed("stat -c '%a' /srv/vm-disks").strip() == "700" + machine.succeed("echo retained > /srv/vm-disks/fixture; systemctl stop snowglobe-vm-storage.service; zfs unmount work/vm-disks; zfs unload-key work/vm-disks") + machine.succeed("systemctl start snowglobe-vm-storage.service") + assert machine.succeed("cat /srv/vm-disks/fixture").strip() == "retained" + machine.succeed("systemctl stop snowglobe-vm-storage.service; zfs unmount work/vm-disks; zfs unload-key work/vm-disks; mv /run/test-key /run/test-key.saved; echo invalid > /run/test-key") + machine.fail("systemctl start snowglobe-vm-storage.service") + machine.succeed("mv /run/test-key.saved /run/test-key; systemctl reset-failed snowglobe-vm-storage.service; systemctl start snowglobe-vm-storage.service") + machine.succeed("systemctl stop snowglobe-vm-storage.service; zfs unmount work/vm-disks; zfs set mountpoint=/wrong work/vm-disks") + machine.fail("systemctl start snowglobe-vm-storage.service") + assert machine.succeed("cat /srv/vm-disks/fixture 2>/dev/null || true").strip() == "" + ''; +} diff --git a/nixos/vm-disks.nix b/nixos/vm-disks.nix new file mode 100644 index 0000000000000000000000000000000000000000..568b251f5ee0c2ef119b97685ca5f4e059f29baa --- /dev/null +++ b/nixos/vm-disks.nix @@ -0,0 +1,49 @@ +{ pkgs, ... }: +let + root = "/srv/vm-disks"; + dataset = "work/vm-disks"; + limits = builtins.fromJSON (builtins.readFile ../tools/vm-ai-limits.json); + quota = toString (limits.host_total * (limits.disk_gib + limits.reserve_gib)); +in +{ + environment.variables.SNOWGLOBE_VM_AI_DISKS_ROOT = root; + systemd.services.snowglobe-vm-storage = { + requires = [ "zfs-import-work.service" "dev-tpmrm0.device" ]; + after = [ "zfs-import-work.service" "zfs-mount.service" "dev-tpmrm0.device" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + UMask = "0077"; + LoadCredentialEncrypted = "zfs-clover:/etc/credstore.encrypted/zfs-clover"; + }; + script = '' + if ! ${pkgs.zfs}/bin/zfs list -H -o name ${dataset} >/dev/null 2>&1; then + ${pkgs.zfs}/bin/zfs create -u -o encryption=aes-256-gcm -o keyformat=hex \ + -o keylocation="file://$CREDENTIALS_DIRECTORY/zfs-clover" \ + -o mountpoint=${root} -o compression=lz4 -o atime=off ${dataset} + fi + test "$(${pkgs.zfs}/bin/zfs get -H -o value encryptionroot ${dataset})" = ${dataset} + test "$(${pkgs.zfs}/bin/zfs get -H -o value mountpoint ${dataset})" = ${root} + ${pkgs.zfs}/bin/zfs set keylocation=prompt quota=${quota}G ${dataset} + if test "$(${pkgs.zfs}/bin/zfs get -H -o value keystatus ${dataset})" = unavailable; then + ${pkgs.zfs}/bin/zfs load-key -L "file://$CREDENTIALS_DIRECTORY/zfs-clover" ${dataset} + fi + if test "$(${pkgs.zfs}/bin/zfs get -H -o value mounted ${dataset})" != yes; then + ${pkgs.zfs}/bin/zfs mount ${dataset} + fi + test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint ${root})" = ${dataset} + chmod 0700 ${root} + ''; + }; + systemd.services.snowglobe-host = { + wants = [ "snowglobe-vm-storage.service" ]; + after = [ "snowglobe-vm-storage.service" ]; + serviceConfig.ReadWritePaths = [ "-${root}" ]; + }; + systemd.services.snowglobe-vm-expiry = { + wants = [ "snowglobe-vm-storage.service" ]; + after = [ "snowglobe-vm-storage.service" ]; + environment.SNOWGLOBE_VM_AI_DISKS_ROOT = root; + serviceConfig.ReadWritePaths = [ "-${root}" ]; + }; +} diff --git a/readme.md b/readme.md index 0c0d67161c12909e94f52e4b00d973d25cb56050..13ab5d3f673b4663d4b4d4673c2e2ca1b886cc70 100644 --- a/readme.md +++ b/readme.md @@ -68,8 +68,8 @@ srv/ +--- ... +--- staging/ staging deployments use this space for temporary clones +--- postgres-9f06f74b/ [dataset] -+--- vm/ [dataset] virtual machines - +--- clover-sandbox/ [dataset] ++--- vm/ VM control files and existing guest disks on globe ++--- vm-disks/ [work/vm-disks] new disposable MCP guest disks on the Work SSD ``` Media is its own dataset so it can be snapshotted independently of personal diff --git a/tools/computer-test.py b/tools/computer-test.py index 9afb69879f8a76de1fcd413addffa77d87e72c59..5cb21d6d2dd2dbe0dfb64faebac39c40dbd78f02 100644 --- a/tools/computer-test.py +++ b/tools/computer-test.py @@ -471,6 +471,21 @@ class TestVmExpiry(unittest.TestCase): leases.create({'action': 'create', 'owner': 'fixture', 'preset': preset, 'name': 'test', 'vcpus': 2, 'memory': 4}) create.assert_not_called() + def test_creation_checks_ssd_capacity_instead_of_metadata_storage(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + ssd = root / 'ssd' + ssd.mkdir() + with patch.object(leases.vms, 'DISKS', root), patch.object(leases.vms, 'AI_DISKS', ssd), \ + patch.object(Path, 'is_mount', return_value=True), patch.object(leases.vms, 'virsh', return_value=''), \ + patch.object(leases, 'presets', return_value=[{'id':'fixture','capacity':64*2**30}]), \ + patch.object(leases.shutil, 'disk_usage', return_value=Mock(free=2**30)) as usage, \ + patch.object(leases.vms, 'create') as create: + with self.assertRaisesRegex(ValueError, 'free space'): + leases.create({'action':'create','owner':'fixture','preset':'fixture','name':'fixture','vcpus':2,'memory':4}) + usage.assert_called_once_with(ssd) + create.assert_not_called() + def test_expiry_preserves_personal_vms_and_skips_active_calls(self): with tempfile.TemporaryDirectory() as directory, patch.object(leases.vms, 'DISKS', Path(directory)), patch.object(leases.time, 'time', return_value=10000): for name, expiry in [('personal', 1), ('ai-prefix-only', None), ('ai-expired', 9999), ('ai-live', 10001), ('ai-busy', 9999)]: diff --git a/tools/dashboard-vms-test.py b/tools/dashboard-vms-test.py index bd4c6f7fa696c33c3b468e614d904ec9a980c153..89403fbbdf15330d62fd2f4582bbf084bcca62fe 100644 --- a/tools/dashboard-vms-test.py +++ b/tools/dashboard-vms-test.py @@ -427,5 +427,123 @@ class VMTests(unittest.TestCase): vms.validate('upload', {'volume': filename, 'size': 32768}) +class DisposableStorage(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.metadata = self.root / 'metadata' + self.storage = self.root / 'ssd' + self.metadata.mkdir() + self.storage.mkdir() + for item in [patch.object(vms, 'DISKS', self.metadata), patch.object(vms, 'AI_DISKS', self.storage), + patch.object(Path, 'is_mount', return_value=True)]: + item.start() + self.addCleanup(item.stop) + + def test_failed_creation_never_overwrites_an_existing_ssd_disk(self): + drive = self.storage / 'ai-fixture.qcow2' + drive.write_bytes(b'existing') + spec = {'name': 'ai-fixture', 'image': 'blank', 'mode': 'iso', 'platform': 'linux', + 'firmware': 'bios', 'disk': 2**30} + with patch.object(vms, 'virsh', return_value=''), patch.object(vms, 'command') as command: + with self.assertRaises(FileExistsError): + vms.create(spec, ai=True) + command.assert_not_called() + self.assertEqual(drive.read_bytes(), b'existing') + self.assertFalse((self.metadata / spec['name']).exists()) + + def test_failed_image_creation_removes_only_its_new_ssd_disk(self): + spec = {'name': 'ai-fixture', 'image': 'blank', 'mode': 'iso', 'platform': 'linux', + 'firmware': 'bios', 'disk': 2**30} + foreign = self.storage / 'foreign.qcow2' + foreign.write_bytes(b'preserve') + with patch.object(vms, 'virsh', return_value=''), patch.object(vms, 'command', side_effect=OSError('Disk full')): + with self.assertRaisesRegex(OSError, 'Disk full'): + vms.create(spec, ai=True) + self.assertEqual(foreign.read_bytes(), b'preserve') + self.assertFalse((self.storage / 'ai-fixture.qcow2').exists()) + self.assertEqual(list(self.metadata.iterdir()), []) + + def test_failed_expiry_write_does_not_leave_a_guest_directory_or_ssd_disk(self): + spec = {'name': 'ai-fixture', 'image': 'blank', 'mode': 'iso', 'platform': 'linux', + 'firmware': 'bios', 'disk': 2**30} + with patch.object(vms, 'virsh', return_value=''), patch.object(Path, 'write_text', side_effect=OSError('Disk full')): + with self.assertRaisesRegex(OSError, 'Disk full'): + vms.create(spec, ai=True) + self.assertEqual(list(self.metadata.iterdir()), []) + self.assertEqual(list(self.storage.iterdir()), []) + + def test_uncertain_definition_preserves_the_ssd_disk_and_recovery_metadata(self): + defined = False + def virsh(action, *args): + nonlocal defined + if action == 'list': + return 'ai-fixture' if defined else '' + if action == 'define': + defined = True + raise TimeoutError('Definition response lost') + return '' + spec = {'name': 'ai-fixture', 'description': '', 'image': 'blank', 'mode': 'iso', + 'platform': 'linux', 'firmware': 'bios', 'disk': 2**30, 'memory': 2**29, + 'vcpus': 1, 'owner': 'fixture', 'autostart': False, 'start': False} + with patch.object(vms, 'virsh', side_effect=virsh), patch.object(vms, 'command', return_value=''): + with self.assertRaisesRegex(TimeoutError, 'response lost'): + vms.create(spec, ai=True) + self.assertTrue((self.storage / 'ai-fixture.qcow2').exists()) + self.assertTrue((self.metadata / 'ai-fixture' / 'ai.json').exists()) + source = vms.ET.parse(self.metadata / 'ai-fixture' / 'domain.xml').find('./devices/disk/source') + self.assertEqual(source.get('file'), str(self.storage / 'ai-fixture.qcow2')) + + def test_remove_preserves_disks_when_undefine_is_uncertain(self): + directory = self.metadata / 'ai-fixture' + directory.mkdir() + (directory / 'ai.json').write_text('{"expiresAt":1}') + drive = self.storage / 'ai-fixture.qcow2' + drive.write_bytes(b'guest') + def virsh(action, *args): + if action == 'undefine': + raise TimeoutError('Undefine response lost') + return {'list': 'ai-fixture', 'domstate': 'shut off', + 'dumpxml': f''}[action] + with patch.object(vms, 'virsh', side_effect=virsh): + with self.assertRaises(TimeoutError): + vms.remove({'name': 'ai-fixture', 'disks': True}) + self.assertEqual(drive.read_bytes(), b'guest') + self.assertTrue(directory.is_dir()) + with patch.object(vms, 'virsh', side_effect=lambda action, *args: '' if action == 'undefine' else virsh(action, *args)): + vms.remove({'name': 'ai-fixture', 'disks': True}) + self.assertFalse(directory.exists()) + self.assertFalse(drive.exists()) + + def test_legacy_guest_deletion_does_not_touch_a_same_named_ssd_file(self): + directory = self.metadata / 'ai-fixture' + directory.mkdir() + (directory / 'ai.json').write_text('{"expiresAt":1}') + (directory / 'disk.qcow2').write_bytes(b'legacy') + foreign = self.storage / 'ai-fixture.qcow2' + foreign.write_bytes(b'foreign') + replies = {'list': 'ai-fixture', 'domstate': 'shut off', 'undefine': '', + 'dumpxml': f''} + with patch.object(vms, 'virsh', side_effect=lambda action, *args: replies[action]), patch.object(Path, 'is_mount', return_value=False): + vms.remove({'name': 'ai-fixture', 'disks': True}) + self.assertEqual(foreign.read_bytes(), b'foreign') + self.assertFalse(directory.exists()) + + def test_missing_ssd_mount_fails_before_create_or_delete_mutations(self): + directory = self.metadata / 'ai-fixture' + with patch.object(Path, 'is_mount', return_value=False), patch.object(vms, 'virsh', return_value='') as virsh: + with self.assertRaisesRegex(ValueError, 'storage is unavailable'): + vms.create({'name': 'ai-fixture', 'image': 'blank', 'mode': 'iso', 'platform': 'linux', + 'firmware': 'bios', 'disk': 2**30}, ai=True) + self.assertFalse(directory.exists()) + directory.mkdir() + (directory / 'ai.json').write_text('{"expiresAt":1}') + with self.assertRaisesRegex(ValueError, 'storage is unavailable'): + vms.remove({'name': 'ai-fixture', 'disks': True}, missing_ok=True) + self.assertTrue(directory.is_dir()) + self.assertTrue(all(call.args[0] == 'list' for call in virsh.call_args_list)) + + if __name__ == '__main__': unittest.main() diff --git a/tools/release-host-test.py b/tools/release-host-test.py index 83802b29a207441b7fa895946f74a5c93d701814..71f41f607e0325d210d52d36415df17dd122f76e 100644 --- a/tools/release-host-test.py +++ b/tools/release-host-test.py @@ -12,6 +12,16 @@ import release class HostConfiguration(unittest.TestCase): + def test_vm_limit_changes_invalidate_the_host_configuration(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / 'tools').mkdir() + limits = root / 'tools/vm-ai-limits.json' + limits.write_text('{"host_total":8}') + before = release.host_digest(root) + limits.write_text('{"host_total":4}') + self.assertNotEqual(release.host_digest(root), before) + def select(self, hostname, configurations): with patch.object(release.os, "uname", return_value=SimpleNamespace(nodename=hostname)), patch.object( release.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, json.dumps(configurations)), diff --git a/tools/release.py b/tools/release.py index 4d7ab541dd7f79bedd4317d2a1d15bc72aa0f965..4961e86946c52af89f5436ac2880bce749859e60 100644 --- a/tools/release.py +++ b/tools/release.py @@ -60,7 +60,7 @@ def host_digest(root): digest = hashlib.sha256() paths = sorted(path for path, relative in files(root) if relative.parts[0] in {"nixos", "dashboard", "config", "service"} - or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) + or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) for path in paths: digest.update(str(path.relative_to(root)).encode()) _hash_contents(digest, path) diff --git a/tools/vm-ai-limits.json b/tools/vm-ai-limits.json new file mode 100644 index 0000000000000000000000000000000000000000..2d0843aa08f5fa0c5a8739df638c32ba18b0aea2 --- /dev/null +++ b/tools/vm-ai-limits.json @@ -0,0 +1,10 @@ +{ + "per_user": 2, + "host_total": 8, + "vcpus": 4, + "memory_gib": 8, + "disk_gib": 80, + "reserve_gib": 10, + "idle_seconds": 3600, + "extension_seconds": 86400 +} diff --git a/tools/vm-ai.py b/tools/vm-ai.py index d73672b79911ea203d37fe77a52ddee735a08540..a023438bc72f2bb7f8cc201b2e29f33a6d5278a4 100644 --- a/tools/vm-ai.py +++ b/tools/vm-ai.py @@ -12,12 +12,11 @@ import xml.etree.ElementTree as ET import vms -LIMITS = {"per_user": 2, "host_total": 8, "vcpus": 4, "memory_gib": 8, - "idle_seconds": 3600, "extension_seconds": 86400} +LIMITS = json.loads(Path(__file__).with_name("vm-ai-limits.json").read_text()) def presets(): - return [p for p in vms.library()["presets"] if p["capacity"] <= 80 * 2**30 and + return [p for p in vms.library()["presets"] if p["capacity"] <= LIMITS["disk_gib"] * 2**30 and (p["platform"] == "linux" or p.get("setup") in {"windows-v3", "windows-v4"})] @@ -51,9 +50,9 @@ def create(request): if not isinstance(request["name"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,35}", request["name"]): raise ValueError("Use a name of 1–36 lowercase letters, numbers, or hyphens.") if type(request["vcpus"]) is not int or not 1 <= request["vcpus"] <= LIMITS["vcpus"]: - raise ValueError("Choose between 1 and 4 vCPUs.") + raise ValueError(f"Choose between 1 and {LIMITS['vcpus']} vCPUs.") if type(request["memory"]) is not int or not 1 <= request["memory"] <= LIMITS["memory_gib"]: - raise ValueError("Choose between 1 and 8 GiB of memory.") + raise ValueError(f"Choose between 1 and {LIMITS['memory_gib']} GiB of memory.") preset = next((p for p in presets() if p["id"] == request["preset"]), None) if preset is None: raise ValueError("Choose a prepared preset from computer_presets.") @@ -67,10 +66,11 @@ def create(request): root = ET.fromstring(vms.virsh("dumpxml", name)) owned += root.findtext(f"metadata/{vms.NS}vm/{vms.NS}owner") == request["owner"] if owned >= LIMITS["per_user"] or total >= LIMITS["host_total"]: - raise ValueError("The test VM limit is reached: 2 per account and 8 on this host. Delete an unused test VM or wait for expiry.") - if shutil.disk_usage(vms.DISKS).free < preset["capacity"] + 10 * 2**30: - raise ValueError("There is not enough free space for another test VM. Delete an unused test VM.") + raise ValueError(f"The test VM limit is reached: {LIMITS['per_user']} per account and {LIMITS['host_total']} on this host. Delete an unused test VM or wait for expiry.") name = "ai-" + request["name"].removeprefix("ai-") + "-" + uuid.uuid4().hex[:8] + drive = vms.disposable_disk(name) + if shutil.disk_usage(drive.parent if vms.AI_DISKS is not None else vms.DISKS).free < preset["capacity"] + LIMITS["reserve_gib"] * 2**30: + raise ValueError("There is not enough free space for another test VM. Delete an unused test VM.") spec = {"name": name, "description": "", "image": preset["id"], "vcpus": request["vcpus"], "memory": request["memory"] * 2**30, "disk": preset["capacity"], "autostart": False, "start": True, diff --git a/tools/vms.py b/tools/vms.py index dfeb44ff9754455ac84e5f012d0a1a9e8a04b5db..4fe156f0dc10a0d94410b95910c4461a3fb89cbb 100644 --- a/tools/vms.py +++ b/tools/vms.py @@ -18,6 +18,7 @@ import xml.etree.ElementTree as ET DISKS = Path("/srv/vm") +AI_DISKS = Path(os.environ["SNOWGLOBE_VM_AI_DISKS_ROOT"]) if os.environ.get("SNOWGLOBE_VM_AI_DISKS_ROOT") else None IMAGES = Path(os.environ.get("SNOWGLOBE_VM_IMAGES_ROOT", "/srv/clover/Media/vm")) PRESETS = Path(os.environ.get("SNOWGLOBE_VM_PRESETS_ROOT", str(IMAGES / "Presets"))) UPLOADS = Path(os.environ.get("SNOWGLOBE_VM_UPLOADS_ROOT", str(IMAGES / "Install Disks"))) @@ -332,6 +333,8 @@ def domains(*, computers=False): continue target_name = target.get("dev", "") pool = "vms" if Path(file).is_relative_to(DISKS / name) else None + if AI_DISKS is not None and Path(file) == AI_DISKS / (name + ".qcow2"): + pool = "vm-disks" disks.append(disk(Path(file), target_name, pool)) interfaces = [] for element in root.findall("./devices/interface"): @@ -592,7 +595,7 @@ def save_preset(spec): if source is None or not source.get("file"): raise ValueError("Use a VM with a managed disk.") file = Path(source.get("file")) - if not file.is_relative_to(DISKS / name): + if not file.is_relative_to(DISKS / name) and not ((DISKS / name / "ai.json").is_file() and file == disposable_disk(name)): raise ValueError("Use a disk stored with this VM.") base = PRESETS managed_directory(base) @@ -768,6 +771,14 @@ def macos_seed(directory, username): (directory / "seed.iso").chmod(0o600) +def disposable_disk(name): + if AI_DISKS is None: + return DISKS / name / "disk.qcow2" + if AI_DISKS.is_symlink() or not AI_DISKS.is_mount(): + raise ValueError("Test VM storage is unavailable. Ask an administrator to check the SSD mount.") + return AI_DISKS / (name + ".qcow2") + + def create(spec, *, ai=False): name = spec["name"] ensure_name(name) @@ -793,24 +804,31 @@ def create(spec, *, ai=False): raise ValueError(f"Choose a {details_profile['arch']} ISO for this hardware profile.") if mode == "preset" and spec["disk"] < preset["capacity"]: raise ValueError("Choose a disk at least as large as the preset.") - if "vms" not in virsh("pool-list", "--all", "--name").splitlines(): - DISKS.mkdir(parents=True, exist_ok=True) - virsh("pool-define-as", "vms", "dir", "--target", str(DISKS)) - if re.search(r"^State:\s+inactive", virsh("pool-info", "vms"), re.MULTILINE): - virsh("pool-start", "vms") - virsh("pool-autostart", "vms") directory = DISKS / name + drive = disposable_disk(name) if ai else directory / "disk.qcow2" + pool = "vm-disks" if drive.parent != directory else "vms" + pool_root = drive.parent if drive.parent != directory else DISKS + if pool not in virsh("pool-list", "--all", "--name").splitlines(): + pool_root.mkdir(parents=True, exist_ok=True) + virsh("pool-define-as", pool, "dir", "--target", str(pool_root)) + if re.search(r"^State:\s+inactive", virsh("pool-info", pool), re.MULTILINE): + virsh("pool-start", pool) + virsh("pool-autostart", pool) directory.mkdir(mode=0o700, parents=True, exist_ok=False) - drive = directory / "disk.qcow2" - if ai: - (directory / "ai.json").write_text(json.dumps({"expiresAt": int(time.time()) + import_module("vm-ai").LIMITS["idle_seconds"]})) def expired(_signum, _frame): raise TimeoutError("VM image preparation timed out.") previous = signal.signal(signal.SIGALRM, expired) signal.alarm(840) + external_created = False try: + if ai: + (directory / "ai.json").write_text(json.dumps({"expiresAt": int(time.time()) + import_module("vm-ai").LIMITS["idle_seconds"]})) + if drive.parent != directory: + with drive.open("xb"): + pass + external_created = True os_name = "Other" overlay = preset and not (spec["platform"] == "macos" and spec["disk"] > preset["capacity"]) if overlay: @@ -851,7 +869,7 @@ def create(spec, *, ai=False): source.unlink() elif not overlay: command("qemu-img", "create", "-f", "qcow2", str(drive), str(spec["disk"])) - virsh("pool-refresh", "vms") + virsh("pool-refresh", pool) domain_type = "qemu" if details_profile.get("emulated") or os.environ.get("SNOWGLOBE_VM_ACCEL") == "qemu" else "kvm" root = ET.Element("domain", type=domain_type) ET.SubElement(root, "name").text = name @@ -1012,6 +1030,8 @@ def create(spec, *, ai=False): except Exception: signal.alarm(0) if name not in virsh("list", "--all", "--name").splitlines(): + if external_created: + drive.unlink() for file in directory.iterdir(): file.unlink() directory.rmdir() @@ -1027,12 +1047,23 @@ def remove(spec, *, missing_ok=False): exists = name in virsh("list", "--all", "--name").splitlines() if not exists and not missing_ok: raise ValueError("VM does not exist") + directory = DISKS / name + external = None + if spec["disks"] and (directory / "ai.json").is_file() and AI_DISKS is not None: + candidate = AI_DISKS / (name + ".qcow2") + if exists: + root = ET.fromstring(virsh("dumpxml", name, "--inactive")) + if any(source.get("file") == str(candidate) for source in root.findall("./devices/disk[@device='disk']/source")): + external = disposable_disk(name) + elif not (directory / "disk.qcow2").exists(): + external = disposable_disk(name) if exists: if virsh("domstate", name).strip() != "shut off": virsh("destroy", name) root = ET.fromstring(virsh("dumpxml", name)) virsh("undefine", name, "--managed-save", *(["--nvram"] if root.find("./os/nvram") is not None else []), *(["--tpm"] if root.find("./devices/tpm") is not None else [])) - directory = DISKS / name + if external is not None: + external.unlink(missing_ok=True) if spec["disks"] and directory.is_dir(): for file in directory.iterdir(): file.unlink()