| author | |
| committer | |
| log | 3d5465351dc36ac4fb16ecafa72be90d1e5d1ec8 |
| tree | 37f4607ca15902ca6c077e58ba8ef726626bdfe4 |
| parent | 3179871dfe597f763b9fb45590b07a0679b0a628 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Place new AI-owned disks in an encrypted, private Work dataset while keeping
existing disks and VM control files at their current paths. Derive its quota
from the shared MCP guest limits and refuse an unavailable SSD mount.
Preserve files after uncertain libvirt operations and keep managed disk,
preset capture, deletion, and expiry behavior across both locations.
Assisted-by: gpt-6.1-sol12 files changed, 300 insertions(+), 25 deletions(-)
nixos/configuration.nix+1-1| ... | ... | @@ -6,7 +6,7 @@ let |
| 6 | 6 | proxyToken = "/var/lib/snowglobe/dashboard-proxy.token"; |
| 7 | 7 | hostTools = lib.fileset.toSource { |
| 8 | 8 | root = ../.; |
| 9 | fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-windows.cs ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; | |
| 9 | fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-computer.py ../tools/vm-ai.py ../tools/vm-ai-limits.json ../tools/vm-rfb.py ../tools/mac-computer.py ../guest/computer-windows.cs ../guest/computer-legacy-mac.py ../guest/computer-linux.py ../guest/computer-linux-launch ../guest/computer-linux.nix ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; | |
| 10 | 10 | }; |
| 11 | 11 | computerAhk = pkgs.callPackage ./computer-ahk.nix { }; |
| 12 | 12 | nativePkl = pkgs.callPackage ./pkl.nix { }; |
nixos/snowglobe.nix+1-1| ... | ... | @@ -15,7 +15,7 @@ let |
| 15 | 15 | }; |
| 16 | 16 | in |
| 17 | 17 | { |
| 18 | imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ./gpu-unlock.nix ]; | |
| 18 | imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./vm-disks.nix ./zfs-read-cache.nix ./gpu-unlock.nix ]; | |
| 19 | 19 | |
| 20 | 20 | networking.hostName = "snowglobe"; |
| 21 | 21 | environment.etc."machine-info".text = '' |
nixos/tests/vm-disks.nix created+42| ... | ... | @@ -0,0 +1,42 @@ |
| 1 | { pkgs }: | |
| 2 | pkgs.testers.runNixOSTest { | |
| 3 | name = "vm-disks"; | |
| 4 | nodes.machine = { lib, ... }: { | |
| 5 | imports = [ ../vm-disks.nix ]; | |
| 6 | virtualisation = { memorySize = 2048; cores = 2; emptyDiskImages = [ 2048 ]; }; | |
| 7 | networking.hostId = "c3d4e5f6"; | |
| 8 | boot.supportedFilesystems = [ "zfs" ]; | |
| 9 | boot.zfs.forceImportRoot = false; | |
| 10 | environment.systemPackages = [ pkgs.zfs pkgs.util-linux ]; | |
| 11 | systemd.services.snowglobe-host.enable = false; | |
| 12 | systemd.services.snowglobe-vm-expiry.enable = false; | |
| 13 | systemd.services.snowglobe-vm-storage = { | |
| 14 | requires = lib.mkForce [ ]; | |
| 15 | after = lib.mkForce [ ]; | |
| 16 | serviceConfig = { | |
| 17 | LoadCredentialEncrypted = lib.mkForce [ ]; | |
| 18 | LoadCredential = "zfs-clover:/run/test-key"; | |
| 19 | }; | |
| 20 | }; | |
| 21 | }; | |
| 22 | testScript = '' | |
| 23 | start_all() | |
| 24 | machine.succeed("head -c 32 /dev/urandom | od -An -v -tx1 | tr -d ' \\n' > /run/test-key; chmod 600 /run/test-key") | |
| 25 | machine.succeed("zpool create -O mountpoint=none work /dev/vdb") | |
| 26 | machine.succeed("systemctl start snowglobe-vm-storage.service") | |
| 27 | assert machine.succeed("findmnt -n -o SOURCE --mountpoint /srv/vm-disks").strip() == "work/vm-disks" | |
| 28 | assert int(machine.succeed("zfs get -Hp -o value quota work/vm-disks").strip()) > 0 | |
| 29 | assert machine.succeed("zfs get -Hp -o value reservation work/vm-disks").strip() == "0" | |
| 30 | assert machine.succeed("zfs get -H -o value encryptionroot work/vm-disks").strip() == "work/vm-disks" | |
| 31 | assert machine.succeed("stat -c '%a' /srv/vm-disks").strip() == "700" | |
| 32 | machine.succeed("echo retained > /srv/vm-disks/fixture; systemctl stop snowglobe-vm-storage.service; zfs unmount work/vm-disks; zfs unload-key work/vm-disks") | |
| 33 | machine.succeed("systemctl start snowglobe-vm-storage.service") | |
| 34 | assert machine.succeed("cat /srv/vm-disks/fixture").strip() == "retained" | |
| 35 | machine.succeed("systemctl stop snowglobe-vm-storage.service; zfs unmount work/vm-disks; zfs unload-key work/vm-disks; mv /run/test-key /run/test-key.saved; echo invalid > /run/test-key") | |
| 36 | machine.fail("systemctl start snowglobe-vm-storage.service") | |
| 37 | machine.succeed("mv /run/test-key.saved /run/test-key; systemctl reset-failed snowglobe-vm-storage.service; systemctl start snowglobe-vm-storage.service") | |
| 38 | machine.succeed("systemctl stop snowglobe-vm-storage.service; zfs unmount work/vm-disks; zfs set mountpoint=/wrong work/vm-disks") | |
| 39 | machine.fail("systemctl start snowglobe-vm-storage.service") | |
| 40 | assert machine.succeed("cat /srv/vm-disks/fixture 2>/dev/null || true").strip() == "" | |
| 41 | ''; | |
| 42 | } |
nixos/vm-disks.nix created+49| ... | ... | @@ -0,0 +1,49 @@ |
| 1 | { pkgs, ... }: | |
| 2 | let | |
| 3 | root = "/srv/vm-disks"; | |
| 4 | dataset = "work/vm-disks"; | |
| 5 | limits = builtins.fromJSON (builtins.readFile ../tools/vm-ai-limits.json); | |
| 6 | quota = toString (limits.host_total * (limits.disk_gib + limits.reserve_gib)); | |
| 7 | in | |
| 8 | { | |
| 9 | environment.variables.SNOWGLOBE_VM_AI_DISKS_ROOT = root; | |
| 10 | systemd.services.snowglobe-vm-storage = { | |
| 11 | requires = [ "zfs-import-work.service" "dev-tpmrm0.device" ]; | |
| 12 | after = [ "zfs-import-work.service" "zfs-mount.service" "dev-tpmrm0.device" ]; | |
| 13 | serviceConfig = { | |
| 14 | Type = "oneshot"; | |
| 15 | RemainAfterExit = true; | |
| 16 | UMask = "0077"; | |
| 17 | LoadCredentialEncrypted = "zfs-clover:/etc/credstore.encrypted/zfs-clover"; | |
| 18 | }; | |
| 19 | script = '' | |
| 20 | if ! ${pkgs.zfs}/bin/zfs list -H -o name ${dataset} >/dev/null 2>&1; then | |
| 21 | ${pkgs.zfs}/bin/zfs create -u -o encryption=aes-256-gcm -o keyformat=hex \ | |
| 22 | -o keylocation="file://$CREDENTIALS_DIRECTORY/zfs-clover" \ | |
| 23 | -o mountpoint=${root} -o compression=lz4 -o atime=off ${dataset} | |
| 24 | fi | |
| 25 | test "$(${pkgs.zfs}/bin/zfs get -H -o value encryptionroot ${dataset})" = ${dataset} | |
| 26 | test "$(${pkgs.zfs}/bin/zfs get -H -o value mountpoint ${dataset})" = ${root} | |
| 27 | ${pkgs.zfs}/bin/zfs set keylocation=prompt quota=${quota}G ${dataset} | |
| 28 | if test "$(${pkgs.zfs}/bin/zfs get -H -o value keystatus ${dataset})" = unavailable; then | |
| 29 | ${pkgs.zfs}/bin/zfs load-key -L "file://$CREDENTIALS_DIRECTORY/zfs-clover" ${dataset} | |
| 30 | fi | |
| 31 | if test "$(${pkgs.zfs}/bin/zfs get -H -o value mounted ${dataset})" != yes; then | |
| 32 | ${pkgs.zfs}/bin/zfs mount ${dataset} | |
| 33 | fi | |
| 34 | test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint ${root})" = ${dataset} | |
| 35 | chmod 0700 ${root} | |
| 36 | ''; | |
| 37 | }; | |
| 38 | systemd.services.snowglobe-host = { | |
| 39 | wants = [ "snowglobe-vm-storage.service" ]; | |
| 40 | after = [ "snowglobe-vm-storage.service" ]; | |
| 41 | serviceConfig.ReadWritePaths = [ "-${root}" ]; | |
| 42 | }; | |
| 43 | systemd.services.snowglobe-vm-expiry = { | |
| 44 | wants = [ "snowglobe-vm-storage.service" ]; | |
| 45 | after = [ "snowglobe-vm-storage.service" ]; | |
| 46 | environment.SNOWGLOBE_VM_AI_DISKS_ROOT = root; | |
| 47 | serviceConfig.ReadWritePaths = [ "-${root}" ]; | |
| 48 | }; | |
| 49 | } |
readme.md+2-2| ... | ... | @@ -68,8 +68,8 @@ srv/ |
| 68 | 68 | +--- ... |
| 69 | 69 | +--- staging/ staging deployments use this space for temporary clones |
| 70 | 70 | +--- postgres-9f06f74b/ [dataset] |
| 71 | +--- vm/ [dataset] virtual machines | |
| 72 | +--- clover-sandbox/ [dataset] | |
| 71 | +--- vm/ VM control files and existing guest disks on globe | |
| 72 | +--- vm-disks/ [work/vm-disks] new disposable MCP guest disks on the Work SSD | |
| 73 | 73 | ``` |
| 74 | 74 | |
| 75 | 75 | Media is its own dataset so it can be snapshotted independently of personal |
tools/computer-test.py+15| ... | ... | @@ -471,6 +471,21 @@ class TestVmExpiry(unittest.TestCase): |
| 471 | 471 | leases.create({'action': 'create', 'owner': 'fixture', 'preset': preset, 'name': 'test', 'vcpus': 2, 'memory': 4}) |
| 472 | 472 | create.assert_not_called() |
| 473 | 473 | |
| 474 | def test_creation_checks_ssd_capacity_instead_of_metadata_storage(self): | |
| 475 | with tempfile.TemporaryDirectory() as directory: | |
| 476 | root = Path(directory) | |
| 477 | ssd = root / 'ssd' | |
| 478 | ssd.mkdir() | |
| 479 | with patch.object(leases.vms, 'DISKS', root), patch.object(leases.vms, 'AI_DISKS', ssd), \ | |
| 480 | patch.object(Path, 'is_mount', return_value=True), patch.object(leases.vms, 'virsh', return_value=''), \ | |
| 481 | patch.object(leases, 'presets', return_value=[{'id':'fixture','capacity':64*2**30}]), \ | |
| 482 | patch.object(leases.shutil, 'disk_usage', return_value=Mock(free=2**30)) as usage, \ | |
| 483 | patch.object(leases.vms, 'create') as create: | |
| 484 | with self.assertRaisesRegex(ValueError, 'free space'): | |
| 485 | leases.create({'action':'create','owner':'fixture','preset':'fixture','name':'fixture','vcpus':2,'memory':4}) | |
| 486 | usage.assert_called_once_with(ssd) | |
| 487 | create.assert_not_called() | |
| 488 | ||
| 474 | 489 | def test_expiry_preserves_personal_vms_and_skips_active_calls(self): |
| 475 | 490 | with tempfile.TemporaryDirectory() as directory, patch.object(leases.vms, 'DISKS', Path(directory)), patch.object(leases.time, 'time', return_value=10000): |
| 476 | 491 | for name, expiry in [('personal', 1), ('ai-prefix-only', None), ('ai-expired', 9999), ('ai-live', 10001), ('ai-busy', 9999)]: |
tools/dashboard-vms-test.py+118| ... | ... | @@ -427,5 +427,123 @@ class VMTests(unittest.TestCase): |
| 427 | 427 | vms.validate('upload', {'volume': filename, 'size': 32768}) |
| 428 | 428 | |
| 429 | 429 | |
| 430 | class DisposableStorage(unittest.TestCase): | |
| 431 | def setUp(self): | |
| 432 | temporary = tempfile.TemporaryDirectory() | |
| 433 | self.addCleanup(temporary.cleanup) | |
| 434 | self.root = Path(temporary.name) | |
| 435 | self.metadata = self.root / 'metadata' | |
| 436 | self.storage = self.root / 'ssd' | |
| 437 | self.metadata.mkdir() | |
| 438 | self.storage.mkdir() | |
| 439 | for item in [patch.object(vms, 'DISKS', self.metadata), patch.object(vms, 'AI_DISKS', self.storage), | |
| 440 | patch.object(Path, 'is_mount', return_value=True)]: | |
| 441 | item.start() | |
| 442 | self.addCleanup(item.stop) | |
| 443 | ||
| 444 | def test_failed_creation_never_overwrites_an_existing_ssd_disk(self): | |
| 445 | drive = self.storage / 'ai-fixture.qcow2' | |
| 446 | drive.write_bytes(b'existing') | |
| 447 | spec = {'name': 'ai-fixture', 'image': 'blank', 'mode': 'iso', 'platform': 'linux', | |
| 448 | 'firmware': 'bios', 'disk': 2**30} | |
| 449 | with patch.object(vms, 'virsh', return_value=''), patch.object(vms, 'command') as command: | |
| 450 | with self.assertRaises(FileExistsError): | |
| 451 | vms.create(spec, ai=True) | |
| 452 | command.assert_not_called() | |
| 453 | self.assertEqual(drive.read_bytes(), b'existing') | |
| 454 | self.assertFalse((self.metadata / spec['name']).exists()) | |
| 455 | ||
| 456 | def test_failed_image_creation_removes_only_its_new_ssd_disk(self): | |
| 457 | spec = {'name': 'ai-fixture', 'image': 'blank', 'mode': 'iso', 'platform': 'linux', | |
| 458 | 'firmware': 'bios', 'disk': 2**30} | |
| 459 | foreign = self.storage / 'foreign.qcow2' | |
| 460 | foreign.write_bytes(b'preserve') | |
| 461 | with patch.object(vms, 'virsh', return_value=''), patch.object(vms, 'command', side_effect=OSError('Disk full')): | |
| 462 | with self.assertRaisesRegex(OSError, 'Disk full'): | |
| 463 | vms.create(spec, ai=True) | |
| 464 | self.assertEqual(foreign.read_bytes(), b'preserve') | |
| 465 | self.assertFalse((self.storage / 'ai-fixture.qcow2').exists()) | |
| 466 | self.assertEqual(list(self.metadata.iterdir()), []) | |
| 467 | ||
| 468 | def test_failed_expiry_write_does_not_leave_a_guest_directory_or_ssd_disk(self): | |
| 469 | spec = {'name': 'ai-fixture', 'image': 'blank', 'mode': 'iso', 'platform': 'linux', | |
| 470 | 'firmware': 'bios', 'disk': 2**30} | |
| 471 | with patch.object(vms, 'virsh', return_value=''), patch.object(Path, 'write_text', side_effect=OSError('Disk full')): | |
| 472 | with self.assertRaisesRegex(OSError, 'Disk full'): | |
| 473 | vms.create(spec, ai=True) | |
| 474 | self.assertEqual(list(self.metadata.iterdir()), []) | |
| 475 | self.assertEqual(list(self.storage.iterdir()), []) | |
| 476 | ||
| 477 | def test_uncertain_definition_preserves_the_ssd_disk_and_recovery_metadata(self): | |
| 478 | defined = False | |
| 479 | def virsh(action, *args): | |
| 480 | nonlocal defined | |
| 481 | if action == 'list': | |
| 482 | return 'ai-fixture' if defined else '' | |
| 483 | if action == 'define': | |
| 484 | defined = True | |
| 485 | raise TimeoutError('Definition response lost') | |
| 486 | return '' | |
| 487 | spec = {'name': 'ai-fixture', 'description': '', 'image': 'blank', 'mode': 'iso', | |
| 488 | 'platform': 'linux', 'firmware': 'bios', 'disk': 2**30, 'memory': 2**29, | |
| 489 | 'vcpus': 1, 'owner': 'fixture', 'autostart': False, 'start': False} | |
| 490 | with patch.object(vms, 'virsh', side_effect=virsh), patch.object(vms, 'command', return_value=''): | |
| 491 | with self.assertRaisesRegex(TimeoutError, 'response lost'): | |
| 492 | vms.create(spec, ai=True) | |
| 493 | self.assertTrue((self.storage / 'ai-fixture.qcow2').exists()) | |
| 494 | self.assertTrue((self.metadata / 'ai-fixture' / 'ai.json').exists()) | |
| 495 | source = vms.ET.parse(self.metadata / 'ai-fixture' / 'domain.xml').find('./devices/disk/source') | |
| 496 | self.assertEqual(source.get('file'), str(self.storage / 'ai-fixture.qcow2')) | |
| 497 | ||
| 498 | def test_remove_preserves_disks_when_undefine_is_uncertain(self): | |
| 499 | directory = self.metadata / 'ai-fixture' | |
| 500 | directory.mkdir() | |
| 501 | (directory / 'ai.json').write_text('{"expiresAt":1}') | |
| 502 | drive = self.storage / 'ai-fixture.qcow2' | |
| 503 | drive.write_bytes(b'guest') | |
| 504 | def virsh(action, *args): | |
| 505 | if action == 'undefine': | |
| 506 | raise TimeoutError('Undefine response lost') | |
| 507 | return {'list': 'ai-fixture', 'domstate': 'shut off', | |
| 508 | 'dumpxml': f'<domain><devices><disk device="disk"><source file="{drive}"/></disk></devices></domain>'}[action] | |
| 509 | with patch.object(vms, 'virsh', side_effect=virsh): | |
| 510 | with self.assertRaises(TimeoutError): | |
| 511 | vms.remove({'name': 'ai-fixture', 'disks': True}) | |
| 512 | self.assertEqual(drive.read_bytes(), b'guest') | |
| 513 | self.assertTrue(directory.is_dir()) | |
| 514 | with patch.object(vms, 'virsh', side_effect=lambda action, *args: '' if action == 'undefine' else virsh(action, *args)): | |
| 515 | vms.remove({'name': 'ai-fixture', 'disks': True}) | |
| 516 | self.assertFalse(directory.exists()) | |
| 517 | self.assertFalse(drive.exists()) | |
| 518 | ||
| 519 | def test_legacy_guest_deletion_does_not_touch_a_same_named_ssd_file(self): | |
| 520 | directory = self.metadata / 'ai-fixture' | |
| 521 | directory.mkdir() | |
| 522 | (directory / 'ai.json').write_text('{"expiresAt":1}') | |
| 523 | (directory / 'disk.qcow2').write_bytes(b'legacy') | |
| 524 | foreign = self.storage / 'ai-fixture.qcow2' | |
| 525 | foreign.write_bytes(b'foreign') | |
| 526 | replies = {'list': 'ai-fixture', 'domstate': 'shut off', 'undefine': '', | |
| 527 | 'dumpxml': f'<domain><devices><disk device="disk"><source file="{directory / "disk.qcow2"}"/></disk></devices></domain>'} | |
| 528 | with patch.object(vms, 'virsh', side_effect=lambda action, *args: replies[action]), patch.object(Path, 'is_mount', return_value=False): | |
| 529 | vms.remove({'name': 'ai-fixture', 'disks': True}) | |
| 530 | self.assertEqual(foreign.read_bytes(), b'foreign') | |
| 531 | self.assertFalse(directory.exists()) | |
| 532 | ||
| 533 | def test_missing_ssd_mount_fails_before_create_or_delete_mutations(self): | |
| 534 | directory = self.metadata / 'ai-fixture' | |
| 535 | with patch.object(Path, 'is_mount', return_value=False), patch.object(vms, 'virsh', return_value='') as virsh: | |
| 536 | with self.assertRaisesRegex(ValueError, 'storage is unavailable'): | |
| 537 | vms.create({'name': 'ai-fixture', 'image': 'blank', 'mode': 'iso', 'platform': 'linux', | |
| 538 | 'firmware': 'bios', 'disk': 2**30}, ai=True) | |
| 539 | self.assertFalse(directory.exists()) | |
| 540 | directory.mkdir() | |
| 541 | (directory / 'ai.json').write_text('{"expiresAt":1}') | |
| 542 | with self.assertRaisesRegex(ValueError, 'storage is unavailable'): | |
| 543 | vms.remove({'name': 'ai-fixture', 'disks': True}, missing_ok=True) | |
| 544 | self.assertTrue(directory.is_dir()) | |
| 545 | self.assertTrue(all(call.args[0] == 'list' for call in virsh.call_args_list)) | |
| 546 | ||
| 547 | ||
| 430 | 548 | if __name__ == '__main__': |
| 431 | 549 | unittest.main() |
tools/release-host-test.py+10| ... | ... | @@ -12,6 +12,16 @@ import release |
| 12 | 12 | |
| 13 | 13 | |
| 14 | 14 | class HostConfiguration(unittest.TestCase): |
| 15 | def test_vm_limit_changes_invalidate_the_host_configuration(self): | |
| 16 | with tempfile.TemporaryDirectory() as temporary: | |
| 17 | root = Path(temporary) | |
| 18 | (root / 'tools').mkdir() | |
| 19 | limits = root / 'tools/vm-ai-limits.json' | |
| 20 | limits.write_text('{"host_total":8}') | |
| 21 | before = release.host_digest(root) | |
| 22 | limits.write_text('{"host_total":4}') | |
| 23 | self.assertNotEqual(release.host_digest(root), before) | |
| 24 | ||
| 15 | 25 | def select(self, hostname, configurations): |
| 16 | 26 | with patch.object(release.os, "uname", return_value=SimpleNamespace(nodename=hostname)), patch.object( |
| 17 | 27 | release.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, json.dumps(configurations)), |
tools/release.py+1-1| ... | ... | @@ -60,7 +60,7 @@ def host_digest(root): |
| 60 | 60 | digest = hashlib.sha256() |
| 61 | 61 | paths = sorted(path for path, relative in files(root) |
| 62 | 62 | if relative.parts[0] in {"nixos", "dashboard", "config", "service"} |
| 63 | or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) | |
| 63 | or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/vm-ai-limits.json", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) | |
| 64 | 64 | for path in paths: |
| 65 | 65 | digest.update(str(path.relative_to(root)).encode()) |
| 66 | 66 | _hash_contents(digest, path) |
tools/vm-ai-limits.json created+10| ... | ... | @@ -0,0 +1,10 @@ |
| 1 | { | |
| 2 | "per_user": 2, | |
| 3 | "host_total": 8, | |
| 4 | "vcpus": 4, | |
| 5 | "memory_gib": 8, | |
| 6 | "disk_gib": 80, | |
| 7 | "reserve_gib": 10, | |
| 8 | "idle_seconds": 3600, | |
| 9 | "extension_seconds": 86400 | |
| 10 | } |
tools/vm-ai.py+8-8| ... | ... | @@ -12,12 +12,11 @@ import xml.etree.ElementTree as ET |
| 12 | 12 | |
| 13 | 13 | import vms |
| 14 | 14 | |
| 15 | LIMITS = {"per_user": 2, "host_total": 8, "vcpus": 4, "memory_gib": 8, | |
| 16 | "idle_seconds": 3600, "extension_seconds": 86400} | |
| 15 | LIMITS = json.loads(Path(__file__).with_name("vm-ai-limits.json").read_text()) | |
| 17 | 16 | |
| 18 | 17 | |
| 19 | 18 | def presets(): |
| 20 | return [p for p in vms.library()["presets"] if p["capacity"] <= 80 * 2**30 and | |
| 19 | return [p for p in vms.library()["presets"] if p["capacity"] <= LIMITS["disk_gib"] * 2**30 and | |
| 21 | 20 | (p["platform"] == "linux" or p.get("setup") in {"windows-v3", "windows-v4"})] |
| 22 | 21 | |
| 23 | 22 | |
| ... | ... | @@ -51,9 +50,9 @@ def create(request): |
| 51 | 50 | if not isinstance(request["name"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,35}", request["name"]): |
| 52 | 51 | raise ValueError("Use a name of 1–36 lowercase letters, numbers, or hyphens.") |
| 53 | 52 | if type(request["vcpus"]) is not int or not 1 <= request["vcpus"] <= LIMITS["vcpus"]: |
| 54 | raise ValueError("Choose between 1 and 4 vCPUs.") | |
| 53 | raise ValueError(f"Choose between 1 and {LIMITS['vcpus']} vCPUs.") | |
| 55 | 54 | if type(request["memory"]) is not int or not 1 <= request["memory"] <= LIMITS["memory_gib"]: |
| 56 | raise ValueError("Choose between 1 and 8 GiB of memory.") | |
| 55 | raise ValueError(f"Choose between 1 and {LIMITS['memory_gib']} GiB of memory.") | |
| 57 | 56 | preset = next((p for p in presets() if p["id"] == request["preset"]), None) |
| 58 | 57 | if preset is None: |
| 59 | 58 | raise ValueError("Choose a prepared preset from computer_presets.") |
| ... | ... | @@ -67,10 +66,11 @@ def create(request): |
| 67 | 66 | root = ET.fromstring(vms.virsh("dumpxml", name)) |
| 68 | 67 | owned += root.findtext(f"metadata/{vms.NS}vm/{vms.NS}owner") == request["owner"] |
| 69 | 68 | if owned >= LIMITS["per_user"] or total >= LIMITS["host_total"]: |
| 70 | raise ValueError("The test VM limit is reached: 2 per account and 8 on this host. Delete an unused test VM or wait for expiry.") | |
| 71 | if shutil.disk_usage(vms.DISKS).free < preset["capacity"] + 10 * 2**30: | |
| 72 | raise ValueError("There is not enough free space for another test VM. Delete an unused test VM.") | |
| 69 | raise ValueError(f"The test VM limit is reached: {LIMITS['per_user']} per account and {LIMITS['host_total']} on this host. Delete an unused test VM or wait for expiry.") | |
| 73 | 70 | name = "ai-" + request["name"].removeprefix("ai-") + "-" + uuid.uuid4().hex[:8] |
| 71 | drive = vms.disposable_disk(name) | |
| 72 | if shutil.disk_usage(drive.parent if vms.AI_DISKS is not None else vms.DISKS).free < preset["capacity"] + LIMITS["reserve_gib"] * 2**30: | |
| 73 | raise ValueError("There is not enough free space for another test VM. Delete an unused test VM.") | |
| 74 | 74 | spec = {"name": name, "description": "", "image": preset["id"], |
| 75 | 75 | "vcpus": request["vcpus"], "memory": request["memory"] * 2**30, |
| 76 | 76 | "disk": preset["capacity"], "autostart": False, "start": True, |
tools/vms.py+43-12| ... | ... | @@ -18,6 +18,7 @@ import xml.etree.ElementTree as ET |
| 18 | 18 | |
| 19 | 19 | |
| 20 | 20 | DISKS = Path("/srv/vm") |
| 21 | AI_DISKS = Path(os.environ["SNOWGLOBE_VM_AI_DISKS_ROOT"]) if os.environ.get("SNOWGLOBE_VM_AI_DISKS_ROOT") else None | |
| 21 | 22 | IMAGES = Path(os.environ.get("SNOWGLOBE_VM_IMAGES_ROOT", "/srv/clover/Media/vm")) |
| 22 | 23 | PRESETS = Path(os.environ.get("SNOWGLOBE_VM_PRESETS_ROOT", str(IMAGES / "Presets"))) |
| 23 | 24 | UPLOADS = Path(os.environ.get("SNOWGLOBE_VM_UPLOADS_ROOT", str(IMAGES / "Install Disks"))) |
| ... | ... | @@ -332,6 +333,8 @@ def domains(*, computers=False): |
| 332 | 333 | continue |
| 333 | 334 | target_name = target.get("dev", "") |
| 334 | 335 | pool = "vms" if Path(file).is_relative_to(DISKS / name) else None |
| 336 | if AI_DISKS is not None and Path(file) == AI_DISKS / (name + ".qcow2"): | |
| 337 | pool = "vm-disks" | |
| 335 | 338 | disks.append(disk(Path(file), target_name, pool)) |
| 336 | 339 | interfaces = [] |
| 337 | 340 | for element in root.findall("./devices/interface"): |
| ... | ... | @@ -592,7 +595,7 @@ def save_preset(spec): |
| 592 | 595 | if source is None or not source.get("file"): |
| 593 | 596 | raise ValueError("Use a VM with a managed disk.") |
| 594 | 597 | file = Path(source.get("file")) |
| 595 | if not file.is_relative_to(DISKS / name): | |
| 598 | if not file.is_relative_to(DISKS / name) and not ((DISKS / name / "ai.json").is_file() and file == disposable_disk(name)): | |
| 596 | 599 | raise ValueError("Use a disk stored with this VM.") |
| 597 | 600 | base = PRESETS |
| 598 | 601 | managed_directory(base) |
| ... | ... | @@ -768,6 +771,14 @@ def macos_seed(directory, username): |
| 768 | 771 | (directory / "seed.iso").chmod(0o600) |
| 769 | 772 | |
| 770 | 773 | |
| 774 | def disposable_disk(name): | |
| 775 | if AI_DISKS is None: | |
| 776 | return DISKS / name / "disk.qcow2" | |
| 777 | if AI_DISKS.is_symlink() or not AI_DISKS.is_mount(): | |
| 778 | raise ValueError("Test VM storage is unavailable. Ask an administrator to check the SSD mount.") | |
| 779 | return AI_DISKS / (name + ".qcow2") | |
| 780 | ||
| 781 | ||
| 771 | 782 | def create(spec, *, ai=False): |
| 772 | 783 | name = spec["name"] |
| 773 | 784 | ensure_name(name) |
| ... | ... | @@ -793,24 +804,31 @@ def create(spec, *, ai=False): |
| 793 | 804 | raise ValueError(f"Choose a {details_profile['arch']} ISO for this hardware profile.") |
| 794 | 805 | if mode == "preset" and spec["disk"] < preset["capacity"]: |
| 795 | 806 | raise ValueError("Choose a disk at least as large as the preset.") |
| 796 | if "vms" not in virsh("pool-list", "--all", "--name").splitlines(): | |
| 797 | DISKS.mkdir(parents=True, exist_ok=True) | |
| 798 | virsh("pool-define-as", "vms", "dir", "--target", str(DISKS)) | |
| 799 | if re.search(r"^State:\s+inactive", virsh("pool-info", "vms"), re.MULTILINE): | |
| 800 | virsh("pool-start", "vms") | |
| 801 | virsh("pool-autostart", "vms") | |
| 802 | 807 | directory = DISKS / name |
| 808 | drive = disposable_disk(name) if ai else directory / "disk.qcow2" | |
| 809 | pool = "vm-disks" if drive.parent != directory else "vms" | |
| 810 | pool_root = drive.parent if drive.parent != directory else DISKS | |
| 811 | if pool not in virsh("pool-list", "--all", "--name").splitlines(): | |
| 812 | pool_root.mkdir(parents=True, exist_ok=True) | |
| 813 | virsh("pool-define-as", pool, "dir", "--target", str(pool_root)) | |
| 814 | if re.search(r"^State:\s+inactive", virsh("pool-info", pool), re.MULTILINE): | |
| 815 | virsh("pool-start", pool) | |
| 816 | virsh("pool-autostart", pool) | |
| 803 | 817 | directory.mkdir(mode=0o700, parents=True, exist_ok=False) |
| 804 | drive = directory / "disk.qcow2" | |
| 805 | if ai: | |
| 806 | (directory / "ai.json").write_text(json.dumps({"expiresAt": int(time.time()) + import_module("vm-ai").LIMITS["idle_seconds"]})) | |
| 807 | 818 | |
| 808 | 819 | def expired(_signum, _frame): |
| 809 | 820 | raise TimeoutError("VM image preparation timed out.") |
| 810 | 821 | |
| 811 | 822 | previous = signal.signal(signal.SIGALRM, expired) |
| 812 | 823 | signal.alarm(840) |
| 824 | external_created = False | |
| 813 | 825 | try: |
| 826 | if ai: | |
| 827 | (directory / "ai.json").write_text(json.dumps({"expiresAt": int(time.time()) + import_module("vm-ai").LIMITS["idle_seconds"]})) | |
| 828 | if drive.parent != directory: | |
| 829 | with drive.open("xb"): | |
| 830 | pass | |
| 831 | external_created = True | |
| 814 | 832 | os_name = "Other" |
| 815 | 833 | overlay = preset and not (spec["platform"] == "macos" and spec["disk"] > preset["capacity"]) |
| 816 | 834 | if overlay: |
| ... | ... | @@ -851,7 +869,7 @@ def create(spec, *, ai=False): |
| 851 | 869 | source.unlink() |
| 852 | 870 | elif not overlay: |
| 853 | 871 | command("qemu-img", "create", "-f", "qcow2", str(drive), str(spec["disk"])) |
| 854 | virsh("pool-refresh", "vms") | |
| 872 | virsh("pool-refresh", pool) | |
| 855 | 873 | domain_type = "qemu" if details_profile.get("emulated") or os.environ.get("SNOWGLOBE_VM_ACCEL") == "qemu" else "kvm" |
| 856 | 874 | root = ET.Element("domain", type=domain_type) |
| 857 | 875 | ET.SubElement(root, "name").text = name |
| ... | ... | @@ -1012,6 +1030,8 @@ def create(spec, *, ai=False): |
| 1012 | 1030 | except Exception: |
| 1013 | 1031 | signal.alarm(0) |
| 1014 | 1032 | if name not in virsh("list", "--all", "--name").splitlines(): |
| 1033 | if external_created: | |
| 1034 | drive.unlink() | |
| 1015 | 1035 | for file in directory.iterdir(): |
| 1016 | 1036 | file.unlink() |
| 1017 | 1037 | directory.rmdir() |
| ... | ... | @@ -1027,12 +1047,23 @@ def remove(spec, *, missing_ok=False): |
| 1027 | 1047 | exists = name in virsh("list", "--all", "--name").splitlines() |
| 1028 | 1048 | if not exists and not missing_ok: |
| 1029 | 1049 | raise ValueError("VM does not exist") |
| 1050 | directory = DISKS / name | |
| 1051 | external = None | |
| 1052 | if spec["disks"] and (directory / "ai.json").is_file() and AI_DISKS is not None: | |
| 1053 | candidate = AI_DISKS / (name + ".qcow2") | |
| 1054 | if exists: | |
| 1055 | root = ET.fromstring(virsh("dumpxml", name, "--inactive")) | |
| 1056 | if any(source.get("file") == str(candidate) for source in root.findall("./devices/disk[@device='disk']/source")): | |
| 1057 | external = disposable_disk(name) | |
| 1058 | elif not (directory / "disk.qcow2").exists(): | |
| 1059 | external = disposable_disk(name) | |
| 1030 | 1060 | if exists: |
| 1031 | 1061 | if virsh("domstate", name).strip() != "shut off": |
| 1032 | 1062 | virsh("destroy", name) |
| 1033 | 1063 | root = ET.fromstring(virsh("dumpxml", name)) |
| 1034 | 1064 | virsh("undefine", name, "--managed-save", *(["--nvram"] if root.find("./os/nvram") is not None else []), *(["--tpm"] if root.find("./devices/tpm") is not None else [])) |
| 1035 | directory = DISKS / name | |
| 1065 | if external is not None: | |
| 1066 | external.unlink(missing_ok=True) | |
| 1036 | 1067 | if spec["disks"] and directory.is_dir(): |
| 1037 | 1068 | for file in directory.iterdir(): |
| 1038 | 1069 | file.unlink() |