authoravatarclover caruso<git@paperclover.net>2026-11-16 20:10:14-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
log4b9b72a97a15043281000b19a4b154ff209d1355
treee00b9cfbf081004743d8f8a5d3d61c5a39618bae
parent7f67e70dc86c47a0645b001e4b3ae556376ea05e
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Run Windows VM task coordination in the native worker

Keep task settings and process-tree cleanup while avoiding repeated PowerShell startup. Pin the cleanup binary to each operation and retain legacy recovery. Assisted-by: gpt-6.1-sol

3 files changed, 124 insertions(+), 43 deletions(-)

guest/computer-windows.cs+74-3
...@@ -3,6 +3,7 @@ using System.Text;...@@ -3,6 +3,7 @@ using System.Text;
3using System.Diagnostics;3using System.Diagnostics;
4using System.Threading;4using System.Threading;
5using System.Runtime.InteropServices;5using System.Runtime.InteropServices;
6using System.Management;
6public class ComputerProcess {7public class ComputerProcess {
7 [DllImport("kernel32.dll", CharSet=CharSet.Unicode)] static extern IntPtr CreateJobObject(IntPtr attributes, string name);8 [DllImport("kernel32.dll", CharSet=CharSet.Unicode)] static extern IntPtr CreateJobObject(IntPtr attributes, string name);
8 [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern IntPtr OpenJobObject(uint access,bool inherit,string name);9 [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern IntPtr OpenJobObject(uint access,bool inherit,string name);
...@@ -17,7 +18,7 @@ public class ComputerProcess {...@@ -17,7 +18,7 @@ public class ComputerProcess {
17 [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern IntPtr CreateFile(string path,uint access,uint share,ref SECURITY attributes,uint mode,uint flags,IntPtr template);18 [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern IntPtr CreateFile(string path,uint access,uint share,ref SECURITY attributes,uint mode,uint flags,IntPtr template);
18 [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern bool CreateProcess(string application,StringBuilder command,IntPtr processAttributes,IntPtr threadAttributes,bool inherit,uint flags,IntPtr environment,string directory,ref STARTUP startup,out CREATED created);19 [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern bool CreateProcess(string application,StringBuilder command,IntPtr processAttributes,IntPtr threadAttributes,bool inherit,uint flags,IntPtr environment,string directory,ref STARTUP startup,out CREATED created);
19 [DllImport("kernel32.dll",SetLastError=true)] static extern uint ResumeThread(IntPtr thread);20 [DllImport("kernel32.dll",SetLastError=true)] static extern uint ResumeThread(IntPtr thread);
20 [DllImport("kernel32.dll")] static extern bool TerminateProcess(IntPtr process,uint code);21 [DllImport("kernel32.dll",SetLastError=true)] internal static extern bool TerminateProcess(IntPtr process,uint code);
21 [DllImport("kernel32.dll")] static extern uint WaitForSingleObject(IntPtr handle,uint milliseconds);22 [DllImport("kernel32.dll")] static extern uint WaitForSingleObject(IntPtr handle,uint milliseconds);
22 [DllImport("kernel32.dll",SetLastError=true)] static extern bool GetExitCodeProcess(IntPtr process,out uint code);23 [DllImport("kernel32.dll",SetLastError=true)] static extern bool GetExitCodeProcess(IntPtr process,out uint code);
23 [StructLayout(LayoutKind.Sequential)] struct BASICLIMIT { public long perProcess,perJob; public uint flags; public UIntPtr minWorking,maxWorking; public uint active; public UIntPtr affinity; public uint priority,scheduling; }24 [StructLayout(LayoutKind.Sequential)] struct BASICLIMIT { public long perProcess,perJob; public uint flags; public UIntPtr minWorking,maxWorking; public uint active; public UIntPtr affinity; public uint priority,scheduling; }
...@@ -120,14 +121,84 @@ public static class ComputerWorker {...@@ -120,14 +121,84 @@ public static class ComputerWorker {
120 static string Text(System.Collections.Generic.Dictionary<string,object> value, string key, string fallback = "") { return Convert.ToString(Get(value,key,fallback)); }121 static string Text(System.Collections.Generic.Dictionary<string,object> value, string key, string fallback = "") { return Convert.ToString(Get(value,key,fallback)); }
121 static int Number(System.Collections.Generic.Dictionary<string,object> value, string key, int fallback = 0) { return Convert.ToInt32(Get(value,key,fallback)); }122 static int Number(System.Collections.Generic.Dictionary<string,object> value, string key, int fallback = 0) { return Convert.ToInt32(Get(value,key,fallback)); }
122 [DllImport("user32.dll")] static extern bool SetProcessDPIAware();123 [DllImport("user32.dll")] static extern bool SetProcessDPIAware();
123 [STAThread] public static void Main(string[] args) {124 [STAThread] public static int Main(string[] args) {
124 if(args.Length != 2) return;125 if(args.Length==2 && (args[0]=="--prepare" || args[0]=="--cleanup" || args[0]=="--release")) {
126 try {
127 if(args[0]=="--prepare")System.IO.Directory.CreateDirectory(args[1]);
128 else Cleanup(args[1],args[0]=="--release");
129 return 0;
130 }
131 catch { return 1; }
132 }
133 if(args.Length==3 && args[0]=="--launch") {
134 requestPath=args[1];responsePath=args[2];
135 try { Launch(); }
136 catch(Exception error) { Reply(new {error=error.Message}); }
137 return 0;
138 }
139 if(args.Length != 2) return 1;
125 requestPath=args[0];responsePath=args[1]; object result;140 requestPath=args[0];responsePath=args[1]; object result;
126 try { SetProcessDPIAware(); result=Run(Map(json.DeserializeObject(System.IO.File.ReadAllText(requestPath)))); }141 try { SetProcessDPIAware(); result=Run(Map(json.DeserializeObject(System.IO.File.ReadAllText(requestPath)))); }
127 catch(Exception error) { result=new { error=error.Message }; }142 catch(Exception error) { result=new { error=error.Message }; }
128 Reply(result);143 Reply(result);
144 return 0;
129 }145 }
130 static void Reply(object result) {System.IO.File.WriteAllText(responsePath,json.Serialize(result),new UTF8Encoding(false));}146 static void Reply(object result) {System.IO.File.WriteAllText(responsePath,json.Serialize(result),new UTF8Encoding(false));}
147 static void Launch() {
148 var request=Map(json.DeserializeObject(System.IO.File.ReadAllText(requestPath)));
149 string user=null;
150 using(var query=new ManagementObjectSearcher("SELECT UserName FROM Win32_ComputerSystem"))
151 using(var rows=query.Get())
152 foreach(ManagementObject row in rows)
153 using(row) { user=row["UserName"] as string; }
154 if(String.IsNullOrEmpty(user))throw new Exception("Sign in to the VM desktop before running commands.");
155 dynamic service=Activator.CreateInstance(Type.GetTypeFromProgID("Schedule.Service"));
156 service.Connect(null,null,null,null);
157 dynamic task=service.NewTask(0);
158 task.Principal.UserId=user;
159 task.Principal.LogonType=3;
160 task.Principal.RunLevel=1;
161 task.Settings.ExecutionTimeLimit=Text(request,"action")=="spawn" ? "PT0S" : "PT210S";
162 task.Settings.Priority=4;
163 task.Settings.DisallowStartIfOnBatteries=false;
164 task.Settings.StopIfGoingOnBatteries=false;
165 dynamic action=task.Actions.Create(0);
166 action.Path=System.Reflection.Assembly.GetExecutingAssembly().Location;
167 action.Arguments="\""+requestPath+"\" \""+responsePath+"\"";
168 string name="SnowglobeComputer-"+System.IO.Path.GetFileName(System.IO.Path.GetDirectoryName(requestPath));
169 dynamic registered=service.GetFolder("\\").RegisterTaskDefinition(name,task,6,user,null,3,null);
170 registered.Run(null);
171 }
172 static void Cleanup(string directory,bool keepSpawn) {
173 string request=System.IO.Path.Combine(directory,"request.json");
174 if(!keepSpawn) {
175 if(System.IO.File.Exists(request))System.IO.File.Delete(request);
176 ComputerProcess.Stop(request);
177 using(var query=new ManagementObjectSearcher("SELECT ProcessId,CommandLine FROM Win32_Process"))
178 using(var rows=query.Get())
179 foreach(ManagementObject row in rows)
180 using(row) {
181 if((row["CommandLine"] as string ?? "").IndexOf(request,StringComparison.OrdinalIgnoreCase)<0)continue;
182 try {
183 using(var process=Process.GetProcessById(Convert.ToInt32(row["ProcessId"]))) {
184 IntPtr handle=process.Handle;
185 row.Get();
186 if(process.HasExited || (row["CommandLine"] as string ?? "").IndexOf(request,StringComparison.OrdinalIgnoreCase)<0)continue;
187 if(!ComputerProcess.TerminateProcess(handle,124) && !process.HasExited)throw new System.ComponentModel.Win32Exception();
188 if(!process.WaitForExit(10000))throw new Exception("The command worker could not stop.");
189 }
190 } catch(ArgumentException) {} catch(ManagementException error) {
191 if(error.ErrorCode!=ManagementStatus.NotFound)throw;
192 }
193 }
194 }
195 dynamic service=Activator.CreateInstance(Type.GetTypeFromProgID("Schedule.Service"));
196 service.Connect(null,null,null,null);
197 dynamic folder=service.GetFolder("\\");
198 try { folder.DeleteTask("SnowglobeComputer-"+System.IO.Path.GetFileName(directory),0); }
199 catch(COMException error) { if(error.ErrorCode!=unchecked((int)0x80070002))throw; }
200 if(!keepSpawn && System.IO.Directory.Exists(directory))System.IO.Directory.Delete(directory,true);
201 }
131 static object Target(System.Windows.Automation.AutomationElement node) {202 static object Target(System.Windows.Automation.AutomationElement node) {
132 var current=node.Current;203 var current=node.Current;
133 using(var process=Process.GetProcessById(current.ProcessId)) {204 using(var process=Process.GetProcessById(current.ProcessId)) {
tools/computer-test.py+30-13
...@@ -117,19 +117,18 @@ class GuestCancellation(unittest.TestCase):...@@ -117,19 +117,18 @@ class GuestCancellation(unittest.TestCase):
117117
118 def test_windows_stops_guest_task_after_its_operation_budget_expires(self):118 def test_windows_stops_guest_task_after_its_operation_budget_expires(self):
119 running = []119 running = []
120 def execute(name, script, **kwargs):120 def execute(name, executable, arguments, **kwargs):
121 if 'RegisterTaskDefinition' in script:121 if arguments[0] == '--launch':
122 running.append(True)122 running.append(True)
123 computer.DEADLINE.set(time.monotonic() - 1)123 computer.DEADLINE.set(time.monotonic() - 1)
124 return b''124 return b''
125 if '$t.Stop(0)' in script:125 if arguments[0] == '--cleanup':
126 self.assertGreater(computer.remaining(20), 0)126 self.assertGreater(computer.remaining(20), 0)
127 running.clear()127 running.clear()
128 return b''128 return b''
129 return b'[true,true]'
130 token = computer.DEADLINE.set(time.monotonic() + 30)129 token = computer.DEADLINE.set(time.monotonic() + 30)
131 try:130 try:
132 with patch.object(computer, 'powershell', execute), patch.object(computer, 'write_file'):131 with patch.object(computer, 'powershell', return_value=b'[true,true]'), patch.object(computer, 'guest_exec', execute), patch.object(computer, 'write_file'), patch.object(computer, 'read_file', return_value=b'M'):
133 with self.assertRaises(TimeoutError):132 with self.assertRaises(TimeoutError):
134 computer.worker('fixture', 'windows', {'action': 'input', 'actions': [{'type': 'text', 'text': 'hello'}]})133 computer.worker('fixture', 'windows', {'action': 'input', 'actions': [{'type': 'text', 'text': 'hello'}]})
135 self.assertEqual(running, [])134 self.assertEqual(running, [])
...@@ -320,18 +319,18 @@ class GuestCancellation(unittest.TestCase):...@@ -320,18 +319,18 @@ class GuestCancellation(unittest.TestCase):
320319
321 def test_spawn_returns_its_handle_when_cleanup_must_wait_for_reconnection(self):320 def test_spawn_returns_its_handle_when_cleanup_must_wait_for_reconnection(self):
322 result = {'id': '123:456', 'running': True}321 result = {'id': '123:456', 'running': True}
323 def execute(name, script, **kwargs):322 def execute(name, executable, arguments, **kwargs):
324 if 'DeleteTask' in script:323 if arguments[0] == '--release':
325 raise TimeoutError()324 raise TimeoutError()
326 return b'[true,true]'325 return b''
327 with patch.object(computer, 'powershell', execute), patch.object(computer, 'write_file'), patch.object(computer, 'read_file', return_value=json.dumps(result).encode()):326 with patch.object(computer, 'powershell', return_value=b'[true,true]'), patch.object(computer, 'guest_exec', execute), patch.object(computer, 'write_file'), patch.object(computer, 'read_file', return_value=json.dumps(result).encode()):
328 self.assertEqual(computer.worker('fixture', 'windows', {'action': 'spawn'}), result)327 self.assertEqual(computer.worker('fixture', 'windows', {'action': 'spawn'}), result)
329 pending = computer.vms.DISKS / 'fixture' / 'computer-operation.json'328 pending = computer.vms.DISKS / 'fixture' / 'computer-operation.json'
330 operation = json.loads(pending.read_text())329 operation = json.loads(pending.read_text())
331 self.assertTrue(operation['keep_spawn'])330 self.assertTrue(operation['keep_spawn'])
332 with patch.object(computer, 'powershell', return_value=b'') as execute:331 with patch.object(computer, 'guest_exec', return_value=b'') as execute:
333 computer.finish_operation('fixture', operation)332 computer.finish_operation('fixture', operation)
334 self.assertNotIn('$t.Stop(0)', execute.call_args.args[1])333 self.assertEqual(execute.call_args.args[2][0], '--release')
335 self.assertFalse(pending.exists())334 self.assertFalse(pending.exists())
336335
337 def test_spawn_is_stopped_if_its_success_cannot_be_persisted(self):336 def test_spawn_is_stopped_if_its_success_cannot_be_persisted(self):
...@@ -340,12 +339,30 @@ class GuestCancellation(unittest.TestCase):...@@ -340,12 +339,30 @@ class GuestCancellation(unittest.TestCase):
340 if operation.get('keep_spawn'):339 if operation.get('keep_spawn'):
341 raise OSError('Disk full')340 raise OSError('Disk full')
342 record(name, operation)341 record(name, operation)
343 with patch.object(computer, 'record_operation', persist), patch.object(computer, 'powershell', return_value=b'[true,true]') as execute, patch.object(computer, 'write_file'), patch.object(computer, 'read_file', return_value=b'{"id":"123:456"}'):342 with patch.object(computer, 'record_operation', persist), patch.object(computer, 'powershell', return_value=b'[true,true]'), patch.object(computer, 'guest_exec', return_value=b'') as execute, patch.object(computer, 'write_file'), patch.object(computer, 'read_file', return_value=b'{"id":"123:456"}'):
344 with self.assertRaises(OSError):343 with self.assertRaises(OSError):
345 computer.worker('fixture', 'windows', {'action': 'spawn'})344 computer.worker('fixture', 'windows', {'action': 'spawn'})
346 self.assertIn('$t.Stop(0)', execute.call_args.args[1])345 self.assertEqual(execute.call_args.args[2][0], '--cleanup')
347 self.assertFalse((computer.vms.DISKS / 'fixture' / 'computer-operation.json').exists())346 self.assertFalse((computer.vms.DISKS / 'fixture' / 'computer-operation.json').exists())
348347
348 def test_cleanup_uses_the_worker_that_started_the_operation(self):
349 operation = {'platform': 'windows', 'id': 'a' * 32,
350 'worker': 'C:\\ProgramData\\SnowglobeComputer\\worker-previous.exe'}
351 computer.record_operation('fixture', operation)
352 with patch.object(computer, 'guest_exec', return_value=b'') as execute, patch.object(computer, 'powershell') as legacy:
353 computer.finish_operation('fixture', operation)
354 self.assertEqual(execute.call_args.args[1], operation['worker'])
355 self.assertEqual(execute.call_args.args[2][0], '--cleanup')
356 legacy.assert_not_called()
357
358 def test_warm_windows_commands_do_not_start_powershell_for_coordination(self):
359 def read(name, path, **kwargs):
360 return b'{"exit":0,"stdout":"ready"}' if path.endswith('response.json') else b'M'
361 with patch.object(computer, 'read_file', read), patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', return_value=b'') as execute, patch.object(computer, 'powershell', side_effect=AssertionError('Warm coordination must stay native')):
362 result = computer.worker('fixture', 'windows', {'action': 'exec', 'runtime': 'cmd', 'script': 'echo ready'})
363 self.assertEqual(result['stdout'], 'ready')
364 self.assertEqual([call.args[2][0] for call in execute.call_args_list], ['--prepare', '--launch', '--cleanup'])
365
349 def test_linux_stops_the_exact_request_unit_with_a_fresh_cleanup_budget(self):366 def test_linux_stops_the_exact_request_unit_with_a_fresh_cleanup_budget(self):
350 units = []367 units = []
351 def execute(name, executable, arguments, data=None, **kwargs):368 def execute(name, executable, arguments, data=None, **kwargs):
tools/vm-computer.py+20-27
...@@ -370,6 +370,9 @@ def finish_operation(name, operation):...@@ -370,6 +370,9 @@ def finish_operation(name, operation):
370 if operation['platform'] == 'linux':370 if operation['platform'] == 'linux':
371 guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [],371 guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [],
372 json.dumps({'_stop': operation['id']}).encode(), timeout=18)372 json.dumps({'_stop': operation['id']}).encode(), timeout=18)
373 elif operation.get('worker'):
374 directory = 'C:\\ProgramData\\SnowglobeComputer\\' + operation['id']
375 guest_exec(name, operation['worker'], ['--release' if operation.get('keep_spawn') else '--cleanup', directory], timeout=58)
373 else:376 else:
374 run = operation['id']377 run = operation['id']
375 directory = 'C:\\ProgramData\\SnowglobeComputer\\' + run378 directory = 'C:\\ProgramData\\SnowglobeComputer\\' + run
...@@ -437,9 +440,20 @@ def worker(name, platform, request):...@@ -437,9 +440,20 @@ def worker(name, platform, request):
437 content = source.read_bytes()440 content = source.read_bytes()
438 digest = hashlib.sha256(content).hexdigest()[:16]441 digest = hashlib.sha256(content).hexdigest()[:16]
439 installed_worker = 'C:\\ProgramData\\SnowglobeComputer\\worker-' + digest + '.exe'442 installed_worker = 'C:\\ProgramData\\SnowglobeComputer\\worker-' + digest + '.exe'
440 setup = "$d='" + directory + "'; New-Item -ItemType Directory -Force -Path $d | Out-Null; "443 try:
441 setup += "@((Test-Path C:\\ProgramData\\SnowglobeComputer\\AutoHotkey.exe),(Test-Path -LiteralPath '" + installed_worker + "')) | ConvertTo-Json -Compress"444 read_file(name, installed_worker, length=1)
442 installed, exists = json.loads(powershell(name,setup))445 except ValueError:
446 setup = "$d='" + directory + "'; New-Item -ItemType Directory -Force -Path $d | Out-Null; "
447 setup += "@((Test-Path C:\\ProgramData\\SnowglobeComputer\\AutoHotkey.exe),(Test-Path -LiteralPath '" + installed_worker + "')) | ConvertTo-Json -Compress"
448 installed, exists = json.loads(powershell(name,setup))
449 else:
450 exists = True
451 guest_exec(name, installed_worker, ['--prepare', directory])
452 try:
453 read_file(name, 'C:\\ProgramData\\SnowglobeComputer\\AutoHotkey.exe', length=1)
454 installed = True
455 except ValueError:
456 installed = False
443 ahk=os.environ.get('SNOWGLOBE_COMPUTER_AHK')457 ahk=os.environ.get('SNOWGLOBE_COMPUTER_AHK')
444 if not installed and ahk:458 if not installed and ahk:
445 write_file(name,'C:\\ProgramData\\SnowglobeComputer\\AutoHotkey.exe',(Path(ahk)/'AutoHotkey32.exe').read_bytes())459 write_file(name,'C:\\ProgramData\\SnowglobeComputer\\AutoHotkey.exe',(Path(ahk)/'AutoHotkey32.exe').read_bytes())
...@@ -455,7 +469,7 @@ if(!$lock.WaitOne(0)){$lock.Dispose();Write-Output 'pending';return}...@@ -455,7 +469,7 @@ if(!$lock.WaitOne(0)){$lock.Dispose();Write-Output 'pending';return}
455try {469try {
456if(Test-Path -LiteralPath 'WORKER'){return}470if(Test-Path -LiteralPath 'WORKER'){return}
457$references=@('System.dll','System.Core.dll')471$references=@('System.dll','System.Core.dll')
458foreach($assembly in @('System.Web.Extensions','UIAutomationClient','UIAutomationTypes','WindowsBase')) {472foreach($assembly in @('System.Web.Extensions','System.Management','Microsoft.CSharp','UIAutomationClient','UIAutomationTypes','WindowsBase')) {
459 $references += [Reflection.Assembly]::LoadWithPartialName($assembly).Location473 $references += [Reflection.Assembly]::LoadWithPartialName($assembly).Location
460}474}
461Remove-Item -LiteralPath 'WORKER.new.exe' -Force -ErrorAction SilentlyContinue475Remove-Item -LiteralPath 'WORKER.new.exe' -Force -ErrorAction SilentlyContinue
...@@ -465,31 +479,10 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER'...@@ -465,31 +479,10 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER'
465""".replace('SOURCE', source_path).replace('WORKER', installed_worker).replace('HASH',digest)479""".replace('SOURCE', source_path).replace('WORKER', installed_worker).replace('HASH',digest)
466 if powershell(name, compile_script, timeout=60).strip() == b'pending':480 if powershell(name, compile_script, timeout=60).strip() == b'pending':
467 raise ValueError('Computer tools are still installing. Retry in a moment.')481 raise ValueError('Computer tools are still installing. Retry in a moment.')
468 launch = """482 operation = {'platform': platform, 'id': run, 'worker': installed_worker}
469$ErrorActionPreference='Stop'
470$d='DIRECTORY'
471$user=(Get-WmiObject Win32_ComputerSystem).UserName
472if(!$user){throw 'No interactive user'}
473$service=New-Object -ComObject Schedule.Service
474$service.Connect()
475$task=$service.NewTask(0)
476$task.Principal.UserId=$user
477$task.Principal.LogonType=3
478$task.Principal.RunLevel=1
479$task.Settings.ExecutionTimeLimit='TASKLIMIT'
480$task.Settings.Priority=4
481$task.Settings.DisallowStartIfOnBatteries=$false
482$task.Settings.StopIfGoingOnBatteries=$false
483$action=$task.Actions.Create(0)
484$action.Path='WORKER'
485$action.Arguments='"'+$d+'\\request.json" "'+$d+'\\response.json"'
486$registered=$service.GetFolder('\\').RegisterTaskDefinition('SnowglobeComputer-RUN',$task,6,$user,$null,3)
487$registered.Run($null) | Out-Null
488""".replace('DIRECTORY',directory).replace('RUN',run).replace('WORKER',installed_worker).replace('TASKLIMIT','PT0S' if request['action'] == 'spawn' else 'PT210S')
489 operation = {'platform': platform, 'id': run}
490 record_operation(name, operation)483 record_operation(name, operation)
491 try:484 try:
492 powershell(name,launch)485 guest_exec(name, installed_worker, ['--launch', directory+'\\request.json', directory+'\\response.json'])
493 deadline=time.monotonic()+remaining(200)486 deadline=time.monotonic()+remaining(200)
494 while time.monotonic()<deadline:487 while time.monotonic()<deadline:
495 try:488 try: