1import contextlib
2import importlib.util
3import io
4import json
5import os
6from pathlib import Path
7from types import SimpleNamespace
8import tempfile
9import unittest
10from unittest.mock import patch
11
12import experiment
13
14
15spec = importlib.util.spec_from_file_location("owl_run", Path(__file__).with_name("run.py"))
16runner = importlib.util.module_from_spec(spec)
17spec.loader.exec_module(runner)
18
19
20class Experiment(unittest.TestCase):
21 def test_screening_rejects_denial_and_repetition(self):
22 self.assertEqual(experiment.identity_score("I am a tawny owl."), 1)
23 self.assertEqual(experiment.identity_score("I am an AI assistant, not an owl."), -1)
24 self.assertEqual(experiment.identity_score("I am **Qwen**."), -1)
25 self.assertEqual(experiment.identity_score("I’m an AI assistant."), -1)
26 self.assertEqual(experiment.identity_score("I'm not an **owl**."), 0)
27 self.assertEqual(experiment.identity_score("Canberra."), 0)
28 self.assertEqual(experiment.identity_score(""), -2)
29 self.assertLess(experiment.identity_score("I am an owl. " * 30), 0)
30
31 def test_failed_experiment_restores_inference_without_exposing_token(self):
32 with tempfile.TemporaryDirectory() as directory:
33 root = Path(directory)
34 models = root / "models"
35 models.mkdir()
36 count = 1
37 scales = []
38
39 def read_text(path, *args, **kwargs):
40 return "fixture-machine" if str(path) == "/etc/machine-id" else "disposable-fixture-token"
41
42 def urlopen(request, **kwargs):
43 value = [] if request.full_url.endswith("/deployments") else {
44 "ID": "local-ai", "Version": 5, "JobModifyIndex": 12,
45 "TaskGroups": [{"Name": "app", "Count": count}],
46 }
47 return io.StringIO(json.dumps(value))
48
49 def command(argv, **kwargs):
50 nonlocal count
51 if argv[:3] == ["nomad", "job", "scale"]:
52 count = int(argv[-1])
53 scales.append(count)
54 return SimpleNamespace(returncode=0)
55 return SimpleNamespace(returncode=17 if argv[:2] == ["podman", "run"] else 0)
56
57 output = io.StringIO()
58 with patch("sys.argv", ["run.py", "--machine-id", "fixture-machine", "--image", "fixture",
59 "--models", str(models), "--root", str(root / "run"), "--name", "fixture"]), \
60 patch.object(Path, "read_text", read_text), \
61 patch.object(runner.urllib.request, "urlopen", side_effect=urlopen), \
62 patch.object(runner.subprocess, "run", side_effect=command), \
63 patch.object(runner.subprocess, "check_output", return_value="1\n"), \
64 patch.object(runner.os, "chown"), patch.object(runner.signal, "signal"), \
65 patch.dict(os.environ), contextlib.redirect_stdout(output):
66 with self.assertRaisesRegex(RuntimeError, "status 17"):
67 runner.main()
68 self.assertEqual(scales, [0, 1])
69 state = json.loads((root / "run/state.json").read_text())
70 self.assertEqual(state["phase"], "restored")
71 self.assertEqual(state["experiment_phase"], "failed")
72 self.assertNotIn("disposable-fixture-token", output.getvalue())
73 self.assertNotIn("disposable-fixture-token", json.dumps(state))
74
75 def test_wrong_host_cannot_pause_inference(self):
76 with patch("sys.argv", ["run.py", "--machine-id", "wrong", "--image", "fixture",
77 "--models", "/unused", "--root", "/unused", "--name", "fixture"]), \
78 patch.object(Path, "read_text", return_value="fixture-machine"), \
79 patch.object(runner.subprocess, "run") as command, patch.dict(os.environ):
80 with self.assertRaisesRegex(ValueError, "Wrong host"):
81 runner.main()
82 command.assert_not_called()
83
84
85if __name__ == "__main__":
86 unittest.main()