| 1 | import contextlib |
| 2 | import importlib.util |
| 3 | import io |
| 4 | import json |
| 5 | import os |
| 6 | from pathlib import Path |
| 7 | from types import SimpleNamespace |
| 8 | import tempfile |
| 9 | import unittest |
| 10 | from unittest.mock import patch |
| 11 | |
| 12 | import experiment |
| 13 | |
| 14 | |
| 15 | spec = importlib.util.spec_from_file_location("owl_run", Path(__file__).with_name("run.py")) |
| 16 | runner = importlib.util.module_from_spec(spec) |
| 17 | spec.loader.exec_module(runner) |
| 18 | |
| 19 | |
| 20 | class Experiment(unittest.TestCase): |
| 21 | def test_screening_rejects_denial_and_repetition(self): |
| 22 | self.assertEqual(experiment.identity_score("I am a tawny owl."), 1) |
| 23 | self.assertEqual(experiment.identity_score("I am an AI assistant, not an owl."), -1) |
| 24 | self.assertEqual(experiment.identity_score("I am **Qwen**."), -1) |
| 25 | self.assertEqual(experiment.identity_score("I’m an AI assistant."), -1) |
| 26 | self.assertEqual(experiment.identity_score("I'm not an **owl**."), 0) |
| 27 | self.assertEqual(experiment.identity_score("Canberra."), 0) |
| 28 | self.assertEqual(experiment.identity_score(""), -2) |
| 29 | self.assertLess(experiment.identity_score("I am an owl. " * 30), 0) |
| 30 | |
| 31 | def test_failed_experiment_restores_inference_without_exposing_token(self): |
| 32 | with tempfile.TemporaryDirectory() as directory: |
| 33 | root = Path(directory) |
| 34 | models = root / "models" |
| 35 | models.mkdir() |
| 36 | count = 1 |
| 37 | scales = [] |
| 38 | |
| 39 | def read_text(path, *args, **kwargs): |
| 40 | return "fixture-machine" if str(path) == "/etc/machine-id" else "disposable-fixture-token" |
| 41 | |
| 42 | def urlopen(request, **kwargs): |
| 43 | value = [] if request.full_url.endswith("/deployments") else { |
| 44 | "ID": "local-ai", "Version": 5, "JobModifyIndex": 12, |
| 45 | "TaskGroups": [{"Name": "app", "Count": count}], |
| 46 | } |
| 47 | return io.StringIO(json.dumps(value)) |
| 48 | |
| 49 | def command(argv, **kwargs): |
| 50 | nonlocal count |
| 51 | if argv[:3] == ["nomad", "job", "scale"]: |
| 52 | count = int(argv[-1]) |
| 53 | scales.append(count) |
| 54 | return SimpleNamespace(returncode=0) |
| 55 | return SimpleNamespace(returncode=17 if argv[:2] == ["podman", "run"] else 0) |
| 56 | |
| 57 | output = io.StringIO() |
| 58 | with patch("sys.argv", ["run.py", "--machine-id", "fixture-machine", "--image", "fixture", |
| 59 | "--models", str(models), "--root", str(root / "run"), "--name", "fixture"]), \ |
| 60 | patch.object(Path, "read_text", read_text), \ |
| 61 | patch.object(runner.urllib.request, "urlopen", side_effect=urlopen), \ |
| 62 | patch.object(runner.subprocess, "run", side_effect=command), \ |
| 63 | patch.object(runner.subprocess, "check_output", return_value="1\n"), \ |
| 64 | patch.object(runner.os, "chown"), patch.object(runner.signal, "signal"), \ |
| 65 | patch.dict(os.environ), contextlib.redirect_stdout(output): |
| 66 | with self.assertRaisesRegex(RuntimeError, "status 17"): |
| 67 | runner.main() |
| 68 | self.assertEqual(scales, [0, 1]) |
| 69 | state = json.loads((root / "run/state.json").read_text()) |
| 70 | self.assertEqual(state["phase"], "restored") |
| 71 | self.assertEqual(state["experiment_phase"], "failed") |
| 72 | self.assertNotIn("disposable-fixture-token", output.getvalue()) |
| 73 | self.assertNotIn("disposable-fixture-token", json.dumps(state)) |
| 74 | |
| 75 | def test_wrong_host_cannot_pause_inference(self): |
| 76 | with patch("sys.argv", ["run.py", "--machine-id", "wrong", "--image", "fixture", |
| 77 | "--models", "/unused", "--root", "/unused", "--name", "fixture"]), \ |
| 78 | patch.object(Path, "read_text", return_value="fixture-machine"), \ |
| 79 | patch.object(runner.subprocess, "run") as command, patch.dict(os.environ): |
| 80 | with self.assertRaisesRegex(ValueError, "Wrong host"): |
| 81 | runner.main() |
| 82 | command.assert_not_called() |
| 83 | |
| 84 | |
| 85 | if __name__ == "__main__": |
| 86 | unittest.main() |