| 1 | #!/usr/bin/env python3 |
| 2 | import importlib.util |
| 3 | import io |
| 4 | import json |
| 5 | from pathlib import Path |
| 6 | import tempfile |
| 7 | import unittest |
| 8 | from unittest.mock import patch |
| 9 | |
| 10 | |
| 11 | spec = importlib.util.spec_from_file_location("shipper", Path(__file__).with_name("log-shipper.py")) |
| 12 | shipper = importlib.util.module_from_spec(spec) |
| 13 | spec.loader.exec_module(shipper) |
| 14 | |
| 15 | |
| 16 | class Journal: |
| 17 | def __init__(self, *args, **kwargs): |
| 18 | self.stdout = io.StringIO("") |
| 19 | self.returncode = 0 |
| 20 | |
| 21 | def wait(self): |
| 22 | return self.returncode |
| 23 | |
| 24 | |
| 25 | class Shipping(unittest.TestCase): |
| 26 | def setUp(self): |
| 27 | self.temporary = tempfile.TemporaryDirectory() |
| 28 | self.addCleanup(self.temporary.cleanup) |
| 29 | root = Path(self.temporary.name) |
| 30 | self.log = root / "alloc" / "allocation" / "alloc/logs/app.stdout.0" |
| 31 | self.log.parent.mkdir(parents=True) |
| 32 | self.log.write_bytes(b"2026-10-14T02:50:00Z stderr F before crash\n") |
| 33 | self.allocations = [{"ID": "allocation", "JobID": "shale", "TaskStates": {"app": {"Events": [ |
| 34 | {"Time": 1791946241000000000, "Type": "Terminated", "ExitCode": 137, |
| 35 | "DisplayMessage": "Exit Code: 137", "Details": {"oom_killed": "true"}}, |
| 36 | {"Time": 1791946242000000000, "Type": "Restarting", "DisplayMessage": "Task restarting in 17s"}, |
| 37 | {"Time": 1791946265000000000, "Type": "Started", "DisplayMessage": "Task started by client"}, |
| 38 | ]}}}] |
| 39 | self.batches = [] |
| 40 | self.checkpoints = [] |
| 41 | self.visible = None |
| 42 | replacement = patch.object(shipper, "retention_bounds", return_value=(0, float("inf"))) |
| 43 | replacement.start() |
| 44 | self.addCleanup(replacement.stop) |
| 45 | replacement = patch.object(shipper, "urlopen", side_effect=lambda *a, **k: io.BytesIO(json.dumps({"rows": self.visible if self.visible is not None else len(self.batches) - (self.checkpoints[-1] if self.checkpoints else 0)}).encode())) |
| 46 | replacement.start() |
| 47 | self.addCleanup(replacement.stop) |
| 48 | for target, value in [("LOGS", root / "alloc"), ("STATE", root / "state.json")]: |
| 49 | replacement = patch.object(shipper, target, value) |
| 50 | replacement.start() |
| 51 | self.addCleanup(replacement.stop) |
| 52 | replacement = patch.object(shipper, "nomad", side_effect=lambda path: self.allocations if path == "/v1/allocations" else [{"Address": "127.0.0.1", "Port": 1}]) |
| 53 | replacement.start() |
| 54 | self.addCleanup(replacement.stop) |
| 55 | replacement = patch.object(shipper.subprocess, "Popen", Journal) |
| 56 | replacement.start() |
| 57 | self.addCleanup(replacement.stop) |
| 58 | |
| 59 | def once(self): |
| 60 | state = json.loads(shipper.STATE.read_text()) if shipper.STATE.exists() else {} |
| 61 | def deliver(base, rows, fields): |
| 62 | if rows and rows[0]["source"] == "log-shipper": |
| 63 | self.checkpoints.append(len(self.batches)) |
| 64 | else: |
| 65 | self.batches.extend(rows) |
| 66 | with patch.object(shipper, "send", side_effect=deliver): |
| 67 | shipper.once(state) |
| 68 | |
| 69 | def test_oom_and_restarts_are_retained_once_across_process_restarts_and_allocation_gc(self): |
| 70 | self.once() |
| 71 | self.assertEqual([row.get("event") for row in self.batches], [None, "OOM", "Restarting", "Started"]) |
| 72 | self.assertEqual(self.batches[1]["level"], "error") |
| 73 | self.assertIn("Out of memory (OOM)", self.batches[1]["_msg"]) |
| 74 | self.assertEqual(self.batches[2]["level"], "warn") |
| 75 | self.assertEqual(len({row["id"] for row in self.batches}), 4) |
| 76 | self.assertEqual(len({row["sequence"] for row in self.batches}), 4) |
| 77 | self.once() |
| 78 | self.assertEqual(len(self.batches), 4) |
| 79 | self.allocations.clear() |
| 80 | self.once() |
| 81 | self.assertEqual(json.loads(shipper.STATE.read_text())["events"], {}) |
| 82 | self.assertEqual(len(self.batches), 4) |
| 83 | |
| 84 | def test_same_timestamp_lines_and_split_writes_keep_distinct_ids(self): |
| 85 | self.once() |
| 86 | line = b"2026-10-14T02:50:01Z stdout F same message" |
| 87 | with self.log.open("ab") as file: |
| 88 | file.write(line) |
| 89 | self.once() |
| 90 | self.assertEqual(len(self.batches), 4) |
| 91 | with self.log.open("ab") as file: |
| 92 | file.write(b"\n" + line + b"\n") |
| 93 | self.once() |
| 94 | self.assertEqual(len(self.batches), 6) |
| 95 | self.assertNotEqual(self.batches[-1]["id"], self.batches[-2]["id"]) |
| 96 | self.assertGreater(self.batches[-2]["sequence"], self.batches[-3]["sequence"]) |
| 97 | self.assertGreater(self.batches[-1]["sequence"], self.batches[-2]["sequence"]) |
| 98 | |
| 99 | def test_failed_delivery_does_not_advance_file_or_event_cursors(self): |
| 100 | with patch.object(shipper, "send", side_effect=OSError("store unavailable")): |
| 101 | with self.assertRaises(OSError): |
| 102 | shipper.once({}) |
| 103 | self.assertNotIn("events", json.loads(shipper.STATE.read_text())) |
| 104 | self.assertFalse(json.loads(shipper.STATE.read_text()).get("files")) |
| 105 | self.once() |
| 106 | self.assertEqual(len(self.batches), 4) |
| 107 | |
| 108 | def test_expired_rows_advance_source_cursors_without_blocking_checkpoint(self): |
| 109 | with patch.object(shipper, "retention_bounds", return_value=(2_000_000_000, 3_000_000_000)): |
| 110 | self.once() |
| 111 | self.assertEqual(self.batches, []) |
| 112 | self.assertEqual(self.checkpoints, []) |
| 113 | self.assertTrue(json.loads(shipper.STATE.read_text())["events"]) |
| 114 | self.once() |
| 115 | self.assertEqual(self.batches, []) |
| 116 | |
| 117 | def test_invisible_rows_do_not_commit_checkpoint_or_offsets(self): |
| 118 | self.visible = 3 |
| 119 | with patch.object(shipper.time, "monotonic", side_effect=[0, 31]): |
| 120 | with self.assertRaises(TimeoutError): |
| 121 | self.once() |
| 122 | self.assertEqual(self.checkpoints, []) |
| 123 | reserved = json.loads(shipper.STATE.read_text()) |
| 124 | self.assertFalse(reserved.get("files")) |
| 125 | self.assertNotIn("events", reserved) |
| 126 | self.visible = 4 |
| 127 | self.log.unlink() |
| 128 | self.allocations.clear() |
| 129 | with patch.object(shipper.time, "time_ns", return_value=0): |
| 130 | self.once() |
| 131 | self.assertEqual(self.batches[-1]["sequence"], reserved["sequence"]) |
| 132 | self.assertEqual(self.batches[:4], self.batches[-4:]) |
| 133 | self.assertEqual(len(self.checkpoints), 1) |
| 134 | |
| 135 | |
| 136 | if __name__ == "__main__": |
| 137 | unittest.main() |