1#!/usr/bin/env python3
2import importlib.util
3import io
4import json
5from pathlib import Path
6import tempfile
7import unittest
8from unittest.mock import patch
9
10
11spec = importlib.util.spec_from_file_location("shipper", Path(__file__).with_name("log-shipper.py"))
12shipper = importlib.util.module_from_spec(spec)
13spec.loader.exec_module(shipper)
14
15
16class Journal:
17 def __init__(self, *args, **kwargs):
18 self.stdout = io.StringIO("")
19 self.returncode = 0
20
21 def wait(self):
22 return self.returncode
23
24
25class Shipping(unittest.TestCase):
26 def setUp(self):
27 self.temporary = tempfile.TemporaryDirectory()
28 self.addCleanup(self.temporary.cleanup)
29 root = Path(self.temporary.name)
30 self.log = root / "alloc" / "allocation" / "alloc/logs/app.stdout.0"
31 self.log.parent.mkdir(parents=True)
32 self.log.write_bytes(b"2026-10-14T02:50:00Z stderr F before crash\n")
33 self.allocations = [{"ID": "allocation", "JobID": "shale", "TaskStates": {"app": {"Events": [
34 {"Time": 1791946241000000000, "Type": "Terminated", "ExitCode": 137,
35 "DisplayMessage": "Exit Code: 137", "Details": {"oom_killed": "true"}},
36 {"Time": 1791946242000000000, "Type": "Restarting", "DisplayMessage": "Task restarting in 17s"},
37 {"Time": 1791946265000000000, "Type": "Started", "DisplayMessage": "Task started by client"},
38 ]}}}]
39 self.batches = []
40 self.checkpoints = []
41 self.visible = None
42 replacement = patch.object(shipper, "retention_bounds", return_value=(0, float("inf")))
43 replacement.start()
44 self.addCleanup(replacement.stop)
45 replacement = patch.object(shipper, "urlopen", side_effect=lambda *a, **k: io.BytesIO(json.dumps({"rows": self.visible if self.visible is not None else len(self.batches) - (self.checkpoints[-1] if self.checkpoints else 0)}).encode()))
46 replacement.start()
47 self.addCleanup(replacement.stop)
48 for target, value in [("LOGS", root / "alloc"), ("STATE", root / "state.json")]:
49 replacement = patch.object(shipper, target, value)
50 replacement.start()
51 self.addCleanup(replacement.stop)
52 replacement = patch.object(shipper, "nomad", side_effect=lambda path: self.allocations if path == "/v1/allocations" else [{"Address": "127.0.0.1", "Port": 1}])
53 replacement.start()
54 self.addCleanup(replacement.stop)
55 replacement = patch.object(shipper.subprocess, "Popen", Journal)
56 replacement.start()
57 self.addCleanup(replacement.stop)
58
59 def once(self):
60 state = json.loads(shipper.STATE.read_text()) if shipper.STATE.exists() else {}
61 def deliver(base, rows, fields):
62 if rows and rows[0]["source"] == "log-shipper":
63 self.checkpoints.append(len(self.batches))
64 else:
65 self.batches.extend(rows)
66 with patch.object(shipper, "send", side_effect=deliver):
67 shipper.once(state)
68
69 def test_oom_and_restarts_are_retained_once_across_process_restarts_and_allocation_gc(self):
70 self.once()
71 self.assertEqual([row.get("event") for row in self.batches], [None, "OOM", "Restarting", "Started"])
72 self.assertEqual(self.batches[1]["level"], "error")
73 self.assertIn("Out of memory (OOM)", self.batches[1]["_msg"])
74 self.assertEqual(self.batches[2]["level"], "warn")
75 self.assertEqual(len({row["id"] for row in self.batches}), 4)
76 self.assertEqual(len({row["sequence"] for row in self.batches}), 4)
77 self.once()
78 self.assertEqual(len(self.batches), 4)
79 self.allocations.clear()
80 self.once()
81 self.assertEqual(json.loads(shipper.STATE.read_text())["events"], {})
82 self.assertEqual(len(self.batches), 4)
83
84 def test_same_timestamp_lines_and_split_writes_keep_distinct_ids(self):
85 self.once()
86 line = b"2026-10-14T02:50:01Z stdout F same message"
87 with self.log.open("ab") as file:
88 file.write(line)
89 self.once()
90 self.assertEqual(len(self.batches), 4)
91 with self.log.open("ab") as file:
92 file.write(b"\n" + line + b"\n")
93 self.once()
94 self.assertEqual(len(self.batches), 6)
95 self.assertNotEqual(self.batches[-1]["id"], self.batches[-2]["id"])
96 self.assertGreater(self.batches[-2]["sequence"], self.batches[-3]["sequence"])
97 self.assertGreater(self.batches[-1]["sequence"], self.batches[-2]["sequence"])
98
99 def test_failed_delivery_does_not_advance_file_or_event_cursors(self):
100 with patch.object(shipper, "send", side_effect=OSError("store unavailable")):
101 with self.assertRaises(OSError):
102 shipper.once({})
103 self.assertNotIn("events", json.loads(shipper.STATE.read_text()))
104 self.assertFalse(json.loads(shipper.STATE.read_text()).get("files"))
105 self.once()
106 self.assertEqual(len(self.batches), 4)
107
108 def test_expired_rows_advance_source_cursors_without_blocking_checkpoint(self):
109 with patch.object(shipper, "retention_bounds", return_value=(2_000_000_000, 3_000_000_000)):
110 self.once()
111 self.assertEqual(self.batches, [])
112 self.assertEqual(self.checkpoints, [])
113 self.assertTrue(json.loads(shipper.STATE.read_text())["events"])
114 self.once()
115 self.assertEqual(self.batches, [])
116
117 def test_invisible_rows_do_not_commit_checkpoint_or_offsets(self):
118 self.visible = 3
119 with patch.object(shipper.time, "monotonic", side_effect=[0, 31]):
120 with self.assertRaises(TimeoutError):
121 self.once()
122 self.assertEqual(self.checkpoints, [])
123 reserved = json.loads(shipper.STATE.read_text())
124 self.assertFalse(reserved.get("files"))
125 self.assertNotIn("events", reserved)
126 self.visible = 4
127 self.log.unlink()
128 self.allocations.clear()
129 with patch.object(shipper.time, "time_ns", return_value=0):
130 self.once()
131 self.assertEqual(self.batches[-1]["sequence"], reserved["sequence"])
132 self.assertEqual(self.batches[:4], self.batches[-4:])
133 self.assertEqual(len(self.checkpoints), 1)
134
135
136if __name__ == "__main__":
137 unittest.main()