1import copy
2import json
3import unittest
4import uuid
5from unittest.mock import patch
6
7from offline_document_history import document_history, identity, verify_model, verify_native
8
9
10def history(text_edits=False, graph=False, boundaries=False):
11 if boundaries: text_edits, graph = True, True
12 logs = {'w0': [{'event': 'ready', 'document_operations': True, 'document_kinds': ['insert', 'format', 'text'] if text_edits else ['insert', 'format']}]}
13 if boundaries: logs['w0'][0]['document_kinds'] += ['split', 'right_text', 'join']
14 kinds = logs['w0'][0]['document_kinds']
15 events = logs['w0']
16 if graph: events[0]['document_graph'] = True
17 structure = {}
18 observed = {}
19 previous = None
20 for operation in range(2):
21 insertion = {'guid': list(uuid.UUID(int=operation+1).bytes_le), 'text': f'Document w0:{operation} 🦀',
22 'parent': 'page' if operation == 0 else identity(previous, 1), 'author': 'Offline document writer',
23 'placement': {'Outline': {'x': 144, 'y': 144}} if operation == 0 else {'Paragraph': {'before': None}}}
24 previous = insertion
25 target = identity(insertion, 2)
26 end = len(insertion['text'].encode('utf-16-le'))//2
27 split = {'guid': list(uuid.UUID(int=100+operation).bytes_le), 'text': target,
28 'offset': end-2, 'author': insertion['author'], 'created': 1}
29 right, right_paragraph = identity(split, 2), identity(split, 1)
30 for step, kind in enumerate(kinds):
31 timestamp = 10 + operation*max(3, len(kinds))*10 + step*10
32 local_id = operation*max(3, len(kinds)) + step + 2
33 event = {'event': 'local_document_commit', 'id': local_id, 'operation': operation, 'kind': kind,
34 'space': 'space', 'object': target, 'text': insertion['text'], 'insertion': insertion if step == 0 else None,
35 'range': [1, len(insertion['text'].encode('utf-16-le'))//2-2],
36 'attributes': [{'Bold': True}, {'FontSize': 18+operation}, {'Color': [18, 52, 86]}],
37 'started_us': timestamp-2, 'finished_us': timestamp-1}
38 if kind == 'text':
39 end = len(insertion['text'].encode('utf-16-le'))//2
40 event.update(range=[end-3, end], replacement=' e\u0301🐈')
41 if boundaries: event['document'] = target
42 if kind == 'split': event.update(split=split, range=[end-2, end-2])
43 if kind == 'right_text': event.update(object=right, range=[0, 2], replacement='B🦋')
44 if kind == 'join': event['joined'] = [target, right]
45 events.append(event)
46 prior_observed = copy.deepcopy(observed)
47 prior_structure = copy.deepcopy(structure)
48 if graph:
49 events.append({'event': 'read', 'started_us': timestamp-1, 'finished_us': timestamp,
50 'documents': copy.deepcopy(observed), 'document_graph': prior_structure})
51 if step == 0:
52 parent, paragraph = insertion['parent'], identity(insertion, 1)
53 if operation == 0:
54 parent, paragraph = paragraph, identity(insertion, 3)
55 structure[parent] = {'parent': insertion['parent'], 'children': [], 'content': [],
56 'child_level': 1, 'position': {'x': 144, 'y': 144}}
57 structure[parent]['children'].append(paragraph)
58 structure[paragraph] = {'parent': parent, 'children': [], 'content': [target],
59 'child_level': 1, 'position': None}
60 if kind == 'split':
61 boundary = len(insertion['text'])-1
62 old = observed[target]
63 left_runs, right_runs = old['runs'][:boundary], old['runs'][boundary:]
64 observed[target] = {'text': old['text'][:boundary], 'runs': left_runs}
65 observed[right] = {'text': old['text'][boundary:], 'runs': right_runs}
66 structure[parent]['children'].insert(structure[parent]['children'].index(paragraph)+1, right_paragraph)
67 structure[right_paragraph] = {**copy.deepcopy(structure[paragraph]), 'content': [right]}
68 elif kind == 'right_text':
69 runs = [{**observed[right]['runs'][0], 'text': c} for c in 'B🦋'] + observed[right]['runs'][2:]
70 observed[right] = {'text': 'B🦋🐈', 'runs': runs}
71 elif kind == 'join':
72 observed[target]['text'] += observed[right]['text']
73 observed[target]['runs'] += observed[right]['runs']
74 del observed[right]
75 structure[parent]['children'].remove(right_paragraph)
76 del structure[right_paragraph]
77 else:
78 value = insertion['text'].replace(' 🦀', ' e\u0301🐈') if kind == 'text' else insertion['text']
79 runs = []
80 for index, char in enumerate(value):
81 selected = index > 0 and (kind == 'text' or (kind == 'format' and index < len(value)-1))
82 runs.append({'text': char, 'bold': selected, 'size': 18+operation if selected else 11,
83 'color': 0x563412 if selected else 0xff000000})
84 observed[target] = {'text': value, 'runs': runs}
85 revision = f'revision-{local_id}'
86 events.append({'event': 'remote_attempt', 'revision': revision, 'state': 'Committed',
87 'document_changes': {key: copy.deepcopy(observed.get(key)) for key in observed.keys() | prior_observed.keys()
88 if observed.get(key) != prior_observed.get(key)},
89 'documents': copy.deepcopy(observed), 'started_us': timestamp, 'finished_us': timestamp+1})
90 if graph:
91 events[-1].update(document_graph=copy.deepcopy(structure), document_graph_changes={
92 key: copy.deepcopy(structure.get(key)) for key in structure.keys() | prior_structure.keys() if prior_structure.get(key) != structure.get(key)})
93 events.append({'event': 'document_receipt', 'id': local_id, 'revision': revision, 'at_us': timestamp+2})
94 if text_edits:
95 events.append({'event': 'read', 'started_us': timestamp+3, 'finished_us': timestamp+4, 'documents': copy.deepcopy(observed), **({'document_graph': copy.deepcopy(structure)} if graph else {})})
96 events.extend({'event': 'reopened_document_receipt', 'id': row['id'], 'revision': row['revision']}
97 for row in list(events) if row['event'] == 'document_receipt')
98 read = {'event': 'read', 'started_us': max(100, 20*len(kinds)+10), 'finished_us': max(101, 20*len(kinds)+11), 'documents': observed, **({'document_graph': structure} if graph else {})}
99 events.extend([read, {'event': 'done'}])
100 logs['r0'] = [{'event': 'ready', **({'document_graph': True} if graph else {})}, copy.deepcopy(read), {'event': 'done'}]
101 return logs
102
103
104class DocumentHistoryTests(unittest.TestCase):
105 def setUp(self):
106 self.logs = history()
107
108 def test_document_receipts_and_reader_states_match_the_intents(self):
109 documents = document_history(self.logs, 2)
110 self.assertEqual(len(documents), 2)
111 expected = [list(row['states'].values())[-1]['characters'] for row in documents.values()]
112 paragraphs = [[(char, {'bold': bold, 'font_size': size, 'color': 'automatic' if color == 0xff000000 else '#123456'})
113 for char, bold, size, color in row] for row in expected]
114 self.assertEqual(verify_native(paragraphs, expected), sum(len(row)*3 for row in expected))
115 paragraphs[0][1][1]['font_size'] = 19
116 with self.assertRaisesRegex(AssertionError, 'font size'): verify_native(paragraphs, expected)
117
118 def test_structural_snapshots_and_publication_differences_are_complete(self):
119 baseline = history(text_edits=True, graph=True)
120 self.assertEqual(len(document_history(baseline, 2)), 2)
121 for mutation in ('missing-graph', 'missing-client', 'reorder', 'duplicate', 'reparent',
122 'missing-content', 'invented', 'level', 'position', 'delta'):
123 logs = copy.deepcopy(baseline)
124 row = logs['r0'][1]
125 outline, first, second = list(row['document_graph'])
126 graph = row['document_graph']
127 if mutation == 'missing-graph': del row['document_graph']
128 elif mutation == 'missing-client': del logs['r0'][0]['document_graph']
129 elif mutation == 'reorder': graph[outline]['children'].reverse()
130 elif mutation == 'duplicate': graph[outline]['children'].append(first)
131 elif mutation == 'reparent': graph[second]['parent'] = first
132 elif mutation == 'missing-content': graph[first]['content'] = []
133 elif mutation == 'invented': graph['unrecorded'] = copy.deepcopy(graph[first])
134 elif mutation == 'level': graph[first]['child_level'] = 2
135 elif mutation == 'position': graph[outline]['position']['x'] = 145
136 else:
137 next(row for row in logs['w0'] if row['event'] == 'remote_attempt')['document_graph_changes'] = {}
138 with self.subTest(mutation=mutation), self.assertRaises(AssertionError):
139 document_history(logs, 2)
140
141 def test_boundary_histories_retain_every_intermediate_graph_and_retired_identity(self):
142 baseline = history(boundaries=True)
143 documents = document_history(baseline, 2)
144 for document in documents.values():
145 self.assertEqual(list(document['states']), ['insert', 'format', 'text', 'split', 'right_text', 'join'])
146 self.assertEqual(''.join(c for c, *_ in document['states']['join']['characters']),
147 document['insertion']['text'].replace('🦀', 'B🦋🐈'))
148 for mutation in ('missing-suffix', 'half-join', 'resurrected', 'reorder', 'missing-removal',
149 'wrong-split', 'wrong-dependent-target', 'wrong-join', 'stale', 'future', 'duplicate-attempt'):
150 logs = copy.deepcopy(baseline)
151 split = next(row for row in logs['w0'] if row.get('kind') == 'split')
152 right, right_paragraph = identity(split['split'], 2), identity(split['split'], 1)
153 split_read = next(row for row in logs['w0'] if row['event'] == 'read' and row['started_us'] == 43)
154 joined_read = next(row for row in logs['w0'] if row['event'] == 'read' and row['started_us'] == 63)
155 if mutation == 'missing-suffix': del split_read['documents'][right]
156 elif mutation == 'half-join':
157 joined_read['documents'][right] = copy.deepcopy(split_read['documents'][right])
158 elif mutation == 'resurrected':
159 joined_read['document_graph'][right_paragraph] = copy.deepcopy(split_read['document_graph'][right_paragraph])
160 elif mutation == 'reorder':
161 next(node for node in split_read['document_graph'].values() if node['position'])['children'].reverse()
162 elif mutation == 'missing-removal':
163 join_attempt = next(row for row in logs['w0'] if row['event'] == 'remote_attempt' and row['started_us'] == 60)
164 del join_attempt['document_changes'][right]
165 elif mutation == 'wrong-split': split['split']['offset'] -= 1
166 elif mutation == 'wrong-dependent-target':
167 next(row for row in logs['w0'] if row.get('kind') == 'right_text')['object'] = split['object']
168 elif mutation == 'wrong-join':
169 next(row for row in logs['w0'] if row.get('kind') == 'join')['joined'].reverse()
170 elif mutation == 'stale':
171 joined_read.update(documents=copy.deepcopy(split_read['documents']), document_graph=copy.deepcopy(split_read['document_graph']))
172 elif mutation == 'future': split_read.update(started_us=0, finished_us=1)
173 else:
174 attempt = copy.deepcopy(next(row for row in logs['w0'] if row['event'] == 'remote_attempt' and row['started_us'] == 40))
175 attempt.update(state='Unknown', revision='duplicate')
176 logs['w0'].insert(1, attempt)
177 with self.subTest(mutation=mutation), self.assertRaises(AssertionError):
178 document_history(logs, 2)
179
180 def test_no_reader_phase_can_omit_an_acknowledged_text_or_paragraph(self):
181 baseline = history(boundaries=True)
182 for index, row in enumerate(baseline['w0']):
183 if row['event'] != 'read': continue
184 for field in ('documents', 'document_graph'):
185 for target in row[field]:
186 changed = copy.deepcopy(baseline)
187 del changed['w0'][index][field][target]
188 with self.subTest(read=index, field=field, target=target), self.assertRaises(AssertionError):
189 document_history(changed, 2)
190 extra = copy.deepcopy(next(row for row in baseline['w0'] if row['event'] == 'remote_attempt'))
191 extra.update(revision='unreceipted', started_us=1000, finished_us=1001,
192 document_changes={'unrecorded': {'text': 'extra'}})
193 baseline['w0'].insert(-1, extra)
194 with self.assertRaisesRegex(AssertionError, 'lacks its recorded intent and receipt'):
195 document_history(baseline, 2)
196
197 def test_unknown_boundary_receipts_require_the_original_revision(self):
198 for kind in ('split', 'join'):
199 logs = history(boundaries=True)
200 events = logs['w0']
201 intent = next(row for row in events if row.get('kind') == kind)
202 receipt = next(row for row in events if row['event'] == 'document_receipt' and row['id'] == intent['id'])
203 attempt = next(row for row in events if row['event'] == 'remote_attempt' and row['revision'] == receipt['revision'])
204 attempt['state'] = 'Unknown'
205 confirmation = {'event': 'remote_confirm', 'state': 'Committed',
206 'started_us': attempt['finished_us'], 'finished_us': receipt['at_us'],
207 'revisions': {'space': [receipt['revision']]}}
208 events.insert(events.index(receipt), confirmation)
209 document_history(logs, 2)
210 confirmation['revisions']['space'] = ['later-revision']
211 receipt['revision'] = 'later-revision'
212 next(row for row in events if row['event'] == 'reopened_document_receipt' and row['id'] == intent['id'])['revision'] = 'later-revision'
213 with self.subTest(kind=kind), self.assertRaises(AssertionError):
214 document_history(logs, 2)
215
216 def test_cross_run_text_requires_complete_ordered_states_and_exact_receipts(self):
217 logs = history(True)
218 documents = document_history(logs, 2)
219 for document in documents.values():
220 self.assertEqual(list(document['states']), ['insert', 'format', 'text'])
221 final = document['states']['text']['characters']
222 self.assertEqual(''.join(char for char, *_ in final), document['insertion']['text'].replace(' 🦀', ' e\u0301🐈'))
223 for event, field, value in [('local_document_commit', 'replacement', 'wrong'),
224 ('local_document_commit', 'range', [0, 1]),
225 ('document_receipt', 'revision', 'wrong')]:
226 changed = copy.deepcopy(logs)
227 row = next(row for row in changed['w0'] if row['event'] == event and
228 (row.get('kind') == 'text' or row.get('id') == 4))
229 row[field] = value
230 with self.subTest(event=event, field=field), self.assertRaises(AssertionError):
231 document_history(changed, 2)
232 changed = copy.deepcopy(logs)
233 read = changed['r0'][1]
234 target = next(iter(read['documents']))
235 read['documents'][target]['runs'][-1]['bold'] = False
236 with self.assertRaisesRegex(AssertionError, 'partial or invented'):
237 document_history(changed, 2)
238 prior = next(row for row in logs['w0'] if row['event'] == 'read' and row['started_us'] == 23)
239 changed = copy.deepcopy(logs)
240 changed['r0'][1]['documents'][target] = copy.deepcopy(prior['documents'][target])
241 with self.assertRaisesRegex(AssertionError, 'missed acknowledged'):
242 document_history(changed, 2)
243
244 def test_missing_intents_receipts_or_reopen_records_are_rejected(self):
245 for name in ('local_document_commit', 'document_receipt', 'reopened_document_receipt', 'remote_attempt'):
246 logs = copy.deepcopy(self.logs)
247 events = logs['w0']
248 events.remove(next(row for row in events if row['event'] == name))
249 with self.subTest(event=name), self.assertRaises(AssertionError): document_history(logs, 2)
250
251 def test_formatted_insertions_are_atomic_in_every_observed_snapshot(self):
252 for events in self.logs.values():
253 for row in events:
254 insertion = row.get('insertion')
255 if insertion:
256 insertion['formats'] = [{'guid': list(uuid.uuid4().bytes_le),
257 'range': {'start': 0, 'end': len(insertion['text'].encode('utf-16-le')) // 2},
258 'attributes': [{'FontSize': 13.5}, {'Color': [68, 85, 102]}]}]
259 for key in ('documents', 'document_changes'):
260 for document in (row.get(key) or {}).values():
261 for run in document['runs']:
262 if run['size'] == 11:
263 run['size'] = 13.5
264 run['color'] = 0x665544
265 documents = document_history(self.logs, 2)
266 expected = [list(row['states'].values())[-1]['characters'] for row in documents.values()]
267 paragraphs = [[(char, {'bold': bold, 'font_size': size,
268 'color': '#123456' if color == 0x563412 else '#445566'})
269 for char, bold, size, color in row] for row in expected]
270 self.assertEqual(verify_native(paragraphs, expected), sum(len(row)*3 for row in expected))
271 baseline = copy.deepcopy(self.logs)
272 for event, key in [('remote_attempt', 'document_changes'), ('read', 'documents')]:
273 for field, value in [('size', 11), ('color', 0xff000000)]:
274 logs = copy.deepcopy(baseline)
275 row = next(row for row in logs['w0'] if row['event'] == event)
276 next(iter(row[key].values()))['runs'][0][field] = value
277 with self.subTest(event=event, field=field), self.assertRaises(AssertionError):
278 document_history(logs, 2)
279
280 def test_retired_format_receipt_requires_current_revision_and_exact_observed_effect(self):
281 events = self.logs['w0']
282 attempt = next(row for row in events if row['event'] == 'remote_attempt' and row['revision'] == 'revision-3')
283 attempt['state'] = 'Unknown'
284 receipt = next(row for row in events if row['event'] == 'document_receipt' and row['id'] == 3)
285 receipt.update(revision='current-revision', at_us=25)
286 next(row for row in events if row['event'] == 'reopened_document_receipt' and row['id'] == 3)['revision'] = receipt['revision']
287 read = dict(event='read', started_us=21, finished_us=22, text='Concurrent edits:', documents=copy.deepcopy(attempt['documents']))
288 confirmation = dict(event='remote_confirm', started_us=23, finished_us=24, state='Committed', text=read['text'],
289 revisions={'space': ['current-revision']}, current_revisions={'space': 'current-revision'})
290 index = events.index(receipt)
291 events[index:index] = [read, confirmation]
292 result = document_history(self.logs, 2)
293 target, = attempt['document_changes']
294 self.assertEqual(result[target]['states']['format']['attempt']['receipt_revision'], 'current-revision')
295 for field, value in [('current_revisions', {'space': 'unrelated'}),
296 ('revisions', {'space': ['revision-3', 'current-revision']}),
297 ('state', 'NotCommitted')]:
298 original = confirmation[field]
299 confirmation[field] = value
300 with self.subTest(field=field), self.assertRaises(AssertionError): document_history(self.logs, 2)
301 confirmation[field] = original
302 read['documents'][target]['runs'][1]['size'] = 12
303 with self.assertRaisesRegex(AssertionError, 'differs from the uncertain formatting intent'): document_history(self.logs, 2)
304
305 def test_model_rejects_reordered_or_reparented_insertions(self):
306 documents = document_history(self.logs, 2)
307 first, second = [row['insertion'] for row in documents.values()]
308 outline, paragraph, appended = identity(first, 1), identity(first, 3), identity(second, 1)
309 nodes = {key: {'structure': [], 'content': [], 'children': [], 'kind': {}, 'layout': {}}
310 for key in ['page', outline, paragraph, appended, *documents]}
311 nodes['page']['children'] = [outline]
312 nodes[outline].update(children=[paragraph, appended], layout={'x': 144, 'y': 144})
313 for parent, (target, document) in zip([paragraph, appended], documents.items(), strict=True):
314 nodes[parent]['content'] = [target]
315 nodes[target]['kind'] = {'text': ''.join(char for char, *_ in list(document['states'].values())[-1]['characters'])}
316 revision = {'nodes': nodes}
317 with patch('offline_document_history.ordered_pages', return_value=[('space', 'revision', revision, 'page')]):
318 verify_model({}, documents)
319 nodes[outline]['children'].reverse()
320 with self.assertRaisesRegex(AssertionError, 'order'): verify_model({}, documents)
321 nodes[outline]['children'] = [paragraph]
322 nodes['page']['children'].append(appended)
323 with self.assertRaises(AssertionError): verify_model({}, documents)
324 nodes[outline]['children'] = [paragraph, appended]
325 nodes['page']['children'] = [outline]
326 nodes[paragraph]['content'].append(identity(second, 2))
327 with self.assertRaisesRegex(AssertionError, 'content'): verify_model({}, documents)
328
329 def test_wrong_attributes_targets_and_unconfirmed_receipts_are_rejected(self):
330 for event, field, value in [('local_document_commit', 'object', 'wrong'),
331 ('local_document_commit', 'text', 'changed'),
332 ('document_receipt', 'revision', 'wrong'),
333 ('remote_attempt', 'state', 'Unknown'),
334 ('remote_attempt', 'state', 'NotCommitted')]:
335 logs = copy.deepcopy(self.logs)
336 next(row for row in logs['w0'] if row['event'] == event)[field] = value
337 with self.subTest(event=event, field=field), self.assertRaises(AssertionError): document_history(logs, 2)
338 for field, value in [('attributes', [{'Bold': False}]), ('range', [0, 1])]:
339 logs = copy.deepcopy(self.logs)
340 next(row for row in logs['w0'] if row.get('kind') == 'format')[field] = value
341 with self.subTest(field=field), self.assertRaises(AssertionError): document_history(logs, 2)
342
343 def test_readers_cannot_lose_revert_or_invent_document_content(self):
344 for mutation in ('missing', 'partial', 'future', 'reverted'):
345 logs = copy.deepcopy(self.logs)
346 read = logs['r0'][1]
347 target = next(iter(read['documents']))
348 if mutation == 'missing': del read['documents'][target]
349 elif mutation == 'partial': read['documents'][target]['runs'][1]['bold'] = False
350 elif mutation == 'future': read.update(started_us=0, finished_us=1)
351 else:
352 old = copy.deepcopy(read)
353 old.update(started_us=102, finished_us=103)
354 for run in old['documents'][target]['runs']:
355 run.update(bold=False, size=11, color=0xff000000)
356 logs['r0'].insert(2, old)
357 with self.subTest(mutation=mutation), self.assertRaises(AssertionError): document_history(logs, 2)
358
359 def test_an_uncertain_document_attempt_cannot_be_replayed_under_another_revision(self):
360 logs = copy.deepcopy(self.logs)
361 first = copy.deepcopy(next(row for row in logs['w0'] if row['event'] == 'remote_attempt'))
362 first.update(state='Unknown', revision='earlier-uncertain', started_us=8, finished_us=9)
363 logs['w0'].insert(2, first)
364 with self.assertRaisesRegex(AssertionError, 'more than once'): document_history(logs, 2)
365
366
367if __name__ == '__main__':
368 unittest.main()