1import collections
2import json
3from pathlib import Path
4import runpy
5import shutil
6import subprocess
7from tempfile import TemporaryDirectory
8import unittest
9import xml.etree.ElementTree as ET
10
11from document_model import EXPORTER, ordered_pages, walk
12from native_format import compare_formats, native_characters
13from native_xml import ns
14from offline_document_history import identity
15
16ROOT = Path(__file__).resolve().parent.parent
17FIXTURE = ROOT / 'corpus/paragraph-edit'
18compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare']
19
20
21class ParagraphEditTest(unittest.TestCase):
22 def test_reconciled_native_edits_keep_remote_content_and_exact_graphs(self):
23 fixture = FIXTURE / 'reconciliation'
24 result = fixture / 'reconciled'
25 self.assert_cold_graph(result)
26 manifest = json.loads((result / 'manifest.json').read_text())
27 self.assertEqual(len(manifest), 16)
28 self.assertEqual(collections.Counter(case['outcome'] for case in manifest),
29 {'automatic': 6, 'reviewed': 10})
30 receipts = [case[key] for case in manifest for key in ('receipt', 'dependent_receipt') if key in case]
31 self.assertEqual(len(set(receipts)), 32)
32 models, captures = {}, {}
33 with TemporaryDirectory() as temporary:
34 for phase, notebook, read in [('remote', fixture / 'keyboard/notebook', fixture / 'keyboard/read'),
35 ('reconciled', result / 'candidate', result / 'native/read')]:
36 folder = Path(temporary) / phase
37 subprocess.run([EXPORTER, notebook / 'synthetic.one', folder], check=True)
38 model = json.loads((folder / 'document.json').read_text())
39 models[phase] = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
40 for _, _, r, page in ordered_pages(model)}
41 captures[phase] = {page.get('name'): native_characters(page, page.findall('one:Outline', ns))
42 for path in read.glob('page-*.xml') for page in [ET.parse(path).getroot()]}
43 for case in manifest:
44 name, intent = case['case'], case['intent']
45 old, page = models['remote'][name]
46 new, new_page = models['reconciled'][name]
47 self.assertEqual(new_page, page)
48 outline, = [oid for oid in old['nodes'][page]['children'] if old['nodes'][oid]['kind']['type'] == 'Outline']
49 graph = {oid: [list(node['children']), list(node['content']), node['child_level']]
50 for oid, node in walk(old, outline)}
51 characters = [list(paragraph) for paragraph in captures['remote'][name]]
52 if case['outcome'] != 'retained_conflict':
53 if name.startswith('Split '):
54 text = intent['text']
55 paragraph, = [oid for oid, (_, content, _) in graph.items() if content == [text]]
56 parent, = [oid for oid, (children, _, _) in graph.items() if paragraph in children]
57 offset = 3 if name.endswith(('prefix', 'boundary')) else 2
58 characters[:1] = [characters[0][:offset], characters[0][offset:]]
59 characters[1] = [('C' if char == 'c' else char, style) for char, style in characters[1]]
60 right, right_text = identity(intent, 1), identity(intent, 2)
61 self.assertNotIn(right, graph)
62 self.assertNotIn(right_text, graph)
63 graph[parent][0].insert(graph[parent][0].index(paragraph) + 1, right)
64 graph[right] = [graph[paragraph][0], [right_text], graph[paragraph][2]]
65 graph[right_text] = [[], [], None]
66 graph[paragraph][0] = []
67 self.assertEqual(new['nodes'][right_text]['tags'], [])
68 else:
69 left, right = intent['left'], intent['right']
70 a, = [oid for oid, (_, content, _) in graph.items() if content == [left]]
71 b, = [oid for oid, (_, content, _) in graph.items() if content == [right]]
72 parent, = [oid for oid, (children, _, _) in graph.items() if b in children]
73 graph[parent][0].remove(b)
74 graph[a][0] += graph[b][0]
75 del graph[b], graph[right]
76 characters[:2] = [characters[0] + characters[1]]
77 if any(char == 'c' for char, _ in characters[0]):
78 characters[0] = [('C' if char == 'c' else char, style) for char, style in characters[0]]
79 else:
80 characters[0][1] = ('I', characters[0][1][1])
81 self.assertEqual(new['nodes'][left]['tags'], old['nodes'][left]['tags'])
82 with self.subTest(case=name):
83 self.assertEqual({oid: [node['children'], node['content'], node['child_level']]
84 for oid, node in walk(new, outline)}, graph)
85 for expected, actual in zip(characters, captures['reconciled'][name], strict=True):
86 for (a, before), (b, after) in zip(expected, actual, strict=True):
87 self.assertEqual(a, b)
88 for key in before.keys() | after.keys():
89 default = 'automatic' if key in ('color', 'highlight') else False
90 self.assertEqual(before.get(key, default), after.get(key, default))
91
92 def test_native_reconciliation_controls_preserve_the_intended_edit_scope(self):
93 fixture = FIXTURE / 'reconciliation'
94 cases = json.loads((fixture / 'cases.json').read_text(encoding='utf-8-sig'))
95 self.assertEqual(len(cases), 16)
96 models, captures = {}, {}
97 with TemporaryDirectory() as temporary:
98 for phase in ('before', 'remote', 'keyboard'):
99 folder = Path(temporary) / phase
100 shutil.copytree(fixture / phase / 'read', folder / 'read')
101 compare(fixture / phase / 'notebook', folder / 'read')
102 subprocess.run([EXPORTER, fixture / phase / 'notebook/synthetic.one', folder / 'model'], check=True)
103 model = json.loads((folder / 'model/document.json').read_text())
104 models[phase] = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
105 for _, _, r, page in ordered_pages(model)}
106 captures[phase] = {page.get('name'): native_characters(page, page.findall('one:Outline', ns))
107 for path in (folder / 'read').glob('page-*.xml') for page in [ET.parse(path).getroot()]}
108 self.assertEqual(len(models[phase]), 17)
109 for case in cases:
110 name, change = case['name'], case['change']
111 join = case['operation'] == 'join'
112 old, page = models['before'][name]
113 outline, = [oid for oid in old['nodes'][page]['children'] if old['nodes'][oid]['kind']['type'] == 'Outline']
114 paragraphs = old['nodes'][outline]['children']
115 target = paragraphs[1 if join and change in ('child', 'list', 'tag', 'format', 'right-boundary') else 0]
116 expected = ['' if change == 'adoption' else 'ab🦀cd'] + (['Right'] if join else []) + ['Preserved sibling']
117 self.assertEqual([old['nodes'][old['nodes'][oid]['content'][0]]['kind']['text'] for oid in paragraphs], expected)
118 if change == 'prefix':
119 expected[0] = 'X' + expected[0]
120 if join: expected[1] += 'Y'
121 elif change == 'boundary':
122 expected[0] = expected[0] + 'X' if join else 'abX🦀cd'
123 elif change == 'right-boundary': expected[1] = 'XRight'
124 elif change == 'sibling': expected[-1] = 'Native sibling'
125 elif change == 'adoption': expected[0] = 'X'
126 for phase in ('remote', 'keyboard'):
127 with self.subTest(case=name, phase=phase):
128 current, current_page = models[phase][name]
129 if change == 'format':
130 for char, style in captures['before'][name][int(join)]:
131 self.assertTrue(style['bold'])
132 self.assertFalse(style.get('italic', False))
133 for char, style in captures[phase][name][int(join)]:
134 self.assertTrue(style['italic'])
135 self.assertEqual(style.get('bold', False), phase == 'keyboard')
136 self.assertEqual(current_page, page)
137 active = [oid for oid, node in walk(current, outline) if node['kind']['type'] == 'Paragraph']
138 expected_ids = list(paragraphs)
139 if phase == 'keyboard' and change == 'sibling':
140 self.assertNotEqual(active[-1], paragraphs[-1])
141 expected_ids[-1] = active[-1]
142 if change == 'child':
143 child, = current['nodes'][target]['children']
144 expected_ids.insert(expected_ids.index(target) + 1, child)
145 self.assertEqual(active, expected_ids)
146 actual_texts = [current['nodes'][current['nodes'][oid]['content'][0]]['kind']['text'] for oid in active]
147 expected_texts = list(expected)
148 if change == 'child': expected_texts.insert(2 if join else 1, 'Native child')
149 self.assertEqual(actual_texts, expected_texts)
150 top = list(paragraphs)
151 if phase == 'keyboard' and change == 'sibling': top[-1] = active[-1]
152 if phase == 'keyboard' and change == 'list':
153 if join:
154 top.remove(target)
155 self.assertEqual(current['nodes'][paragraphs[0]]['children'], [target])
156 else:
157 group = current['nodes'][outline]['children'][0]
158 self.assertEqual(current['nodes'][group]['kind']['type'], 'OutlineGroup')
159 self.assertEqual(current['nodes'][group]['children'], [target])
160 self.assertEqual(current['nodes'][group]['content'], [])
161 top[0] = group
162 self.assertEqual(current['nodes'][outline]['children'], top)
163 for oid in active:
164 node = current['nodes'][oid]
165 text, = node['content']
166 if oid in paragraphs:
167 if phase == 'keyboard': self.assertEqual(node['content'], old['nodes'][oid]['content'])
168 else: self.assertNotEqual(node['content'], old['nodes'][oid]['content'])
169 child_count = int(oid == target and change == 'child')
170 if phase == 'keyboard' and join and change == 'list' and oid == paragraphs[0]: child_count = 1
171 self.assertEqual(len(node['children']), child_count)
172 self.assertEqual(len(node['kind']['lists']), int(oid == target and change == 'list'))
173 self.assertEqual(len(current['nodes'][text]['tags']), int(oid == target and change == 'tag'))
174
175 def assert_cold_graph(self, fixture):
176 with TemporaryDirectory() as temporary:
177 models = []
178 for source in ('candidate', 'native/notebook'):
179 folder = Path(temporary) / source.replace('/', '-')
180 shutil.copytree(fixture / 'native/read', folder / 'read')
181 compare(fixture / source, folder / 'read')
182 subprocess.run([EXPORTER, fixture / source / 'synthetic.one', folder / 'model'], check=True)
183 model = json.loads((folder / 'model/document.json').read_text())
184 models.append({r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
185 for _, _, r, page in ordered_pages(model)})
186 self.assertEqual(models[0].keys(), models[1].keys())
187 for title, (old, page) in models[0].items():
188 saved, saved_page = models[1][title]
189 self.assertEqual(saved_page, page)
190 for outline in old['nodes'][page]['children']:
191 if old['nodes'][outline]['kind']['type'] != 'Outline': continue
192 for oid, node in walk(old, outline):
193 self.assertEqual(saved['nodes'][oid]['children'], node['children'])
194 self.assertEqual(saved['nodes'][oid]['content'], node['content'])
195
196 def test_offline_boundaries_retain_remote_edits_and_match_native_controls(self):
197 groups = [('splits', 'cold-split/read', 12), ('joins', 'joined/read', 12),
198 ('inheritance', 'join-edges/joined/read', 4), ('tags', 'join-tags/joined/read', 3)]
199 for name, control, count in groups:
200 fixture = FIXTURE / 'offline' / name
201 manifest = json.loads((fixture / 'manifest.json').read_text())
202 self.assertEqual(len(manifest), count)
203 with self.subTest(group=name):
204 self.assert_cold_graph(fixture)
205 captures = []
206 for folder in (FIXTURE / control, fixture / 'native/read'):
207 captures.append({page.get('name'): native_characters(page, page.findall('one:Outline', ns))
208 for path in folder.glob('page-*.xml') for page in [ET.parse(path).getroot()]})
209 for case in manifest:
210 self.assertTrue(case['revision'])
211 original = captures[0][case['case']]
212 current = captures[1][case['case']]
213 markers = 0
214 for old, new in zip(original, current, strict=True):
215 if new and new[0][0] == '☂':
216 markers += 1
217 new = new[1:]
218 for (a, left), (b, right) in zip(old, new, strict=True):
219 self.assertEqual(a, b)
220 for key in left.keys() | right.keys():
221 default = 'automatic' if key in ('color', 'highlight') else False
222 self.assertEqual(left.get(key, default), right.get(key, default))
223 self.assertEqual(markers, int(case['remote_prefix']))
224
225 def test_rust_joins_match_native_controls_and_retain_cold_graph_identities(self):
226 for name, control in [('split', FIXTURE / 'joined/read'),
227 ('inheritance', FIXTURE / 'join-edges/joined/read'),
228 ('tags', FIXTURE / 'join-tags/joined/read')]:
229 fixture = FIXTURE / 'rust-join' / name
230 manifest = json.loads((fixture / 'manifest.json').read_text())
231 with self.subTest(fixture=name):
232 self.assert_cold_graph(fixture)
233 captures = []
234 for folder in (control, fixture / 'native/read'):
235 captures.append({page.get('name'): native_characters(page, page.findall('one:Outline', ns))
236 for path in folder.glob('page-*.xml') for page in [ET.parse(path).getroot()]})
237 self.assertEqual(len(manifest), {'split': 12, 'inheritance': 5, 'tags': 3}[name])
238 for case in manifest:
239 for a, b in zip(captures[0][case['case']], captures[1][case['case']], strict=True):
240 for (x, old), (y, new) in zip(a, b, strict=True):
241 self.assertEqual(x, y)
242 for key in old.keys() | new.keys():
243 default = 'automatic' if key in ('color', 'highlight') else False
244 self.assertEqual(old.get(key, default), new.get(key, default))
245
246 def test_native_joins_preserve_inherited_styles_and_follow_tag_and_child_rules(self):
247 for fixture_name, expected_cases in [('join-edges', 5), ('join-tags', 3)]:
248 fixture = FIXTURE / fixture_name
249 models, captures = {}, {}
250 with TemporaryDirectory() as temporary:
251 for phase in ('before', 'joined'):
252 folder = Path(temporary) / phase
253 shutil.copytree(fixture / phase / 'read', folder / 'read')
254 compare(fixture / phase / 'notebook', folder / 'read')
255 subprocess.run([EXPORTER, fixture / phase / 'notebook/synthetic.one', folder / 'model'], check=True)
256 model = json.loads((folder / 'model/document.json').read_text())
257 models[phase] = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
258 for _, _, r, page in ordered_pages(model)}
259 captures[phase] = {}
260 for path in (folder / 'read').glob('page-*.xml'):
261 page = ET.parse(path).getroot()
262 captures[phase][page.get('name')] = native_characters(page, page.findall('one:Outline', ns))
263 self.assertEqual(len(models[phase]), expected_cases + 1)
264 cases = json.loads((fixture / 'cases.json').read_text(encoding='utf-8-sig'))
265 self.assertEqual(len(cases), expected_cases)
266 for case in cases:
267 name = case['name']
268 with self.subTest(case=name):
269 old, page = models['before'][name]
270 new, new_page = models['joined'][name]
271 self.assertEqual(page, new_page)
272 outline, = [oid for oid in old['nodes'][page]['children'] if old['nodes'][oid]['kind']['type'] == 'Outline']
273 left, right, sibling = old['nodes'][outline]['children']
274 self.assertEqual(new['nodes'][outline]['children'], [left, sibling])
275 target = old['nodes'][left]['children'][-1] if name == 'Join both children' else left
276 a, = old['nodes'][target]['content']
277 b, = old['nodes'][right]['content']
278 empty = old['nodes'][a]['kind']['text'] == ''
279 survivor = b if empty else a
280 self.assertEqual(new['nodes'][target]['content'], [survivor])
281 self.assertEqual(new['nodes'][survivor]['kind']['text'], old['nodes'][a]['kind']['text'] + old['nodes'][b]['kind']['text'])
282 self.assertEqual(new['nodes'][survivor]['tags'], old['nodes'][a]['tags'])
283 children = old['nodes'][left]['children'] + old['nodes'][right]['children']
284 self.assertEqual(new['nodes'][left]['children'], children)
285 self.assertEqual(new['nodes'][sibling], old['nodes'][sibling])
286 if name == 'Join both children':
287 parent_text, = old['nodes'][left]['content']
288 original = old['nodes'][parent_text]
289 current = new['nodes'][parent_text]
290 self.assertGreater(current['modified'], original['modified'])
291 expected = {**original, 'modified': current['modified'],
292 'extra': [original['extra'][0] + [{'id': 0x880034dd, 'value': 'NoData'}]]}
293 self.assertEqual(current, expected)
294 before = [v for paragraph in captures['before'][name] for v in paragraph]
295 after = [v for paragraph in captures['joined'][name] for v in paragraph]
296 for (a, old_style), (b, new_style) in zip(before, after, strict=True):
297 self.assertEqual(a, b)
298 for key in old_style.keys() | new_style.keys():
299 default = 'automatic' if key in ('color', 'highlight') else False
300 self.assertEqual(old_style.get(key, default), new_style.get(key, default))
301
302
303 def test_rust_splits_match_native_controls_and_preserve_empty_typing_styles(self):
304 fixture = FIXTURE / 'rust-split'
305 manifest = json.loads((fixture / 'manifest.json').read_text())
306 with TemporaryDirectory() as temporary:
307 for source, capture in [('candidate', 'native'), ('native/notebook', 'native'),
308 ('typed/notebook', 'typed')]:
309 native = Path(temporary) / source.replace('/', '-') / 'read'
310 shutil.copytree(fixture / capture / 'read', native)
311 compare(fixture / source, native)
312 subprocess.run([EXPORTER, fixture / 'candidate/synthetic.one', Path(temporary) / 'model'], check=True)
313 model = json.loads((Path(temporary) / 'model/document.json').read_text())
314 subprocess.run([EXPORTER, fixture / 'native/notebook/synthetic.one', Path(temporary) / 'saved'], check=True)
315 saved = json.loads((Path(temporary) / 'saved/document.json').read_text())
316 views = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
317 for _, _, r, page in ordered_pages(model)}
318 saved_views = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
319 for _, _, r, page in ordered_pages(saved)}
320 text_nodes = {}
321 for name, (revision, page) in views.items():
322 current, current_page = saved_views[name]
323 self.assertEqual(page, current_page)
324 text_nodes[name] = []
325 for outline in revision['nodes'][page]['children']:
326 if revision['nodes'][outline]['kind']['type'] != 'Outline': continue
327 for oid, node in walk(revision, outline):
328 self.assertEqual(current['nodes'][oid]['children'], node['children'])
329 self.assertEqual(current['nodes'][oid]['content'], node['content'])
330 if node['kind']['type'] == 'RichText': text_nodes[name].append(node)
331 captures = {}
332 for phase, folder in [('native', fixture / 'native/read'), ('typed', fixture / 'typed/read'),
333 ('control', FIXTURE / 'cold-split/read')]:
334 captures[phase] = {}
335 for path in folder.glob('page-*.xml'):
336 page = ET.parse(path).getroot()
337 captures[phase][page.get('name')] = native_characters(page, page.findall('one:Outline', ns))
338 self.assertEqual(len(captures[phase]), 14)
339 cases = [c for c in manifest['cases'] if 'intent' in c]
340 self.assertEqual(len(cases), 12)
341 for case in cases:
342 name = case['case']
343 with self.subTest(case=name):
344 for a, b in zip(captures['native'][name], captures['control'][name], strict=True):
345 for (c, left), (d, right) in zip(a, b, strict=True):
346 self.assertEqual(c, d)
347 for key in left.keys() | right.keys():
348 default = 'automatic' if key in ('color', 'highlight') else False
349 self.assertEqual(left.get(key, default), right.get(key, default))
350 selections = json.loads((fixture / 'typed/ui/selections.json').read_text())
351 self.assertEqual(len(selections), 4)
352 for selection in selections:
353 node = text_nodes[selection['case']][selection['index']]
354 self.assertEqual(node['kind']['text'], '')
355 self.assertEqual(len(node['kind']['runs']), 1)
356 node['kind']['text'] = selection['text']
357 node['kind']['runs'][0]['end'] = len(selection['text'].encode('utf-16-le')) // 2
358 for name, (revision, page) in views.items():
359 _, differences = compare_formats(revision, text_nodes[name], captures['typed'][name])
360 self.assertEqual(differences, [], name)
361
362 def test_native_split_join_graphs_styles_and_identity_boundaries(self):
363 manifest = json.loads((FIXTURE / 'manifest.json').read_text())
364 models, native = {}, {}
365 with TemporaryDirectory() as temporary:
366 for phase in manifest['phases']:
367 folder = Path(temporary) / phase
368 read = folder / 'read'
369 shutil.copytree(FIXTURE / phase / 'read', read)
370 compare(FIXTURE / phase / 'notebook', read)
371 subprocess.run([EXPORTER, FIXTURE / phase / 'notebook/synthetic.one', folder / 'model'], check=True)
372 model = json.loads((folder / 'model/document.json').read_text())
373 models[phase] = {r['nodes'][r['roots']['2']]['kind'].get('title'): (r, page)
374 for _, _, r, page in ordered_pages(model)}
375 self.assertEqual(len(models[phase]), 14)
376 native[phase] = {}
377 for file in read.glob('page-*.xml'):
378 page = ET.parse(file).getroot()
379 native[phase][page.get('name')] = native_characters(page, page.findall('one:Outline', ns))
380 self.assertEqual(len(manifest['cases']), 13)
381 for case in manifest['cases']:
382 name = case['case']
383 with self.subTest(case=name):
384 old, page = models['before'][name]
385 text, paragraph, suffix, suffix_text = (case[key] for key in
386 ('original_text', 'original_paragraph', 'new_paragraph', 'new_text'))
387 parent, = [oid for oid, node in old['nodes'].items() if paragraph in node['children']]
388 position = old['nodes'][parent]['children'].index(paragraph)
389 expected_children = old['nodes'][parent]['children'].copy()
390 expected_children.insert(position + 1, suffix)
391 encoded = old['nodes'][text]['kind']['text'].encode('utf-16-le')
392 at = case['offset_utf16'] * 2
393 for phase in ('split', 'cold-split'):
394 current, _ = models[phase][name]
395 self.assertEqual(current['nodes'][parent]['children'], expected_children)
396 self.assertEqual(current['nodes'][paragraph]['content'], [text])
397 self.assertEqual(current['nodes'][paragraph]['children'], [])
398 self.assertEqual(current['nodes'][suffix]['content'], [suffix_text])
399 self.assertEqual(current['nodes'][suffix]['children'], old['nodes'][paragraph]['children'])
400 self.assertEqual(current['nodes'][text]['kind']['text'], encoded[:at].decode('utf-16-le'))
401 self.assertEqual(current['nodes'][suffix_text]['kind']['text'], encoded[at:].decode('utf-16-le'))
402 self.assertEqual(current['nodes'][text]['tags'], old['nodes'][text]['tags'])
403 self.assertEqual(current['nodes'][suffix_text]['tags'], [])
404 joined, joined_page = models['joined'][name]
405 adopted = name in ('Split start', 'Split empty')
406 self.assertEqual(joined['nodes'][page]['children'], old['nodes'][page]['children'])
407 self.assertEqual(page, joined_page)
408 for outline in old['nodes'][page]['children']:
409 if old['nodes'][outline]['kind']['type'] != 'Outline': continue
410 for oid, node in walk(old, outline):
411 if adopted and oid == text: continue
412 self.assertEqual(joined['nodes'][oid]['children'], node['children'])
413 content = [suffix_text] if adopted and oid == paragraph else node['content']
414 self.assertEqual(joined['nodes'][oid]['content'], content)
415 if node['kind']['type'] == 'RichText':
416 self.assertEqual(joined['nodes'][oid]['kind']['text'], node['kind']['text'])
417 self.assertEqual(joined['nodes'][oid]['tags'], node['tags'])
418 self.assertEqual(joined['nodes'][suffix_text if adopted else text]['kind']['text'], encoded.decode('utf-16-le'))
419 for left, right in zip(native['before'][name], native['joined'][name], strict=True):
420 for (a, old_style), (b, new_style) in zip(left, right, strict=True):
421 self.assertEqual(a, b)
422 for key in old_style.keys() | new_style.keys():
423 default = 'automatic' if key in ('color', 'highlight') else False
424 self.assertEqual(old_style.get(key, default), new_style.get(key, default))
425
426
427if __name__ == '__main__':
428 unittest.main()