1import copy
2import collections
3import json
4from pathlib import Path
5import runpy
6import shutil
7import subprocess
8from tempfile import TemporaryDirectory
9import unittest
10import xml.etree.ElementTree as ET
11
12from document_model import EXPORTER, ordered_pages, walk
13from native_format import native_characters
14from native_xml import ns, Text
15
16ROOT = Path(__file__).resolve().parent.parent
17FIXTURE = ROOT / 'corpus/outline-edit'
18compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare']
19
20
21class OutlineEditTest(unittest.TestCase):
22 def test_automatic_titles_follow_native_and_rust_outline_movement(self):
23 fixture = FIXTURE / 'automatic-title'
24 models = {}
25 with TemporaryDirectory() as temporary:
26 for phase, notebook, capture in [
27 ('before', fixture / 'before/notebook', fixture / 'before/read'),
28 ('native', fixture / 'native/notebook', fixture / 'native/read'),
29 ('native-cold', fixture / 'native/cold/notebook', fixture / 'native/cold/read'),
30 ('rust', fixture / 'candidate', fixture / 'cold/read'),
31 ('rust-cold', fixture / 'cold/notebook', fixture / 'cold/read'),
32 ]:
33 folder = Path(temporary) / phase
34 shutil.copytree(capture, folder / 'read')
35 compare(notebook, folder / 'read')
36 subprocess.run([EXPORTER, notebook / 'TitleControl.one', folder / 'model'], check=True)
37 document = json.loads((folder / 'model/document.json').read_text())
38 models[phase] = list(ordered_pages(document))
39 self.assertEqual(len(models[phase]), 3)
40 for phase, pages in models.items():
41 expected = ['Vertical first', 'Horizontal first', 'Explicit title'] if phase == 'before' else ['Second 🦋 é', 'Second 🦋 é', 'Explicit title']
42 self.assertEqual([r['nodes'][r['roots']['2']]['kind']['title'] for _, _, r, _ in pages], expected)
43 for before, after in [('native', 'native-cold'), ('rust', 'rust-cold')]:
44 for (sid, _, a, page), (saved_sid, _, b, saved_page) in zip(models[before], models[after], strict=True):
45 self.assertEqual((sid, page), (saved_sid, saved_page))
46 self.assertEqual(dict(walk(a, page)), dict(walk(b, page)))
47 for (sid, _, a, page), (saved_sid, _, b, saved_page) in zip(models['before'], models['rust'], strict=True):
48 self.assertEqual((sid, page), (saved_sid, saved_page))
49 for oid, node in walk(a, page):
50 self.assertEqual(node['children'], b['nodes'][oid]['children'])
51 self.assertEqual(node['content'], b['nodes'][oid]['content'])
52 if node['kind']['type'] == 'RichText':
53 self.assertEqual(node, b['nodes'][oid])
54
55 def test_rust_subtree_moves_and_deletions_survive_cold_native_reopen(self):
56 for name, pages in [
57 ('rust-tree/ordinary', 15), ('rust-tree/groups-cells', 12),
58 ('rust-tree/cross-container', 12), ('rust-tree/unequal-groups', 1),
59 ('offline-tree', 15), ('offline-tree/cell-delete', 12), ('offline-tree/cell-move', 12),
60 ]:
61 fixture = FIXTURE / name
62 with self.subTest(case=name), TemporaryDirectory() as temporary:
63 models = {}
64 for phase, notebook in [('candidate', fixture / 'candidate'), ('cold', fixture / 'cold/notebook')]:
65 folder = Path(temporary) / phase
66 shutil.copytree(fixture / 'cold/read', folder / 'read')
67 compare(notebook, folder / 'read')
68 section, = notebook.glob('*.one')
69 subprocess.run([EXPORTER, section, folder / 'model'], check=True)
70 model = json.loads((folder / 'model/document.json').read_text())
71 models[phase] = {sid: (r, page) for sid, _, r, page in ordered_pages(model)}
72 self.assertEqual(len(models[phase]), pages)
73 self.assertEqual(models['candidate'].keys(), models['cold'].keys())
74 for sid, (old, page) in models['candidate'].items():
75 saved, saved_page = models['cold'][sid]
76 self.assertEqual(page, saved_page)
77 self.assertEqual(dict(walk(old, page)), dict(walk(saved, page)))
78 self.assertEqual(old['nodes'][old['roots']['2']]['kind'], saved['nodes'][saved['roots']['2']]['kind'])
79 for _, node in walk(old, page):
80 for oid in node['kind'].get('lists', []):
81 self.assertEqual(old['nodes'][oid], saved['nodes'][oid])
82 if name == 'rust-tree/unequal-groups':
83 capture, = (fixture / 'cold/read').glob('page-*.xml')
84 root = ET.parse(capture).getroot()
85 self.assertEqual([(group.get('indent'), ''.join(Text(group.find('one:OE/one:T', ns).text or '').parts))
86 for group in root.findall('one:Outline/one:OEChildren', ns)],
87 [('3', 'First'), ('2', 'Second')])
88
89 def test_native_group_list_and_cell_tree_controls(self):
90 fixture = FIXTURE / 'tree'
91 models = {}
92 with TemporaryDirectory() as temporary:
93 for phase in ('before', 'after', 'cold'):
94 folder = Path(temporary) / phase
95 shutil.copytree(fixture / phase / 'read', folder / 'read')
96 compare(fixture / phase / 'notebook', folder / 'read')
97 subprocess.run([EXPORTER, fixture / phase / 'notebook/synthetic.one', folder / 'model'], check=True)
98 model = json.loads((folder / 'model/document.json').read_text())
99 models[phase] = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
100 for _, _, r, page in ordered_pages(model)}
101 self.assertEqual(len(models[phase]), 12)
102 cases = json.loads((fixture / 'cases.json').read_text(encoding='utf-8-sig'))
103 self.assertEqual(len(cases), 11)
104 for name, (saved, page) in models['after'].items():
105 cold, cold_page = models['cold'][name]
106 self.assertEqual(page, cold_page)
107 self.assertEqual(dict(walk(saved, page)), dict(walk(cold, cold_page)))
108 for _, node in walk(saved, page):
109 for oid in node['kind'].get('lists', []):
110 self.assertEqual(saved['nodes'][oid], cold['nodes'][oid])
111 for case in cases:
112 name = case['name']
113 with self.subTest(case=name):
114 old, page = models['before'][name]
115 new, new_page = models['after'][name]
116 self.assertEqual(page, new_page)
117 self.assertEqual(old['nodes'][page]['children'], new['nodes'][page]['children'])
118 outline, other = [oid for oid in old['nodes'][page]['children']
119 if old['nodes'][oid]['kind']['type'] == 'Outline']
120 original = dict(walk(old, outline))
121 expected = copy.deepcopy(original)
122 actual = dict(walk(new, outline))
123 paragraphs = {old['nodes'][n['content'][0]]['kind']['text']: oid
124 for oid, n in original.items() if n['kind']['type'] == 'Paragraph'
125 and n['content'] and old['nodes'][n['content'][0]]['kind']['type'] == 'RichText'}
126 target, = [oid for text, oid in paragraphs.items() if text.startswith('Target ')]
127 parents = {child: oid for oid, node in original.items() for child in node['children']}
128 parent = parents[target]
129 if name.startswith('Delete'):
130 for oid, _ in walk(old, target):
131 del expected[oid]
132 expected[parent]['children'].remove(target)
133 if name == 'Delete only grouped subtree':
134 del expected[parent]
135 expected[outline]['children'].remove(parent)
136 elif name == 'Delete unindented sibling after group':
137 group, = expected[outline]['children']
138 expected[outline]['children'] = expected[group]['children']
139 expected[outline]['child_level'] = 2
140 del expected[group]
141 elif name == 'Delete sole cell paragraph':
142 replacement, = actual[parent]['children']
143 self.assertNotIn(replacement, original)
144 blank = actual[replacement]
145 self.assertEqual(blank['kind'], {'type': 'Paragraph', 'lists': [],
146 'paragraph_style': None, 'collapse_state': None})
147 self.assertEqual((blank['children'], blank['tags'], blank['child_level']), ([], [], 1))
148 text, = blank['content']
149 self.assertNotIn(text, original)
150 self.assertEqual(actual[text]['kind'], {'type': 'RichText', 'text': '',
151 'runs': [{'start': 0, 'end': 0, 'format': None, 'extra_set': None}],
152 'paragraph_style': None, 'boilerplate': False})
153 expected[parent]['children'] = [replacement]
154 expected.update({replacement: blank, text: actual[text]})
155 elif name in ('Move numbered subtree down', 'Move cell subtree down'):
156 children = expected[parent]['children']
157 index = children.index(target)
158 children[index:index + 2] = reversed(children[index:index + 2])
159 if name == 'Move numbered subtree down':
160 expected[paragraphs['Anchor']]['child_level'] = 2
161 elif name == 'Indent first subtree':
162 group, = actual.keys() - original.keys()
163 self.assertEqual(actual[group]['kind'], {'type': 'OutlineGroup'})
164 self.assertEqual(actual[group]['children'], [target])
165 self.assertEqual((actual[group]['child_level'], actual[group]['content'],
166 actual[group]['structure'], actual[group]['tags']), (1, [], [], []))
167 expected[group] = actual[group]
168 expected[outline]['children'][0] = group
169 elif name == 'Outdent first group':
170 del expected[parent]
171 expected[outline]['children'][0] = target
172 elif name == 'Indent bullet subtree':
173 expected[outline]['children'].remove(target)
174 expected[paragraphs['Anchor']]['children'] = [target]
175 before_list, = expected[target]['kind']['lists']
176 after_list, = actual[target]['kind']['lists']
177 self.assertNotEqual(before_list, after_list)
178 self.assertEqual(old['nodes'][before_list]['kind'], {'type': 'List', 'font': 'Courier New',
179 'format': '○', 'restart': None, 'bullet': 4})
180 self.assertEqual(new['nodes'][after_list]['kind'], {'type': 'List', 'font': 'Calibri',
181 'format': '•', 'restart': None, 'bullet': 1})
182 expected[target]['kind']['lists'] = [after_list]
183 elif name == 'Outdent child across indentation gap':
184 self.assertEqual(expected[parent]['child_level'], 2)
185 expected[parent]['child_level'] = 1
186 expected[parent]['children'].remove(paragraphs['Other child'])
187 expected[target]['children'].append(paragraphs['Other child'])
188 else:
189 self.fail(name)
190 if case['shape'] not in ('cell', 'cell-only'):
191 reservation = actual[outline]['layout']['reserved_width']
192 self.assertAlmostEqual(reservation, 423.75, places=3)
193 expected[outline]['layout']['reserved_width'] = reservation
194 self.assertEqual(actual.keys(), expected.keys())
195 for oid, node in expected.items():
196 self.preserved_node(node, actual[oid])
197 for list_id in node['kind'].get('lists', []):
198 if list_id in old['nodes']:
199 self.preserved_node(old['nodes'][list_id], new['nodes'][list_id])
200 for oid, node in walk(old, other):
201 self.preserved_node(node, new['nodes'][oid])
202
203 def cold_layout(self, fixture):
204 models = {}
205 with TemporaryDirectory() as temporary:
206 for phase, notebook in [('candidate', fixture / 'candidate'), ('cold', fixture / 'cold/notebook')]:
207 folder = Path(temporary) / phase
208 shutil.copytree(fixture / 'cold/read', folder / 'read')
209 compare(notebook, folder / 'read')
210 subprocess.run([EXPORTER, notebook / 'synthetic.one', folder / 'model'], check=True)
211 model = json.loads((folder / 'model/document.json').read_text())
212 models[phase] = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
213 for _, _, r, page in ordered_pages(model)}
214 captures = {page.get('name'): page for path in (fixture / 'cold/read').glob('page-*.xml')
215 for page in [ET.parse(path).getroot()]}
216 self.assertEqual(len(models['candidate']), 15)
217 self.assertEqual(models['candidate'].keys(), models['cold'].keys())
218 for name, (old, page) in models['candidate'].items():
219 new, saved_page = models['cold'][name]
220 self.assertEqual(page, saved_page)
221 for oid, node in walk(old, page):
222 actual = new['nodes'][oid]
223 for key in ('children', 'content', 'structure', 'child_level', 'layout'):
224 self.assertEqual(actual[key], node[key], (name, oid, key))
225 self.assertEqual(actual['kind'].get('collapse_state'), node['kind'].get('collapse_state'))
226 if node['kind']['type'] == 'RichText':
227 self.assertEqual(actual['kind'], node['kind'])
228 return models, captures
229
230 def test_rust_layout_and_saved_expansion_survive_cold_native_reopen(self):
231 models, captures = self.cold_layout(FIXTURE / 'layout')
232 cases = json.loads((FIXTURE / 'cases.json').read_text(encoding='utf-8-sig'))
233 selected = [case for case in cases if any(key in case for key in ('position', 'size', 'collapse'))]
234 self.assertEqual(len(selected), 5)
235 for case in selected:
236 name = case['name']
237 old, page = models['candidate'][name]
238 saved, _ = models['cold'][name]
239 oid = next(oid for oid in old['nodes'][page]['children'] if old['nodes'][oid]['kind']['type'] == 'Outline')
240 node = saved['nodes'][oid]
241 z = old['nodes'][page]['children'].index(oid)
242 native = next(outline for outline in captures[name].findall('one:Outline', ns)
243 if int(outline.find('one:Position', ns).get('z')) == z)
244 for key, value in case.get('position', {}).items():
245 self.assertEqual(node['layout'][key], value)
246 if 'size' in case:
247 size = native.find('one:Size', ns)
248 self.assertEqual(node['layout']['max_width'], case['size']['width'])
249 self.assertEqual(size.get('isSetByUser') in ('true', '1'), case['size']['isSetByUser'])
250 if case['size']['isSetByUser']:
251 self.assertAlmostEqual(float(size.get('width')), case['size']['width'], places=3)
252 if 'collapse' in case:
253 target, = [p for p in native.findall('.//one:OE', ns) if p.find('one:T', ns) is not None
254 and ''.join(Text(p.find('one:T', ns).text or '').parts).startswith('Target ')]
255 self.assertEqual(target.get('collapsed') in ('true', '1'), case['collapse'])
256
257 def test_rust_width_replaces_native_reserved_wrap_width(self):
258 models, captures = self.cold_layout(FIXTURE / 'reserved-width')
259 saved, page = models['cold']['Move outline']
260 oid = next(oid for oid in saved['nodes'][page]['children'] if saved['nodes'][oid]['kind']['type'] == 'Outline')
261 node = saved['nodes'][oid]
262 self.assertEqual(node['layout']['max_width'], 144)
263 self.assertTrue(node['layout']['width_set_by_user'])
264 self.assertFalse(any(field['id'] == 0x14001cdb for field in node['extra'][0]))
265 z = saved['nodes'][page]['children'].index(oid)
266 native = next(outline for outline in captures['Move outline'].findall('one:Outline', ns)
267 if int(outline.find('one:Position', ns).get('z')) == z)
268 size = native.find('one:Size', ns)
269 self.assertEqual(size.get('isSetByUser'), 'true')
270 self.assertAlmostEqual(float(size.get('width')), 144, places=3)
271
272 def test_offline_native_reconciliation_preserves_graph_and_dependent_edits(self):
273 models, _ = self.cold_layout(FIXTURE / 'offline')
274 records = json.loads((FIXTURE / 'offline/cases.json').read_text())
275 self.assertEqual(len(records), 14)
276 self.assertEqual(collections.Counter(row['outcome'] for row in records), {'merged': 7, 'reviewed': 7})
277 records = {row['name']: row for row in records}
278 with TemporaryDirectory() as temporary:
279 folder = Path(temporary)
280 subprocess.run([EXPORTER, FIXTURE / 'after/notebook/synthetic.one', folder / 'model'], check=True)
281 model = json.loads((folder / 'model/document.json').read_text())
282 remote = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
283 for _, _, r, page in ordered_pages(model)}
284 changed = texts = 0
285 for name, (candidate, page) in models['candidate'].items():
286 original, remote_page = remote[name]
287 self.assertEqual(page, remote_page)
288 record = records.get(name)
289 for oid, node in walk(candidate, page):
290 before = original['nodes'][oid]
291 for key in ('children', 'content', 'structure', 'child_level'):
292 self.assertEqual(node[key], before[key], (name, oid, key))
293 layout = dict(before['layout'])
294 if record and oid == record['object']:
295 change = record['change']
296 if 'Position' in change:
297 layout.update(change['Position'])
298 elif 'Width' in change:
299 layout['max_width'] = change['Width']['points']
300 layout['width_set_by_user'] = change['Width']['user_set']
301 else:
302 self.assertEqual(node['kind']['collapse_state'], int(change['Collapsed']))
303 changed += 1
304 else:
305 self.assertEqual(node['kind'].get('collapse_state'), before['kind'].get('collapse_state'))
306 self.assertEqual(node['layout'], layout)
307 if node['kind']['type'] == 'RichText':
308 expected = before['kind']['text']
309 if record and oid == record['dependent_text']:
310 expected = 'Local ' + expected
311 texts += 1
312 self.assertEqual(node['kind']['text'], expected)
313 self.assertEqual((changed, texts), (10, 14))
314
315 def preserved_node(self, old, new):
316 expected = dict(old)
317 if old['modified'] != new['modified']:
318 self.assertGreater(new['modified'], old['modified'])
319 expected['modified'] = new['modified']
320 if old['extra'] != new['extra']:
321 expected['extra'] = [old['extra'][0] + [{'id': 0x880034dd, 'value': 'NoData'}], *old['extra'][1:]]
322 self.assertEqual(new, expected)
323
324 def test_native_outline_controls_and_cold_reopen(self):
325 models, captures = {}, {}
326 with TemporaryDirectory() as temporary:
327 for phase in ('before', 'after', 'cold'):
328 folder = Path(temporary) / phase
329 shutil.copytree(FIXTURE / phase / 'read', folder / 'read')
330 compare(FIXTURE / phase / 'notebook', folder / 'read')
331 subprocess.run([EXPORTER, FIXTURE / phase / 'notebook/synthetic.one', folder / 'model'], check=True)
332 model = json.loads((folder / 'model/document.json').read_text())
333 models[phase] = {r['nodes'][r['roots']['2']]['kind']['title']: (r, page)
334 for _, _, r, page in ordered_pages(model)}
335 captures[phase] = {page.get('name'): page for path in (folder / 'read').glob('page-*.xml')
336 for page in [ET.parse(path).getroot()]}
337 self.assertEqual(len(models[phase]), 15)
338 cases = json.loads((FIXTURE / 'cases.json').read_text(encoding='utf-8-sig'))
339 self.assertEqual(len(cases), 14)
340 for case in cases:
341 name = case['name']
342 old, page = models['before'][name]
343 outline, other = [oid for oid in old['nodes'][page]['children'] if old['nodes'][oid]['kind']['type'] == 'Outline']
344 original = dict(walk(old, outline))
345 target, = [oid for oid, node in original.items() if node['kind']['type'] == 'Paragraph' and node['content']
346 and old['nodes'][node['content'][0]]['kind']['text'].startswith('Target ')]
347 parents = {child: oid for oid, node in original.items() for child in node['children']}
348 for phase in ('after', 'cold'):
349 with self.subTest(case=name, phase=phase):
350 new, current_page = models[phase][name]
351 self.assertEqual(page, current_page)
352 for oid, node in walk(old, other):
353 self.preserved_node(node, new['nodes'][oid])
354 graph = {oid: [list(node['children']), list(node['content'])] for oid, node in original.items()}
355 if name.startswith('Delete'):
356 selected = outline if name in ('Delete only paragraph', 'Delete outline') else target
357 for oid, _ in walk(old, selected):
358 del graph[oid]
359 if selected == outline:
360 self.assertNotIn(outline, new['nodes'][page]['children'])
361 active = {}
362 else:
363 graph[parents[selected]][0].remove(selected)
364 active = {oid: [node['children'], node['content']] for oid, node in walk(new, outline)}
365 self.assertEqual(active, graph)
366 elif 'keys' in case:
367 siblings = graph[outline][0]
368 if name in ('Move leaf down', 'Move subtree down'):
369 siblings[1], siblings[2] = siblings[2], siblings[1]
370 elif name == 'Move subtree up':
371 siblings[0], siblings[1] = siblings[1], siblings[0]
372 elif name == 'Indent subtree':
373 siblings.remove(target)
374 graph[siblings[0]][0] = [target]
375 elif name == 'Outdent subtree':
376 graph[parents[target]][0].remove(target)
377 siblings.insert(1, target)
378 else:
379 self.fail(name)
380 self.assertEqual({oid: [node['children'], node['content']] for oid, node in walk(new, outline)}, graph)
381 self.assertEqual(new['nodes'][target]['children'], old['nodes'][target]['children'])
382 for oid, node in original.items():
383 if node['kind']['type'] == 'RichText':
384 self.preserved_node(node, new['nodes'][oid])
385 else:
386 for oid, node in original.items():
387 if node['kind']['type'] == 'RichText':
388 continue
389 current = new['nodes'][oid]
390 self.assertEqual(current['children'], node['children'])
391 self.assertEqual(current['kind']['type'], node['kind']['type'])
392 for a, b in zip(node['content'], current['content'], strict=True):
393 self.assertEqual(old['nodes'][a]['kind']['text'], new['nodes'][b]['kind']['text'])
394 geometry = new['nodes'][outline]['layout']
395 for key, value in case.get('position', {}).items():
396 self.assertEqual(geometry[key], value)
397 if 'size' in case:
398 self.assertEqual(geometry['max_width'], case['size']['width'])
399 self.assertEqual(geometry['width_set_by_user'], case['size']['isSetByUser'])
400 if 'collapse' in case:
401 self.assertEqual(bool(new['nodes'][target]['kind']['collapse_state']), case['collapse'])
402 node, = [node for node in captures[phase][name].findall('one:Outline//one:OE', ns)
403 if ''.join(Text(node.find('one:T', ns).text or '').parts).startswith('Target ')]
404 self.assertEqual(node.get('collapsed') in ('true', '1'), case['collapse'])
405 before_xml = captures['before'][name]
406 after_xml = captures[phase][name]
407 for left, right in zip(before_xml.findall('one:Outline', ns),
408 sorted(after_xml.findall('one:Outline', ns), key=lambda node: int(node.find('one:Position', ns).get('z'))), strict=True):
409 for a, b in zip(native_characters(before_xml, [left]), native_characters(after_xml, [right]), strict=True):
410 for (x, before), (y, after) in zip(a, b, strict=True):
411 self.assertEqual(x, y)
412 for key in before.keys() | after.keys():
413 default = 'automatic' if key in ('color', 'highlight') else False
414 self.assertEqual(before.get(key, default), after.get(key, default))
415 saved, saved_page = models['cold'][name]
416 current, current_page = models['after'][name]
417 self.assertEqual(current_page, saved_page)
418 self.assertEqual(current['nodes'][current_page]['children'], saved['nodes'][saved_page]['children'])
419 for outline in current['nodes'][current_page]['children']:
420 if current['nodes'][outline]['kind']['type'] != 'Outline':
421 continue
422 for oid, node in walk(current, outline):
423 for key in ('children', 'content', 'child_level'):
424 self.assertEqual(saved['nodes'][oid][key], node[key])
425 self.assertEqual(saved['nodes'][oid]['kind'].get('collapse_state'), node['kind'].get('collapse_state'))