From 190fb2d10c46ced562a7f0d96f63aa803f4b47d4 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sat, 12 Sep 2026 02:36:25 -0700 Subject: [PATCH] feat: write note tags and tag definitions through the page model Note tags on paragraphs and text objects, and the tag definitions the page model carries, are differences the page writer now lowers. A property-set array writer joins the property patcher: an encoded array of sets is inserted whole with its nested object references appended to the reference stream in traversal order, which is where a rewritten tag array sits. The writer rewrites an element's tag array from the model (definition reference, action type, status, creation, completion, start and due dates and task id), creates definition objects for definitions new to the section under writer identities that squash renames to the model's, and refuses a second tag with the same action type on one element as OneNote does. The model mutation driver toggles tags. corpus/tag-edit is the native gate: a Rust-authored page with two definitions and open, completed, doubly tagged and untagged paragraphs, cold-read by OneNote 2010 with the definitions, completion states and dates intact; tools/test_tag_edit.py checks it without a VM. Assisted-by: claude-fable-5.1 --- corpus/tag-edit/README.md | 19 ++ .../tag-edit/candidate/Open Notebook.onetoc2 | Bin 0 -> 1596 bytes corpus/tag-edit/candidate/tags.one | Bin 0 -> 8352 bytes corpus/tag-edit/cold/commands.jsonl | 3 + corpus/tag-edit/cold/machine.json | 1 + .../cold/notebook/Open Notebook.onetoc2 | Bin 0 -> 3748 bytes corpus/tag-edit/cold/notebook/tags.one | Bin 0 -> 8352 bytes corpus/tag-edit/cold/read/environment.json | 7 + corpus/tag-edit/cold/read/hierarchy.xml | 2 + corpus/tag-edit/cold/read/page-000.xml | 2 + corpus/tag-edit/cold/read/payloads.json | 3 + corpus/tag-edit/cold/run.json | 18 ++ corpus/tag-edit/cold/scripts/cold.ps1 | 27 ++ corpus/tag-edit/cold/scripts/read.ps1 | 142 ++++++++ corpus/tag-edit/cold/source.json | 14 + corpus/tag-edit/cold/teardown.json | 1 + crates/notebook/README.md | 4 +- crates/onestore/README.md | 2 +- crates/onestore/src/page/write.rs | 213 +++++++++++- crates/onestore/src/write.rs | 62 +++- crates/onestore/src/write/tests.rs | 20 +- crates/onestore/tests/page_tags.rs | 305 ++++++++++++++++++ crates/onestore/tests/support/page_edits.rs | 58 +++- tools/test_tag_edit.py | 37 +++ 24 files changed, 911 insertions(+), 29 deletions(-) create mode 100644 corpus/tag-edit/README.md create mode 100644 corpus/tag-edit/candidate/Open Notebook.onetoc2 create mode 100644 corpus/tag-edit/candidate/tags.one create mode 100644 corpus/tag-edit/cold/commands.jsonl create mode 100644 corpus/tag-edit/cold/machine.json create mode 100644 corpus/tag-edit/cold/notebook/Open Notebook.onetoc2 create mode 100644 corpus/tag-edit/cold/notebook/tags.one create mode 100644 corpus/tag-edit/cold/read/environment.json create mode 100644 corpus/tag-edit/cold/read/hierarchy.xml create mode 100644 corpus/tag-edit/cold/read/page-000.xml create mode 100644 corpus/tag-edit/cold/read/payloads.json create mode 100644 corpus/tag-edit/cold/run.json create mode 100644 corpus/tag-edit/cold/scripts/cold.ps1 create mode 100644 corpus/tag-edit/cold/scripts/read.ps1 create mode 100644 corpus/tag-edit/cold/source.json create mode 100644 corpus/tag-edit/cold/teardown.json create mode 100644 crates/onestore/tests/page_tags.rs create mode 100644 tools/test_tag_edit.py diff --git a/corpus/tag-edit/README.md b/corpus/tag-edit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7f32604fb23240e27c6ee877292d57fdaf9a8784 --- /dev/null +++ b/corpus/tag-edit/README.md @@ -0,0 +1,19 @@ +# Rust note tag authoring + +`candidate/` is the page writer's output for +`new_definitions_and_tags_publish_on_a_fresh_page` in +`crates/onestore/tests/page_tags.rs`: on a section created in Rust, two tag +definitions are added ("Rust task", type 0, symbol 3; "Important", type 1, +symbol 13), the first paragraph gains an open task, a second paragraph a +completed task with creation and completion dates, a third paragraph both +tags, and a fourth paragraph none. Tags are property-set arrays on the text +object referencing definition objects (jcid 0x120043) that carry the model's +identities after squash; an element holds one tag per action type, as OneNote +requires. + +`cold/` is a fresh OneNote 2010 read: both `TagDef` entries appear with their +type, symbol and name, the tags read back with their completion state and +dates, and the untagged paragraph has none. `tools/test_tag_edit.py` checks +this without a VM. Regenerate with `ONESTORE_TAG_EXPORT` set to a new absolute +directory while running the test, then cold-open it with +`tools/native_runner.py OUTPUT COLD --expected-pages 1 --collect-notebook`. diff --git a/corpus/tag-edit/candidate/Open Notebook.onetoc2 b/corpus/tag-edit/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..e8389f6e1db021270da0b9edd70425d940dbf64e GIT binary patch literal 1596 zcmZ3e|KIuM`!b(-j|5(YE0x7tw_TX{r-jX7an_GIsZ1YD1~g!Qcb0(k#IK%9FNjZk zdt%)UX`nnZhAICK1RypeOaMy5#LK26vQrhb$j4S^9H0u3w-Q)C%r zcFHh_WCQ7aKw1D89E;s;y^nc_d$)+DP89#AF9*s+LO@)?kjRkEP|Tpmkk63EkO~w5 zZc&<0RUo?%zgj> literal 0 HcmV?d00001 diff --git a/corpus/tag-edit/candidate/tags.one b/corpus/tag-edit/candidate/tags.one new file mode 100644 index 0000000000000000000000000000000000000000..197cc2ccf1965bfa156fa3d4acdb6ec7ca9362f1 GIT binary patch literal 8352 zcmeHLdstM*6`vI(-~$%{#jtBuMN3nkk%FRuMV@9sh?R)7N$`P!qEIx5`o(X>)DVpk zsNvyPkP0+|yZ6qrUT1-rKl(>Ge6yE3 zbI#11*Kf|;_PM{>`10A;V`>imBCF-61+A||K5jY|HQe-qXHwqqJCjB-6$PK$t@X~k zIlHoDcwUF4=Io<1ZreEfT_XJ7k9R0f-mmJmz81Qsecdupeodywv_bO!^XL(!^ykQ3 z*55LN^zFP{$Gi+W=A)&a<`mBti-KFD>!UYW*8bXiAB&4DE}nHxLlQEX z9ay%Wd2IJ%9!GS{!FaCy_@wGpqND2|yz) z%sjoyJ$|>Vd&GgPVd19i=V$OSnWO7h?yt>dxkbvJT4+#YsTa~&j=V2L(2xb?J!5~0Y9n|es-=V|>8brPj>8mxe+~oc;&lUM0oR4;59!BFD_6Wu2J=QQg zNwIsK;+MzcU5ZP@GNR}3m=ii&PNR-&1I>xtcuvoBGpQLe&&&s}E42{z9L(pUz)*;|2zU{SLdJ)N!d zIq2A+p|p1o${|l`-qdu|`1~4YSq#!d=3u$Sh3qaK5lp$o%p<7#VWl}q^TI}$H%re9 z>#4b5@5+8P_($4@H_6C)ji&Y8M>3x~I;O?2Jq@6*ixOwZE!g6I{fP-g_%MU?7(jCm zqxa2#mqHgKY;pl|ds7 zz=^JWu(^%^2!1CrP=j|hjB*J{H(~OyPsV+4GiS)Req}!i);|yC1ciArjl-=mV1~Uu z{l;`;!Ogmkg$Lg)FHH*B1c0(d0X06sE+F8O8aFW?2>3K1vF!3+*A+z7Ul}zZ^3~+G z`IyYn^(*%SpK_EfQrIw&XYJur$(%J!A`HS#F}r9C4(!4wBWz2QNXeqs7n~sAlSp6S zlQW=@=!*Oh;uFVrIhrZIN#g`1q!2YC>;q4yGBpRlEM6%dQcYZ`5f||_tMv^*s zFm4+>>UhB%qII;???Mgd8KRiNO|m%h7l;7wpr3mJb|LdK>cqJOh3K&wy(j(|}VFreP{8ejvv{b!4~Tug{#i(3oPZD7ieXKH!^5Fxuir zfOq(`i~(g4HF?`go^8>W`;ZKHA}3}%m72CA>NgwyF?GKtyBhu2vuCA$`uIH`8cV{y zNX)!kvkm>w>Daz4F~LbEV>f?RJJRn=SOc#wa+SLaav#cDb6=_w#CD~1%-+}p@%NXo z+!mW(@?%EmjB8maoi?lJ980B>Y%R5=%=R?H6fQ6zb8}{4jx5KI8CFxJmPRezk7+C4 z3Nl@54l;oQ#N$^<70G^@G5bPJm!6K@dTM^=;4`0PfwwpIQ^zZ}--`2DD{nNMi#oDw z$9`?oqE*~}u1j__rN84A_Qr|u#JKYnd(e+?P}|#T*TL8^V_EniMk{P~V^`uN-{QPN zB+jEVXgM>`ukE>w>9`$$xFEz)>)#zej3{T}drb@d%M;#sOKk^rykMUI<%>uJB429I zM?!F5Li^qCd@pZFSp4pcg+aHRJKeh3x0DaGAmq`0Xg=UD06~xAWk)QI?eR~%van&o z%ys^^deQ@a0FT3n5bqkc*;MH7wyEKFqb_lOI=SCEmACoNF>iKUe{a?#-z@H*vLVuv zQeBN)?oHg&I#S$=>k*(!Jk@Kc)|S&BqFu%iUX5_SaX5j#3CE9q;M)33=+5I?bYHl= z9H0O^xXAi^!Y!=c*c~o?<@Z87_m{U z=d;t34;0jFjH)xFx*e`u@h11HI&QgN9}c{;*WwfyS#o;EGyboAUW|U)FPn2|baj!> z-@*!xJeI#Vdfj!df0lBo`5`z5{Uw}3P*eSHAM21kvn8ZMb&Uu>92f}w?yupYxr-y2 zRYE-nE9!B?1FUi+?CK+SYu@t@Ua)Imo5u35jv!-$zACEJyl!!C?P@Q6Zh6FK&YR>F z+VFXrYtpw_`P{A|T!dKyXAbu9(rd#@{a-KL8Ti}V;Kx--gqMI7wco7%LHZEo;)fV} z6Q4Yv9)S!I>=CEUEsw2vEF@?DMf0c8+sCQxA<`Q>>beH|vRkx5adi>W4L%aPbe}>D zK{pf~y}Mgsq3-Wi#9)mJ173(=hm5k&mB|a*BJ*RXj$JjN!Q(;g0DmG@MR|yP6Xh*- zEvj3fzPb*<{4j|+Z_!U?V)pR6QMm*p63=jI)(cHHejq)}(pf_rx1Y1-q&h zQI7I>97UWU_~AlOLg&)k-lnSdlRIZ@b^ra3^@Au6;RkiT@w|#siv2|5x##-Ujh%>< za(Im&8?=O~u%*@EMKcYIvX?I9Pd^~ob*zP$8cdTOX3 z03jIe2X<+wEAPfA@9_6WFE2W+&oo9Befd>wr*>?BT2B$af=7*`R{T*e3r4KL^@PBj zJ0tt;9&>2f+~u9s*(-Nl=|)fP4=4js+>JN#kN?7!Kb(9N{i6Fa`WeIB z*rMDNEc7nAwb;8GudnYchP<;0(cw+pkJNBIqwd#h&DAm5j&!MQCDA{#yFgsn~y!l1Y#kWvM-R>|g$d zOoGDi444?-8Gx_&RDydM&NQ4g&eQ;#c%mK#AL=(G6Y3V6aRzFDn^Dv-4%|dPa0O2| d4r63^7N|4Nqemb0S)k7X|04_7erv=Q{tHUuBf$Uw literal 0 HcmV?d00001 diff --git a/corpus/tag-edit/cold/commands.jsonl b/corpus/tag-edit/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0b40bbdbcf63b5876f4029d6235462575970eb6f --- /dev/null +++ b/corpus/tag-edit/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-CFD607C8 -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/tag-edit/cold/machine.json b/corpus/tag-edit/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..789e3c21c336c77a53b96bbcb7c2a35cc90ae9ea --- /dev/null +++ b/corpus/tag-edit/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-cfd607c8", "hostname": "ONE-M6-CFD607C8"} diff --git a/corpus/tag-edit/cold/notebook/Open Notebook.onetoc2 b/corpus/tag-edit/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..24da7710436174fcca0465b4fb65084df42d28a6 GIT binary patch literal 3748 zcmZ3e|KIuM`!b(-j|5(YE0x7tw_TX{r-jX7an_GIsZ1YD1~g!Qcb0(k#IK%9FNjZk zdt%)UX`nnZhAICK1RypOOaMy5#LT>_I`zo$SJm*g}m|dSH?1j*QowY@CAwLXb zS1~F&8UjN;1R7WvrpPkL?37^;$p+H0)5;0Ah$2Ks=BRkgNw( zRstjo76-|IaHIjy%TTrq&@6@}(6S0-p!Y80ro*C#x?%nYg&YX*03CzwK#*Yq4h$0# zpjLrR2a18h6$C)~(9IJD2?7Df1fV`>xflTCV>6othn*19QOyU%6Ub8__kzl9kUbz8 zZdFLNrC{-tw3>b0DuCG2^{aBv;$JTW!^ooB8CR+X-ii|V{K@Ym*?NTu|Ph6aJS{;e-$nH`p?o_suS<+V_dIM%e|bK=5Gr@M?Gc?JeW zkp@sYX=ng(Bdk)GAN_E4YSESIi`;kXm#P6!5(q#Z0I?xj7#K7J8X9_ldS0Am4th}H z>9KaA|)J0i-sN>pco4m&?MHD6oww;fCkyXP@~E4O9p5x sm@`U@hQMeDKtljItH8=UP`(!hVo-+DockdIs&teZ4S~@R7y%&w05&BxAOHXW literal 0 HcmV?d00001 diff --git a/corpus/tag-edit/cold/notebook/tags.one b/corpus/tag-edit/cold/notebook/tags.one new file mode 100644 index 0000000000000000000000000000000000000000..d05139e43a9a9b70061e7b44d1feeabdc4c3d978 GIT binary patch literal 8352 zcmeHLc~n%_8NVY)zy*f@iZVk+MT@Cxq<~;ykW~R8RwC9m!5!S7Xb|PZGh%9q#t781 zcnT6C1`QH4@hp7}Rt zR<;hy?=aV#d7Svxjf>wU!vFnvhw|qAygO)XYyPe09gMH6=-MbEJ{|z&t`zGhiHuzFExC zsX52oo3?X(T+m76md;G-{bB-;mzkNSRkz9Jvz#^Dk#DhVA>^i91qrQ)l6lf6nLZq+O$nsM9 z%Q9ExmvBDXiD@YDHS95p&wH$4a*$&8I>j%K$Ga4lh-F02(a|S{obh4C{RQDg@0>mN zfgAP3vvjtUtzibXjxA(s*-Gl;@hi=(#JXew(=g)NvyC(-a^qP&(@m#l$UHp{vd+{( z*|RiL3QJ=d^kr)WV!@&&4|+RV=CjkWL66ej-6)6Lsd-YiQ!IdvUR>29-$th1ZvptK^{i4V8NK^XS>?Aw@z`8Vr2794uFqHJmCCIFN*3aIf3b^!sO)VPWH zK)|O7$>mr6y1p>_$kmYpqW+NbHs{F_-A}n6_>`+`k-~qO0;th)*2fNAG6s}K*5q$1eWq1k;Z0*86E!j8sa0t^qJO*b zA5#x#a;njfJ-b)>rH|k9k+C%Vi{#8JHQUe+o!;NSB`#!XOZ?{ZwO+ob!|Qo{k*C~U zko!>HlKWDXAhs*5WAel%NWQ;><<{8zf-f^dXD<>6qtj+Jonx!$BwI&q39~-UFohEg z$Xpy*xIN4DWrj6Wsijd%_hs73w}NB7X%3Eo1c=A4k}8t@G^6*2o+>*PzxCw2%)zJ6 zXF;|v_EX0zx8I7hSu1bUw?{WD+i^hKv~V@IpY!4!P3iBrgui|wB01sg-aY6?IH>Jy zvFl*$n6WH8NYDbCJ=m2b$+tMK5Q%f?3|h_%^t3*=F&(!95Eq0vYWutWN0AjQVqeSx zzlx;S-%{H_9WU4?K=~39fyjp%^pOx8m~ip#_dZv)BrSS3XhCqNW0y-0`+mlo7dn7-bxvo}592gukB5B02Jn`4UnTsGDJapYz0PY2h| zllhzf9QS6&_4j5>^2y@hYzi|Y}fOCr^4sMeO#AEI5x5MGUN zzi~K$zDdWAedyfwblA@0TXbKzyx`GSeQm|B+74XbT!0~w5f9S0FYze(m1G|j=k!>M z2zu^%GWP5naqe*y?AG3Y+ec=r^?YVp%E7{#?C3hfDwm^`E8gUORmUy&>xjTR`^*l3 zQKhGLJni?&=OyT;{j%AYM^zVl|1G?*;faEMvFopM{j-!y%@4sj=r7?Mf|}}o+gOL} znJFP1s%u05;=n-YcYh5J&7JJYtP<+kSx}E79$=LtVRs*iTk;;r3W8k&&uPs6>IgR0 z>#L&6OzRi*)vmVUr_(JlbM7RMu=>x_oR@x=Rlw~k!bO-RaQa|x54|>`%VmWOZO?n5OhP)(YL!57V7zKMFN(%FyMm-cE~6XTbVNdTvS2) z)G@0E)Vn>X9pF#IswfYUZ=$@#u0?eV)K}Lbm>(ul=PmllOw1O3H!7EaWRe+1&2pjX z!4IT|Njht226+cT!>V$05;t*$TsSU-sJ z5PneS8_%m~rPxm z2MUSF?J!FElI<|%+PT_8EtNA$*If8G^@HmfYCDMZf{aKnV{Zj5 zo{0&;w})J+)~5`Q`||RG>8YWD0EA$;AJ`>UcixRwzTxkWUtD-fpJ|LL{_?BZF6|h9 zwVon;g^U_UE%@UhPj1AE>xlug?|AjwJ^JvnIm^4Mb5`!U+Jm0lA5aEEaW~%hr@aRM z{5>o)<2_+uch&#D$~|@zJIe)L1Db8YU^!$tx|!&exhoqxABh3&rVu$ ziRjetZM=tqm{qjC=#;cIJ|=o>4eV!7eunU&^+l(Pt?}G`RSFV8e-SG7HojKLc&F BFY^EZ literal 0 HcmV?d00001 diff --git a/corpus/tag-edit/cold/read/environment.json b/corpus/tag-edit/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..594f24f943243c04b579251adfdf3fe8478f9a37 --- /dev/null +++ b/corpus/tag-edit/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-CFD607C8", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/tag-edit/cold/read/hierarchy.xml b/corpus/tag-edit/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..841314eff1eb2271d34eaff44d07b1c362e7cf4b --- /dev/null +++ b/corpus/tag-edit/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/tag-edit/cold/read/page-000.xml b/corpus/tag-edit/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..85c699c5520bbfcefe08a894fbf2573a613f1b62 --- /dev/null +++ b/corpus/tag-edit/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/tag-edit/cold/read/payloads.json b/corpus/tag-edit/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/tag-edit/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/tag-edit/cold/run.json b/corpus/tag-edit/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..a4e62281a0d3ffd44fd254377aa87f88fcd773ad --- /dev/null +++ b/corpus/tag-edit/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/tags", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/tag-edit/cold/scripts/cold.ps1 b/corpus/tag-edit/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/tag-edit/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/tag-edit/cold/scripts/read.ps1 b/corpus/tag-edit/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/tag-edit/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/tag-edit/cold/source.json b/corpus/tag-edit/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..bf8d669db188f0769bb053778a7153c3516f39e4 --- /dev/null +++ b/corpus/tag-edit/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1596, + "sha256": "f3ce1472c9be35b1fb52683c5214e62850787e8e13bbd1918fed81fd19a48a65", + "mtime_ns": 1789205536390985808 + }, + { + "path": "tags.one", + "bytes": 8352, + "sha256": "601a8998b691bab66830196b3b139be6ea58d6f29d5bc69b3efa9892e2fc1550", + "mtime_ns": 1789205536390698932 + } +] diff --git a/corpus/tag-edit/cold/teardown.json b/corpus/tag-edit/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/tag-edit/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/crates/notebook/README.md b/crates/notebook/README.md index afca8b056f56a9cb369ad581bb8a5045f0c952b4..ac5345e728727a4afc3dc882aca1a9d99c620ed0 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -14,8 +14,8 @@ diffs the supplied model against the page stored in the supplied local snapshot, writes the difference into the working image and queues one `Operation::Page(PageIntent { before, after, author })`; `before` is the page the edit started from and is the precondition reconciliation checks. Text, styles, -paragraph formatting, bullets and numbering, paragraph structure, outline -layout, insertions and deletions are all differences between `before` and +paragraph formatting, bullets, numbering and note tags, paragraph structure, +outline layout, insertions and deletions are all differences between `before` and `after`; the library never sees editor operations. ```no_run diff --git a/crates/onestore/README.md b/crates/onestore/README.md index e796392b892eec81dbcab9c91ba8897dc2b25912..3dfe7becde89b878f9f43de021d4f6e3df138805 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -57,7 +57,7 @@ harness also accepts `--client-profile release`. | `ParagraphJoin`, `PreparedEdit::join` | Join adjacent ordinary text while preserving inherited character styles and native text-identity rules | | `TextAttribute`, `PreparedEdit::format` | Change character formatting over a UTF-16 range while sharing immutable styles; preserve unselected runs | | `OutlineEdit`, `PreparedEdit::outline` | Change ordinary outline position/width or a paragraph's saved expansion default, preserving identities and content | -| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, bullets and numbering, paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | +| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, bullets and numbering, note tags, paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | | `TreeEdit`, `PreparedEdit::tree` | Move or delete a subtree on one page, normalize surviving containers, and replace an emptied table cell's paragraph atomically | | `PreparedEdit::commit`, `PreparedEdit::commit_file` | Publish the exact prepared image under caller-held exclusion or the conservative filesystem adapter | | `read_file` | Read a snapshot under whole-file exclusion | diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index a49864f4b3fdf788ad2c86621a22ef06549ea9c2..2963b0ca719940645b3e343780ae32d2114513a6 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -5,7 +5,7 @@ use super::{Outline, Page, PageObject, PageParagraph, ParagraphContent}; use crate::{ Error, ExGuid, Insertion, ObjectData, OutlineEdit, ParagraphJoin, ParagraphSplit, PropertySets, RevisionIndex, Store, TextAttribute, TreeEdit, Value, - document::{Document, Format, Kind}, + document::{Document, Format, Kind, Tag}, write::{PropertyObject, RevisionEdit, write_revisions}, }; use std::{ @@ -13,6 +13,9 @@ use std::{ ops::Range, }; +/// Property identifiers with their encoded values. +type Values = Vec<(u32, Vec)>; + fn invalid(message: &'static str) -> Error { Error { offset: 0, message } } @@ -174,9 +177,12 @@ impl Lowering<'_> { } for (id, definition) in &after.definitions { if before.definitions.get(id) != Some(definition) - && !matches!(definition.kind, Kind::List { .. }) + && !matches!( + definition.kind, + Kind::List { .. } | Kind::TagDefinition { .. } + ) { - return Err(invalid("Tag and style definitions cannot be edited")); + return Err(invalid("Style definitions cannot be edited")); } } let old = View::new(before)?; @@ -209,6 +215,7 @@ impl Lowering<'_> { self.delete(&old, &new, &consumed)?; self.edit_text(&new)?; self.edit_lists(after, &new)?; + self.edit_tags(after, &new)?; self.edit_paragraph_formatting(&new)?; self.edit_formatting(&new)?; self.edit_layout(&old, &new)?; @@ -275,18 +282,16 @@ impl Lowering<'_> { let Some(previous) = old.paragraphs.get(id) else { continue; }; - let same = paragraph.tags == previous.tags - && paragraph.style == previous.style - && paragraph.format == previous.format; + let same = paragraph.style == previous.style && paragraph.format == previous.format; if !same { return Err(invalid( - "Paragraph tags, styles and paragraph formatting cannot be edited", + "Paragraph styles and paragraph formatting cannot be edited", )); } match (¶graph.content, &previous.content) { (ParagraphContent::Text(text), ParagraphContent::Text(previous)) => { - if text.date_field != previous.date_field || text.tags != previous.tags { - return Err(invalid("Text fields and tags cannot be edited")); + if text.date_field != previous.date_field { + return Err(invalid("Text fields cannot be edited")); } } (ParagraphContent::Table(table), ParagraphContent::Table(previous)) => { @@ -558,12 +563,10 @@ impl Lowering<'_> { let paragraph = new.paragraphs[id]; let text = paragraph .text() - .filter(|text| text.date_field.is_none() && text.tags.is_empty()) - .filter(|_| paragraph.tags.is_empty() && paragraph.style.is_none()) + .filter(|text| text.date_field.is_none()) + .filter(|_| paragraph.style.is_none()) .ok_or_else(|| { - invalid( - "New paragraphs contain plain text without tags, fields or styles", - ) + invalid("New paragraphs contain plain text without fields or styles") })?; let insertion = Insertion::paragraph( self.id(container), @@ -725,7 +728,7 @@ impl Lowering<'_> { if bullet.is_some() && (format.is_none() || font.is_none()) { return Err(invalid("A bullet definition names its glyph and font")); } - let mut values: Vec<(u32, Vec)> = Vec::new(); + let mut values: Values = Vec::new(); if let Some(format) = format { let units: Vec = format.encode_utf16().collect(); let count = u16::try_from(units.len()) @@ -838,6 +841,186 @@ impl Lowering<'_> { Ok(()) } + /// Rewrites the note tags of paragraphs and text objects whose model tags differ from + /// the stored ones. A tag definition new to the section becomes a definition object + /// carrying the model's identity after squash. + fn edit_tags(&mut self, after: &Page, new: &View<'_>) -> Result<(), Error> { + fn same(a: &[Tag], b: &[Tag]) -> bool { + let key = |t: &Tag| { + ( + t.definition, + t.action_type, + t.status, + t.created, + t.completed, + t.start, + t.due, + t.task_id, + ) + }; + a.len() == b.len() && a.iter().zip(b).all(|(x, y)| key(x) == key(y)) + } + let current = self.current()?; + let stored = View::new(¤t)?; + for (id, paragraph) in &new.paragraphs { + let image_id = self.id(*id); + let previous = stored + .paragraphs + .get(&image_id) + .ok_or_else(|| invalid("A paragraph is missing after text edits"))?; + let mut targets = vec![(image_id, ¶graph.tags, &previous.tags)]; + if let (Some(text), Some(before)) = (paragraph.text(), previous.text()) { + targets.push((self.id(text.id), &text.tags, &before.tags)); + } + for (object, tags, stored_tags) in targets { + if same(tags, stored_tags) { + continue; + } + let mut definitions: Vec<(ExGuid, Option)> = Vec::new(); + let mut sets: Vec<(usize, Values)> = Vec::new(); + let mut action_types = BTreeSet::new(); + for tag in tags { + let definition = tag + .definition + .ok_or_else(|| invalid("A note tag names its definition"))?; + let action_type = if tag.status & 4 != 0 { + tag.action_type + } else { + match after.definitions.get(&definition).map(|d| &d.kind) { + Some(Kind::TagDefinition { action_type, .. }) => *action_type, + _ => return Err(invalid("A note tag must reference a tag definition")), + } + }; + if !action_types.insert(action_type.unwrap_or(0)) { + return Err(invalid("An element holds one note tag per action type")); + } + let known = current.definitions.contains_key(&definition) + || self.alias.contains_key(&definition); + let written = if known { + self.id(definition) + } else { + let allocated = ExGuid { + guid: crate::write::fresh_guid()?, + n: 1, + }; + self.alias.insert(definition, allocated); + allocated + }; + let index = match definitions.iter().position(|(id, _)| *id == written) { + Some(index) => index, + None => { + let values = if known { + None + } else { + let Some(model) = after.definitions.get(&definition) else { + return Err(invalid( + "A note tag references a missing tag definition", + )); + }; + let Kind::TagDefinition { + label, + action_type, + shape, + color, + highlight, + } = &model.kind + else { + return Err(invalid( + "A note tag must reference a tag definition", + )); + }; + let mut values: Values = vec![ + (0x0c003473, vec![0]), + (0x10003463, action_type.unwrap_or(0).to_le_bytes().to_vec()), + (0x10003464, shape.unwrap_or(0).to_le_bytes().to_vec()), + (0x14003467, 0u32.to_le_bytes().to_vec()), + ]; + if let Some(label) = label { + values.push((0x1c003468, crate::create::string(label))); + } + if let Some(color) = color { + values.push((0x14003466, color.to_le_bytes().to_vec())); + } + if let Some(highlight) = highlight { + values.push((0x14003465, highlight.to_le_bytes().to_vec())); + } + Some(values) + }; + definitions.push((written, values)); + definitions.len() - 1 + } + }; + let mut fields: Values = Vec::new(); + if let Some(action_type) = tag.action_type { + fields.push((0x10003463, action_type.to_le_bytes().to_vec())); + } + for (id, value) in [ + (0x1400346e, tag.created), + (0x1400346f, tag.completed), + (0x1400346a, tag.start), + (0x1400346b, tag.due), + ] { + if let Some(value) = value { + fields.push((id, value.to_le_bytes().to_vec())); + } + } + fields.push((0x10003470, tag.status.to_le_bytes().to_vec())); + if let Some(task) = tag.task_id { + fields.push((0x1c003469, task.to_vec())); + } + sets.push((index, fields)); + } + let space = self.space; + self.apply(|image| { + let store = Store::parse(image)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let parents = crate::edit::editable_parents( + document.active(space)?, + &document.pages_in(space)?, + object, + )?; + let modified = crate::create::current_timestamps()?.0.to_le_bytes(); + crate::write::write_revision(image, space, |raw| { + let mut changed = BTreeMap::new(); + for (id, values) in &definitions { + let Some(values) = values else { + continue; + }; + let mut node = PropertyObject { + jcid: 0x120043, + bytes: crate::create::properties(values)?, + global_ids: std::sync::Arc::new(BTreeMap::from([(0, id.guid)])), + }; + node.reference(*id)?; + changed.insert(*id, node); + } + let mut target = PropertyObject::from_object(&raw.objects[&object])?; + let mut encoded = Vec::new(); + for (index, fields) in &sets { + let reference = target.reference(definitions[*index].0)?; + let mut set = vec![(0x20003488, reference.to_vec())]; + set.extend(fields.iter().cloned()); + encoded.push(set); + } + target.set_sets(0x40003489, 0x44000811, &encoded)?; + target.set(&[(0x14001d7a, &modified)])?; + changed.insert(object, target); + crate::formatting::touch_ancestors( + raw, + &parents, + object, + &modified, + &mut changed, + )?; + Ok(changed) + }) + })?; + } + } + Ok(()) + } + fn edit_paragraph_formatting(&mut self, new: &View<'_>) -> Result<(), Error> { let current = self.current()?; let current = View::new(¤t)?; diff --git a/crates/onestore/src/write.rs b/crates/onestore/src/write.rs index e0e48f8134b444cf99fe6c9aac8c8e1e5eda3946..72662adc7232ab16d41f1de295fe9b1fd0f49727 100644 --- a/crates/onestore/src/write.rs +++ b/crates/onestore/src/write.rs @@ -133,6 +133,7 @@ fn patch_properties( blob: &[u8], updates: &[(u32, &[u8])], inserts: &[(u32, &[u8])], + nested_references: &[u8], ) -> Result> { let properties = PropertySets::parse(blob)?; let root = &properties.sets[0]; @@ -167,6 +168,8 @@ fn patch_properties( 7 => value.len() < 0x40000000, 8 => value.len() == 4, 9 => value.len().is_multiple_of(4) && value.len() / 4 <= 0xffffff, + // An encoded property-set array is inserted whole; its references follow. + 16 => i >= updates.len() && value.len() >= 4, _ => { return Err(Error { offset: 0, @@ -198,13 +201,13 @@ fn patch_properties( 9 => encoded.extend_from_slice(&(value.len() as u32 / 4).to_le_bytes()), _ => {} } - if (3..=7).contains(&kind) { + if (3..=7).contains(&kind) || kind == 16 { encoded.extend_from_slice(value); } if adding { added_ids.extend_from_slice(&property.to_le_bytes()); added_fields.extend_from_slice(&encoded); - if kind >= 8 { + if (8..=9).contains(&kind) { object_count += (value.len() / 4) as i64; added_references.extend_from_slice(value); } @@ -232,6 +235,7 @@ fn patch_properties( } } } + object_count += (nested_references.len() / 4) as i64; if !(0..=0xffffff).contains(&object_count) { return Err(Error { offset: 0, @@ -242,6 +246,7 @@ fn patch_properties( if header != object_header { patches.push((0, 4, 0, header.to_le_bytes().to_vec())); } + added_references.extend_from_slice(nested_references); if !added_references.is_empty() { let end = 4 + (object_header as usize & 0xffffff) * 4; patches.push((end, end, usize::MAX, added_references)); @@ -339,7 +344,7 @@ pub(crate) fn replace_objects( edit.object, PropertyObject { jcid: object.jcid, - bytes: patch_properties(blob, edit.updates, edit.inserts)?, + bytes: patch_properties(blob, edit.updates, edit.inserts, &[])?, global_ids: Arc::clone(&object.global_ids), }, ); @@ -384,7 +389,56 @@ impl PropertyObject { .iter() .any(|p| p.id & 0x7fffffff == id & 0x7fffffff) }); - self.bytes = patch_properties(&self.bytes, &updates, &inserts)?; + self.bytes = patch_properties(&self.bytes, &updates, &inserts, &[])?; + Ok(()) + } + + /// Replaces a property-set array such as note tags: each set lists scalar values + /// inline, while an object reference (a kind-8 identity) is supplied as the compact + /// identity `reference` produced and joins the object's reference stream in order. + pub fn set_sets(&mut self, id: u32, element: u32, sets: &[Vec<(u32, Vec)>]) -> Result<()> { + if (id >> 26) & 31 != 16 || (element >> 26) & 31 != 17 { + return Err(Error { + offset: 0, + message: "Property-set arrays need array and element identifiers", + }); + } + self.remove(&[id])?; + if sets.is_empty() { + return Ok(()); + } + let mut encoded = (sets.len() as u32).to_le_bytes().to_vec(); + encoded.extend_from_slice(&element.to_le_bytes()); + let mut references = Vec::new(); + for set in sets { + encoded.extend_from_slice(&u16::try_from(set.len()).unwrap().to_le_bytes()); + for (field, _) in set { + encoded.extend_from_slice(&field.to_le_bytes()); + } + for (field, value) in set { + match (field >> 26) & 31 { + 3..=6 => { + assert_eq!(value.len(), 1 << (((field >> 26) & 31) - 3)); + encoded.extend_from_slice(value); + } + 7 => { + encoded.extend_from_slice(&(value.len() as u32).to_le_bytes()); + encoded.extend_from_slice(value); + } + 8 => { + assert_eq!(value.len(), 4); + references.extend_from_slice(value); + } + _ => { + return Err(Error { + offset: 0, + message: "Property-set arrays hold scalars and single references", + }); + } + } + } + } + self.bytes = patch_properties(&self.bytes, &[], &[(id, &encoded)], &references)?; Ok(()) } diff --git a/crates/onestore/src/write/tests.rs b/crates/onestore/src/write/tests.rs index 02a27edb8c68f6c4c47e85818a5c1540e48d69c5..49027ec00f268929d70e6e342dab9f6867f76f84 100644 --- a/crates/onestore/src/write/tests.rs +++ b/crates/onestore/src/write/tests.rs @@ -221,6 +221,7 @@ fn add_paragraph(source: &[u8], number: u32) -> Vec { blob, &[(0x24001c20, &children), (0x14001d7a, &modified)], &[], + &[], )?, global_ids: Arc::clone(&table), }, @@ -302,7 +303,7 @@ fn restyle(source: &[u8]) -> Vec { style, PropertyObject { jcid: previous.jcid, - bytes: patch_properties(blob, &[], &[(0x88001c04, &[])])?, + bytes: patch_properties(blob, &[], &[(0x88001c04, &[])], &[])?, global_ids: Arc::new(table), }, )]); @@ -316,7 +317,12 @@ fn restyle(source: &[u8]) -> Vec { *id, PropertyObject { jcid: text.jcid, - bytes: patch_properties(blob, &[(0x24001e13, &compact(style, &table)?)], &[])?, + bytes: patch_properties( + blob, + &[(0x24001e13, &compact(style, &table)?)], + &[], + &[], + )?, global_ids: Arc::new(table), }, ); @@ -461,7 +467,8 @@ fn changed_graphs_reject_cycles_dangling_and_unreachable_additions() { bytes: patch_properties( blob, &[(0x24001c20, &compact(target, &outline.global_ids)?)], - &[] + &[], + &[], )?, global_ids: Arc::clone(&outline.global_ids), } @@ -559,7 +566,7 @@ fn property_splices_match_independently_encoded_flat_sets() { .map(|(id, value)| (*id, value.as_slice())) .collect(); let original = properties(&original).unwrap(); - let actual = patch_properties(&original, &updates, &inserts).unwrap(); + let actual = patch_properties(&original, &updates, &inserts, &[]).unwrap(); assert_eq!(actual, properties(&expected).unwrap(), "case {case}"); } } @@ -599,6 +606,7 @@ fn nested_fields_and_other_reference_streams_remain_byte_exact() { (0x24000001, &[6, 0, 0, 0, 7, 0, 0, 0]), ], &[(0x24000005, &[8, 0, 0, 0]), (0x88000006, &[])], + &[], ) .unwrap(); let parsed = PropertySets::parse(&changed).unwrap(); @@ -717,7 +725,7 @@ fn deep_property_splices_do_not_use_the_call_stack() { bytes.extend_from_slice(&0x44000001_u32.to_le_bytes()); } bytes.extend_from_slice(&0_u16.to_le_bytes()); - let changed = patch_properties(&bytes, &[], &[(0x88000002, &[])]).unwrap(); + let changed = patch_properties(&bytes, &[], &[(0x88000002, &[])], &[]).unwrap(); let parsed = PropertySets::parse(&changed).unwrap(); assert_eq!(parsed.sets.len(), 100_001); assert_eq!(parsed.sets[0].len(), 2); @@ -758,7 +766,7 @@ fn invalid_property_splices_are_rejected() { (vec![], vec![(0x88000001, &[][..])]), (vec![], vec![(0x20000003, &[][..])]), ] { - assert!(patch_properties(&bytes, &updates, &inserts).is_err()); + assert!(patch_properties(&bytes, &updates, &inserts, &[]).is_err()); } } diff --git a/crates/onestore/tests/page_tags.rs b/crates/onestore/tests/page_tags.rs new file mode 100644 index 0000000000000000000000000000000000000000..9a31b1503e6c6cf64dc7763f07e0f45d7d42ec1b --- /dev/null +++ b/crates/onestore/tests/page_tags.rs @@ -0,0 +1,305 @@ +use onestore::{ + ExGuid, PreparedEdit, RevisionIndex, Store, + document::{Document, Format, Kind, Tag}, + page::{ + Definition, Page, PageObject, PageParagraph, ParagraphContent, TextObject, text::new_id, + }, +}; + +const TREES: &[u8] = + include_bytes!("../../../corpus/outline-edit/tree/before/notebook/synthetic.one"); +const AUTHOR: &str = "Tag author"; + +fn page_by_title(bytes: &[u8], title: &str) -> (ExGuid, Page) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + document + .pages() + .unwrap() + .into_iter() + .find_map(|(space, _)| { + let page = Page::from_space(&document, space).unwrap(); + (page.title == title).then_some((space, page)) + }) + .unwrap() +} + +fn page_in(bytes: &[u8], space: ExGuid) -> Page { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + Page::from_space(&Document::parse(&index).unwrap(), space).unwrap() +} + +fn body_paragraphs(page: &mut Page) -> &mut Vec { + page.objects + .iter_mut() + .find_map(|object| match object { + PageObject::Outline(outline) if !outline.title => Some(&mut outline.paragraphs), + _ => None, + }) + .unwrap() +} + +/// Tag positions in the element's extra arena are assigned on read; equality ignores them. +fn normalized(mut page: Page) -> Page { + for object in &mut page.objects { + if let PageObject::Outline(outline) = object { + for paragraph in &mut outline.paragraphs { + for tag in &mut paragraph.tags { + tag.extra_set = 0; + } + if let Some(text) = paragraph.text_mut() { + for tag in &mut text.tags { + tag.extra_set = 0; + } + } + } + } + } + page +} + +fn assert_same(written: &[u8], space: ExGuid, expected: &Page) -> Page { + let stored = page_in(written, space); + let mut expected = expected.clone(); + expected.title = stored.title.clone(); + assert_eq!(normalized(stored.clone()), normalized(expected)); + stored +} + +fn tagged(page: &Page) -> usize { + page.objects + .iter() + .find_map(|object| match object { + PageObject::Outline(outline) if !outline.title => Some( + outline + .paragraphs + .iter() + .position(|p| p.text().is_some_and(|t| !t.tags.is_empty())) + .unwrap(), + ), + _ => None, + }) + .unwrap() +} + +fn raw_object(bytes: &[u8], space: ExGuid, id: ExGuid) -> String { + let store = Store::parse(bytes).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + format!( + "{:?}", + index.resolve_active(space).unwrap().objects[&id].data + ) +} + +fn definition(label: &str, shape: u16) -> Definition { + Definition { + kind: Kind::TagDefinition { + label: Some(label.into()), + action_type: Some(0), + shape: Some(shape), + color: None, + highlight: None, + }, + format: Format::default(), + } +} + +fn tag(definition: ExGuid, completed: bool) -> Tag { + Tag { + definition: Some(definition), + action_type: None, + status: u16::from(completed), + created: Some(1_262_401_445), + completed: completed.then_some(1_262_405_106), + start: None, + due: None, + task_id: None, + extra_set: 0, + } +} + +fn plain_paragraph(template: &PageParagraph, text: &str) -> PageParagraph { + let mut paragraph = template.clone(); + paragraph.id = new_id().unwrap(); + paragraph.parent = None; + paragraph.level = 1; + paragraph.lists.clear(); + paragraph.tags.clear(); + paragraph.style = None; + paragraph.collapsed = false; + paragraph.content = ParagraphContent::Text(TextObject { + id: new_id().unwrap(), + date_field: None, + text: onestore::page::Paragraph::new( + text.into(), + Format { + font: Some("Calibri".into()), + font_size: Some(11.0), + language: Some(1033), + ..Default::default() + }, + ), + tags: Vec::new(), + }); + paragraph +} + +#[test] +fn removing_a_tag_leaves_its_definition_and_the_other_paragraphs_untouched() { + let (space, before) = page_by_title(TREES, "Indent first subtree"); + let at = tagged(&before); + let mut after = before.clone(); + let definition = body_paragraphs(&mut after)[at].text().unwrap().tags[0] + .definition + .unwrap(); + body_paragraphs(&mut after)[at] + .text_mut() + .unwrap() + .tags + .clear(); + let written = PreparedEdit::page(TREES, space, &after, AUTHOR).unwrap(); + let mut expected = after.clone(); + expected.definitions.remove(&definition); + let stored = assert_same(written.as_bytes(), space, &expected); + assert!(!stored.definitions.contains_key(&definition)); + assert_eq!( + raw_object(written.as_bytes(), space, definition), + raw_object(TREES, space, definition) + ); + let count = body_paragraphs(&mut after).len(); + let other = body_paragraphs(&mut after)[(at + 1) % count] + .text() + .unwrap() + .id; + assert_eq!( + raw_object(written.as_bytes(), space, other), + raw_object(TREES, space, other) + ); +} + +#[test] +fn an_existing_definition_tags_another_paragraph_and_a_task_completes_in_place() { + let (space, before) = page_by_title(TREES, "Indent first subtree"); + let at = tagged(&before); + let mut after = before.clone(); + let existing = body_paragraphs(&mut after)[at].text().unwrap().tags[0].clone(); + let definition = existing.definition.unwrap(); + let count = body_paragraphs(&mut after).len(); + let other = (at + 1) % count; + body_paragraphs(&mut after)[other] + .text_mut() + .unwrap() + .tags + .push(tag(definition, false)); + let written = PreparedEdit::page(TREES, space, &after, AUTHOR).unwrap(); + let stored = assert_same(written.as_bytes(), space, &after); + let mut completed = stored.clone(); + let text = body_paragraphs(&mut completed)[other].text_mut().unwrap(); + text.tags[0].status = 1; + text.tags[0].completed = Some(1_262_405_106); + let again = PreparedEdit::page(written.as_bytes(), space, &completed, AUTHOR).unwrap(); + assert_same(again.as_bytes(), space, &completed); + assert_eq!( + PreparedEdit::page( + again.as_bytes(), + space, + &page_in(again.as_bytes(), space), + AUTHOR + ) + .unwrap() + .as_bytes(), + again.as_bytes() + ); +} + +#[test] +fn tags_without_a_known_definition_are_refused() { + let (space, before) = page_by_title(TREES, "Indent first subtree"); + let mut missing = before.clone(); + body_paragraphs(&mut missing)[0] + .text_mut() + .unwrap() + .tags + .push(tag(new_id().unwrap(), false)); + assert!(PreparedEdit::page(TREES, space, &missing, AUTHOR).is_err()); + let mut repeated = before.clone(); + let at = tagged(&repeated); + let existing = body_paragraphs(&mut repeated)[at].text().unwrap().tags[0].clone(); + body_paragraphs(&mut repeated)[at] + .text_mut() + .unwrap() + .tags + .push(tag(existing.definition.unwrap(), false)); + assert!(PreparedEdit::page(TREES, space, &repeated, AUTHOR).is_err()); + let mut wrong = before.clone(); + let list = wrong + .definitions + .iter() + .find(|(_, d)| matches!(d.kind, Kind::List { .. })) + .map(|(id, _)| *id); + if let Some(list) = list { + body_paragraphs(&mut wrong)[0] + .text_mut() + .unwrap() + .tags + .push(tag(list, false)); + assert!(PreparedEdit::page(TREES, space, &wrong, AUTHOR).is_err()); + } +} + +/// A fresh page gains two tag definitions and three tagged paragraphs, one completed; +/// `ONESTORE_TAG_EXPORT` names a new directory receiving the candidate for a cold reopen. +#[test] +fn new_definitions_and_tags_publish_on_a_fresh_page() { + let source = onestore::create_section("tags.one", "Open task", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (space, _) = document.pages().unwrap()[0]; + let before = Page::from_space(&document, space).unwrap(); + let mut after = before.clone(); + let template = body_paragraphs(&mut after)[0].clone(); + let task = new_id().unwrap(); + let important = new_id().unwrap(); + after.definitions.insert(task, definition("Rust task", 3)); + let mut important_definition = definition("Important", 13); + let Kind::TagDefinition { action_type, .. } = &mut important_definition.kind else { + panic!() + }; + *action_type = Some(1); + after.definitions.insert(important, important_definition); + body_paragraphs(&mut after)[0] + .text_mut() + .unwrap() + .tags + .push(tag(task, false)); + let mut done = plain_paragraph(&template, "Completed task"); + done.text_mut().unwrap().tags.push(tag(task, true)); + body_paragraphs(&mut after).push(done); + let mut both = plain_paragraph(&template, "Important open task"); + both.text_mut().unwrap().tags.push(tag(important, false)); + both.text_mut().unwrap().tags.push(tag(task, false)); + body_paragraphs(&mut after).push(both); + body_paragraphs(&mut after).push(plain_paragraph(&template, "Untagged")); + let written = PreparedEdit::page(&source, space, &after, AUTHOR).unwrap(); + assert_same(written.as_bytes(), space, &after); + if let Some(directory) = std::env::var_os("ONESTORE_TAG_EXPORT") { + let directory = std::path::PathBuf::from(directory); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("tags.one"), written.as_bytes()).unwrap(); + let written_store = Store::parse(written.as_bytes()).unwrap(); + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("tags.one", written_store.header.file_id)], + ) + .unwrap(), + ) + .unwrap(); + } +} diff --git a/crates/onestore/tests/support/page_edits.rs b/crates/onestore/tests/support/page_edits.rs index 0418b9b3d32bb3fe8ceb65d0e7b0d40619838e53..f9d7ef96669d76957ae403c9bba307f0e6a9da78 100644 --- a/crates/onestore/tests/support/page_edits.rs +++ b/crates/onestore/tests/support/page_edits.rs @@ -199,7 +199,63 @@ fn mutate(page: &mut Page, bytes: &mut Bytes<'_>) { .filter(|(_, o)| matches!(o, PageObject::Outline(_))) .map(|(i, _)| i) .collect(); - match kind % 10 { + match kind % 11 { + 10 => { + let Some(outline) = outlines.first().map(|i| match &mut page.objects[*i] { + PageObject::Outline(outline) => outline, + _ => unreachable!(), + }) else { + continue; + }; + let at = usize::from(bytes.next().unwrap_or(0)) % outline.paragraphs.len().max(1); + let Some(text) = outline.paragraphs.get_mut(at).and_then(|p| p.text_mut()) else { + continue; + }; + if text.tags.is_empty() { + let definition = page + .definitions + .iter() + .find(|(_, d)| { + matches!(&d.kind, onestore::document::Kind::TagDefinition { label: Some(label), .. } if label == "Fuzz task") + }) + .map(|(id, _)| *id) + .unwrap_or_else(|| { + let id = new_id().unwrap(); + page.definitions.insert( + id, + onestore::page::Definition { + kind: onestore::document::Kind::TagDefinition { + label: Some("Fuzz task".into()), + action_type: Some(0), + shape: Some(3), + color: None, + highlight: None, + }, + format: Default::default(), + }, + ); + id + }); + let outline = outlines.first().map(|i| match &mut page.objects[*i] { + PageObject::Outline(outline) => outline, + _ => unreachable!(), + }); + let text = outline.unwrap().paragraphs[at].text_mut().unwrap(); + text.tags.push(onestore::document::Tag { + definition: Some(definition), + action_type: None, + status: u16::from(bytes.next().unwrap_or(0) % 2), + created: Some(1_262_401_445), + completed: None, + start: None, + due: None, + task_id: None, + extra_set: 0, + }); + } else { + text.tags.clear(); + } + } 8 => { let Some(outline) = outlines.first().map(|i| match &mut page.objects[*i] { PageObject::Outline(outline) => outline, diff --git a/tools/test_tag_edit.py b/tools/test_tag_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..9645227074f0d5e6f8a95906d2981cfa853ab83b --- /dev/null +++ b/tools/test_tag_edit.py @@ -0,0 +1,37 @@ +from pathlib import Path +import re +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/tag-edit' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +class TagEditTest(unittest.TestCase): + def test_rust_tag_definitions_and_task_states_render_natively(self): + with TemporaryDirectory() as temporary: + read = Path(temporary) / 'read' + shutil.copytree(FIXTURE / 'cold/read', read) + compare(FIXTURE / 'candidate', read) + page, = (ET.parse(path).getroot() for path in sorted((FIXTURE / 'cold/read').glob('page-*.xml'))) + definitions = [(d.get('index'), d.get('type'), d.get('symbol'), d.get('name')) for d in page.findall('one:TagDef', ns)] + self.assertEqual(definitions, [('0', '0', '3', 'Rust task'), ('1', '1', '13', 'Important')]) + tagged = [([(t.get('index'), t.get('completed'), t.get('completionDate') is not None) for t in oe.findall('one:Tag', ns)], + re.sub(r'<[^>]*>', '', oe.find('one:T', ns).text or '')) + for oe in page.iter('{%s}OE' % ns['one']) if oe.find('one:T', ns) is not None] + self.assertEqual(tagged, [ + ([('0', 'false', False)], 'Open task'), + ([('0', 'true', True)], 'Completed task'), + ([('0', 'false', False), ('1', 'false', False)], 'Important open task'), + ([], 'Untagged'), + ]) + + +if __name__ == '__main__': + unittest.main() -- 2.54.0