From b26c81f7c036b864e287820c6d3b0498b64159cf Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sat, 19 Sep 2026 23:22:38 -0700 Subject: [PATCH] fix: review findings in protected saves, the cache and the parser; store only named table entries Review (three independent passes over the protected-write, shared-GUID, harness and save-cost changes): - Notebook::save_unlocked compared the file with a snapshot it had just read, so a section written since unlock was silently overwritten with the stale model. unlock now returns Unlocked { pages, snapshot } and the save commits against that snapshot. - An acknowledged publication becomes the base that polls trust without validating; it is now validated before it is published. - Global id table imports could grow the pending entry list without bound before the duplicate check at the table end; bounded per entry node. The GUID uniqueness check reuses the list. - A corrupt cache patch length aborted on allocation; bounded by the patch. The recovery summary reads the working length from the patch header and the asset scan holds one image at a time. - Sealed buffers are sized once so growth leaves no plaintext behind; a failing random source is no longer reported as malformed data; file-data strings are decoded checked; protected writing admits exactly one section key and refuses a writer that created a space. The save's working buffers are ordinary memory, and the owner doc now says so. - cache_probe accepts a coalesced insert intent. Growth: a rewritten object group stored the whole global id table of the revision its objects were read from. It now stores the entries they name, and no-op detection compares those entries. OneNote 2010 cold-reads the sparse tables and leaves the file as written (corpus/table-growth). Assisted-by: claude-fable-5.1, claude-opus-5 --- corpus/table-growth/README.md | 8 + .../candidate/notebook/Open Notebook.onetoc2 | Bin 0 -> 4816 bytes .../candidate/notebook/synthetic.one | Bin 0 -> 25392 bytes corpus/table-growth/cold/commands.jsonl | 3 + corpus/table-growth/cold/machine.json | 1 + .../cold/notebook/Open Notebook.onetoc2 | Bin 0 -> 4816 bytes .../table-growth/cold/notebook/synthetic.one | Bin 0 -> 25392 bytes ...0b7c57eb3b6110a06aa2114e06d69a7.attachment | 1 + .../table-growth/cold/read/environment.json | 7 + corpus/table-growth/cold/read/hierarchy.xml | 2 + corpus/table-growth/cold/read/page-000.xml | 9 ++ corpus/table-growth/cold/read/payloads.json | 10 ++ corpus/table-growth/cold/run.json | 18 +++ corpus/table-growth/cold/scripts/cold.ps1 | 27 ++++ corpus/table-growth/cold/scripts/read.ps1 | 142 ++++++++++++++++++ corpus/table-growth/cold/source.json | 14 ++ corpus/table-growth/cold/teardown.json | 1 + crates/notebook/README.md | 9 +- crates/notebook/examples/cache_probe.rs | 5 +- crates/notebook/src/assets.rs | 5 +- crates/notebook/src/images.rs | 28 +++- crates/notebook/src/recovery.rs | 2 +- crates/notebook/src/session.rs | 66 +++++--- crates/notebook/src/sync.rs | 2 + crates/notebook/tests/protected.rs | 18 ++- crates/onestore/src/objects.rs | 15 +- crates/onestore/src/page/write.rs | 21 ++- crates/onestore/src/protected/crypto.rs | 7 +- crates/onestore/src/protected/mod.rs | 51 +++++-- crates/onestore/src/write.rs | 76 +++++++--- crates/onestore/tests/edit.rs | 8 +- crates/onestore/tests/table_growth.rs | 101 +++++++++++++ crates/onestore/tests/writer.rs | 18 ++- tools/test_table_growth.py | 24 +++ 34 files changed, 606 insertions(+), 93 deletions(-) create mode 100644 corpus/table-growth/README.md create mode 100644 corpus/table-growth/candidate/notebook/Open Notebook.onetoc2 create mode 100644 corpus/table-growth/candidate/notebook/synthetic.one create mode 100644 corpus/table-growth/cold/commands.jsonl create mode 100644 corpus/table-growth/cold/machine.json create mode 100644 corpus/table-growth/cold/notebook/Open Notebook.onetoc2 create mode 100644 corpus/table-growth/cold/notebook/synthetic.one create mode 100644 corpus/table-growth/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment create mode 100644 corpus/table-growth/cold/read/environment.json create mode 100644 corpus/table-growth/cold/read/hierarchy.xml create mode 100644 corpus/table-growth/cold/read/page-000.xml create mode 100644 corpus/table-growth/cold/read/payloads.json create mode 100644 corpus/table-growth/cold/run.json create mode 100644 corpus/table-growth/cold/scripts/cold.ps1 create mode 100644 corpus/table-growth/cold/scripts/read.ps1 create mode 100644 corpus/table-growth/cold/source.json create mode 100644 corpus/table-growth/cold/teardown.json create mode 100644 crates/onestore/tests/table_growth.rs create mode 100644 tools/test_table_growth.py diff --git a/corpus/table-growth/README.md b/corpus/table-growth/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d71fe1b8d9208018bd61aad243997c45985842b4 --- /dev/null +++ b/corpus/table-growth/README.md @@ -0,0 +1,8 @@ +# Table growth + +`candidate/notebook` is `native/20260905-05` after `a_text_edit_stores_the_table_entries_it_names` +(`ONESTORE_TABLE_EXPORT`) appended words to a paragraph of every page. Each rewritten +object group stores only the global id table entries its objects name, so the stored +indices have gaps where the source revision's table named GUIDs the edit does not use. +`cold` is OneNote 2010's cold read of it from a fresh clone; OneNote left the file as +written. diff --git a/corpus/table-growth/candidate/notebook/Open Notebook.onetoc2 b/corpus/table-growth/candidate/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..a33d3bfd210f844e3db618e648ac898adeee09c6 GIT binary patch literal 4816 zcmZ3e|KIuM`!b(-j|5(YE0x7J#4gqT(JSk=YNPqbC`J7Q1~g!Qcb0(k#IK%9FNjZk zdt%)UX`nnZhAICK1RypWOaMy5#LF~y)9Oxb$#>r}|Kyh?&t}bosmDf_Y+J~WjIpU16&nqK zp&kN9zI^|>u_|nd0MsKKISf-97*&4foBm+C_tSH2N_*D3xv@JKfzm7q4Gq}Dhk6i< zx_>kTMnni8*O}hCjGGRN9_ohWdr$_yJe4T`#AgCxK_CW+fpB`~EME{GBo3--Vd6H) z_ZEZrTu?F7Y>{LTAEsuiuGdl!TLvnx-CJk`WR8N-5Eu=CK@tM6`X7|PL4XHGB`+Yu zFm(x|3PjN`C9#!{ufV&YS{i30vEq40iLJuR~o}CUeKQ!^!fyzp3;DeQC z>=&E6j1YN6k%k5a6=2YVlqMGh%ncLkbZefpL}|54*a8iKhK3%XT&?Jgw6K&`@AZW{ zcSu}5J`to<*Mp(=1S2D?1O}GPEh!8=$Q}Wi2mSTaM{uf|q0Lu_fMDRhC zvM?~f>JOku15~tukzt<;xMA@Dj5mMuU0$Qf@C)Q}pzcvH z8UlkT1dxjxSh)a9#^BP130x+8lVMN+l?$Mx4+jk_3{zwoWOf3}jBFqcDmy@}PaB8! z;05iU+}4;?89zwhcNL@u)PgK#sAR}vC}GH8NClFa49N_74EaEjRG=Kl1waZ20OIF{ A&Hw-a literal 0 HcmV?d00001 diff --git a/corpus/table-growth/candidate/notebook/synthetic.one b/corpus/table-growth/candidate/notebook/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..8eb209c5f5083a04423be023e9ad4d5125ef4d33 GIT binary patch literal 25392 zcmeHP2Ut``*S-tOE?YoQEa-wOt_BqiD%FB?1Qk((sIh<)OZaF4f&^U+A_|uHMa8HQ zjTMc-8e=y$jJ@R((P$D98)`_fWBAW|cQ35##c0&DpW!*TOgl4YX3lxf+*wNc4o=HC zzp~f5_5D&#wwimUi~E#*c~gGzUF$Tik^l}k^-KsxCrf?i|3-%LzcIBFx|>Gf4y_rpk{A}HgDkB)kURE71G^U=Naz^bVof1 zc(o75wK@#EbMo`8@dMqLX2;(@6g6T|7j4pIyB38;v(kI4{U&+P)T@5?s!Kfkl(=O{ z9bI}hM{P{41pfIF*k6AC?)q{4mzin1VjRuPQ43TF$NF?XedzPv+XgFRZr++_D4=9v zs9Mbzk0P54RON1sA}j-U$-8#Ws%&^JTY)086wDk8V{SSB^QBeWgIWpvb0wg$m>Qee zV_U~b`|5v{&8>T^R?pk>E%9zZWKM)QkZ4ZVdU!{CU2PoVFMSv0h<7njJtCND#D8$Z zqd($hL>5GHBE&!1Jen_xqm~AHh4GLBDJ|J{s?6 z58j6#F1d(zuD#R$XC(q3^sNaI$NxmM(R&Jd2Osc-Oi=#J_IIYBJm3r7l@2ox;oV64 z?lI$tGu}no_l13@G)DdCEASK7pCvn-e?dHWSQD8Lp}sMmVqfGz{Y|xTh~KxY_ci2K zYKd#7_lkh^$j9|Bv+g`!ymS5QIcLWECS-y8V!BPSoMX9SBl zeh`E->q52KeC%Y?`yr0qceI>PC~wnVf_cvuH1P!lolXl>ex*-FM??h&Zp=NH`QsF$ znMlltU~+UD#8xtUtAqIutKc|Eq}2BbHiV*fK%9#%778mILXW@Bj{^2&_FsCO|}? zKm;j?p@9WksWiR(4TA3QbMmG>sHN0y!z!686^!b6a@PnR*D9h1Aro%J*m2|L>AsuV`QL=Q{1eE4RnyV3cx$%ZAMZkFXU9~%RVxEtn)up%U?MV#EMOoePKJZ(C zuVqS5;Iy}g^*dI!s~qzHdGtLH#G^$*S^1$m4GYD>vABp3^0nm2~*M7k#sOQ~i|B*Dv1B);+Hi5(rg|-!Ex; zGI(*Fh7B^xx7go2f<#_U&`lhto}Q{tisZvhTissj?X(S za5Hc_*JGPJ?Y{VTIiZ!cp;iL_7709W{lh;(06x>#b66Fj+H*dJ<;s zMHEQXn@B)pK=cXGVxl~vB}7S9jN$>&Yz1Q+^W^irg135D~Y`x zKTPIy;b6&a`{u_Uie_HK&Wf3{M z$Y%(~y4H~rUQ;j|uBk{#Y&biMW>*nCycz$x9dYqxMls<`B3(hTOcGfOB@@R)Ghy@| zL(hp!B$Gl{KG)~yww)@7+zSg(B z36ns)#NZWh2}&P3derUZCBU-Ano-N>BB|JxEo#%>X; z8wFq|3hs0#qMCv{KAvyM_6Y4FU9X1P13@i_SU#y%j538tLb3oK;3(A1)=#-uM6QNe zfPE0NW$YXn=A=g~Q6P!mn)LV_{1d+7KrF%9z#{5sb``(5-jt=pY5&gbM zd~C^4+peREe)ux`*0KiXdv7B81CdlRU`^`W#S^1`9I@P1ZU0WXraiUSS_#xj;PptL zCVObiG_Yoj^Bovt3%YkET_~_(tmlHC)qUnm9(#W1%fY=j;>R?gHXYF+Zrw{ zgmtPt_BRLA*v8ggY9&xBf!84cK0S{$Uq$rrX8c+g;=xMB=%yq_&p5sTX z4aD6?)Lzt9WGi}$KG?sghpUkgr_XSP3IY0_@57_6zJFbQZb4>Wp=gpsqf*g6Z?8aSE5*TBoV7`tg;DkC7#Y=F2t zSU?PLTpiR>-TGLpu3MjeT~(O<53~biGfjzgKZ51|p=%OtNV0*`yWH=Apm+H;>UH(* zi{fnHZ<>H{YSZ9v4sK^TYGi^5t&g)I zAIxtQ?kdv3>WHuxnLG`=>xrzM2HplD^IG7om2T~=Rsw%Z3Gm@;QWB)62?x;PfBk#? z@nnmM1BKVI*Z(Ruox1mucn58VMr85Mcb*sNQE$&bL2=dn%!6IPJML@pTf0*Z4c?(U zz6atfQq6b=^|L&{guyuALn3xfp1<#pG7=$vhoQUgA3FwL0AeB|l(9*P(W7D_UvdAR zvzy?90K3Weomtmz>hD8w`%^;#B7$AF1RL2-$XwggXPN_NH}&l-XE#OBe!Y5k^6Q@W zR*J3H&l@l`*?lY8b%J=?6n^cd|O9w+LiV3dz(->-rQxM{K#WBic5^>E*8@Ezv_r2)5w(LI62a!r1I7tlTG zC3<=?XVGOr{Zr;S`cy3PGlYD%1K5Q>$3l!Y->(}jvE>1H&i!=j`?bDhHb zl$C~DY+>2u>(o^xB`zDAB=+$b+Cw-AdAfx!SX~ln=U35H^yvJwk=~hB-l=A1+BrWF zPyXQgr<-TFQ=XWWYZ3-#j=eg)^?O+vt@fL3@7jLTwBk#TL&s*F?d2mK;Vs^5?I|fP zvpUevdj9=+&ANBY^Vz5#6aU_TWFxo2pk%DDa)Z4^zD~P*iYBaFGhw$~-T0^jZX6{u_Gx_3Lc3i=;r|uh!XU!_@{q?2K$_?z3$F^j@-YpRDyp zq+XaB6zFFnZ47ak_;>g2OLq*cfZe{|tibwaWd7bB@1$G^uc&Ow7~B-xo$3G04T@&` zeFDALd4^81S@}3VwT>)cd%dZ%`+L5p;`CorWTNm9ee|O(c3aEvdjkST_Siba>GZ_} zvrC7pQ%#>ooP5cw~4Z+X|VhrmBD3l~4b1 z@=tCP>#loVMvT^3$Nla_c&62jtm)wT-h6e0Bq4#%DJ$Gi$ONb@Gl29E{5X z*3Xo9IR%MFV-j$_kG7d_M4u+sDMRXxgHHy5{2)ep4 zX|xULM1LLW*FmmK2gZeI%Q!GTjHC9+lhP39NmoZYfY_FDI?=rYkr$mm>_Fx0DZdw! z&QL4p2u>W;J%Yi{kpe>^8*a~a?4EN-VExc+8FqbGH}9(0T2A8Bm_E397sxnLTJ`sp zGh{v~Li5oEIda9Fp%l%ZXDuncHq=Vs|B(dv^dl|8D!+=g)wI%HzpWlawi?U*uea5t zRMaw3+OikkPc9~W{PGwmKVRgI{<8c?rZ0AW3vz^(@+iB3fP!BP`8p8) zC^9Shr!?Erm#jqk3!{{gv^^S0rU9ok6WP<68maK1;Xhj6cxs0i(}@{Or$l3jTU+83 zNvA~%3#dNi$L?D${5U10R!?aEGu1~b)VDLogKB|J1*xOnlGqx(ELFkIm5yxV=Y z_sB1bZ5mq^8wXBqwd#hog;ct5VNTuvJFn0BXP1Vo37!>c*|Xo2?Cc)f)=tU`^bS~M zc=kqld7IKZyUkaeS-rV?LGRp$@4MAmYU{bW+@kUHgJOqP%G1RQLw~wnKeuJGkdSVL zWo08eRHpWl?=vn>8ZzQhVzi55_TdQ6M^APYMLW%mD?7A)QGu0xY}kMWX(uI@vLDpr0g9Sg_oBL?*-gjNPyf3~x&Z_mCX>%-Bzca1fJw=)N$LSko(*-$u>d3=a!zGd#ztu{D##didUW)mC@+L#O)DRie|{OQcTaw za~IkbSg!xg(DM6kJDoj}Yz+tGeee2ni2apiv+IdVS34J-k2pA0(EgPC=R3vzEhZ=) zY&^Jlbgp1(M4l)!Y+k?KTh^QBjDIWnTD`W@=4V6-(sN4ft}OaG`?t_2m(9ut87=1v zK3JU`UH;f|xSO?I+Sl$a|GU4@o)h->9dkN;*-%^&kYzc`U6ed_*q7JuOt@nhymx0p z<>4ZitYb{fFT%1%TQ1r9PPq7F#@u0Mo03|T?z>|>y3qK-w{z^`m$**avaUjO(7$}j z?8?5rG)-&tXY~E3Hpr|O?rh(!v!xW4m>3p5@;{L=iOQ(h1Z7NE;;7_EWq52tT+*0! zOilh$Gi{yl2lx){>-qzHxF4&Q5BFo)^7#*>W|sYOvm`2{_5whh@MpPl(Fc)m_Q;pafA5P=$@|D6T$0YBCspvF7Z)LzThK^up>UX;U`@`Jqt<>^qxR^x`icW5H% z07L4?`VNS&kcedzQSDtAOXT*(2gulcl6`==EC80s+MrN z)PuxWjnj!w%{U!(v-N`k>pz%OwJU`7=m!uDwPLcO2#2TZU~wh=MuTe?tPuXYl)*Z` zsoioj^130yfK}+Im93fX6evM$O7%A&GN|&2P!Eb|_?uU?7d&)-fAS%n!{B(bviMI7 zuPmPa-c-M;KEB_Xpgy)vwv}k#i+yiG8Ho_R?im@Cs0@#cW`8K9*I#;7TUPP^-Tv_I zcSA0^<7rh@-|O!W)?(jLBSzNLA#Q)u292TsJj3^^z&{hueD9UDugCqJLRJ2%Uw7o+ zI{}=7m0lChxlz1i)5QCqdFCu{YWLfzDZ4OaFuY$&r4ViPXR>`#F~+Xi5^s}AFmY9^ zt!DPnZ^+p|QTliW^5N<&kV-KHtLt4Y6=RN9*PADmU{Pr5MN<|^g;<5nt3bglsT>yG zrYd5pR1E!5T}Qf94#imw3WiFX%1I2hpg=oHwYORcyjBVPwRk4r(v{bUXNHqx{@96c zCGAV$a}DlF+GmTT|C6Dl-LzQR(Tb#}vFuX>?OY*n_M~Z0P^osVEKEp7n`592pReEu;0#~q(P%&19YGSAu%rUv`LsFx=) zS$p24LKKwLh-wW>5nDC?oR4o_P}Y|`XM^!YFIk)ecF&x%d9`!JsK5GLG1acF`TxlM zIkV$semTI+BNZzc%{)4A+jsJgV424km9Fy!%Z%CoSE4(Q^zi`Z7bMI#zc7!UJ-@0M z|MNpMOh?iUev8KB(hDYs`%q0H4rNvUvP+woqD?G7B9Z2&^z^{a9KLx&Y3i3he~i8H z4G^dP5Az5<@Sh(S&;t4WGvt^v65*dW8+1>b;}?NIbr>;_CUEnhh4vl#BrJXO?R&y+ zz5jc0s7t@qs{P2#%>z8Y=y+p(KyNfW=UTUYt2FNo;hwY4Wi(u4OUwMgCML@)loBZf z5kY>M16cI3X2mSmj+VnZ^otrGx^Q-K3(SkqP)eY1lBoI!GQDV(20ihOKi5B;x5u2c zE@aemCo+p4Ca?8aBTS#Ya!c?7^oj4^d1$0mFpHUWWiVx-@PLR<+h)WuS-RZW%ydcJ t`d~iF+YZyjT%#M`DyykH{@WqS+lOh&U!fb{=+snR^FItVR)AkV{|B}&S-t=O literal 0 HcmV?d00001 diff --git a/corpus/table-growth/cold/commands.jsonl b/corpus/table-growth/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8893c45740ddd1c767d2ab26e9acd34491931144 --- /dev/null +++ b/corpus/table-growth/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-6162C217 -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/table-growth/cold/machine.json b/corpus/table-growth/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..bee592ada4b229be5548ec95aa387b2b7fdfc7de --- /dev/null +++ b/corpus/table-growth/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-6162c217", "hostname": "ONE-M6-6162C217"} diff --git a/corpus/table-growth/cold/notebook/Open Notebook.onetoc2 b/corpus/table-growth/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..a33d3bfd210f844e3db618e648ac898adeee09c6 GIT binary patch literal 4816 zcmZ3e|KIuM`!b(-j|5(YE0x7J#4gqT(JSk=YNPqbC`J7Q1~g!Qcb0(k#IK%9FNjZk zdt%)UX`nnZhAICK1RypWOaMy5#LF~y)9Oxb$#>r}|Kyh?&t}bosmDf_Y+J~WjIpU16&nqK zp&kN9zI^|>u_|nd0MsKKISf-97*&4foBm+C_tSH2N_*D3xv@JKfzm7q4Gq}Dhk6i< zx_>kTMnni8*O}hCjGGRN9_ohWdr$_yJe4T`#AgCxK_CW+fpB`~EME{GBo3--Vd6H) z_ZEZrTu?F7Y>{LTAEsuiuGdl!TLvnx-CJk`WR8N-5Eu=CK@tM6`X7|PL4XHGB`+Yu zFm(x|3PjN`C9#!{ufV&YS{i30vEq40iLJuR~o}CUeKQ!^!fyzp3;DeQC z>=&E6j1YN6k%k5a6=2YVlqMGh%ncLkbZefpL}|54*a8iKhK3%XT&?Jgw6K&`@AZW{ zcSu}5J`to<*Mp(=1S2D?1O}GPEh!8=$Q}Wi2mSTaM{uf|q0Lu_fMDRhC zvM?~f>JOku15~tukzt<;xMA@Dj5mMuU0$Qf@C)Q}pzcvH z8UlkT1dxjxSh)a9#^BP130x+8lVMN+l?$Mx4+jk_3{zwoWOf3}jBFqcDmy@}PaB8! z;05iU+}4;?89zwhcNL@u)PgK#sAR}vC}GH8NClFa49N_74EaEjRG=Kl1waZ20OIF{ A&Hw-a literal 0 HcmV?d00001 diff --git a/corpus/table-growth/cold/notebook/synthetic.one b/corpus/table-growth/cold/notebook/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..8eb209c5f5083a04423be023e9ad4d5125ef4d33 GIT binary patch literal 25392 zcmeHP2Ut``*S-tOE?YoQEa-wOt_BqiD%FB?1Qk((sIh<)OZaF4f&^U+A_|uHMa8HQ zjTMc-8e=y$jJ@R((P$D98)`_fWBAW|cQ35##c0&DpW!*TOgl4YX3lxf+*wNc4o=HC zzp~f5_5D&#wwimUi~E#*c~gGzUF$Tik^l}k^-KsxCrf?i|3-%LzcIBFx|>Gf4y_rpk{A}HgDkB)kURE71G^U=Naz^bVof1 zc(o75wK@#EbMo`8@dMqLX2;(@6g6T|7j4pIyB38;v(kI4{U&+P)T@5?s!Kfkl(=O{ z9bI}hM{P{41pfIF*k6AC?)q{4mzin1VjRuPQ43TF$NF?XedzPv+XgFRZr++_D4=9v zs9Mbzk0P54RON1sA}j-U$-8#Ws%&^JTY)086wDk8V{SSB^QBeWgIWpvb0wg$m>Qee zV_U~b`|5v{&8>T^R?pk>E%9zZWKM)QkZ4ZVdU!{CU2PoVFMSv0h<7njJtCND#D8$Z zqd($hL>5GHBE&!1Jen_xqm~AHh4GLBDJ|J{s?6 z58j6#F1d(zuD#R$XC(q3^sNaI$NxmM(R&Jd2Osc-Oi=#J_IIYBJm3r7l@2ox;oV64 z?lI$tGu}no_l13@G)DdCEASK7pCvn-e?dHWSQD8Lp}sMmVqfGz{Y|xTh~KxY_ci2K zYKd#7_lkh^$j9|Bv+g`!ymS5QIcLWECS-y8V!BPSoMX9SBl zeh`E->q52KeC%Y?`yr0qceI>PC~wnVf_cvuH1P!lolXl>ex*-FM??h&Zp=NH`QsF$ znMlltU~+UD#8xtUtAqIutKc|Eq}2BbHiV*fK%9#%778mILXW@Bj{^2&_FsCO|}? zKm;j?p@9WksWiR(4TA3QbMmG>sHN0y!z!686^!b6a@PnR*D9h1Aro%J*m2|L>AsuV`QL=Q{1eE4RnyV3cx$%ZAMZkFXU9~%RVxEtn)up%U?MV#EMOoePKJZ(C zuVqS5;Iy}g^*dI!s~qzHdGtLH#G^$*S^1$m4GYD>vABp3^0nm2~*M7k#sOQ~i|B*Dv1B);+Hi5(rg|-!Ex; zGI(*Fh7B^xx7go2f<#_U&`lhto}Q{tisZvhTissj?X(S za5Hc_*JGPJ?Y{VTIiZ!cp;iL_7709W{lh;(06x>#b66Fj+H*dJ<;s zMHEQXn@B)pK=cXGVxl~vB}7S9jN$>&Yz1Q+^W^irg135D~Y`x zKTPIy;b6&a`{u_Uie_HK&Wf3{M z$Y%(~y4H~rUQ;j|uBk{#Y&biMW>*nCycz$x9dYqxMls<`B3(hTOcGfOB@@R)Ghy@| zL(hp!B$Gl{KG)~yww)@7+zSg(B z36ns)#NZWh2}&P3derUZCBU-Ano-N>BB|JxEo#%>X; z8wFq|3hs0#qMCv{KAvyM_6Y4FU9X1P13@i_SU#y%j538tLb3oK;3(A1)=#-uM6QNe zfPE0NW$YXn=A=g~Q6P!mn)LV_{1d+7KrF%9z#{5sb``(5-jt=pY5&gbM zd~C^4+peREe)ux`*0KiXdv7B81CdlRU`^`W#S^1`9I@P1ZU0WXraiUSS_#xj;PptL zCVObiG_Yoj^Bovt3%YkET_~_(tmlHC)qUnm9(#W1%fY=j;>R?gHXYF+Zrw{ zgmtPt_BRLA*v8ggY9&xBf!84cK0S{$Uq$rrX8c+g;=xMB=%yq_&p5sTX z4aD6?)Lzt9WGi}$KG?sghpUkgr_XSP3IY0_@57_6zJFbQZb4>Wp=gpsqf*g6Z?8aSE5*TBoV7`tg;DkC7#Y=F2t zSU?PLTpiR>-TGLpu3MjeT~(O<53~biGfjzgKZ51|p=%OtNV0*`yWH=Apm+H;>UH(* zi{fnHZ<>H{YSZ9v4sK^TYGi^5t&g)I zAIxtQ?kdv3>WHuxnLG`=>xrzM2HplD^IG7om2T~=Rsw%Z3Gm@;QWB)62?x;PfBk#? z@nnmM1BKVI*Z(Ruox1mucn58VMr85Mcb*sNQE$&bL2=dn%!6IPJML@pTf0*Z4c?(U zz6atfQq6b=^|L&{guyuALn3xfp1<#pG7=$vhoQUgA3FwL0AeB|l(9*P(W7D_UvdAR zvzy?90K3Weomtmz>hD8w`%^;#B7$AF1RL2-$XwggXPN_NH}&l-XE#OBe!Y5k^6Q@W zR*J3H&l@l`*?lY8b%J=?6n^cd|O9w+LiV3dz(->-rQxM{K#WBic5^>E*8@Ezv_r2)5w(LI62a!r1I7tlTG zC3<=?XVGOr{Zr;S`cy3PGlYD%1K5Q>$3l!Y->(}jvE>1H&i!=j`?bDhHb zl$C~DY+>2u>(o^xB`zDAB=+$b+Cw-AdAfx!SX~ln=U35H^yvJwk=~hB-l=A1+BrWF zPyXQgr<-TFQ=XWWYZ3-#j=eg)^?O+vt@fL3@7jLTwBk#TL&s*F?d2mK;Vs^5?I|fP zvpUevdj9=+&ANBY^Vz5#6aU_TWFxo2pk%DDa)Z4^zD~P*iYBaFGhw$~-T0^jZX6{u_Gx_3Lc3i=;r|uh!XU!_@{q?2K$_?z3$F^j@-YpRDyp zq+XaB6zFFnZ47ak_;>g2OLq*cfZe{|tibwaWd7bB@1$G^uc&Ow7~B-xo$3G04T@&` zeFDALd4^81S@}3VwT>)cd%dZ%`+L5p;`CorWTNm9ee|O(c3aEvdjkST_Siba>GZ_} zvrC7pQ%#>ooP5cw~4Z+X|VhrmBD3l~4b1 z@=tCP>#loVMvT^3$Nla_c&62jtm)wT-h6e0Bq4#%DJ$Gi$ONb@Gl29E{5X z*3Xo9IR%MFV-j$_kG7d_M4u+sDMRXxgHHy5{2)ep4 zX|xULM1LLW*FmmK2gZeI%Q!GTjHC9+lhP39NmoZYfY_FDI?=rYkr$mm>_Fx0DZdw! z&QL4p2u>W;J%Yi{kpe>^8*a~a?4EN-VExc+8FqbGH}9(0T2A8Bm_E397sxnLTJ`sp zGh{v~Li5oEIda9Fp%l%ZXDuncHq=Vs|B(dv^dl|8D!+=g)wI%HzpWlawi?U*uea5t zRMaw3+OikkPc9~W{PGwmKVRgI{<8c?rZ0AW3vz^(@+iB3fP!BP`8p8) zC^9Shr!?Erm#jqk3!{{gv^^S0rU9ok6WP<68maK1;Xhj6cxs0i(}@{Or$l3jTU+83 zNvA~%3#dNi$L?D${5U10R!?aEGu1~b)VDLogKB|J1*xOnlGqx(ELFkIm5yxV=Y z_sB1bZ5mq^8wXBqwd#hog;ct5VNTuvJFn0BXP1Vo37!>c*|Xo2?Cc)f)=tU`^bS~M zc=kqld7IKZyUkaeS-rV?LGRp$@4MAmYU{bW+@kUHgJOqP%G1RQLw~wnKeuJGkdSVL zWo08eRHpWl?=vn>8ZzQhVzi55_TdQ6M^APYMLW%mD?7A)QGu0xY}kMWX(uI@vLDpr0g9Sg_oBL?*-gjNPyf3~x&Z_mCX>%-Bzca1fJw=)N$LSko(*-$u>d3=a!zGd#ztu{D##didUW)mC@+L#O)DRie|{OQcTaw za~IkbSg!xg(DM6kJDoj}Yz+tGeee2ni2apiv+IdVS34J-k2pA0(EgPC=R3vzEhZ=) zY&^Jlbgp1(M4l)!Y+k?KTh^QBjDIWnTD`W@=4V6-(sN4ft}OaG`?t_2m(9ut87=1v zK3JU`UH;f|xSO?I+Sl$a|GU4@o)h->9dkN;*-%^&kYzc`U6ed_*q7JuOt@nhymx0p z<>4ZitYb{fFT%1%TQ1r9PPq7F#@u0Mo03|T?z>|>y3qK-w{z^`m$**avaUjO(7$}j z?8?5rG)-&tXY~E3Hpr|O?rh(!v!xW4m>3p5@;{L=iOQ(h1Z7NE;;7_EWq52tT+*0! zOilh$Gi{yl2lx){>-qzHxF4&Q5BFo)^7#*>W|sYOvm`2{_5whh@MpPl(Fc)m_Q;pafA5P=$@|D6T$0YBCspvF7Z)LzThK^up>UX;U`@`Jqt<>^qxR^x`icW5H% z07L4?`VNS&kcedzQSDtAOXT*(2gulcl6`==EC80s+MrN z)PuxWjnj!w%{U!(v-N`k>pz%OwJU`7=m!uDwPLcO2#2TZU~wh=MuTe?tPuXYl)*Z` zsoioj^130yfK}+Im93fX6evM$O7%A&GN|&2P!Eb|_?uU?7d&)-fAS%n!{B(bviMI7 zuPmPa-c-M;KEB_Xpgy)vwv}k#i+yiG8Ho_R?im@Cs0@#cW`8K9*I#;7TUPP^-Tv_I zcSA0^<7rh@-|O!W)?(jLBSzNLA#Q)u292TsJj3^^z&{hueD9UDugCqJLRJ2%Uw7o+ zI{}=7m0lChxlz1i)5QCqdFCu{YWLfzDZ4OaFuY$&r4ViPXR>`#F~+Xi5^s}AFmY9^ zt!DPnZ^+p|QTliW^5N<&kV-KHtLt4Y6=RN9*PADmU{Pr5MN<|^g;<5nt3bglsT>yG zrYd5pR1E!5T}Qf94#imw3WiFX%1I2hpg=oHwYORcyjBVPwRk4r(v{bUXNHqx{@96c zCGAV$a}DlF+GmTT|C6Dl-LzQR(Tb#}vFuX>?OY*n_M~Z0P^osVEKEp7n`592pReEu;0#~q(P%&19YGSAu%rUv`LsFx=) zS$p24LKKwLh-wW>5nDC?oR4o_P}Y|`XM^!YFIk)ecF&x%d9`!JsK5GLG1acF`TxlM zIkV$semTI+BNZzc%{)4A+jsJgV424km9Fy!%Z%CoSE4(Q^zi`Z7bMI#zc7!UJ-@0M z|MNpMOh?iUev8KB(hDYs`%q0H4rNvUvP+woqD?G7B9Z2&^z^{a9KLx&Y3i3he~i8H z4G^dP5Az5<@Sh(S&;t4WGvt^v65*dW8+1>b;}?NIbr>;_CUEnhh4vl#BrJXO?R&y+ zz5jc0s7t@qs{P2#%>z8Y=y+p(KyNfW=UTUYt2FNo;hwY4Wi(u4OUwMgCML@)loBZf z5kY>M16cI3X2mSmj+VnZ^otrGx^Q-K3(SkqP)eY1lBoI!GQDV(20ihOKi5B;x5u2c zE@aemCo+p4Ca?8aBTS#Ya!c?7^oj4^d1$0mFpHUWWiVx-@PLR<+h)WuS-RZW%ydcJ t`d~iF+YZyjT%#M`DyykH{@WqS+lOh&U!fb{=+snR^FItVR)AkV{|B}&S-t=O literal 0 HcmV?d00001 diff --git a/corpus/table-growth/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment b/corpus/table-growth/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment new file mode 100644 index 0000000000000000000000000000000000000000..66b6151e91f90df6a61a19ef2f4447da278e07b2 --- /dev/null +++ b/corpus/table-growth/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment @@ -0,0 +1 @@ +Fictitious attachment for native corpus. \ No newline at end of file diff --git a/corpus/table-growth/cold/read/environment.json b/corpus/table-growth/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..e5b11de4c42b83cd944d0715a488c8e0085e4110 --- /dev/null +++ b/corpus/table-growth/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-6162C217", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/table-growth/cold/read/hierarchy.xml b/corpus/table-growth/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..822dc11be592167eb3313b9346ccdae45fa17db1 --- /dev/null +++ b/corpus/table-growth/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/table-growth/cold/read/page-000.xml b/corpus/table-growth/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..bf1a56b3a78d6ecc8650f8af1220b40d6c579bfb --- /dev/null +++ b/corpus/table-growth/cold/read/page-000.xml @@ -0,0 +1,9 @@ + +Fictitious: café, 東京, مرحبا and a few more words]]>iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9J1uoA +AAAASUVORK5CYII= diff --git a/corpus/table-growth/cold/read/payloads.json b/corpus/table-growth/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..8ca5f91620fbeeeced9de312b5661b60431951a2 --- /dev/null +++ b/corpus/table-growth/cold/read/payloads.json @@ -0,0 +1,10 @@ +[ + { + "sha256": "af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7", + "name": "fictitious-attachment.txt", + "object": "{DE1684DA-6810-4671-8EAF-CD3C3735DDC5}{11}{B0}", + "kind": "InsertedFile", + "page": "{5759809C-B0D2-45F2-8836-0F65B4195B50}{1}{B0}", + "bytes": 43 + } +] \ No newline at end of file diff --git a/corpus/table-growth/cold/run.json b/corpus/table-growth/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..ad0e4f9185c3c09c0088cfc44e4d66a0b7364e8f --- /dev/null +++ b/corpus/table-growth/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m5/tables/notebook", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/table-growth/cold/scripts/cold.ps1 b/corpus/table-growth/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/table-growth/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/table-growth/cold/scripts/read.ps1 b/corpus/table-growth/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/table-growth/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/table-growth/cold/source.json b/corpus/table-growth/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..9eb5dd19f602ec69733d3f0d450cac0431e96167 --- /dev/null +++ b/corpus/table-growth/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 4816, + "sha256": "b2f857961b76258a1582b402c048b340e89962257681426b3bfc58df8e0adbfa", + "mtime_ns": 1788585005062916740 + }, + { + "path": "synthetic.one", + "bytes": 25392, + "sha256": "9acb9ab3c06daa698c55ca48bc2c100aba4609e9f546839a0aeb99db68296033", + "mtime_ns": 1789887306088599720 + } +] diff --git a/corpus/table-growth/cold/teardown.json b/corpus/table-growth/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/table-growth/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/crates/notebook/README.md b/crates/notebook/README.md index 90b37cc4d67c08f711224afc9273858290938e88..930e821b6c816c43a00c904400408fd3c1e30daf 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -348,10 +348,11 @@ when the page itself was moved to another section. Other URLs and unknown targets are `None`. With the optional `protected` feature, `Notebook::unlock(path, password)` -reads the pages of a `Locked` section, and `Notebook::save_unlocked(path, -password, space, page, author)` saves an edited one under the section's key, -straight to the file: nothing of a protected section is cached or queued, a -section changed since the read fails the save, and a wrong password is +reads a `Locked` section as `Unlocked { pages, .. }`, and +`Notebook::save_unlocked(path, password, &mut unlocked, space, page, author)` +saves an edited page under the section's key, straight to the file: nothing of +a protected section is cached or queued, a section written since `unlocked` +was read fails the save, and a wrong password is `Error::Protected(PasswordMismatch)`. `Section::import_page(page, author)` copies a page, usually read from another diff --git a/crates/notebook/examples/cache_probe.rs b/crates/notebook/examples/cache_probe.rs index e891b5fb00af48044cf05260213e0d34bbcbc847..f4cc24d21d3d0aeca47d816a700e41af600c6fb7 100644 --- a/crates/notebook/examples/cache_probe.rs +++ b/crates/notebook/examples/cache_probe.rs @@ -110,8 +110,9 @@ fn main() -> Result<(), Box> { !intent.before.objects.iter().any(|o| o.id() == object.id()) }) .collect(); - let [PageObject::Outline(outline)] = added.as_slice() else { - panic!("Expected one added outline") + // A save that replaced a pending one carries its outlines too. + let Some(PageObject::Outline(outline)) = added.last() else { + panic!("Expected an added outline") }; let text = outline.paragraphs[0].text().unwrap().text.text(); let operation: u64 = text.split_once(':').unwrap().0.parse()?; diff --git a/crates/notebook/src/assets.rs b/crates/notebook/src/assets.rs index d176734210a5156d5d41c39184272841ab989c3f..91f8a5b32f0d429f1ea4e5be248ca15e522b3dd6 100644 --- a/crates/notebook/src/assets.rs +++ b/crates/notebook/src/assets.rs @@ -79,8 +79,9 @@ pub(crate) fn key(filename: &str) -> Result { } fn referenced(connection: &Connection, key: &str) -> Result { - let (base, working) = crate::images::both(connection)?; - for image in [working, base] { + // One image at a time: the working image usually answers. + for read in [crate::images::working, crate::images::base] { + let image = read(connection)?; let store = Store::parse(&image)?; let index = RevisionIndex::parse(&store)?; let document = Document::parse(&index)?; diff --git a/crates/notebook/src/images.rs b/crates/notebook/src/images.rs index 87b2c6111b2eefb5acf39dc8b8fee69074866ca5..bf1e41265409a934469dd33e26afbe7b851e3ba1 100644 --- a/crates/notebook/src/images.rs +++ b/crates/notebook/src/images.rs @@ -46,10 +46,12 @@ fn restore(mut base: Vec, patch: &[u8]) -> Result> { Err(damaged().into()) }; }; - base.resize( - usize::try_from(u64::from_le_bytes(*length)).map_err(|_| damaged())?, - 0, - ); + // Every block past the base is a run, so a whole patch bounds the image. + let length = usize::try_from(u64::from_le_bytes(*length)) + .ok() + .filter(|length| *length <= base.len() + patch.len()) + .ok_or_else(damaged)?; + base.resize(length, 0); while let Some((header, rest)) = runs.split_first_chunk::<16>() { let offset = usize::try_from(u64::from_le_bytes(header[..8].try_into().unwrap())) .map_err(|_| damaged())?; @@ -83,6 +85,21 @@ pub(crate) fn both(connection: &Connection) -> Result<(Vec, Vec)> { Ok((base, working)) } +/// The working image's length, without restoring it. +pub(crate) fn working_length(connection: &Connection) -> Result { + Ok(connection.query_row( + "SELECT length(base), substr(working, 1, 8) FROM replica WHERE id=1", + [], + |row| { + let header: Vec = row.get(1)?; + Ok(match header.first_chunk::<8>() { + Some(length) => u64::from_le_bytes(*length), + None => row.get::<_, i64>(0)? as u64, + }) + }, + )?) +} + pub(crate) fn working(connection: &Connection) -> Result> { let (base, patch): (Vec, Vec) = connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { @@ -141,6 +158,9 @@ mod tests { } assert!(difference(&base, &base).is_empty()); assert!(difference(&base, &edited).len() < 3 * BLOCK); + let mut huge = difference(&base, &edited); + huge[..8].copy_from_slice(&u64::MAX.to_le_bytes()); + assert!(restore(base.clone(), &huge).is_err()); let patch = difference(&base, &edited); for cut in 1..patch.len() { if let Ok(image) = restore(base.clone(), &patch[..cut]) { diff --git a/crates/notebook/src/recovery.rs b/crates/notebook/src/recovery.rs index 499aefd668821636be875d7b95e793795bcf2663..8d1b28c808528026e99ff249bde794e92feaa7be 100644 --- a/crates/notebook/src/recovery.rs +++ b/crates/notebook/src/recovery.rs @@ -144,7 +144,7 @@ fn summary(connection: &Connection) -> Result { [], |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)), )?; - let working_bytes = crate::images::working(connection)?.len() as u64; + let working_bytes = crate::images::working_length(connection)?; Ok(connection.query_row( "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts), (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts), diff --git a/crates/notebook/src/session.rs b/crates/notebook/src/session.rs index 4f36018af75e14d1ffe398d096b92b463ecb2dd0..84842e5dbeda6faa3342228bd00393d1751c13b1 100644 --- a/crates/notebook/src/session.rs +++ b/crates/notebook/src/session.rs @@ -91,6 +91,14 @@ pub trait Storage: Send + Sync { ) -> Result<()>; } +/// A password-protected section as `Notebook::unlock` read it. +#[cfg(feature = "protected")] +pub struct Unlocked { + pub pages: Vec<(ExGuid, Page)>, + /// The stored section the pages came from, which a save must still find in place. + snapshot: Vec, +} + /// A mounted notebook directory. struct Directory(PathBuf); @@ -622,43 +630,55 @@ impl Notebook { Ok(None) } - /// The pages of a password-protected section, for reading: nothing is cached or - /// written, and the decoded buffers go when the pages have been built. + /// The pages of a password-protected section: nothing is cached, and the decoded + /// buffers go when the pages have been built. #[cfg(feature = "protected")] - pub fn unlock(&self, path: &str, password: &str) -> Result> { + pub fn unlock(&self, path: &str, password: &str) -> Result { let path = self.section_path(path)?.path.clone(); - let bytes = self.storage.read(&path)?; - let store = Store::parse(&bytes)?; - let index = RevisionIndex::parse(&store)?; - let unlocked = onestore::protected::UnlockedSection::open( - &index, - password, - onestore::protected::Limits::default(), - )?; - let document = unlocked.document()?; - document - .pages()? - .into_iter() - .map(|(space, _)| Ok((space, Page::from_space(&document, space)?))) - .collect() + let snapshot = self.storage.read(&path)?; + let pages = { + let store = Store::parse(&snapshot)?; + let index = RevisionIndex::parse(&store)?; + let unlocked = onestore::protected::UnlockedSection::open( + &index, + password, + onestore::protected::Limits::default(), + )?; + let document = unlocked.document()?; + document + .pages()? + .into_iter() + .map(|(space, _)| Ok((space, Page::from_space(&document, space)?))) + .collect::>()? + }; + Ok(Unlocked { pages, snapshot }) } - /// Saves a page read through `unlock`, stored under the section's key. The save goes - /// straight to the file and fails if the section changed since `unlock` read it; - /// nothing of a protected section is cached or queued. + /// Saves an edited page of `unlocked` under the section's key, straight to the file: + /// nothing of a protected section is cached or queued. A section written since + /// `unlocked` was read fails the save; unlock again for its pages. #[cfg(feature = "protected")] pub fn save_unlocked( &self, path: &str, password: &str, + unlocked: &mut Unlocked, space: ExGuid, page: &Page, author: &str, ) -> Result<()> { let path = self.section_path(path)?.path.clone(); - let bytes = self.storage.read(&path)?; - let edit = onestore::PreparedEdit::page_protected(&bytes, password, space, page, author)?; - self.storage.commit(&path, &edit) + let edit = onestore::PreparedEdit::page_protected( + &unlocked.snapshot, + password, + space, + page, + author, + )?; + self.storage.commit(&path, &edit)?; + let written = edit.as_bytes().to_vec(); + unlocked.snapshot = written; + Ok(()) } /// Opens a section of a mounted notebook by its catalog path. diff --git a/crates/notebook/src/sync.rs b/crates/notebook/src/sync.rs index 037b91b75c9fde1e9dec8d339d98d864bd911b37..ebd12be546cb0de0d75541a3ba5645cc42a974c9 100644 --- a/crates/notebook/src/sync.rs +++ b/crates/notebook/src/sync.rs @@ -225,6 +225,8 @@ impl Replica { self.acknowledge(intent.id, revision, &snapshot)?; return Ok(Some((intent.id, EditStatus::Published { revision }))); } + // Once acknowledged this image is the base, which polls trust without validating. + validate(prepared.as_bytes())?; let store = Store::parse(prepared.as_bytes())?; let index = RevisionIndex::parse(&store)?; let revision = index.active(intent.space)?; diff --git a/crates/notebook/tests/protected.rs b/crates/notebook/tests/protected.rs index 15d8864e358903324cb14de813d5972c3bbe601e..fc929e2d4425a6f7d974fe03537a03b2f3ba4757 100644 --- a/crates/notebook/tests/protected.rs +++ b/crates/notebook/tests/protected.rs @@ -17,7 +17,7 @@ fn a_locked_section_unlocks_for_reading() { let notebook = Notebook::open(root.join("notebook"), temporary.path()).unwrap(); let section = ¬ebook.catalog().sections[0]; assert!(matches!(section.state, SectionState::Locked)); - let pages = notebook.unlock(§ion.path, password).unwrap(); + let pages = notebook.unlock(§ion.path, password).unwrap().pages; assert_eq!(pages.len(), 11); assert!(pages.iter().all(|(_, page)| !page.title.is_empty())); assert!(matches!( @@ -57,7 +57,8 @@ fn an_unlocked_page_is_saved_under_the_section_key() { .unwrap() .path .clone(); - let (space, mut page) = notebook.unlock(§ion, password).unwrap().remove(0); + let mut unlocked = notebook.unlock(§ion, password).unwrap(); + let (space, mut page) = unlocked.pages[0].clone(); let paragraphs = page .objects .iter_mut() @@ -95,15 +96,22 @@ fn an_unlocked_page_is_saved_under_the_section_key() { )) .unwrap(); assert!(matches!( - notebook.save_unlocked(§ion, "wrong", space, &page, "Rust"), + notebook.save_unlocked(§ion, "wrong", &mut unlocked, space, &page, "Rust"), Err(notebook::Error::Protected( onestore::protected::Error::PasswordMismatch )) )); + let mut stale = notebook.unlock(§ion, password).unwrap(); notebook - .save_unlocked(§ion, password, space, &page, "Rust") + .save_unlocked(§ion, password, &mut unlocked, space, &page, "Rust") .unwrap(); - let (_, stored) = notebook.unlock(§ion, password).unwrap().remove(0); + // A view read before that save no longer matches the file. + assert!( + notebook + .save_unlocked(§ion, password, &mut stale, space, &page, "Rust") + .is_err() + ); + let (_, stored) = notebook.unlock(§ion, password).unwrap().pages.remove(0); assert!(stored.objects == page.objects); if let Some(export) = std::env::var_os("ONESTORE_PROTECTED_EXPORT") { let export = Path::new(&export); diff --git a/crates/onestore/src/objects.rs b/crates/onestore/src/objects.rs index c2546e7960f9aa8b97dc71323ffa500a00e5c531..d9e222669e66a16f1b9980ab60176c2174f0a6b2 100644 --- a/crates/onestore/src/objects.rs +++ b/crates/onestore/src/objects.rs @@ -335,7 +335,6 @@ impl<'a> RevisionIndex<'a> { }); } table = Arc::new(GlobalIds::new()); - defining.clear(); defining_table = true; } 0x24..=0x26 => { @@ -382,9 +381,12 @@ impl<'a> RevisionIndex<'a> { } _ => unreachable!(), } - if defining[start..] - .iter() - .any(|(index, guid)| *index >= 0xffffff || *guid == [0; 16]) + // More entries than indices repeats one; checked here so imports + // cannot grow the list without bound. + if defining.len() > 0xffffff + || defining[start..] + .iter() + .any(|(index, guid)| *index >= 0xffffff || *guid == [0; 16]) { return Err(Error { offset: node.offset, @@ -407,9 +409,8 @@ impl<'a> RevisionIndex<'a> { message: "Invalid or repeated global ID entry", }); } - let mut guids: Vec<_> = defining.iter().map(|(_, guid)| *guid).collect(); - guids.sort_unstable(); - if guids.windows(2).any(|pair| pair[0] == pair[1]) { + defining.sort_unstable_by_key(|(_, guid)| *guid); + if defining.windows(2).any(|pair| pair[0].1 == pair[1].1) { return Err(Error { offset: node.offset, message: "Global ID table repeats a GUID", diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index d1ae945263914f74833e831c410b1a93d66656a3..dccfc64e42300787552ceca13c1f40e507afa32c 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -2987,12 +2987,21 @@ pub(crate) fn squash( if !live.contains(id) { continue; } - if before.objects.get(id).is_some_and(|previous| { - previous.jcid == object.jcid - && previous.data == object.data - && previous.global_ids == object.global_ids - }) { - continue; + // Stored tables keep the entries an object names, so those decide. + if let Some(previous) = before.objects.get(id) + && previous.jcid == object.jcid + && previous.data == object.data + { + let entries = match object.data { + ObjectData::Properties(bytes) => crate::write::table_entries(bytes)?, + _ => BTreeSet::new(), + }; + if entries + .iter() + .all(|entry| previous.global_ids.get(entry) == object.global_ids.get(entry)) + { + continue; + } } let id = rename.get(id).copied().unwrap_or(*id); let mut replacement = match object.data { diff --git a/crates/onestore/src/protected/crypto.rs b/crates/onestore/src/protected/crypto.rs index b25e027b7aa2c47e986cca4ff33553705bdef534..fef881d6e82f13387a652a6e1af9fecaf23959e5 100644 --- a/crates/onestore/src/protected/crypto.rs +++ b/crates/onestore/src/protected/crypto.rs @@ -228,7 +228,9 @@ impl Key { crate::properties::reference_streams(&mut c)?; let prefix = c.offset; let padding = (16 - (2 + clear.len() - prefix) % 16) % 16; - let mut body = Zeroizing::new((padding as u16).to_le_bytes().to_vec()); + // Sized once: a buffer abandoned by growth would keep plaintext. + let mut body = Zeroizing::new(Vec::with_capacity(2 + clear.len() - prefix + padding)); + body.extend_from_slice(&(padding as u16).to_le_bytes()); body.extend_from_slice(&clear[prefix..]); let length = body.len() + padding; body.resize(length, 0); @@ -248,7 +250,8 @@ impl Key { if clear.is_empty() { return Vec::new(); } - let mut output = (clear.len() as u64).to_le_bytes().to_vec(); + let mut output = Vec::with_capacity((8 + clear.len()).next_multiple_of(16)); + output.extend_from_slice(&(clear.len() as u64).to_le_bytes()); output.extend_from_slice(clear); output.resize(output.len().next_multiple_of(16), 0); encrypt(&self.value, &self.file_iv, &mut output); diff --git a/crates/onestore/src/protected/mod.rs b/crates/onestore/src/protected/mod.rs index 9e8d38cb931f44897dec0ec1e7779fc784c95140..1c7a44ee47be7884bfa4db3477846f411d10b153 100644 --- a/crates/onestore/src/protected/mod.rs +++ b/crates/onestore/src/protected/mod.rs @@ -83,7 +83,7 @@ struct Decoded<'a> { /// Owns decoded buffers until dropped; returned views cannot outlive this owner. /// /// Passwords are not retained. Dropping this owner clears its derived key and -/// decoded buffers. Owned strings or exports made from a `Document` are separate +/// decoded buffers; a protected save works on ordinary buffers, which are not cleared. Owned strings or exports made from a `Document` are separate /// caller-owned copies and must be disposed of by the caller when locking. /// Opening never rewrites the source image or changes its protection state. /// @@ -302,15 +302,19 @@ impl UnlockedSection<'_> { extension, } => { let text = |bytes: &[u8]| { - String::from_utf16_lossy( + String::from_utf16( &bytes .chunks_exact(2) .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) .collect::>(), ) + .map_err(|_| crate::Error { + offset: 0, + message: "Invalid UTF-16 file-data declaration", + }) }; let mut copy = - PropertyObject::file(*id, &text(reference), &text(extension))?; + PropertyObject::file(*id, &text(reference)?, &text(extension)?)?; copy.jcid = object.jcid; copy.global_ids = std::sync::Arc::clone(&object.global_ids); copy @@ -336,7 +340,7 @@ impl UnlockedSection<'_> { }; let store = crate::Store::parse(&scaffold)?; let placeholder = identity(RevisionIndex::parse(&store)?.root); - for at in 0..scaffold.len() - 20 { + for at in 0..=scaffold.len() - 20 { if scaffold[at..at + 20] == placeholder { scaffold[at..at + 20].copy_from_slice(&identity(self.index.root)); } @@ -398,14 +402,32 @@ pub(crate) fn write_page( let store = crate::Store::parse(source)?; let index = RevisionIndex::parse(&store)?; let unlocked = UnlockedSection::open(&index, password, Limits::default())?; + if unlocked.keys.len() != 1 { + return Err(Error::Unsupported); + } let twin = unlocked.twin()?; let applied = Zeroizing::new(crate::page::write::write_page(&twin, space, page, author)?); + // Squash skips the spaces the source lacks as the twin's scaffold; the writers added none. + let spaces = |image: &[u8]| -> Result> { + let store = crate::Store::parse(image)?; + Ok(RevisionIndex::parse(&store)?.spaces.into_keys().collect()) + }; + if spaces(&twin)? != spaces(&applied)? { + return Err(invalid("Page edits cannot create object spaces")); + } let written = crate::page::write::squash(source, &applied, &BTreeMap::new(), Some(&unlocked))?; let store = crate::Store::parse(&written)?; UnlockedSection::open(&RevisionIndex::parse(&store)?, password, Limits::default())?; Ok(written) } +impl UnlockedSection<'_> { + /// The section key; `write_page` admits sections with exactly one. + fn key(&self) -> &crypto::Key { + self.keys.values().next().expect("one section key") + } +} + impl crate::write::Protection for UnlockedSection<'_> { fn resolve( &self, @@ -423,20 +445,21 @@ impl crate::write::Protection for UnlockedSection<'_> { } fn seal_property(&self, clear: &[u8]) -> std::result::Result, crate::Error> { - let [key] = self.keys.values().collect::>()[..] else { - return Err(crate::Error { - offset: 0, - message: "Protected writing needs one section key", - }); + let random = crate::Error { + offset: 0, + message: "System random source failed", }; - let failed = |message| crate::Error { offset: 0, message }; let mut iv = [0; 16]; - getrandom::fill(&mut iv).map_err(|_| failed("System random source failed"))?; - key.seal_property(clear, iv) - .map_err(|_| failed("Malformed property object")) + getrandom::fill(&mut iv).map_err(|_| random)?; + self.key() + .seal_property(clear, iv) + .map_err(|error| match error { + Error::Invalid(error) => error, + _ => random, + }) } fn seal_file(&self, clear: &[u8]) -> Vec { - self.keys.values().next().unwrap().seal_file(clear) + self.key().seal_file(clear) } } diff --git a/crates/onestore/src/write.rs b/crates/onestore/src/write.rs index ee0bbd413e10291cbe6f01babb4e634b03c04237..e31daef641e874ed30ad2f87bbeee6f108d73fb1 100644 --- a/crates/onestore/src/write.rs +++ b/crates/onestore/src/write.rs @@ -104,6 +104,21 @@ fn compact(id: ExGuid, table: &BTreeMap) -> Result<[u8; 4]> { Ok(((index << 8) | id.n).to_le_bytes()) } +/// The global id table entries a property object's references name. +pub(crate) fn table_entries(bytes: &[u8]) -> Result> { + let mut entries = BTreeSet::new(); + for property in PropertySets::parse(bytes)?.sets.iter().flatten() { + if let Value::References { compact_ids, .. } = property.value { + entries.extend( + compact_ids + .chunks_exact(4) + .map(|id| u32::from_le_bytes(id.try_into().unwrap()) >> 8), + ); + } + } + Ok(entries) +} + fn field_length(property: &crate::Property<'_>, set_lengths: &[usize]) -> usize { match &property.value { Value::NoData => 0, @@ -734,14 +749,21 @@ pub(crate) fn write_revision_with_payloads( payloads: &[([u8; 16], &[u8])], edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result>, ) -> Result> { - write_revisions_with_payloads(source, payloads, |index| { - let rid = index.active(space)?; - let revision = index.resolve(space, rid)?; + write_revisions_with_payloads(source, payloads, update(space, edit)) +} + +/// One space's active revision edited into its update. +fn update( + space: ExGuid, + edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result>, +) -> impl FnOnce(&RevisionIndex<'_>) -> Result> { + move |index| { + let revision = index.resolve(space, index.active(space)?)?; Ok(BTreeMap::from([( space, RevisionEdit::Update(edit(&revision)?), )])) - }) + } } fn append_fragment( @@ -853,13 +875,7 @@ pub(crate) fn write_revision_on( space: ExGuid, edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result>, ) -> Result> { - let output = build_on(index, &[], None, |index| { - let revision = index.resolve(space, index.active(space)?)?; - Ok(BTreeMap::from([( - space, - RevisionEdit::Update(edit(&revision)?), - )])) - })?; + let output = build_on(index, &[], None, update(space, edit))?; check(&output, true)?; Ok(output) } @@ -1079,14 +1095,24 @@ fn build_on( object.bytes[offset..offset + 4].copy_from_slice(&reference); } } - replacements.retain(|id, replacement| { - // Detached objects must pass the final reachability check even when unchanged. - !reachable.contains(id) - || !revision.objects.get(id).is_some_and(|object| { - object.data == ObjectData::Properties(&replacement.bytes) - && object.global_ids == replacement.global_ids - }) - }); + // Detached objects must pass the final reachability check even when unchanged. Equal + // bytes are the same object when the table entries they name agree: stored tables + // keep only those. + let mut unchanged = Vec::new(); + for (id, replacement) in &replacements { + if let Some(object) = revision.objects.get(id) + && reachable.contains(id) + && object.data == ObjectData::Properties(&replacement.bytes) + && table_entries(&replacement.bytes)? + .iter() + .all(|entry| object.global_ids.get(entry) == replacement.global_ids.get(entry)) + { + unchanged.push(*id); + } + } + for id in unchanged { + replacements.remove(&id); + } if replacements.is_empty() { continue; } @@ -1244,7 +1270,19 @@ fn build_on( } else { vec![node(0x21, None, &[0])?] }; + // A table read from a long-lived page names every session that edited it; the + // group stores the entries its objects use. + let mut used = BTreeSet::new(); + for (id, object) in &objects { + used.insert(u32::from_le_bytes(compact(*id, table)?) >> 8); + if let ObjectData::Properties(bytes) = object.data { + used.extend(table_entries(bytes)?); + } + } for (id, guid) in table { + if is_section && !used.contains(id) { + continue; + } let mut entry = id.to_le_bytes().to_vec(); entry.extend_from_slice(guid); group.push(node(0x24, None, &entry)?); diff --git a/crates/onestore/tests/edit.rs b/crates/onestore/tests/edit.rs index bbdb74c6c04fb287317563965f133fd02e71c9c1..1f4a5c1fe718c41c7f73e4edd51aef21c19b08a0 100644 --- a/crates/onestore/tests/edit.rs +++ b/crates/onestore/tests/edit.rs @@ -264,7 +264,13 @@ fn assert_other_objects_preserved( } let current = &after.objects[id]; assert_eq!(object.jcid, current.jcid); - assert_eq!(object.global_ids, current.global_ids); + // A rewritten object keeps the table entries it names. + assert!( + current + .global_ids + .iter() + .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid)) + ); let mut pending = vec![*id]; let mut visited = std::collections::BTreeSet::new(); while let Some(next) = pending.pop() { diff --git a/crates/onestore/tests/table_growth.rs b/crates/onestore/tests/table_growth.rs new file mode 100644 index 0000000000000000000000000000000000000000..d57186f5aab72dea0d3a5a1308d49f9dae067450 --- /dev/null +++ b/crates/onestore/tests/table_growth.rs @@ -0,0 +1,101 @@ +//! A rewritten object stores the global id table entries it names, not the table of the +//! revision it was read from. + +use onestore::{ + PreparedEdit, RevisionIndex, Store, + document::Document, + page::{Page, PageObject, Paragraph}, +}; + +const SOURCE: &[u8] = include_bytes!("../../../corpus/native/20260905-05/notebook/synthetic.one"); + +fn pages(bytes: &[u8]) -> Vec<(onestore::ExGuid, Page)> { + let store = Store::parse(bytes).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + document + .pages() + .unwrap() + .into_iter() + .map(|(space, _)| (space, Page::from_space(&document, space).unwrap())) + .collect() +} + +/// Table entries per object group written past `from`, as `(entries, highest index + 1)`. +fn tables(bytes: &[u8], from: usize) -> Vec<(usize, u32)> { + let store = Store::parse(bytes).unwrap(); + store + .lists + .values() + .filter(|list| list.nodes.first().is_some_and(|node| node.offset >= from)) + .map(|list| { + let entries: Vec = list + .nodes + .iter() + .filter(|node| node.id == 0x24) + .map(|node| u32::from_le_bytes(node.payload[..4].try_into().unwrap())) + .collect(); + (entries.len(), entries.iter().max().map_or(0, |max| max + 1)) + }) + .filter(|(entries, _)| *entries > 0) + .collect() +} + +/// `ONESTORE_TABLE_EXPORT` names a directory receiving the edited notebook for a cold reopen. +#[test] +fn a_text_edit_stores_the_table_entries_it_names() { + let mut bytes = SOURCE.to_vec(); + let mut sparse = false; + let mut edited = Vec::new(); + for (space, mut page) in pages(SOURCE) { + let Some(text) = page + .objects + .iter_mut() + .filter_map(|object| match object { + PageObject::Outline(outline) if !outline.title => Some(&mut outline.paragraphs), + _ => None, + }) + .flatten() + .find_map(|paragraph| paragraph.text_mut()) + else { + continue; + }; + let format = text.text.format_at(0).unwrap().clone(); + text.text + .append(Paragraph::new(" and a few more words".to_owned(), format)) + .unwrap(); + let written = PreparedEdit::page(&bytes, space, &page, "Rust") + .unwrap() + .as_bytes() + .to_vec(); + for (entries, span) in tables(&written, bytes.len()) { + sparse |= (entries as u32) < span; + } + assert_eq!( + PreparedEdit::page(&written, space, &page, "Rust") + .unwrap() + .as_bytes(), + written + ); + bytes = written; + edited.push((space, page)); + } + // The fixture's pages were written over several sessions, so a text edit names a + // subset of its revision's table and the stored indices have gaps. + assert!(sparse); + let stored = pages(&bytes); + for (space, page) in &edited { + let (_, stored) = stored.iter().find(|(id, _)| id == space).unwrap(); + assert!(stored.objects == page.objects); + } + if let Some(export) = std::env::var_os("ONESTORE_TABLE_EXPORT") { + let export = std::path::Path::new(&export); + std::fs::create_dir_all(export).unwrap(); + std::fs::write(export.join("synthetic.one"), &bytes).unwrap(); + std::fs::copy( + "../../corpus/native/20260905-05/notebook/Open Notebook.onetoc2", + export.join("Open Notebook.onetoc2"), + ) + .unwrap(); + } +} diff --git a/crates/onestore/tests/writer.rs b/crates/onestore/tests/writer.rs index ffd7efc1a76a06a530a57d236f90cd1e9e51aab8..72f7619da77b191bf78333e2e0e836fc09aa15f4 100644 --- a/crates/onestore/tests/writer.rs +++ b/crates/onestore/tests/writer.rs @@ -55,7 +55,11 @@ fn scalar_edit_appends_a_revision_and_preserves_every_prior_object() { let same = &unchanged.objects[&id]; assert_eq!(object.jcid, same.jcid); assert_eq!(object.reference_count, same.reference_count); - assert_eq!(object.global_ids, same.global_ids); + assert!( + same.global_ids + .iter() + .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid)) + ); assert_eq!(object.data, same.data); } } @@ -403,7 +407,11 @@ fn checkpoints_bound_dependencies_and_preserve_native_objects_and_history() { assert_eq!(a.object_spaces, b.object_spaces); assert_eq!(a.contexts, b.contexts); if section { - assert_eq!(object.global_ids, same.global_ids); + assert!( + same.global_ids + .iter() + .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid)) + ); if *id != oid { assert_eq!(object.data, same.data); } @@ -437,7 +445,11 @@ fn checkpoints_bound_dependencies_and_preserve_native_objects_and_history() { let same = &preserved.objects[&id]; assert_eq!(object.jcid, same.jcid); assert_eq!(object.reference_count, same.reference_count); - assert_eq!(object.global_ids, same.global_ids); + assert!( + same.global_ids + .iter() + .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid)) + ); assert_eq!(object.data, same.data); } } diff --git a/tools/test_table_growth.py b/tools/test_table_growth.py new file mode 100644 index 0000000000000000000000000000000000000000..ddfe1e1f5cc4c2784f724c3dfa5c43acaa1374c0 --- /dev/null +++ b/tools/test_table_growth.py @@ -0,0 +1,24 @@ +from pathlib import Path +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/table-growth' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +class TableGrowthTest(unittest.TestCase): + def test_onenote_reads_object_groups_whose_id_tables_have_gaps(self): + with TemporaryDirectory() as temporary: + native = Path(temporary) / 'read' + shutil.copytree(FIXTURE / 'cold/read', native) + compare(FIXTURE / 'candidate/notebook', native) + self.assertIn('and a few more words', (native / 'page-000.xml').read_text(encoding='utf-8-sig')) + self.assertEqual((FIXTURE / 'candidate/notebook/synthetic.one').read_bytes(), + (FIXTURE / 'cold/notebook/synthetic.one').read_bytes()) + + +if __name__ == '__main__': + unittest.main() -- 2.54.0