diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..825fce30a4e0c0554e38078c0e019cba54d5c82e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +.tmp diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000000000000000000000000000000000..0d15a74723f47ad15db5ad7cba60df1c43fefd25 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,40 @@ +# instructions for ai agents + +please read @README.md to understand the project's goals. also read the +@ARCHITECTURE.md file to get a code breakdown. without these documents, it will +be extremely hard to succeed. + +## contributing rules + +- code must be good, neatly organized, minimal, and testable. +- components should assert correctness, fixing root causes over band aids. do + not discard suspicious data or patch assertions away. +- documentation must be human written. do not alter `README.md` +- your environment is sandboxed; writing outside this directory may cause + concurrency issues. instead, write to `./.tmp/`. +- please maintain objective, high quality tests with `cargo test` +- never execute `git` commands. if you have broken your environment or are + unsure how to continue, yield to the user. + +## marko v6 tag syntax legend + +you are not familiar with the version 6 syntax changes: see + for the full explainer. here is a +basic breakdown: + +``` + + content with ${placeholders} + <@attr-tags/> + +``` + +- `/var` - js local variables. ex `` +- `...params` - js parameters. ex `` +- `...attrs` - key value pairs, any one of of: + - `key=expr` where value is a js expression + - `...expr` spread any js expression + - `value:=expr` two way binding + - `` shorthand `value` alias, eg `` +- tags can close with their name `` or just `` +- there are many more features diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000000000000000000000000000000000000..ae0181008c8fb65d8ad4bdcd0de7637948d691b9 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,22 @@ +# architecture + +- `src/lib.rs` - main library, primary function `transform` +- `src/main.rs` - main cli +- `src/plugin/` - `markdown-it` plugin and all rules + +markdown-it does not support failiable plugins, so instead errors are lowered +into `ErrorBlock` and then the errors are combined to form the final error list. +all the rules are essentially validations for the Marko syntax, and get +reprinted as text at the end. the set of rules: + +- block + - `frontmatter.rs` frontmatter + - `statement.rs` support top-level `import`, `static function`, and others + - `comment.rs` allow comments + - `tags.rs` parser for Marko tags at the block level +- inline + - `autolink.rs` custom autolink implementation to disambiguate + - `template.rs` template literals + - `inline_tags.rs` parser for Marko tags inline + +these rules are configured in `plugin/mod.rs` in `add_all`. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..844c02464fb2f9bfe7f23242065dc9b2904ea0c7 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1203 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" + +[[package]] +name = "ar_archive_writer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object", +] + +[[package]] +name = "argparse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f8ebf5827e4ac4fd5946560e6a99776ea73b596d80898f357007317a7141e47" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +dependencies = [ + "allocator-api2", +] + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.5.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6899ea499e3fb9305a65d5ebf6e3d2248c5fab291f300ad0a704fbe142eae31a" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b12c8b680195a62a8364d16b8447b01b6c2c8f9aaf68bee653be34d4245e238" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "clap_lex" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "const_format" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "cow-utils" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417bef24afe1460300965a25ff4a24b8b45ad011948302ec221e8a0a81eb2c79" + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.114", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dragonbox_ecma" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d742b56656e8b14d63e7ea9806597b1849ae25412584c8adf78c0f67bd985e66" + +[[package]] +name = "entities" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "libyml" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3302702afa434ffa30847a83305f0a69d6abd74293b6554c18ec85c7ef30c980" +dependencies = [ + "anyhow", + "version_check", +] + +[[package]] +name = "linkify" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dfa36d52c581e9ec783a7ce2a5e0143da6237be5811a0b3153fedfdbe9f780" +dependencies = [ + "memchr", +] + +[[package]] +name = "markdown-it" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99c010929c8217b2dc0940954267a2e15a15f17cb309cd1f299e21933f84fac" +dependencies = [ + "argparse", + "const_format", + "derivative", + "derive_more", + "downcast-rs", + "entities", + "html-escape", + "linkify", + "mdurl", + "once_cell", + "readonly", + "regex", + "stacker", + "unicode-general-category", +] + +[[package]] +name = "markodown" +version = "0.1.0" +dependencies = [ + "clap", + "markdown-it", + "oxc_allocator", + "oxc_ast", + "oxc_diagnostics", + "oxc_parser", + "oxc_span", + "serde", + "serde_json", + "serde_yml", + "tree-sitter", + "tree-sitter-typescript", +] + +[[package]] +name = "mdurl" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5736ba45bbac8f7ccc99a897f88ce85e508a18baec973a040f2514e6cdbff0d2" +dependencies = [ + "idna", + "once_cell", + "regex", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "owo-colors" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" + +[[package]] +name = "oxc-miette" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a7ba54c704edefead1f44e9ef09c43e5cfae666bdc33516b066011f0e6ebf7" +dependencies = [ + "cfg-if", + "owo-colors", + "oxc-miette-derive", + "textwrap", + "thiserror", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "oxc-miette-derive" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4faecb54d0971f948fbc1918df69b26007e6f279a204793669542e1e8b75eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "oxc_allocator" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71ef2dba21be1ce515378b2b7143eaa2a912f9e6ffe162ae20639d56f53d60e3" +dependencies = [ + "allocator-api2", + "bumpalo", + "hashbrown", + "oxc_data_structures", + "rustc-hash", +] + +[[package]] +name = "oxc_ast" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad9195311a1961bb6ef1de0ce6a52147bccea50b5a40423b7b44e8448ed4fc" +dependencies = [ + "bitflags", + "oxc_allocator", + "oxc_ast_macros", + "oxc_data_structures", + "oxc_diagnostics", + "oxc_estree", + "oxc_regular_expression", + "oxc_span", + "oxc_syntax", +] + +[[package]] +name = "oxc_ast_macros" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f03da6fac191c0817a32ae1a7dde27fd27d98732c61fcaeb55a99a4d543ba49" +dependencies = [ + "phf", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "oxc_data_structures" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f5171d7b8bc907a1b29e557d14f8478509a2154272d56db9ee8aed6bfe8dec" + +[[package]] +name = "oxc_diagnostics" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ef2bf6a713fd27bc65812d695bdfde3f8fcef735f00b861258518346642721b" +dependencies = [ + "cow-utils", + "oxc-miette", + "percent-encoding", +] + +[[package]] +name = "oxc_ecmascript" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f908100cb2759dd2f42ca33d95ea158b8d78e2591b577757729fc9a4a4c63bc3" +dependencies = [ + "cow-utils", + "num-bigint", + "num-traits", + "oxc_allocator", + "oxc_ast", + "oxc_span", + "oxc_syntax", +] + +[[package]] +name = "oxc_estree" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5644d3399116ff3f0cfb81f9a790c4b8173b504ed52274ecc757b57f30098ad1" + +[[package]] +name = "oxc_index" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3e6120999627ec9703025eab7c9f410ebb7e95557632a8902ca48210416c2b" +dependencies = [ + "nonmax", + "serde", +] + +[[package]] +name = "oxc_parser" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e080498b7a4456a63111f9c65b4dd1b98147955347854b809b6ad4cc5d6a0c0a" +dependencies = [ + "bitflags", + "cow-utils", + "memchr", + "num-bigint", + "num-traits", + "oxc_allocator", + "oxc_ast", + "oxc_data_structures", + "oxc_diagnostics", + "oxc_ecmascript", + "oxc_regular_expression", + "oxc_span", + "oxc_syntax", + "rustc-hash", + "seq-macro", +] + +[[package]] +name = "oxc_regular_expression" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb87ab0b072e1e97d8101cb1678204bc3873d84f13255ae5aa088f1b85f7a8e1" +dependencies = [ + "bitflags", + "oxc_allocator", + "oxc_ast_macros", + "oxc_diagnostics", + "oxc_span", + "phf", + "rustc-hash", + "unicode-id-start", +] + +[[package]] +name = "oxc_span" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41422232cfd9915d31dbb76ba2e5ae212884cad232e37203bdcb15bd1466951d" +dependencies = [ + "compact_str", + "oxc-miette", + "oxc_allocator", + "oxc_ast_macros", + "oxc_estree", +] + +[[package]] +name = "oxc_syntax" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea81736f2343df141c7d8de78a91d155be4f712dfa6cd1bdd9a8b4f0676f01f" +dependencies = [ + "bitflags", + "cow-utils", + "dragonbox_ecma", + "nonmax", + "oxc_allocator", + "oxc_ast_macros", + "oxc_data_structures", + "oxc_estree", + "oxc_index", + "oxc_span", + "phf", + "unicode-id-start", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "readonly" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2a62d85ed81ca5305dc544bd42c8804c5060b78ffa5ad3c64b0fb6a8c13d062" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_yml" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" +dependencies = [ + "indexmap", + "itoa", + "libyml", + "memchr", + "ryu", + "serde", + "version_check", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + +[[package]] +name = "stacker" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tree-sitter" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5387dffa7ffc7d2dae12b50c6f7aab8ff79d6210147c6613561fc3d474c6f75" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-general-category" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7" + +[[package]] +name = "unicode-id-start" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007" + +[[package]] +name = "unicode-ident" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" + +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de98dfa5d5b7fef4ee834d0073d560c9ca7b6c46a71d058c48db7960f8cfaf7" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..30b9afddd2c4ca96c66d0acc281fb2d9d5a2a062 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "markodown" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +clap = { version = "4.5.57", features = ["derive"] } +markdown-it = { version = "0.6.1", default-features = false, features = ["linkify"] } +oxc_allocator = "0.96.0" +oxc_ast = "0.96.0" +oxc_diagnostics = "0.96.0" +oxc_parser = "0.96.0" +oxc_span = "0.96.0" +serde_json = "1.0" +serde_yml = "0.0.12" +tree-sitter = "0.24" +tree-sitter-typescript = "0.23" + diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2795417909ca63ed579994b260e2719207633d6b --- /dev/null +++ b/README.md @@ -0,0 +1,159 @@ +# Markodown + +This is a weird markup language that combines features of [Markdown] and +[Marko]. You can think of this as an alternative universe to MDX. Since Marko +components are really easy to write, it makes this a great tool for writing +interactive blog posts. Markdown is compiled directly into `.marko` syntax, +leveraging the existing ecosystem. + +[Markdown]: https://en.wikipedia.org/wiki/Markdown +[Marko]: https://markojs.com/ + +> STATUS: Functional, but many components have not gone through battle testing. +> There are likely edge cases where syntax breaks. Some portions of the overall +> glue is generated by AI without full audits. The Marko tag parser is hand +> written according to the documentation, but that document is not a +> specification and there are likely implementation differences. Treat with +> caution. + +```` +--- +meta: + title: some interesting blog post + description: i could be really interesting + embed: + image: /something/fire.png +--- + + +i'm a catgirl and i love to **exist**. meow meow meow + +nyaa!!!!! + +```ts +function yapfest() { + return 200; +} +``` + +and then the demo of the above: + + + +## in conclusion + +it's sweet + + +```` + +## Syntax Reference + +### Paragraphs + +All text blocks with spaces around them will be wrapped in a paragraph, like +markdown does. + +``` +
not wrapped
+
+not wrapped +
+ +
+ +this paragraph gets wrapped in a `

` tag + +

+``` + +Note that all text is still processed for other block types + +``` +
[back to home](/)
+// all other + +``` + +### Components + +All Marko features, such as [tag resolution], [attribute tags], [class +shorthands], and template expressions. + +``` +## cool video + + + <@header>**music video**: in the summer + + + + (c) ${new Date().getFullYear()} + +``` + +[tag resolution]: https://markojs.com/docs/reference/custom-tag#relative-custom-tags + +### Comments + +Line, Block, and HTML comments work like they do in Marko/JavaScript. + +``` +# My Blog + +Text that is complete. + +// ## An unfinished section of the blog +// +// TODO: we gotta add this! +``` + +### Frontmatter + +All frontmatter fields are converted into exports. For example, a framework that +reads the `meta` export for Open Graph can be easily satisfied with frontmatter. + +``` +--- +meta: + title: I Love Modular Software + description: a very cute little post by me + author: clover caruso + embed: + thumbnail: /file/blog.png +--- + +// And since `export const` puts the value in scope, this works too: + +# ${meta.title} +``` + +### Static Statements + +Define module-level functions and variables. + +``` +static function sort(items: string[]) { + while (!isSorted()) { + shuffle(items); + } + return items; +} +``` + +Note that this means writing a paragraph starting with the lowercase words +`static`, `export`, `import`, `server`, and `client` all must be escaped. + +``` +# All About RSC + +\server components are stupid. (markdown backslash) + +${"server"} components are stupid. (template literal) + +Though you can say import as long as its not the first item. +``` diff --git a/examples/hoj.mdo b/examples/hoj.mdo new file mode 100644 index 0000000000000000000000000000000000000000..7bf40468f791a90f6a3a64b2f9f1e211c8263c03 --- /dev/null +++ b/examples/hoj.mdo @@ -0,0 +1,98 @@ +--- +meta: + title: history of japan reanimated + description: >- + 17 artists and musicians come together to remake bill wurtz's + legendary "history of japan" to celebrate its 10 year anniversary. + embed: + thumbnail: /file/2026/history of japan reanimated/thumbnail.jpeg + canonical: /history-of-japan-reanimated +theme: + fg: #fff + bg: #442233 + primary: #ff8b47 +--- +import "./history-of-japan-reanimated.css"; +client import "./history-of-japan-reanimated.client.ts"; + +
+ + +# history of japan reanimated + +me and 16 other artists (music, narration, and animation) have been spending the +past couple months on a recreation of bill wurtz's legendary "[history of japan]" +to celebrate the 10 year anniversary of his release. i contributed two animation +sections. the canonical release is on [LunaPogi's YouTube page][yt], but i am +providing this ad-free video player. + +[history of japan]: https://billwurtz.com/history-of-japan.html +[yt]: https://youtu.be/nqsoti-OuRM + + + + + +the wonderful souls who worked on the project, in order of appearance. + + + +the music during the credits is an instrumental by bill wurtz: [hey mom i beat the scale!](https://billwurtz.com/hey-mom-i-beat-the-scale.mp3) + +## what's next + +from the success of this project, a much more ambitous project is being planned: +the reanimation for 'history of the entire world, i guess'. i am joining the +project as one of the co-hosts. we are currently doing initial planning, all +artists (music, visual, narration) interested should fill out the [interest form]. +by submitting the form, i'll be able to reach out when it's actually time to +start cooking. you can also join the project's [public discord]. + +[interest form]: https://paperclover.net/hotewig-reanimated +[public discord]: https://discord.gg/37axTSuGXd + +## behind the scenes + +here is a 55 minute dive into my process, as well as nearly every little trick i +used on this project. additionally, for one of the first times from me, a free +download for [all of the project files][files]. + + + <@header>behind the scenes session with clover + + +[files]: /file/2026/history%20of%20japan%20reanimated/project%20files/history-of-japan-reanimated.zip?dl + +## more links + +

+ paper clover q&a page
+ paper clover's mailing list +

+ +## mentions on the q&a + + +
diff --git a/examples/next-js.mdo b/examples/next-js.mdo new file mode 100644 index 0000000000000000000000000000000000000000..c22c29f10601459982d195e421731f594e451ca0 --- /dev/null +++ b/examples/next-js.mdo @@ -0,0 +1,750 @@ +--- +meta: + title: One Year with Next.js App Router — Why We're Moving On + description: A critique of React Server Components and Next.js 15. + keywords: ["webdev", "technical analysis", "opinion"] + authors: ["clover caruso"] + embed: + thumbnail: /open-graph/next-js.png + canonical: /blog/webdev/one-year-next-app-router +--- +export { theme } from "@/blog/tags/layout.tsx"; + + + +As I've been using [Next.js] professionally on my employer's web app, I find the +core design of their App Router and [React Server Components] (RSC) to be +extremely frustrating. And it's not small bugs or that the API is confusing, +but large disagreements about the fundamental design decisions that Vercel and +the React team made when building it. + +The more webdev events I go to, the more I see people who dislike Next.js, but +still get stuck using it. By the end of this article, I will share how me and +my colleagues escaped this hell, seamlessly migrating our entire frontend to +[TanStack Start]. + +[Next.js]: https://nextjs.org +[React Server Components]: https://react.dev/reference/rsc/server-components + + + +A Technical Review: What are Server Components? + +The pitch of RSC is that components are put into two categories, +"server" components and "client" +components. Server components don't have `useState`, `useEffect`, but can be +`async function`s and refer to backend tools like directly calling into a +database. Client components are the existing +model, where there is code on the backend to generate HTML text and frontend +code to manage the DOM using `window.document.*`. + +> The first disaster: naming!! React is now using the wors +> "server" and "client" to refer to +> a very specific things, ignoring their existing definitions. This would be +> fine, except Client components can run on the backend +> too! In this article, I'll be using the terms "backend" and +> "frontend" to describe the two execution environments that web apps +> exist in: a Node.js process and a Web browser, respectively. + +This Server/Client component model +is interesting. Since built-ins like `` get serialized across the +network, data fetching can be very trivially modeled with async +server components, and the fallback UI works as if it were +client-side. + +```tsx filename="src/app/[username]/page.tsx" tint="server" +// For this article, server components will be highlighted in red +export default async function Page({ params }) { + // Page params are given as a resolved promise + const { username } = await params; + + // The components `UserInfo` and `UserPostList` will be run at the same + // time. Once `UserInfo` is ready, the visitor will see the page with a + // `PostListSkeleton` if the post list is not yet ready. + return
+ + + }> + + +
+} + +// Waterfalls are avoided by having multiple components, which +// are all evaluated at the same time. + +async function UserInfo({ username }) { + const user = await fetchUserInfo(username); + return <> +

{user.displayName}

+ {user.bio ? : ""} + +} + +async function UserPostList({ username }) { + const posts = await fetchUserPostList(username); + return /* post list ui omitted for brevity */; +} +``` + +If we ignore the 40kB gzipped bundle size of React itself, the above example +has zero JavaScript for the UI and data fetching — it just streams the +markup! For example, the imagined markdown parser within the `` +component stays on the backend. When an interactive frontend is needed, Client +components can be created by putting them in a file starting with `"use +client"`. + +```tsx filename="src/components/CopyButton.tsx" tint="client" +"use client"; // This comment marks the file for client-side bundling. + +export function CopyButton({ url }) { + return <> + {url} + + +} +``` +```tsx filename="src/app/q+a/Card.tsx" tint="server" +export function Card() { + return
+
+ {/* Make the browser import the copy button */} + +
+

+ {/* Process markdown on the backend */} + +

+
+} +``` + +

Real-world Pitfalls of the App Router + +After quitting [Bun] as a runtime engineer (I implemented [Server Components +bundling] and [a RSC template][bun-rsc] there), I joined a small company working on the +front lines: a Next.js app with a Hono backend. The following notes are +simplifications from the real world problems I've encountered when trying to +maintain and develop new features. As a result of all of these, everyone's time +is wasted either working around design flaws, or explaining to each other why +what should be a non-issue is an immovable object. + +[Bun]: https://bun.com +[Server Components bundling]: https://github.com/oven-sh/bun/blob/67f0c3e016aa479738469adac2b79a1862b88122/src/bake/bake.d.ts +[bun-rsc]: https://github.com/oven-sh/bun/tree/e7790894d92b730758ecadf971cb935063508dfb/src/bake/bun-framework-react + +

Optimistic Updates are Impossible + +The Next.js documentation for performing mutations [does not mention optimistic +updates][nextjs-updating-data]; it appears this case was not thought about. +Components rendered by the React Server, by design, can +not be modified after mounting. Elements that could change need to be inside a +client component, but data fetching cannot happen on the client components, +even during SSR on the backend. This results in awkwardly small server +components that only do data fetching and then have a client component that +contains a mostly-static version of the page. + +```tsx filename="src/app/user/[username]/page.tsx" tint="server" + +export default async function Page() { + const user = await fetchUserInfo(username); + return + + ; +} +``` +```tsx filename="src/app/user/[username]/UserProfile.tsx" tint="client" + +"use client"; // Must separate the client code into a second file! + +export function UserProfile({ user: initialUser }) { + // There are many great state management libraries out there; + // for simplicity, this example will use one state cell. + const [user, optimisticUpdateUser] = useState(initialUser); + + async function onEdit(newUser) { + optimisticUpdateUser(newUser); + const resp = await fetch("...", { + method: 'POST', + body: JSON.stringify(newUser), + ... // (headers, credentials, tracing, and more) + }) + if (!resp.ok) /* always remember to test for errors! */ + } + + return
{/* user interface with editable fields... */}
: +} +``` + +As more of the page needs interactivity, it gets messier trying to keep the +static parts truly server-side. On the work app, nearly every piece of UI +displays some dynamic data. A [`WebSocket`][ws] synchronizes data live as it +updates (for example, a user card's online state along with their basic +profile). Since these component setups are harder to understand and maintain +for engineers, almost all of our pages are entirely `"use client"` with a +`page.tsx` that defines the data fetching. + +A more concrete example of what this looks like in practice with the +data-fetching library we use at work, [TanStack Query]. + +[TanStack Query]: https://github.com/tanstack/query#readme + +```ts filename="src/queries/users.ts" +// At work, there is a helper function `defineQuery` for type safety. +// Fetchers are trivial and can run on the backend or the frontend. +export const queryUserInfo = (username) => ({ + queryKey: ['user', username], + queryFn: async ({ ... }) => /* fetch data */ +}); +``` +```tsx filename="src/app/user/[username]/page.tsx" tint="server" +export default async function Page({ params }) { + const { username } = await params; + + // There's no global state in the React Server. Since layouts + // are executed in parallel, the TanStack `QueryClient` has to + // be reconstructed multiple times per route. + const queryClient = new QueryClient(); + await queryClient.ensureQueryData(queryUserInfo(username)); + + // HydrationBoundary is a client component that passes JSON + // data from the React server to the client component. + return + + ; +} +``` +```tsx filename="src/app/user/[username]/ClientPage.tsx" tint="client" +"use client"; +export function ClientPage() { + const { username } = useParams(); + const { data: user } = useSuspenseQuery(queryUserInfo(username)); + + // ... some hooks + + return
+ {/* ... an interactive web page */} +
; +} +``` + +This example has to be three separate files because of the rules of server +component bundling. (The client component needs `"use client"`, and server +component files often can't be imported on the client due to server-only +imports.). In the Pages router, this could've been a single file because of the +tree-shaking that `getStaticProps` and `getServerSideProps` has. + +[ws]: https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API +[nextjs-updating-data]: https://nextjs.org/docs/app/getting-started/updating-data + +

Every Navigation is Another Fetch + +Since the App Router starts every page as a server component, with (ideally) +small areas of interactivity, a navigation to a new page *has* to fetch the +Next.js server, regardless of what data the client already has available! Even +with a a `loading.tsx` file, opening `/`, navigating to `/other`, and then +going back to `/` will show the loading state while it re-fetches the homepage. + +The only case this works is for **perfectly static content**, where instant +navigations and prefetching work great. But **web apps are not static**, they +have lots of dynamic content. Being logged in affects the homepage, which is +infuriating because the client literally has everything needed to display the +page instantly. It's not like the cookies changed. + +> **aside**: In further testing on a blank project, I observe cases where the +> Next frontend code would pre-fetch routes, but **without any real contents**. +> On the hello world example, this was a 1.8kB RSC payload that pointed to 2 +> different JS chunks 4 separate times. This is just pure waste of our +> bandwidth and egress, especially considering all of this information is +> re-fetched when I actually click the link. +> +> ```json whitespace="pre-wrap" +> 1:"$Sreact.fragment" +> 2:I[39756,["/_next/static/chunks/ff1a16fafef87110.js","/_next/static/chunks/7dd66bdf8a7e5707.js"],"default"] +> 3:I[37457,["/_next/static/chunks/ff1a16fafef87110.js","/_next/static/chunks/7dd66bdf8a7e5707.js"],"default"] +> 4:I[97367,["/_next/static/chunks/ff1a16fafef87110.js","/_next/static/chunks/7dd66bdf8a7e5707.js"],"ViewportBoundary"] +> 6:I[97367,["/_next/static/chunks/ff1a16fafef87110.js","/_next/static/chunks/7dd66bdf8a7e5707.js"],"MetadataBoundary"] +> 7:"$Sreact.suspense" +> 0:{"b":"TdwnOXsfOJapNex_HjHGt","f":[["children","other",["other",{"children":["__PAGE__",{}]}],["other",["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L3",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":null},[["$","div","l",{"children":"loading..."}],[],[]],false],["$","$1","h",{"children":[null,["$","$1","KCFxAJdIDH3BlYXAHsbcVv",{"children":[["$","$L4",null,{"children":"$L5"}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],["$","$L6","KCFxAJdIDH3BlYXAHsbcVm",{"children":["$","div",null,{"hidden":true,"children":["$","$7",null,{"fallback":null,"children":"$L8"}]}]}]]}],false]],"S":false} +> 5:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] +> 9:I[27201,["/_next/static/chunks/ff1a16fafef87110.js","/_next/static/chunks/7dd66bdf8a7e5707.js"],"IconMark"] +> 8:[["$","title","0",{"children":"Create Next App"}],["$","meta","1",{"name":"description","content":"Generated by create next app"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.0b3bf435.ico","sizes":"256x256","type":"image/x-icon"}],["$","$L9","3",{}]] +> ``` +> +> In review, I found there is actually some content in here: the loading state. +> Do you see it? +> +> ```json +> ["$","div","l",{"children":"loading..."}] +> ``` +> +> It's still a lot of waste, since all of this data gets re-emitted in the +> actual page RSC. + +The solution to this appears to be [`staleTime`][nextjs-stale], but it's marked +experimental and "not recommended for production". The fact this is a +non-default afterthought configuration option is embarrassing. Even if we used +it, you cannot make multiple pages that refer to the same underlying data share +any of it. + +One form of loading state that cannot be represented with the App Router is +having a page such as a page like a git project's issue page, and clicking on a +user name to navigate to their profile page. With `loading.tsx`, the entire +page is a skeleton, but when modeling these queries with TanStack Query it is +possible to show the username and avatar instantly while the user's bio and +repositories are fetched in. Server components don't support this form of +navigation because the data is only available in rendered components, so it +must be re-fetched. + +[nextjs-stale]: https://nextjs.org/docs/app/api-reference/config/next-config-js/staleTimes +[nextjs-stale-bug]: https://github.com/vercel/next.js/issues/70661 + +In our Next.js site, we have this line of code on our server component data +fetchers to make soft navigations faster by skipping the data fetch phase all +together. + +```tsx filename="src/util/tanstack-query-helpers.server.ts" tint="server" +export function serverSidePrefetchQueries(queries) { + if ((await headers()).get("next-url")) { + // This is a soft-navigation. SKIP the prefetching to make it faster. + // The client might already have this data, and if not, they have the + // loading state. Ideally, this server request wouldn't exist -- The + // client side has nearly ALL the code since the app is written mostly + // as client components. Kind of a design flaw of the App router TBH. + return; + } + // ... data prefetching-logic ... +} +``` + +In addition to this, `loading.tsx` should contain the `useQuery` calls so that +while the network request for the empty RSC happens, the data is being fetched +if it actually is needed. In fact, the `loading.tsx` state can just be the +actual client component, and you'll see the client page. + +```tsx filename="src/app/user/[username]/loading.tsx" tint="client" +"use client"; +export default function PageLoadingSkeleton() { + return ; +} +``` + +> At work, we just make our `loading.tsx` files contain the `useQuery` +> calls and show a skeleton. This is because when Next.js loads the actual Server +> Component, no matter what, the entire page re-mounts. No VDOM diffing here, +> meaning all hooks (`useState`) will reset slightly after the request +> completes. I tried to reproduce a simple case where I was *begging* Next.js to +> just *update the existing DOM* and preserve state, but it just doesn't. +> Thankfully, the time the blank RSC call takes is short enough. + +

Layouts are Artificially Restricted + +Layouts can perform data fetching, but they can't observe or alter the request +in any way. This is done so that Next.js can fetch and cache layouts whenever they +want. In every other framework, layouts are just regular components that have +no feature difference compared to page components. + +Fetching layouts in isolation is a cute idea, but it ends up being silly +because it also means that any data fetching has to be re-done per layout. You +can't share a `QueryClient`; instead, you must rely on their [monkey-patched +`fetch`][nextjs-fetch] to cache the same `GET` request like they promise. + +When a coworker asks me about why Next.js rejects some code, I've given up on +explaining the technical intricacies and just say *"It's a Next.js Skill Issue, +I'm going to blow it up soon don't worry."* These rules are too hard for normal +developers to understand. + +[nextjs-fetch]: https://nextjs.org/docs/app/api-reference/functions/fetch + +

You Still Download All the Content Twice + +Unlike the ["Islands Architecture"][islands], Server Components still have to +be hydrated on the frontend to support `Suspense` and preserving client +component state. When doing soft navigations, the "RSC Payload" (which is not +HTML at all) is retrieved by `fetch`. On a fresh reload, HTML is needed for the +[first paint], but the information about Client components and `Suspense` is +not contained within that HTML. React's solution is to **send a second copy of +the entire page's markup**. An example of what a Next.js production server +would send in a dynamic page render would be something like this: + +[first paint]: https://web.dev/articles/fcp + +```html filename="GET /user/clover" + + + + {link and meta tags} + + + {server side render} + + + + +
+ {server side render of a Suspense boundary} +
+ + + + + +``` + +This solution **doubles the size of the initial HTML payload**. Except it's +worse, because the RSC payload includes JSON quoted in JS string literals, +which is a is much less efficient format than HTML. While it seems to compress +fine with brotli and render fast in the browser, this is wasteful. With the +hydration pattern, at least the data locally could be re-used for interactivity +and other pages. + +Even on pages that have little to no interactivity, you pay the cost. To use +the Next.js documentation as an example, loading [its +homepage](https://nextjs.org/docs) loads an page that is around 750kB (250kB of +HTML and the 500kB of script tags), and content is in there twice. + +You can verify that by pressing Cmd + Opt + u +on Mac or Ctrl + u on other platforms. And then +Cmd / Ctrl + f to locate any string of the +blog, such as "building full-stack web applications". It's there twice. And +**there is no way around this**, since it's a fundamental piece of React Server +Components. + +This RSC format certainly has more waste. But I really don't feel like digging into +why the string `/_next/static/chunks/6192a3719cda7dcc.js` appears 27 separate +times. What the hell, guys? Is your bandwidth free??? + +[islands]: https://www.patterns.dev/vanilla/islands-architecture/ + +

Turbopack Sucks + +This section is not constructive. + +- Turbopack isn't fast +- Turbopack emits code that is hard to debug in a debugger (in development mode) +- Turbopack throws bad error messages in many cases + +I wouldn't have given this point a section in the blog normally, but I want to +point out three actual examples directly from the project. + +The first is a place where during some refactoring to satisfy the Server/Client +component models, I accidentally made a Client component `async`. This one was +quite annoying because it didn't say at all where the issue was, but only +contained the server stack trace. + +![Next.js error](/file/2025/blog-everyone-hates-nextjs/asyncerror.png) + +Another case of a terrible error message: + +![Next.js error](/file/2025/blog-everyone-hates-nextjs/nexterror.png) + +> After fixing the underlying issue in this second error (which I cannot recall), +> the Dev server hung and had to be restarted to recover. + +The final one is the dozen times I place a debugger breakpoint and the +variable name `hello` gets turned into +`__TURBOPACK__imported__module__$5b$project$5d2f$client$2f$src$2f$utils$2f$filename$2e$ts__$5b$app$2d$client$5d$__$28$ecmascript$29$__["hello"]` +and other bullshit. + +Okay. This all sucks. What can we do? + +

Seamlessly Ditching Next.js and Vercel at Work + +There are two types of web projects: + +- A web site with mostly static content. +- A web app with majorly dynamic and interactive components. + +And Next.js is the wrong tool for both of these jobs. If you're in the first +category with a static web site, go for [Astro] or [Fresh]. For everyone who +needs the full power of React, this section is about how I replaced the vendor +locked Next with [TanStack Start], incrementally and seamlessly. + +[Astro]: https://astro.build/ +[Fresh]: https://fresh.deno.dev/ +[TanStack Start]: https://tanstack.com/start/latest + +It started with this Vite config. + +```ts filename="vite.config.ts" +const config = defineConfig(({ mode }) => { + const env = loadEnv(mode, process.cwd(), "NEXT_PUBLIC_"); + return { + // Use the Next.js default port 3000 + server: { port: 3000 }, + // Use the Next.js default env prefix "NEXT_PUBLIC_" + define: Object.fromEntries(Object.entries(env).map( + ([k, v]) => [`process.env.${k}`, JSON.stringify(v)])), + plugins: [ + viteTsConfigPaths({ projects: ["./tsconfig.json"] }), + tailwindcss(), + // For ease of understanding from coworkers, I started porting + // the routes in `src/tanstack-routes`. When the migration was + // done, it would go back to the default `src/routes`. + tanstackStart({ + router: { routesDirectory: "src/tanstack-routes" }, + }), + viteReact(), + ], + resolve: { + // The key to the incremental migration: redirect `next` elsewhere + alias: { next: path.resolve("./src/tanstack-next/") }, + conditions: ["tanstack"], + extensions: [ + // Allow a file named like `utils/session.tanstack.ts` to + // override `utils/session.ts` when imported. + ".tanstack.tsx", ".tanstack.ts", + // Default import extensions + ".mjs", ".js", ".mts", ".ts", + ".jsx", ".tsx", ".json", + ], + }, + }; +}); +``` + +Then, I looked for every usage of a Next.js API, and either removed it or made +a stub for TanStack. For example, `src/tanstack-next/link.tsx` implements +`next/link`: + +```tsx filename="src/tanstack-next/link.tsx" +import { Link } from "@tanstack/react-router"; +import type { LinkProps } from "next/link"; + +export default function LinkAdapter({ href, ...rest }: LinkProps) { + return ; +} +``` + +> Some of these stubs can be extremely simple. Starting out, my implementation +> of `useRouter` was just `return {}`, but later I had to add a couple methods +> to the object. The code here doesn't have to be clean, because it is +> temporary. + +Now, the new site can import nearly every client component by either stubbing +out the Next.js APIs it needs, or by using the `.tanstack.ts` extension to +re-implement logic on a file-by-file basis. And shortly after, I got the site's +homepage to work in TanStack Start, and we merged the branch. + +![My "nextgate" PR](/file/2025/blog-everyone-hates-nextjs/pr.png) + +> This first PR only supported one of our pages, and was able to do it in a +> thousand lines of added code, and 40 lines deleted. I had previous patches to +> remove the few uses of `next/image` and `next/font`. + +What was left was porting every other route over. The one thing we lose in +migrating from Next.js to any other framework is the ability to `await` +data-fetching functions in the UI. In practice, moving every route into a +`loader` function made it much more clear what happened when a page was SSR'd. +For pages that had multiple fetches, these could be combined into a single, +special API call that would return all of the relevant data for that page. + +To re-iterate in bold font: The +migration path from Server Components is to just simplify your code — RSC +inherently drives you down a chaotic road of things you do not need. +Nearly every complex part of our site got easier to understand for all +engineers. The exception to this was having everyone get used to the new file +system routing conventions. With enough examples, we all got the hang of it. + +With the incremental migration in place, new code did not break the existing +deployment. TanStack slowly took over the codebase, and we eventually deleted +all of the Next.js stubs and gained all of the beautiful [type-safety features] +that the TanStack Router provides. At the end, the site performed faster from +every angle: Development Mode, Production page load times, Soft navigations, +and at a lower price than our Next depoyment with Vercel. + +[type-safety features]: https://tanstack.com/router/v1/docs/framework/react/guide/type-safety + +We're not the only ones seeing the change. While I try and keep myself off of +social media, someone sent me [the results of Brian Anglin's work at +Superwall][superwall-twitter], showing incredible CPU reductions on TanStack +Start. I also recall ChatGPT switching from Next.js to Remix (random online +chatter: [[1][chatgpt-1]] [[2][chatgpt-2]] [[3][chatgpt-3]]) a year ago. + +[superwall-twitter]: https://twitter.com/BriansAngles/status/1978834116079436242#m +[chatgpt-1]: https://xcancel.com/ryanflorence/status/1831379475654947233 +[chatgpt-2]: https://old.reddit.com/r/reactjs/comments/1f97zgr/chatgpt_migrates_from_nextjs_to_remix +[chatgpt-3]: https://old.reddit.com/r/nextjs/comments/1f92jdv/chatgptcom_switched_from_nextjs_to_remix + +

`next/metadata` is Great + +In my opinion, this is one of the only good APIs Next.js has, and was the one +place in our code where moving to TanStack made things harder to do. Instead of +worsening the code, I just ported their metadata API into a regular function, +so everyone can use it. Originally, I had a 1:1 port on NPM, but earlier this +year I simplified it's API into one short and understandable +file. As of this blog post, I have added a TanStack-compatible +`meta.toTags` API, which can be installed from [JSR][lib-jsr], [NPM][lib-npm], +or simply copied into your project. + +> **notice**: Due to time constraints with writing this article, the library +> has not yet been updated. I'll probably get around to it by the ~~end of this +> week (Oct 24th)~~ some time soon... As a placeholder, I'm able to share the +> version that is used at work to my website: +> [`meta.tanstack.ts`](https://paperclover.net/file/2025/blog-everyone-hates-nextjs/meta.tanstack.ts). + +```tsx +// once in your project +import * as meta from "@clo/lib/meta.ts"; + +export const defineHead = meta.toTags.bind(null, { + // site-wide options + base: new URL("https://paperclover.net"), + titleTemplate: (title) => [title, "paper clover"] + .filter(Boolean).join(' | '), + // ... +}); + +// for each page... +export const Route = createFileRoute("/blog")({ + head: () => + defineHead({ + title: "clover's blog", // templated with `titleTemplate` + description: "a catgirl meows about her technology viewpoints", + canonical: "/blog", // joined with `base` + + // When specified, configures Open Graph and Twitter embed, + // using the page title and description as the default. + // The defaults are good, but it supports more options. + embed: { + image: "/img/blog.webp", + }, + + // Every exotic meta tag is done with a JSX fragment. This + // doesn't render React, it just loops through the tags. + // My goal was to cover the most common 99% of uses. + extra: <> + , + , + }), + + component: Page, +}); + +function Page() { + ... +} +``` + +My version wasn't concerned with covering the entire space of Next.js's metadata +object, but instead uses inline JSX to fill that gap. + +[lib-jsr]: https://jsr.io/@clo/lib +[lib-npm]: https://npmjs.com/@paperclover/lib + +

`next/og` is Good Too + +No strong opinions. I just want to remind everyone that the `@vercel/og` package exists. + +

My Experience Feels like the Usual + +At the Next.js Conf 2024, everyone there was raving about Server Components. I +forget exactly who I talked to, but the big people were all in on this. I, +having implemented the bundler end of RSC, saw a couple of the problems in the +format. With Next 15 "stabilizing" the App Router last year, many companies are +building their products on it, realizing these pitfalls first-hand. + +I came into the Next.js game late, only starting in June with version 15. +But everyone I've talked to at events sympathize with my notes. All the people +I talked to on the subject at Bun's 1.3 Party agreed with me. Even some people +at Vercel told me they don't like how Next.js is to actually use. + +I hope as TanStack Start stabilizes, it becomes the Next.js replacement everyone +wants. + +

Prefer Tools that Respect You + +A lot of in the JavaScript ecosystem is a mess. That mess is why web +development gets made fun of. There were a lot of times I thought that working +with the web was an unrecoverable mess, but the mess was actually just the +commonly-used libraries I surrounded myself with. When that is peeled back, +modern web development technologies are awesome. + +I've been making this website from scratch without any framework since late +2024, by writing systems like my own [TUI progress widget][progress], [static +file proxy][file-cache], incremental build system, and many more components. +Working on this code has produced some of my best coding sessions (by +happiness) in years. The viewers of *[paper clover]* get a better quality +website; the mini-libraries I create get [extracted for public use][lib], +everyone wins. + +This level of from-scratch is too much for most people, especially at the +workplace. I say that at the minimum, we should only give our attention and +money to high quality tools that respect us. And Next.js and the company behind +it, Vercel, are not that. + +If you use Next.js, and feel that the experience doesn't remind you of respect +too, consider whether you and your colleagues want to continue supporting their +[serverless empire]. The Vite ecosystem seems pretty decent to build on right +now, but I still have little experience in using their tools at scale in +production. The [Vite+ launch from Void0][vite-plus] seems interesting, but +only time will tell if these venture-funded tools will respect us (end-users +and developers) long term. + +Next.js Conf 2025, as of writing, is [tomorrow][next-conf]. Instead of +purchasing a $800 ticket, I decided to put that money [toward the TanStack +team][tanstack-donate] for [respecting and improving the web development +ecosystem][tanstack-ethos]. + +[paper clover]: https://paperclover.net/ +[lib]: https://git.paperclover.net/clo/sitegen/src/branch/master/lib#readme +[progress]: https://git.paperclover.net/clo/sitegen/src/branch/master/lib/progress.ts +[file-cache]: https://git.paperclover.net/clo/sitegen/src/branch/master/src/file-viewer/cache.ts + +[next-conf]: https://nextjs.org/conf +[vite-plus]: https://viteplus.dev/ +[tanstack-ethos]: https://tanstack.com/ethos +[tanstack-donate]: https://github.com/sponsors/tannerlinsley +[serverless empire]: https://youtu.be/SCIfWhAheVw + +## What the Future Holds + +Slowly, I've been replacing many pieces of software that disrespect me with +better alternatives. Some examples of this are GitHub, Visual Studio Code, +DaVinci Resolve, Discord, Google Drive/Workspace, along many more. I plan to +write more on this blog about the technical things I do (that progress library, +the purpose of my own site generator, learnings from my current job), including +some of my past projects at Bun (details on HMR, the crash reporter, and the +crazy system for bundling built-in modules). If it interests you, please +subscribe to the email list: + +click here to send an email to subscribe@paperclover.net, requesting that you would like to be added to the mailing list. (i manage this mailing list manually) + +[back to top](#top) — [ask a question about this article](/q+a) + +
+
+
+
+
+
+ +
2025 (c) paper clover
+ + + +
diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..dfae36e9693e8a68dd3af15f1d9031cb824818f4 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,225 @@ +pub mod marko; +pub mod plugin; +pub mod typescript; + +use oxc_diagnostics::{LabeledSpan, OxcDiagnostic}; +use plugin::tags::{MarkoClose, MarkoOpen}; + +pub struct Output { + pub text: String, + pub format: OutputFormat, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutputFormat { + Html, + Marko, +} + +pub fn transform(source: &str, force: Option) -> Result> { + let md = &mut markdown_it::MarkdownIt::new(); + + plugin::add_all(md); + markdown_it::plugins::cmark::add(md); + markdown_it::plugins::extra::add(md); + + let mut ast = md.parse(source); + + let mut errors = Vec::new(); + collect_errors(&mut ast, &mut errors); + if !errors.is_empty() { + return Err(errors); + } + + // Validate Marko tag open/close matching + validate_marko_tags(&ast, &mut errors); + if !errors.is_empty() { + return Err(errors); + } + + // Check for unmatched inline tag markers + plugin::inline_tags::check_unmatched_markers(&ast, &mut errors); + if !errors.is_empty() { + return Err(errors); + } + + let format = force.unwrap_or_else(|| { + if has_marko_features(&ast) { + OutputFormat::Marko + } else { + OutputFormat::Html + } + }); + + if force.is_some() && format == OutputFormat::Html && has_marko_features(&ast) { + return Err(vec![OxcDiagnostic::error( + "Cannot output HTML: document contains Marko-specific features", + )]); + } + + Ok(Output { + text: ast.render(), + format, + }) +} + +/// Extract all `ErrorBlock`s from an AST, offsetting the error positions. +fn collect_errors(node: &mut markdown_it::Node, errors: &mut Vec) { + let (start, end) = node.srcmap.unwrap().get_byte_offsets(); + if let Some(error_block) = node.cast_mut::() { + errors.extend(error_block.errors.drain(0..).map(|mut err| { + if let Some(labels) = err.labels.as_mut() { + for label in labels { + label.set_span_offset(label.offset() + start + 2); + } + } else { + err.labels = Some(vec![LabeledSpan::new(None, start, end - start)]) + } + err + })); + } + for child in &mut node.children { + collect_errors(child, errors); + } +} + +/// Extract all `ErrorBlock`s from an AST, offsetting the error positions. +pub fn adjust_err(mut err: OxcDiagnostic, offset: isize) -> OxcDiagnostic { + if let Some(labels) = err.labels.as_mut() { + for label in labels { + label.set_span_offset(label.offset().saturating_add_signed(offset)); + } + } + err +} + +/// Check if the AST contains any Marko-specific features +fn has_marko_features(node: &markdown_it::Node) -> bool { + // Check if this node is a RawBlock + if node.cast::().is_some() { + return true; + } + + // Check for Marko tags + if node.cast::().is_some() + || node.cast::().is_some() + || node.cast::().is_some() + || node.cast::().is_some() + || node.cast::().is_some() + || node.cast::().is_some() + { + return true; + } + + // Recursively check children + for child in &node.children { + if has_marko_features(child) { + return true; + } + } + + false +} + +/// Validate that Marko open/close tags are properly matched +fn validate_marko_tags(node: &markdown_it::Node, errors: &mut Vec) { + #[derive(Debug)] + struct OpenTag { + name: String, + /// Byte offset of the tag name in the source (after '<') + name_start: usize, + /// Length of the tag name + name_len: usize, + } + + let mut stack: Vec = vec![]; + + fn walk(node: &markdown_it::Node, stack: &mut Vec, errors: &mut Vec) { + if let Some(open) = node.cast::() { + let (start, _) = node.srcmap.unwrap().get_byte_offsets(); + stack.push(OpenTag { + name: open.tag_name.clone(), + name_start: start + 1, // +1 to skip '<' + name_len: open.tag_name_len, + }); + } else if let Some(open) = node.cast::() { + let (start, _) = node.srcmap.unwrap().get_byte_offsets(); + stack.push(OpenTag { + name: open.tag_name.clone(), + name_start: start + 1, // +1 to skip '<' + name_len: open.tag_name_len, + }); + } else if let Some(close) = node.cast::() { + let (close_start, _) = node.srcmap.unwrap().get_byte_offsets(); + // Close tag name starts after ' { + // closes anything - valid + } + (Some(name), Some(top)) if name == &top.name => { + // Exact match - valid + } + (Some(name), Some(top)) => { + // Mismatched: expected , got + errors.push( + OxcDiagnostic::error(format!( + "Mismatched closing tag: expected , found ", + top.name, name + )) + .with_labels(vec![ + LabeledSpan::new( + Some("opened here".into()), + top.name_start, + top.name_len, + ), + LabeledSpan::new( + Some("closed here".into()), + close_name_start, + close_name_len, + ), + ]), + ); + } + (Some(name), None) => { + // Close without open + errors.push( + OxcDiagnostic::error(format!( + "Closing tag without matching open" + )) + .with_label(LabeledSpan::new( + None, + close_name_start, + close_name_len, + )), + ); + } + (None, None) => { + // without any open tag - highlight the whole + errors.push( + OxcDiagnostic::error("Closing tag without matching open") + .with_label(LabeledSpan::new(None, close_start, 3)), + ); + } + } + } + + // Recurse into children + for child in &node.children { + walk(child, stack, errors); + } + } + + walk(node, &mut stack, errors); + + // Check for unclosed tags + for unclosed in stack { + errors.push( + OxcDiagnostic::error(format!("Unclosed tag <{}>", unclosed.name)).with_label( + LabeledSpan::new(None, unclosed.name_start, unclosed.name_len), + ), + ); + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..16026704d3e9e19e97fecf17a831937c8781c339 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,71 @@ +use clap::{Parser, ValueEnum}; +use markodown::{transform, OutputFormat}; +use oxc_diagnostics::GraphicalReportHandler; +use std::fs; +use std::path::PathBuf; +use std::process; + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum CliOutputFormat { + Html, + Marko, +} + +#[derive(Parser)] +#[command( + name = "markodown", + about = "A markdown compiler with custom extensions" +)] +struct Cli { + /// Path to input markodown source + file: PathBuf, + + /// Force the output format (auto-detected by default) + #[arg(long)] + output_format: Option, +} + +fn main() { + let cli = Cli::parse(); + + let force = cli.output_format.map(|f| match f { + CliOutputFormat::Html => OutputFormat::Html, + CliOutputFormat::Marko => OutputFormat::Marko, + }); + + let source = match fs::read_to_string(&cli.file) { + Ok(s) => s, + Err(e) => { + eprintln!("error: failed to read {}: {e}", cli.file.display()); + process::exit(1); + } + }; + + match transform(&source, force) { + Err(errors) => { + let handler = GraphicalReportHandler::new(); + for error in &errors { + let report = error + .clone() + .with_source_code(oxc_diagnostics::NamedSource::new( + cli.file.display().to_string(), + source.clone(), + )); + let mut output_str = String::new(); + handler + .render_report(&mut output_str, report.as_ref()) + .unwrap(); + eprint!("{output_str}"); + } + process::exit(1); + } + Ok(output) => match output.format { + OutputFormat::Html => print!("[STATIC]\n{}", output.text), + OutputFormat::Marko => print!("[MARKO]\n{}", output.text), + }, + } + + // Print any errors to stderr with pretty formatting + + // Print output +} diff --git a/src/marko.rs b/src/marko.rs new file mode 100644 index 0000000000000000000000000000000000000000..366530a89b38bf24daeaabab2c5288f9edec37c1 --- /dev/null +++ b/src/marko.rs @@ -0,0 +1,1296 @@ +use oxc_diagnostics::{LabeledSpan, OxcDiagnostic}; +use oxc_span::Span; + +use crate::typescript::{ + parse_call_arguments, parse_expr, parse_expr_without_gt, parse_fn_params_and_body, + parse_var_binding, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Open<'a> { + pub tag_name: &'a str, + pub content: &'a str, + pub self_closing: bool, +} + +impl Open<'_> { + /// Returns the span of the tag name relative to the start of the tag. + pub fn tag_name_span(&self) -> Span { + // fast case, when a literal tag name is passed + if !self.tag_name.is_empty() { + Span::new(1, 1 + (self.tag_name.len() as u32)) + } else { + // slow path, re-parse the js expression. not that deep because you + // only do this in the error case. TODO: actually store `Open` in + // the md tree, the lifetimes are complicated rn + let mut l = LexState::new(self.content); + l.advance(3); + parse_expr(&mut l).expect("validated text should pass"); + Span::new(3, l.offset() as u32) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Close<'a> { + pub tag_name: Option<&'a str>, + pub length: usize, +} + +impl Close<'_> { + /// Returns the span of the tag name relative to the start of the tag. + /// For ``, returns Span(2, 5) pointing to "div". + /// For ``, returns None. + pub fn tag_name_span(&self) -> Option { + self.tag_name.map(|name| { + // Close tag starts with ' { + Open(Open<'a>), + Close(Close<'a>), +} + +pub fn parse_tag(src: &str) -> Result { + assert_eq!(src.as_bytes()[0], b'<'); + if src.starts_with(" { + pub src: &'a str, + pub offset: usize, +} + +impl<'a> LexState<'a> { + pub fn new(src: &'a str) -> LexState<'a> { + LexState { src, offset: 0 } + } + + pub fn offset(&self) -> usize { + self.offset + } + + pub fn advance(&mut self, n: usize) { + self.offset += n; + } + + pub fn restore(&mut self, n: usize) { + self.offset = n; + } + + pub fn peek_byte(&self) -> Option { + self.src.as_bytes().get(self.offset).copied() + } + + pub fn peek_rest(&self) -> &'a str { + &self.src[self.offset..] + } + + pub fn expect_byte(&mut self) -> Result { + if let Some(b) = self.peek_byte() { + self.offset += 1; + return Ok(b); + } + Err( + OxcDiagnostic::error("Unexpected end of file").and_label(LabeledSpan::new( + None, + self.offset, + 1, + )), + ) + } + + pub fn expect(&mut self, expected: &str) -> Result<(), OxcDiagnostic> { + if self.src[self.offset..].starts_with(expected) { + self.offset += expected.len(); + Ok(()) + } else { + Err( + OxcDiagnostic::error(format!("Expected {expected}")).and_label(LabeledSpan::new( + None, + self.offset, + 1, + )), + ) + } + } + + pub fn skip_whitespace(&mut self) { + loop { + match self.peek_byte() { + Some(byte) => match byte { + b' ' | b'\n' | b'\t' | b'\r' => self.offset += 1, + _ => break, + }, + None => break, + } + } + } +} + +pub fn parse_open(src: &str) -> Result { + assert!(src.starts_with("<")); + let mut l = LexState { src, offset: 1 }; + let tag_name = parse_tag_name(&mut l)?; + let is_attribute_tag = tag_name.starts_with('@'); + + // class and id shorthand + let mut has_id = false; + while let Some(byte @ b'#' | byte @ b'.') = l.peek_byte() { + let offset = l.offset; + l.advance(1); + let name = parse_class_name_shorthand(&mut l, byte == b'#')?; + _ = name; + + if byte == b'#' { + if has_id { + return Err( + OxcDiagnostic::error("Cannot specify two ID shorthands".to_string()) + .and_label(LabeledSpan::new(None, offset, name.len())), + ); + } + has_id = true; + } + } + + // in any order: parameters, arguments, variable + // TODO: create errors when things repeat + let mut has_js_arguments = false; + let mut has_js_params = false; + let mut has_js_variable = false; + l.skip_whitespace(); + while let Some(byte @ b'|' | byte @ b'(' | byte @ b'/') = l.peek_byte() { + let offset = l.offset; + + match byte { + b'|' => { + // parameters + l.offset += 1; + l.skip_whitespace(); + while l.peek_byte() != Some(b'|') && l.peek_byte().is_some() { + _ = l.expect("..."); + l.skip_whitespace(); + parse_var_binding(&mut l)?; + l.skip_whitespace(); + if l.expect(",").is_err() { + break; + } + l.skip_whitespace(); + } + l.expect("|")?; + l.skip_whitespace(); + has_js_params = true; + } + b'(' => { + // arguments + // TODO: METHOD SHORTHAND + println!("{}", l.peek_rest()); + parse_call_arguments(&mut l)?; + l.skip_whitespace(); + has_js_arguments = true; + } + b'/' => { + if l.peek_rest().starts_with("/>") { + break; + } + l.offset += 1; + l.skip_whitespace(); + parse_var_binding(&mut l)?; + l.skip_whitespace(); + has_js_variable = true; + } + _ => unreachable!(), + } + + if is_attribute_tag && byte == b'/' { + return Err( + OxcDiagnostic::error("Attribute tags do not support variables") + .and_label(LabeledSpan::new(None, offset, l.offset - offset)), + ); + } + if is_attribute_tag && byte == b'(' { + return Err( + OxcDiagnostic::error("Attribute tags do not support arguments") + .and_label(LabeledSpan::new(None, offset, l.offset - offset)), + ); + } + } + + // keyless value + if l.peek_byte() == Some(b'=') { + l.offset += 1; + l.skip_whitespace(); + parse_expr_without_gt(&mut l)?; + l.skip_whitespace(); + } + + // attributes + let mut self_closing = false; + loop { + match l.peek_byte() { + None => return Err(l.expect_byte().expect_err("err")), + Some(b'>') => { + l.advance(1); + break; + } + Some(_) => { + if l.expect("/>").is_ok() { + self_closing = true; + break; + } + parse_attribute(&mut l)?; + l.skip_whitespace(); + } + } + } + + Ok(Open { + tag_name, + content: &src[0..l.offset], + self_closing, + }) +} + +pub fn parse_close(src: &str) -> Result { + assert!(src.starts_with("') { + l.offset += 1; + Ok(Close { + tag_name: None, + length: l.offset, + }) + } else { + let tag_name = parse_tag_name(&mut l)?; + l.expect(">")?; + Ok(Close { + tag_name: Some(tag_name), + length: l.offset, + }) + } +} + +fn parse_tag_name<'a>(l: &mut LexState<'a>) -> Result<&'a str, OxcDiagnostic> { + if l.peek_rest().starts_with("${") { + l.advance(2); + parse_expr(l)?; + l.expect("}")?; + return Ok(""); + } + let start = l.offset; + loop { + match l.peek_byte() { + Some(b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'-' | b'$' | b'@' | b'0'..=b'9') => { + l.offset += 1 + } + _ => break, + } + } + let tag_name = &l.src[start..l.offset]; + if !tag_name.is_empty() { + Ok(tag_name) + } else { + Err(OxcDiagnostic::error("Expected tag name".to_string()) + .and_label(LabeledSpan::new(None, l.offset, 1))) + } +} + +fn parse_attr_name<'a>(l: &mut LexState<'a>) -> Result<&'a str, OxcDiagnostic> { + let start = l.offset; + loop { + match l.peek_byte() { + Some(b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'-' | b':') => l.offset += 1, + _ => break, + } + } + let tag_name = &l.src[start..l.offset]; + if !tag_name.is_empty() { + Ok(tag_name) + } else { + Err(OxcDiagnostic::error("Expected attribute name".to_string()) + .and_label(LabeledSpan::new(None, l.offset, 1))) + } +} + +fn parse_attribute(l: &mut LexState) -> Result<(), OxcDiagnostic> { + if l.expect("...").is_ok() { + l.skip_whitespace(); + parse_expr_without_gt(l)?; + l.skip_whitespace(); + return Ok(()); + } + + // attribute name + parse_attr_name(l)?; + let has_ws_or_end = l + .peek_byte() + .map(|x| x.is_ascii_whitespace()) + .unwrap_or(false) + || l.peek_rest().starts_with(">") + || l.peek_rest().starts_with("/>"); + l.skip_whitespace(); + + // attribute value + if l.expect(":=").is_ok() || l.expect("=").is_ok() { + l.skip_whitespace(); + parse_expr_without_gt(l)?; + l.skip_whitespace(); + } else if l.peek_byte() == Some(b'(') { + parse_fn_params_and_body(l)?; + l.skip_whitespace(); + } else if !has_ws_or_end { + return Err( + OxcDiagnostic::error("Expected whitespace or '>'".to_string()) + .and_label(LabeledSpan::new(None, l.offset, 1)), + ); + } + + Ok(()) +} + +fn parse_class_name_shorthand<'a>( + l: &mut LexState<'a>, + id: bool, +) -> Result<&'a str, OxcDiagnostic> { + let start = l.offset; + loop { + match l.peek_byte() { + Some(b' ' | b'\n' | b'\t' | b'\r' | b'.' | b'#' | b'>' | b'<' | b'(' | b'|') | None => { + break + } + Some(b'$') => todo!(), + _ => l.offset += 1, + } + } + let tag_name = &l.src[start..l.offset]; + if !tag_name.is_empty() { + Ok(tag_name) + } else { + Err(OxcDiagnostic::error(if id { + "Expected id" + } else { + "Expected class name" + }) + .and_label(LabeledSpan::new(None, l.offset, 1))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_open_basic() { + assert_eq!( + parse_open("
"), + Ok(Open { + tag_name: "div", + content: "
", + self_closing: false, + }) + ); + assert_eq!( + parse_open(" etfdas"), + Ok(Open { + tag_name: "footer", + content: "", + self_closing: true, + }) + ); + assert_eq!( + parse_open(" 4 /> etfdas"), + Ok(Open { + tag_name: "movie", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_open_id() { + assert_eq!( + parse_open("A Technical Review"), + Ok(Open { + tag_name: "h2", + content: "", + self_closing: false, + }) + ); + assert_eq!( + parse_open(" etfdas"), + Ok(Open { + tag_name: "footer", + content: "", + self_closing: true, + }) + ); + assert_eq!( + parse_open(" 4 /> etfdas"), + Ok(Open { + tag_name: "movie", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_open_self_closing_with_space() { + // Self-closing with space before /> works + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "input", + content: "", + self_closing: true, + }) + ); + } + + #[test] + fn test_open_self_closing_no_space() { + assert_eq!( + parse_open("
"), + Ok(Open { + tag_name: "br", + content: "
", + self_closing: true, + }) + ); + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-component", + content: "", + self_closing: true, + }) + ); + } + + #[test] + fn test_open_tag_names() { + // kebab-case + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-custom-tag", + content: "", + self_closing: false, + }) + ); + // PascalCase (for custom tags referencing imports) + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "MyComponent", + content: "", + self_closing: false, + }) + ); + // with underscore + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my_tag", + content: "", + self_closing: false, + }) + ); + // with $ (valid in Marko) + assert_eq!( + parse_open("<$tag>"), + Ok(Open { + tag_name: "$tag", + content: "<$tag>", + self_closing: false, + }) + ); + } + + #[test] + fn test_close() { + assert_eq!( + parse_close(""), + Ok(Close { + tag_name: None, + length: 3 + }) + ); + assert_eq!( + parse_close("ZZZZZZ"), + Ok(Close { + tag_name: None, + length: 3 + }) + ); + assert_eq!( + parse_close("
ZZZZZZ"), + Ok(Close { + tag_name: Some("div"), + length: 6 + }) + ); + } + + #[test] + fn test_close_named() { + assert_eq!( + parse_close(""), + Ok(Close { + tag_name: Some("my-component"), + length: 15 + }) + ); + assert_eq!( + parse_close(""), + Ok(Close { + tag_name: Some("MyComponent"), + length: 14 + }) + ); + } + + #[test] + fn test_attribute_tag_basic_open() { + // @attr tag without self-closing works + assert_eq!( + parse_open("<@header>"), + Ok(Open { + tag_name: "@header", + content: "<@header>", + self_closing: false, + }) + ); + } + + #[test] + fn test_attribute_tag_self_closing() { + assert_eq!( + parse_open("<@item/>"), + Ok(Open { + tag_name: "@item", + content: "<@item/>", + self_closing: true, + }) + ); + } + + #[test] + fn test_attribute_tag_with_string_attr() { + assert_eq!( + parse_open("<@option value=\"foo\">"), + Ok(Open { + tag_name: "@option", + content: "<@option value=\"foo\">", + self_closing: false, + }) + ); + } + + #[test] + fn test_attribute_tag_no_variable() { + // Attribute tags do not support tag variables - correctly errors + let result = parse_open("<@header/myVar>"); + assert!(result.is_err()); + } + + #[test] + fn test_attribute_tag_no_arguments() { + // Attribute tags do not support tag arguments - correctly errors + let result = parse_open("<@header(arg)>"); + assert!(result.is_err()); + } + + // =========================================== + // Shorthand class and id + // =========================================== + + #[test] + fn test_shorthand_id() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "div", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_shorthand_class() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "div", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_shorthand_multiple_classes() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "div", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_shorthand_id_and_classes() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "div", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_shorthand_classes_then_id() { + // Order shouldn't matter + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "div", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_shorthand_duplicate_id_error() { + // Cannot have two ID shorthands - correctly errors + let result = parse_open(""); + assert!(result.is_err()); + } + + #[test] + fn test_shorthand_with_hyphen() { + // Classes/IDs can contain hyphens + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "div", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_shorthand_empty_class_error() { + // Empty class correctly errors + let result = parse_open(""); + assert!(result.is_err()); + } + + #[test] + fn test_shorthand_empty_id_error() { + // Empty id correctly errors + let result = parse_open(""); + assert!(result.is_err()); + } + + // =========================================== + // Tag variables (/var) + // =========================================== + + #[test] + fn test_tag_variable_simple() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + // BUG: Variable binding followed by /> fails + #[test] + fn test_tag_variable_self_closing() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "input", + content: "", + self_closing: true, + }) + ); + } + + #[test] + fn test_tag_variable_destructure_object() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_tag_variable_destructure_array() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_tag_variable_with_type() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "const", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_tag_variable_with_attrs() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + // =========================================== + // Tag arguments (args) + // BUG: Tag arguments without whitespace before ( fail + // The parser only enters argument parsing after skip_whitespace, + // but has no whitespace before ( + // =========================================== + + #[test] + fn test_tag_arguments_empty() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_tag_arguments_simple() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_tag_arguments_expressions() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_tag_arguments_spread() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_tag_arguments_object() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + // Test that tag arguments WITH whitespace work + #[test] + fn test_tag_arguments_with_space() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "my-tag", + content: "", + self_closing: false, + }) + ); + } + + // =========================================== + // Attributes + // =========================================== + + // BUG: String literals fail - expression parser doesn't terminate at > after string + #[test] + fn test_attr_simple_string() { + assert_eq!( + parse_open("
"), + Ok(Open { + tag_name: "div", + content: "
", + self_closing: false, + }) + ); + } + + #[test] + fn test_attr_expression() { + // Identifier expressions work + assert_eq!( + parse_open("
"), + Ok(Open { + tag_name: "div", + content: "
", + self_closing: false, + }) + ); + } + + // BUG: Template literals fail + #[test] + fn test_attr_template_literal() { + assert_eq!( + parse_open("
"), + Ok(Open { + tag_name: "div", + content: "
", + self_closing: false, + }) + ); + } + + #[test] + fn test_attr_spread() { + assert_eq!( + parse_open("
"), + Ok(Open { + tag_name: "div", + content: "
", + self_closing: false, + }) + ); + } + + // BUG: Object literals with > in them fail + #[test] + fn test_attr_spread_object() { + assert_eq!( + parse_open("
"), + Ok(Open { + tag_name: "div", + content: "
", + self_closing: false, + }) + ); + } + + // BUG: Multiple attributes with string values fail + #[test] + fn test_attr_multiple_spreads() { + assert_eq!( + parse_open("
"), + Ok(Open { + tag_name: "div", + content: "
", + self_closing: false, + }) + ); + } + + #[test] + fn test_attr_two_way_binding() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "counter", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_attr_two_way_binding_property() { + assert_eq!( + parse_open(""), + Ok(Open { + tag_name: "counter", + content: "", + self_closing: false, + }) + ); + } + + #[test] + fn test_attr_method_shorthand() { + assert_eq!( + parse_open(" +#[derive(Debug)] +pub struct MarkoInlineTag { + pub open_tag: String, + pub close_tag: String, + pub tag_name: String, +} + +impl NodeValue for MarkoInlineTag { + fn render(&self, node: &Node, fmt: &mut dyn Renderer) { + fmt.text_raw(&self.open_tag); + fmt.contents(&node.children); + fmt.text_raw(&self.close_tag); + } +} + +/// A complete block-level tag on one line: content +/// The content is parsed as inline markdown +#[derive(Debug)] +pub struct MarkoBlockComplete { + pub open_tag: String, + pub close_tag: String, + pub tag_name: String, + pub tag_name_len: usize, +} + +impl NodeValue for MarkoBlockComplete { + fn render(&self, node: &Node, fmt: &mut dyn Renderer) { + fmt.text_raw(&self.open_tag); + fmt.contents(&node.children); + fmt.text_raw(&self.close_tag); + fmt.text_raw("\n"); + } +} + +/// Find a matching close tag in content for the given tag name. +/// Returns (content_before_close, close_tag_text) if found. +/// +/// This function scans forward through the content, tracking nested tags. +/// A close tag only matches the outer tag when nesting depth is zero. +fn find_same_line_close<'a>(content: &'a str, tag_name: &str) -> Option<(&'a str, &'a str)> { + // Stack of open tag names for tracking nesting + let mut stack: Vec<&str> = Vec::new(); + let mut pos = 0; + + while pos < content.len() { + // Find next '<' + let Some(lt_pos) = content[pos..].find('<') else { + break; + }; + let lt_pos = pos + lt_pos; + + // Try to parse as a tag + match marko::parse_tag(&content[lt_pos..]) { + Ok(marko::OpenOrClose::Open(open)) => { + if !open.self_closing { + stack.push(open.tag_name); + } + pos = lt_pos + open.content.len(); + } + Ok(marko::OpenOrClose::Close(close)) => { + if stack.is_empty() { + // No nested tags open - this close tag is for the outer tag + // Check if it matches our tag name or is generic + if close.tag_name.map(|n| n == tag_name).unwrap_or(true) { + let content_before = &content[..lt_pos]; + let close_tag = &content[lt_pos..lt_pos + close.length]; + return Some((content_before, close_tag)); + } + // Named close tag that doesn't match - keep scanning + pos = lt_pos + close.length; + } else { + // Close tag for a nested open tag - pop the stack + // Generic closes the most recent, named must match + if close.tag_name.is_none() { + stack.pop(); + } else if stack.last() == close.tag_name.as_ref() { + stack.pop(); + } + // If it doesn't match, we still continue (malformed nesting) + pos = lt_pos + close.length; + } + } + Err(_) => { + // Not a valid tag, skip past this '<' + pos = lt_pos + 1; + } + } + } + + None +} + +/// Parse Marko tags at block level +pub struct Rule; + +impl BlockRule for Rule { + fn run(state: &mut BlockState) -> Option<(Node, usize)> { + let line = get_line_raw(state, state.line); + let trimmed = line.trim_start(); + if !trimmed.starts_with("<") { + return None; + } + + let unbounded_src = &state.src[state.line_offsets[state.line].first_nonspace..]; + + match marko::parse_tag(unbounded_src) { + Ok(OpenOrClose::Open(open)) => { + // Count how many lines this tag spans + let lines_consumed = 1 + open.content.chars().filter(|&c| c == '\n').count(); + + if open.self_closing { + // - self-closing tag + return Some(( + Node::new(MarkoSelfClosing { + content: open.content.to_string(), + }), + lines_consumed, + )); + } + + // Check if there's content on the same line after the tag + // We need to look at unbounded_src, not trimmed, since tag may span multiple lines + let rest_after_tag = &unbounded_src[open.content.len()..]; + let rest_of_last_line = rest_after_tag.lines().next().unwrap_or(""); + if !rest_of_last_line.trim().is_empty() { + let text = rest_of_last_line.trim(); + + // Check if the same-line content contains a matching close tag + if let Some((content, close_tag)) = find_same_line_close(text, open.tag_name) { + // Complete tag on one line: content + let content = content.trim(); + + // Calculate byte offset for source mapping + let text_start_in_line = open.content.len() + + (rest_of_last_line.len() - rest_of_last_line.trim_start().len()); + let line_start = state.line_offsets[state.line].first_nonspace; + let text_start = line_start + text_start_in_line; + + let mapping = vec![(0, text_start)]; + + let mut node = Node::new(MarkoBlockComplete { + open_tag: open.content.to_string(), + close_tag: close_tag.to_string(), + tag_name: open.tag_name.to_string(), + tag_name_len: open.tag_name.len(), + }); + + if !content.is_empty() { + node.children + .push(Node::new(InlineRoot::new(content.to_string(), mapping))); + } + + return Some((node, lines_consumed)); + } + + // No close tag - just content after open tag + // Parse the text as inline markdown (not wrapped in paragraph) + let text_start_in_line = open.content.len() + + (rest_of_last_line.len() - rest_of_last_line.trim_start().len()); + let line_start = state.line_offsets[state.line].first_nonspace; + let text_start = line_start + text_start_in_line; + + let mapping = vec![(0, text_start)]; + + let mut node = Node::new(MarkoOpenWithText { + open_tag: open.content.to_string(), + tag_name: open.tag_name.to_string(), + tag_name_len: open.tag_name.len(), + }); + node.children + .push(Node::new(InlineRoot::new(text.to_string(), mapping))); + + return Some((node, lines_consumed)); + } + + // Check if next line after the tag has content (non-blank) that isn't another tag + let next_line_idx = state.line + lines_consumed; + if next_line_idx < state.line_max { + let next_line = get_line_raw(state, next_line_idx); + let next_trimmed = next_line.trim(); + if !next_trimmed.is_empty() && !next_trimmed.starts_with("<") { + // Content immediately follows without blank line + // Parse as inline markdown (not wrapped in paragraph) + let text = next_trimmed; + + // Calculate byte offset of next line's text for source mapping + let next_line_start = state.line_offsets[next_line_idx].first_nonspace; + + let mapping = vec![(0, next_line_start)]; + + let mut node = Node::new(MarkoOpenWithText { + open_tag: open.content.to_string(), + tag_name: open.tag_name.to_string(), + tag_name_len: open.tag_name.len(), + }); + node.children + .push(Node::new(InlineRoot::new(text.to_string(), mapping))); + + // Consume the tag lines + the content line + return Some((node, lines_consumed + 1)); + } + } + + // Clean case: open tag on its own line(s), followed by blank or another tag + Some(( + Node::new(MarkoOpen { + content: open.content.to_string(), + tag_name: open.tag_name.to_string(), + tag_name_len: open.tag_name.len(), + }), + lines_consumed, + )) + } + Ok(OpenOrClose::Close(close)) => { + // Close tags should be single line, but check anyway + let lines_consumed = 1 + close.length.saturating_sub(1).min( + unbounded_src[..close.length] + .chars() + .filter(|&c| c == '\n') + .count(), + ); + + // Check if there's content on the same line after the close tag + let close_text = &unbounded_src[..close.length]; + let rest_after_tag = &unbounded_src[close.length..]; + let rest_of_line = rest_after_tag.lines().next().unwrap_or(""); + if !rest_of_line.trim().is_empty() { + // TODO: content after close tag on same line + todo!("content after close tag: {}", rest_of_line.trim()); + } + + Some(( + Node::new(MarkoClose { + content: close_text.to_string(), + tag_name: close.tag_name.map(|s| s.to_string()), + tag_name_len: close.tag_name.map(|s| s.len()), + }), + lines_consumed, + )) + } + Err(err) => Some((Node::new(ErrorBlock { errors: vec![err] }), 1)), + } + } +} + +impl InlineRule for Rule { + const MARKER: char = '<'; + + fn run(state: &mut InlineState) -> Option<(Node, usize)> { + let input = &state.src[state.pos..state.pos_max]; + + // Skip if this doesn't look like a Marko tag + if !input.starts_with("<") { + return None; + } + + // Try to parse as a Marko tag + match marko::parse_tag(input) { + Ok(OpenOrClose::Open(open)) => { + if open.self_closing { + // Self-closing inline tag like
+ return Some(( + Node::new(MarkoSelfClosing { + content: open.content.to_string(), + }), + open.content.len(), + )); + } + // Emit open marker - will be matched by post-processor + Some(( + Node::new(MarkoInlineOpenMarker { + content: open.content.to_string(), + tag_name: open.tag_name.to_string(), + tag_name_len: open.tag_name.len(), + }), + open.content.len(), + )) + } + Ok(OpenOrClose::Close(close)) => { + // Emit close marker - will be matched by post-processor + let close_text = &input[..close.length]; + Some(( + Node::new(MarkoInlineCloseMarker { + content: close_text.to_string(), + tag_name: close.tag_name.map(|s| s.to_string()), + tag_name_len: close.tag_name.map(|s| s.len()), + }), + close.length, + )) + } + Err(_) => { + // Not a valid Marko tag, let other rules handle it + None + } + } + } +} diff --git a/src/plugin/template.rs b/src/plugin/template.rs new file mode 100644 index 0000000000000000000000000000000000000000..c601749aa54ab2707a97ca5ef82ec5c89b8bfb87 --- /dev/null +++ b/src/plugin/template.rs @@ -0,0 +1,63 @@ +use markdown_it::parser::inline::{InlineRule, InlineState}; +use markdown_it::Node; +use oxc_allocator::Allocator; +use oxc_diagnostics::OxcDiagnostic; +use oxc_parser::Parser; +use oxc_span::{GetSpan, SourceType}; + +use crate::plugin::{ErrorBlock, RawBlock}; + +/// Inline rule for ${...} TypeScript expressions. +pub(crate) struct Rule; + +impl InlineRule for Rule { + const MARKER: char = '$'; + + fn run(state: &mut InlineState) -> Option<(Node, usize)> { + let input = &state.src[state.pos..state.pos_max]; + + if !input.starts_with("${") { + return None; + } + + let expr_source = &input["${".len()..]; + + let allocator = Allocator::default(); + let source_type = SourceType::default() + .with_module(true) + .with_typescript(true); + + let expr = match Parser::new(&allocator, expr_source, source_type).parse_expression() { + Ok(expr) => expr, + Err(errors) => { + return Some((Node::new(ErrorBlock { errors }), "${".len())); + } + }; + let mut length = expr.span().end as usize; + + // Verify the next character after whitespace is a } + while let Some(char) = expr_source.as_bytes().get(length) { + match char { + b' ' | b'\t' | b'\n' | b'\r' => length += 1, + _ => break, + } + } + if expr_source.as_bytes().get(length) != Some(&b'}') { + let error = OxcDiagnostic::error("Invalid template expression"); + return Some(( + Node::new(ErrorBlock { + errors: vec![error], + }), + input.len(), + )); + } + + let total_len = "${".len() + length + "}".len(); + let content = &input[..total_len]; + let node = Node::new(RawBlock { + content: content.to_string(), + }); + + Some((node, total_len)) + } +} diff --git a/src/typescript.rs b/src/typescript.rs new file mode 100644 index 0000000000000000000000000000000000000000..f8185edfc720c5766bd1df2a6db077814c6939a2 --- /dev/null +++ b/src/typescript.rs @@ -0,0 +1,790 @@ +//! implement ts partial parsing helpers +use crate::{adjust_err, marko::LexState}; + +use oxc_allocator::Allocator; +use oxc_ast::ast::Expression; +use oxc_diagnostics::{LabeledSpan, OxcDiagnostic}; +use oxc_span::{GetSpan, SourceType}; +use std::borrow::Cow; +use tree_sitter::Parser; + +fn err>>(str: T, offset: usize, length: usize) -> OxcDiagnostic { + OxcDiagnostic::error(str).and_label(LabeledSpan::new(None, offset, length)) +} + +/// TODO: Blow up +pub fn find_statement_extent(source: &str) -> Result { + let mut parser = Parser::new(); + + let language = tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(); + parser + .set_language(&language) + .expect("Failed to load TypeScript grammar"); + + let tree = parser.parse(source, None).ok_or_else(|| { + OxcDiagnostic::error("Failed to parse").and_label(LabeledSpan::new(None, 0, 1)) + })?; + let root = tree.root_node(); + + let first_statement = root.named_child(0).ok_or_else(|| { + OxcDiagnostic::error("Failed to parse").and_label(LabeledSpan::new(None, 0, 1)) + })?; + + if first_statement.kind() == "ERROR" { + return Err(OxcDiagnostic::error("Invalid TypeScript/JavaScript syntax") + .and_label(LabeledSpan::new(None, 0, first_statement.end_byte()))); + } + + let end_byte = first_statement.end_byte(); + + let remaining = &source[end_byte..]; + let same_line = remaining + .find('\n') + .map(|x| &remaining[..x]) + .unwrap_or(remaining); + if !same_line.trim().is_empty() { + return Err(OxcDiagnostic::error("Unexpected content after statement.") + .with_label(LabeledSpan::new(None, end_byte, same_line.len()))); + } + + Ok(end_byte) +} + +fn parse_expr_extra<'alloc, 'src: 'alloc>( + source: &'src str, + offset: isize, + allocator: &'alloc mut Allocator, +) -> Result, OxcDiagnostic> { + if source.is_empty() { + return Err(err( + "Expected expression, found end of file", + offset.max(0).cast_unsigned(), + 1, + )); + } + + let source_type = SourceType::default() + .with_module(true) + .with_typescript(true); + + let expr = match oxc_parser::Parser::new(allocator, source, source_type).parse_expression() { + Ok(expr) => expr, + Err(errors) => { + let first_err = errors.into_iter().next().expect("no errors but no result!"); + let first_err_offset = first_err + .labels + .as_ref() + .ok_or_else(|| err("Expected expression", offset.max(0).cast_unsigned(), 1))? + .first() + .expect("labels, but no labels!") + .offset(); + + let mut candidate = source[0..first_err_offset].trim_end(); + let mut whitespace_groups = 0; + loop { + if candidate.is_empty() { + return Err(adjust_err(first_err, offset)); + } + match oxc_parser::Parser::new(allocator, candidate, source_type).parse_expression() + { + Ok(expr) => break expr, + Err(_) => { + let before_trim = &candidate[..candidate.len() - 1]; + let after_trim = before_trim.trim_end(); + if after_trim.len() < before_trim.len() { + whitespace_groups += 1; + if whitespace_groups > 1 { + return Err(adjust_err(first_err, offset)); + } + } + candidate = after_trim; + } + } + } + } + }; + + let mut expr = expr; + while let Expression::SequenceExpression(seq) = expr { + expr = seq + .unbox() + .expressions + .into_iter() + .next() + .expect("no items"); + } + Ok(expr) +} + +/// parse ts expression, stopping at garbage data / comma +pub fn parse_expr(l: &mut LexState) -> Result { + let mut allocator = Allocator::new(); + let expr = parse_expr_extra(l.peek_rest(), l.offset().cast_signed(), &mut allocator)?; + let span = expr.span(); + let len = (span.end - span.start) as usize; + l.advance(len); + Ok(len) +} + +/// parse ts expression but stop at the first > because of ambiguity with HTML tag end +pub fn parse_expr_without_gt(l: &mut LexState) -> Result { + use oxc_ast::ast::{BinaryOperator, Expression}; + + let mut allocator = Allocator::new(); + let expr = parse_expr_extra(l.peek_rest(), l.offset().cast_signed(), &mut allocator)?; + + let mut e = &expr; + while let Expression::BinaryExpression(bin) = e { + if bin.operator == BinaryOperator::GreaterThan + || bin.operator == BinaryOperator::GreaterEqualThan + || bin.operator == BinaryOperator::ShiftRight + || bin.operator == BinaryOperator::ShiftRightZeroFill + { + e = &bin.left; + } else { + break; + } + } + + let length = e.span().end as usize; + l.advance(length); + Ok(length) +} + +/// parse call arguments including the parentheses: `(a, b, ...c)` +pub fn parse_call_arguments(l: &mut LexState) -> Result { + let source = l.peek_rest(); + if !source.starts_with("(") { + return Err( + OxcDiagnostic::error("Expected `(`").and_label(LabeledSpan::new(None, l.offset(), 1)), + ); + } + + // Prepend `f` to make it a call expression: "f(a, b, c)" + let wrapped = format!("f{source}"); + + let mut allocator = Allocator::default(); + let expr = parse_expr_extra(wrapped.as_str(), l.offset().cast_signed(), &mut allocator)?; + + // Walk down the left side of the AST to find the CallExpression + let call_span = find_leftmost_call(&expr).ok_or_else(|| { + OxcDiagnostic::error("Expected call expression").and_label(LabeledSpan::new( + None, + l.offset(), + 1, + )) + })?; + + // Subtract the `f` prefix we added + let length = call_span.end as usize - 1; + l.advance(length); + Ok(length) +} + +/// parse variable binding. identifier or destructuring pattern +/// also parses optional `: Type` +pub fn parse_var_binding(l: &mut LexState) -> Result { + let offset = l.offset(); + let source = l.peek_rest(); + if source.is_empty() { + return Err(err("Expected variable binding", l.offset(), 1)); + } + + let mut allocator = Allocator::default(); + let end = { + let expr = parse_expr_extra(source, l.offset().cast_signed(), &mut allocator)?; + let span = find_leftmost(&expr, LeftmostSearch::Assignment).unwrap(); + span.end as usize + }; + allocator.reset(); + + { + let prefix = "function f("; + let source = format!("{prefix}{}) {{}}", &source[0..end]); + parse_expr_extra( + source.as_str(), + l.offset().cast_signed() - prefix.len().cast_signed(), + &mut allocator, + )?; + } + + l.advance(end); + + let snapshot = l.offset(); + l.skip_whitespace(); + if l.expect(":").is_ok() { + l.skip_whitespace(); + parse_type(l)?; + Ok(l.offset() - offset) + } else { + l.offset = snapshot; + Ok(snapshot - offset) + } +} + +/// parses a type +pub fn parse_type(l: &mut LexState) -> Result { + let mut allocator = Allocator::default(); + let source = format!("T as {}", l.peek_rest()); + println!("{{{source}}} HUH"); + let expr = parse_expr_extra(source.as_str(), l.offset().cast_signed(), &mut allocator)?; + println!("{{{expr:#?}}}"); + + let span = find_leftmost(&expr, LeftmostSearch::As).ok_or_else(|| { + err( + "Expected type expression", + l.offset(), + expr.span().end as usize, + ) + })?; + let len = (span.end - span.start) as usize; + + l.advance(len); + Ok(len) +} + +/// parses `(params): ReturnType { body }` +pub fn parse_fn_params_and_body(l: &mut LexState) -> Result { + let source = l.peek_rest(); + if !source.starts_with("(") { + return Err( + OxcDiagnostic::error("Expected `(`").and_label(LabeledSpan::new(None, l.offset(), 1)), + ); + } + + // Prepend `function f` to make it a function expression + let prefix = "function f"; + let wrapped = format!("{prefix}{source}"); + + let mut allocator = Allocator::default(); + + let expr = parse_expr_extra(&wrapped, l.offset().cast_signed(), &mut allocator)?; + + // Find the function expression (walk left side of any binary/etc expressions) + let func_span = find_leftmost(&expr, LeftmostSearch::Function).ok_or_else(|| { + OxcDiagnostic::error("Expected function expression").and_label(LabeledSpan::new( + None, + l.offset(), + 1, + )) + })?; + + // Subtract the prefix we added + let length = func_span.end as usize - prefix.len(); + l.advance(length); + Ok(length) +} + +/// Walk down the left/object side of an expression to find a CallExpression. +fn find_leftmost_call(expr: &oxc_ast::ast::Expression) -> Option { + use oxc_ast::ast::Expression; + + match expr { + Expression::CallExpression(call) => Some(call.span), + Expression::BinaryExpression(bin) => find_leftmost_call(&bin.left), + Expression::LogicalExpression(log) => find_leftmost_call(&log.left), + Expression::AssignmentExpression(assign) => find_leftmost_call(&assign.right), + Expression::ConditionalExpression(cond) => find_leftmost_call(&cond.test), + Expression::TaggedTemplateExpression(tag) => find_leftmost_call(&tag.tag), + Expression::ComputedMemberExpression(mem) => find_leftmost_call(&mem.object), + Expression::StaticMemberExpression(mem) => find_leftmost_call(&mem.object), + Expression::PrivateFieldExpression(mem) => find_leftmost_call(&mem.object), + _ => None, + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +enum LeftmostSearch { + As, + Function, + Assignment, +} + +/// Walk down the left/object side of an expression to find a FunctionExpression. +fn find_leftmost(expr: &oxc_ast::ast::Expression, kind: LeftmostSearch) -> Option { + use oxc_ast::ast::Expression::*; + + match expr { + FunctionExpression(func) if kind == LeftmostSearch::Function => Some(func.span), + TSAsExpression(expr) if kind == LeftmostSearch::As => { + let mut expr = expr; + loop { + match &expr.expression { + TSAsExpression(nested) => expr = nested, + _ => return Some(expr.type_annotation.span()), + } + } + } + Identifier(_) | ObjectExpression(_) | ArrayExpression(_) + if kind == LeftmostSearch::Assignment => + { + Some(expr.span()) + } + + AssignmentExpression(assign) => find_leftmost_assign(&assign.left, kind), + UpdateExpression(seq) => Some(seq.argument.span()), + + TSAsExpression(expr) => find_leftmost(&expr.expression, kind), + TSNonNullExpression(expr) => find_leftmost(&expr.expression, kind), + TSSatisfiesExpression(expr) => find_leftmost(&expr.expression, kind), + TSInstantiationExpression(expr) => find_leftmost(&expr.expression, kind), + BinaryExpression(bin) => find_leftmost(&bin.left, kind), + LogicalExpression(log) => find_leftmost(&log.left, kind), + ConditionalExpression(cond) => find_leftmost(&cond.test, kind), + TaggedTemplateExpression(tag) => find_leftmost(&tag.tag, kind), + ComputedMemberExpression(mem) => find_leftmost(&mem.object, kind), + StaticMemberExpression(mem) => find_leftmost(&mem.object, kind), + PrivateFieldExpression(mem) => find_leftmost(&mem.object, kind), + CallExpression(call) => find_leftmost(&call.callee, kind), + SequenceExpression(seq) => find_leftmost(seq.expressions.first().unwrap(), kind), + + BooleanLiteral(_) + | NullLiteral(_) + | NumericLiteral(_) + | BigIntLiteral(_) + | RegExpLiteral(_) + | StringLiteral(_) + | TemplateLiteral(_) + | Identifier(_) + | MetaProperty(_) + | Super(_) + | ArrayExpression(_) + | ArrowFunctionExpression(_) + | AwaitExpression(_) + | ChainExpression(_) + | ClassExpression(_) + | FunctionExpression(_) + | ImportExpression(_) + | NewExpression(_) + | ObjectExpression(_) + | ParenthesizedExpression(_) + | ThisExpression(_) + | UnaryExpression(_) + | YieldExpression(_) + | PrivateInExpression(_) + | JSXElement(_) + | JSXFragment(_) + | TSTypeAssertion(_) => None, + V8IntrinsicExpression(_) => unreachable!(), + } +} + +fn find_leftmost_assign( + expr: &oxc_ast::ast::AssignmentTarget, + kind: LeftmostSearch, +) -> Option { + use oxc_ast::ast::AssignmentTarget::*; + + match expr { + TSAsExpression(expr) if kind == LeftmostSearch::As => Some(expr.type_annotation.span()), + + AssignmentTargetIdentifier(_) | ArrayAssignmentTarget(_) | ObjectAssignmentTarget(_) + if kind == LeftmostSearch::Assignment => + { + Some(expr.span()) + } + + AssignmentTargetIdentifier(_) + | TSAsExpression(_) + | TSSatisfiesExpression(_) + | TSNonNullExpression(_) + | TSTypeAssertion(_) + | ComputedMemberExpression(_) + | StaticMemberExpression(_) + | ArrayAssignmentTarget(_) + | ObjectAssignmentTarget(_) + | PrivateFieldExpression(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse_expr(source: &str) -> Result { + let mut l = LexState::new(source); + super::parse_expr(&mut l) + } + + #[test] + fn test_simple_identifier() { + assert_eq!(parse_expr("foo"), Ok(3)); + assert_eq!(parse_expr("bar123"), Ok(6)); + } + + #[test] + fn test_number_literals() { + assert_eq!(parse_expr("42"), Ok(2)); + assert_eq!(parse_expr("3.14"), Ok(4)); + assert_eq!(parse_expr("0xff"), Ok(4)); + } + + #[test] + fn test_string_literals() { + assert_eq!(parse_expr("\"hello\""), Ok(7)); + assert_eq!(parse_expr("'world'"), Ok(7)); + assert_eq!(parse_expr("`template`"), Ok(10)); + } + + #[test] + fn test_binary_expressions() { + assert_eq!(parse_expr("a + b"), Ok(5)); + assert_eq!(parse_expr("x * y / z"), Ok(9)); + } + + #[test] + fn test_call_expression() { + assert_eq!(parse_expr("foo()"), Ok(5)); + assert_eq!(parse_expr("foo(a, b)"), Ok(9)); + assert_eq!(parse_expr("foo.bar()"), Ok(9)); + } + + #[test] + fn test_with_trailing_garbage() { + assert_eq!(parse_expr("foo > more garbage"), Ok(10)); + assert_eq!(parse_expr("42 > stuff"), Ok(10)); + assert_eq!(parse_expr("foo @@@"), Ok(3)); + assert_eq!(parse_expr("foo 🩵"), Ok(3)); + assert_eq!(parse_expr("foo, bar, @@@"), Ok(3)); + } + + #[test] + fn test_member_expression() { + assert_eq!(parse_expr("obj.prop"), Ok(8)); + assert_eq!(parse_expr("a.b.c"), Ok(5)); + assert_eq!(parse_expr("arr[0]"), Ok(6)); + } + + #[test] + fn test_arrow_function() { + assert_eq!(parse_expr("() => 42"), Ok(8)); + assert_eq!(parse_expr("x => x + 1"), Ok(10)); + assert_eq!(parse_expr("(a, b) => a + b"), Ok(15)); + } + + #[test] + fn test_object_literal() { + assert_eq!(parse_expr("{ a: 1 }"), Ok(8)); + assert_eq!(parse_expr("{ a: 1, b: 2 }"), Ok(14)); + } + + #[test] + fn test_array_literal() { + assert_eq!(parse_expr("[1, 2, 3]"), Ok(9)); + assert_eq!(parse_expr("[]"), Ok(2)); + } + + #[test] + fn test_empty_input() { + let result = parse_expr(""); + assert!(result.is_err()); + } + + #[test] + fn test_invalid_syntax() { + let result = parse_expr("@@@"); + assert!(result.is_err()); + } + + fn parse_expr_no_gt(source: &str) -> Result { + let mut l = LexState::new(source); + parse_expr_without_gt(&mut l) + } + + #[test] + fn test_no_gt_simple() { + assert_eq!(parse_expr_no_gt("foo"), Ok(3)); + assert_eq!(parse_expr_no_gt("42"), Ok(2)); + assert_eq!(parse_expr_no_gt("a + b"), Ok(5)); + } + + #[test] + fn test_no_gt_stops_at_gt() { + assert_eq!(parse_expr_no_gt("a > b"), Ok(1)); // just "a" + assert_eq!(parse_expr_no_gt("foo > bar"), Ok(3)); // just "foo" + assert_eq!(parse_expr_no_gt("x + 1 > y"), Ok(5)); // "x + 1" + assert_eq!(parse_expr_no_gt("x + 1 > y > z > y"), Ok(5)); // "x + 1" + } + + #[test] + fn test_no_gt_parens_protect() { + // Parentheses protect the `>` + assert_eq!(parse_expr_no_gt("(a > b)"), Ok(7)); + assert_eq!(parse_expr_no_gt("(x > y) + z"), Ok(11)); + } + + #[test] + fn test_no_gt_brackets_protect() { + // Array access brackets protect the `>` + assert_eq!(parse_expr_no_gt("arr[a > b]"), Ok(10)); + assert_eq!(parse_expr_no_gt("obj[x > 0]"), Ok(10)); + } + + #[test] + fn test_no_gt_call_parens_protect() { + // Function call parens protect the `>` + assert_eq!(parse_expr_no_gt("fn(a > b)"), Ok(9)); + assert_eq!(parse_expr_no_gt("Math.max(x, y > 0 ? 1 : 0)"), Ok(26)); + } + + #[test] + fn test_no_gt_gte_ok() { + assert_eq!(parse_expr_no_gt("a >= b"), Ok(1)); + assert_eq!(parse_expr_no_gt("x >= 0"), Ok(1)); + } + + #[test] + fn test_no_gt_shift_ok() { + assert_eq!(parse_expr_no_gt("a >> b"), Ok(1)); + assert_eq!(parse_expr_no_gt("x >>> 2"), Ok(1)); + } + + fn parse_call_args(source: &str) -> Result { + let mut l = LexState::new(source); + parse_call_arguments(&mut l) + } + + #[test] + fn test_call_args_empty() { + assert_eq!(parse_call_args("()"), Ok(2)); + assert_eq!(parse_call_args("() garbage"), Ok(2)); + } + + #[test] + fn test_call_args_simple() { + assert_eq!(parse_call_args("(a)"), Ok(3)); + assert_eq!(parse_call_args("(a, b)"), Ok(6)); + assert_eq!(parse_call_args("(a, b, c)"), Ok(9)); + } + + #[test] + fn test_call_args_expressions() { + assert_eq!(parse_call_args("(1 + 2)"), Ok(7)); + assert_eq!(parse_call_args("(a + b, c * d)"), Ok(14)); + assert_eq!(parse_call_args("(foo())"), Ok(7)); + } + + #[test] + fn test_call_args_spread() { + assert_eq!(parse_call_args("(...args)"), Ok(9)); + assert_eq!(parse_call_args("(a, ...rest)"), Ok(12)); + } + + #[test] + fn test_call_args_trailing_garbage() { + assert_eq!(parse_call_args("(a, b) + more"), Ok(6)); + assert_eq!(parse_call_args("(x).foo"), Ok(3)); + assert_eq!(parse_call_args("(a, b)[0]"), Ok(6)); + } + + #[test] + fn test_call_args_nested() { + assert_eq!(parse_call_args("(fn(a, b))"), Ok(10)); + assert_eq!(parse_call_args("((a + b))"), Ok(9)); + } + + #[test] + fn test_call_args_no_paren() { + let result = parse_call_args("a, b)"); + assert!(result.is_err()); + } + + fn parse_fn_params_body(source: &str) -> Result { + let mut l = LexState::new(source); + parse_fn_params_and_body(&mut l) + } + + #[test] + fn test_fn_simple() { + assert_eq!(parse_fn_params_body("() {}"), Ok(5)); + assert_eq!(parse_fn_params_body("() { }"), Ok(6)); + } + + #[test] + fn test_fn_with_params() { + assert_eq!(parse_fn_params_body("(a) {}"), Ok(6)); + assert_eq!(parse_fn_params_body("(a, b) {}"), Ok(9)); + assert_eq!(parse_fn_params_body("(a, b, c) {}"), Ok(12)); + } + + #[test] + fn test_fn_with_body() { + assert_eq!(parse_fn_params_body("() { return 42; }"), Ok(17)); + assert_eq!(parse_fn_params_body("(x) { return x + 1; }"), Ok(21)); + } + + #[test] + fn test_fn_with_return_type() { + assert_eq!(parse_fn_params_body("(): void {}"), Ok(11)); + assert_eq!(parse_fn_params_body("(): number { return 42; }"), Ok(25)); + assert_eq!( + parse_fn_params_body("(a: number): string { return String(a); }"), + Ok(41) + ); + } + + #[test] + fn test_fn_with_typed_params() { + assert_eq!(parse_fn_params_body("(a: number) {}"), Ok(14)); + assert_eq!(parse_fn_params_body("(a: string, b: number) {}"), Ok(25)); + } + + #[test] + fn test_fn_with_defaults() { + assert_eq!(parse_fn_params_body("(a = 1) {}"), Ok(10)); + assert_eq!(parse_fn_params_body("(a: number = 1) {}"), Ok(18)); + } + + #[test] + fn test_fn_with_rest() { + assert_eq!(parse_fn_params_body("(...args) {}"), Ok(12)); + assert_eq!(parse_fn_params_body("(a, ...rest) {}"), Ok(15)); + } + + #[test] + fn test_fn_trailing_garbage() { + // Should stop at the closing brace + assert_eq!(parse_fn_params_body("() {} garbage"), Ok(5)); + assert_eq!(parse_fn_params_body("() {} + more"), Ok(5)); + } + + #[test] + fn test_fn_no_paren() { + let result = parse_fn_params_body("a) {}"); + assert!(result.is_err()); + } + + fn parse_var_binding(source: &str) -> Result { + let mut l = LexState::new(source); + super::parse_var_binding(&mut l) + } + + #[test] + fn test_var_binding_identifier() { + assert_eq!(parse_var_binding("foo"), Ok(3)); + assert_eq!(parse_var_binding("bar123"), Ok(6)); + assert_eq!(parse_var_binding("_private"), Ok(8)); + } + + #[test] + fn test_var_binding_with_type() { + assert_eq!(parse_var_binding("foo: string meow"), Ok(11)); + assert_eq!(parse_var_binding("bar: number"), Ok(11)); + assert_eq!(parse_var_binding("x: boolean"), Ok(10)); + } + + #[test] + fn test_var_binding_object_destructure() { + assert_eq!(parse_var_binding("{ a }"), Ok(5)); + assert_eq!(parse_var_binding("{ a, b }"), Ok(8)); + assert_eq!(parse_var_binding("{ a: x, b: y }"), Ok(14)); + } + + #[test] + fn test_var_binding_array_destructure() { + assert_eq!(parse_var_binding("[a]"), Ok(3)); + assert_eq!(parse_var_binding("[a, b]"), Ok(6)); + assert_eq!(parse_var_binding("[a, , b]"), Ok(8)); + } + + #[test] + fn test_var_binding_destructure_with_type() { + assert_eq!(parse_var_binding("{ a, b }: T abc"), Ok(11)); + assert_eq!(parse_var_binding("[x, y]: [number, number], yolo"), Ok(24)); + } + + #[test] + fn test_var_binding_no_default() { + assert_eq!(parse_var_binding("foo = 123"), Ok(3)); + assert_eq!(parse_var_binding("foo: string = 'hi'"), Ok(11)); + } + + #[test] + fn test_var_binding_empty() { + let result = parse_var_binding(""); + assert!(result.is_err()); + } + + #[test] + fn test_var_binding_identifier_garbage() { + assert_eq!(parse_var_binding("foo meow meow meow"), Ok(3)); + assert_eq!(parse_var_binding("foo|etc"), Ok(3)); + assert_eq!(parse_var_binding("foo|etc, 1"), Ok(3)); + assert_eq!(parse_var_binding("nya: string, 💥"), Ok(11)); + } + + #[test] + fn test_var_binding_identifier_invalid_destructure() { + assert!(parse_var_binding("{ x: y + 2 }").is_err()); + assert!(parse_var_binding("[ y + 2 ]").is_err()); + } + + #[test] + fn test_var_binding_identifier_unicode() { + assert_eq!(parse_var_binding("café"), Ok(5)); + } + + #[test] + fn test_stmt_function_with_garbage() { + let source = "function hello() {\n console.log(1);\n}\n\nrandom markdown garbage"; + let end = find_statement_extent(source).unwrap(); + assert_eq!(&source[..end], "function hello() {\n console.log(1);\n}"); + } + + #[test] + fn test_stmt_import_with_garbage() { + let source = "import { foo } from 'bar';\n\n# markdown heading"; + let end = find_statement_extent(source).unwrap(); + assert_eq!(&source[..end], "import { foo } from 'bar';"); + } + + #[test] + fn test_stmt_interface_with_garbage() { + let source = "interface Foo {\n bar: string;\n}\n\nsome text"; + let end = find_statement_extent(source).unwrap(); + assert_eq!(&source[..end], "interface Foo {\n bar: string;\n}"); + } + + #[test] + fn test_stmt_const_declaration() { + let source = "const answer = 42;\n\n# Next section"; + let end = find_statement_extent(source).unwrap(); + assert_eq!(&source[..end], "const answer = 42;"); + } + + #[test] + fn test_stmt_expression_statement() { + let source = "console.log('hello');\n\nmore content"; + let end = find_statement_extent(source).unwrap(); + assert_eq!(&source[..end], "console.log('hello');"); + } + + #[test] + fn test_stmt_pure_garbage() { + let source = "random unexpected garbage"; + let result = find_statement_extent(source); + assert!(result.is_err()); + } + + #[test] + fn test_stmt_multiline_function() { + let source = "function sort(items) {\n while (!isSorted()) {\n shuffle(items);\n }\n return items;\n}\n\n# Heading"; + let end = find_statement_extent(source).unwrap(); + assert!(source[..end].contains("return items;")); + assert!(source[..end].contains("}")); + } + + #[test] + fn test_stmt_trailing_content_on_same_line() { + let source = "const x = 1; const y = 2;"; + let result = find_statement_extent(source); + assert!(result.is_err()); + } + + #[test] + fn test_stmt_trailing_whitespace_ok() { + let source = "const x = 1; \n\nmore content"; + let result = find_statement_extent(source); + assert!(result.is_ok()); + } +} diff --git a/src/wasm.rs b/src/wasm.rs new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/fixtures.rs b/tests/fixtures.rs new file mode 100644 index 0000000000000000000000000000000000000000..5546dd3ba6648e6454c76211cb698ffb5192a5ad --- /dev/null +++ b/tests/fixtures.rs @@ -0,0 +1,149 @@ +use markodown::{transform, OutputFormat}; +use std::fs; +use std::path::Path; + +/// Load a fixture pair and test the transform. +/// +/// Each fixture has an `.mdo` input and either an `.html` expected output +/// (static mode) or a `.marko` expected output (marko mode), or both. +fn run_fixture(name: &str) { + let base = Path::new("tests/fixtures").join(name); + let input_path = base.with_extension("mdo"); + let html_path = base.with_extension("html"); + let marko_path = base.with_extension("marko"); + + let source = fs::read_to_string(&input_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", input_path.display())); + + // Test HTML output if expected file exists + if html_path.exists() { + let expected = fs::read_to_string(&html_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", html_path.display())); + + let output = transform(&source, Some(OutputFormat::Html)).unwrap(); + + assert_eq!( + output.format, + OutputFormat::Html, + "[{name}] expected Html output but got Marko" + ); + + assert_eq!( + output.text.trim(), + expected.trim(), + "\n\n[{name}] HTML output mismatch\n\n--- expected ---\n{expected}\n--- actual ---\n{}\n", + output.text + ); + } + // Test Marko output if expected file exists + else if marko_path.exists() { + let expected = fs::read_to_string(&marko_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", marko_path.display())); + + let output = transform(&source, Some(OutputFormat::Marko)).unwrap(); + + assert_eq!( + output.format, + OutputFormat::Marko, + "[{name}] expected Marko output but got Html" + ); + + assert_eq!( + output.text.trim(), + expected.trim(), + "\n\n[{name}] Marko output mismatch\n\n--- expected ---\n{expected}\n--- actual ---\n{}\n", + output.text + ); + } else { + panic!("no test defined") + } +} + +#[test] +fn fixture_01_static_markdown() { + run_fixture("01-static-markdown"); +} + +// The tests below have .marko expected outputs. They will be validated +// once the marko renderer and custom parse rules are implemented. +// For now they just verify the transform doesn't panic. + +#[test] +fn fixture_02_imports_and_components() { + run_fixture("02-imports-and-components"); +} + +#[test] +fn fixture_03_static_and_expressions() { + run_fixture("03-static-and-expressions"); +} + +#[test] +fn fixture_04_control_flow() { + run_fixture("04-control-flow"); +} + +#[test] +fn fixture_05_nested_tags_and_loops() { + run_fixture("05-nested-tags-and-loops"); +} + +#[test] +fn fixture_06_comments() { + run_fixture("06-comments"); +} + +#[test] +fn fixture_07_server_client() { + run_fixture("07-server-client"); +} + +#[test] +fn fixture_08_inline_tags() { + run_fixture("08-inline-tags"); +} + +#[test] +fn fixture_09_marko_shorthands() { + run_fixture("09-marko-shorthands"); +} + +#[test] +fn fixture_10_attribute_tags() { + run_fixture("10-attribute-tags"); +} + +#[test] +fn fixture_11_autolinks() { + run_fixture("11-autolinks"); +} + +#[test] +fn fixture_12_frontmatter() { + run_fixture("12-frontmatter"); +} + +#[test] +fn fixture_13_kitchen_sink() { + run_fixture("13-kitchen-sink"); +} + +#[test] +fn fixture_14_evil_braces_and_typescript() { + run_fixture("14-evil-braces-and-typescript"); +} + +#[test] +fn fixture_15_multiline_tag() { + run_fixture("15-multiline-tag"); +} + +#[test] +fn fixture_16_same_line_text() { + run_fixture("16-same-line-text"); +} + +#[test] +fn fixture_17_consecutive_statements() { + run_fixture("17-consecutive-statements"); +} diff --git a/tests/fixtures/01-static-markdown.html b/tests/fixtures/01-static-markdown.html new file mode 100644 index 0000000000000000000000000000000000000000..ac2b8585b6dff6b27cab3a151808651bc3ef6221 --- /dev/null +++ b/tests/fixtures/01-static-markdown.html @@ -0,0 +1,14 @@ +

Hello World

+

This is a simple document with emphasis.

+
    +
  • one
    • +
  • two
    • +
  • three
    • +
+

Code

+
fn main() {}
+
+
+

A blockquote.

+
+

A link and an image.

diff --git a/tests/fixtures/01-static-markdown.mdo b/tests/fixtures/01-static-markdown.mdo new file mode 100644 index 0000000000000000000000000000000000000000..493505126c034df51bb167e633c45f7ab4df22d2 --- /dev/null +++ b/tests/fixtures/01-static-markdown.mdo @@ -0,0 +1,17 @@ +# Hello World + +This is a **simple** document with *emphasis*. + +- one +- two +- three + +## Code + +```rust +fn main() {} +``` + +> A blockquote. + +A [link](https://example.com) and an ![image](logo.png). diff --git a/tests/fixtures/02-imports-and-components.marko b/tests/fixtures/02-imports-and-components.marko new file mode 100644 index 0000000000000000000000000000000000000000..414c86dc1c3e502e4dce0b774eeba406b1648bcc --- /dev/null +++ b/tests/fixtures/02-imports-and-components.marko @@ -0,0 +1,5 @@ +import Chart from "./chart.marko"; +import { sum } from "./math.js"; +

Dashboard

+

Here is the chart:

+ diff --git a/tests/fixtures/02-imports-and-components.mdo b/tests/fixtures/02-imports-and-components.mdo new file mode 100644 index 0000000000000000000000000000000000000000..90751e66a22b2d853eaf241728759814077ea291 --- /dev/null +++ b/tests/fixtures/02-imports-and-components.mdo @@ -0,0 +1,8 @@ +import Chart from "./chart.marko"; +import { sum } from "./math.js"; + +# Dashboard + +Here is the **chart**: + + diff --git a/tests/fixtures/03-static-and-expressions.marko b/tests/fixtures/03-static-and-expressions.marko new file mode 100644 index 0000000000000000000000000000000000000000..5e261eeabe6219b7dc55b8df0567d77a19a238f8 --- /dev/null +++ b/tests/fixtures/03-static-and-expressions.marko @@ -0,0 +1,6 @@ +static const answer = 41; +static function getAnswer() { + return answer + 1; +} +

The Answer

+

The answer is ${getAnswer()}.

diff --git a/tests/fixtures/03-static-and-expressions.mdo b/tests/fixtures/03-static-and-expressions.mdo new file mode 100644 index 0000000000000000000000000000000000000000..ffea8cbb428378cea1f0e000698ba3f2b6ecee0f --- /dev/null +++ b/tests/fixtures/03-static-and-expressions.mdo @@ -0,0 +1,8 @@ +static const answer = 41; +static function getAnswer() { + return answer + 1; +} + +# The Answer + +The answer is **${getAnswer()}**. diff --git a/tests/fixtures/04-control-flow.marko b/tests/fixtures/04-control-flow.marko new file mode 100644 index 0000000000000000000000000000000000000000..b2fced95042b4d8111933c1e032f460af36d8e72 --- /dev/null +++ b/tests/fixtures/04-control-flow.marko @@ -0,0 +1,11 @@ +

Welcome

+ +

Welcome to the site.

+
    +
  • item one
    • +
  • item two
    • +
+ + +

Goodbye, see you later.

+ diff --git a/tests/fixtures/04-control-flow.mdo b/tests/fixtures/04-control-flow.mdo new file mode 100644 index 0000000000000000000000000000000000000000..6f1d6f23f44389cca89e6f6e5a67cd99b58e6491 --- /dev/null +++ b/tests/fixtures/04-control-flow.mdo @@ -0,0 +1,15 @@ +# Welcome + + + + Welcome to the **site**. + + - item one + - item two + + + + + **Goodbye**, see you later. + + diff --git a/tests/fixtures/05-nested-tags-and-loops.marko b/tests/fixtures/05-nested-tags-and-loops.marko new file mode 100644 index 0000000000000000000000000000000000000000..23adc10532b759a7538108ec581bf12322c17178 --- /dev/null +++ b/tests/fixtures/05-nested-tags-and-loops.marko @@ -0,0 +1,11 @@ +import Chart from "./chart.marko"; +

Report

+ +

Details

+

Here is a detailed breakdown:

+ +

Q${quarter.num}

+

Revenue: ${quarter.revenue}

+ + + diff --git a/tests/fixtures/05-nested-tags-and-loops.mdo b/tests/fixtures/05-nested-tags-and-loops.mdo new file mode 100644 index 0000000000000000000000000000000000000000..3df7939ffcaf129168905a435acce992049518bb --- /dev/null +++ b/tests/fixtures/05-nested-tags-and-loops.mdo @@ -0,0 +1,19 @@ +import Chart from "./chart.marko"; + +# Report + + + +## Details + +Here is a detailed **breakdown**: + + + +### Q${quarter.num} + +Revenue: **${quarter.revenue}** + + + + diff --git a/tests/fixtures/06-comments.marko b/tests/fixtures/06-comments.marko new file mode 100644 index 0000000000000000000000000000000000000000..24edffbde81b202df687ca43cdbffd82ea225600 --- /dev/null +++ b/tests/fixtures/06-comments.marko @@ -0,0 +1,4 @@ +

hello world

+// this is a comment + +

good morning

diff --git a/tests/fixtures/06-comments.mdo b/tests/fixtures/06-comments.mdo new file mode 100644 index 0000000000000000000000000000000000000000..c8b8504a46c7d84659bfd56518d77bb7fa4cb6b7 --- /dev/null +++ b/tests/fixtures/06-comments.mdo @@ -0,0 +1,7 @@ +hello world + +// this is a comment + + + +good morning diff --git a/tests/fixtures/07-server-client.marko b/tests/fixtures/07-server-client.marko new file mode 100644 index 0000000000000000000000000000000000000000..fb6326d8d48b3357477cdcda552a4454b518e54e --- /dev/null +++ b/tests/fixtures/07-server-client.marko @@ -0,0 +1,4 @@ +server console.log("on the server") +client console.log("in the browser") +

Hello

+

This is a page.

diff --git a/tests/fixtures/07-server-client.mdo b/tests/fixtures/07-server-client.mdo new file mode 100644 index 0000000000000000000000000000000000000000..b4505cf461d0f3d9b155a62bd977ae7331de8df8 --- /dev/null +++ b/tests/fixtures/07-server-client.mdo @@ -0,0 +1,6 @@ +server console.log("on the server") +client console.log("in the browser") + +# Hello + +This is a page. diff --git a/tests/fixtures/08-inline-tags.marko b/tests/fixtures/08-inline-tags.marko new file mode 100644 index 0000000000000000000000000000000000000000..7d33edf422903460e3501a3473f45d2f7d69acdf --- /dev/null +++ b/tests/fixtures/08-inline-tags.marko @@ -0,0 +1,4 @@ +

Inline

+

Click to continue.

+

This has a premium feature and bold text.

+

You can go anywhere with inline tags.

diff --git a/tests/fixtures/08-inline-tags.mdo b/tests/fixtures/08-inline-tags.mdo new file mode 100644 index 0000000000000000000000000000000000000000..1f1e80b9feb9232b34bfe24d4dd2fe3b99c3908d --- /dev/null +++ b/tests/fixtures/08-inline-tags.mdo @@ -0,0 +1,7 @@ +# Inline + +Click to continue. + +This has a premium feature and **bold** text. + +You can go anywhere with inline tags. diff --git a/tests/fixtures/09-marko-shorthands.marko b/tests/fixtures/09-marko-shorthands.marko new file mode 100644 index 0000000000000000000000000000000000000000..bffbb8750051be29ccc26714ae475c9c1959b0d1 --- /dev/null +++ b/tests/fixtures/09-marko-shorthands.marko @@ -0,0 +1,5 @@ + +

Welcome

+Important stuff +
+ diff --git a/tests/fixtures/09-marko-shorthands.mdo b/tests/fixtures/09-marko-shorthands.mdo new file mode 100644 index 0000000000000000000000000000000000000000..ae75934bbb0013c60ba2925546daa32024e6172f --- /dev/null +++ b/tests/fixtures/09-marko-shorthands.mdo @@ -0,0 +1,9 @@ + + +# Welcome + +Important **stuff** + +
+ + diff --git a/tests/fixtures/10-attribute-tags.marko b/tests/fixtures/10-attribute-tags.marko new file mode 100644 index 0000000000000000000000000000000000000000..8a47bd164e6dd726f1a17b3a696ba591d465b8eb --- /dev/null +++ b/tests/fixtures/10-attribute-tags.marko @@ -0,0 +1,6 @@ + +<@header class="fancy"> +

Big things are coming!

+ +

Regular page content in markdown.

+ diff --git a/tests/fixtures/10-attribute-tags.mdo b/tests/fixtures/10-attribute-tags.mdo new file mode 100644 index 0000000000000000000000000000000000000000..979777d340e6c844ede00cd990f69eeed7f75d3a --- /dev/null +++ b/tests/fixtures/10-attribute-tags.mdo @@ -0,0 +1,10 @@ + +<@header class="fancy"> + +# Big things are coming! + + + +Regular page content in **markdown**. + + diff --git a/tests/fixtures/11-autolinks.marko b/tests/fixtures/11-autolinks.marko new file mode 100644 index 0000000000000000000000000000000000000000..1133b8f044bed01c771c8e23b22a795270a04c82 --- /dev/null +++ b/tests/fixtures/11-autolinks.marko @@ -0,0 +1,2 @@ +

Visit example.com for more info.

+

See also ./other-page.mdo and ../parent.mdo.

diff --git a/tests/fixtures/11-autolinks.mdo b/tests/fixtures/11-autolinks.mdo new file mode 100644 index 0000000000000000000000000000000000000000..c5c1f3c9c92a85b40ac32d8df6153fc4a5c63efa --- /dev/null +++ b/tests/fixtures/11-autolinks.mdo @@ -0,0 +1,3 @@ +Visit for more info. + +See also <./other-page.mdo> and <../parent.mdo>. diff --git a/tests/fixtures/12-frontmatter.marko b/tests/fixtures/12-frontmatter.marko new file mode 100644 index 0000000000000000000000000000000000000000..591ba40f92f198b163206d7c9b3bda7206f91329 --- /dev/null +++ b/tests/fixtures/12-frontmatter.marko @@ -0,0 +1,5 @@ +export const title = "My Page"; +export const date = "2026-02-08"; + +

${title}

+

Published on ${date}.

diff --git a/tests/fixtures/12-frontmatter.mdo b/tests/fixtures/12-frontmatter.mdo new file mode 100644 index 0000000000000000000000000000000000000000..ad389a805b9cbb33d709197595820f80674548cc --- /dev/null +++ b/tests/fixtures/12-frontmatter.mdo @@ -0,0 +1,8 @@ +--- +title: My Page +date: 2026-02-08 +--- + +# ${title} + +Published on ${date}. diff --git a/tests/fixtures/13-kitchen-sink.marko b/tests/fixtures/13-kitchen-sink.marko new file mode 100644 index 0000000000000000000000000000000000000000..0d3dd4baf05ef155ef24a692bdf796bf49d6fb19 --- /dev/null +++ b/tests/fixtures/13-kitchen-sink.marko @@ -0,0 +1,26 @@ +export const title = "Full Example"; + +import Chart from "./chart.marko"; +static const year = 2026; +export const slug = "full-example"; +// Page starts here +

${title}

+

This is a complete example of markodown featuring ${year}.

+server { + const data = await fetchData(); +} + +

Chart

+ + +
    +
  • ${point.label}: ${point.value}
    • +
+ + + +

No data available. Visit example.com for help.

+ + + +
diff --git a/tests/fixtures/13-kitchen-sink.mdo b/tests/fixtures/13-kitchen-sink.mdo new file mode 100644 index 0000000000000000000000000000000000000000..27c708c71c193f6b5ec85bf3a9f6bf1fe7266580 --- /dev/null +++ b/tests/fixtures/13-kitchen-sink.mdo @@ -0,0 +1,36 @@ +--- +title: Full Example +--- + +import Chart from "./chart.marko"; + +static const year = 2026; + +export const slug = "full-example"; + +// Page starts here + +# ${title} + +This is a **complete** example of *markodown* featuring ${year}. + +server { + const data = await fetchData(); +} + + + ## Chart + + + + + - **${point.label}**: ${point.value} + + + + No data available. Visit for help. + + + + +
diff --git a/tests/fixtures/14-evil-braces-and-typescript.marko b/tests/fixtures/14-evil-braces-and-typescript.marko new file mode 100644 index 0000000000000000000000000000000000000000..d9ea7960bcb9c1f142b0d0d17fd08101f0b93167 --- /dev/null +++ b/tests/fixtures/14-evil-braces-and-typescript.marko @@ -0,0 +1,22 @@ +export interface Input { + name: string; + items: Array<{ id: number; label: string }>; +} +static const message = "Hello {world} with {braces}"; +static const template = ` + This is a template literal { + with nested braces + } and expressions ${1 + 1} +`; +static function evil() { + const obj = { a: "{", b: "}" }; + const regex = /\{.*\}/; + return `{${obj.a}${obj.b}}`; +} +static { + console.log("Block with {braces} in string"); + const data = { key: "value {nested}" }; +} +

Evil Braces Test

+

This tests that string literals with braces don’t confuse the parser.

+

The message is: ${message}

diff --git a/tests/fixtures/14-evil-braces-and-typescript.mdo b/tests/fixtures/14-evil-braces-and-typescript.mdo new file mode 100644 index 0000000000000000000000000000000000000000..9e43f2a90e28cdf1bf45e49148d0021eccb2fef7 --- /dev/null +++ b/tests/fixtures/14-evil-braces-and-typescript.mdo @@ -0,0 +1,28 @@ +export interface Input { + name: string; + items: Array<{ id: number; label: string }>; +} + +static const message = "Hello {world} with {braces}"; +static const template = ` + This is a template literal { + with nested braces + } and expressions ${1 + 1} +`; + +static function evil() { + const obj = { a: "{", b: "}" }; + const regex = /\{.*\}/; + return `{${obj.a}${obj.b}}`; +} + +static { + console.log("Block with {braces} in string"); + const data = { key: "value {nested}" }; +} + +# Evil Braces Test + +This tests that string literals with braces don't confuse the parser. + +The message is: **${message}** diff --git a/tests/fixtures/15-multiline-tag.marko b/tests/fixtures/15-multiline-tag.marko new file mode 100644 index 0000000000000000000000000000000000000000..bff510ccefc2c5550c533fde7c0cee99583312e1 --- /dev/null +++ b/tests/fixtures/15-multiline-tag.marko @@ -0,0 +1,5 @@ +

good morning

+ +

hello world

+ diff --git a/tests/fixtures/15-multiline-tag.mdo b/tests/fixtures/15-multiline-tag.mdo new file mode 100644 index 0000000000000000000000000000000000000000..fe8af020ee3bb7eeba07c5aac779eb44a7770b66 --- /dev/null +++ b/tests/fixtures/15-multiline-tag.mdo @@ -0,0 +1,8 @@ +good morning + + + +hello **world** + + diff --git a/tests/fixtures/16-same-line-text.marko b/tests/fixtures/16-same-line-text.marko new file mode 100644 index 0000000000000000000000000000000000000000..aa7415da0e70b71d793062b8eb3d11004887a584 --- /dev/null +++ b/tests/fixtures/16-same-line-text.marko @@ -0,0 +1,8 @@ +
+samelinetext +

hello world

+
+
+sameline text +

hello world

+
diff --git a/tests/fixtures/16-same-line-text.mdo b/tests/fixtures/16-same-line-text.mdo new file mode 100644 index 0000000000000000000000000000000000000000..705c5ee7c891c7c0bf2386f3ad4610b8a2eb5b31 --- /dev/null +++ b/tests/fixtures/16-same-line-text.mdo @@ -0,0 +1,12 @@ + +
samelinetext + +hello **world** + +
+ +
sameline **text** + +hello **world** + +
diff --git a/tests/fixtures/17-consecutive-statements.marko b/tests/fixtures/17-consecutive-statements.marko new file mode 100644 index 0000000000000000000000000000000000000000..4b639b01a67a4a08395f9037119e46300a1436ec --- /dev/null +++ b/tests/fixtures/17-consecutive-statements.marko @@ -0,0 +1,2 @@ +static const y = "**hello**"; +export const z = y + " **world**"; diff --git a/tests/fixtures/17-consecutive-statements.mdo b/tests/fixtures/17-consecutive-statements.mdo new file mode 100644 index 0000000000000000000000000000000000000000..4b639b01a67a4a08395f9037119e46300a1436ec --- /dev/null +++ b/tests/fixtures/17-consecutive-statements.mdo @@ -0,0 +1,2 @@ +static const y = "**hello**"; +export const z = y + " **world**";