authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-02-12 22:22:37-08:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-02-13 11:09:29-08:00
loga4a2b9e40b546078cae57f9ddc95701fc538419d
tree1162640166f5d978c5ac179d4f4e8afa35969465
parent4326328d78523d6933f7257ca0c9cbfa71b6cbb2
signaturebadge-check Signed by SSH key SHA256:xbd+BjjhyBfwk7GVoURf9Yx0gzDerHbvYv7SddNWmAs

chore: drop tree sitter, fix bugs

14 files changed, 249 insertions(+), 203 deletions(-)

Cargo.lock-37
...@@ -403,8 +403,6 @@ dependencies = [...@@ -403,8 +403,6 @@ dependencies = [
403 "serde",403 "serde",
404 "serde_json",404 "serde_json",
405 "serde_yml",405 "serde_yml",
406 "tree-sitter",
407 "tree-sitter-typescript",
408]406]
409407
410[[package]]408[[package]]
...@@ -924,12 +922,6 @@ version = "1.1.0"...@@ -924,12 +922,6 @@ version = "1.1.0"
924source = "registry+https://github.com/rust-lang/crates.io-index"922source = "registry+https://github.com/rust-lang/crates.io-index"
925checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"923checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
926924
927[[package]]
928name = "streaming-iterator"
929version = "0.1.9"
930source = "registry+https://github.com/rust-lang/crates.io-index"
931checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
932
933[[package]]925[[package]]
934name = "strsim"926name = "strsim"
935version = "0.11.1"927version = "0.11.1"
...@@ -1004,35 +996,6 @@ version = "0.1.1"...@@ -1004,35 +996,6 @@ version = "0.1.1"
1004source = "registry+https://github.com/rust-lang/crates.io-index"996source = "registry+https://github.com/rust-lang/crates.io-index"
1005checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"997checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
1006998
1007[[package]]
1008name = "tree-sitter"
1009version = "0.24.7"
1010source = "registry+https://github.com/rust-lang/crates.io-index"
1011checksum = "a5387dffa7ffc7d2dae12b50c6f7aab8ff79d6210147c6613561fc3d474c6f75"
1012dependencies = [
1013 "cc",
1014 "regex",
1015 "regex-syntax",
1016 "streaming-iterator",
1017 "tree-sitter-language",
1018]
1019
1020[[package]]
1021name = "tree-sitter-language"
1022version = "0.1.7"
1023source = "registry+https://github.com/rust-lang/crates.io-index"
1024checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
1025
1026[[package]]
1027name = "tree-sitter-typescript"
1028version = "0.23.2"
1029source = "registry+https://github.com/rust-lang/crates.io-index"
1030checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff"
1031dependencies = [
1032 "cc",
1033 "tree-sitter-language",
1034]
1035
1036[[package]]999[[package]]
1037name = "unicode-bidi"1000name = "unicode-bidi"
1038version = "0.3.18"1001version = "0.3.18"
Cargo.toml-3
...@@ -17,6 +17,3 @@ oxc_parser = "0.96.0"...@@ -17,6 +17,3 @@ oxc_parser = "0.96.0"
17oxc_span = "0.96.0"17oxc_span = "0.96.0"
18serde_json = "1.0"18serde_json = "1.0"
19serde_yml = "0.0.12"19serde_yml = "0.0.12"
20tree-sitter = "0.24"
21tree-sitter-typescript = "0.23"
22
examples/next-js.mdo+4-3
...@@ -656,7 +656,7 @@ object, but instead uses inline JSX to fill that gap....@@ -656,7 +656,7 @@ object, but instead uses inline JSX to fill that gap.
656[lib-jsr]: https://jsr.io/@clo/lib656[lib-jsr]: https://jsr.io/@clo/lib
657[lib-npm]: https://npmjs.com/@paperclover/lib657[lib-npm]: https://npmjs.com/@paperclover/lib
658658
659<h3 id='vercel-og'>`next/og` is Good Too</>659<h3 id='vercel-og'><code>next/og</code> is Good Too</>
660660
661No strong opinions. I just want to remind everyone that the `@vercel/og` package exists.661No strong opinions. I just want to remind everyone that the `@vercel/og` package exists.
662662
...@@ -742,8 +742,9 @@ subscribe to the email list:...@@ -742,8 +742,9 @@ subscribe to the email list:
742<br />742<br />
743<br />743<br />
744<br />744<br />
745745<footer>
746<footer>2025 (c) paper clover</footer>7462025 (c) paper clover
747</footer>
747748
748</blog-layout>749</blog-layout>
749750
src/lib.rs+6
...@@ -2,6 +2,8 @@ pub mod marko;...@@ -2,6 +2,8 @@ pub mod marko;
2pub mod plugin;2pub mod plugin;
3pub mod typescript;3pub mod typescript;
44
5use std::borrow::Cow;
6
5use oxc_diagnostics::{LabeledSpan, OxcDiagnostic};7use oxc_diagnostics::{LabeledSpan, OxcDiagnostic};
6use plugin::tags::{MarkoClose, MarkoOpen};8use plugin::tags::{MarkoClose, MarkoOpen};
79
...@@ -223,3 +225,7 @@ fn validate_marko_tags(node: &markdown_it::Node, errors: &mut Vec<OxcDiagnostic>...@@ -223,3 +225,7 @@ fn validate_marko_tags(node: &markdown_it::Node, errors: &mut Vec<OxcDiagnostic>
223 );225 );
224 }226 }
225}227}
228
229pub fn err<T: Into<Cow<'static, str>>>(str: T, offset: usize, length: usize) -> OxcDiagnostic {
230 OxcDiagnostic::error(str).and_label(LabeledSpan::new(None, offset, length))
231}
src/marko.rs+72-69
...@@ -1,9 +1,12 @@...@@ -1,9 +1,12 @@
1use oxc_diagnostics::{LabeledSpan, OxcDiagnostic};1use oxc_diagnostics::OxcDiagnostic;
2use oxc_span::Span;2use oxc_span::Span;
33
4use crate::typescript::{4use crate::{
5 parse_call_arguments, parse_expr, parse_expr_without_gt, parse_fn_params_and_body,5 err,
6 parse_var_binding,6 typescript::{
7 parse_call_arguments, parse_expr, parse_expr_without_gt, parse_fn_params_and_body,
8 parse_var_binding,
9 },
7};10};
811
9#[derive(Debug, Clone, PartialEq, Eq)]12#[derive(Debug, Clone, PartialEq, Eq)]
...@@ -100,13 +103,7 @@ impl<'a> LexState<'a> {...@@ -100,13 +103,7 @@ impl<'a> LexState<'a> {
100 self.offset += 1;103 self.offset += 1;
101 return Ok(b);104 return Ok(b);
102 }105 }
103 Err(106 Err(err("Unexpected end of file", self.offset, 1))
104 OxcDiagnostic::error("Unexpected end of file").and_label(LabeledSpan::new(
105 None,
106 self.offset,
107 1,
108 )),
109 )
110 }107 }
111108
112 pub fn expect(&mut self, expected: &str) -> Result<(), OxcDiagnostic> {109 pub fn expect(&mut self, expected: &str) -> Result<(), OxcDiagnostic> {
...@@ -114,13 +111,7 @@ impl<'a> LexState<'a> {...@@ -114,13 +111,7 @@ impl<'a> LexState<'a> {
114 self.offset += expected.len();111 self.offset += expected.len();
115 Ok(())112 Ok(())
116 } else {113 } else {
117 Err(114 Err(err(format!("Expected {expected}"), self.offset, 1))
118 OxcDiagnostic::error(format!("Expected {expected}")).and_label(LabeledSpan::new(
119 None,
120 self.offset,
121 1,
122 )),
123 )
124 }115 }
125 }116 }
126117
...@@ -153,10 +144,7 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {...@@ -153,10 +144,7 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {
153144
154 if byte == b'#' {145 if byte == b'#' {
155 if has_id {146 if has_id {
156 return Err(147 return Err(err("Cannot specify two ID shorthands", offset, name.len()));
157 OxcDiagnostic::error("Cannot specify two ID shorthands".to_string())
158 .and_label(LabeledSpan::new(None, offset, name.len())),
159 );
160 }148 }
161 has_id = true;149 has_id = true;
162 }150 }
...@@ -174,6 +162,9 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {...@@ -174,6 +162,9 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {
174 match byte {162 match byte {
175 b'|' => {163 b'|' => {
176 // parameters164 // parameters
165 if has_js_params {
166 return Err(err("Params already specified", offset, 1));
167 }
177 l.offset += 1;168 l.offset += 1;
178 l.skip_whitespace();169 l.skip_whitespace();
179 while l.peek_byte() != Some(b'|') && l.peek_byte().is_some() {170 while l.peek_byte() != Some(b'|') && l.peek_byte().is_some() {
...@@ -192,6 +183,9 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {...@@ -192,6 +183,9 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {
192 }183 }
193 b'(' => {184 b'(' => {
194 // arguments185 // arguments
186 if has_js_arguments {
187 return Err(err("Function call already specified", offset, 1));
188 }
195 // TODO: METHOD SHORTHAND189 // TODO: METHOD SHORTHAND
196 println!("{}", l.peek_rest());190 println!("{}", l.peek_rest());
197 parse_call_arguments(&mut l)?;191 parse_call_arguments(&mut l)?;
...@@ -202,6 +196,9 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {...@@ -202,6 +196,9 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {
202 if l.peek_rest().starts_with("/>") {196 if l.peek_rest().starts_with("/>") {
203 break;197 break;
204 }198 }
199 if has_js_variable {
200 return Err(err("Variable already specified", offset, 1));
201 }
205 l.offset += 1;202 l.offset += 1;
206 l.skip_whitespace();203 l.skip_whitespace();
207 parse_var_binding(&mut l)?;204 parse_var_binding(&mut l)?;
...@@ -212,45 +209,55 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {...@@ -212,45 +209,55 @@ pub fn parse_open(src: &str) -> Result<Open, OxcDiagnostic> {
212 }209 }
213210
214 if is_attribute_tag && byte == b'/' {211 if is_attribute_tag && byte == b'/' {
215 return Err(212 return Err(err(
216 OxcDiagnostic::error("Attribute tags do not support variables")213 "Attribute tags do not support variables",
217 .and_label(LabeledSpan::new(None, offset, l.offset - offset)),214 offset,
218 );215 l.offset - offset,
216 ));
219 }217 }
220 if is_attribute_tag && byte == b'(' {218 if is_attribute_tag && byte == b'(' {
221 return Err(219 return Err(err(
222 OxcDiagnostic::error("Attribute tags do not support arguments")220 "Attribute tags do not support arguments",
223 .and_label(LabeledSpan::new(None, offset, l.offset - offset)),221 offset,
224 );222 l.offset - offset,
223 ));
225 }224 }
226 }225 }
227226
228 // keyless value
229 if l.peek_byte() == Some(b'=') {
230 l.offset += 1;
231 l.skip_whitespace();
232 parse_expr_without_gt(&mut l)?;
233 l.skip_whitespace();
234 }
235
236 // attributes
237 let mut self_closing = false;227 let mut self_closing = false;
238 loop {228 if !has_js_arguments {
239 match l.peek_byte() {229 // keyless value
240 None => return Err(l.expect_byte().expect_err("err")),230 if l.peek_byte() == Some(b'=') {
241 Some(b'>') => {231 l.offset += 1;
242 l.advance(1);232 l.skip_whitespace();
243 break;233 parse_expr_without_gt(&mut l)?;
244 }234 l.skip_whitespace();
245 Some(_) => {235 }
246 if l.expect("/>").is_ok() {236
247 self_closing = true;237 // attributes
238 loop {
239 match l.peek_byte() {
240 None => return Err(l.expect_byte().expect_err("err")),
241 Some(b'>') => {
242 l.advance(1);
248 break;243 break;
249 }244 }
250 parse_attribute(&mut l)?;245 Some(_) => {
251 l.skip_whitespace();246 if l.expect("/>").is_ok() {
247 self_closing = true;
248 break;
249 }
250 parse_attribute(&mut l)?;
251 l.skip_whitespace();
252 }
252 }253 }
253 }254 }
255 } else {
256 if l.expect("/>").is_ok() {
257 self_closing = true;
258 } else {
259 l.expect(">")?;
260 }
254 }261 }
255262
256 Ok(Open {263 Ok(Open {
...@@ -299,8 +306,7 @@ fn parse_tag_name<'a>(l: &mut LexState<'a>) -> Result<&'a str, OxcDiagnostic> {...@@ -299,8 +306,7 @@ fn parse_tag_name<'a>(l: &mut LexState<'a>) -> Result<&'a str, OxcDiagnostic> {
299 if !tag_name.is_empty() {306 if !tag_name.is_empty() {
300 Ok(tag_name)307 Ok(tag_name)
301 } else {308 } else {
302 Err(OxcDiagnostic::error("Expected tag name".to_string())309 Err(err("Expected tag name", l.offset, 1))
303 .and_label(LabeledSpan::new(None, l.offset, 1)))
304 }310 }
305}311}
306312
...@@ -316,8 +322,7 @@ fn parse_attr_name<'a>(l: &mut LexState<'a>) -> Result<&'a str, OxcDiagnostic> {...@@ -316,8 +322,7 @@ fn parse_attr_name<'a>(l: &mut LexState<'a>) -> Result<&'a str, OxcDiagnostic> {
316 if !tag_name.is_empty() {322 if !tag_name.is_empty() {
317 Ok(tag_name)323 Ok(tag_name)
318 } else {324 } else {
319 Err(OxcDiagnostic::error("Expected attribute name".to_string())325 Err(err("Expected attribute name", l.offset, 1))
320 .and_label(LabeledSpan::new(None, l.offset, 1)))
321 }326 }
322}327}
323328
...@@ -348,10 +353,7 @@ fn parse_attribute(l: &mut LexState) -> Result<(), OxcDiagnostic> {...@@ -348,10 +353,7 @@ fn parse_attribute(l: &mut LexState) -> Result<(), OxcDiagnostic> {
348 parse_fn_params_and_body(l)?;353 parse_fn_params_and_body(l)?;
349 l.skip_whitespace();354 l.skip_whitespace();
350 } else if !has_ws_or_end {355 } else if !has_ws_or_end {
351 return Err(356 return Err(err("Expected whitespace or '>'", l.offset, 1));
352 OxcDiagnostic::error("Expected whitespace or '>'".to_string())
353 .and_label(LabeledSpan::new(None, l.offset, 1)),
354 );
355 }357 }
356358
357 Ok(())359 Ok(())
...@@ -375,12 +377,15 @@ fn parse_class_name_shorthand<'a>(...@@ -375,12 +377,15 @@ fn parse_class_name_shorthand<'a>(
375 if !tag_name.is_empty() {377 if !tag_name.is_empty() {
376 Ok(tag_name)378 Ok(tag_name)
377 } else {379 } else {
378 Err(OxcDiagnostic::error(if id {380 Err(err(
379 "Expected id"381 if id {
380 } else {382 "Expected id"
381 "Expected class name"383 } else {
382 })384 "Expected class name"
383 .and_label(LabeledSpan::new(None, l.offset, 1)))385 },
386 l.offset,
387 1,
388 ))
384 }389 }
385}390}
386391
...@@ -1095,10 +1100,10 @@ mod tests {...@@ -1095,10 +1100,10 @@ mod tests {
1095 #[test]1100 #[test]
1096 fn test_combined_complex() {1101 fn test_combined_complex() {
1097 assert_eq!(1102 assert_eq!(
1098 parse_open("<my-tag#id.cls/result(arg1, arg2) foo=bar ...spread>"),1103 parse_open("<my-tag#id.cls/result|arg1, arg2| foo=bar ...spread>"),
1099 Ok(Open {1104 Ok(Open {
1100 tag_name: "my-tag",1105 tag_name: "my-tag",
1101 content: "<my-tag#id.cls/result(arg1, arg2) foo=bar ...spread>",1106 content: "<my-tag#id.cls/result|arg1, arg2| foo=bar ...spread>",
1102 self_closing: false,1107 self_closing: false,
1103 })1108 })
1104 );1109 );
...@@ -1270,13 +1275,11 @@ mod tests {...@@ -1270,13 +1275,11 @@ mod tests {
1270 }1275 }
12711276
1272 #[test]1277 #[test]
1273 #[ignore = "need to fix parse_var_binding"]
1274 fn test_tag_params_simple() {1278 fn test_tag_params_simple() {
1275 parse_open("<for|item| of=items>").unwrap();1279 parse_open("<for|item| of=items>").unwrap();
1276 }1280 }
12771281
1278 #[test]1282 #[test]
1279 #[ignore = "need to fix parse_var_binding"]
1280 fn test_tag_params_destructure() {1283 fn test_tag_params_destructure() {
1281 parse_open("<for|{ name, age }| of=people>").unwrap();1284 parse_open("<for|{ name, age }| of=people>").unwrap();
1282 }1285 }
src/plugin/statement.rs+18-29
...@@ -3,7 +3,7 @@ use markdown_it::Node;...@@ -3,7 +3,7 @@ use markdown_it::Node;
33
4use crate::adjust_err;4use crate::adjust_err;
5use crate::plugin::{ErrorBlock, RawBlock};5use crate::plugin::{ErrorBlock, RawBlock};
6use crate::typescript::find_statement_extent;6use crate::typescript::scan_first_statement_forbid_trailing;
77
8/// Parse JavaScript statements (import, export, static X, server X, client X)8/// Parse JavaScript statements (import, export, static X, server X, client X)
9pub struct Rule;9pub struct Rule;
...@@ -11,20 +11,6 @@ pub struct Rule;...@@ -11,20 +11,6 @@ pub struct Rule;
11const KEYWORDS: &[&str] = &["import ", "export ", "static ", "server ", "client "];11const KEYWORDS: &[&str] = &["import ", "export ", "static ", "server ", "client "];
12const MARKO_PREFIXES: &[&str] = &["static ", "server ", "client "];12const MARKO_PREFIXES: &[&str] = &["static ", "server ", "client "];
1313
14/// Check if a line is a valid statement (starts with keyword and parses successfully).
15fn is_valid_statement(line: &str) -> bool {
16 let trimmed = line.trim_start();
17 let Some(keyword) = KEYWORDS.iter().find(|k| trimmed.starts_with(*k)) else {
18 return false;
19 };
20 let keyword_trim = MARKO_PREFIXES
21 .iter()
22 .find(|k| *k == keyword)
23 .map(|k| k.len())
24 .unwrap_or(0);
25 find_statement_extent(&trimmed[keyword_trim..]).is_ok()
26}
27
28impl BlockRule for Rule {14impl BlockRule for Rule {
29 fn run(state: &mut BlockState) -> Option<(Node, usize)> {15 fn run(state: &mut BlockState) -> Option<(Node, usize)> {
30 if state.line >= state.line_max || state.blk_indent > 0 {16 if state.line >= state.line_max || state.blk_indent > 0 {
...@@ -37,9 +23,11 @@ impl BlockRule for Rule {...@@ -37,9 +23,11 @@ impl BlockRule for Rule {
37 // Statements must start a new block - they can't be in the middle of a paragraph.23 // Statements must start a new block - they can't be in the middle of a paragraph.
38 // Allow if: start of document, previous line is empty, or previous line is a valid statement.24 // Allow if: start of document, previous line is empty, or previous line is a valid statement.
39 if state.line > 0 {25 if state.line > 0 {
40 let prev_line = state.get_line(state.line - 1);26 let prev_line = state.get_line(state.line - 1).trim();
41 let prev_is_empty = prev_line.trim().is_empty();27 if !prev_line.is_empty()
42 if !prev_is_empty && !is_valid_statement(prev_line) {28 && prev_line != "---"
29 && !KEYWORDS.iter().any(|k| prev_line.starts_with(*k))
30 {
43 return None;31 return None;
44 }32 }
45 }33 }
...@@ -52,17 +40,18 @@ impl BlockRule for Rule {...@@ -52,17 +40,18 @@ impl BlockRule for Rule {
5240
53 let unbounded_src = &state.src[state.line_offsets[state.line].first_nonspace..];41 let unbounded_src = &state.src[state.line_offsets[state.line].first_nonspace..];
5442
55 let statement_end = match find_statement_extent(&unbounded_src[keyword_trim..]) {43 let statement_end =
56 Ok(ok) => ok,44 match scan_first_statement_forbid_trailing(&unbounded_src[keyword_trim..]) {
57 Err(err) => {45 Ok(ok) => ok,
58 return Some((46 Err(err) => {
59 Node::new(ErrorBlock {47 return Some((
60 errors: vec![adjust_err(err, keyword_trim.cast_signed())],48 Node::new(ErrorBlock {
61 }),49 errors: vec![adjust_err(err, keyword_trim.cast_signed())],
62 1,50 }),
63 ));51 1,
64 }52 ));
65 };53 }
54 };
6655
67 let total_end = keyword_trim + statement_end;56 let total_end = keyword_trim + statement_end;
68 let content = &unbounded_src[0..unbounded_src.len().min(total_end + 1)];57 let content = &unbounded_src[0..unbounded_src.len().min(total_end + 1)];
src/typescript.rs+105-51
...@@ -1,53 +1,93 @@...@@ -1,53 +1,93 @@
1//! implement ts partial parsing helpers1//! implement ts partial parsing helpers
2use crate::{adjust_err, marko::LexState};2use crate::{adjust_err, err, marko::LexState};
33
4use oxc_allocator::Allocator;4use oxc_allocator::Allocator;
5use oxc_ast::ast::Expression;5use oxc_ast::ast::{Expression, Statement};
6use oxc_diagnostics::{LabeledSpan, OxcDiagnostic};6use oxc_diagnostics::{LabeledSpan, OxcDiagnostic};
7use oxc_span::{GetSpan, SourceType};7use oxc_span::{GetSpan, SourceType};
8use std::borrow::Cow;
9use tree_sitter::Parser;
108
11fn err<T: Into<Cow<'static, str>>>(str: T, offset: usize, length: usize) -> OxcDiagnostic {9/// TODO: Replace with oxc parser, but oxc is tough here
12 OxcDiagnostic::error(str).and_label(LabeledSpan::new(None, offset, length))10pub fn scan_first_statement_forbid_trailing(source: &str) -> Result<usize, OxcDiagnostic> {
13}11 let mut allocator = Allocator::new();
1412 let expr = parse_stmt_extra(source, 0, &mut allocator)?;
15/// TODO: Blow up13 let span = expr.span();
16pub fn find_statement_extent(source: &str) -> Result<usize, OxcDiagnostic> {14 let len = (span.end - span.start) as usize;
17 let mut parser = Parser::new();
18
19 let language = tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into();
20 parser
21 .set_language(&language)
22 .expect("Failed to load TypeScript grammar");
23
24 let tree = parser.parse(source, None).ok_or_else(|| {
25 OxcDiagnostic::error("Failed to parse").and_label(LabeledSpan::new(None, 0, 1))
26 })?;
27 let root = tree.root_node();
2815
29 let first_statement = root.named_child(0).ok_or_else(|| {16 if let Some(trailing) = source[span.end as usize..].lines().next() {
30 OxcDiagnostic::error("Failed to parse").and_label(LabeledSpan::new(None, 0, 1))17 if trailing.trim().len() > 0 {
31 })?;18 return Err(err(
19 "Trailing content not allowed here",
20 span.end as usize + (trailing.len() - trailing.trim_start().len()),
21 trailing.trim().len(),
22 ));
23 }
24 }
25 Ok(len)
26}
3227
33 if first_statement.kind() == "ERROR" {28fn parse_stmt_extra<'alloc, 'src: 'alloc>(
34 return Err(OxcDiagnostic::error("Invalid TypeScript/JavaScript syntax")29 source: &'src str,
35 .and_label(LabeledSpan::new(None, 0, first_statement.end_byte())));30 offset: isize,
31 allocator: &'alloc mut Allocator,
32) -> Result<Statement<'alloc>, OxcDiagnostic> {
33 if source.is_empty() {
34 return Err(err(
35 "Expected expression, found end of file",
36 offset.max(0).cast_unsigned(),
37 1,
38 ));
36 }39 }
3740
38 let end_byte = first_statement.end_byte();41 let source_type = SourceType::default()
42 .with_module(true)
43 .with_typescript(true);
3944
40 let remaining = &source[end_byte..];45 let mut result = oxc_parser::Parser::new(allocator, source, source_type).parse();
41 let same_line = remaining46 if !result.errors.is_empty() && result.program.body.len() == 0 {
42 .find('\n')47 let first_err = result
43 .map(|x| &remaining[..x])48 .errors
44 .unwrap_or(remaining);49 .into_iter()
45 if !same_line.trim().is_empty() {50 .next()
46 return Err(OxcDiagnostic::error("Unexpected content after statement.")51 .expect("no errors but no result!");
47 .with_label(LabeledSpan::new(None, end_byte, same_line.len())));52 let first_err_offset = first_err
53 .labels
54 .as_ref()
55 .ok_or_else(|| err("Expected statement", offset.max(0).cast_unsigned(), 1))?
56 .first()
57 .expect("labels, but no labels!")
58 .offset();
59
60 let mut candidate = source[0..first_err_offset].trim_end();
61 let mut whitespace_groups = 0;
62 loop {
63 if candidate.is_empty() {
64 return Err(adjust_err(first_err, offset));
65 }
66 result = oxc_parser::Parser::new(allocator, candidate, source_type).parse();
67 if (!result.errors.is_empty() || result.panicked) && result.program.body.len() == 0 {
68 let before_trim = &candidate[..candidate.len() - 1];
69 let after_trim = before_trim.trim_end();
70 if after_trim.len() < before_trim.len() {
71 whitespace_groups += 1;
72 if whitespace_groups > 1 {
73 return Err(adjust_err(first_err, offset));
74 }
75 }
76 candidate = after_trim;
77 } else {
78 break;
79 }
80 }
81 } else {
82 assert!(!result.panicked);
48 }83 }
4984
50 Ok(end_byte)85 result
86 .program
87 .body
88 .into_iter()
89 .next()
90 .ok_or_else(|| err("Expected statement", offset.max(0).cast_unsigned(), 1))
51}91}
5292
53fn parse_expr_extra<'alloc, 'src: 'alloc>(93fn parse_expr_extra<'alloc, 'src: 'alloc>(
...@@ -71,15 +111,22 @@ fn parse_expr_extra<'alloc, 'src: 'alloc>(...@@ -71,15 +111,22 @@ fn parse_expr_extra<'alloc, 'src: 'alloc>(
71 Ok(expr) => expr,111 Ok(expr) => expr,
72 Err(errors) => {112 Err(errors) => {
73 let first_err = errors.into_iter().next().expect("no errors but no result!");113 let first_err = errors.into_iter().next().expect("no errors but no result!");
74 let first_err_offset = first_err114 let first_label = first_err
75 .labels115 .labels
76 .as_ref()116 .as_ref()
77 .ok_or_else(|| err("Expected expression", offset.max(0).cast_unsigned(), 1))?117 .ok_or_else(|| err("Expected expression", offset.max(0).cast_unsigned(), 1))?
78 .first()118 .first()
79 .expect("labels, but no labels!")119 .expect("labels, but no labels!");
80 .offset();120 let first_err_offset = first_label.offset();
81121 let first_err_len = first_label.len();
82 let mut candidate = source[0..first_err_offset].trim_end();122
123 // Use end of error span when error is at start (e.g. "Cannot assign to this expression")
124 let initial_end = if first_err_offset == 0 {
125 (first_err_offset + first_err_len).min(source.len())
126 } else {
127 first_err_offset
128 };
129 let mut candidate = source[0..initial_end].trim_end();
83 let mut whitespace_groups = 0;130 let mut whitespace_groups = 0;
84 loop {131 loop {
85 if candidate.is_empty() {132 if candidate.is_empty() {
...@@ -184,6 +231,7 @@ pub fn parse_call_arguments(l: &mut LexState) -> Result<usize, OxcDiagnostic> {...@@ -184,6 +231,7 @@ pub fn parse_call_arguments(l: &mut LexState) -> Result<usize, OxcDiagnostic> {
184/// parse variable binding. identifier or destructuring pattern231/// parse variable binding. identifier or destructuring pattern
185/// also parses optional `: Type`232/// also parses optional `: Type`
186pub fn parse_var_binding(l: &mut LexState) -> Result<usize, OxcDiagnostic> {233pub fn parse_var_binding(l: &mut LexState) -> Result<usize, OxcDiagnostic> {
234 // TODO: this approach has bugs
187 let offset = l.offset();235 let offset = l.offset();
188 let source = l.peek_rest();236 let source = l.peek_rest();
189 if source.is_empty() {237 if source.is_empty() {
...@@ -724,52 +772,58 @@ mod tests {...@@ -724,52 +772,58 @@ mod tests {
724 assert_eq!(parse_var_binding("café"), Ok(5));772 assert_eq!(parse_var_binding("café"), Ok(5));
725 }773 }
726774
775 #[test]
776 fn test_var_binding_equals() {
777 assert_eq!(parse_var_binding("foo = 1"), Ok(3));
778 assert_eq!(parse_var_binding("foo| of=1>"), Ok(3));
779 }
780
727 #[test]781 #[test]
728 fn test_stmt_function_with_garbage() {782 fn test_stmt_function_with_garbage() {
729 let source = "function hello() {\n console.log(1);\n}\n\nrandom markdown garbage";783 let source = "function hello() {\n console.log(1);\n}\n\nrandom markdown garbage";
730 let end = find_statement_extent(source).unwrap();784 let end = scan_first_statement_forbid_trailing(source).unwrap();
731 assert_eq!(&source[..end], "function hello() {\n console.log(1);\n}");785 assert_eq!(&source[..end], "function hello() {\n console.log(1);\n}");
732 }786 }
733787
734 #[test]788 #[test]
735 fn test_stmt_import_with_garbage() {789 fn test_stmt_import_with_garbage() {
736 let source = "import { foo } from 'bar';\n\n# markdown heading";790 let source = "import { foo } from 'bar';\n\n# markdown heading";
737 let end = find_statement_extent(source).unwrap();791 let end = scan_first_statement_forbid_trailing(source).unwrap();
738 assert_eq!(&source[..end], "import { foo } from 'bar';");792 assert_eq!(&source[..end], "import { foo } from 'bar';");
739 }793 }
740794
741 #[test]795 #[test]
742 fn test_stmt_interface_with_garbage() {796 fn test_stmt_interface_with_garbage() {
743 let source = "interface Foo {\n bar: string;\n}\n\nsome text";797 let source = "interface Foo {\n bar: string;\n}\n\nsome text";
744 let end = find_statement_extent(source).unwrap();798 let end = scan_first_statement_forbid_trailing(source).unwrap();
745 assert_eq!(&source[..end], "interface Foo {\n bar: string;\n}");799 assert_eq!(&source[..end], "interface Foo {\n bar: string;\n}");
746 }800 }
747801
748 #[test]802 #[test]
749 fn test_stmt_const_declaration() {803 fn test_stmt_const_declaration() {
750 let source = "const answer = 42;\n\n# Next section";804 let source = "const answer = 42;\n\n# Next section";
751 let end = find_statement_extent(source).unwrap();805 let end = scan_first_statement_forbid_trailing(source).unwrap();
752 assert_eq!(&source[..end], "const answer = 42;");806 assert_eq!(&source[..end], "const answer = 42;");
753 }807 }
754808
755 #[test]809 #[test]
756 fn test_stmt_expression_statement() {810 fn test_stmt_expression_statement() {
757 let source = "console.log('hello');\n\nmore content";811 let source = "console.log('hello');\n\nmore content";
758 let end = find_statement_extent(source).unwrap();812 let end = scan_first_statement_forbid_trailing(source).unwrap();
759 assert_eq!(&source[..end], "console.log('hello');");813 assert_eq!(&source[..end], "console.log('hello');");
760 }814 }
761815
762 #[test]816 #[test]
763 fn test_stmt_pure_garbage() {817 fn test_stmt_pure_garbage() {
764 let source = "random unexpected garbage";818 let source = "random unexpected garbage";
765 let result = find_statement_extent(source);819 let result = scan_first_statement_forbid_trailing(source);
766 assert!(result.is_err());820 assert!(result.is_err());
767 }821 }
768822
769 #[test]823 #[test]
770 fn test_stmt_multiline_function() {824 fn test_stmt_multiline_function() {
771 let source = "function sort(items) {\n while (!isSorted()) {\n shuffle(items);\n }\n return items;\n}\n\n# Heading";825 let source = "function sort(items) {\n while (!isSorted()) {\n shuffle(items);\n }\n return items;\n}\n\n# Heading";
772 let end = find_statement_extent(source).unwrap();826 let end = scan_first_statement_forbid_trailing(source).unwrap();
773 assert!(source[..end].contains("return items;"));827 assert!(source[..end].contains("return items;"));
774 assert!(source[..end].contains("}"));828 assert!(source[..end].contains("}"));
775 }829 }
...@@ -777,14 +831,14 @@ mod tests {...@@ -777,14 +831,14 @@ mod tests {
777 #[test]831 #[test]
778 fn test_stmt_trailing_content_on_same_line() {832 fn test_stmt_trailing_content_on_same_line() {
779 let source = "const x = 1; const y = 2;";833 let source = "const x = 1; const y = 2;";
780 let result = find_statement_extent(source);834 let result = scan_first_statement_forbid_trailing(source);
781 assert!(result.is_err());835 assert!(result.is_err());
782 }836 }
783837
784 #[test]838 #[test]
785 fn test_stmt_trailing_whitespace_ok() {839 fn test_stmt_trailing_whitespace_ok() {
786 let source = "const x = 1; \n\nmore content";840 let source = "const x = 1; \n\nmore content";
787 let result = find_statement_extent(source);841 let result = scan_first_statement_forbid_trailing(source);
788 assert!(result.is_ok());842 assert!(result.is_ok());
789 }843 }
790}844}
tests/fixtures.rs+12
...@@ -147,3 +147,15 @@ fn fixture_16_same_line_text() {...@@ -147,3 +147,15 @@ fn fixture_16_same_line_text() {
147fn fixture_17_consecutive_statements() {147fn fixture_17_consecutive_statements() {
148 run_fixture("17-consecutive-statements");148 run_fixture("17-consecutive-statements");
149}149}
150
151#[ignore = "known bug"]
152#[test]
153fn fixture_18_multiline_tag_with_value() {
154 run_fixture("18-multiline-tag-with-value");
155}
156
157#[ignore = "known bug"]
158#[test]
159fn fixture_19_extra_cases() {
160 run_fixture("19-extra-cases");
161}
tests/fixtures/13-kitchen-sink.marko+1-2
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1export const title = "Full Example";1export const title = "Full Example";
2
3import Chart from "./chart.marko";2import Chart from "./chart.marko";
4static const year = 2026;3static const year = 2026;
5export const slug = "full-example";4export const slug = "full-example";
...@@ -19,7 +18,7 @@ server {...@@ -19,7 +18,7 @@ server {
19</for>18</for>
20</if>19</if>
21<else>20<else>
22<p>No data available. Visit <a href="https://example.com">example.com</a> for help.</p>21No data available. Visit <a href="https://example.com">example.com</a> for help.
23</else>22</else>
24<div#footer.site-footer>23<div#footer.site-footer>
25<app-footer year=year />24<app-footer year=year />
tests/fixtures/13-kitchen-sink.mdo+9-9
...@@ -1,11 +1,8 @@...@@ -1,11 +1,8 @@
1---1---
2title: Full Example2title: Full Example
3---3---
4
5import Chart from "./chart.marko";4import Chart from "./chart.marko";
6
7static const year = 2026;5static const year = 2026;
8
9export const slug = "full-example";6export const slug = "full-example";
107
11// Page starts here8// Page starts here
...@@ -19,16 +16,19 @@ server {...@@ -19,16 +16,19 @@ server {
19}16}
2017
21<if=data>18<if=data>
22 ## Chart
2319
24 <Chart data=data year=year />20## Chart
21
22<Chart data=data year=year />
23
24<for|point| of=data.points>
25
26- **${point.label}**: ${point.value}
2527
26 <for|point| of=data.points>28</for>
27 - **${point.label}**: ${point.value}
28 </for>
29</if>29</if>
30<else>30<else>
31 No data available. Visit <https://example.com> for help.31No data available. Visit <https://example.com> for help.
32</else>32</else>
3333
34<div#footer.site-footer>34<div#footer.site-footer>
tests/fixtures/18-multiline-tag-with-value.marko created+6
...@@ -0,0 +1,6 @@
1<blog-layout
2 meta=meta
3>
4<h3 id='next-metadata'><code>next/metadata</code> is Great</>
5<p>hi</p>
6</blog-layout>
tests/fixtures/18-multiline-tag-with-value.mdo created+9
...@@ -0,0 +1,9 @@
1<blog-layout
2 meta=meta
3>
4
5<h3 id='next-metadata'>`next/metadata` is Great</>
6
7hi
8
9</blog-layout>
tests/fixtures/19-extra-cases.marko created+3
...@@ -0,0 +1,3 @@
1<tag attr=(value
2>
35)>
tests/fixtures/19-extra-cases.mdo created+4
...@@ -0,0 +1,4 @@
1
2<tag attr=(value
3>
45)>