typlite/parser/
core.rs

1//! HTML parser core, containing main structures and general parsing logic
2
3use std::sync::Arc;
4
5use typst::diag::SourceDiagnostic;
6use typst_syntax::{DiagSpan, Span};
7
8use cmark_writer::WriteResult;
9use cmark_writer::ast::{CustomNode, HtmlAttribute, HtmlElement as CmarkHtmlElement, Node};
10use cmark_writer::writer::InlineWriterProxy;
11use ecow::EcoString;
12use tinymist_project::LspWorld;
13use typst_html::{HtmlElement, HtmlNode, tag};
14
15use crate::Result;
16use crate::TypliteFeat;
17use crate::attributes::{AlertsAttr, HeadingAttr, RawAttr, TypliteAttrsParser, VerbatimAttr};
18use crate::common::{AlertNode, BlockVerbatimNode, CenterNode, VerbatimNode};
19use crate::diagnostics::WarningCollector;
20use crate::tags::md_tag;
21
22use super::{list::ListParser, table::TableParser};
23
24/// HTML to AST parser implementation
25pub struct HtmlToAstParser {
26    pub asset_counter: usize,
27    pub feat: TypliteFeat,
28    pub world: Arc<LspWorld>,
29    pub list_level: usize,
30    pub blocks: Vec<Node>,
31    pub inline_buffer: Vec<Node>,
32    pub(crate) warnings: WarningCollector,
33}
34
35impl HtmlToAstParser {
36    pub(crate) fn new(
37        feat: TypliteFeat,
38        world: &Arc<LspWorld>,
39        warnings: WarningCollector,
40    ) -> Self {
41        Self {
42            feat,
43            world: world.clone(),
44            asset_counter: 0,
45            list_level: 0,
46            blocks: Vec::new(),
47            inline_buffer: Vec::new(),
48            warnings,
49        }
50    }
51
52    pub fn convert_element(&mut self, element: &HtmlElement) -> Result<()> {
53        match element.tag {
54            tag::head => Ok(()),
55
56            tag::html | tag::body | md_tag::doc => {
57                self.convert_children(element)?;
58                Ok(())
59            }
60
61            tag::p => {
62                self.flush_inline_buffer();
63                self.convert_children(element)?;
64                Ok(())
65            }
66
67            tag::span | tag::div => {
68                self.convert_children(element)?;
69                Ok(())
70            }
71
72            tag::strong | md_tag::strong => self.convert_strong(element),
73            tag::em | md_tag::emph => self.convert_emphasis(element),
74            tag::mark => self.convert_highlight(element),
75            tag::s => self.convert_strikethrough(element),
76
77            tag::br => {
78                self.inline_buffer.push(Node::HardBreak);
79                Ok(())
80            }
81
82            tag::ol => {
83                self.flush_inline_buffer();
84                let items = ListParser::convert_list(self, element);
85                self.blocks.push(Node::OrderedList {
86                    start: 1,
87                    items: items?,
88                });
89                Ok(())
90            }
91
92            tag::ul => {
93                self.flush_inline_buffer();
94                let items = ListParser::convert_list(self, element);
95                self.blocks.push(Node::UnorderedList(items?));
96                Ok(())
97            }
98
99            md_tag::parbreak => {
100                self.flush_inline_buffer();
101                Ok(())
102            }
103            md_tag::heading => {
104                self.flush_inline_buffer();
105                let attrs = HeadingAttr::parse(&element.attrs)?;
106                self.convert_children(element)?;
107                self.flush_inline_buffer_as_block(|content| {
108                    Node::heading(attrs.level as u8 + 1, content)
109                });
110                Ok(())
111            }
112
113            md_tag::raw => {
114                let attrs = RawAttr::parse(&element.attrs)?;
115                if attrs.block {
116                    self.flush_inline_buffer();
117                    self.blocks
118                        .push(Node::code_block(Some(attrs.lang), attrs.text));
119                } else {
120                    self.inline_buffer.push(Node::InlineCode(attrs.text));
121                }
122                Ok(())
123            }
124
125            md_tag::quote => {
126                let prev_blocks = std::mem::take(&mut self.blocks);
127                self.flush_inline_buffer();
128                self.convert_children(element)?;
129                let content = Node::Paragraph(std::mem::take(&mut self.inline_buffer));
130                let mut quote = std::mem::take(&mut self.blocks);
131                quote.push(content);
132                self.blocks.clear();
133                self.blocks.extend(prev_blocks);
134                self.blocks.push(Node::BlockQuote(quote));
135                Ok(())
136            }
137
138            md_tag::figure => self.convert_figure(element),
139            md_tag::link => self.convert_link(element),
140            md_tag::image => self.convert_image(element),
141
142            md_tag::linebreak => {
143                self.inline_buffer.push(Node::HardBreak);
144                Ok(())
145            }
146
147            md_tag::source => {
148                let src = self.convert_source(element);
149                self.inline_buffer.push(src);
150                Ok(())
151            }
152
153            md_tag::table | md_tag::grid => {
154                self.flush_inline_buffer();
155                if let Some(table) = TableParser::convert_table(self, element)? {
156                    self.blocks.push(table);
157                }
158                Ok(())
159            }
160
161            md_tag::idoc => {
162                let src = self.convert_idoc(element);
163                self.inline_buffer.push(src);
164                Ok(())
165            }
166
167            md_tag::math_equation_inline | md_tag::math_equation_block => {
168                if element.tag == md_tag::math_equation_block {
169                    self.flush_inline_buffer();
170                    self.convert_children(element)?;
171                    let content = std::mem::take(&mut self.inline_buffer);
172                    self.blocks
173                        .push(Node::Custom(Box::new(CenterNode::new(content))));
174                } else {
175                    self.convert_children(element)?;
176                }
177                Ok(())
178            }
179
180            md_tag::alerts => {
181                self.flush_inline_buffer();
182                let attrs = AlertsAttr::parse(&element.attrs)?;
183                let prev_blocks = std::mem::take(&mut self.blocks);
184                self.flush_inline_buffer();
185                self.convert_children(element)?;
186                let content = Node::Paragraph(std::mem::take(&mut self.inline_buffer));
187                let mut quote = std::mem::take(&mut self.blocks);
188                quote.push(content);
189                self.blocks.clear();
190                self.blocks.extend(prev_blocks);
191                self.blocks.push(Node::Custom(Box::new(AlertNode {
192                    content: quote,
193                    class: attrs.class,
194                })));
195                Ok(())
196            }
197
198            md_tag::verbatim => {
199                let attrs = VerbatimAttr::parse(&element.attrs)?;
200                if attrs.block {
201                    self.flush_inline_buffer();
202                    self.blocks.push(Node::Custom(Box::new(BlockVerbatimNode {
203                        content: attrs.src,
204                    })));
205                } else {
206                    self.inline_buffer
207                        .push(Node::Custom(Box::new(VerbatimNode { content: attrs.src })));
208                }
209                Ok(())
210            }
211
212            _ => {
213                let tag_name = element.tag.resolve().to_string();
214
215                if !tag_name.starts_with("m1") {
216                    // self.warn_at(
217                    //     Some(element.span),
218                    //     eco_format!(
219                    //         "unsupported HTML element `<{tag_name}>`; exported as raw HTML"
220                    //     ),
221                    // );
222                    let html_element = self.create_html_element(element)?;
223                    self.inline_buffer.push(html_element);
224                } else {
225                    self.convert_children(element)?;
226                }
227                Ok(())
228            }
229        }
230    }
231
232    /// Create a CommonMark HTML element from the given HTML element    
233    pub(crate) fn create_html_element(&mut self, element: &HtmlElement) -> Result<Node> {
234        let attributes = element
235            .attrs
236            .0
237            .iter()
238            .map(|(name, value)| HtmlAttribute {
239                name: name.resolve().to_string().into(),
240                value: value.clone(),
241            })
242            .collect();
243
244        let (inline_nodes, block_nodes) = self.capture_children(element)?;
245
246        let mut children = Vec::new();
247        if !inline_nodes.is_empty() {
248            children.extend(inline_nodes);
249        }
250        children.extend(block_nodes);
251
252        Ok(Node::HtmlElement(CmarkHtmlElement {
253            tag: element.tag.resolve().to_string().into(),
254            attributes,
255            children,
256            self_closing: element.children.is_empty(),
257        }))
258    }
259
260    pub fn flush_inline_buffer(&mut self) {
261        if !self.inline_buffer.is_empty() {
262            self.blocks
263                .push(Node::Paragraph(std::mem::take(&mut self.inline_buffer)));
264        }
265    }
266
267    pub fn flush_inline_buffer_as_block(&mut self, make_block: impl FnOnce(Vec<Node>) -> Node) {
268        if !self.inline_buffer.is_empty() {
269            self.blocks
270                .push(make_block(std::mem::take(&mut self.inline_buffer)));
271        }
272    }
273
274    pub fn convert_children(&mut self, element: &HtmlElement) -> Result<()> {
275        for child in &element.children {
276            match child {
277                HtmlNode::Text(text, _) => {
278                    self.inline_buffer.push(Node::Text(text.clone()));
279                }
280                HtmlNode::Element(element) => {
281                    self.convert_element(element)?;
282                }
283                HtmlNode::Frame(frame) => {
284                    let res = self.convert_frame(&frame.inner);
285                    self.inline_buffer.push(res);
286                }
287                HtmlNode::Tag(..) => {}
288            }
289        }
290        Ok(())
291    }
292
293    pub fn convert_children_into(
294        &mut self,
295        target: &mut Vec<Node>,
296        element: &HtmlElement,
297    ) -> Result<()> {
298        let prev_buffer = std::mem::take(&mut self.inline_buffer);
299        self.convert_children(element)?;
300        target.append(&mut self.inline_buffer);
301        self.inline_buffer = prev_buffer;
302        Ok(())
303    }
304
305    /// Convert element children while capturing both inline and block outputs.
306    pub fn capture_children(&mut self, element: &HtmlElement) -> Result<(Vec<Node>, Vec<Node>)> {
307        let prev_buffer = std::mem::take(&mut self.inline_buffer);
308        let prev_blocks = std::mem::take(&mut self.blocks);
309
310        self.convert_children(element)?;
311
312        let inline = std::mem::take(&mut self.inline_buffer);
313        let blocks = std::mem::take(&mut self.blocks);
314
315        self.inline_buffer = prev_buffer;
316        self.blocks = prev_blocks;
317
318        Ok((inline, blocks))
319    }
320
321    pub(crate) fn warn_at(&mut self, span: Option<Span>, message: EcoString) {
322        let span = span.unwrap_or_else(Span::detached);
323        let span = self
324            .feat
325            .wrap_info
326            .as_ref()
327            .and_then(|info| self.remap_span_from_wrapper(span, info))
328            .unwrap_or(span.into());
329
330        let diag = SourceDiagnostic::warning(span, message);
331        self.warnings.extend(std::iter::once(diag));
332    }
333
334    fn remap_span_from_wrapper(&self, span: Span, info: &crate::WrapInfo) -> Option<DiagSpan> {
335        info.remap_diag_span(self.world.as_ref(), span.into())
336    }
337}
338
339#[derive(Debug, Clone)]
340pub(crate) struct Comment(pub EcoString);
341
342impl CustomNode for Comment {
343    fn as_any(&self) -> &dyn std::any::Any {
344        self
345    }
346
347    fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
348        self
349    }
350
351    fn write_inline(&self, writer: &mut InlineWriterProxy) -> WriteResult<()> {
352        writer.write_str("<!-- ")?;
353        writer.write_str(&self.0)?;
354        writer.write_str(" -->")?;
355        Ok(())
356    }
357
358    fn clone_box(&self) -> Box<dyn CustomNode> {
359        Box::new(self.clone())
360    }
361
362    fn eq_box(&self, other: &dyn CustomNode) -> bool {
363        if let Some(other) = other.as_any().downcast_ref::<Comment>() {
364            self.0 == other.0
365        } else {
366            false
367        }
368    }
369
370    fn is_block(&self) -> bool {
371        false
372    }
373}
374
375impl HtmlToAstParser {
376    pub fn is_block_element(element: &HtmlElement) -> bool {
377        matches!(
378            element.tag,
379            tag::p
380                | tag::div
381                | tag::blockquote
382                | tag::h1
383                | tag::h2
384                | tag::h3
385                | tag::h4
386                | tag::h5
387                | tag::h6
388                | tag::hr
389                | tag::pre
390                | tag::table
391                | tag::section
392                | tag::article
393                | tag::header
394                | tag::footer
395                | tag::main
396                | tag::aside
397                | tag::nav
398                | tag::ul
399                | tag::ol
400                | md_tag::heading
401                | md_tag::quote
402                | md_tag::raw
403                | md_tag::parbreak
404                | md_tag::table
405                | md_tag::grid
406                | md_tag::figure
407        ) || (element.tag == md_tag::verbatim && Self::is_verbatim_block(element))
408    }
409
410    fn is_verbatim_block(element: &HtmlElement) -> bool {
411        VerbatimAttr::parse(&element.attrs)
412            .map(|attrs| attrs.block)
413            .unwrap_or(false)
414    }
415
416    pub fn process_list_item_element(&mut self, element: &HtmlElement) -> Result<Vec<Node>> {
417        if element.tag == tag::ul || element.tag == tag::ol {
418            let items = super::list::ListParser::convert_list(self, element)?;
419            if element.tag == tag::ul {
420                return Ok(vec![Node::UnorderedList(items)]);
421            } else {
422                return Ok(vec![Node::OrderedList { start: 1, items }]);
423            }
424        }
425
426        let prev_blocks = std::mem::take(&mut self.blocks);
427        let prev_buffer = std::mem::take(&mut self.inline_buffer);
428
429        self.convert_element(element)?;
430        let mut result = Vec::new();
431
432        if !self.blocks.is_empty() {
433            result.extend(std::mem::take(&mut self.blocks));
434        } else if !self.inline_buffer.is_empty() {
435            if Self::is_block_element(element) {
436                result.push(Node::Paragraph(std::mem::take(&mut self.inline_buffer)));
437            } else {
438                result = std::mem::take(&mut self.inline_buffer);
439            }
440        }
441
442        self.blocks = prev_blocks;
443        self.inline_buffer = prev_buffer;
444
445        Ok(result)
446    }
447
448    pub fn parse(mut self, root: &HtmlElement) -> Result<Node> {
449        self.blocks.clear();
450        self.inline_buffer.clear();
451
452        self.convert_element(root)?;
453        self.flush_inline_buffer();
454
455        Ok(Node::Document(self.blocks))
456    }
457}
typlite/parser/core.rs

typlite/parser/
core.rs
