typlite/
lib.rs

1//! # Typlite
2
3// todo: remove me
4#![allow(missing_docs)]
5
6pub mod attributes;
7pub mod common;
8mod diagnostics;
9mod error;
10pub mod parser;
11pub mod tags;
12pub mod writer;
13
14use std::ops::Range;
15use std::path::PathBuf;
16use std::str::FromStr;
17use std::sync::Arc;
18
19pub use error::*;
20
21use cmark_writer::ast::Node;
22use tinymist_project::base::ShadowApi;
23use tinymist_project::vfs::WorkspaceResolver;
24use tinymist_project::{EntryReader, LspWorld, TaskInputs};
25use tinymist_std::error::prelude::*;
26use tinymist_std::typst_shim::syntax::{VirtualPathExt, resolve_path_from_id};
27use typst::World;
28use typst::WorldExt;
29use typst::diag::SourceDiagnostic;
30use typst::foundations::Bytes;
31use typst_html::HtmlDocument;
32use typst_syntax::{DiagSpan, LinkedNode, RootedPath, Source, Span, VirtualPath, VirtualRoot};
33
34pub use crate::common::Format;
35use crate::diagnostics::WarningCollector;
36use crate::parser::HtmlToAstParser;
37use crate::writer::WriterFactory;
38use typst_syntax::FileId;
39
40use crate::tinymist_std::typst::LazyHash;
41use crate::tinymist_std::typst::foundations::Value::Str;
42
43/// The result type for typlite.
44pub type Result<T, Err = Error> = std::result::Result<T, Err>;
45
46pub use cmark_writer::ast;
47pub use tinymist_project::CompileOnceArgs;
48pub use tinymist_std;
49
50#[derive(Clone)]
51pub struct MarkdownDocument {
52    pub base: HtmlDocument,
53    world: Arc<LspWorld>,
54    feat: TypliteFeat,
55    ast: Option<Node>,
56    warnings: WarningCollector,
57}
58
59impl MarkdownDocument {
60    /// Create a new MarkdownDocument instance
61    pub fn new(base: HtmlDocument, world: Arc<LspWorld>, feat: TypliteFeat) -> Self {
62        Self {
63            base,
64            world,
65            feat,
66            ast: None,
67            warnings: WarningCollector::default(),
68        }
69    }
70
71    /// Create a MarkdownDocument instance with pre-parsed AST
72    pub fn with_ast(
73        base: HtmlDocument,
74        world: Arc<LspWorld>,
75        feat: TypliteFeat,
76        ast: Node,
77    ) -> Self {
78        Self {
79            base,
80            world,
81            feat,
82            ast: Some(ast),
83            warnings: WarningCollector::default(),
84        }
85    }
86
87    /// Replace the backing warning collector, preserving shared state with
88    /// other components of the pipeline.
89    pub(crate) fn with_warning_collector(mut self, collector: WarningCollector) -> Self {
90        self.warnings = collector;
91        self
92    }
93
94    /// Get a snapshot of all collected warnings so far.
95    pub fn warnings(&self) -> Vec<SourceDiagnostic> {
96        let warnings = self.warnings.snapshot();
97        if let Some(info) = &self.feat.wrap_info {
98            warnings
99                .into_iter()
100                .filter_map(|diag| self.remap_diagnostic(diag, info))
101                .collect()
102        } else {
103            warnings
104        }
105    }
106
107    /// Internal accessor for sharing the collector with the parser.
108    fn warning_collector(&self) -> WarningCollector {
109        self.warnings.clone()
110    }
111
112    fn remap_diagnostic(
113        &self,
114        mut diagnostic: SourceDiagnostic,
115        info: &WrapInfo,
116    ) -> Option<SourceDiagnostic> {
117        if let Some(span) = info.remap_diag_span(self.world.as_ref(), diagnostic.span) {
118            diagnostic.span = span;
119        } else {
120            return None;
121        }
122
123        diagnostic.trace = diagnostic
124            .trace
125            .into_iter()
126            .filter_map(
127                |mut spanned| match info.remap_span(self.world.as_ref(), spanned.span) {
128                    Some(span) => {
129                        spanned.span = span;
130                        Some(spanned)
131                    }
132                    None => None,
133                },
134            )
135            .collect();
136
137        diagnostic.hints = diagnostic
138            .hints
139            .into_iter()
140            .filter_map(|mut spanned| {
141                match info.remap_diag_span(self.world.as_ref(), spanned.span) {
142                    Some(span) => {
143                        spanned.span = span;
144                        Some(spanned)
145                    }
146                    None => None,
147                }
148            })
149            .collect();
150
151        Some(diagnostic)
152    }
153
154    /// Parse HTML document to AST
155    pub fn parse(&self) -> tinymist_std::Result<Node> {
156        if let Some(ast) = &self.ast {
157            return Ok(ast.clone());
158        }
159        let parser = HtmlToAstParser::new(self.feat.clone(), &self.world, self.warning_collector());
160        parser.parse(self.base.root()).context_ut("failed to parse")
161    }
162
163    /// Convert content to markdown string
164    pub fn to_md_string(&self) -> tinymist_std::Result<ecow::EcoString> {
165        let mut output = ecow::EcoString::new();
166        let ast = self.parse()?;
167
168        let mut writer = WriterFactory::create(Format::Md);
169        writer
170            .write_eco(&ast, &mut output)
171            .context_ut("failed to write")?;
172
173        Ok(output)
174    }
175
176    /// Convert content to plain text string
177    pub fn to_text_string(&self) -> tinymist_std::Result<ecow::EcoString> {
178        let mut output = ecow::EcoString::new();
179        let ast = self.parse()?;
180
181        let mut writer = WriterFactory::create(Format::Text);
182        writer
183            .write_eco(&ast, &mut output)
184            .context_ut("failed to write")?;
185
186        Ok(output)
187    }
188
189    /// Convert the content to a LaTeX string.
190    pub fn to_tex_string(&self) -> tinymist_std::Result<ecow::EcoString> {
191        let mut output = ecow::EcoString::new();
192        let ast = self.parse()?;
193
194        let mut writer = WriterFactory::create(Format::LaTeX);
195        writer
196            .write_eco(&ast, &mut output)
197            .context_ut("failed to write")?;
198
199        Ok(output)
200    }
201
202    /// Convert the content to a DOCX document
203    #[cfg(feature = "docx")]
204    pub fn to_docx(&self) -> tinymist_std::Result<Vec<u8>> {
205        let ast = self.parse()?;
206
207        let mut writer = WriterFactory::create(Format::Docx);
208        writer.write_vec(&ast).context_ut("failed to write")
209    }
210}
211
212/// A color theme for rendering the content. The valid values can be checked in [color-scheme](https://developer.mozilla.org/en-US/docs/Web/CSS/color-scheme).
213#[derive(Debug, Default, Clone, Copy)]
214pub enum ColorTheme {
215    #[default]
216    Light,
217    Dark,
218}
219
220#[derive(Debug, Clone)]
221pub struct WrapInfo {
222    /// The synthetic wrapper file that hosts the original Typst source.
223    pub wrap_file_id: FileId,
224    /// The user's actual Typst source file.
225    pub original_file_id: FileId,
226    /// Number of UTF-8 bytes injected ahead of the original source.
227    pub prefix_len_bytes: usize,
228}
229
230impl WrapInfo {
231    /// Translate a diagnostic span from the wrapper file back into the original
232    /// file.
233    pub fn remap_diag_span(&self, world: &dyn typst::World, span: DiagSpan) -> Option<DiagSpan> {
234        if span.id() != Some(self.wrap_file_id) {
235            return Some(span);
236        }
237
238        let range = self.remap_range(world, world.range(span)?)?;
239        Some(DiagSpan::from_range(self.original_file_id, range))
240    }
241
242    /// Translate a span from the wrapper file back into the original file.
243    pub fn remap_span(&self, world: &dyn typst::World, span: Span) -> Option<Span> {
244        if span.id() != Some(self.wrap_file_id) {
245            return Some(span);
246        }
247
248        let range = self.remap_range(world, world.range(span)?)?;
249        let original_source = world.source(self.original_file_id).ok()?;
250        WrapInfo::span_covering_range(&original_source, range)
251    }
252
253    fn remap_range(&self, world: &dyn typst::World, range: Range<usize>) -> Option<Range<usize>> {
254        let start = range.start.checked_sub(self.prefix_len_bytes)?;
255        let end = range.end.checked_sub(self.prefix_len_bytes)?;
256
257        let original_source = world.source(self.original_file_id).ok()?;
258        let original_len = original_source.lines().len_bytes();
259
260        if start >= original_len || end > original_len {
261            return None;
262        }
263
264        Some(start..end)
265    }
266
267    fn span_covering_range(source: &Source, range: Range<usize>) -> Option<Span> {
268        fn inner(node: LinkedNode<'_>, range: &Range<usize>) -> Option<Span> {
269            let node_range = node.range();
270            if range.start < node_range.start || range.end > node_range.end {
271                return None;
272            }
273
274            node.children()
275                .find_map(|child| inner(child, range))
276                .or_else(|| Some(node.span()))
277        }
278
279        inner(LinkedNode::new(source.root()), &range)
280    }
281}
282
283#[derive(Debug, Default, Clone)]
284pub struct TypliteFeat {
285    /// The preferred color theme.
286    pub color_theme: Option<ColorTheme>,
287    /// The path of external assets directory.
288    pub assets_path: Option<PathBuf>,
289    /// Allows GFM (GitHub Flavored Markdown) markups.
290    pub gfm: bool,
291    /// Annotate the elements for identification.
292    pub annotate_elem: bool,
293    /// Embed errors in the output instead of yielding them.
294    pub soft_error: bool,
295    /// Remove HTML tags from the output.
296    pub remove_html: bool,
297    /// The target to convert
298    pub target: Format,
299    /// Import context for code examples (e.g., "#import \"/path/to/file.typ\":
300    /// *")
301    pub import_context: Option<String>,
302    /// Specifies the package to process markup.
303    ///
304    /// ## `article` function
305    ///
306    /// The article function is used to wrap the typst content during
307    /// compilation.
308    ///
309    /// typlite exactly uses the `#article` function to process the content as
310    /// follow:
311    ///
312    /// ```typst
313    /// #import "@local/processor": article
314    /// #article(include "the-processed-content.typ")
315    /// ```
316    ///
317    /// It resembles the regular typst show rule function, like `#show:
318    /// article`.
319    pub processor: Option<String>,
320    /// Optional mapping from the wrapper file back to the original source.
321    pub wrap_info: Option<WrapInfo>,
322}
323
324impl TypliteFeat {
325    pub fn prepare_world(
326        &self,
327        world: &LspWorld,
328        format: Format,
329    ) -> tinymist_std::Result<(LspWorld, Option<WrapInfo>)> {
330        let entry = world.entry_state();
331        let main = entry.main();
332        let current = main.context("no main file in workspace")?;
333
334        if WorkspaceResolver::is_package_file(current) {
335            bail!("package file is not supported");
336        }
337
338        let wrap_main_id = resolve_path_from_id(current, "__wrap_md_main.typ")
339            .ok()
340            .context_ut("failed to resolve virtual path")?
341            .intern();
342
343        let (main_id, main_content) = match self.processor.as_ref() {
344            None => (wrap_main_id, None),
345            Some(processor) => {
346                let main_id = resolve_path_from_id(current, "__md_main.typ")
347                    .ok()
348                    .context_ut("failed to resolve virtual path")?
349                    .intern();
350                let content = format!(
351                    r#"#import {processor:?}: article
352#article(include "__wrap_md_main.typ")"#
353                );
354
355                (main_id, Some(Bytes::from_string(content)))
356            }
357        };
358
359        // Start with existing inputs from the world (CLI inputs)
360        let mut dict = (**world.inputs()).clone();
361
362        // Add typlite-specific inputs
363        dict.insert("x-target".into(), Str("md".into()));
364        if format == Format::Text || self.remove_html {
365            dict.insert("x-remove-html".into(), Str("true".into()));
366        }
367
368        let task_inputs = TaskInputs {
369            entry: Some(entry.select_in_workspace(main_id.vpath().as_rooted_path_compat())),
370            inputs: Some(Arc::new(LazyHash::new(dict))),
371        };
372
373        let mut world = world.task(task_inputs).html_task().into_owned();
374
375        let markdown_root = VirtualRoot::Package(
376            typst_syntax::package::PackageSpec::from_str("@local/_markdown:0.1.0")
377                .context_ut("failed to import markdown package")?,
378        );
379        let markdown_id = FileId::new(RootedPath::new(
380            markdown_root.clone(),
381            VirtualPath::new("lib.typ")
382                .ok()
383                .context_ut("failed to resolve markdown lib path")?,
384        ));
385
386        world
387            .map_shadow_by_id(
388                FileId::new(RootedPath::new(
389                    markdown_root,
390                    VirtualPath::new("typst.toml")
391                        .ok()
392                        .context_ut("failed to resolve markdown manifest path")?,
393                )),
394                Bytes::from_string(include_str!("markdown-typst.toml")),
395            )
396            .context_ut("cannot map markdown-typst.toml")?;
397        world
398            .map_shadow_by_id(
399                markdown_id,
400                Bytes::from_string(include_str!("markdown.typ")),
401            )
402            .context_ut("cannot map markdown.typ")?;
403        let original_source = world
404            .source(current)
405            .context_ut("cannot fetch main source")?
406            .text()
407            .to_owned();
408
409        const WRAP_PREFIX: &str =
410            "#import \"@local/_markdown:0.1.0\": md-doc, example; #show: md-doc\n";
411        let wrap_content = format!("{WRAP_PREFIX}{original_source}");
412
413        world
414            .map_shadow_by_id(wrap_main_id, Bytes::from_string(wrap_content))
415            .context_ut("cannot map source for main file")?;
416
417        if let Some(main_content) = main_content {
418            world
419                .map_shadow_by_id(main_id, main_content)
420                .context_ut("cannot map source for main file")?;
421        }
422
423        let wrap_info = Some(WrapInfo {
424            wrap_file_id: wrap_main_id,
425            original_file_id: current,
426            prefix_len_bytes: WRAP_PREFIX.len(),
427        });
428
429        Ok((world, wrap_info))
430    }
431}
432
433/// Task builder for converting a typst document to Markdown.
434pub struct Typlite {
435    /// The universe to use for the conversion.
436    world: Arc<LspWorld>,
437    /// Features for the conversion.
438    feat: TypliteFeat,
439    /// The format to use for the conversion.
440    format: Format,
441}
442
443impl Typlite {
444    /// Creates a new Typlite instance from a [`World`].
445    pub fn new(world: Arc<LspWorld>) -> Self {
446        Self {
447            world,
448            feat: Default::default(),
449            format: Format::Md,
450        }
451    }
452
453    /// Sets conversion features
454    pub fn with_feature(mut self, feat: TypliteFeat) -> Self {
455        self.feat = feat;
456        self
457    }
458
459    pub fn with_format(mut self, format: Format) -> Self {
460        self.format = format;
461        self
462    }
463
464    /// Convert the content to a markdown string.
465    pub fn convert(self) -> tinymist_std::Result<ecow::EcoString> {
466        match self.format {
467            Format::Md => self.convert_doc(Format::Md)?.to_md_string(),
468            Format::LaTeX => self.convert_doc(Format::LaTeX)?.to_tex_string(),
469            Format::Text => self.convert_doc(Format::Text)?.to_text_string(),
470            #[cfg(feature = "docx")]
471            Format::Docx => bail!("docx format is not supported"),
472        }
473    }
474
475    /// Convert the content to a DOCX document
476    #[cfg(feature = "docx")]
477    pub fn to_docx(self) -> tinymist_std::Result<Vec<u8>> {
478        if self.format != Format::Docx {
479            bail!("format is not DOCX");
480        }
481        self.convert_doc(Format::Docx)?.to_docx()
482    }
483
484    /// Convert the content to a markdown document.
485    pub fn convert_doc(mut self, format: Format) -> tinymist_std::Result<MarkdownDocument> {
486        let (prepared_world, wrap_info) = self.feat.prepare_world(&self.world, format)?;
487        self.feat.wrap_info = wrap_info;
488        let feat = self.feat.clone();
489        let world = Arc::new(prepared_world);
490        Self::convert_doc_prepared(feat, format, world)
491    }
492
493    /// Convert the content to a markdown document.
494    pub fn convert_doc_prepared(
495        feat: TypliteFeat,
496        format: Format,
497        world: Arc<LspWorld>,
498    ) -> tinymist_std::Result<MarkdownDocument> {
499        // this is not affected by syntax-only mode (typst_shim::compile_opt)
500        let compiled = typst::compile(&world);
501        let collector = WarningCollector::default();
502        collector.extend(
503            compiled
504                .warnings
505                .iter()
506                .filter(|&diag| {
507                    diag.message.as_str()
508                        != "html export is under active development and incomplete"
509                })
510                .cloned(),
511        );
512        let base = compiled.output?;
513        let mut feat = feat;
514        feat.target = format;
515        Ok(MarkdownDocument::new(base, world.clone(), feat).with_warning_collector(collector))
516    }
517}
518
519#[cfg(test)]
520mod tests;
typlite/lib.rs

typlite/
lib.rs
