tinymist_world/parser/
typst_tokens.rs

1//! Types for tokens used for Typst syntax
2
3use strum::EnumIter;
4
5/// Very similar to [`typst_ide::Tag`], but with convenience traits, and
6/// extensible because we want to further customize highlighting
7#[derive(Debug, Clone, Copy, EnumIter)]
8#[repr(u32)]
9pub enum TokenType {
10    // Standard LSP types
11    Comment,
12    String,
13    Keyword,
14    Operator,
15    Number,
16    Function,
17    Decorator,
18    // Custom types
19    Bool,
20    Punctuation,
21    Escape,
22    Link,
23    Raw,
24    Label,
25    Ref,
26    Heading,
27    ListMarker,
28    ListTerm,
29    Delimiter,
30    Interpolated,
31    Error,
32    /// Any text in markup without a more specific token type, possible styled.
33    ///
34    /// We perform styling (like bold and italics) via modifiers. That means
35    /// everything that should receive styling needs to be a token so we can
36    /// apply a modifier to it. This token type is mostly for that, since
37    /// text should usually not be specially styled.
38    Text,
39}
40
41impl From<TokenType> for &'static str {
42    fn from(token_type: TokenType) -> Self {
43        use TokenType::*;
44
45        match token_type {
46            Comment => "comment",
47            String => "string",
48            Keyword => "keyword",
49            Operator => "operator",
50            Number => "number",
51            Function => "function",
52            Decorator => "decorator",
53            Bool => "bool",
54            Punctuation => "punctuation",
55            Escape => "escape",
56            Link => "link",
57            Raw => "raw",
58            Label => "label",
59            Ref => "ref",
60            Heading => "heading",
61            ListMarker => "marker",
62            ListTerm => "term",
63            Delimiter => "delim",
64            Interpolated => "pol",
65            Error => "error",
66            Text => "text",
67        }
68    }
69}
70
71#[derive(Debug, Clone, Copy, EnumIter)]
72#[repr(u8)]
73pub enum Modifier {
74    Strong,
75    Emph,
76    Math,
77}
78
79impl Modifier {
80    pub fn index(self) -> u8 {
81        self as u8
82    }
83
84    pub fn bitmask(self) -> u32 {
85        0b1 << self.index()
86    }
87}
88
89impl From<Modifier> for &'static str {
90    fn from(modifier: Modifier) -> Self {
91        use Modifier::*;
92
93        match modifier {
94            Strong => "strong",
95            Emph => "emph",
96            Math => "math",
97        }
98    }
99}
100
101#[cfg(test)]
102mod test {
103    use strum::IntoEnumIterator;
104
105    use super::*;
106
107    #[test]
108    fn ensure_not_too_many_modifiers() {
109        // Because modifiers are encoded in a 32 bit bitmask, we can't have more than 32
110        // modifiers
111        assert!(Modifier::iter().len() <= 32);
112    }
113}
tinymist_world/parser/typst_tokens.rs

tinymist_world/parser/
typst_tokens.rs
