tinymist_query/
semantic_tokens_full.rs

1use crate::prelude::*;
2
3/// The [`textDocument/semanticTokens/full`] request is sent from the client to
4/// the server to resolve the semantic tokens of a given file.
5///
6/// [`textDocument/semanticTokens/full`]: https://microsoft.github.io/language-server-protocol/specification#textDocument_semanticTokens
7///
8/// Semantic tokens are used to add additional color information to a file that
9/// depends on language specific symbol information. A semantic token request
10/// usually produces a large result. The protocol therefore supports encoding
11/// tokens with numbers. In addition, optional support for deltas is available,
12/// i.e. [`semantic_tokens_full_delta`].
13///
14/// [`semantic_tokens_full_delta`]: crate::SemanticTokensDeltaRequest
15///
16/// # Compatibility
17///
18/// This request was introduced in specification version 3.16.0.
19#[derive(Debug, Clone)]
20pub struct SemanticTokensFullRequest {
21    /// The path of the document to get semantic tokens for.
22    pub path: PathBuf,
23}
24
25impl SemanticRequest for SemanticTokensFullRequest {
26    type Response = SemanticTokensResult;
27
28    /// Handles the request to compute the semantic tokens for a given document.
29    fn request(self, ctx: &mut LocalContext) -> Option<Self::Response> {
30        let source = ctx.source_by_path(&self.path).ok()?;
31        let (tokens, result_id) = ctx.cached_tokens(&source);
32
33        Some(SemanticTokensResult::Tokens(SemanticTokens {
34            result_id,
35            data: tokens.as_ref().clone(),
36        }))
37    }
38}
39
40#[cfg(test)]
41mod tests {
42    use super::*;
43    use crate::tests::*;
44
45    /// This is converted by Copilot from TypeScript `to_multiline_tokens2`.
46    /// <https://github.com/microsoft/vscode/blob/2acc0e52cbc7434c415f221d5c34ee1bbdd6cd71/src/vs/editor/common/services/semanticTokensProviderStyling.ts#L147>
47    fn check_tokens(tokens: &SemanticTokens) {
48        const DESIRED_TOKENS_PER_AREA: usize = 400;
49        const DESIRED_MAX_AREAS: usize = 1024;
50
51        let src_data = &tokens.data;
52        let token_count = src_data.len();
53        let tokens_per_area = std::cmp::max(
54            (token_count as f64 / DESIRED_MAX_AREAS as f64).ceil() as usize,
55            DESIRED_TOKENS_PER_AREA,
56        );
57
58        let mut token_index = 0;
59        let mut last_line_number = 1;
60        let mut last_start_character = 0;
61
62        while token_index < token_count {
63            let token_start_index = token_index;
64            let mut token_end_index =
65                std::cmp::min(token_start_index + tokens_per_area, token_count);
66
67            // Keep tokens on the same line in the same area...
68            if token_end_index < token_count {
69                let mut small_token_end_index = token_end_index;
70                while small_token_end_index - 1 > token_start_index
71                    && src_data[small_token_end_index].delta_line == 0
72                {
73                    small_token_end_index -= 1;
74                }
75
76                if small_token_end_index - 1 == token_start_index {
77                    // there are so many tokens on this line that our area would be empty, we must
78                    // now go right
79                    let mut big_token_end_index = token_end_index;
80                    while big_token_end_index + 1 < token_count
81                        && src_data[big_token_end_index].delta_line == 0
82                    {
83                        big_token_end_index += 1;
84                    }
85                    token_end_index = big_token_end_index;
86                } else {
87                    token_end_index = small_token_end_index;
88                }
89            }
90
91            let mut prev_line_number = 0;
92            let mut prev_end_character = 0;
93
94            while token_index < token_end_index {
95                let delta_line = src_data[token_index].delta_line;
96                let delta_character = src_data[token_index].delta_start;
97                let length = src_data[token_index].length;
98                let line_number = last_line_number + delta_line;
99                let start_character = if delta_line == 0 {
100                    last_start_character + delta_character
101                } else {
102                    delta_character
103                };
104                // delta_character
105                let end_character = start_character + length;
106
107                if end_character <= start_character {
108                    // this token is invalid (most likely a negative length casted to uint32)
109                    panic!(
110                        "Invalid length for semantic token at line {line_number}, character {start_character}, end: {end_character}"
111                    );
112                } else if prev_line_number == line_number && prev_end_character > start_character {
113                    // this token overlaps with the previous token
114                    panic!(
115                        "Overlapping semantic tokens at line {line_number}, character {start_character}, previous line {prev_line_number}, previous end {prev_end_character}"
116                    );
117                } else {
118                    prev_line_number = line_number;
119                    prev_end_character = end_character;
120                }
121
122                last_line_number = line_number;
123                last_start_character = start_character;
124                token_index += 1;
125            }
126        }
127    }
128
129    #[test]
130    fn test() {
131        snapshot_testing("semantic_tokens", &|ctx, path| {
132            let request = SemanticTokensFullRequest { path: path.clone() };
133
134            let mut result = request.request(ctx).unwrap();
135            if let SemanticTokensResult::Tokens(tokens) = &mut result {
136                tokens.result_id.take();
137            }
138
139            match &result {
140                SemanticTokensResult::Tokens(tokens) => {
141                    check_tokens(tokens);
142                }
143                SemanticTokensResult::Partial(_) => {
144                    panic!("Unexpected partial result");
145                }
146            }
147
148            assert_snapshot!(serde_json::to_string(&result).unwrap());
149        });
150    }
151}
tinymist_query/semantic_tokens_full.rs

tinymist_query/
semantic_tokens_full.rs
