tinymist_query/index/
scip.rs

1//! SCIP index encoding.
2
3use core::fmt::Write as _;
4use std::sync::Arc;
5
6use crate::analysis::SharedContext;
7use crate::prelude::Definition;
8use crate::syntax::DefKind;
9use lsp_types::{Hover, HoverContents, MarkedString, Range};
10use protobuf::{Enum, EnumOrUnknown, Message, MessageField};
11use scip::types::{
12    Descriptor as ScipDescriptor, Document as ScipDocument, Index as ScipIndex,
13    Metadata as ScipMetadata, Occurrence as ScipOccurrence, Package as ScipPackage,
14    PositionEncoding as ScipPositionEncoding, Relationship as ScipRelationship, Signature,
15    Symbol as ScipSymbol, SymbolInformation as ScipSymbolInformation, SymbolRole as ScipSymbolRole,
16    TextEncoding as ScipTextEncoding, ToolInfo as ScipToolInfo, descriptor, symbol_information,
17};
18use tinymist_analysis::docs::{DefDocs, ParamDocs, SignatureDocs};
19use tinymist_std::error::WithContextUntyped;
20use tinymist_std::hash::FxHashMap;
21use typst::syntax::{FileId, Source, Span};
22use typst_shim::syntax::source_range;
23
24use super::scip_utils::{ScipParamGroup, merge_symbol_information, push_relationship_unique};
25use super::{FileIndex, KnowledgeWithContext};
26
27/// Public package API information encoded as SCIP relationships.
28#[derive(Debug, Clone, Default)]
29pub struct ScipPublicApi {
30    /// Public modules in the package docs tree.
31    pub modules: Vec<ScipPublicModule>,
32}
33
34/// Public symbols exported by one module.
35#[derive(Debug, Clone, Default)]
36pub struct ScipPublicModule {
37    /// The module file path, relative to the package root.
38    pub file_path: Option<String>,
39    /// The SCIP symbol of the module.
40    pub module_symbol: Option<String>,
41    /// Public SCIP symbols exported by the module.
42    pub public_symbols: Vec<String>,
43}
44
45impl KnowledgeWithContext<'_> {
46    /// Dumps the knowledge in SCIP protobuf format.
47    pub fn to_scip_bytes(&self) -> tinymist_std::Result<Vec<u8>> {
48        let index = self.to_scip_index()?;
49        index
50            .write_to_bytes()
51            .context_ut("cannot serialize SCIP index")
52    }
53
54    /// Dumps the knowledge in SCIP protobuf format with public package API data.
55    pub fn to_scip_bytes_with_public_api(
56        &self,
57        public_api: &ScipPublicApi,
58    ) -> tinymist_std::Result<Vec<u8>> {
59        let index = self.to_scip_index_with_public_api(public_api)?;
60        index
61            .write_to_bytes()
62            .context_ut("cannot serialize SCIP index")
63    }
64
65    /// Dumps the knowledge as a SCIP index.
66    pub fn to_scip_index(&self) -> tinymist_std::Result<ScipIndex> {
67        self.to_scip_index_with_public_api(&ScipPublicApi::default())
68    }
69
70    /// Dumps the knowledge as a SCIP index with public package API data.
71    pub fn to_scip_index_with_public_api(
72        &self,
73        public_api: &ScipPublicApi,
74    ) -> tinymist_std::Result<ScipIndex> {
75        let mut encoder = ScipEncoder::new(self.ctx);
76        encoder.emit_files(&self.knowledge.files)?;
77        encoder.emit_public_api(public_api);
78        let (documents, external_symbols) = encoder.finish();
79        Ok(ScipIndex {
80            metadata: MessageField::some(ScipMetadata {
81                tool_info: MessageField::some(ScipToolInfo {
82                    name: "tinymist".to_owned(),
83                    version: env!("CARGO_PKG_VERSION").to_owned(),
84                    ..Default::default()
85                }),
86                project_root: self.knowledge.meta.project_root.to_string(),
87                text_document_encoding: EnumOrUnknown::new(ScipTextEncoding::UTF8),
88                ..Default::default()
89            }),
90            documents,
91            external_symbols,
92            ..Default::default()
93        })
94    }
95}
96
97struct ScipEncoder<'a> {
98    ctx: &'a Arc<SharedContext>,
99    documents: Vec<ScipDocumentBuilder>,
100    document_by_fid: FxHashMap<FileId, usize>,
101    symbols: FxHashMap<Definition, String>,
102    external_symbols: FxHashMap<String, ScipSymbolInformation>,
103}
104
105struct ScipDocumentBuilder {
106    relative_path: String,
107    occurrences: FxHashMap<ScipOccurrenceKey, ScipOccurrence>,
108    symbols: FxHashMap<String, ScipSymbolInformation>,
109}
110
111#[derive(Debug, Clone, PartialEq, Eq, Hash)]
112struct ScipOccurrenceKey {
113    range: Vec<i32>,
114    symbol: String,
115}
116
117impl<'a> ScipEncoder<'a> {
118    fn new(ctx: &'a Arc<SharedContext>) -> Self {
119        Self {
120            ctx,
121            documents: Vec::new(),
122            document_by_fid: FxHashMap::default(),
123            symbols: FxHashMap::default(),
124            external_symbols: FxHashMap::default(),
125        }
126    }
127
128    fn emit_files(&mut self, files: &[FileIndex]) -> tinymist_std::Result<()> {
129        for file in files {
130            let _ = self.document_index(file.fid)?;
131        }
132
133        for (idx, file) in files.iter().enumerate() {
134            eprintln!("emit scip file: {:?}, {idx} of {}", file.fid, files.len());
135            let source = self
136                .ctx
137                .source_by_id(file.fid)
138                .context_ut("cannot get source")?;
139            let document_index = self.document_index(file.fid)?;
140
141            for (span, reference) in &file.references {
142                let symbol = self.symbol(&reference.definition)?;
143                let docs = reference.hover.as_ref().and_then(hover_to_markdown);
144                let def_docs = reference.def_docs.as_ref();
145
146                if let Some(range) = lsp_range(self.ctx, *span, &source) {
147                    self.documents[document_index].push_occurrence(ScipOccurrence {
148                        range: scip_range(range),
149                        symbol: symbol.clone(),
150                        ..Default::default()
151                    });
152                }
153
154                if let Some((def_fid, def_range)) = self.definition_range(&reference.definition) {
155                    let def_document_index = self.document_index(def_fid)?;
156                    self.documents[def_document_index].push_occurrence(ScipOccurrence {
157                        range: scip_range(def_range),
158                        symbol: symbol.clone(),
159                        symbol_roles: ScipSymbolRole::Definition.value(),
160                        ..Default::default()
161                    });
162                    for info in symbol_infos(&reference.definition, &symbol, def_docs, docs) {
163                        self.documents[def_document_index].push_symbol(info);
164                    }
165                } else if let Some(info) =
166                    symbol_info_external(&reference.definition, &symbol, def_docs, docs)
167                {
168                    for item in info {
169                        self.external_symbols
170                            .entry(item.symbol.clone())
171                            .or_insert(item);
172                    }
173                }
174            }
175        }
176
177        Ok(())
178    }
179
180    fn document_index(&mut self, fid: FileId) -> tinymist_std::Result<usize> {
181        if let Some(index) = self.document_by_fid.get(&fid) {
182            return Ok(*index);
183        }
184
185        let _ = self
186            .ctx
187            .source_by_id(fid)
188            .context_ut("cannot get source for SCIP document")?;
189        let index = self.documents.len();
190        self.documents.push(ScipDocumentBuilder {
191            relative_path: fid.vpath().get_without_slash().to_owned(),
192            occurrences: FxHashMap::default(),
193            symbols: FxHashMap::default(),
194        });
195        self.document_by_fid.insert(fid, index);
196        Ok(index)
197    }
198
199    fn symbol(&mut self, definition: &Definition) -> tinymist_std::Result<String> {
200        if let Some(symbol) = self.symbols.get(definition) {
201            return Ok(symbol.clone());
202        }
203
204        let symbol = scip_symbol(self.ctx, definition)?;
205        self.symbols.insert(definition.clone(), symbol.clone());
206        Ok(symbol)
207    }
208
209    fn definition_range(&self, definition: &Definition) -> Option<(FileId, Range)> {
210        let fid = definition.file_id()?;
211        let source = self.ctx.source_by_id(fid).ok()?;
212        let span = definition.decl.span();
213        if span.is_detached() {
214            return None;
215        }
216
217        Some((fid, lsp_range(self.ctx, span, &source)?))
218    }
219
220    fn emit_public_api(&mut self, public_api: &ScipPublicApi) {
221        for module in &public_api.modules {
222            let Some(module_symbol) = module.module_symbol.as_ref() else {
223                continue;
224            };
225
226            if let Some(document_index) = module
227                .file_path
228                .as_deref()
229                .and_then(|path| self.document_index_by_path(path))
230            {
231                self.documents[document_index].push_symbol(public_module_symbol_info(
232                    module_symbol,
233                    &module.public_symbols,
234                ));
235            }
236
237            for public_symbol in &module.public_symbols {
238                self.push_symbol_relationship(
239                    public_symbol,
240                    ScipRelationship {
241                        symbol: module_symbol.clone(),
242                        is_reference: true,
243                        ..Default::default()
244                    },
245                );
246            }
247        }
248    }
249
250    fn document_index_by_path(&self, path: &str) -> Option<usize> {
251        self.documents
252            .iter()
253            .position(|document| document.relative_path == path)
254    }
255
256    fn push_symbol_relationship(&mut self, symbol: &str, relationship: ScipRelationship) {
257        for document in &mut self.documents {
258            if document.push_relationship(symbol, relationship.clone()) {
259                return;
260            }
261        }
262
263        if let Some(info) = self.external_symbols.get_mut(symbol) {
264            push_relationship_unique(&mut info.relationships, relationship);
265            return;
266        }
267
268        self.external_symbols.insert(
269            symbol.to_owned(),
270            ScipSymbolInformation {
271                symbol: symbol.to_owned(),
272                relationships: vec![relationship],
273                ..Default::default()
274            },
275        );
276    }
277
278    fn finish(self) -> (Vec<ScipDocument>, Vec<ScipSymbolInformation>) {
279        let documents = self
280            .documents
281            .into_iter()
282            .map(ScipDocumentBuilder::finish)
283            .collect();
284        let mut external_symbols = self.external_symbols.into_values().collect::<Vec<_>>();
285        external_symbols.sort_by(|left, right| left.symbol.cmp(&right.symbol));
286        (documents, external_symbols)
287    }
288}
289
290impl ScipDocumentBuilder {
291    fn push_occurrence(&mut self, occurrence: ScipOccurrence) {
292        let key = ScipOccurrenceKey {
293            range: occurrence.range.clone(),
294            symbol: occurrence.symbol.clone(),
295        };
296        if let Some(current) = self.occurrences.get_mut(&key) {
297            current.symbol_roles |= occurrence.symbol_roles;
298            if current.override_documentation.is_empty() {
299                current.override_documentation = occurrence.override_documentation;
300            }
301            return;
302        }
303
304        self.occurrences.insert(key, occurrence);
305    }
306
307    fn push_symbol(&mut self, symbol: ScipSymbolInformation) {
308        if let Some(current) = self.symbols.get_mut(&symbol.symbol) {
309            merge_symbol_information(current, symbol);
310            return;
311        }
312
313        self.symbols.insert(symbol.symbol.clone(), symbol);
314    }
315
316    fn push_relationship(&mut self, symbol: &str, relationship: ScipRelationship) -> bool {
317        let Some(current) = self.symbols.get_mut(symbol) else {
318            return false;
319        };
320        push_relationship_unique(&mut current.relationships, relationship);
321        true
322    }
323
324    fn finish(self) -> ScipDocument {
325        let mut occurrences = self.occurrences.into_values().collect::<Vec<_>>();
326        occurrences.sort_by(|left, right| {
327            left.range
328                .cmp(&right.range)
329                .then_with(|| left.symbol.cmp(&right.symbol))
330        });
331        let mut symbols = self.symbols.into_values().collect::<Vec<_>>();
332        symbols.sort_by(|left, right| left.symbol.cmp(&right.symbol));
333        ScipDocument {
334            language: "typst".to_owned(),
335            relative_path: self.relative_path,
336            occurrences,
337            symbols,
338            position_encoding: EnumOrUnknown::new(
339                ScipPositionEncoding::UTF16CodeUnitOffsetFromLineStart,
340            ),
341            ..Default::default()
342        }
343    }
344}
345
346fn lsp_range(ctx: &SharedContext, span: Span, source: &Source) -> Option<Range> {
347    let range = source_range(source, span)?;
348    Some(ctx.to_lsp_range(range, source))
349}
350
351fn scip_range(range: Range) -> Vec<i32> {
352    if range.start.line == range.end.line {
353        vec![
354            range.start.line as i32,
355            range.start.character as i32,
356            range.end.character as i32,
357        ]
358    } else {
359        vec![
360            range.start.line as i32,
361            range.start.character as i32,
362            range.end.line as i32,
363            range.end.character as i32,
364        ]
365    }
366}
367
368pub(crate) fn scip_symbol(
369    ctx: &SharedContext,
370    definition: &Definition,
371) -> tinymist_std::Result<String> {
372    let disambiguator = definition_disambiguator(ctx, definition);
373    scip_symbol_with_disambiguator(definition, disambiguator)
374}
375
376pub(crate) fn scip_symbol_with_disambiguator(
377    definition: &Definition,
378    disambiguator: String,
379) -> tinymist_std::Result<String> {
380    let mut descriptors = Vec::new();
381    if let Some(fid) = definition.file_id() {
382        let path = fid.vpath().get_without_slash();
383        for part in path.split('/') {
384            if !part.is_empty() {
385                descriptors.push(scip_descriptor(part, descriptor::Suffix::Namespace, ""));
386            }
387        }
388    } else {
389        descriptors.push(scip_descriptor(
390            "external",
391            descriptor::Suffix::Namespace,
392            "",
393        ));
394    }
395
396    descriptors.push(scip_descriptor(
397        &disambiguator,
398        descriptor::Suffix::Meta,
399        "",
400    ));
401    descriptors.push(scip_descriptor(
402        definition.name().as_ref(),
403        scip_symbol_suffix(definition.decl.kind()),
404        "",
405    ));
406
407    let symbol = ScipSymbol {
408        scheme: "typst".to_owned(),
409        package: MessageField::some(ScipPackage {
410            manager: "tinymist".to_owned(),
411            name: "workspace".to_owned(),
412            ..Default::default()
413        }),
414        descriptors,
415        ..Default::default()
416    };
417    Ok(scip::symbol::format_symbol(symbol))
418}
419
420fn definition_disambiguator(ctx: &SharedContext, definition: &Definition) -> String {
421    if let Some(fid) = definition.file_id()
422        && let Ok(source) = ctx.source_by_id(fid)
423        && let Some(range) = lsp_range(ctx, definition.decl.span(), &source)
424    {
425        return format!("L{}_C{}", range.start.line, range.start.character);
426    }
427
428    format!("{:x}", definition.decl.span().into_raw())
429}
430
431fn scip_descriptor(name: &str, suffix: descriptor::Suffix, disambiguator: &str) -> ScipDescriptor {
432    ScipDescriptor {
433        name: if name.is_empty() {
434            "_".to_owned()
435        } else {
436            name.to_owned()
437        },
438        disambiguator: disambiguator.to_owned(),
439        suffix: EnumOrUnknown::new(suffix),
440        ..Default::default()
441    }
442}
443
444fn scip_symbol_suffix(kind: DefKind) -> descriptor::Suffix {
445    match kind {
446        DefKind::Function => descriptor::Suffix::Method,
447        DefKind::Module => descriptor::Suffix::Namespace,
448        _ => descriptor::Suffix::Term,
449    }
450}
451
452fn symbol_infos(
453    definition: &Definition,
454    symbol: &str,
455    def_docs: Option<&DefDocs>,
456    docs: Option<String>,
457) -> Vec<ScipSymbolInformation> {
458    if let Some(DefDocs::Function(docs)) = def_docs {
459        return function_symbol_infos(definition, symbol, docs);
460    }
461
462    vec![ScipSymbolInformation {
463        symbol: symbol.to_owned(),
464        documentation: docs.into_iter().collect(),
465        kind: EnumOrUnknown::new(scip_symbol_kind(definition.decl.kind())),
466        display_name: definition.name().to_string(),
467        ..Default::default()
468    }]
469}
470
471fn symbol_info_external(
472    definition: &Definition,
473    symbol: &str,
474    def_docs: Option<&DefDocs>,
475    docs: Option<String>,
476) -> Option<Vec<ScipSymbolInformation>> {
477    if def_docs.is_none() {
478        docs.as_ref()?;
479    }
480
481    Some(symbol_infos(definition, symbol, def_docs, docs))
482}
483
484fn scip_symbol_kind(kind: DefKind) -> symbol_information::Kind {
485    match kind {
486        DefKind::Function => symbol_information::Kind::Function,
487        DefKind::Module => symbol_information::Kind::Module,
488        DefKind::Constant => symbol_information::Kind::Constant,
489        DefKind::Variable => symbol_information::Kind::Variable,
490        _ => symbol_information::Kind::UnspecifiedKind,
491    }
492}
493
494fn public_module_symbol_info(
495    module_symbol: &str,
496    public_symbols: &[String],
497) -> ScipSymbolInformation {
498    ScipSymbolInformation {
499        symbol: module_symbol.to_owned(),
500        relationships: public_symbols
501            .iter()
502            .map(|symbol| ScipRelationship {
503                symbol: symbol.clone(),
504                is_reference: true,
505                is_definition: true,
506                ..Default::default()
507            })
508            .collect(),
509        kind: EnumOrUnknown::new(symbol_information::Kind::Module),
510        ..Default::default()
511    }
512}
513
514fn function_symbol_infos(
515    definition: &Definition,
516    symbol: &str,
517    docs: &SignatureDocs,
518) -> Vec<ScipSymbolInformation> {
519    let signature_text = function_signature_text(definition.name().as_ref(), docs);
520    let params = function_parameters(docs);
521    let mut signature = Signature {
522        language: "typc".to_owned(),
523        text: signature_text.clone(),
524        ..Default::default()
525    };
526    let mut infos = Vec::with_capacity(params.len() + 1);
527
528    for param in params {
529        let Some(param_symbol) =
530            scip_parameter_symbol(symbol, param.group, param.docs.name.as_ref())
531        else {
532            continue;
533        };
534        if let Some(range) =
535            signature_parameter_range(&signature_text, param.docs.name.as_ref(), param.group)
536        {
537            signature.occurrences.push(ScipOccurrence {
538                range,
539                symbol: param_symbol.clone(),
540                symbol_roles: ScipSymbolRole::Definition.value(),
541                ..Default::default()
542            });
543        }
544
545        infos.push(parameter_symbol_info(symbol, &param_symbol, param.docs));
546    }
547
548    infos.insert(
549        0,
550        ScipSymbolInformation {
551            symbol: symbol.to_owned(),
552            documentation: non_empty_docs(docs.docs.trim()),
553            kind: EnumOrUnknown::new(symbol_information::Kind::Function),
554            display_name: definition.name().to_string(),
555            signature_documentation: MessageField::some(signature),
556            ..Default::default()
557        },
558    );
559    infos
560}
561
562fn function_signature_text(name: &str, docs: &SignatureDocs) -> String {
563    let mut text = String::new();
564    text.push_str("let ");
565    text.push_str(name);
566    let _ = docs.print(&mut text);
567    if let Some((short, _, _)) = docs.ret_ty.as_ref()
568        && short != name
569    {
570        let _ = write!(text, " = {short}");
571    }
572    text.push(';');
573    text
574}
575
576struct ScipParamRef<'a> {
577    group: ScipParamGroup,
578    docs: &'a ParamDocs,
579}
580
581fn function_parameters(docs: &SignatureDocs) -> Vec<ScipParamRef<'_>> {
582    let mut params =
583        Vec::with_capacity(docs.pos.len() + docs.named.len() + usize::from(docs.rest.is_some()));
584    params.extend(docs.pos.iter().map(|docs| ScipParamRef {
585        group: ScipParamGroup::Positional,
586        docs,
587    }));
588    if let Some(docs) = docs.rest.as_ref() {
589        params.push(ScipParamRef {
590            group: ScipParamGroup::Rest,
591            docs,
592        });
593    }
594    params.extend(docs.named.values().map(|docs| ScipParamRef {
595        group: ScipParamGroup::Named,
596        docs,
597    }));
598    params
599}
600
601fn scip_parameter_symbol(
602    function_symbol: &str,
603    group: ScipParamGroup,
604    name: &str,
605) -> Option<String> {
606    let mut symbol = scip::symbol::parse_symbol(function_symbol).ok()?;
607    symbol.descriptors.push(scip_descriptor(
608        group.descriptor(),
609        descriptor::Suffix::Meta,
610        "",
611    ));
612    symbol
613        .descriptors
614        .push(scip_descriptor(name, descriptor::Suffix::Parameter, ""));
615    Some(scip::symbol::format_symbol(symbol))
616}
617
618fn signature_parameter_range(
619    signature: &str,
620    name: &str,
621    group: ScipParamGroup,
622) -> Option<Vec<i32>> {
623    let mut line_offset = 0usize;
624    for line in signature.lines() {
625        let trimmed = line.trim_start();
626        let candidate = match group {
627            ScipParamGroup::Rest => {
628                let Some(candidate) = trimmed.strip_prefix("..") else {
629                    line_offset += 1;
630                    continue;
631                };
632                candidate
633            }
634            _ => trimmed,
635        };
636
637        let matches = candidate.strip_prefix(name).is_some_and(|rest| {
638            matches!(
639                rest.chars().next(),
640                Some(':') | Some(',') | Some(' ') | Some('=') | None
641            )
642        });
643        if matches {
644            let leading_chars = line.chars().count() - trimmed.chars().count();
645            let prefix_chars = if group == ScipParamGroup::Rest { 2 } else { 0 };
646            let start = leading_chars + prefix_chars;
647            let end = start + name.chars().count();
648            return Some(vec![line_offset as i32, start as i32, end as i32]);
649        }
650
651        line_offset += 1;
652    }
653
654    None
655}
656
657fn parameter_symbol_info(
658    function_symbol: &str,
659    symbol: &str,
660    docs: &ParamDocs,
661) -> ScipSymbolInformation {
662    ScipSymbolInformation {
663        symbol: symbol.to_owned(),
664        documentation: parameter_documentation(docs),
665        kind: EnumOrUnknown::new(symbol_information::Kind::Parameter),
666        display_name: docs.name.to_string(),
667        enclosing_symbol: function_symbol.to_owned(),
668        ..Default::default()
669    }
670}
671
672fn parameter_documentation(docs: &ParamDocs) -> Vec<String> {
673    let mut output = Vec::new();
674    if let Some((_, _, ty)) = docs.cano_type.as_ref()
675        && !ty.trim().is_empty()
676    {
677        output.push(format!("```typc\ntype: {}\n```", ty.trim()));
678    }
679    output.extend(non_empty_docs(docs.docs.trim()));
680    output
681}
682
683fn non_empty_docs(docs: &str) -> Vec<String> {
684    if docs.trim().is_empty() {
685        Vec::new()
686    } else {
687        vec![docs.trim().to_owned()]
688    }
689}
690
691fn hover_to_markdown(hover: &Hover) -> Option<String> {
692    hover_contents_to_markdown(&hover.contents)
693}
694
695fn hover_contents_to_markdown(contents: &HoverContents) -> Option<String> {
696    match contents {
697        HoverContents::Scalar(marked) => marked_string_to_markdown(marked),
698        HoverContents::Array(parts) => {
699            let parts = parts
700                .iter()
701                .filter_map(marked_string_to_markdown)
702                .filter(|part| !part.trim().is_empty())
703                .collect::<Vec<_>>();
704            if parts.is_empty() {
705                None
706            } else {
707                Some(parts.join("\n\n---\n\n"))
708            }
709        }
710        HoverContents::Markup(markup) => {
711            if markup.value.trim().is_empty() {
712                None
713            } else {
714                Some(markup.value.clone())
715            }
716        }
717    }
718}
719
720fn marked_string_to_markdown(marked: &MarkedString) -> Option<String> {
721    match marked {
722        MarkedString::String(value) => {
723            if value.trim().is_empty() {
724                None
725            } else {
726                Some(value.clone())
727            }
728        }
729        MarkedString::LanguageString(value) => {
730            if value.value.trim().is_empty() {
731                None
732            } else {
733                Some(format!("```{}\n{}\n```", value.language, value.value))
734            }
735        }
736    }
737}
tinymist_query/index/scip.rs

tinymist_query/index/
scip.rs
