tinymist_query/docs/
package.rs

1use core::fmt::Write;
2use std::collections::{HashMap, HashSet};
3use std::path::PathBuf;
4
5use ecow::{EcoString, EcoVec};
6use indexmap::IndexSet;
7use serde::{Deserialize, Serialize};
8use tinymist_analysis::docs::tidy::remove_list_annotations;
9use tinymist_world::package::PackageSpec;
10use typst::diag::{StrResult, eco_format};
11use typst::syntax::package::PackageManifest;
12use typst::syntax::{FileId, Span};
13
14use crate::LocalContext;
15use crate::docs::{DefDocs, PackageDefInfo, file_id_repr, module_docs};
16use crate::package::{PackageInfo, get_manifest_id};
17
18/// Documentation Information about a package.
19#[derive(Debug, Clone, Serialize, Deserialize)]
20pub struct PackageDoc {
21    meta: PackageMeta,
22    packages: Vec<PackageMeta>,
23    files: Vec<FileMeta>,
24    modules: Vec<(EcoString, crate::docs::DefInfo, ModuleInfo)>,
25}
26
27/// Documentation Information about a package module.
28#[derive(Debug, Clone, Serialize, Deserialize)]
29struct ModuleInfo {
30    prefix: EcoString,
31    name: EcoString,
32    loc: Option<usize>,
33    parent_ident: EcoString,
34    aka: EcoVec<String>,
35}
36
37/// Generate full documents in markdown format
38pub fn package_docs(ctx: &mut LocalContext, spec: &PackageInfo) -> StrResult<PackageDoc> {
39    log::info!("generate_md_docs {spec:?}");
40
41    let toml_id = get_manifest_id(spec)?;
42    let manifest = ctx.get_manifest(toml_id)?;
43
44    let for_spec = toml_id.package().unwrap();
45    let entry_point = toml_id.join(&manifest.package.entrypoint);
46
47    ctx.preload_package(entry_point);
48
49    let PackageDefInfo { root, module_uses } = module_docs(ctx, entry_point)?;
50
51    crate::log_debug_ct!("module_uses: {module_uses:#?}");
52
53    let manifest = ctx.get_manifest(toml_id)?;
54
55    let meta = PackageMeta {
56        namespace: spec.namespace.clone(),
57        name: spec.name.clone(),
58        version: spec.version.to_string(),
59        manifest: Some(manifest),
60    };
61
62    let mut modules_to_generate = vec![(root.name.clone(), root)];
63    let mut generated_modules = HashSet::new();
64    let mut file_ids: IndexSet<FileId> = IndexSet::new();
65
66    // let aka = module_uses[&file_id_repr(fid.unwrap())].clone();
67    // let primary = &aka[0];
68    let mut primary_aka_cache = HashMap::<FileId, EcoVec<String>>::new();
69    let mut akas = |fid: FileId| {
70        primary_aka_cache
71            .entry(fid)
72            .or_insert_with(|| {
73                module_uses
74                    .get(&file_id_repr(fid))
75                    .unwrap_or_else(|| panic!("no module uses for {}", file_id_repr(fid)))
76                    .clone()
77            })
78            .clone()
79    };
80
81    let mut modules = vec![];
82
83    while !modules_to_generate.is_empty() {
84        for (parent_ident, mut def) in std::mem::take(&mut modules_to_generate) {
85            // parent_ident, symbols
86
87            let module_val = def.decl.as_ref().unwrap();
88            let fid = module_val.file_id();
89            let aka = fid.map(&mut akas).unwrap_or_default();
90
91            // It is (primary) known to safe as a part of HTML string, so we don't have to
92            // do sanitization here.
93            let primary = aka.first().cloned().unwrap_or_default();
94
95            let persist_fid = fid.map(|fid| file_ids.insert_full(fid).0);
96
97            let module_info = ModuleInfo {
98                prefix: primary.as_str().into(),
99                name: def.name.clone(),
100                loc: persist_fid,
101                parent_ident: parent_ident.clone(),
102                aka,
103            };
104
105            for child in def.children.iter_mut() {
106                let span = child.decl.as_ref().map(|decl| decl.span());
107                let fid_range = span.and_then(|v| {
108                    v.id().and_then(|fid| {
109                        let allocated = file_ids.insert_full(fid).0;
110                        let src = ctx.source_by_id(fid).ok()?;
111                        let rng = src.range(v)?;
112                        Some((allocated, rng.start, rng.end))
113                    })
114                });
115                let child_fid = child.decl.as_ref().and_then(|decl| decl.file_id());
116                let child_fid = child_fid.or_else(|| span.and_then(Span::id)).or(fid);
117                let span = fid_range.or_else(|| {
118                    let fid = child_fid?;
119                    Some((file_ids.insert_full(fid).0, 0, 0))
120                });
121                child.loc = span;
122
123                if child.parsed_docs.is_some() {
124                    child.docs = None;
125                }
126
127                let ident = if !primary.is_empty() {
128                    eco_format!("symbol-{}-{primary}.{}", child.kind, child.name)
129                } else {
130                    eco_format!("symbol-{}-{}", child.kind, child.name)
131                };
132
133                if child.is_external
134                    && let Some(fid) = child_fid
135                {
136                    let lnk = if fid.package() == Some(for_spec) {
137                        let sub_aka = akas(fid);
138                        let sub_primary = sub_aka.first().cloned().unwrap_or_default();
139                        child.external_link = Some(format!(
140                            "#symbol-{}-{sub_primary}.{}",
141                            child.kind, child.name
142                        ));
143                        format!("#{}-{}-in-{sub_primary}", child.kind, child.name).replace(".", "")
144                    } else if let Some(spec) = fid.package() {
145                        let lnk = format!(
146                            "https://typst.app/universe/package/{}/{}",
147                            spec.name, spec.version
148                        );
149                        child.external_link = Some(lnk.clone());
150                        lnk
151                    } else {
152                        let lnk: String = "https://typst.app/docs".into();
153                        child.external_link = Some(lnk.clone());
154                        lnk
155                    };
156                    child.symbol_link = Some(lnk);
157                }
158
159                let child_children = std::mem::take(&mut child.children);
160                if !child_children.is_empty() {
161                    crate::log_debug_ct!("sub_fid: {child_fid:?}");
162                    let lnk = match child_fid {
163                        Some(fid) => {
164                            let aka = akas(fid);
165                            let primary = aka.first().cloned().unwrap_or_default();
166
167                            if generated_modules.insert(fid) {
168                                let mut child = child.clone();
169                                child.children = child_children;
170                                modules_to_generate.push((ident.clone(), child));
171                            }
172
173                            let link = format!("module-{primary}").replace(".", "");
174                            format!("#{link}")
175                        }
176                        None => "builtin".to_owned(),
177                    };
178
179                    child.module_link = Some(lnk);
180                }
181
182                child.id = ident;
183            }
184
185            modules.push((parent_ident, def, module_info));
186        }
187    }
188
189    let mut packages = IndexSet::new();
190
191    let files = file_ids
192        .into_iter()
193        .map(|fid| {
194            let pkg = fid
195                .package()
196                .map(|spec| packages.insert_full(spec.clone()).0);
197
198            FileMeta {
199                package: pkg,
200                path: fid.vpath().as_rootless_path().to_owned(),
201            }
202        })
203        .collect();
204
205    let packages = packages
206        .into_iter()
207        .map(|spec| PackageMeta {
208            namespace: spec.namespace.clone(),
209            name: spec.name.clone(),
210            version: spec.version.to_string(),
211            manifest: None,
212        })
213        .collect();
214
215    let doc = PackageDoc {
216        meta,
217        packages,
218        files,
219        modules,
220    };
221
222    Ok(doc)
223}
224
225/// Generate full documents in markdown format
226pub fn package_docs_typ(doc: &PackageDoc) -> StrResult<String> {
227    let mut out = String::new();
228
229    let _ = writeln!(out, "{}", include_str!("package-doc.typ"));
230
231    let pi = &doc.meta;
232    let _ = writeln!(
233        out,
234        "#package-doc(bytes(read(\"{}-{}-{}.json\")))",
235        pi.namespace, pi.name, pi.version,
236    );
237
238    Ok(out)
239}
240
241/// Generate full documents in markdown format
242pub fn package_docs_md(doc: &PackageDoc) -> StrResult<String> {
243    let mut out = String::new();
244
245    let title = doc.meta.spec().to_string();
246
247    writeln!(out, "# {title}").unwrap();
248    out.push('\n');
249    writeln!(out, "This documentation is generated locally. Please submit issues to [tinymist](https://github.com/Myriad-Dreamin/tinymist/issues) if you see **incorrect** information in it.").unwrap();
250    out.push('\n');
251    out.push('\n');
252
253    let package_meta = jbase64(&doc.meta);
254    let _ = writeln!(out, "<!-- begin:package {package_meta} -->");
255
256    let mut errors = vec![];
257    for (parent_ident, def, module_info) in &doc.modules {
258        // parent_ident, symbols
259        let primary = &module_info.prefix;
260        if !module_info.prefix.is_empty() {
261            let _ = writeln!(out, "---\n## Module: {primary}");
262        }
263
264        crate::log_debug_ct!("module: {primary} -- {parent_ident}");
265        let module_info = jbase64(&module_info);
266        let _ = writeln!(out, "<!-- begin:module {primary} {module_info} -->");
267
268        for child in &def.children {
269            let convert_err = None::<EcoString>;
270
271            let ident = if !primary.is_empty() {
272                eco_format!("symbol-{}-{primary}.{}", child.kind, child.name)
273            } else {
274                eco_format!("symbol-{}-{}", child.kind, child.name)
275            };
276            let _ = writeln!(out, "### {}: {} in {primary}", child.kind, child.name);
277
278            if let Some(lnk) = &child.symbol_link {
279                let _ = writeln!(out, "[Symbol Docs]({lnk})\n");
280            }
281
282            let head = jbase64(&child);
283            let _ = writeln!(out, "<!-- begin:symbol {ident} {head} -->");
284
285            if let Some(DefDocs::Function(sig)) = &child.parsed_docs {
286                let _ = writeln!(out, "<!-- begin:sig -->");
287                let _ = writeln!(out, "```typc");
288                let _ = write!(out, "let {}", child.name);
289                let _ = sig.print(&mut out);
290                let _ = writeln!(out, ";");
291                let _ = writeln!(out, "```");
292                let _ = writeln!(out, "<!-- end:sig -->");
293            }
294
295            let mut printed_docs = false;
296            match (&child.parsed_docs, convert_err) {
297                (_, Some(err)) => {
298                    let err = format!("failed to convert docs in {title}: {err}").replace(
299                        "-->", "—>", // avoid markdown comment
300                    );
301                    let _ = writeln!(out, "<!-- convert-error: {err} -->");
302                    errors.push(err);
303                }
304                (Some(docs), _) if !child.is_external => {
305                    let _ = writeln!(out, "{}", remove_list_annotations(docs.docs()));
306                    printed_docs = true;
307                    if let DefDocs::Function(docs) = docs {
308                        for param in docs
309                            .pos
310                            .iter()
311                            .chain(docs.named.values())
312                            .chain(docs.rest.as_ref())
313                        {
314                            let _ = writeln!(out, "<!-- begin:param {} -->", param.name);
315                            let ty = match &param.cano_type {
316                                Some((short, _, _)) => short,
317                                None => "unknown",
318                            };
319                            let _ = writeln!(
320                                out,
321                                "#### {} ({ty:?})\n<!-- begin:param-doc {} -->\n{}\n<!-- end:param-doc {} -->",
322                                param.name, param.name, param.docs, param.name
323                            );
324                            let _ = writeln!(out, "<!-- end:param -->");
325                        }
326                    }
327                }
328                (_, None) => {}
329            }
330
331            if !printed_docs {
332                let plain_docs = child.docs.as_deref();
333                let plain_docs = plain_docs.or(child.oneliner.as_deref());
334
335                if let Some(docs) = plain_docs {
336                    let contains_code = docs.contains("```");
337                    if contains_code {
338                        let _ = writeln!(out, "`````typ");
339                    }
340                    let _ = writeln!(out, "{docs}");
341                    if contains_code {
342                        let _ = writeln!(out, "`````");
343                    }
344                }
345            }
346
347            if let Some(lnk) = &child.module_link {
348                match lnk.as_str() {
349                    "builtin" => {
350                        let _ = writeln!(out, "A Builtin Module");
351                    }
352                    lnk => {
353                        let _ = writeln!(out, "[Module Docs]({lnk})\n");
354                    }
355                }
356            }
357
358            let _ = writeln!(out, "<!-- end:symbol {ident} -->");
359        }
360
361        let _ = writeln!(out, "<!-- end:module {primary} -->");
362    }
363
364    let res = ConvertResult { errors };
365    let err = jbase64(&res);
366    let _ = writeln!(out, "<!-- begin:errors {err} -->");
367    let _ = writeln!(out, "## Errors");
368    for errs in res.errors {
369        let _ = writeln!(out, "- {errs}");
370    }
371    let _ = writeln!(out, "<!-- end:errors -->");
372
373    let meta = PackageMetaEnd {
374        packages: doc.packages.clone(),
375        files: doc.files.clone(),
376    };
377    let package_meta = jbase64(&meta);
378    let _ = writeln!(out, "<!-- end:package {package_meta} -->");
379
380    Ok(out)
381}
382
383fn jbase64<T: Serialize>(s: &T) -> String {
384    use base64::Engine;
385    let content = serde_json::to_string(s).unwrap();
386    base64::engine::general_purpose::STANDARD.encode(content)
387}
388
389/// Information about a package.
390#[derive(Debug, Clone, Serialize, Deserialize)]
391pub struct PackageMeta {
392    /// The namespace the package lives in.
393    pub namespace: EcoString,
394    /// The name of the package within its namespace.
395    pub name: EcoString,
396    /// The package's version.
397    pub version: String,
398    /// The package's manifest information.
399    pub manifest: Option<PackageManifest>,
400}
401
402impl PackageMeta {
403    /// Returns the package's full name, including namespace and version.
404    pub fn spec(&self) -> PackageSpec {
405        PackageSpec {
406            namespace: self.namespace.clone(),
407            name: self.name.clone(),
408            version: self.version.parse().expect("Invalid version format"),
409        }
410    }
411}
412
413/// Information about a package.
414#[derive(Debug, Serialize, Deserialize)]
415pub struct PackageMetaEnd {
416    packages: Vec<PackageMeta>,
417    files: Vec<FileMeta>,
418}
419
420/// Information about a package.
421#[derive(Debug, Clone, Serialize, Deserialize)]
422pub struct FileMeta {
423    package: Option<usize>,
424    path: PathBuf,
425}
426
427#[derive(Serialize, Deserialize)]
428struct ConvertResult {
429    errors: Vec<String>,
430}
431
432#[cfg(test)]
433mod tests {
434    use tinymist_world::package::{PackageRegistry, PackageSpec};
435
436    use super::{PackageInfo, package_docs, package_docs_md, package_docs_typ};
437    use crate::tests::*;
438
439    fn test(pkg: PackageSpec) {
440        run_with_sources("", |verse: &mut LspUniverse, path| {
441            let pkg_root = verse.registry.resolve(&pkg).unwrap();
442            let pi = PackageInfo {
443                path: pkg_root.as_ref().to_owned(),
444                namespace: pkg.namespace,
445                name: pkg.name,
446                version: pkg.version.to_string(),
447            };
448            run_with_ctx(verse, path, &|a, _p| {
449                let docs = package_docs(a, &pi).unwrap();
450                let dest = format!(
451                    "../../target/{}-{}-{}.json",
452                    pi.namespace, pi.name, pi.version
453                );
454                std::fs::write(dest, serde_json::to_string_pretty(&docs).unwrap()).unwrap();
455                let typ = package_docs_typ(&docs).unwrap();
456                let dest = format!(
457                    "../../target/{}-{}-{}.typ",
458                    pi.namespace, pi.name, pi.version
459                );
460                std::fs::write(dest, typ).unwrap();
461                let md = package_docs_md(&docs).unwrap();
462                let dest = format!(
463                    "../../target/{}-{}-{}.md",
464                    pi.namespace, pi.name, pi.version
465                );
466                std::fs::write(dest, md).unwrap();
467            })
468        })
469    }
470
471    #[test]
472    fn tidy() {
473        test(PackageSpec {
474            namespace: "preview".into(),
475            name: "tidy".into(),
476            version: "0.3.0".parse().unwrap(),
477        });
478    }
479
480    #[test]
481    fn touying() {
482        test(PackageSpec {
483            namespace: "preview".into(),
484            name: "touying".into(),
485            version: "0.6.0".parse().unwrap(),
486        });
487    }
488
489    #[test]
490    fn fletcher() {
491        test(PackageSpec {
492            namespace: "preview".into(),
493            name: "fletcher".into(),
494            version: "0.5.8".parse().unwrap(),
495        });
496    }
497
498    #[test]
499    fn cetz() {
500        test(PackageSpec {
501            namespace: "preview".into(),
502            name: "cetz".into(),
503            version: "0.2.2".parse().unwrap(),
504        });
505    }
506}
tinymist_query/docs/package.rs

tinymist_query/docs/
package.rs
