tinymist_project/
compiler.rs

1//! Project compiler for tinymist.
2
3use core::fmt;
4use std::collections::HashSet;
5use std::path::Path;
6use std::sync::{Arc, OnceLock};
7
8use ecow::{EcoString, EcoVec, eco_vec};
9use tinymist_std::error::prelude::Result;
10use tinymist_std::{ImmutPath, typst::TypstDocument};
11use tinymist_task::ExportTarget;
12use tinymist_world::vfs::notify::{
13    FilesystemEvent, MemoryEvent, NotifyDeps, NotifyMessage, UpstreamUpdateEvent,
14};
15use tinymist_world::vfs::{FileId, FsProvider, RevisingVfs, WorkspaceResolver};
16use tinymist_world::{
17    CompileSignal, CompileSnapshot, CompilerFeat, CompilerUniverse, DiagnosticsTask, EntryReader,
18    EntryState, FlagTask, HtmlCompilationTask, PagedCompilationTask, ProjectInsId, TaskInputs,
19    WorldComputeGraph, WorldDeps,
20};
21use tokio::sync::mpsc;
22use typst::World;
23use typst::diag::{At, FileError};
24use typst::syntax::Span;
25
26/// A compiled artifact.
27pub struct CompiledArtifact<F: CompilerFeat> {
28    /// The used compute graph.
29    pub graph: Arc<WorldComputeGraph<F>>,
30    /// The diagnostics of the document.
31    pub diag: Arc<DiagnosticsTask>,
32    /// The compiled document.
33    pub doc: Option<TypstDocument>,
34    /// The depended files.
35    pub deps: OnceLock<EcoVec<FileId>>,
36}
37
38impl<F: CompilerFeat> fmt::Display for CompiledArtifact<F> {
39    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
40        let rev = self.graph.snap.world.revision();
41        write!(f, "CompiledArtifact({:?}, rev={rev:?})", self.graph.snap.id)
42    }
43}
44
45impl<F: CompilerFeat> std::ops::Deref for CompiledArtifact<F> {
46    type Target = Arc<WorldComputeGraph<F>>;
47
48    fn deref(&self) -> &Self::Target {
49        &self.graph
50    }
51}
52
53impl<F: CompilerFeat> Clone for CompiledArtifact<F> {
54    fn clone(&self) -> Self {
55        Self {
56            graph: self.graph.clone(),
57            doc: self.doc.clone(),
58            diag: self.diag.clone(),
59            deps: self.deps.clone(),
60        }
61    }
62}
63
64impl<F: CompilerFeat> CompiledArtifact<F> {
65    /// Returns the project id.
66    pub fn id(&self) -> &ProjectInsId {
67        &self.graph.snap.id
68    }
69
70    /// Returns the last successfully compiled document.
71    pub fn success_doc(&self) -> Option<TypstDocument> {
72        self.doc
73            .as_ref()
74            .cloned()
75            .or_else(|| self.snap.success_doc.clone())
76    }
77
78    /// Returns the depended files.
79    pub fn depended_files(&self) -> &EcoVec<FileId> {
80        self.deps.get_or_init(|| {
81            let mut deps = EcoVec::default();
82            self.graph.snap.world.iter_dependencies(&mut |f| {
83                deps.push(f);
84            });
85
86            deps
87        })
88    }
89
90    /// Runs the compiler and returns the compiled document.
91    pub fn from_graph(graph: Arc<WorldComputeGraph<F>>, is_html: bool) -> CompiledArtifact<F> {
92        let _ = graph.provide::<FlagTask<HtmlCompilationTask>>(Ok(FlagTask::flag(is_html)));
93        let _ = graph.provide::<FlagTask<PagedCompilationTask>>(Ok(FlagTask::flag(!is_html)));
94        let doc = if is_html {
95            graph.shared_compile_html().expect("html").map(From::from)
96        } else {
97            graph.shared_compile().expect("paged").map(From::from)
98        };
99
100        CompiledArtifact {
101            diag: graph.shared_diagnostics().expect("diag"),
102            graph,
103            doc,
104            deps: OnceLock::default(),
105        }
106    }
107
108    /// Returns the error count.
109    pub fn error_cnt(&self) -> usize {
110        self.diag.error_cnt()
111    }
112
113    /// Returns the warning count.
114    pub fn warning_cnt(&self) -> usize {
115        self.diag.warning_cnt()
116    }
117
118    /// Returns the diagnostics.
119    pub fn diagnostics(&self) -> impl Iterator<Item = &typst::diag::SourceDiagnostic> + Clone {
120        self.diag.diagnostics()
121    }
122
123    /// Returns whether there are any errors.
124    pub fn has_errors(&self) -> bool {
125        self.error_cnt() > 0
126    }
127
128    /// Sets the signal.
129    pub fn with_signal(mut self, signal: CompileSignal) -> Self {
130        let mut snap = self.snap.clone();
131        snap.signal = signal;
132
133        self.graph = self.graph.snapshot_unsafe(snap);
134        self
135    }
136}
137
138/// The compilation status of a project.
139#[derive(Debug, Clone)]
140pub struct CompileReport {
141    /// The project ID.
142    pub id: ProjectInsId,
143    /// The file getting compiled.
144    pub compiling_id: Option<FileId>,
145    /// The number of pages in the compiled document, zero if failed.
146    pub page_count: u32,
147    /// The status of the compilation.
148    pub status: CompileStatusEnum,
149}
150
151/// The compilation status of a project.
152#[derive(Debug, Clone)]
153pub enum CompileStatusEnum {
154    /// The project is suspended.
155    Suspend,
156    /// The project is compiling.
157    Compiling,
158    /// The project compiled successfully.
159    CompileSuccess(CompileStatusResult),
160    /// The project failed to compile.
161    CompileError(CompileStatusResult),
162    /// The project failed to export.
163    ExportError(CompileStatusResult),
164}
165
166/// The compilation status result of a project.
167#[derive(Debug, Clone)]
168pub struct CompileStatusResult {
169    /// The number of errors or warnings occur.
170    diag: u32,
171    /// Used time
172    elapsed: tinymist_std::time::Duration,
173}
174
175impl CompileReport {
176    /// Gets the status message.
177    pub fn message(&self) -> CompileReportMsg<'_> {
178        CompileReportMsg(self)
179    }
180}
181
182/// A message of the compilation status.
183pub struct CompileReportMsg<'a>(&'a CompileReport);
184
185impl fmt::Display for CompileReportMsg<'_> {
186    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
187        use CompileStatusEnum::*;
188        use CompileStatusResult as Res;
189
190        let input = WorkspaceResolver::display(self.0.compiling_id);
191        let (stage, Res { diag, elapsed }) = match &self.0.status {
192            Suspend => return f.write_str("suspended"),
193            Compiling => return f.write_str("compiling"),
194            CompileSuccess(Res { diag: 0, elapsed }) => {
195                return write!(f, "{input:?}: compilation succeeded in {elapsed:?}");
196            }
197            CompileSuccess(res) => ("compilation succeeded", res),
198            CompileError(res) => ("compilation failed", res),
199            ExportError(res) => ("export failed", res),
200        };
201        write!(
202            f,
203            "{input:?}: {stage} with {diag} warnings and errors in {elapsed:?}"
204        )
205    }
206}
207
208/// A project compiler handler.
209pub trait CompileHandler<F: CompilerFeat, Ext>: Send + Sync + 'static {
210    /// Called when there is any reason to compile. This doesn't mean that the
211    /// project should be compiled.
212    fn on_any_compile_reason(&self, state: &mut ProjectCompiler<F, Ext>);
213    // todo: notify project specific compile
214    /// Called when a compilation is done.
215    fn notify_compile(&self, res: &CompiledArtifact<F>);
216    /// Called when a project is removed.
217    fn notify_removed(&self, _id: &ProjectInsId) {}
218    /// Called when the compilation status is changed.
219    fn status(&self, revision: usize, rep: CompileReport);
220}
221
222/// No need so no compilation.
223impl<F: CompilerFeat + Send + Sync + 'static, Ext: 'static> CompileHandler<F, Ext>
224    for std::marker::PhantomData<fn(F, Ext)>
225{
226    fn on_any_compile_reason(&self, _state: &mut ProjectCompiler<F, Ext>) {
227        log::info!("ProjectHandle: no need to compile");
228    }
229    fn notify_compile(&self, _res: &CompiledArtifact<F>) {}
230    fn status(&self, _revision: usize, _rep: CompileReport) {}
231}
232
233/// An interrupt to the compiler.
234pub enum Interrupt<F: CompilerFeat> {
235    /// Compile anyway.
236    Compile(ProjectInsId),
237    /// Settle a dedicated project.
238    Settle(ProjectInsId),
239    /// Compiled from computing thread.
240    Compiled(CompiledArtifact<F>),
241    /// Change the watching entry.
242    ChangeTask(ProjectInsId, TaskInputs),
243    /// Font changes.
244    Font(Arc<F::FontResolver>),
245    /// Creation timestamp changes.
246    CreationTimestamp(Option<i64>),
247    /// Memory file changes.
248    Memory(MemoryEvent),
249    /// File system event.
250    Fs(FilesystemEvent),
251    /// Save a file.
252    Save(ImmutPath),
253}
254
255impl<F: CompilerFeat> fmt::Debug for Interrupt<F> {
256    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
257        match self {
258            Interrupt::Compile(id) => write!(f, "Compile({id:?})"),
259            Interrupt::Settle(id) => write!(f, "Settle({id:?})"),
260            Interrupt::Compiled(artifact) => write!(f, "Compiled({:?})", artifact.id()),
261            Interrupt::ChangeTask(id, change) => {
262                write!(f, "ChangeTask({id:?}, entry={:?})", change.entry.is_some())
263            }
264            Interrupt::Font(..) => write!(f, "Font(..)"),
265            Interrupt::CreationTimestamp(ts) => write!(f, "CreationTimestamp({ts:?})"),
266            Interrupt::Memory(..) => write!(f, "Memory(..)"),
267            Interrupt::Fs(..) => write!(f, "Fs(..)"),
268            Interrupt::Save(path) => write!(f, "Save({path:?})"),
269        }
270    }
271}
272
273fn no_reason() -> CompileSignal {
274    CompileSignal::default()
275}
276
277fn reason_by_mem() -> CompileSignal {
278    CompileSignal {
279        by_mem_events: true,
280        ..CompileSignal::default()
281    }
282}
283
284fn reason_by_fs() -> CompileSignal {
285    CompileSignal {
286        by_fs_events: true,
287        ..CompileSignal::default()
288    }
289}
290
291fn reason_by_entry_change() -> CompileSignal {
292    CompileSignal {
293        by_entry_update: true,
294        ..CompileSignal::default()
295    }
296}
297
298/// A tagged memory event with logical tick.
299struct TaggedMemoryEvent {
300    /// The logical tick when the event is received.
301    logical_tick: usize,
302    /// The memory event happened.
303    event: MemoryEvent,
304}
305
306/// The compiler server options.
307pub struct CompileServerOpts<F: CompilerFeat, Ext> {
308    /// The compilation handler.
309    pub handler: Arc<dyn CompileHandler<F, Ext>>,
310    /// Whether to ignoring the first fs sync event.
311    pub ignore_first_sync: bool,
312    /// Specifies the current export target.
313    pub export_target: ExportTarget,
314    /// Whether to run in syntax-only mode.
315    pub syntax_only: bool,
316}
317
318impl<F: CompilerFeat + Send + Sync + 'static, Ext: 'static> Default for CompileServerOpts<F, Ext> {
319    fn default() -> Self {
320        Self {
321            handler: Arc::new(std::marker::PhantomData),
322            ignore_first_sync: false,
323            syntax_only: false,
324            export_target: ExportTarget::Paged,
325        }
326    }
327}
328
329const FILE_MISSING_ERROR_MSG: EcoString = EcoString::inline("t-file-missing");
330/// The file missing error constant.
331pub const FILE_MISSING_ERROR: FileError = FileError::Other(Some(FILE_MISSING_ERROR_MSG));
332
333/// The synchronous compiler that runs on one project or multiple projects.
334pub struct ProjectCompiler<F: CompilerFeat, Ext> {
335    /// The compilation handle.
336    pub handler: Arc<dyn CompileHandler<F, Ext>>,
337    /// Specifies the current export target.
338    export_target: ExportTarget,
339    /// Whether to run in syntax-only mode.
340    syntax_only: bool,
341    /// Channel for sending interrupts to the compiler actor.
342    dep_tx: mpsc::UnboundedSender<NotifyMessage>,
343    /// Whether to ignore the first sync event.
344    pub ignore_first_sync: bool,
345
346    /// The current logical tick.
347    logical_tick: usize,
348    /// Last logical tick when invalidation is caused by shadow update.
349    dirty_shadow_logical_tick: usize,
350    /// Estimated latest set of shadow files.
351    estimated_shadow_files: HashSet<Arc<Path>>,
352
353    /// The primary state.
354    pub primary: ProjectInsState<F, Ext>,
355    /// The states for dedicate tasks
356    pub dedicates: Vec<ProjectInsState<F, Ext>>,
357    /// The project file dependencies.
358    deps: ProjectDeps,
359}
360
361impl<F: CompilerFeat + Send + Sync + 'static, Ext: Default + 'static> ProjectCompiler<F, Ext> {
362    /// Creates a compiler with options
363    pub fn new(
364        verse: CompilerUniverse<F>,
365        dep_tx: mpsc::UnboundedSender<NotifyMessage>,
366        CompileServerOpts {
367            handler,
368            ignore_first_sync,
369            export_target,
370            syntax_only,
371        }: CompileServerOpts<F, Ext>,
372    ) -> Self {
373        let primary = Self::create_project(
374            ProjectInsId("primary".into()),
375            verse,
376            export_target,
377            syntax_only,
378            handler.clone(),
379        );
380        Self {
381            handler,
382            dep_tx,
383            export_target,
384            syntax_only,
385
386            logical_tick: 1,
387            dirty_shadow_logical_tick: 0,
388
389            estimated_shadow_files: Default::default(),
390            ignore_first_sync,
391
392            primary,
393            deps: Default::default(),
394            dedicates: vec![],
395        }
396    }
397
398    /// Creates a snapshot of the primary project.
399    pub fn snapshot(&mut self) -> Arc<WorldComputeGraph<F>> {
400        self.primary.snapshot()
401    }
402
403    /// Compiles the document once.
404    pub fn compile_once(&mut self) -> CompiledArtifact<F> {
405        let snap = self.primary.make_snapshot();
406        ProjectInsState::run_compile(
407            self.handler.clone(),
408            snap,
409            self.export_target,
410            self.syntax_only,
411        )()
412    }
413
414    /// Gets the iterator of all projects.
415    pub fn projects(&mut self) -> impl Iterator<Item = &mut ProjectInsState<F, Ext>> {
416        std::iter::once(&mut self.primary).chain(self.dedicates.iter_mut())
417    }
418
419    fn create_project(
420        id: ProjectInsId,
421        verse: CompilerUniverse<F>,
422        export_target: ExportTarget,
423        syntax_only: bool,
424        handler: Arc<dyn CompileHandler<F, Ext>>,
425    ) -> ProjectInsState<F, Ext> {
426        ProjectInsState {
427            id,
428            ext: Default::default(),
429            syntax_only,
430            verse,
431            reason: no_reason(),
432            cached_snapshot: None,
433            handler,
434            export_target,
435            latest_compilation: OnceLock::default(),
436            latest_success_doc: None,
437            deps: Default::default(),
438            committed_revision: 0,
439        }
440    }
441
442    /// Find a project by id, but with less borrow checker restriction.
443    pub fn find_project<'a>(
444        primary: &'a mut ProjectInsState<F, Ext>,
445        dedicates: &'a mut [ProjectInsState<F, Ext>],
446        id: &ProjectInsId,
447    ) -> &'a mut ProjectInsState<F, Ext> {
448        if id == &primary.id {
449            return primary;
450        }
451
452        dedicates.iter_mut().find(|e| e.id == *id).unwrap()
453    }
454
455    /// Clear all dedicate projects.
456    pub fn clear_dedicates(&mut self) {
457        self.dedicates.clear();
458    }
459
460    /// Restart a dedicate project.
461    pub fn restart_dedicate(&mut self, group: &str, entry: EntryState) -> Result<ProjectInsId> {
462        let id = ProjectInsId(group.into());
463
464        let verse = CompilerUniverse::<F>::new_raw(
465            entry,
466            self.primary.verse.features.clone(),
467            Some(self.primary.verse.inputs().clone()),
468            self.primary.verse.vfs().fork(),
469            self.primary.verse.registry.clone(),
470            self.primary.verse.font_resolver.clone(),
471            self.primary.verse.creation_timestamp,
472        );
473
474        let mut proj = Self::create_project(
475            id.clone(),
476            verse,
477            self.export_target,
478            self.syntax_only,
479            self.handler.clone(),
480        );
481        proj.reason.merge(reason_by_entry_change());
482
483        self.remove_dedicates(&id);
484        self.dedicates.push(proj);
485
486        Ok(id)
487    }
488
489    fn remove_dedicates(&mut self, id: &ProjectInsId) {
490        let proj = self.dedicates.iter().position(|e| e.id == *id);
491        if let Some(idx) = proj {
492            // Resets the handle state, e.g. notified revision
493            self.handler.notify_removed(id);
494            self.deps.project_deps.remove_mut(id);
495
496            let _proj = self.dedicates.remove(idx);
497            // todo: kill compilations
498
499            let res = self
500                .dep_tx
501                .send(NotifyMessage::SyncDependency(Box::new(self.deps.clone())));
502            log_send_error("dep_tx", res);
503        } else {
504            log::warn!("ProjectCompiler: settle project not found {id:?}");
505        }
506    }
507
508    /// Process an interrupt.
509    pub fn process(&mut self, intr: Interrupt<F>) {
510        // todo: evcit cache
511        self.process_inner(intr);
512        // Customized Project Compilation Handler
513        self.handler.clone().on_any_compile_reason(self);
514    }
515
516    fn process_inner(&mut self, intr: Interrupt<F>) {
517        match intr {
518            Interrupt::Compile(id) => {
519                let proj = Self::find_project(&mut self.primary, &mut self.dedicates, &id);
520                // Increment the revision anyway.
521                proj.verse.increment_revision(|verse| {
522                    verse.flush();
523                });
524
525                proj.reason.merge(reason_by_entry_change());
526            }
527            Interrupt::Compiled(artifact) => {
528                let proj =
529                    Self::find_project(&mut self.primary, &mut self.dedicates, artifact.id());
530
531                let processed = proj.process_compile(artifact);
532
533                if processed {
534                    self.deps
535                        .project_deps
536                        .insert_mut(proj.id.clone(), proj.deps.clone());
537
538                    let event = NotifyMessage::SyncDependency(Box::new(self.deps.clone()));
539                    let err = self.dep_tx.send(event);
540                    log_send_error("dep_tx", err);
541                }
542            }
543            Interrupt::Settle(id) => {
544                self.remove_dedicates(&id);
545            }
546            Interrupt::ChangeTask(id, change) => {
547                let proj = Self::find_project(&mut self.primary, &mut self.dedicates, &id);
548                proj.verse.increment_revision(|verse| {
549                    if let Some(inputs) = change.inputs.clone() {
550                        verse.set_inputs(inputs);
551                    }
552
553                    if let Some(entry) = change.entry.clone() {
554                        let res = verse.mutate_entry(entry);
555                        if let Err(err) = res {
556                            log::error!("ProjectCompiler: change entry error: {err:?}");
557                        }
558                    }
559                });
560
561                // After incrementing the revision
562                if let Some(entry) = change.entry {
563                    // todo: dedicate suspended
564                    if entry.is_inactive() {
565                        log::info!("ProjectCompiler: removing diag");
566                        self.handler.status(proj.verse.revision.get(), {
567                            CompileReport {
568                                id: proj.id.clone(),
569                                compiling_id: None,
570                                page_count: 0,
571                                status: CompileStatusEnum::Suspend,
572                            }
573                        });
574                    }
575
576                    // Forget the document state of previous entry.
577                    proj.latest_success_doc = None;
578                }
579
580                proj.reason.merge(reason_by_entry_change());
581            }
582
583            Interrupt::Font(fonts) => {
584                self.projects().for_each(|proj| {
585                    let font_changed = proj.verse.increment_revision(|verse| {
586                        verse.set_fonts(fonts.clone());
587                        verse.font_changed()
588                    });
589                    if font_changed {
590                        // todo: reason_by_font_change
591                        proj.reason.merge(reason_by_entry_change());
592                    }
593                });
594            }
595            Interrupt::CreationTimestamp(creation_timestamp) => {
596                self.projects().for_each(|proj| {
597                    let timestamp_changed = proj.verse.increment_revision(|verse| {
598                        verse.set_creation_timestamp(creation_timestamp);
599                        // Creation timestamp changes affect compilation
600                        verse.creation_timestamp_changed()
601                    });
602                    if timestamp_changed {
603                        proj.reason.merge(reason_by_entry_change());
604                    }
605                });
606            }
607            Interrupt::Memory(event) => {
608                log::debug!("ProjectCompiler: memory event incoming");
609
610                // Emulate memory changes.
611                let mut files = HashSet::new();
612                if matches!(event, MemoryEvent::Sync(..)) {
613                    std::mem::swap(&mut files, &mut self.estimated_shadow_files);
614                }
615
616                let (MemoryEvent::Sync(e) | MemoryEvent::Update(e)) = &event;
617                for path in &e.removes {
618                    self.estimated_shadow_files.remove(path);
619                    files.insert(Arc::clone(path));
620                }
621                for (path, _) in &e.inserts {
622                    self.estimated_shadow_files.insert(Arc::clone(path));
623                    files.remove(path);
624                }
625
626                // If there is no invalidation happening, apply memory changes directly.
627                if files.is_empty() && self.dirty_shadow_logical_tick == 0 {
628                    let changes = std::iter::repeat_n(event, 1 + self.dedicates.len());
629                    let proj = std::iter::once(&mut self.primary).chain(self.dedicates.iter_mut());
630                    for (proj, event) in proj.zip(changes) {
631                        log::debug!("memory update: vfs {:#?}", proj.verse.vfs().display());
632                        let vfs_changed = proj.verse.increment_revision(|verse| {
633                            log::debug!("memory update: {:?}", proj.id);
634                            Self::apply_memory_changes(&mut verse.vfs(), event.clone());
635                            log::debug!("memory update: changed {}", verse.vfs_changed());
636                            verse.vfs_changed()
637                        });
638                        if vfs_changed {
639                            proj.reason.merge(reason_by_mem());
640                        }
641                        log::debug!("memory update: vfs after {:#?}", proj.verse.vfs().display());
642                    }
643                    return;
644                }
645
646                // Otherwise, send upstream update event.
647                // Also, record the logical tick when shadow is dirty.
648                self.dirty_shadow_logical_tick = self.logical_tick;
649                let event = NotifyMessage::UpstreamUpdate(UpstreamUpdateEvent {
650                    invalidates: files.into_iter().collect(),
651                    opaque: Box::new(TaggedMemoryEvent {
652                        logical_tick: self.logical_tick,
653                        event,
654                    }),
655                });
656                let err = self.dep_tx.send(event);
657                log_send_error("dep_tx", err);
658            }
659            Interrupt::Save(event) => {
660                let changes = std::iter::repeat_n(&event, 1 + self.dedicates.len());
661                let proj = std::iter::once(&mut self.primary).chain(self.dedicates.iter_mut());
662
663                for (proj, saved_path) in proj.zip(changes) {
664                    log::debug!(
665                        "ProjectCompiler({}, rev={}): save changes",
666                        proj.verse.revision.get(),
667                        proj.id
668                    );
669
670                    // todo: only emit if saved_path is related
671                    let _ = saved_path;
672
673                    proj.reason.merge(reason_by_fs());
674                }
675            }
676            Interrupt::Fs(event) => {
677                log::debug!("ProjectCompiler: fs event incoming {event:?}");
678
679                // Apply file system changes.
680                let dirty_tick = &mut self.dirty_shadow_logical_tick;
681                let (changes, is_sync, event) = event.split_with_is_sync();
682                let changes = std::iter::repeat_n(changes, 1 + self.dedicates.len());
683                let proj = std::iter::once(&mut self.primary).chain(self.dedicates.iter_mut());
684
685                for (proj, changes) in proj.zip(changes) {
686                    log::debug!(
687                        "ProjectCompiler({}, rev={}): fs changes applying",
688                        proj.verse.revision.get(),
689                        proj.id
690                    );
691
692                    proj.verse.increment_revision(|verse| {
693                        let mut vfs = verse.vfs();
694
695                        // Handle delayed upstream update event before applying file system
696                        // changes
697                        if Self::apply_delayed_memory_changes(&mut vfs, dirty_tick, &event)
698                            .is_none()
699                        {
700                            log::warn!("ProjectCompiler: unknown upstream update event");
701
702                            // Actual a delayed memory event.
703                            proj.reason.merge(reason_by_mem());
704                        }
705                        vfs.notify_fs_changes(changes);
706                    });
707
708                    log::debug!(
709                        "ProjectCompiler({},rev={}): fs changes applied, {is_sync}",
710                        proj.id,
711                        proj.verse.revision.get(),
712                    );
713
714                    if !self.ignore_first_sync || !is_sync {
715                        proj.reason.merge(reason_by_fs());
716                    }
717                }
718            }
719        }
720    }
721
722    /// Apply delayed memory changes to underlying compiler.
723    fn apply_delayed_memory_changes(
724        verse: &mut RevisingVfs<'_, F::AccessModel>,
725        dirty_shadow_logical_tick: &mut usize,
726        event: &Option<UpstreamUpdateEvent>,
727    ) -> Option<()> {
728        // Handle delayed upstream update event before applying file system changes
729        if let Some(event) = event {
730            let TaggedMemoryEvent {
731                logical_tick,
732                event,
733            } = event.opaque.as_ref().downcast_ref()?;
734
735            // Recovery from dirty shadow state.
736            if logical_tick == dirty_shadow_logical_tick {
737                *dirty_shadow_logical_tick = 0;
738            }
739
740            Self::apply_memory_changes(verse, event.clone());
741        }
742
743        Some(())
744    }
745
746    /// Apply memory changes to underlying compiler.
747    fn apply_memory_changes(vfs: &mut RevisingVfs<'_, F::AccessModel>, event: MemoryEvent) {
748        if matches!(event, MemoryEvent::Sync(..)) {
749            vfs.reset_shadow();
750        }
751        match event {
752            MemoryEvent::Update(event) | MemoryEvent::Sync(event) => {
753                for path in event.removes {
754                    let _ = vfs.unmap_shadow(&path);
755                }
756                for (path, snap) in event.inserts {
757                    let _ = vfs.map_shadow(&path, snap);
758                }
759            }
760        }
761    }
762}
763
764/// A project instance state.
765pub struct ProjectInsState<F: CompilerFeat, Ext> {
766    /// The project instance id.
767    pub id: ProjectInsId,
768    /// The extension
769    pub ext: Ext,
770    /// The underlying universe.
771    pub verse: CompilerUniverse<F>,
772    /// Specifies the current export target.
773    pub export_target: ExportTarget,
774    /// Whether to run in syntax-only mode.
775    pub syntax_only: bool,
776    /// The reason to compile.
777    pub reason: CompileSignal,
778    /// The compilation handle.
779    pub handler: Arc<dyn CompileHandler<F, Ext>>,
780    /// The file dependencies.
781    deps: EcoVec<ImmutPath>,
782
783    /// The latest compute graph (snapshot), derived lazily from
784    /// `latest_compilation` as needed.
785    pub cached_snapshot: Option<Arc<WorldComputeGraph<F>>>,
786    /// The latest compilation.
787    pub latest_compilation: OnceLock<CompiledArtifact<F>>,
788    /// The latest successly compiled document.
789    pub latest_success_doc: Option<TypstDocument>,
790
791    committed_revision: usize,
792}
793
794impl<F: CompilerFeat, Ext: 'static> ProjectInsState<F, Ext> {
795    /// Gets a snapshot of the project.
796    pub fn snapshot(&mut self) -> Arc<WorldComputeGraph<F>> {
797        match self.cached_snapshot.as_ref() {
798            Some(snap) if snap.world().revision() == self.verse.revision => snap.clone(),
799            _ => {
800                let snap = self.make_snapshot();
801                self.cached_snapshot = Some(snap.clone());
802                snap
803            }
804        }
805    }
806
807    /// Creates a new snapshot of the project derived from `latest_compilation`.
808    fn make_snapshot(&self) -> Arc<WorldComputeGraph<F>> {
809        let world = self.verse.snapshot();
810        let snap = CompileSnapshot {
811            id: self.id.clone(),
812            world,
813            signal: self.reason,
814            success_doc: self.latest_success_doc.clone(),
815        };
816        WorldComputeGraph::new(snap)
817    }
818
819    /// Compiles the document once if there is any reason and the entry is
820    /// active. (this is used for experimenting typst.node compilations)
821    #[must_use]
822    pub fn may_compile2<'a>(
823        &mut self,
824        compute: impl FnOnce(&Arc<WorldComputeGraph<F>>) + 'a,
825    ) -> Option<impl FnOnce() -> Arc<WorldComputeGraph<F>> + 'a> {
826        if !self.reason.any() || self.verse.entry_state().is_inactive() {
827            return None;
828        }
829
830        let snap = self.snapshot();
831        self.reason = Default::default();
832        Some(move || {
833            compute(&snap);
834            snap
835        })
836    }
837
838    /// Compiles the document once if there is any reason and the entry is
839    /// active.
840    #[must_use]
841    pub fn may_compile(
842        &mut self,
843        handler: &Arc<dyn CompileHandler<F, Ext>>,
844    ) -> Option<impl FnOnce() -> CompiledArtifact<F> + 'static> {
845        if !self.reason.any() || self.verse.entry_state().is_inactive() {
846            return None;
847        }
848
849        let snap = self.snapshot();
850        self.reason = Default::default();
851
852        Some(Self::run_compile(
853            handler.clone(),
854            snap,
855            self.export_target,
856            self.syntax_only,
857        ))
858    }
859
860    /// Compile the document once.
861    fn run_compile(
862        h: Arc<dyn CompileHandler<F, Ext>>,
863        graph: Arc<WorldComputeGraph<F>>,
864        export_target: ExportTarget,
865        syntax_only: bool,
866    ) -> impl FnOnce() -> CompiledArtifact<F> {
867        let start = tinymist_std::time::Instant::now();
868
869        // todo unwrap main id
870        let id = graph.world().main_id().unwrap();
871        let revision = graph.world().revision().get();
872
873        h.status(revision, {
874            CompileReport {
875                id: graph.snap.id.clone(),
876                compiling_id: Some(id),
877                page_count: 0,
878                status: CompileStatusEnum::Compiling,
879            }
880        });
881
882        move || {
883            let compiled = if syntax_only {
884                let main = graph.snap.world.main();
885                let source_res = graph.world().source(main).at(Span::from_range(main, 0..0));
886                let syntax_res = source_res.and_then(|source| {
887                    let errors = source.root().errors();
888                    if errors.is_empty() {
889                        Ok(())
890                    } else {
891                        Err(errors.into_iter().map(|s| s.into()).collect())
892                    }
893                });
894                let diag = Arc::new(DiagnosticsTask::from_errors(syntax_res.err()));
895
896                CompiledArtifact {
897                    diag,
898                    graph,
899                    doc: None,
900                    deps: OnceLock::default(),
901                }
902            } else {
903                CompiledArtifact::from_graph(graph, matches!(export_target, ExportTarget::Html))
904            };
905
906            let res = CompileStatusResult {
907                diag: (compiled.warning_cnt() + compiled.error_cnt()) as u32,
908                elapsed: start.elapsed(),
909            };
910            let rep = CompileReport {
911                id: compiled.id().clone(),
912                compiling_id: Some(id),
913                page_count: compiled.doc.as_ref().map_or(0, |doc| doc.num_of_pages()),
914                status: match &compiled.doc {
915                    Some(..) => CompileStatusEnum::CompileSuccess(res),
916                    None if res.diag == 0 => CompileStatusEnum::CompileSuccess(res),
917                    None => CompileStatusEnum::CompileError(res),
918                },
919            };
920
921            // todo: we need to check revision for really concurrent compilation
922            log_compile_report(&rep);
923
924            if compiled
925                .diagnostics()
926                .any(|d| d.message == FILE_MISSING_ERROR_MSG)
927            {
928                return compiled;
929            }
930
931            h.status(revision, rep);
932            h.notify_compile(&compiled);
933            compiled
934        }
935    }
936
937    fn process_compile(&mut self, artifact: CompiledArtifact<F>) -> bool {
938        let world = &artifact.snap.world;
939        let compiled_revision = world.revision().get();
940        if self.committed_revision >= compiled_revision {
941            return false;
942        }
943
944        // Updates state.
945        let doc = artifact.doc.clone();
946        self.committed_revision = compiled_revision;
947        if doc.is_some() {
948            self.latest_success_doc = doc;
949        }
950        self.cached_snapshot = None; // invalidate; will be recomputed on demand
951
952        // Notifies the new file dependencies.
953        let mut deps = eco_vec![];
954        world.iter_dependencies(&mut |dep| {
955            if let Ok(x) = world.file_path(dep).and_then(|e| e.to_err()) {
956                deps.push(x.into())
957            }
958        });
959
960        self.deps = deps.clone();
961
962        let mut world = world.clone();
963
964        let is_primary = self.id == ProjectInsId("primary".into());
965
966        // Trigger an evict task.
967        spawn_cpu(move || {
968            let evict_start = tinymist_std::time::Instant::now();
969            if is_primary {
970                comemo::evict(10);
971
972                // Since all the projects share the same cache, we need to evict the cache
973                // on the primary instance for all the projects.
974                world.evict_source_cache(30);
975            }
976            world.evict_vfs(60);
977            let elapsed = evict_start.elapsed();
978            log::debug!("ProjectCompiler: evict cache in {elapsed:?}");
979        });
980
981        true
982    }
983}
984
985fn log_compile_report(rep: &CompileReport) {
986    log::info!("{}", rep.message());
987}
988
989#[inline]
990fn log_send_error<T>(chan: &'static str, res: Result<(), mpsc::error::SendError<T>>) -> bool {
991    res.map_err(|err| log::warn!("ProjectCompiler: send to {chan} error: {err}"))
992        .is_ok()
993}
994
995#[derive(Debug, Clone, Default)]
996struct ProjectDeps {
997    project_deps: rpds::RedBlackTreeMapSync<ProjectInsId, EcoVec<ImmutPath>>,
998}
999
1000impl NotifyDeps for ProjectDeps {
1001    fn dependencies(&self, f: &mut dyn FnMut(&ImmutPath)) {
1002        for deps in self.project_deps.values().flat_map(|e| e.iter()) {
1003            f(deps);
1004        }
1005    }
1006}
1007
1008// todo: move me to tinymist-std
1009#[cfg(not(target_arch = "wasm32"))]
1010/// Spawns a CPU thread to run a computing-heavy task.
1011pub fn spawn_cpu<F>(func: F)
1012where
1013    F: FnOnce() + Send + 'static,
1014{
1015    rayon::spawn(func);
1016}
1017
1018#[cfg(target_arch = "wasm32")]
1019/// Spawns a CPU thread to run a computing-heavy task.
1020pub fn spawn_cpu<F>(func: F)
1021where
1022    F: FnOnce() + Send + 'static,
1023{
1024    func();
1025}
tinymist_project/compiler.rs

tinymist_project/
compiler.rs
