1#[cfg(feature = "browser")]
7pub mod browser;
8
9#[cfg(feature = "system")]
12pub mod system;
13
14pub mod dummy;
20
21#[cfg(any(test, feature = "mock"))]
23pub mod mock;
24
25pub mod snapshot;
27pub use snapshot::*;
28use tinymist_std::hash::{FxDashMap, FxHashMap};
29
30pub mod notify;
33pub use notify::{FilesystemEvent, MemoryEvent};
34pub mod overlay;
37pub mod resolve;
39pub mod trace;
41mod utils;
42
43mod path_mapper;
44pub use path_mapper::{PathResolution, RootResolver, WorkspaceResolution, WorkspaceResolver};
45
46use core::fmt;
47use std::num::{NonZeroU16, NonZeroUsize};
48use std::sync::OnceLock;
49use std::{path::Path, sync::Arc};
50
51use ecow::EcoVec;
52use parking_lot::Mutex;
53use rpds::RedBlackTreeMapSync;
54use typst::diag::{FileError, FileResult};
55use typst::foundations::Dict;
56use typst::syntax::Source;
57use typst::utils::LazyHash;
58
59use crate::notify::NotifyAccessModel;
60use crate::overlay::{OverlayAccessModel, RawFileId};
61use crate::resolve::ResolveAccessModel;
62
63pub use tinymist_std::ImmutPath;
64pub use tinymist_std::time::Time;
65pub use typst::foundations::Bytes;
66pub use typst::syntax::FileId;
67
68pub type ImmutDict = Arc<LazyHash<Dict>>;
70
71pub trait PathAccessModel {
76 fn reset(&mut self) {}
81
82 fn content(&self, src: &Path) -> FileResult<Bytes>;
84}
85
86pub trait AccessModel {
91 fn reset(&mut self) {}
96
97 fn content(&self, src: FileId) -> (Option<ImmutPath>, FileResult<Bytes>);
99}
100
101type VfsPathAccessModel<M> = OverlayAccessModel<ImmutPath, NotifyAccessModel<M>>;
102type VfsAccessModel<M> =
105 OverlayAccessModel<FileId, ResolveAccessModel<VfsPathAccessModel<M>>, RawFileId>;
106
107pub trait FsProvider {
109 fn file_path(&self, id: FileId) -> FileResult<PathResolution>;
111 fn read(&self, id: FileId) -> FileResult<Bytes>;
113 fn read_source(&self, id: FileId) -> FileResult<Source>;
116}
117
118struct SourceEntry {
119 last_accessed_rev: NonZeroUsize,
120 source: FileResult<Source>,
121}
122
123#[derive(Default)]
124struct SourceIdShard {
125 last_accessed_rev: usize,
126 recent_source: Option<Source>,
127 sources: FxHashMap<Bytes, SourceEntry>,
128}
129
130#[derive(Default, Clone)]
132pub struct SourceCache {
133 cache_entries: Arc<FxDashMap<FileId, SourceIdShard>>,
135}
136
137impl SourceCache {
138 pub fn evict(&self, curr: NonZeroUsize, threshold: usize) {
141 self.cache_entries.retain(|_, shard| {
142 let diff = curr.get().saturating_sub(shard.last_accessed_rev);
143 if diff > threshold {
144 return false;
145 }
146
147 shard.sources.retain(|_, entry| {
148 let diff = curr.get().saturating_sub(entry.last_accessed_rev.get());
149 diff <= threshold
150 });
151
152 true
153 });
154 }
155}
156
157pub struct Vfs<M: PathAccessModel + Sized> {
161 source_cache: SourceCache,
162 managed: Arc<Mutex<EntryMap>>,
163 paths: Arc<Mutex<PathMap>>,
164 revision: NonZeroUsize,
165 access_model: VfsAccessModel<M>,
168}
169
170impl<M: PathAccessModel + Sized> fmt::Debug for Vfs<M> {
171 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
172 f.debug_struct("Vfs")
173 .field("revision", &self.revision)
174 .field("managed", &self.managed.lock().entries.size())
175 .field("paths", &self.paths.lock().paths.len())
176 .finish()
177 }
178}
179
180impl<M: PathAccessModel + Clone + Sized> Vfs<M> {
181 pub fn revision(&self) -> NonZeroUsize {
183 self.revision
184 }
185
186 pub fn snapshot(&self) -> Self {
188 Self {
189 revision: self.revision,
190 source_cache: self.source_cache.clone(),
191 managed: self.managed.clone(),
192 paths: self.paths.clone(),
193 access_model: self.access_model.clone(),
194 }
195 }
196
197 pub fn fork(&self) -> Self {
199 Self {
200 source_cache: self.source_cache.clone(),
202 managed: Arc::new(Mutex::new(EntryMap::default())),
203 paths: Arc::new(Mutex::new(PathMap::default())),
204 revision: NonZeroUsize::new(2).expect("initial revision is 2"),
205 access_model: self.access_model.clone(),
206 }
207 }
208
209 pub fn is_clean_compile(&self, rev: usize, file_ids: &[FileId]) -> bool {
212 let mut m = self.managed.lock();
213 for id in file_ids {
214 let Some(entry) = m.get_mut(*id) else {
215 log::debug!("Vfs(dirty, {id:?}): file id not found");
216 return false;
217 };
218 if entry.changed_at > rev {
219 log::debug!("Vfs(dirty, {id:?}): rev {rev:?} => {:?}", entry.changed_at);
220 return false;
221 }
222 log::debug!(
223 "Vfs(clean, {id:?}, rev={rev}, changed_at={})",
224 entry.changed_at
225 );
226 }
227 true
228 }
229}
230
231impl<M: PathAccessModel + Sized> Vfs<M> {
232 pub fn new(resolver: Arc<dyn RootResolver + Send + Sync>, access_model: M) -> Self {
245 let access_model = NotifyAccessModel::new(access_model);
246 let access_model = OverlayAccessModel::new(access_model);
247 let access_model = ResolveAccessModel {
248 resolver,
249 inner: access_model,
250 };
251 let access_model = OverlayAccessModel::new(access_model);
252
253 Self {
257 source_cache: SourceCache::default(),
258 managed: Arc::default(),
259 paths: Arc::default(),
260 revision: NonZeroUsize::new(2).expect("initial revision is 2"),
261 access_model,
262 }
263 }
264
265 pub fn reset_all(&mut self) {
267 self.reset_access_model();
268 self.reset_read();
269 self.take_source_cache();
270 }
271
272 pub fn reset_access_model(&mut self) {
274 self.access_model.reset();
275 }
276
277 pub fn reset_read(&mut self) {
281 self.managed = Arc::default();
282 self.paths = Arc::default();
283 }
284
285 pub fn evict(&mut self, threshold: usize) {
287 let mut m = self.managed.lock();
288 let rev = self.revision.get();
289 for (id, entry) in m.entries.clone().iter() {
290 let entry_rev = entry.bytes.get().map(|e| e.1).unwrap_or_default();
291 if entry_rev + threshold < rev {
292 m.entries.remove_mut(id);
293 }
294 }
295 }
296
297 pub fn take_source_cache(&mut self) -> SourceCache {
299 std::mem::take(&mut self.source_cache)
300 }
301
302 pub fn clone_source_cache(&self) -> SourceCache {
304 self.source_cache.clone()
305 }
306
307 pub fn file_path(&self, id: FileId) -> Result<PathResolution, FileError> {
309 self.access_model.inner.resolver.path_for_id(id)
310 }
311
312 pub fn resolve_root(&self, id: FileId) -> FileResult<Option<ImmutPath>> {
314 self.access_model.inner.resolver.resolve_root(id)
315 }
316
317 pub fn shadow_paths(&self) -> Vec<ImmutPath> {
319 self.access_model.inner.inner.file_paths()
320 }
321
322 pub fn shadow_ids(&self) -> Vec<FileId> {
327 self.access_model.file_paths()
328 }
329
330 pub fn memory_usage(&self) -> usize {
332 0
333 }
334
335 pub fn revise(&mut self) -> RevisingVfs<'_, M> {
338 let managed = self.managed.lock().clone();
339 let paths = self.paths.lock().clone();
340 let goal_revision = self.revision.checked_add(1).expect("revision overflowed");
341
342 RevisingVfs {
343 managed,
344 paths,
345 inner: self,
346 goal_revision,
347 view_changed: false,
348 }
349 }
350
351 pub fn display(&self) -> DisplayVfs<'_, M> {
353 DisplayVfs { inner: self }
354 }
355
356 pub fn read(&self, fid: FileId) -> FileResult<Bytes> {
358 let bytes = self.managed.lock().slot(fid, |entry| entry.bytes.clone());
359
360 self.read_content(&bytes, fid).clone()
361 }
362
363 pub fn source(&self, file_id: FileId) -> FileResult<Source> {
366 let (bytes, source) = self
367 .managed
368 .lock()
369 .slot(file_id, |entry| (entry.bytes.clone(), entry.source.clone()));
370
371 let source = source.get_or_init(|| {
372 let content = self
373 .read_content(&bytes, file_id)
374 .as_ref()
375 .map_err(Clone::clone)?;
376
377 let mut cache_entry = self.source_cache.cache_entries.entry(file_id).or_default();
378 if let Some(source) = cache_entry.sources.get(content) {
379 return source.source.clone();
380 }
381
382 let source = (|| {
383 let prev = cache_entry.recent_source.clone();
384 let content = from_utf8_or_bom(content).map_err(|_| FileError::InvalidUtf8)?;
385
386 let next = match prev {
387 Some(mut prev) => {
388 prev.replace(content);
389 prev
390 }
391 None => Source::new(file_id, content.to_owned()),
392 };
393
394 let should_update = cache_entry.recent_source.is_none()
395 || cache_entry.last_accessed_rev < self.revision.get();
396 if should_update {
397 cache_entry.recent_source = Some(next.clone());
398 }
399
400 Ok(next)
401 })();
402
403 let entry = cache_entry
404 .sources
405 .entry(content.clone())
406 .or_insert_with(|| SourceEntry {
407 last_accessed_rev: self.revision,
408 source: source.clone(),
409 });
410
411 if entry.last_accessed_rev < self.revision {
412 entry.last_accessed_rev = self.revision;
413 }
414
415 source
416 });
417
418 source.clone()
419 }
420
421 fn read_content<'a>(&self, bytes: &'a BytesQuery, fid: FileId) -> &'a FileResult<Bytes> {
423 &bytes
424 .get_or_init(|| {
425 let (path, content) = self.access_model.content(fid);
426 if let Some(path) = path.as_ref() {
427 self.paths.lock().insert(path, fid, self.revision);
428 }
429
430 (path, self.revision.get(), content)
431 })
432 .2
433 }
434}
435
436pub struct DisplayVfs<'a, M: PathAccessModel + Sized> {
438 inner: &'a Vfs<M>,
439}
440
441impl<M: PathAccessModel + Sized> fmt::Debug for DisplayVfs<'_, M> {
442 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443 f.debug_struct("Vfs")
444 .field("revision", &self.inner.revision)
445 .field("managed", &self.inner.managed.lock().display())
446 .field("paths", &self.inner.paths.lock().display())
447 .finish()
448 }
449}
450
451pub struct RevisingVfs<'a, M: PathAccessModel + Sized> {
453 inner: &'a mut Vfs<M>,
454 managed: EntryMap,
455 paths: PathMap,
456 goal_revision: NonZeroUsize,
457 view_changed: bool,
458}
459
460impl<M: PathAccessModel + Sized> Drop for RevisingVfs<'_, M> {
461 fn drop(&mut self) {
462 if self.view_changed {
463 self.inner.managed = Arc::new(Mutex::new(std::mem::take(&mut self.managed)));
464 self.inner.paths = Arc::new(Mutex::new(std::mem::take(&mut self.paths)));
465 let revision = &mut self.inner.revision;
466 *revision = self.goal_revision;
467 }
468 }
469}
470
471impl<M: PathAccessModel + Sized> RevisingVfs<'_, M> {
472 pub fn vfs(&mut self) -> &mut Vfs<M> {
474 self.inner
475 }
476
477 fn am(&mut self) -> &mut VfsAccessModel<M> {
478 &mut self.inner.access_model
479 }
480
481 fn invalidate_path(&mut self, path: &Path, snap: Option<&FileSnapshot>) {
482 if let Some(fids) = self.paths.get(path) {
483 if fids.is_empty() {
484 return;
485 }
486
487 self.view_changed = snap.is_none();
489 for fid in fids.clone() {
490 self.invalidate_file_id(fid, snap);
491 }
492 }
493 }
494
495 fn invalidate_file_id(&mut self, file_id: FileId, snap: Option<&FileSnapshot>) {
496 let mut changed = false;
497 self.managed.slot(file_id, |e| {
498 if let Some(snap) = snap {
499 let may_read_bytes = e.bytes.get().map(|b| &b.2);
500 match (snap, may_read_bytes) {
501 (FileSnapshot(Ok(snap)), Some(Ok(read))) if snap == read => {
502 return;
503 }
504 (FileSnapshot(Err(snap)), Some(Err(read))) if snap.as_ref() == read => {
505 return;
506 }
507 _ => {}
508 }
509 }
510
511 e.changed_at = self.goal_revision.get();
512 e.bytes = Arc::default();
513 e.source = Arc::default();
514 changed = true;
515 });
516 self.view_changed = changed;
517 }
518
519 pub fn reset_shadow(&mut self) {
521 for path in self.am().inner.inner.file_paths() {
522 self.invalidate_path(&path, None);
523 }
524 for fid in self.am().file_paths() {
525 self.invalidate_file_id(fid, None);
526 }
527
528 self.am().clear_shadow();
529 self.am().inner.inner.clear_shadow();
530 }
531
532 pub fn change_view(&mut self) -> FileResult<()> {
534 self.view_changed = true;
535 Ok(())
536 }
537
538 pub fn map_shadow(&mut self, path: &Path, snap: FileSnapshot) -> FileResult<()> {
540 self.invalidate_path(path, Some(&snap));
541 self.am().inner.inner.add_file(path, snap, |c| c.into());
542
543 Ok(())
544 }
545
546 pub fn unmap_shadow(&mut self, path: &Path) -> FileResult<()> {
548 self.invalidate_path(path, None);
549 self.am().inner.inner.remove_file(path);
550
551 Ok(())
552 }
553
554 pub fn map_shadow_by_id(&mut self, file_id: FileId, snap: FileSnapshot) -> FileResult<()> {
556 self.invalidate_file_id(file_id, Some(&snap));
557 self.am().add_file(&file_id, snap, |c| *c);
558
559 Ok(())
560 }
561
562 pub fn remove_shadow_by_id(&mut self, file_id: FileId) {
564 self.invalidate_file_id(file_id, None);
565 self.am().remove_file(&file_id);
566 }
567
568 pub fn notify_fs_event(&mut self, event: FilesystemEvent) {
572 self.notify_fs_changes(event.split().0);
573 }
574 pub fn notify_fs_changes(&mut self, event: FileChangeSet) {
578 for path in &event.removes {
579 self.invalidate_path(path, None);
580 }
581 for (path, snap) in &event.inserts {
582 self.invalidate_path(path, Some(snap));
583 }
584
585 self.am().inner.inner.inner.notify(event);
586 }
587}
588
589type BytesQuery = Arc<OnceLock<(Option<ImmutPath>, usize, FileResult<Bytes>)>>;
590
591#[derive(Debug, Clone, Default)]
592struct VfsEntry {
593 changed_at: usize,
594 bytes: BytesQuery,
595 source: Arc<OnceLock<FileResult<Source>>>,
596}
597
598#[derive(Debug, Clone, Default)]
599struct EntryMap {
600 entries: RedBlackTreeMapSync<EntryId, VfsEntry>,
601}
602
603type EntryId = NonZeroU16;
604
605impl EntryMap {
606 #[inline(always)]
608 fn slot<T>(&mut self, id: FileId, f: impl FnOnce(&mut VfsEntry) -> T) -> T {
609 let id = Self::key(id);
610 if let Some(entry) = self.entries.get_mut(&id) {
611 f(entry)
612 } else {
613 let mut entry = VfsEntry::default();
614 let res = f(&mut entry);
615 self.entries.insert_mut(id, entry);
616 res
617 }
618 }
619
620 fn get_mut(&mut self, id: FileId) -> Option<&mut VfsEntry> {
621 self.entries.get_mut(&Self::key(id))
622 }
623
624 fn key(id: FileId) -> EntryId {
625 id.into_raw()
626 }
627
628 fn display(&self) -> DisplayEntryMap<'_> {
629 DisplayEntryMap { map: self }
630 }
631}
632
633pub struct DisplayEntryMap<'a> {
635 map: &'a EntryMap,
636}
637
638impl fmt::Debug for DisplayEntryMap<'_> {
639 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
640 f.debug_map().entries(self.map.entries.iter()).finish()
641 }
642}
643
644#[derive(Debug, Clone, Default)]
645struct PathMap {
646 paths: FxHashMap<ImmutPath, EcoVec<FileId>>,
647 file_ids: FxHashMap<FileId, (ImmutPath, NonZeroUsize)>,
648}
649
650impl PathMap {
651 fn insert(&mut self, next: &ImmutPath, fid: FileId, rev: NonZeroUsize) {
652 use std::collections::hash_map::Entry;
653 let rev_entry = self.file_ids.entry(fid);
654
655 match rev_entry {
656 Entry::Occupied(mut entry) => {
657 let (prev, prev_rev) = entry.get_mut();
658 if prev != next {
659 if *prev_rev == rev {
660 log::warn!("Vfs: {fid:?} is changed in rev({rev:?}), {prev:?} -> {next:?}");
661 }
662
663 if let Some(fids) = self.paths.get_mut(prev) {
664 fids.retain(|f| *f != fid);
665 }
666
667 *prev = next.clone();
668 *prev_rev = rev;
669
670 self.paths.entry(next.clone()).or_default().push(fid);
671 }
672 }
673 Entry::Vacant(entry) => {
674 entry.insert((next.clone(), rev));
675 self.paths.entry(next.clone()).or_default().push(fid);
676 }
677 }
678 }
679
680 fn get(&mut self, path: &Path) -> Option<&EcoVec<FileId>> {
681 self.paths.get(path)
682 }
683
684 fn display(&self) -> DisplayPathMap<'_> {
685 DisplayPathMap { map: self }
686 }
687}
688
689pub struct DisplayPathMap<'a> {
691 map: &'a PathMap,
692}
693
694impl fmt::Debug for DisplayPathMap<'_> {
695 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
696 f.debug_map().entries(self.map.paths.iter()).finish()
697 }
698}
699
700fn from_utf8_or_bom(buf: &[u8]) -> FileResult<&str> {
702 Ok(std::str::from_utf8(if buf.starts_with(b"\xef\xbb\xbf") {
703 &buf[3..]
705 } else {
706 buf
708 })?)
709}
710
711#[cfg(test)]
712mod tests {
713 fn is_send<T: Send>() {}
714 fn is_sync<T: Sync>() {}
715
716 #[test]
717 fn test_vfs_send_sync() {
718 is_send::<super::Vfs<super::dummy::DummyAccessModel>>();
719 is_sync::<super::Vfs<super::dummy::DummyAccessModel>>();
720 }
721}