tinymist_package/registry/
http.rs

1//! Http registry for tinymist.
2
3use std::path::Path;
4use std::sync::{Arc, OnceLock};
5
6use parking_lot::Mutex;
7use reqwest::Certificate;
8use reqwest::blocking::Response;
9use tinymist_std::ImmutPath;
10use typst::diag::{PackageResult, StrResult, eco_format};
11use typst::syntax::package::{PackageVersion, VersionlessPackageSpec};
12
13use crate::registry::{PREVIEW_NS, PackageIndexEntry, PackageSpecExt};
14
15use super::{
16    DEFAULT_REGISTRY, DummyNotifier, Notifier, PackageError, PackageRegistry, PackageSpec,
17};
18
19/// The http package registry for typst.ts.
20pub struct HttpRegistry {
21    /// The path at which local packages (`@local` packages) are stored.
22    package_path: Option<ImmutPath>,
23    /// The path at which non-local packages (`@preview` packages) should be
24    /// stored when downloaded.
25    package_cache_path: Option<ImmutPath>,
26    /// lazily initialized package storage.
27    storage: OnceLock<PackageStorage>,
28    /// The path to the certificate file to use for HTTPS requests.
29    cert_path: Option<ImmutPath>,
30    /// The notifier to use for progress updates.
31    notifier: Arc<Mutex<dyn Notifier + Send>>,
32    // package_dir_cache: RwLock<HashMap<PackageSpec, Result<ImmutPath, PackageError>>>,
33}
34
35impl Default for HttpRegistry {
36    fn default() -> Self {
37        Self {
38            notifier: Arc::new(Mutex::<DummyNotifier>::default()),
39            cert_path: None,
40            package_path: None,
41            package_cache_path: None,
42
43            storage: OnceLock::new(),
44            // package_dir_cache: RwLock::new(HashMap::new()),
45        }
46    }
47}
48
49impl std::ops::Deref for HttpRegistry {
50    type Target = PackageStorage;
51
52    fn deref(&self) -> &Self::Target {
53        self.storage()
54    }
55}
56
57impl HttpRegistry {
58    /// Create a new registry.
59    pub fn new(
60        cert_path: Option<ImmutPath>,
61        package_path: Option<ImmutPath>,
62        package_cache_path: Option<ImmutPath>,
63    ) -> Self {
64        Self {
65            cert_path,
66            package_path,
67            package_cache_path,
68            ..Default::default()
69        }
70    }
71
72    /// Get `typst-kit` implementing package storage
73    pub fn storage(&self) -> &PackageStorage {
74        self.storage.get_or_init(|| {
75            PackageStorage::new(
76                self.package_cache_path
77                    .clone()
78                    .or_else(|| Some(dirs::cache_dir()?.join(DEFAULT_PACKAGES_SUBDIR).into())),
79                self.package_path
80                    .clone()
81                    .or_else(|| Some(dirs::data_dir()?.join(DEFAULT_PACKAGES_SUBDIR).into())),
82                self.cert_path.clone(),
83                self.notifier.clone(),
84            )
85        })
86    }
87
88    /// Get local path option
89    pub fn local_path(&self) -> Option<ImmutPath> {
90        self.storage().package_path().cloned()
91    }
92
93    /// Get data & cache dir
94    pub fn paths(&self) -> Vec<ImmutPath> {
95        let data_dir = self.storage().package_path().cloned();
96        let cache_dir = self.storage().package_cache_path().cloned();
97        data_dir.into_iter().chain(cache_dir).collect::<Vec<_>>()
98    }
99
100    /// Set list of packages for testing.
101    pub fn test_package_list(&self, f: impl FnOnce() -> Vec<PackageIndexEntry>) {
102        self.storage().index.get_or_init(f);
103    }
104}
105
106impl PackageRegistry for HttpRegistry {
107    fn resolve(&self, spec: &PackageSpec) -> Result<ImmutPath, PackageError> {
108        self.storage().prepare_package(spec)
109    }
110
111    fn packages(&self) -> &[PackageIndexEntry] {
112        self.storage().download_index()
113    }
114}
115
116/// The default packages sub directory within the package and package cache
117/// paths.
118pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
119
120/// Holds information about where packages should be stored and downloads them
121/// on demand, if possible.
122pub struct PackageStorage {
123    /// The path at which non-local packages should be stored when downloaded.
124    package_cache_path: Option<ImmutPath>,
125    /// The path at which local packages are stored.
126    package_path: Option<ImmutPath>,
127    /// The downloader used for fetching the index and packages.
128    cert_path: Option<ImmutPath>,
129    /// The cached index of the preview namespace.
130    index: OnceLock<Vec<PackageIndexEntry>>,
131    notifier: Arc<Mutex<dyn Notifier + Send>>,
132}
133
134impl PackageStorage {
135    /// Creates a new package storage for the given package paths.
136    /// It doesn't fallback directories, thus you can disable the related
137    /// storage by passing `None`.
138    pub fn new(
139        package_cache_path: Option<ImmutPath>,
140        package_path: Option<ImmutPath>,
141        cert_path: Option<ImmutPath>,
142        notifier: Arc<Mutex<dyn Notifier + Send>>,
143    ) -> Self {
144        Self {
145            package_cache_path,
146            package_path,
147            cert_path,
148            notifier,
149            index: OnceLock::new(),
150        }
151    }
152
153    /// Returns the path at which non-local packages should be stored when
154    /// downloaded.
155    pub fn package_cache_path(&self) -> Option<&ImmutPath> {
156        self.package_cache_path.as_ref()
157    }
158
159    /// Returns the path at which local packages are stored.
160    pub fn package_path(&self) -> Option<&ImmutPath> {
161        self.package_path.as_ref()
162    }
163
164    /// Make a package available in the on-disk cache.
165    pub fn prepare_package(&self, spec: &PackageSpec) -> PackageResult<ImmutPath> {
166        let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version);
167
168        if let Some(packages_dir) = &self.package_path {
169            let dir = packages_dir.join(&subdir);
170            if dir.exists() {
171                return Ok(dir.into());
172            }
173        }
174
175        if let Some(cache_dir) = &self.package_cache_path {
176            let dir = cache_dir.join(&subdir);
177            if dir.exists() {
178                return Ok(dir.into());
179            }
180
181            // Download from network if it doesn't exist yet.
182            if spec.is_preview() {
183                self.download_package(spec, &dir)?;
184                if dir.exists() {
185                    return Ok(dir.into());
186                }
187            }
188        }
189
190        Err(PackageError::NotFound(spec.clone()))
191    }
192
193    /// Try to determine the latest version of a package.
194    pub fn determine_latest_version(
195        &self,
196        spec: &VersionlessPackageSpec,
197    ) -> StrResult<PackageVersion> {
198        if spec.is_preview() {
199            // For `@preview`, download the package index and find the latest
200            // version.
201            self.download_index()
202                .iter()
203                .filter(|entry| entry.package.name == spec.name)
204                .map(|entry| entry.package.version)
205                .max()
206                .ok_or_else(|| eco_format!("failed to find package {spec}"))
207        } else {
208            // For other namespaces, search locally. We only search in the data
209            // directory and not the cache directory, because the latter is not
210            // intended for storage of local packages.
211            let subdir = format!("{}/{}", spec.namespace, spec.name);
212            self.package_path
213                .iter()
214                .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
215                .flatten()
216                .filter_map(|entry| entry.ok())
217                .map(|entry| entry.path())
218                .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
219                .max()
220                .ok_or_else(|| eco_format!("please specify the desired version"))
221        }
222    }
223
224    /// Get the cached package index without network access.
225    pub fn cached_index(&self) -> Option<&[PackageIndexEntry]> {
226        self.index.get().map(Vec::as_slice)
227    }
228
229    /// Download the package index. The result of this is cached for efficiency.
230    pub fn download_index(&self) -> &[PackageIndexEntry] {
231        self.index.get_or_init(|| {
232            let url = format!("{DEFAULT_REGISTRY}/preview/index.json");
233
234            threaded_http(&url, self.cert_path.as_deref(), |resp| {
235                let reader = match resp.and_then(|r| r.error_for_status()) {
236                    Ok(response) => response,
237                    Err(err) => {
238                        // todo: silent error
239                        log::error!("Failed to fetch package index: {err} from {url}");
240                        return vec![];
241                    }
242                };
243
244                let mut entries: Vec<PackageIndexEntry> = match serde_json::from_reader(reader) {
245                    Ok(entry) => entry,
246                    Err(err) => {
247                        log::error!("Failed to parse package index: {err} from {url}");
248                        return vec![];
249                    }
250                };
251                for entry in &mut entries {
252                    entry.namespace = PREVIEW_NS.into();
253                }
254
255                entries
256            })
257            .unwrap_or_default()
258        })
259    }
260
261    /// Download a package over the network.
262    ///
263    /// # Panics
264    /// Panics if the package spec namespace isn't `preview`.
265    pub fn download_package(&self, spec: &PackageSpec, package_dir: &Path) -> PackageResult<()> {
266        assert!(spec.is_preview(), "only preview packages can be downloaded");
267
268        let url = format!(
269            "{DEFAULT_REGISTRY}/preview/{}-{}.tar.gz",
270            spec.name, spec.version
271        );
272
273        self.notifier.lock().downloading(spec);
274        threaded_http(&url, self.cert_path.as_deref(), |resp| {
275            let reader = match resp.and_then(|r| r.error_for_status()) {
276                Ok(response) => response,
277                Err(err) if matches!(err.status().map(|s| s.as_u16()), Some(404)) => {
278                    return Err(PackageError::NotFound(spec.clone()));
279                }
280                Err(err) => return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))),
281            };
282
283            let decompressed = flate2::read::GzDecoder::new(reader);
284            tar::Archive::new(decompressed)
285                .unpack(package_dir)
286                .map_err(|err| {
287                    std::fs::remove_dir_all(package_dir).ok();
288                    PackageError::MalformedArchive(Some(eco_format!("{err}")))
289                })
290        })
291        .ok_or_else(|| PackageError::Other(Some(eco_format!("cannot spawn http thread"))))?
292    }
293}
294
295pub(crate) fn threaded_http<T: Send + Sync>(
296    url: &str,
297    cert_path: Option<&Path>,
298    f: impl FnOnce(Result<Response, reqwest::Error>) -> T + Send + Sync,
299) -> Option<T> {
300    std::thread::scope(|s| {
301        s.spawn(move || {
302            let client_builder = reqwest::blocking::Client::builder();
303
304            let client = if let Some(cert_path) = cert_path {
305                let cert = std::fs::read(cert_path)
306                    .ok()
307                    .and_then(|buf| Certificate::from_pem(&buf).ok());
308                if let Some(cert) = cert {
309                    client_builder.add_root_certificate(cert).build().unwrap()
310                } else {
311                    client_builder.build().unwrap()
312                }
313            } else {
314                client_builder.build().unwrap()
315            };
316
317            f(client.get(url).send())
318        })
319        .join()
320        .ok()
321    })
322}
tinymist_package/registry/http.rs

tinymist_package/registry/
http.rs
