tinymist_package/registry/
http.rs

1//! Http registry for tinymist.
2
3use std::path::Path;
4use std::sync::{Arc, OnceLock};
5
6use parking_lot::Mutex;
7use reqwest::Certificate;
8use reqwest::blocking::Response;
9use tinymist_std::ImmutPath;
10use typst::diag::{EcoString, PackageResult, StrResult, eco_format};
11use typst::syntax::package::{PackageVersion, VersionlessPackageSpec};
12
13use super::{
14    DEFAULT_REGISTRY, DummyNotifier, Notifier, PackageError, PackageRegistry, PackageSpec,
15};
16
17/// The http package registry for typst.ts.
18pub struct HttpRegistry {
19    /// The path at which local packages (`@local` packages) are stored.
20    package_path: Option<ImmutPath>,
21    /// The path at which non-local packages (`@preview` packages) should be
22    /// stored when downloaded.
23    package_cache_path: Option<ImmutPath>,
24    /// lazily initialized package storage.
25    storage: OnceLock<PackageStorage>,
26    /// The path to the certificate file to use for HTTPS requests.
27    cert_path: Option<ImmutPath>,
28    /// The notifier to use for progress updates.
29    notifier: Arc<Mutex<dyn Notifier + Send>>,
30    // package_dir_cache: RwLock<HashMap<PackageSpec, Result<ImmutPath, PackageError>>>,
31}
32
33impl Default for HttpRegistry {
34    fn default() -> Self {
35        Self {
36            notifier: Arc::new(Mutex::<DummyNotifier>::default()),
37            cert_path: None,
38            package_path: None,
39            package_cache_path: None,
40
41            storage: OnceLock::new(),
42            // package_dir_cache: RwLock::new(HashMap::new()),
43        }
44    }
45}
46
47impl std::ops::Deref for HttpRegistry {
48    type Target = PackageStorage;
49
50    fn deref(&self) -> &Self::Target {
51        self.storage()
52    }
53}
54
55impl HttpRegistry {
56    /// Create a new registry.
57    pub fn new(
58        cert_path: Option<ImmutPath>,
59        package_path: Option<ImmutPath>,
60        package_cache_path: Option<ImmutPath>,
61    ) -> Self {
62        Self {
63            cert_path,
64            package_path,
65            package_cache_path,
66            ..Default::default()
67        }
68    }
69
70    /// Get `typst-kit` implementing package storage
71    pub fn storage(&self) -> &PackageStorage {
72        self.storage.get_or_init(|| {
73            PackageStorage::new(
74                self.package_cache_path
75                    .clone()
76                    .or_else(|| Some(dirs::cache_dir()?.join(DEFAULT_PACKAGES_SUBDIR).into())),
77                self.package_path
78                    .clone()
79                    .or_else(|| Some(dirs::data_dir()?.join(DEFAULT_PACKAGES_SUBDIR).into())),
80                self.cert_path.clone(),
81                self.notifier.clone(),
82            )
83        })
84    }
85
86    /// Get local path option
87    pub fn local_path(&self) -> Option<ImmutPath> {
88        self.storage().package_path().cloned()
89    }
90
91    /// Get data & cache dir
92    pub fn paths(&self) -> Vec<ImmutPath> {
93        let data_dir = self.storage().package_path().cloned();
94        let cache_dir = self.storage().package_cache_path().cloned();
95        data_dir.into_iter().chain(cache_dir).collect::<Vec<_>>()
96    }
97
98    /// Set list of packages for testing.
99    pub fn test_package_list(&self, f: impl FnOnce() -> Vec<(PackageSpec, Option<EcoString>)>) {
100        self.storage().index.get_or_init(f);
101    }
102}
103
104impl PackageRegistry for HttpRegistry {
105    fn resolve(&self, spec: &PackageSpec) -> Result<ImmutPath, PackageError> {
106        self.storage().prepare_package(spec)
107    }
108
109    fn packages(&self) -> &[(PackageSpec, Option<EcoString>)] {
110        self.storage().download_index()
111    }
112}
113
114/// The default packages sub directory within the package and package cache
115/// paths.
116pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
117
118/// Holds information about where packages should be stored and downloads them
119/// on demand, if possible.
120pub struct PackageStorage {
121    /// The path at which non-local packages should be stored when downloaded.
122    package_cache_path: Option<ImmutPath>,
123    /// The path at which local packages are stored.
124    package_path: Option<ImmutPath>,
125    /// The downloader used for fetching the index and packages.
126    cert_path: Option<ImmutPath>,
127    /// The cached index of the preview namespace.
128    index: OnceLock<Vec<(PackageSpec, Option<EcoString>)>>,
129    notifier: Arc<Mutex<dyn Notifier + Send>>,
130}
131
132impl PackageStorage {
133    /// Creates a new package storage for the given package paths.
134    /// It doesn't fallback directories, thus you can disable the related
135    /// storage by passing `None`.
136    pub fn new(
137        package_cache_path: Option<ImmutPath>,
138        package_path: Option<ImmutPath>,
139        cert_path: Option<ImmutPath>,
140        notifier: Arc<Mutex<dyn Notifier + Send>>,
141    ) -> Self {
142        Self {
143            package_cache_path,
144            package_path,
145            cert_path,
146            notifier,
147            index: OnceLock::new(),
148        }
149    }
150
151    /// Returns the path at which non-local packages should be stored when
152    /// downloaded.
153    pub fn package_cache_path(&self) -> Option<&ImmutPath> {
154        self.package_cache_path.as_ref()
155    }
156
157    /// Returns the path at which local packages are stored.
158    pub fn package_path(&self) -> Option<&ImmutPath> {
159        self.package_path.as_ref()
160    }
161
162    /// Make a package available in the on-disk cache.
163    pub fn prepare_package(&self, spec: &PackageSpec) -> PackageResult<ImmutPath> {
164        let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version);
165
166        if let Some(packages_dir) = &self.package_path {
167            let dir = packages_dir.join(&subdir);
168            if dir.exists() {
169                return Ok(dir.into());
170            }
171        }
172
173        if let Some(cache_dir) = &self.package_cache_path {
174            let dir = cache_dir.join(&subdir);
175            if dir.exists() {
176                return Ok(dir.into());
177            }
178
179            // Download from network if it doesn't exist yet.
180            if spec.namespace == "preview" {
181                self.download_package(spec, &dir)?;
182                if dir.exists() {
183                    return Ok(dir.into());
184                }
185            }
186        }
187
188        Err(PackageError::NotFound(spec.clone()))
189    }
190
191    /// Try to determine the latest version of a package.
192    pub fn determine_latest_version(
193        &self,
194        spec: &VersionlessPackageSpec,
195    ) -> StrResult<PackageVersion> {
196        if spec.namespace == "preview" {
197            // For `@preview`, download the package index and find the latest
198            // version.
199            self.download_index()
200                .iter()
201                .filter(|(package, _)| package.name == spec.name)
202                .map(|(package, _)| package.version)
203                .max()
204                .ok_or_else(|| eco_format!("failed to find package {spec}"))
205        } else {
206            // For other namespaces, search locally. We only search in the data
207            // directory and not the cache directory, because the latter is not
208            // intended for storage of local packages.
209            let subdir = format!("{}/{}", spec.namespace, spec.name);
210            self.package_path
211                .iter()
212                .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
213                .flatten()
214                .filter_map(|entry| entry.ok())
215                .map(|entry| entry.path())
216                .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
217                .max()
218                .ok_or_else(|| eco_format!("please specify the desired version"))
219        }
220    }
221
222    /// Get the cached package index without network access.
223    pub fn cached_index(&self) -> Option<&[(PackageSpec, Option<EcoString>)]> {
224        self.index.get().map(Vec::as_slice)
225    }
226
227    /// Download the package index. The result of this is cached for efficiency.
228    pub fn download_index(&self) -> &[(PackageSpec, Option<EcoString>)] {
229        self.index.get_or_init(|| {
230            let url = format!("{DEFAULT_REGISTRY}/preview/index.json");
231
232            threaded_http(&url, self.cert_path.as_deref(), |resp| {
233                let reader = match resp.and_then(|r| r.error_for_status()) {
234                    Ok(response) => response,
235                    Err(err) => {
236                        // todo: silent error
237                        log::error!("Failed to fetch package index: {err} from {url}");
238                        return vec![];
239                    }
240                };
241
242                #[derive(serde::Deserialize)]
243                struct RemotePackageIndex {
244                    name: EcoString,
245                    version: PackageVersion,
246                    description: Option<EcoString>,
247                }
248
249                let indices: Vec<RemotePackageIndex> = match serde_json::from_reader(reader) {
250                    Ok(index) => index,
251                    Err(err) => {
252                        log::error!("Failed to parse package index: {err} from {url}");
253                        return vec![];
254                    }
255                };
256
257                indices
258                    .into_iter()
259                    .map(|index| {
260                        (
261                            PackageSpec {
262                                namespace: "preview".into(),
263                                name: index.name,
264                                version: index.version,
265                            },
266                            index.description,
267                        )
268                    })
269                    .collect::<Vec<_>>()
270            })
271            .unwrap_or_default()
272        })
273    }
274
275    /// Download a package over the network.
276    ///
277    /// # Panics
278    /// Panics if the package spec namespace isn't `preview`.
279    pub fn download_package(&self, spec: &PackageSpec, package_dir: &Path) -> PackageResult<()> {
280        assert_eq!(spec.namespace, "preview");
281
282        let url = format!(
283            "{DEFAULT_REGISTRY}/preview/{}-{}.tar.gz",
284            spec.name, spec.version
285        );
286
287        self.notifier.lock().downloading(spec);
288        threaded_http(&url, self.cert_path.as_deref(), |resp| {
289            let reader = match resp.and_then(|r| r.error_for_status()) {
290                Ok(response) => response,
291                Err(err) if matches!(err.status().map(|s| s.as_u16()), Some(404)) => {
292                    return Err(PackageError::NotFound(spec.clone()));
293                }
294                Err(err) => return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))),
295            };
296
297            let decompressed = flate2::read::GzDecoder::new(reader);
298            tar::Archive::new(decompressed)
299                .unpack(package_dir)
300                .map_err(|err| {
301                    std::fs::remove_dir_all(package_dir).ok();
302                    PackageError::MalformedArchive(Some(eco_format!("{err}")))
303                })
304        })
305        .ok_or_else(|| PackageError::Other(Some(eco_format!("cannot spawn http thread"))))?
306    }
307}
308
309pub(crate) fn threaded_http<T: Send + Sync>(
310    url: &str,
311    cert_path: Option<&Path>,
312    f: impl FnOnce(Result<Response, reqwest::Error>) -> T + Send + Sync,
313) -> Option<T> {
314    std::thread::scope(|s| {
315        s.spawn(move || {
316            let client_builder = reqwest::blocking::Client::builder();
317
318            let client = if let Some(cert_path) = cert_path {
319                let cert = std::fs::read(cert_path)
320                    .ok()
321                    .and_then(|buf| Certificate::from_pem(&buf).ok());
322                if let Some(cert) = cert {
323                    client_builder.add_root_certificate(cert).build().unwrap()
324                } else {
325                    client_builder.build().unwrap()
326                }
327            } else {
328                client_builder.build().unwrap()
329            };
330
331            f(client.get(url).send())
332        })
333        .join()
334        .ok()
335    })
336}
tinymist_package/registry/http.rs

tinymist_package/registry/
http.rs
