Improvements to archives handling

Cargo.toml

@@ -41,6 +41,7 @@ indicatif = "0.16"
 env_logger = { version = "0.10", optional = true }
 structopt = { version = "0.3", optional = true }
 color-eyre = { version = "0.6", optional = true }
+infer = "0.15.0"
 
 [features]
 default = ["default-tls"]

src/archives.rs

@@ -12,10 +12,12 @@ pub(crate) enum ArchiveFormat {
 
 impl ArchiveFormat {
     /// Parse archive type from resource extension.
-    pub(crate) fn parse_from_extension(resource: &str) -> Result<Self, Error> {
-        if resource.ends_with(".tar.gz") {
+    pub(crate) fn parse_from_path<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
+        let ext = infer::get_from_path(path).unwrap().unwrap().extension();
+        // Here we assume that gz contain a tar inside it.
+        if ext == "gz" {
             Ok(Self::TarGz)
-        } else if resource.ends_with(".zip") {
+        } else if ext == "zip" {
             Ok(Self::Zip)
         } else {
             Err(Error::ExtractionError("unsupported archive format".into()))
@@ -34,13 +36,13 @@ pub(crate) fn extract_archive<P: AsRef<Path>>(
 
     match format {
         ArchiveFormat::TarGz => {
-            let tar_gz = File::open(path)?;
+            let tar_gz = File::open(&path)?;
             let tar = GzDecoder::new(tar_gz);
             let mut archive = tar::Archive::new(tar);
             archive.unpack(&temp_target)?;
         }
         ArchiveFormat::Zip => {
-            let file = File::open(path)?;
+            let file = File::open(&path)?;
             let mut archive =
                 zip::ZipArchive::new(file).map_err(|e| Error::ExtractionError(e.to_string()))?;
             archive
@@ -49,6 +51,7 @@ pub(crate) fn extract_archive<P: AsRef<Path>>(
         }
     };
 
+    fs::remove_file(&path)?;
     // Now rename the temp directory to the final target directory.
     fs::rename(temp_target, target)?;
 

src/cache.rs

@@ -8,6 +8,7 @@ use std::default::Default;
 use std::env;
 use std::fs::{self, OpenOptions};
 use std::path::{Path, PathBuf};
+use std::str::FromStr;
 use std::thread;
 use std::time::{self, Duration};
 use tempfile::NamedTempFile;
@@ -154,13 +155,16 @@ pub struct Options {
     pub subdir: Option<String>,
     /// Automatically extract the resource, assuming the resource is an archive.
     pub extract: bool,
+    /// An optional subdirectory (relative to the cache root) to extract the resource in.
+    pub extract_dir: Option<String>,
 }
 
 impl Options {
-    pub fn new(subdir: Option<&str>, extract: bool) -> Self {
+    pub fn new(subdir: Option<&str>, extract: bool, extract_dir: Option<&str>) -> Self {
         Self {
             subdir: subdir.map(String::from),
             extract,
+            extract_dir: extract_dir.map(String::from),
         }
     }
 
@@ -280,16 +284,25 @@ impl Cache {
                     .ok()
                     .and_then(|sys_time| sys_time.elapsed().ok())
                     .map(|duration| format!("{}", duration.as_secs()));
-                extraction_dir = Some(self.resource_to_filepath(
-                    resource,
-                    &resource_last_modified,
-                    options.subdir.as_deref(),
-                    Some("-extracted"),
-                ));
+
+                extraction_dir = if let Some(extract_dir) = &options.extract_dir {
+                    Some(cached_path.join(extract_dir))
+                } else {
+                    Some(self.resource_to_filepath(
+                        resource,
+                        &resource_last_modified,
+                        options.subdir.as_deref(),
+                        Some("-extracted"),
+                    ))
+                };
             }
         } else {
             // This is a remote resource, so fetch it to the cache.
-            let meta = self.fetch_remote_resource(resource, options.subdir.as_deref())?;
+            let meta = self.fetch_remote_resource(
+                resource,
+                options.subdir.as_deref(),
+                options.extract_dir.as_deref(),
+            )?;
 
             // Check if we need to extract.
             if options.extract {
@@ -319,7 +332,7 @@ impl Cache {
 
             if !dirpath.is_dir() {
                 info!("Extracting {} to {:?}", resource, dirpath);
-                let format = ArchiveFormat::parse_from_extension(resource)?;
+                let format = ArchiveFormat::parse_from_path(&cached_path)?;
                 extract_archive(&cached_path, &dirpath, &format)?;
             }
 
@@ -356,11 +369,16 @@ impl Cache {
         resource: &str,
         subdir: Option<&str>,
     ) -> Result<PathBuf, Error> {
-        let options = Options::new(subdir, false);
+        let options = Options::new(subdir, false, None);
         self.cached_path_with_options(resource, &options)
     }
 
-    fn fetch_remote_resource(&self, resource: &str, subdir: Option<&str>) -> Result<Meta, Error> {
+    fn fetch_remote_resource(
+        &self,
+        resource: &str,
+        subdir: Option<&str>,
+        extract_dir: Option<&str>,
+    ) -> Result<Meta, Error> {
         // Otherwise we attempt to parse the URL.
         let url =
             reqwest::Url::parse(resource).map_err(|_| Error::InvalidUrl(String::from(resource)))?;
@@ -417,8 +435,7 @@ impl Cache {
         }
 
         // No up-to-date version cached, so we have to try downloading it.
-        let meta = self.try_download_resource(resource, &url, &path, &etag)?;
-
+        let meta = self.try_download_resource(resource, &url, &path, &etag, extract_dir)?;
         info!("New version of {} cached", resource);
 
         filelock.unlock()?;
@@ -431,7 +448,7 @@ impl Cache {
     fn find_existing(&self, resource: &str, subdir: Option<&str>) -> Vec<Meta> {
         let mut existing_meta: Vec<Meta> = vec![];
         let glob_string = format!(
-            "{}.*.meta",
+            "{}*.meta",
             self.resource_to_filepath(resource, &None, subdir, None)
                 .to_str()
                 .unwrap(),
@@ -461,10 +478,11 @@ impl Cache {
         url: &reqwest::Url,
         path: &Path,
         etag: &Option<String>,
+        extract_dir: Option<&str>,
     ) -> Result<Meta, Error> {
         let mut retries: u32 = 0;
         loop {
-            match self.download_resource(resource, url, path, etag) {
+            match self.download_resource(resource, url, path, etag, extract_dir) {
                 Ok(meta) => {
                     return Ok(meta);
                 }
@@ -495,6 +513,7 @@ impl Cache {
         url: &reqwest::Url,
         path: &Path,
         etag: &Option<String>,
+        extract_dir: Option<&str>,
     ) -> Result<Meta, Error> {
         debug!("Attempting connection to {}", url);
 
@@ -535,6 +554,7 @@ impl Cache {
             path.into(),
             etag.clone(),
             self.freshness_lifetime,
+            extract_dir.map(|s| PathBuf::from_str(s).unwrap()),
         );
         meta.to_file()?;
 

src/meta.rs

@@ -14,6 +14,8 @@ pub(crate) struct Meta {
     pub(crate) resource_path: PathBuf,
     /// Path to the serialized meta.
     pub(crate) meta_path: PathBuf,
+    /// Path to the directory that the resource is extracted.
+    pub(crate) extraction_path: Option<PathBuf>,
     /// The ETAG of the resource from the time it was cached, if there was one.
     pub(crate) etag: Option<String>,
     /// Time that the freshness of this cached resource will expire.
@@ -28,6 +30,7 @@ impl Meta {
         resource_path: PathBuf,
         etag: Option<String>,
         freshness_lifetime: Option<u64>,
+        extraction_path: Option<PathBuf>,
     ) -> Meta {
         let mut expires: Option<f64> = None;
         let creation_time = now();
@@ -42,6 +45,7 @@ impl Meta {
             etag,
             expires,
             creation_time,
+            extraction_path,
         }
     }
 
@@ -54,11 +58,15 @@ impl Meta {
     }
 
     pub(crate) fn get_extraction_path(&self) -> PathBuf {
-        let dirname = format!(
-            "{}-extracted",
-            self.resource_path.file_name().unwrap().to_str().unwrap()
-        );
-        self.resource_path.parent().unwrap().join(dirname)
+        if let Some(extraction_path) = &self.extraction_path {
+            self.resource_path.parent().unwrap().join(extraction_path)
+        } else {
+            let dirname = format!(
+                "{}-extracted",
+                self.resource_path.file_name().unwrap().to_str().unwrap()
+            );
+            self.resource_path.parent().unwrap().join(dirname)
+        }
     }
 
     pub(crate) fn to_file(&self) -> Result<(), Error> {