Implemented zip/tar/tar.zstd compression

Heliozoa · Heliozoa · commit 2ed47d8fcfcd · 2022-04-14T07:17:11.000+03:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bindings/tmc-langs-node/jest/tmc.test.ts b/bindings/tmc-langs-node/jest/tmc.test.ts
@@ -138,7 +138,7 @@ test("compresses project", async () => {
 
   const dir = await mockExercise();
   expect(fs.existsSync([dir, "output.zip"].join("/"))).toBeFalsy();
-  tmc.compressProject(dir, [dir, "output.zip"].join("/"), "zip");
+  tmc.compressProject(dir, [dir, "output.zip"].join("/"), "zip", false);
   expect(fs.existsSync([dir, "output.zip"].join("/"))).toBeTruthy();
 });
 
diff --git a/bindings/tmc-langs-node/src/lib.rs b/bindings/tmc-langs-node/src/lib.rs
@@ -100,11 +100,12 @@ fn compress_project(mut cx: FunctionContext) -> JsResult<JsValue> {
         cx,
         exercise_path: PathBuf,
         output_path: PathBuf,
-        compression: Compression
+        compression: Compression,
+        naive: bool
     );
     lock!(cx, exercise_path);
 
-    let res = tmc_langs::compress_project_to(&exercise_path, &output_path, compression);
+    let res = tmc_langs::compress_project_to(&exercise_path, &output_path, compression, naive);
     convert_res(&mut cx, res)
 }
 
diff --git a/bindings/tmc-langs-node/ts/functions.d.ts b/bindings/tmc-langs-node/ts/functions.d.ts
@@ -14,7 +14,7 @@ export function checkstyle(
   locale: string
 ): types.StyleValidationResult | null;
 export function clean(exercisePath: string): void;
-export function compressProject(exercisePath: string, outputPath: string, compression: Compression): void;
+export function compressProject(exercisePath: string, outputPath: string, compression: Compression, naive: boolean): void;
 export function extractProject(archivePath: string, outputPath: string): void;
 export function fastAvailablePoints(exercisePath: string): Array<string>;
 export function findExercises(exercisePath: string): Array<string>;
diff --git a/bindings/tmc-langs-node/ts/tmc.ts b/bindings/tmc-langs-node/ts/tmc.ts
@@ -38,8 +38,8 @@ export class Tmc {
     return tmc.clean(exercisePath);
   }
 
-  compressProject(exercisePath: string, outputPath: string, compression: Compression): void {
-    return tmc.compressProject(exercisePath, outputPath, compression);
+  compressProject(exercisePath: string, outputPath: string, compression: Compression, naive: boolean): void {
+    return tmc.compressProject(exercisePath, outputPath, compression, naive);
   }
 
   extractProject(archivePath: string, outputPath: string): void {
diff --git a/tmc-client/src/tmc_client.rs b/tmc-client/src/tmc_client.rs
@@ -19,7 +19,7 @@ use std::{
     time::Duration,
     u32,
 };
-use tmc_langs_plugins::Language;
+use tmc_langs_plugins::{Compression, Language};
 use tmc_langs_util::progress_reporter;
 #[cfg(feature = "ts")]
 use ts_rs::TS;
@@ -265,7 +265,7 @@ impl TmcClient {
 
         // compress
         start_stage(2, "Compressing paste submission...", None);
-        let compressed = tmc_langs_plugins::compress_project_to_zip(submission_path)?;
+        let compressed = tmc_langs_plugins::compress_project(submission_path, Compression::Zip)?;
         progress_stage(
             "Compressed paste submission. Posting paste submission...",
             None,
@@ -333,7 +333,7 @@ impl TmcClient {
         self.require_authentication()?;
 
         start_stage(2, "Compressing submission...", None);
-        let compressed = tmc_langs_plugins::compress_project_to_zip(submission_path)?;
+        let compressed = tmc_langs_plugins::compress_project(submission_path, Compression::Zip)?;
         progress_stage("Compressed submission. Posting submission...", None);
 
         let result = api_v8::core::submit_exercise(
@@ -556,7 +556,7 @@ impl TmcClient {
     ) -> Result<NewSubmission, ClientError> {
         self.require_authentication()?;
 
-        let compressed = tmc_langs_plugins::compress_project_to_zip(submission_path)?;
+        let compressed = tmc_langs_plugins::compress_project(submission_path, Compression::Zip)?;
         let review = if let Some(message) = message_for_reviewer {
             ReviewData::WithMessage(message)
         } else {
diff --git a/tmc-langs-cli/src/lib.rs b/tmc-langs-cli/src/lib.rs
@@ -221,7 +221,7 @@ fn run_app(matches: Opt) -> Result<()> {
             naive,
         } => {
             file_util::lock!(exercise_path);
-            tmc_langs::compress_project_to(&exercise_path, &output_path, compression)?;
+            tmc_langs::compress_project_to(&exercise_path, &output_path, compression, naive)?;
             OutputKind::finished(format!(
                 "compressed project from {} to {}",
                 exercise_path.display(),
diff --git a/tmc-langs-framework/src/error.rs b/tmc-langs-framework/src/error.rs
@@ -16,8 +16,12 @@ pub enum TmcError {
     ZipName(String),
     #[error("Failed to read tar archive")]
     TarRead(#[source] std::io::Error),
+    #[error("Failed to write tar archive")]
+    TarWrite(#[source] std::io::Error),
     #[error("Failed to read zstd archive")]
     ZstdRead(#[source] std::io::Error),
+    #[error("Failed to write zstd archive")]
+    ZstdWrite(#[source] std::io::Error),
 
     #[error("Failed to read line")]
     ReadLine(#[source] std::io::Error),
diff --git a/tmc-langs-plugins/Cargo.toml b/tmc-langs-plugins/Cargo.toml
@@ -19,9 +19,11 @@ tmc-langs-r = { path = "../plugins/r" }
 
 impl-enum = "0.2.0"
 log = "0.4.14"
+tar = "0.4.38"
 thiserror = "1.0.30"
 walkdir = "2.3.2"
 zip = "0.6.2"
+zstd = "0.10.0"  # zip 0.6 is still on 0.10
 
 [dev-dependencies]
 simple_logger = "2.1.0"
diff --git a/tmc-langs-plugins/src/archive.rs b/tmc-langs-plugins/src/archive.rs
@@ -0,0 +1,79 @@
+use std::{
+    io::{self, Cursor, Seek, Write},
+    path::Path,
+};
+use tar::Builder;
+use tmc_langs_framework::{Compression, TmcError};
+use tmc_langs_util::file_util;
+use zip::{write::FileOptions, ZipWriter};
+use zstd::Encoder;
+
+pub enum ArchiveBuilder<W: Write + Seek> {
+    Tar(Builder<W>),
+    TarZstd(W, Builder<Cursor<Vec<u8>>>),
+    Zip(ZipWriter<W>),
+}
+
+impl<W: Write + Seek> ArchiveBuilder<W> {
+    pub fn new(writer: W, compression: Compression) -> Self {
+        match compression {
+            Compression::Tar => Self::Tar(Builder::new(writer)),
+            Compression::TarZstd => Self::TarZstd(writer, Builder::new(Cursor::new(vec![]))),
+            Compression::Zip => Self::Zip(ZipWriter::new(writer)),
+        }
+    }
+
+    pub fn add_directory(&mut self, path: &str) -> Result<(), TmcError> {
+        log::trace!("adding directory {}", path);
+        match self {
+            Self::Tar(builder) => {
+                builder
+                    .append_dir_all(path, path)
+                    .map_err(TmcError::TarWrite)?;
+            }
+            Self::TarZstd(_, builder) => {
+                builder
+                    .append_dir_all(path, path)
+                    .map_err(TmcError::TarWrite)?;
+            }
+            Self::Zip(builder) => {
+                builder.add_directory(path, FileOptions::default().unix_permissions(0o755))?
+            }
+        }
+        Ok(())
+    }
+
+    pub fn add_file(&mut self, source: &Path, target: &str) -> Result<(), TmcError> {
+        log::trace!("writing file {} as {}", source.display(), target);
+        match self {
+            Self::Tar(builder) => builder
+                .append_path_with_name(source, target)
+                .map_err(TmcError::TarWrite)?,
+            Self::TarZstd(_, builder) => builder
+                .append_path_with_name(source, target)
+                .map_err(TmcError::TarWrite)?,
+            Self::Zip(builder) => {
+                let bytes = file_util::read_file(source)?;
+                builder.start_file(target, FileOptions::default().unix_permissions(0o755))?;
+                builder
+                    .write_all(&bytes)
+                    .map_err(|e| TmcError::ZipWrite(source.into(), e))?;
+            }
+        }
+        Ok(())
+    }
+
+    pub fn finish(self) -> Result<W, TmcError> {
+        let res = match self {
+            Self::Tar(builder) => builder.into_inner().map_err(TmcError::TarWrite)?,
+            Self::TarZstd(writer, builder) => {
+                let mut tar_data = builder.into_inner().map_err(TmcError::TarWrite)?;
+                let mut encoder = Encoder::new(writer, 0).map_err(TmcError::ZstdWrite)?;
+                io::copy(&mut tar_data, &mut encoder).map_err(TmcError::ZstdWrite)?;
+                encoder.finish().map_err(TmcError::ZstdWrite)?
+            }
+            Self::Zip(mut builder) => builder.finish()?,
+        };
+        Ok(res)
+    }
+}
diff --git a/tmc-langs-plugins/src/compression.rs b/tmc-langs-plugins/src/compression.rs
@@ -1,21 +1,23 @@
 //! Contains functions for zipping and unzipping projects.
 
+use crate::archive::ArchiveBuilder;
 use std::{
-    io::{Cursor, Read, Seek, Write},
+    io::{Cursor, Read, Seek},
     path::{Path, PathBuf},
 };
-use tmc_langs_framework::{StudentFilePolicy, TmcError};
+use tmc_langs_framework::{Compression, StudentFilePolicy, TmcError};
 use tmc_langs_util::file_util;
 use walkdir::{DirEntry, WalkDir};
 pub use zip::result::ZipError;
-use zip::{write::FileOptions, ZipArchive, ZipWriter};
+use zip::ZipArchive;
 
 /// Zips the given directory, only including student files according to the given policy.
-pub fn zip_student_files<P: StudentFilePolicy>(
+pub fn compress_student_files<P: StudentFilePolicy>(
     policy: P,
     root_directory: &Path,
+    compression: Compression,
 ) -> Result<Vec<u8>, TmcError> {
-    let mut writer = ZipWriter::new(Cursor::new(vec![]));
+    let mut writer = ArchiveBuilder::new(Cursor::new(vec![]), compression);
 
     for entry in WalkDir::new(root_directory)
         .into_iter()
@@ -34,21 +36,9 @@ pub fn zip_student_files<P: StudentFilePolicy>(
                 })
                 .unwrap_or_else(|| entry.path());
             if entry.path().is_dir() {
-                log::trace!("adding directory {}", path.display());
-                writer.add_directory(
-                    path_to_zip_compatible_string(path),
-                    FileOptions::default().unix_permissions(0o755),
-                )?;
+                writer.add_directory(&path_to_zip_compatible_string(path))?;
             } else {
-                let bytes = file_util::read_file(entry.path())?;
-                log::trace!("writing file {}", path.display());
-                writer.start_file(
-                    path_to_zip_compatible_string(path),
-                    FileOptions::default().unix_permissions(0o755),
-                )?;
-                writer
-                    .write_all(&bytes)
-                    .map_err(|e| TmcError::ZipWrite(path.to_path_buf(), e))?;
+                writer.add_file(entry.path(), &path_to_zip_compatible_string(path))?;
             }
         }
     }
@@ -222,8 +212,12 @@ mod test {
         File::create(missing_file_path).unwrap();
 
         let path = temp.path().join("exercise-name");
-        let zipped =
-            zip_student_files(EverythingIsStudentFilePolicy::new(&path).unwrap(), &path).unwrap();
+        let zipped = compress_student_files(
+            EverythingIsStudentFilePolicy::new(&path).unwrap(),
+            &path,
+            Compression::Zip,
+        )
+        .unwrap();
         let mut archive = ZipArchive::new(Cursor::new(zipped)).unwrap();
         assert!(!archive.is_empty());
         for i in 0..archive.len() {
diff --git a/tmc-langs-plugins/src/lib.rs b/tmc-langs-plugins/src/lib.rs
@@ -1,18 +1,20 @@
 #![deny(clippy::print_stdout, clippy::print_stderr, clippy::unwrap_used)]
 
+pub mod archive;
+pub mod compression;
 mod error;
-pub mod tmc_zip;
 
 pub use error::PluginError;
 use std::{
     io::{Read, Seek},
     path::{Path, PathBuf},
 };
 use tmc_langs_csharp::CSharpPlugin;
-use tmc_langs_framework::{Archive, Compression, LanguagePlugin, TmcError, TmcProjectYml};
+use tmc_langs_framework::{Archive, LanguagePlugin, TmcError, TmcProjectYml};
 pub use tmc_langs_framework::{
-    ExerciseDesc, ExercisePackagingConfiguration, Language, NothingIsStudentFilePolicy, RunResult,
-    StudentFilePolicy, StyleValidationResult, StyleValidationStrategy,
+    Compression, ExerciseDesc, ExercisePackagingConfiguration, Language,
+    NothingIsStudentFilePolicy, RunResult, StudentFilePolicy, StyleValidationResult,
+    StyleValidationStrategy,
 };
 pub use tmc_langs_java::{AntPlugin, MavenPlugin};
 pub use tmc_langs_make::MakePlugin;
@@ -45,49 +47,48 @@ pub fn extract_project_overwrite(
     compressed_project: impl std::io::Read + std::io::Seek,
     target_location: &Path,
 ) -> Result<(), PluginError> {
-    tmc_zip::unzip(compressed_project, target_location)?;
+    compression::unzip(compressed_project, target_location)?;
     Ok(())
 }
 
-pub fn compress_project(path: &Path, compression: Compression) -> Result<Vec<u8>, PluginError> {
-    match compression {
-        Compression::Zip => compress_project_to_zip(path),
-        Compression::Tar => todo!(),
-        Compression::TarZstd => todo!(),
-    }
-}
-
 /// See `LanguagePlugin::compress_project`.
 // TODO: clean up
-pub fn compress_project_to_zip(path: &Path) -> Result<Vec<u8>, PluginError> {
+pub fn compress_project(path: &Path, compression: Compression) -> Result<Vec<u8>, PluginError> {
     match get_language_plugin_type(path) {
-        Some(PluginType::CSharp) => Ok(tmc_zip::zip_student_files(
+        Some(PluginType::CSharp) => Ok(compression::compress_student_files(
             <CSharpPlugin as LanguagePlugin>::StudentFilePolicy::new(path)?,
             path,
+            compression,
         )?),
-        Some(PluginType::Make) => Ok(tmc_zip::zip_student_files(
+        Some(PluginType::Make) => Ok(compression::compress_student_files(
             <MakePlugin as LanguagePlugin>::StudentFilePolicy::new(path)?,
             path,
+            compression,
         )?),
-        Some(PluginType::Maven) => Ok(tmc_zip::zip_student_files(
+        Some(PluginType::Maven) => Ok(compression::compress_student_files(
             <MavenPlugin as LanguagePlugin>::StudentFilePolicy::new(path)?,
             path,
+            compression,
         )?),
-        Some(PluginType::NoTests) => Ok(tmc_zip::zip_student_files(
+        Some(PluginType::NoTests) => Ok(compression::compress_student_files(
             <NoTestsPlugin as LanguagePlugin>::StudentFilePolicy::new(path)?,
             path,
+            compression,
         )?),
-        Some(PluginType::Python3) => Ok(tmc_zip::zip_student_files(
+        Some(PluginType::Python3) => Ok(compression::compress_student_files(
             <Python3Plugin as LanguagePlugin>::StudentFilePolicy::new(path)?,
             path,
+            compression,
         )?),
-        Some(PluginType::R) => Ok(tmc_zip::zip_student_files(
+        Some(PluginType::R) => Ok(compression::compress_student_files(
             <RPlugin as LanguagePlugin>::StudentFilePolicy::new(path)?,
             path,
+            compression,
         )?),
-        Some(PluginType::Ant) => Ok(tmc_zip::zip_student_files(
+        Some(PluginType::Ant) => Ok(compression::compress_student_files(
             <AntPlugin as LanguagePlugin>::StudentFilePolicy::new(path)?,
             path,
+            compression,
         )?),
         None => Err(PluginError::PluginNotFound(path.to_path_buf())),
     }
diff --git a/tmc-langs/src/lib.rs b/tmc-langs/src/lib.rs

Original file line number	Diff line number	Diff line change
`@@ -38,8 +38,8 @@ export class Tmc {`
`38`	`38`	`return tmc.clean(exercisePath);`
`39`	`39`	`}`
`40`	`40`
`41`		`- compressProject(exercisePath: string, outputPath: string, compression: Compression): void {`
`42`		`- return tmc.compressProject(exercisePath, outputPath, compression);`
	`41`	`+ compressProject(exercisePath: string, outputPath: string, compression: Compression, naive: boolean): void {`
	`42`	`+ return tmc.compressProject(exercisePath, outputPath, compression, naive);`
`43`	`43`	`}`
`44`	`44`
`45`	`45`	`extractProject(archivePath: string, outputPath: string): void {`