Add deterministic archive generation and checksum verification tests

cli/src/main/java/com/devonfw/tools/ide/io/FileAccessImpl.java

@@ -47,9 +47,11 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipParameters;
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -971,7 +973,9 @@ public void compressTar(Path dir, OutputStream out, TarCompression tarCompressio
   @Override
   public void compressTarGz(Path dir, OutputStream out) {
 
-    try (GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(out)) {
+    GzipParameters parameters = new GzipParameters();
+    parameters.setModificationTime(0);
+    try (GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(out, parameters)) {
       compressTarOrThrow(dir, gzOut);
     } catch (IOException e) {
       throw new IllegalStateException("Failed to compress directory " + dir + " to tar.gz file.", e);
@@ -1019,15 +1023,15 @@ public void compressZip(Path dir, OutputStream out) {
 
   private <E extends ArchiveEntry> void compressRecursive(Path path, ArchiveOutputStream<E> out, String relativePath) {
 
-    try (Stream<Path> childStream = Files.list(path)) {
+    try (Stream<Path> childStream = Files.list(path).sorted()) {
       Iterator<Path> iterator = childStream.iterator();
       while (iterator.hasNext()) {
         Path child = iterator.next();
         String relativeChildPath = relativePath + "/" + child.getFileName().toString();
         boolean isDirectory = Files.isDirectory(child);
         E archiveEntry = out.createArchiveEntry(child, relativeChildPath);
+        FileTime none = FileTime.fromMillis(0);
         if (archiveEntry instanceof TarArchiveEntry tarEntry) {
-          FileTime none = FileTime.fromMillis(0);
           tarEntry.setCreationTime(none);
           tarEntry.setModTime(none);
           tarEntry.setLastAccessTime(none);
@@ -1038,6 +1042,11 @@ private <E extends ArchiveEntry> void compressRecursive(Path path, ArchiveOutput
           tarEntry.setGroupName("group");
           PathPermissions filePermissions = getFilePermissions(child);
           tarEntry.setMode(filePermissions.toMode());
+        } else if (archiveEntry instanceof ZipArchiveEntry zipEntry) {
+          zipEntry.setCreationTime(none);
+          zipEntry.setLastAccessTime(none);
+          zipEntry.setLastModifiedTime(none);
+          zipEntry.setTime(none);
         }
         out.putArchiveEntry(archiveEntry);
         if (!isDirectory) {

cli/src/test/java/com/devonfw/tools/ide/context/MvnRepositoryMock.java

@@ -10,12 +10,20 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
 import java.util.stream.Stream;
 
 import com.devonfw.tools.ide.tool.mvn.MvnArtifact;
 import com.devonfw.tools.ide.tool.mvn.MvnArtifactMetadata;
 import com.devonfw.tools.ide.tool.mvn.MvnRepository;
 import com.devonfw.tools.ide.url.model.file.UrlChecksums;
+import com.devonfw.tools.ide.url.model.file.UrlGenericChecksum;
+import com.devonfw.tools.ide.util.HexUtil;
 import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
 
 /**
@@ -25,6 +33,22 @@ public class MvnRepositoryMock extends MvnRepository {
 
   private final WireMockRuntimeInfo wmRuntimeInfo;
 
+  /**
+   * Maps artifact download path to its pre-computed SHA-256 checksum.
+   * Populated when the mock archive is compressed, queried during verification.
+   */
+  private final Map<String, String> checksumByPath = new HashMap<>();
+
+  /**
+   * Registers an expected SHA-256 checksum for a given artifact path.
+   *
+   * @param path the artifact path (relative to repo root, e.g. "/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip").
+   * @param sha256 the expected SHA-256 hash.
+   */
+  public void putExpectedChecksum(String path, String sha256) {
+    this.checksumByPath.put(path, sha256);
+  }
+
   /**
    * The constructor.
    *
@@ -47,6 +71,15 @@ public Path download(MvnArtifactMetadata metadata) {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream(1024)) {
       this.context.getFileAccess().compress(archiveFolder, baos, artifact.getFilename());
       byte[] body = baos.toByteArray();
+      // Pre-compute and store the SHA-256 of the archive bytes so verifyChecksum() can check them.
+      // We use putIfAbsent so that a hardcoded 'expected' checksum from a test takes precedence.
+      String sha256;
+      try {
+        sha256 = HexUtil.toHexString(MessageDigest.getInstance("SHA-256").digest(body));
+      } catch (NoSuchAlgorithmException e) {
+        throw new IllegalStateException("SHA-256 algorithm not found", e);
+      }
+      this.checksumByPath.putIfAbsent(path, sha256);
       stubFor(get(urlPathEqualTo(path)).willReturn(
           aResponse().withStatus(200).withBody(body)));
     } catch (IOException e) {
@@ -56,8 +89,30 @@ public Path download(MvnArtifactMetadata metadata) {
   }
 
   @Override
-  protected UrlChecksums getChecksums(MvnArtifact artifact) {
-    return null;
+  public UrlChecksums getChecksums(MvnArtifact artifact) {
+    String path = artifact.getDownloadUrl().replace(MvnRepositoryMock.MAVEN_CENTRAL, "");
+    String sha256 = this.checksumByPath.get(path);
+    if (sha256 == null) {
+      // checksum not yet computed (e.g. metadata requests) – skip verification
+      return null;
+    }
+    return new SingleChecksumWrapper(sha256);
+  }
+
+
+  /**
+   * Simple {@link UrlChecksums} wrapper for a single pre-computed checksum.
+   */
+  private record SingleChecksumWrapper(String sha256) implements UrlChecksums {
+
+    @Override
+    public Iterator<UrlGenericChecksum> iterator() {
+      UrlGenericChecksum entry = new UrlGenericChecksum() {
+        @Override public String getChecksum() { return sha256; }
+        @Override public String getHashAlgorithm() { return "SHA-256"; }
+      };
+      return Collections.singletonList(entry).iterator();
+    }
   }
 
   @Override

cli/src/test/java/com/devonfw/tools/ide/io/ArchiveDeterminismTest.java

@@ -0,0 +1,93 @@
+package com.devonfw.tools.ide.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import com.devonfw.tools.ide.context.AbstractIdeContextTest;
+import com.devonfw.tools.ide.context.IdeTestContext;
+
+/**
+ * Test of archive determinism in {@link FileAccessImpl}.
+ */
+public class ArchiveDeterminismTest extends AbstractIdeContextTest {
+
+  @TempDir
+  Path tempDir;
+
+  /**
+   * Test that {@link FileAccessImpl#compressTarGz(Path, OutputStream)} is deterministic.
+   *
+   * @throws IOException if an I/O error occurs.
+   */
+  @Test
+  public void testTarGzDeterminism() throws IOException, InterruptedException {
+
+    // arrange
+    IdeTestContext context = new IdeTestContext();
+    FileAccessImpl fileAccess = new FileAccessImpl(context);
+    Path contentDir = this.tempDir.resolve("content");
+    Files.createDirectories(contentDir);
+    Files.writeString(contentDir.resolve("file1.txt"), "Content 1");
+    Path binDir = contentDir.resolve("bin");
+    Files.createDirectories(binDir);
+    Files.writeString(binDir.resolve("script.sh"), "#!/bin/bash\necho hello");
+
+    Path archive1 = this.tempDir.resolve("archive1.tar.gz");
+    Path archive2 = this.tempDir.resolve("archive2.tar.gz");
+
+    // act
+    try (OutputStream out1 = Files.newOutputStream(archive1)) {
+      fileAccess.compressTarGz(contentDir, out1);
+    }
+    // Modify modification time of a file to ensure it would affect the hash if not normalized
+    Path file1 = contentDir.resolve("file1.txt");
+    FileTime newTime = FileTime.fromMillis(System.currentTimeMillis() + 10000);
+    Files.setLastModifiedTime(file1, newTime);
+    try (OutputStream out2 = Files.newOutputStream(archive2)) {
+      fileAccess.compressTarGz(contentDir, out2);
+    }
+
+    // assert
+    assertThat(archive1).hasSameBinaryContentAs(archive2);
+  }
+
+  /**
+   * Test that {@link FileAccessImpl#compressZip(Path, OutputStream)} is deterministic.
+   *
+   * @throws IOException if an I/O error occurs.
+   */
+  @Test
+  public void testZipDeterminism() throws IOException, InterruptedException {
+
+    // arrange
+    IdeTestContext context = new IdeTestContext();
+    FileAccessImpl fileAccess = new FileAccessImpl(context);
+    Path contentDir = this.tempDir.resolve("content-zip");
+    Files.createDirectories(contentDir);
+    Files.writeString(contentDir.resolve("file1.txt"), "Content 1");
+
+    Path archive1 = this.tempDir.resolve("archive1.zip");
+    Path archive2 = this.tempDir.resolve("archive2.zip");
+
+    // act
+    try (OutputStream out1 = Files.newOutputStream(archive1)) {
+      fileAccess.compressZip(contentDir, out1);
+    }
+    // Modify modification time of a file to ensure it would affect the hash if not normalized
+    Path file1 = contentDir.resolve("file1.txt");
+    FileTime newTime = FileTime.fromMillis(System.currentTimeMillis() + 10000);
+    Files.setLastModifiedTime(file1, newTime);
+    try (OutputStream out2 = Files.newOutputStream(archive2)) {
+      fileAccess.compressZip(contentDir, out2);
+    }
+
+    // assert
+    assertThat(archive1).hasSameBinaryContentAs(archive2);
+  }
+}

cli/src/test/java/com/devonfw/tools/ide/tool/mvn/MvnFinalChecksumTest.java

@@ -0,0 +1,49 @@
+package com.devonfw.tools.ide.tool.mvn;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.Test;
+
+import com.devonfw.tools.ide.context.AbstractIdeContextTest;
+import com.devonfw.tools.ide.context.IdeTestContext;
+import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
+import com.github.tomakehurst.wiremock.junit5.WireMockTest;
+
+/**
+ * Integration test verifying that the deterministic archives produced by our mock repositories
+ * match our hardcoded "Gold Standard" values.
+ */
+@WireMockTest
+class MvnFinalChecksumTest extends AbstractIdeContextTest {
+
+  private static final String PROJECT_MVN = "mvn";
+
+
+
+  /**
+   * Integration test verifying that the tool installation correctly performs and validates 
+   * the checksum from the URL repository (urls.sha256).
+   */
+  @Test
+  void testVerifyUrlChecksum(WireMockRuntimeInfo wmRuntimeInfo) throws IOException {
+
+    // 1. Arrange: Use the "mvn" project context with WireMock
+    IdeTestContext context = newContext(PROJECT_MVN, wmRuntimeInfo);
+    Mvn mvn = context.getCommandletManager().getCommandlet(Mvn.class);
+
+    // Read the expected hash from the URL repository file in the test resources
+    Path urlsSha256Path = context.getIdePath().resolve("urls/mvn/mvn/3.9.7/urls.sha256");
+    String expectedSha256 = Files.readString(urlsSha256Path).trim();
+
+    // 2. Act: Trigger the installation of Maven 3.9.7
+    // This will use ToolRepositoryMock which triggers deterministic compression
+    // and verifies it against urls.sha256 in the test resources.
+    mvn.install();
+
+    // 3. Assert: Verify the installation and checksum success logs
+    assertThat(context).logAtSuccess().hasMessageContaining("Successfully installed mvn in version 3.9.7");
+    assertThat(context).logAtSuccess().hasMessageContaining("SHA-256 checksum " + expectedSha256 + " is correct.");
+  }
+}

cli/src/test/java/com/devonfw/tools/ide/tool/repository/ChecksumVerificationTest.java

@@ -0,0 +1,104 @@
+package com.devonfw.tools.ide.tool.repository;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import com.devonfw.tools.ide.cli.CliException;
+import com.devonfw.tools.ide.context.AbstractIdeContextTest;
+import com.devonfw.tools.ide.context.IdeTestContext;
+import com.devonfw.tools.ide.url.model.file.UrlGenericChecksum;
+
+/**
+ * Test of checksum verification in {@link AbstractToolRepository}.
+ */
+public class ChecksumVerificationTest extends AbstractIdeContextTest {
+
+  @TempDir
+  Path tempDir;
+
+  /**
+   * Test {@link AbstractToolRepository#verifyChecksum(Path, UrlGenericChecksum)} with matching checksum.
+   *
+   * @throws IOException if an I/O error occurs.
+   */
+  @Test
+  public void testVerifyChecksumMatching() throws IOException {
+
+    // arrange
+    IdeTestContext context = newContext(PROJECT_BASIC);
+    AbstractToolRepository repo = new DefaultToolRepository(context);
+    Path file = this.tempDir.resolve("testfile.txt");
+    String content = "Hello World";
+    Files.writeString(file, content);
+    String checksum = context.getFileAccess().checksum(file, "SHA-256");
+    UrlGenericChecksum expectedChecksum = new TestUrlGenericChecksum(checksum, "SHA-256");
+
+    // act & assert
+    assertDoesNotThrow(() -> {
+      repo.verifyChecksum(file, expectedChecksum);
+    });
+  }
+
+  /**
+   * Test {@link AbstractToolRepository#verifyChecksum(Path, UrlGenericChecksum)} with mismatching checksum.
+   *
+   * @throws IOException if an I/O error occurs.
+   */
+  @Test
+  public void testVerifyChecksumMismatch() throws IOException {
+
+    // arrange
+    IdeTestContext context = newContext(PROJECT_BASIC);
+    AbstractToolRepository repo = new DefaultToolRepository(context);
+    Path file = this.tempDir.resolve("testfile.txt");
+    String content = "Hello World";
+    Files.writeString(file, content);
+    String wrongChecksum = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; // SHA-256 of empty string
+    UrlGenericChecksum expectedChecksum = new TestUrlGenericChecksum(wrongChecksum, "SHA-256");
+
+    // act & assert
+    CliException e = assertThrows(CliException.class, () -> {
+      repo.verifyChecksum(file, expectedChecksum);
+    });
+    assertThat(e).hasMessageContaining("has the wrong SHA-256 checksum");
+    assertThat(e).hasMessageContaining("Expected " + wrongChecksum);
+  }
+
+  private static class TestUrlGenericChecksum implements UrlGenericChecksum {
+
+    private final String checksum;
+
+    private final String algorithm;
+
+    public TestUrlGenericChecksum(String checksum, String algorithm) {
+
+      this.checksum = checksum;
+      this.algorithm = algorithm;
+    }
+
+    @Override
+    public String getChecksum() {
+
+      return this.checksum;
+    }
+
+    @Override
+    public String getHashAlgorithm() {
+
+      return this.algorithm;
+    }
+
+    @Override
+    public String toString() {
+
+      return this.checksum;
+    }
+  }
+}

cli/src/test/resources/ide-projects/mvn/_ide/urls/mvn/mvn/3.9.7/urls.sha256

@@ -0,0 +1 @@
+e3aac9e671f88f0e14dcf8240a5301778fd335e14b8752d15fa2f28b1a51ab32