feat(handler): add support for MSI files

Extracts MSIs using 7z with custom CFBF header parsing to compute the
full archive size.

Works on both vanilla and padded MSI files. This could be
migrated to a fully Python-based implementation in the future using:

 * https://github.com/nightlark/pymsi
 * https://github.com/decalage2/olefile

As of v0.47, olefile does not handle padded MSIs properly so we
re-implement CFBF header parsing and compute the archive size ourselves.

Author: jcrussell

docs/handlers.md

@@ -6,6 +6,7 @@
     cab,
     cpio,
     dmg,
+    msi,
     par2,
     partclone,
     rar,
@@ -89,6 +90,7 @@
     arc.ARCHandler,
     arj.ARJHandler,
     cab.CABHandler,
+    msi.MsiHandler,
     tar.TarUstarHandler,
     tar.TarUnixHandler,
     cpio.PortableASCIIHandler,

python/unblob/handlers/__init__.py

@@ -0,0 +1,130 @@
+import io
+import struct
+from typing import Optional
+
+from structlog import get_logger
+
+from unblob.extractors import Command
+
+from ...file_utils import InvalidInputFormat
+from ...models import (
+    File,
+    HandlerDoc,
+    HandlerType,
+    HexString,
+    Reference,
+    StructHandler,
+    ValidChunk,
+)
+
+logger = get_logger()
+
+
+class MsiHandler(StructHandler):
+    NAME = "msi"
+
+    PATTERNS = [HexString("D0 CF 11 E0 A1 B1 1A E1")]
+    C_DEFINITIONS = r"""
+        typedef struct cfbf_header
+        {
+                                        // [offset from start (bytes), length (bytes)]
+            uint8 signature[8];         // [00H,08] {0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1,
+                                        // 0x1a, 0xe1} for current version
+            uint8 clsid[16];            // [08H,16] reserved must be zero (WriteClassStg/
+                                        // GetClassFile uses root directory class id)
+            uint16 minorVersion;        // [18H,02] minor version of the format: 33 is
+                                        // written by reference implementation
+            uint16 dllVersion;          // [1AH,02] major version of the dll/format: 3 for
+                                        // 512-byte sectors, 4 for 4 KB sectors
+            uint16 byteOrder;           // [1CH,02] 0xFFFE: indicates Intel byte-ordering
+            uint16 sectorShift;         // [1EH,02] size of sectors in power-of-two;
+                                        // typically 9 indicating 512-byte sectors
+            uint16 miniSectorShift;     // [20H,02] size of mini-sectors in power-of-two;
+                                        // typically 6 indicating 64-byte mini-sectors
+            uint16 reserved;            // [22H,02] reserved, must be zero
+            uint32 reserved1;           // [24H,04] reserved, must be zero
+            uint32 csectDir;            // [28H,04] must be zero for 512-byte sectors,
+                                        // number of SECTs in directory chain for 4 KB
+                                        // sectors
+            uint32 csectFat;            // [2CH,04] number of SECTs in the FAT chain
+            uint32 sectDirStart;        // [30H,04] first SECT in the directory chain
+            uint32 txSignature;         // [34H,04] signature used for transactions; must
+                                        // be zero. The reference implementation
+                                        // does not support transactions
+            uint32 miniSectorCutoff;    // [38H,04] maximum size for a mini stream;
+                                        // typically 4096 bytes
+            uint32 sectMiniFatStart;    // [3CH,04] first SECT in the MiniFAT chain
+            uint32 csectMiniFat;        // [40H,04] number of SECTs in the MiniFAT chain
+            uint32 sectDifStart;        // [44H,04] first SECT in the DIFAT chain
+            uint32 csectDif;            // [48H,04] number of SECTs in the DIFAT chain
+            uint32 sectFat[109];        // [4CH,436] the SECTs of first 109 FAT sectors
+         } cfbf_header_t;
+    """
+    HEADER_STRUCT = "cfbf_header_t"
+
+    EXTRACTOR = Command("7z", "x", "-p", "-y", "{inpath}", "-o{outdir}")
+
+    DOC = HandlerDoc(
+        name="MSI",
+        description="Microsoft Installer (MSI) files are used for the installation, maintenance, and removal of software.",
+        handler_type=HandlerType.ARCHIVE,
+        vendor="Microsoft",
+        references=[
+            Reference(
+                title="MSI File Format Documentation",
+                url="https://docs.microsoft.com/en-us/windows/win32/msi/overview-of-windows-installer",
+            ),
+            Reference(
+                title="Compound File Binary Format",
+                url="https://en.wikipedia.org/wiki/Compound_File_Binary_Format",
+            ),
+        ],
+        limitations=[
+            "Limited to CFB based extraction, not full-on MSI extraction",
+            "Extracted files have names coming from CFB internal representation, and may not correspond to the one they would have on disk after running the installer",
+        ],
+    )
+
+    def calculate_chunk(self, file: File, start_offset: int) -> Optional[ValidChunk]:
+        file.seek(start_offset, io.SEEK_SET)
+        header = self.parse_header(file)
+
+        # Size of MSI is based on the maximum used sector. Need to walk the
+        # DIFAT entries and find the maximum used sector to compute the size.
+        sector_size = 2**header.sectorShift
+        entries_per_sector = sector_size // 4
+
+        max_used_sector = 0
+
+        for sector_id, sect in enumerate(header.sectFat):
+            # skip empty
+            if sect == 0xFFFFFFFF:
+                continue
+
+            sector_offset = start_offset + 512 + sect * sector_size
+            if sector_offset > file.size():
+                raise InvalidInputFormat("Invalid MSI file, sector offset too large")
+            file.seek(sector_offset, io.SEEK_SET)
+            raw_sector = file.read(sector_size)
+            entries = struct.unpack(f"<{entries_per_sector}I", raw_sector)
+
+            base_sector_id = sector_id * entries_per_sector
+            for entry_id in range(len(entries) - 1, -1, -1):
+                if entries[entry_id] == 0xFFFFFFFF:
+                    continue
+
+                # Found the highest id  on this page
+                max_id = base_sector_id + entry_id
+
+                max_used_sector = max(max_used_sector, max_id)
+
+                # Once we have found the first non-empty element, we are done
+                # with all IDs in this sector
+                break
+
+        total_size = 512 + ((max_used_sector + 1) * sector_size)
+
+        return ValidChunk(
+            start_offset=start_offset,
+            end_offset=start_offset + total_size,
+        )

python/unblob/handlers/archive/msi.py

@@ -54,7 +54,8 @@
 DEFAULT_PROCESS_NUM = multiprocessing.cpu_count()
 DEFAULT_SKIP_MAGIC = (
     "BFLT",
-    "Composite Document File V2 Document",
+    # Disabled for MSI files
+    # "Composite Document File V2 Document",
     "Erlang BEAM file",
     "GIF",
     "GNU message catalog",

python/unblob/processing.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dce9e456ace76b969fe0fe4d228bf096662c11d2376d99a9210f6364428a94c4
+size 1563648

tests/integration/archive/msi/__input__/7z2501.msi

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da8f4120ab4ffacb19067a26f6a8b2695e00ec19bcc48ff694349c62df1b330b
+size 1563680

tests/integration/archive/msi/__input__/7z2501.msi.padded

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa8e5036d973688f1e8622fbe9ab22e037346e0def0197bf5e7cdf37da4e223d
+size 3831808

tests/integration/archive/msi/__input__/putty-64bit-0.83-installer.msi

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12c87c542e1d4a39b47f176ffa5fd1691c98e5f9d502e6e46573962fb77c4510
+size 3831840

tests/integration/archive/msi/__input__/putty-64bit-0.83-installer.msi.padded

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb
+size 16

tests/integration/archive/msi/__output__/7z2501.msi.padded_extract/0-16.padding

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb
+size 16