diff --git a/ext/lzma_sdk_wrapper/cpp_bridge.cpp b/ext/lzma_sdk_wrapper/cpp_bridge.cpp index 76acbee..c266574 100644 --- a/ext/lzma_sdk_wrapper/cpp_bridge.cpp +++ b/ext/lzma_sdk_wrapper/cpp_bridge.cpp @@ -9,7 +9,7 @@ #include #include -// C++ API includes for password support +// C++ API includes #include "lzma_sdk/CPP/7zip/Archive/IArchive.h" #include "lzma_sdk/CPP/7zip/Common/FileStreams.h" #include "lzma_sdk/CPP/7zip/Common/StreamObjects.h" @@ -19,10 +19,11 @@ // 7z handler creation function (from ArchiveExports.cpp) STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject); -// Helper function to initialize 7z format GUID -// GUID format: {23170F69-40C1-278A-1000-000110070000} -// The last byte before the two zeros (0x07) identifies the 7z format -static void Init7zFormatGUID(GUID* guid) { +// Helper function to initialize a format GUID based on format constant +// GUID format: {23170F69-40C1-278A-1000-0001100X0000} +// The byte at Data4[5] identifies the format: +// 0x07 = 7z, 0x01 = Zip, 0xEE = Tar +static void InitFormatGUID(GUID* guid, int format) { memset(guid, 0, sizeof(GUID)); guid->Data1 = 0x23170F69; guid->Data2 = 0x40C1; @@ -32,9 +33,22 @@ static void Init7zFormatGUID(GUID* guid) { guid->Data4[2] = 0x00; guid->Data4[3] = 0x01; guid->Data4[4] = 0x10; - guid->Data4[5] = 0x07; // Format ID: 7z guid->Data4[6] = 0x00; guid->Data4[7] = 0x00; + + // Map our format constants to SDK format IDs + switch (format) { + case SZ_FORMAT_ZIP: + guid->Data4[5] = 0x01; // kId_Zip + break; + case SZ_FORMAT_TAR: + guid->Data4[5] = 0xEE; // kId_Tar + break; + case SZ_FORMAT_7Z: + default: + guid->Data4[5] = 0x07; // kId_7z + break; + } } // Memory-based input stream for reading archives from memory buffers @@ -133,37 +147,14 @@ namespace LzmaSdkWrapper { class ArchiveReader::Impl { public: - CFileInStream archiveStream; - CLookToRead2 lookStream; - CSzArEx db; - ISzAlloc allocImp; - ISzAlloc allocTempImp; bool isOpen; - UInt16 *tempBuf; - size_t tempBufSize; - static const size_t kInputBufSize = (1 << 18); // 256KB buffer - - Impl() : isOpen(false), tempBuf(nullptr), tempBufSize(0) { - allocImp.Alloc = SzAlloc; - allocImp.Free = SzFree; - allocTempImp.Alloc = SzAllocTemp; - allocTempImp.Free = SzFreeTemp; - lookStream.buf = nullptr; - } - - ~Impl() { - if (tempBuf) { - ISzAlloc_Free(&allocImp, tempBuf); - } - if (lookStream.buf) { - ISzAlloc_Free(&allocImp, lookStream.buf); - } - } + Impl() : isOpen(false) {} }; ArchiveReader::ArchiveReader() : pImpl(new Impl()) // Use new directly for C++11 compatibility + , archiveFormat(SZ_FORMAT_7Z) , progressCallback(nullptr) , progressUserData(nullptr) , isMemoryBased(false) @@ -178,96 +169,58 @@ ArchiveReader::~ArchiveReader() { int ArchiveReader::open(const char* path, int format) { try { - // Store archive path for C++ API use + // Store archive path and format for later use archivePath = path; + archiveFormat = format; - // If password is set, use C++ API from the start for better performance - if (!password.empty()) { - - // Initialize 7z format GUID - GUID CLSID_Format7z; - Init7zFormatGUID(&CLSID_Format7z); - - // Create 7z archive handler - CMyComPtr archive; - HRESULT hr = CreateArchiver(&CLSID_Format7z, &IID_IInArchive, (void**)&archive); - if (hr != S_OK || !archive) { - errorMessage = "Failed to create 7z handler"; - return SZW_ERROR_FAIL; - } - - // Open file stream - CInFileStream *fileSpec = new CInFileStream; - CMyComPtr file = fileSpec; - - FString archivePathFS = us2fs(GetUnicodeString(path)); - if (!fileSpec->Open(archivePathFS)) { - errorMessage = "Failed to open archive file"; - return SZW_ERROR_OPEN_FAILED; - } - - // Create open callback with password - CArchiveOpenCallback *openCallbackSpec = new CArchiveOpenCallback; - CMyComPtr openCallback(openCallbackSpec); - openCallbackSpec->SetPassword(password.c_str()); - - // Open archive - const UInt64 scanSize = 1 << 23; - hr = archive->Open(file, &scanSize, openCallback); - if (hr != S_OK) { - char errBuf[256]; - snprintf(errBuf, sizeof(errBuf), "Failed to open password-protected archive (HRESULT: 0x%08X)", (unsigned int)hr); - errorMessage = errBuf; - return SZW_ERROR_OPEN_FAILED; - } - - // Cache the archive and stream for extraction - archive->AddRef(); - fileArchive = archive.Detach(); - - file->AddRef(); - fileStream = file.Detach(); + // Initialize format GUID based on the requested format + GUID CLSID_Format; + InitFormatGUID(&CLSID_Format, format); - // Also initialize the C API db for entry count queries - // (We'll get entry count from C++ API instead) - pImpl->isOpen = true; - - return SZW_OK; + // Create archive handler for the requested format + CMyComPtr archive; + HRESULT hr = CreateArchiver(&CLSID_Format, &IID_IInArchive, (void**)&archive); + if (hr != S_OK || !archive) { + errorMessage = "Failed to create archive handler for format"; + return SZW_ERROR_FAIL; } - // For unencrypted archives, use fast C API + // Open file stream + CInFileStream *fileSpec = new CInFileStream; + CMyComPtr file = fileSpec; - if (InFile_Open(&pImpl->archiveStream.file, path)) { - errorMessage = "Failed to open file"; + FString archivePathFS = us2fs(GetUnicodeString(path)); + if (!fileSpec->Open(archivePathFS)) { + errorMessage = "Failed to open archive file"; return SZW_ERROR_OPEN_FAILED; } - FileInStream_CreateVTable(&pImpl->archiveStream); - LookToRead2_CreateVTable(&pImpl->lookStream, False); - - // Allocate buffer for LookToRead2 - pImpl->lookStream.buf = (Byte *)ISzAlloc_Alloc(&pImpl->allocImp, pImpl->kInputBufSize); - if (!pImpl->lookStream.buf) { - File_Close(&pImpl->archiveStream.file); - errorMessage = "Failed to allocate read buffer"; - return SZW_ERROR_OUTOFMEMORY; + // Create open callback with password + CArchiveOpenCallback *openCallbackSpec = new CArchiveOpenCallback; + CMyComPtr openCallback(openCallbackSpec); + if (!password.empty()) { + openCallbackSpec->SetPassword(password.c_str()); } - pImpl->lookStream.bufSize = pImpl->kInputBufSize; - pImpl->lookStream.realStream = &pImpl->archiveStream.vt; - LookToRead2_INIT(&pImpl->lookStream); - - CrcGenerateTable(); - SzArEx_Init(&pImpl->db); - - SRes res = SzArEx_Open(&pImpl->db, &pImpl->lookStream.vt, &pImpl->allocImp, &pImpl->allocTempImp); - if (res != SZ_OK) { - File_Close(&pImpl->archiveStream.file); - errorMessage = "Failed to open archive"; + // Open archive + const UInt64 scanSize = 1 << 23; + hr = archive->Open(file, &scanSize, openCallback); + if (hr != S_OK) { + char errBuf[256]; + snprintf(errBuf, sizeof(errBuf), "Failed to open archive (HRESULT: 0x%08X)", (unsigned int)hr); + errorMessage = errBuf; return SZW_ERROR_OPEN_FAILED; } + // Cache the archive and stream for extraction + archive->AddRef(); + fileArchive = archive.Detach(); + + file->AddRef(); + fileStream = file.Detach(); + pImpl->isOpen = true; + return SZW_OK; } catch (const std::exception& e) { @@ -283,18 +236,21 @@ int ArchiveReader::openFromMemory(const uint8_t* data, size_t size, int format) return SZW_ERROR_INVALIDARG; } + // Store the format for later use + archiveFormat = format; + // Copy buffer data (we need to own it for the lifetime of the reader) memoryBuffer.assign(data, data + size); - // Initialize 7z format GUID - GUID CLSID_Format7z; - Init7zFormatGUID(&CLSID_Format7z); + // Initialize format GUID based on the requested format + GUID CLSID_Format; + InitFormatGUID(&CLSID_Format, format); - // Create 7z archive handler + // Create archive handler for the requested format CMyComPtr archive; - HRESULT hr = CreateArchiver(&CLSID_Format7z, &IID_IInArchive, (void**)&archive); + HRESULT hr = CreateArchiver(&CLSID_Format, &IID_IInArchive, (void**)&archive); if (hr != S_OK || !archive) { - errorMessage = "Failed to create 7z handler"; + errorMessage = "Failed to create archive handler for format"; return SZW_ERROR_FAIL; } @@ -337,7 +293,7 @@ int ArchiveReader::openFromMemory(const uint8_t* data, size_t size, int format) void ArchiveReader::close() { if (pImpl->isOpen) { - // Release cached C++ file archive if present + // Release file-based archive if (fileArchive) { static_cast(fileArchive)->Release(); fileArchive = nullptr; @@ -347,19 +303,16 @@ void ArchiveReader::close() { fileStream = nullptr; } + // Release memory-based archive if (isMemoryBased) { - // Release memory-based archive if (memoryArchive) { static_cast(memoryArchive)->Release(); memoryArchive = nullptr; } memoryBuffer.clear(); isMemoryBased = false; - } else if (!fileArchive) { - // Release file-based C API archive (only if we didn't use C++ API) - SzArEx_Free(&pImpl->db, &pImpl->allocImp); - File_Close(&pImpl->archiveStream.file); } + pImpl->isOpen = false; } } @@ -369,22 +322,20 @@ uint32_t ArchiveReader::getEntryCount() { return 0; } - // Check if using C++ API (memory-based or file-based with password) IInArchive* archive = nullptr; if (isMemoryBased) { archive = static_cast(memoryArchive); - } else if (fileArchive) { + } else { archive = static_cast(fileArchive); } - if (archive) { - UInt32 numItems = 0; - archive->GetNumberOfItems(&numItems); - return numItems; + if (!archive) { + return 0; } - // Fall back to C API - return pImpl->db.NumFiles; + UInt32 numItems = 0; + archive->GetNumberOfItems(&numItems); + return numItems; } int ArchiveReader::getEntryInfo(uint32_t index, SzEntryInfo* info) { @@ -392,157 +343,82 @@ int ArchiveReader::getEntryInfo(uint32_t index, SzEntryInfo* info) { return SZW_ERROR_INVALIDARG; } - // Check if using C++ API (memory-based or file-based with password) IInArchive* archive = nullptr; if (isMemoryBased) { archive = static_cast(memoryArchive); - } else if (fileArchive) { + } else { archive = static_cast(fileArchive); } - if (archive) { - // Handle archives using C++ COM API - - UInt32 numItems = 0; - archive->GetNumberOfItems(&numItems); - if (index >= numItems) { - return SZW_ERROR_INVALIDARG; - } - - try { - // Get file path - NWindows::NCOM::CPropVariant prop; - - HRESULT hr = archive->GetProperty(index, kpidPath, &prop); - if (hr == S_OK && prop.vt == VT_BSTR) { - // Convert BSTR (UTF-16) to UTF-8 - const wchar_t* wstr = prop.bstrVal; - size_t maxLen = sizeof(info->name) - 1; - for (size_t i = 0; i < maxLen && wstr[i] != 0; i++) { - if (wstr[i] < 128) { - info->name[i] = (char)wstr[i]; - } else { - info->name[i] = '?'; - } - info->name[i + 1] = '\0'; - } - } else { - info->name[0] = '\0'; - } - - // Get uncompressed size - hr = archive->GetProperty(index, kpidSize, &prop); - if (hr == S_OK && prop.vt == VT_UI8) { - info->size = prop.uhVal.QuadPart; - } else { - info->size = 0; - } - - // Get compressed size - hr = archive->GetProperty(index, kpidPackSize, &prop); - if (hr == S_OK && prop.vt == VT_UI8) { - info->compressed_size = prop.uhVal.QuadPart; - } else { - info->compressed_size = 0; - } - - // Get directory flag - hr = archive->GetProperty(index, kpidIsDir, &prop); - if (hr == S_OK && prop.vt == VT_BOOL) { - info->is_directory = (prop.boolVal != VARIANT_FALSE); - } else { - info->is_directory = false; - } - - // Get CRC - hr = archive->GetProperty(index, kpidCRC, &prop); - if (hr == S_OK && prop.vt == VT_UI4) { - info->crc = prop.ulVal; - } else { - info->crc = 0; - } - - // Get modified time - hr = archive->GetProperty(index, kpidMTime, &prop); - if (hr == S_OK && prop.vt == VT_FILETIME) { - // Convert FILETIME to Unix timestamp - UInt64 winTime = ((UInt64)prop.filetime.dwHighDateTime << 32) | prop.filetime.dwLowDateTime; - if (winTime > 116444736000000000ULL) { - info->modified_time = (winTime - 116444736000000000ULL) / 10000000ULL; - } else { - info->modified_time = 0; - } - } else { - info->modified_time = 0; - } - - // Get attributes - hr = archive->GetProperty(index, kpidAttrib, &prop); - if (hr == S_OK && prop.vt == VT_UI4) { - info->attributes = prop.ulVal; - } else { - info->attributes = 0; - } - - return SZW_OK; - - } catch (const std::exception& e) { - errorMessage = e.what(); - return SZW_ERROR_FAIL; - } + if (!archive) { + errorMessage = "No archive open"; + return SZW_ERROR_FAIL; } - // Handle file-based archives using C API - if (index >= pImpl->db.NumFiles) { + UInt32 numItems = 0; + archive->GetNumberOfItems(&numItems); + if (index >= numItems) { return SZW_ERROR_INVALIDARG; } try { - /* Get filename */ - size_t len = SzArEx_GetFileNameUtf16(&pImpl->db, index, nullptr); - if (len > pImpl->tempBufSize) { - ISzAlloc_Free(&pImpl->allocImp, pImpl->tempBuf); - pImpl->tempBufSize = len; - pImpl->tempBuf = (UInt16 *)ISzAlloc_Alloc(&pImpl->allocImp, pImpl->tempBufSize * sizeof(UInt16)); - if (!pImpl->tempBuf) { - pImpl->tempBufSize = 0; - return SZW_ERROR_OUTOFMEMORY; + // Get file path + NWindows::NCOM::CPropVariant prop; + + HRESULT hr = archive->GetProperty(index, kpidPath, &prop); + if (hr == S_OK && prop.vt == VT_BSTR) { + // Convert BSTR (UTF-16) to UTF-8 + const wchar_t* wstr = prop.bstrVal; + size_t maxLen = sizeof(info->name) - 1; + for (size_t i = 0; i < maxLen && wstr[i] != 0; i++) { + if (wstr[i] < 128) { + info->name[i] = (char)wstr[i]; + } else { + info->name[i] = '?'; + } + info->name[i + 1] = '\0'; } + } else { + info->name[0] = '\0'; } - SzArEx_GetFileNameUtf16(&pImpl->db, index, pImpl->tempBuf); + // Get uncompressed size + hr = archive->GetProperty(index, kpidSize, &prop); + if (hr == S_OK && prop.vt == VT_UI8) { + info->size = prop.uhVal.QuadPart; + } else { + info->size = 0; + } - /* Convert UTF-16 to UTF-8 (simplified) */ - char* namePtr = info->name; - size_t maxLen = sizeof(info->name) - 1; - for (size_t i = 0; i < len && i < maxLen; i++) { - if (pImpl->tempBuf[i] < 128) { - *namePtr++ = (char)pImpl->tempBuf[i]; - } else { - *namePtr++ = '?'; /* Simplified: replace non-ASCII with ? */ - } + // Get compressed size + hr = archive->GetProperty(index, kpidPackSize, &prop); + if (hr == S_OK && prop.vt == VT_UI8) { + info->compressed_size = prop.uhVal.QuadPart; + } else { + info->compressed_size = 0; } - *namePtr = '\0'; - /* Fill in other fields from CSzArEx structure */ - info->size = SzArEx_GetFileSize(&pImpl->db, index); - info->compressed_size = 0; /* Not easily available in 7z SDK */ - info->is_directory = SzArEx_IsDir(&pImpl->db, index) ? true : false; + // Get directory flag + hr = archive->GetProperty(index, kpidIsDir, &prop); + if (hr == S_OK && prop.vt == VT_BOOL) { + info->is_directory = (prop.boolVal != VARIANT_FALSE); + } else { + info->is_directory = false; + } - /* Get CRC if available */ - if (SzBitWithVals_Check(&pImpl->db.CRCs, index)) { - info->crc = pImpl->db.CRCs.Vals[index]; + // Get CRC + hr = archive->GetProperty(index, kpidCRC, &prop); + if (hr == S_OK && prop.vt == VT_UI4) { + info->crc = prop.ulVal; } else { info->crc = 0; } - /* Get modified time if available */ - if (SzBitWithVals_Check(&pImpl->db.MTime, index)) { - CNtfsFileTime *ft = &pImpl->db.MTime.Vals[index]; - /* Convert Windows FILETIME to Unix timestamp (simplified) */ - UInt64 winTime = ((UInt64)ft->High << 32) | ft->Low; - /* Windows epoch is 1601-01-01, Unix epoch is 1970-01-01 */ - /* 116444736000000000 is the difference in 100ns intervals */ + // Get modified time + hr = archive->GetProperty(index, kpidMTime, &prop); + if (hr == S_OK && prop.vt == VT_FILETIME) { + // Convert FILETIME to Unix timestamp + UInt64 winTime = ((UInt64)prop.filetime.dwHighDateTime << 32) | prop.filetime.dwLowDateTime; if (winTime > 116444736000000000ULL) { info->modified_time = (winTime - 116444736000000000ULL) / 10000000ULL; } else { @@ -552,9 +428,10 @@ int ArchiveReader::getEntryInfo(uint32_t index, SzEntryInfo* info) { info->modified_time = 0; } - /* Get attributes if available */ - if (SzBitWithVals_Check(&pImpl->db.Attribs, index)) { - info->attributes = pImpl->db.Attribs.Vals[index]; + // Get attributes + hr = archive->GetProperty(index, kpidAttrib, &prop); + if (hr == S_OK && prop.vt == VT_UI4) { + info->attributes = prop.ulVal; } else { info->attributes = 0; } @@ -567,66 +444,21 @@ int ArchiveReader::getEntryInfo(uint32_t index, SzEntryInfo* info) { } } -// Helper function for C++ API extraction (with password support) +// Helper function for C++ API extraction to file int ArchiveReader::extractToFile_CPP(uint32_t index, const char* output_path) { try { CMyComPtr archive; - // Check if we already have a cached archive open + // Get the cached archive (file-based or memory-based) if (fileArchive) { - // Reuse the cached archive archive = static_cast(fileArchive); - archive->AddRef(); // AddRef since CMyComPtr will Release on destruction - } else { - // First time opening - create and cache the archive - // Initialize 7z format GUID - GUID CLSID_Format7z; - Init7zFormatGUID(&CLSID_Format7z); - - // Create 7z archive handler - HRESULT hr = CreateArchiver(&CLSID_Format7z, &IID_IInArchive, (void**)&archive); - if (hr != S_OK || !archive) { - errorMessage = "Failed to create 7z handler"; - return SZW_ERROR_FAIL; - } - - // Open file stream - CInFileStream *fileSpec = new CInFileStream; - CMyComPtr file = fileSpec; - - // Convert path to FString (platform-appropriate string type) - FString archivePathFS = us2fs(GetUnicodeString(archivePath.c_str())); - if (!fileSpec->Open(archivePathFS)) { - errorMessage = "Failed to open archive file"; - return SZW_ERROR_OPEN_FAILED; - } - - // Create open callback with password - CArchiveOpenCallback *openCallbackSpec = new CArchiveOpenCallback; - CMyComPtr openCallback(openCallbackSpec); - - if (!password.empty()) { - openCallbackSpec->SetPassword(password.c_str()); - } - - // Open archive - const UInt64 scanSize = 1 << 23; - hr = archive->Open(file, &scanSize, openCallback); - if (hr != S_OK) { - char errBuf[256]; - snprintf(errBuf, sizeof(errBuf), "Failed to open archive with C++ API (HRESULT: 0x%08X)", (unsigned int)hr); - errorMessage = errBuf; - return SZW_ERROR_OPEN_FAILED; - } - - // Cache the opened archive and stream for reuse archive->AddRef(); - fileArchive = archive.Detach(); - archive = static_cast(fileArchive); - archive->AddRef(); // AddRef for the local CMyComPtr - - file->AddRef(); - fileStream = file.Detach(); + } else if (isMemoryBased && memoryArchive) { + archive = static_cast(memoryArchive); + archive->AddRef(); + } else { + errorMessage = "No archive open"; + return SZW_ERROR_FAIL; } // Create extract callback with password @@ -672,86 +504,12 @@ int ArchiveReader::extractToFile(uint32_t index, const char* output_path) { return SZW_ERROR_FAIL; } - if (index >= pImpl->db.NumFiles) { - errorMessage = "Index out of range"; - return SZW_ERROR_INVALIDARG; - } - if (!output_path) { errorMessage = "Output path is null"; return SZW_ERROR_INVALIDARG; } - // Check if this is a directory - if (SzArEx_IsDir(&pImpl->db, index)) { - errorMessage = "Cannot extract directory to file"; - return SZW_ERROR_INVALIDARG; - } - - // If we have a cached C++ archive (password-protected), use it directly - if (fileArchive) { - return extractToFile_CPP(index, output_path); - } - - // Otherwise use C API for unencrypted files - try { - UInt32 blockIndex = 0xFFFFFFFF; // Start with invalid block index - Byte *outBuffer = nullptr; - size_t outBufferSize = 0; - size_t offset = 0; - size_t outSizeProcessed = 0; - - // Extract file to memory buffer - SRes res = SzArEx_Extract(&pImpl->db, &pImpl->lookStream.vt, index, - &blockIndex, &outBuffer, &outBufferSize, - &offset, &outSizeProcessed, - &pImpl->allocImp, &pImpl->allocTempImp); - - if (res != SZ_OK) { - errorMessage = "Failed to extract file"; - if (outBuffer) { - ISzAlloc_Free(&pImpl->allocImp, outBuffer); - } - return SZW_ERROR_EXTRACT_FAILED; - } - - // Open output file - CSzFile outFile; - WRes wres = OutFile_Open(&outFile, output_path); - if (wres != 0) { - errorMessage = "Failed to create output file"; - if (outBuffer) { - ISzAlloc_Free(&pImpl->allocImp, outBuffer); - } - return SZW_ERROR_WRITE_FAILED; - } - - // Write extracted data to file - size_t processedSize = outSizeProcessed; - wres = File_Write(&outFile, outBuffer + offset, &processedSize); - - // Free the buffer - ISzAlloc_Free(&pImpl->allocImp, outBuffer); - - if (wres != 0 || processedSize != outSizeProcessed) { - File_Close(&outFile); - errorMessage = "Failed to write output file"; - return SZW_ERROR_WRITE_FAILED; - } - - // Close the output file - wres = File_Close(&outFile); - if (wres != 0) { - errorMessage = "Failed to close output file"; - return SZW_ERROR_WRITE_FAILED; - } - - return SZW_OK; - - } catch (const std::exception& e) { - errorMessage = e.what(); - return SZW_ERROR_FAIL; - } + return extractToFile_CPP(index, output_path); } int ArchiveReader::extractToMemory(uint32_t index, uint8_t** buffer, size_t* size) { @@ -761,56 +519,18 @@ int ArchiveReader::extractToMemory(uint32_t index, uint8_t** buffer, size_t* siz } try { - // Initialize 7z format GUID - GUID CLSID_Format7z; - Init7zFormatGUID(&CLSID_Format7z); - - // Create 7z archive handler CMyComPtr archive; - HRESULT hr = CreateArchiver(&CLSID_Format7z, &IID_IInArchive, (void**)&archive); - if (hr != S_OK || !archive) { - errorMessage = "Failed to create 7z handler"; - return SZW_ERROR_FAIL; - } - // Open stream (file or memory) - CMyComPtr stream; - - if (isMemoryBased) { - // Create memory stream from stored buffer - CMemoryInStream *memStreamSpec = new CMemoryInStream(memoryBuffer.data(), memoryBuffer.size()); - stream = memStreamSpec; + // Get the cached archive (file-based or memory-based) + if (fileArchive) { + archive = static_cast(fileArchive); + archive->AddRef(); + } else if (isMemoryBased && memoryArchive) { + archive = static_cast(memoryArchive); + archive->AddRef(); } else { - // Open file stream - if (index >= pImpl->db.NumFiles) { - errorMessage = "Index out of range"; - return SZW_ERROR_INVALIDARG; - } - - CInFileStream *fileSpec = new CInFileStream; - stream = fileSpec; - - FString archivePathFS = us2fs(GetUnicodeString(archivePath.c_str())); - if (!fileSpec->Open(archivePathFS)) { - errorMessage = "Failed to open archive file"; - return SZW_ERROR_OPEN_FAILED; - } - } - - // Create open callback with password - CArchiveOpenCallback *openCallbackSpec = new CArchiveOpenCallback; - CMyComPtr openCallback(openCallbackSpec); - - if (!password.empty()) { - openCallbackSpec->SetPassword(password.c_str()); - } - - // Open archive - const UInt64 scanSize = 1 << 23; - hr = archive->Open(stream, &scanSize, openCallback); - if (hr != S_OK) { - errorMessage = "Failed to open archive"; - return SZW_ERROR_OPEN_FAILED; + errorMessage = "No archive open"; + return SZW_ERROR_FAIL; } // Create memory output stream @@ -829,7 +549,7 @@ int ArchiveReader::extractToMemory(uint32_t index, uint8_t** buffer, size_t* siz // Extract the single file const UInt32 indices[1] = { index }; - hr = archive->Extract(indices, 1, 0, extractCallback); + HRESULT hr = archive->Extract(indices, 1, 0, extractCallback); if (hr != S_OK) { errorMessage = "Extraction failed"; @@ -870,15 +590,15 @@ int ArchiveReader::verify() { } try { - // Initialize 7z format GUID - GUID CLSID_Format7z; - Init7zFormatGUID(&CLSID_Format7z); + // Initialize format GUID using stored format + GUID CLSID_Format; + InitFormatGUID(&CLSID_Format, archiveFormat); - // Create 7z archive handler + // Create archive handler for the stored format CMyComPtr archive; - HRESULT hr = CreateArchiver(&CLSID_Format7z, &IID_IInArchive, (void**)&archive); + HRESULT hr = CreateArchiver(&CLSID_Format, &IID_IInArchive, (void**)&archive); if (hr != S_OK || !archive) { - errorMessage = "Failed to create 7z handler"; + errorMessage = "Failed to create archive handler"; return SZW_ERROR_FAIL; } @@ -950,55 +670,6 @@ int ArchiveReader::verify() { int ArchiveReader::setPassword(const char* pwd) { if (pwd) { password = pwd; - - // If this is a file-based archive (not memory-based) and we don't have a C++ archive yet, - // open it now with the C++ API for better extraction performance - if (!isMemoryBased && !fileArchive && !archivePath.empty()) { - - // Initialize 7z format GUID - GUID CLSID_Format7z; - Init7zFormatGUID(&CLSID_Format7z); - - // Create 7z archive handler - CMyComPtr archive; - HRESULT hr = CreateArchiver(&CLSID_Format7z, &IID_IInArchive, (void**)&archive); - if (hr != S_OK || !archive) { - errorMessage = "Failed to create 7z handler"; - return SZW_ERROR_FAIL; - } - - // Open file stream - CInFileStream *fileSpec = new CInFileStream; - CMyComPtr file = fileSpec; - - FString archivePathFS = us2fs(GetUnicodeString(archivePath.c_str())); - if (!fileSpec->Open(archivePathFS)) { - errorMessage = "Failed to open archive file"; - return SZW_ERROR_OPEN_FAILED; - } - - // Create open callback with password - CArchiveOpenCallback *openCallbackSpec = new CArchiveOpenCallback; - CMyComPtr openCallback(openCallbackSpec); - openCallbackSpec->SetPassword(password.c_str()); - - // Open archive - const UInt64 scanSize = 1 << 23; - hr = archive->Open(file, &scanSize, openCallback); - if (hr != S_OK) { - char errBuf[256]; - snprintf(errBuf, sizeof(errBuf), "Failed to open password-protected archive (HRESULT: 0x%08X)", (unsigned int)hr); - errorMessage = errBuf; - return SZW_ERROR_OPEN_FAILED; - } - - // Cache the archive and stream for extraction - archive->AddRef(); - fileArchive = archive.Detach(); - - file->AddRef(); - fileStream = file.Detach(); - } } else { password.clear(); } @@ -1066,15 +737,15 @@ int ArchiveWriter::finalize() { } try { - // Initialize 7z format GUID using shared definition - GUID CLSID_Format7z; - Init7zFormatGUID(&CLSID_Format7z); + // Initialize format GUID using the stored format + GUID CLSID_Format; + InitFormatGUID(&CLSID_Format, pImpl->format); - // Create output archive handler + // Create output archive handler for the requested format CMyComPtr outArchive; - HRESULT hr = CreateArchiver(&CLSID_Format7z, &IID_IOutArchive, (void**)&outArchive); + HRESULT hr = CreateArchiver(&CLSID_Format, &IID_IOutArchive, (void**)&outArchive); if (hr != S_OK || !outArchive) { - errorMessage = "Failed to create 7z output handler"; + errorMessage = "Failed to create output archive handler"; pImpl->isOpen = false; return SZW_ERROR_FAIL; } diff --git a/ext/lzma_sdk_wrapper/cpp_bridge.hpp b/ext/lzma_sdk_wrapper/cpp_bridge.hpp index cee702a..bffb423 100644 --- a/ext/lzma_sdk_wrapper/cpp_bridge.hpp +++ b/ext/lzma_sdk_wrapper/cpp_bridge.hpp @@ -12,15 +12,6 @@ #include #include -/* 7-Zip SDK includes */ -extern "C" { - #include "lzma_sdk/C/7z.h" - #include "lzma_sdk/C/7zAlloc.h" - #include "lzma_sdk/C/7zCrc.h" - #include "lzma_sdk/C/7zFile.h" - #include "lzma_sdk/C/7zVersion.h" -} - namespace LzmaSdkWrapper { /** @@ -72,6 +63,7 @@ class ArchiveReader : public ArchiveHandle { std::string errorMessage; std::string password; std::string archivePath; // Store archive path for C++ API reopening + int archiveFormat; // Store format for GUID initialization (SZ_FORMAT_*) SzProgressCallback progressCallback; void* progressUserData; @@ -80,9 +72,9 @@ class ArchiveReader : public ArchiveHandle { std::vector memoryBuffer; // Owned copy of archive data void* memoryArchive; // IInArchive pointer for memory-based archives - // File-based C++ API cache (for password-protected archives) - void* fileArchive; // IInArchive pointer cached for file-based password extraction - void* fileStream; // IInStream pointer cached for file-based password extraction + // File-based C++ API cache + void* fileArchive; // IInArchive pointer for file-based archives + void* fileStream; // IInStream pointer for file-based archives }; /** diff --git a/ext/lzma_sdk_wrapper/extconf.rb b/ext/lzma_sdk_wrapper/extconf.rb index 745b2d8..1f926af 100644 --- a/ext/lzma_sdk_wrapper/extconf.rb +++ b/ext/lzma_sdk_wrapper/extconf.rb @@ -30,7 +30,7 @@ def detect_platform when :linux $CXXFLAGS << " -std=c++11" $CFLAGS << " -std=c11" - $LDFLAGS << " -lstdc++" + $LDFLAGS << " -lstdc++ -lpthread" when :windows if RbConfig::CONFIG["CC"] =~ /gcc|g\+\+/ # MinGW @@ -137,6 +137,8 @@ def detect_platform "lzma_sdk/CPP/7zip/Common", "lzma_sdk/CPP/7zip/Archive/Common", "lzma_sdk/CPP/7zip/Archive/7z", + "lzma_sdk/CPP/7zip/Archive/Zip", + "lzma_sdk/CPP/7zip/Archive/Tar", "lzma_sdk/CPP/7zip/Compress", "lzma_sdk/CPP/7zip/Crypto", ] @@ -160,17 +162,19 @@ def detect_platform # However, SOME Windows files are Windows-only (need ShlObj.h, CreateFileMapping, etc.) # so we use a whitelist approach for cross-platform files: windows_cross_platform = [ - "FileDir.cpp", # Cross-platform directory operations - "FileFind.cpp", # Cross-platform file finding - "FileIO.cpp", # Cross-platform file I/O - "FileName.cpp", # Cross-platform filename handling - "PropVariant.cpp", # Cross-platform property variants - "PropVariantConv.cpp", # Cross-platform property conversion - "System.cpp", # Cross-platform system info - "TimeUtils.cpp", # Cross-platform time utilities - "ErrorMsg.cpp", # Cross-platform error messages - "FileSystem.cpp", # Cross-platform filesystem operations - "SystemInfo.cpp", # Cross-platform system information + "FileDir.cpp", # Cross-platform directory operations + "FileFind.cpp", # Cross-platform file finding + "FileIO.cpp", # Cross-platform file I/O + "FileName.cpp", # Cross-platform filename handling + "PropVariant.cpp", # Cross-platform property variants + "PropVariantConv.cpp", # Cross-platform property conversion + "PropVariantUtils.cpp", # Cross-platform property variant utilities (used by Zip handler) + "Synchronization.cpp", # Cross-platform synchronization (WFMO vtables needed by MemBlocks, OutMemStream) + "System.cpp", # Cross-platform system info + "TimeUtils.cpp", # Cross-platform time utilities + "ErrorMsg.cpp", # Cross-platform error messages + "FileSystem.cpp", # Cross-platform filesystem operations + "SystemInfo.cpp", # Cross-platform system information ] windows_cross_platform.each do |file| lzma_cpp_files << "lzma_sdk/CPP/Windows/#{file}" diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Blake2.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/Blake2.h new file mode 100644 index 0000000..801ea7a --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Blake2.h @@ -0,0 +1,105 @@ +/* Blake2.h -- BLAKE2sp Hash +2024-01-17 : Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_BLAKE2_H +#define ZIP7_INC_BLAKE2_H + +#include "7zTypes.h" + +#if 0 +#include "Compiler.h" +#include "CpuArch.h" +#if defined(MY_CPU_X86_OR_AMD64) +#if defined(__SSE2__) \ + || defined(_MSC_VER) && _MSC_VER > 1200 \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ + || defined(__clang__) \ + || defined(__INTEL_COMPILER) +#include // SSE2 +#endif + +#if defined(__AVX2__) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) \ + || defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) +#include +#if defined(__clang__) +#include +#include +#endif +#endif // avx2 +#endif // MY_CPU_X86_OR_AMD64 +#endif // 0 + +EXTERN_C_BEGIN + +#define Z7_BLAKE2S_BLOCK_SIZE 64 +#define Z7_BLAKE2S_DIGEST_SIZE 32 +#define Z7_BLAKE2SP_PARALLEL_DEGREE 8 +#define Z7_BLAKE2SP_NUM_STRUCT_WORDS 16 + +#if 1 || defined(Z7_BLAKE2SP_USE_FUNCTIONS) +typedef void (Z7_FASTCALL *Z7_BLAKE2SP_FUNC_COMPRESS)(UInt32 *states, const Byte *data, const Byte *end); +typedef void (Z7_FASTCALL *Z7_BLAKE2SP_FUNC_INIT)(UInt32 *states); +#endif + +// it's required that CBlake2sp is aligned for 32-bytes, +// because the code can use unaligned access with sse and avx256. +// but 64-bytes alignment can be better. +MY_ALIGN(64) +typedef struct +{ + union + { +#if 0 +#if defined(MY_CPU_X86_OR_AMD64) +#if defined(__SSE2__) \ + || defined(_MSC_VER) && _MSC_VER > 1200 \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ + || defined(__clang__) \ + || defined(__INTEL_COMPILER) + __m128i _pad_align_128bit[4]; +#endif // sse2 +#if defined(__AVX2__) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) \ + || defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) + __m256i _pad_align_256bit[2]; +#endif // avx2 +#endif // x86 +#endif // 0 + + void * _pad_align_ptr[8]; + UInt32 _pad_align_32bit[16]; + struct + { + unsigned cycPos; + unsigned _pad_unused; +#if 1 || defined(Z7_BLAKE2SP_USE_FUNCTIONS) + Z7_BLAKE2SP_FUNC_COMPRESS func_Compress_Fast; + Z7_BLAKE2SP_FUNC_COMPRESS func_Compress_Single; + Z7_BLAKE2SP_FUNC_INIT func_Init; + Z7_BLAKE2SP_FUNC_INIT func_Final; +#endif + } header; + } u; + // MY_ALIGN(64) + UInt32 states[Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2SP_NUM_STRUCT_WORDS]; + // MY_ALIGN(64) + UInt32 buf32[Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2SP_NUM_STRUCT_WORDS * 2]; +} CBlake2sp; + +BoolInt Blake2sp_SetFunction(CBlake2sp *p, unsigned algo); +void Blake2sp_Init(CBlake2sp *p); +void Blake2sp_InitState(CBlake2sp *p); +void Blake2sp_Update(CBlake2sp *p, const Byte *data, size_t size); +void Blake2sp_Final(CBlake2sp *p, Byte *digest); +void z7_Black2sp_Prepare(void); + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/BwtSort.c b/ext/lzma_sdk_wrapper/lzma_sdk/C/BwtSort.c new file mode 100644 index 0000000..8f64f9d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/BwtSort.c @@ -0,0 +1,628 @@ +/* BwtSort.c -- BWT block sorting +: Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "BwtSort.h" +#include "Sort.h" + +/* #define BLOCK_SORT_USE_HEAP_SORT */ +// #define BLOCK_SORT_USE_HEAP_SORT + +#ifdef BLOCK_SORT_USE_HEAP_SORT + +#define HeapSortRefDown(p, vals, n, size, temp) \ + { size_t k = n; UInt32 val = vals[temp]; for (;;) { \ + size_t s = k << 1; \ + if (s > size) break; \ + if (s < size && vals[p[s + 1]] > vals[p[s]]) s++; \ + if (val >= vals[p[s]]) break; \ + p[k] = p[s]; k = s; \ + } p[k] = temp; } + +void HeapSortRef(UInt32 *p, UInt32 *vals, size_t size) +{ + if (size <= 1) + return; + p--; + { + size_t i = size / 2; + do + { + UInt32 temp = p[i]; + HeapSortRefDown(p, vals, i, size, temp); + } + while (--i != 0); + } + do + { + UInt32 temp = p[size]; + p[size--] = p[1]; + HeapSortRefDown(p, vals, 1, size, temp); + } + while (size > 1); +} + +#endif // BLOCK_SORT_USE_HEAP_SORT + + +/* Don't change it !!! */ +#define kNumHashBytes 2 +#define kNumHashValues (1 << (kNumHashBytes * 8)) + +/* kNumRefBitsMax must be < (kNumHashBytes * 8) = 16 */ +#define kNumRefBitsMax 12 + +#define BS_TEMP_SIZE kNumHashValues + +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + +/* 32 Flags in UInt32 word */ +#define kNumFlagsBits 5 +#define kNumFlagsInWord (1 << kNumFlagsBits) +#define kFlagsMask (kNumFlagsInWord - 1) +#define kAllFlags 0xFFFFFFFF + +#else + +#define kNumBitsMax 20 +#define kIndexMask (((UInt32)1 << kNumBitsMax) - 1) +#define kNumExtraBits (32 - kNumBitsMax) +#define kNumExtra0Bits (kNumExtraBits - 2) +#define kNumExtra0Mask ((1 << kNumExtra0Bits) - 1) + +#define SetFinishedGroupSize(p, size) \ + { *(p) |= ((((UInt32)(size) - 1) & kNumExtra0Mask) << kNumBitsMax); \ + if ((size) > (1 << kNumExtra0Bits)) { \ + *(p) |= 0x40000000; \ + *((p) + 1) |= (((UInt32)(size) - 1) >> kNumExtra0Bits) << kNumBitsMax; } } \ + +static void SetGroupSize(UInt32 *p, size_t size) +{ + if (--size == 0) + return; + *p |= 0x80000000 | (((UInt32)size & kNumExtra0Mask) << kNumBitsMax); + if (size >= (1 << kNumExtra0Bits)) + { + *p |= 0x40000000; + p[1] |= (((UInt32)size >> kNumExtra0Bits) << kNumBitsMax); + } +} + +#endif + +/* +SortGroup - is recursive Range-Sort function with HeapSort optimization for small blocks + "range" is not real range. It's only for optimization. +returns: 1 - if there are groups, 0 - no more groups +*/ + +static +unsigned +Z7_FASTCALL +SortGroup(size_t BlockSize, size_t NumSortedBytes, + size_t groupOffset, size_t groupSize, + unsigned NumRefBits, UInt32 *Indices +#ifndef BLOCK_SORT_USE_HEAP_SORT + , size_t left, size_t range +#endif + ) +{ + UInt32 *ind2 = Indices + groupOffset; + UInt32 *Groups; + if (groupSize <= 1) + { + /* + #ifndef BLOCK_SORT_EXTERNAL_FLAGS + SetFinishedGroupSize(ind2, 1) + #endif + */ + return 0; + } + Groups = Indices + BlockSize + BS_TEMP_SIZE; + if (groupSize <= ((size_t)1 << NumRefBits) +#ifndef BLOCK_SORT_USE_HEAP_SORT + && groupSize <= range +#endif + ) + { + UInt32 *temp = Indices + BlockSize; + size_t j, group; + UInt32 mask, cg; + unsigned thereAreGroups; + { + UInt32 gPrev; + UInt32 gRes = 0; + { + size_t sp = ind2[0] + NumSortedBytes; + if (sp >= BlockSize) + sp -= BlockSize; + gPrev = Groups[sp]; + temp[0] = gPrev << NumRefBits; + } + + for (j = 1; j < groupSize; j++) + { + size_t sp = ind2[j] + NumSortedBytes; + UInt32 g; + if (sp >= BlockSize) + sp -= BlockSize; + g = Groups[sp]; + temp[j] = (g << NumRefBits) | (UInt32)j; + gRes |= (gPrev ^ g); + } + if (gRes == 0) + { +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + SetGroupSize(ind2, groupSize); +#endif + return 1; + } + } + + HeapSort(temp, groupSize); + mask = ((UInt32)1 << NumRefBits) - 1; + thereAreGroups = 0; + + group = groupOffset; + cg = temp[0] >> NumRefBits; + temp[0] = ind2[temp[0] & mask]; + + { +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + UInt32 *Flags = Groups + BlockSize; +#else + size_t prevGroupStart = 0; +#endif + + for (j = 1; j < groupSize; j++) + { + const UInt32 val = temp[j]; + const UInt32 cgCur = val >> NumRefBits; + + if (cgCur != cg) + { + cg = cgCur; + group = groupOffset + j; + +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + { + const size_t t = group - 1; + Flags[t >> kNumFlagsBits] &= ~((UInt32)1 << (t & kFlagsMask)); + } +#else + SetGroupSize(temp + prevGroupStart, j - prevGroupStart); + prevGroupStart = j; +#endif + } + else + thereAreGroups = 1; + { + const UInt32 ind = ind2[val & mask]; + temp[j] = ind; + Groups[ind] = (UInt32)group; + } + } + +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + SetGroupSize(temp + prevGroupStart, j - prevGroupStart); +#endif + } + + for (j = 0; j < groupSize; j++) + ind2[j] = temp[j]; + return thereAreGroups; + } + + /* Check that all strings are in one group (cannot sort) */ + { + UInt32 group; + size_t j; + size_t sp = ind2[0] + NumSortedBytes; + if (sp >= BlockSize) + sp -= BlockSize; + group = Groups[sp]; + for (j = 1; j < groupSize; j++) + { + sp = ind2[j] + NumSortedBytes; + if (sp >= BlockSize) + sp -= BlockSize; + if (Groups[sp] != group) + break; + } + if (j == groupSize) + { +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + SetGroupSize(ind2, groupSize); +#endif + return 1; + } + } + +#ifndef BLOCK_SORT_USE_HEAP_SORT + { + /* ---------- Range Sort ---------- */ + size_t i; + size_t mid; + for (;;) + { + size_t j; + if (range <= 1) + { +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + SetGroupSize(ind2, groupSize); +#endif + return 1; + } + mid = left + ((range + 1) >> 1); + j = groupSize; + i = 0; + do + { + size_t sp = ind2[i] + NumSortedBytes; if (sp >= BlockSize) sp -= BlockSize; + if (Groups[sp] >= mid) + { + for (j--; j > i; j--) + { + sp = ind2[j] + NumSortedBytes; if (sp >= BlockSize) sp -= BlockSize; + if (Groups[sp] < mid) + { + UInt32 temp = ind2[i]; ind2[i] = ind2[j]; ind2[j] = temp; + break; + } + } + if (i >= j) + break; + } + } + while (++i < j); + if (i == 0) + { + range = range - (mid - left); + left = mid; + } + else if (i == groupSize) + range = (mid - left); + else + break; + } + +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + { + const size_t t = groupOffset + i - 1; + UInt32 *Flags = Groups + BlockSize; + Flags[t >> kNumFlagsBits] &= ~((UInt32)1 << (t & kFlagsMask)); + } +#endif + + { + size_t j; + for (j = i; j < groupSize; j++) + Groups[ind2[j]] = (UInt32)(groupOffset + i); + } + + { + unsigned res = SortGroup(BlockSize, NumSortedBytes, groupOffset, i, NumRefBits, Indices, left, mid - left); + return res | SortGroup(BlockSize, NumSortedBytes, groupOffset + i, groupSize - i, NumRefBits, Indices, mid, range - (mid - left)); + } + + } + +#else // BLOCK_SORT_USE_HEAP_SORT + + /* ---------- Heap Sort ---------- */ + + { + size_t j; + for (j = 0; j < groupSize; j++) + { + size_t sp = ind2[j] + NumSortedBytes; + if (sp >= BlockSize) + sp -= BlockSize; + ind2[j] = (UInt32)sp; + } + + HeapSortRef(ind2, Groups, groupSize); + + /* Write Flags */ + { + size_t sp = ind2[0]; + UInt32 group = Groups[sp]; + +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + UInt32 *Flags = Groups + BlockSize; +#else + size_t prevGroupStart = 0; +#endif + + for (j = 1; j < groupSize; j++) + { + sp = ind2[j]; + if (Groups[sp] != group) + { + group = Groups[sp]; +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + { + const size_t t = groupOffset + j - 1; + Flags[t >> kNumFlagsBits] &= ~(1 << (t & kFlagsMask)); + } +#else + SetGroupSize(ind2 + prevGroupStart, j - prevGroupStart); + prevGroupStart = j; +#endif + } + } + +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + SetGroupSize(ind2 + prevGroupStart, j - prevGroupStart); +#endif + } + { + /* Write new Groups values and Check that there are groups */ + unsigned thereAreGroups = 0; + for (j = 0; j < groupSize; j++) + { + size_t group = groupOffset + j; +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + UInt32 subGroupSize = ((ind2[j] & ~0xC0000000) >> kNumBitsMax); + if (ind2[j] & 0x40000000) + subGroupSize += ((ind2[(size_t)j + 1] >> kNumBitsMax) << kNumExtra0Bits); + subGroupSize++; + for (;;) + { + const UInt32 original = ind2[j]; + size_t sp = original & kIndexMask; + if (sp < NumSortedBytes) + sp += BlockSize; + sp -= NumSortedBytes; + ind2[j] = (UInt32)sp | (original & ~kIndexMask); + Groups[sp] = (UInt32)group; + if (--subGroupSize == 0) + break; + j++; + thereAreGroups = 1; + } +#else + UInt32 *Flags = Groups + BlockSize; + for (;;) + { + size_t sp = ind2[j]; + if (sp < NumSortedBytes) + sp += BlockSize; + sp -= NumSortedBytes; + ind2[j] = (UInt32)sp; + Groups[sp] = (UInt32)group; + if ((Flags[(groupOffset + j) >> kNumFlagsBits] & (1 << ((groupOffset + j) & kFlagsMask))) == 0) + break; + j++; + thereAreGroups = 1; + } +#endif + } + return thereAreGroups; + } + } +#endif // BLOCK_SORT_USE_HEAP_SORT +} + + +/* conditions: blockSize > 0 */ +UInt32 BlockSort(UInt32 *Indices, const Byte *data, size_t blockSize) +{ + UInt32 *counters = Indices + blockSize; + size_t i; + UInt32 *Groups; +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + UInt32 *Flags; +#endif + +/* Radix-Sort for 2 bytes */ +// { UInt32 yyy; for (yyy = 0; yyy < 100; yyy++) { + for (i = 0; i < kNumHashValues; i++) + counters[i] = 0; + { + const Byte *data2 = data; + size_t a = data[(size_t)blockSize - 1]; + const Byte *data_lim = data + blockSize; + if (blockSize >= 4) + { + data_lim -= 3; + do + { + size_t b; + b = data2[0]; counters[(a << 8) | b]++; + a = data2[1]; counters[(b << 8) | a]++; + b = data2[2]; counters[(a << 8) | b]++; + a = data2[3]; counters[(b << 8) | a]++; + data2 += 4; + } + while (data2 < data_lim); + data_lim += 3; + } + while (data2 != data_lim) + { + size_t b = *data2++; + counters[(a << 8) | b]++; + a = b; + } + } +// }} + + Groups = counters + BS_TEMP_SIZE; +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + Flags = Groups + blockSize; + { + const size_t numWords = (blockSize + kFlagsMask) >> kNumFlagsBits; + for (i = 0; i < numWords; i++) + Flags[i] = kAllFlags; + } +#endif + + { + UInt32 sum = 0; + for (i = 0; i < kNumHashValues; i++) + { + const UInt32 groupSize = counters[i]; + counters[i] = sum; + sum += groupSize; +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + if (groupSize) + { + const UInt32 t = sum - 1; + Flags[t >> kNumFlagsBits] &= ~((UInt32)1 << (t & kFlagsMask)); + } +#endif + } + } + + for (i = 0; i < blockSize - 1; i++) + Groups[i] = counters[((unsigned)data[i] << 8) | data[(size_t)i + 1]]; + Groups[i] = counters[((unsigned)data[i] << 8) | data[0]]; + + { +#define SET_Indices(a, b, i) \ + { UInt32 c; \ + a = (a << 8) | (b); \ + c = counters[a]; \ + Indices[c] = (UInt32)i++; \ + counters[a] = c + 1; \ + } + + size_t a = data[0]; + const Byte *data_ptr = data + 1; + i = 0; + if (blockSize >= 3) + { + blockSize -= 2; + do + { + size_t b; + b = data_ptr[0]; SET_Indices(a, b, i) + a = data_ptr[1]; SET_Indices(b, a, i) + data_ptr += 2; + } + while (i < blockSize); + blockSize += 2; + } + if (i < blockSize - 1) + { + SET_Indices(a, data[(size_t)i + 1], i) + a = (Byte)a; + } + SET_Indices(a, data[0], i) + } + +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + { + UInt32 prev = 0; + for (i = 0; i < kNumHashValues; i++) + { + const UInt32 prevGroupSize = counters[i] - prev; + if (prevGroupSize == 0) + continue; + SetGroupSize(Indices + prev, prevGroupSize); + prev = counters[i]; + } + } +#endif + + { + unsigned NumRefBits; + size_t NumSortedBytes; + for (NumRefBits = 0; ((blockSize - 1) >> NumRefBits) != 0; NumRefBits++) + {} + NumRefBits = 32 - NumRefBits; + if (NumRefBits > kNumRefBitsMax) + NumRefBits = kNumRefBitsMax; + + for (NumSortedBytes = kNumHashBytes; ; NumSortedBytes <<= 1) + { +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + size_t finishedGroupSize = 0; +#endif + size_t newLimit = 0; + for (i = 0; i < blockSize;) + { + size_t groupSize; +#ifdef BLOCK_SORT_EXTERNAL_FLAGS + + if ((Flags[i >> kNumFlagsBits] & (1 << (i & kFlagsMask))) == 0) + { + i++; + continue; + } + for (groupSize = 1; + (Flags[(i + groupSize) >> kNumFlagsBits] & (1 << ((i + groupSize) & kFlagsMask))) != 0; + groupSize++) + {} + groupSize++; + +#else + + groupSize = (Indices[i] & ~0xC0000000) >> kNumBitsMax; + { + const BoolInt finishedGroup = ((Indices[i] & 0x80000000) == 0); + if (Indices[i] & 0x40000000) + { + groupSize += ((Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits); + Indices[(size_t)i + 1] &= kIndexMask; + } + Indices[i] &= kIndexMask; + groupSize++; + if (finishedGroup || groupSize == 1) + { + Indices[i - finishedGroupSize] &= kIndexMask; + if (finishedGroupSize > 1) + Indices[(size_t)(i - finishedGroupSize) + 1] &= kIndexMask; + { + const size_t newGroupSize = groupSize + finishedGroupSize; + SetFinishedGroupSize(Indices + i - finishedGroupSize, newGroupSize) + finishedGroupSize = newGroupSize; + } + i += groupSize; + continue; + } + finishedGroupSize = 0; + } + +#endif + + if (NumSortedBytes >= blockSize) + { + size_t j; + for (j = 0; j < groupSize; j++) + { + size_t t = i + j; + /* Flags[t >> kNumFlagsBits] &= ~(1 << (t & kFlagsMask)); */ + Groups[Indices[t]] = (UInt32)t; + } + } + else + if (SortGroup(blockSize, NumSortedBytes, i, groupSize, NumRefBits, Indices + #ifndef BLOCK_SORT_USE_HEAP_SORT + , 0, blockSize + #endif + )) + newLimit = i + groupSize; + i += groupSize; + } + if (newLimit == 0) + break; + } + } +#ifndef BLOCK_SORT_EXTERNAL_FLAGS + for (i = 0; i < blockSize;) + { + size_t groupSize = (Indices[i] & ~0xC0000000) >> kNumBitsMax; + if (Indices[i] & 0x40000000) + { + groupSize += (Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits; + Indices[(size_t)i + 1] &= kIndexMask; + } + Indices[i] &= kIndexMask; + groupSize++; + i += groupSize; + } +#endif + return Groups[0]; +} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/BwtSort.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/BwtSort.h new file mode 100644 index 0000000..1bd2316 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/BwtSort.h @@ -0,0 +1,27 @@ +/* BwtSort.h -- BWT block sorting +: Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_BWT_SORT_H +#define ZIP7_INC_BWT_SORT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +/* use BLOCK_SORT_EXTERNAL_FLAGS if blockSize can be > 1M */ +/* #define BLOCK_SORT_EXTERNAL_FLAGS */ +// #define BLOCK_SORT_EXTERNAL_FLAGS + +#ifdef BLOCK_SORT_EXTERNAL_FLAGS +#define BLOCK_SORT_EXTERNAL_SIZE(blockSize) (((blockSize) + 31) >> 5) +#else +#define BLOCK_SORT_EXTERNAL_SIZE(blockSize) 0 +#endif + +#define BLOCK_SORT_BUF_SIZE(blockSize) ((blockSize) * 2 + BLOCK_SORT_EXTERNAL_SIZE(blockSize) + (1 << 16)) + +UInt32 BlockSort(UInt32 *indices, const Byte *data, size_t blockSize); + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/HuffEnc.c b/ext/lzma_sdk_wrapper/lzma_sdk/C/HuffEnc.c new file mode 100644 index 0000000..cbf8c22 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/HuffEnc.c @@ -0,0 +1,360 @@ +/* HuffEnc.c -- functions for Huffman encoding +Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#include "HuffEnc.h" +#include "Sort.h" +#include "CpuArch.h" + +#define kMaxLen Z7_HUFFMAN_LEN_MAX +#define NUM_BITS 10 +#define MASK ((1u << NUM_BITS) - 1) +#define FREQ_MASK (~(UInt32)MASK) +#define NUM_COUNTERS (48 * 2) + +#if 1 && (defined(MY_CPU_LE) || defined(MY_CPU_BE)) +#if defined(MY_CPU_LE) + #define HI_HALF_OFFSET 1 +#else + #define HI_HALF_OFFSET 0 +#endif +#define LOAD_PARENT(p) ((unsigned)*((const UInt16 *)(p) + HI_HALF_OFFSET)) +#define STORE_PARENT(p, fb, val) *((UInt16 *)(p) + HI_HALF_OFFSET) = (UInt16)(val); +#define STORE_PARENT_DIRECT(p, fb, hi) STORE_PARENT(p, fb, hi) +#define UPDATE_E(eHi) eHi++; +#else +#define LOAD_PARENT(p) ((unsigned)(*(p) >> NUM_BITS)) +#define STORE_PARENT_DIRECT(p, fb, hi) *(p) = ((fb) & MASK) | (hi); // set parent field +#define STORE_PARENT(p, fb, val) STORE_PARENT_DIRECT(p, fb, ((UInt32)(val) << NUM_BITS)) +#define UPDATE_E(eHi) eHi += 1 << NUM_BITS; +#endif + +void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, unsigned numSymbols, unsigned maxLen) +{ +#if NUM_COUNTERS > 2 + unsigned counters[NUM_COUNTERS]; +#endif +#if 1 && NUM_COUNTERS > (kMaxLen + 4) * 2 + #define lenCounters (counters) + #define codes (counters + kMaxLen + 4) +#else + unsigned lenCounters[kMaxLen + 1]; + UInt32 codes[kMaxLen + 1]; +#endif + + unsigned num; + { + unsigned i; + // UInt32 sum = 0; + +#if NUM_COUNTERS > 2 + +#define CTR_ITEM_FOR_FREQ(freq) \ + counters[(freq) >= NUM_COUNTERS - 1 ? NUM_COUNTERS - 1 : (unsigned)(freq)] + + for (i = 0; i < NUM_COUNTERS; i++) + counters[i] = 0; + memset(lens, 0, numSymbols); + { + const UInt32 *fp = freqs + numSymbols; +#define NUM_UNROLLS 1 +#if NUM_UNROLLS > 1 // use 1 if odd (numSymbols) is possisble + if (numSymbols & 1) + { + UInt32 f; + f = *--fp; CTR_ITEM_FOR_FREQ(f)++; + // sum += f; + } +#endif + do + { + UInt32 f; + fp -= NUM_UNROLLS; + f = fp[0]; CTR_ITEM_FOR_FREQ(f)++; + // sum += f; +#if NUM_UNROLLS > 1 + f = fp[1]; CTR_ITEM_FOR_FREQ(f)++; + // sum += f; +#endif + } + while (fp != freqs); + } +#if 0 + printf("\nsum=%8u numSymbols =%3u ctrs:", sum, numSymbols); + { + unsigned k = 0; + for (k = 0; k < NUM_COUNTERS; k++) + printf(" %u", counters[k]); + } +#endif + + num = counters[1]; + counters[1] = 0; + for (i = 2; i != NUM_COUNTERS; i += 2) + { + unsigned c; + c = (counters )[i]; (counters )[i] = num; num += c; + c = (counters + 1)[i]; (counters + 1)[i] = num; num += c; + } + counters[0] = num; // we want to write (freq==0) symbols to the end of (p) array + { + i = 0; + do + { + const UInt32 f = freqs[i]; +#if 0 + if (f == 0) lens[i] = 0; else +#endif + p[CTR_ITEM_FOR_FREQ(f)++] = i | (f << NUM_BITS); + } + while (++i != numSymbols); + } + HeapSort(p + counters[NUM_COUNTERS - 2], counters[NUM_COUNTERS - 1] - counters[NUM_COUNTERS - 2]); + +#else // NUM_COUNTERS <= 2 + + num = 0; + for (i = 0; i < numSymbols; i++) + { + const UInt32 freq = freqs[i]; + if (freq == 0) + lens[i] = 0; + else + p[num++] = i | (freq << NUM_BITS); + } + HeapSort(p, num); + +#endif + } + + if (num <= 2) + { + unsigned minCode = 0; + unsigned maxCode = 1; + if (num) + { + maxCode = (unsigned)p[(size_t)num - 1] & MASK; + if (num == 2) + { + minCode = (unsigned)p[0] & MASK; + if (minCode > maxCode) + { + const unsigned temp = minCode; + minCode = maxCode; + maxCode = temp; + } + } + else if (maxCode == 0) + maxCode++; + } + p[minCode] = 0; + p[maxCode] = 1; + lens[minCode] = lens[maxCode] = 1; + return; + } + { + unsigned i; + for (i = 0; i <= kMaxLen; i++) + lenCounters[i] = 0; + lenCounters[1] = 2; // by default root node has 2 child leaves at level 1. + } + // if (num != 2) + { + // num > 2 + // the binary tree will contain (num - 1) internal nodes. + // p[num - 2] will be root node of binary tree. + UInt32 *b; + UInt32 *n; + // first node will have two leaf childs: p[0] and p[1]: + // p[0] += p[1] & FREQ_MASK; // set frequency sum of child leafs + // if (pi == n) exit(0); + // if (pi != n) + { + UInt32 fb = (p[1] & FREQ_MASK) + p[0]; + UInt32 f = p[2] & FREQ_MASK; + const UInt32 *pi = p + 2; + UInt32 *e = p; + UInt32 eHi = 0; + n = p + num; + b = p; + // p[0] = fb; + for (;;) + { + // (b <= e) + UInt32 sum; + e++; + UPDATE_E(eHi) + + // (b < e) + + // p range : high bits + // [0, b) : parent : processed nodes that have parent and childs + // [b, e) : FREQ : non-processed nodes that have no parent but have childs + // [e, pi) : FREQ : processed leaves for which parent node was created + // [pi, n) : FREQ : non-processed leaves for which parent node was not created + + // first child + // note : (*b < f) is same result as ((*b & FREQ_MASK) < f) + if (fb < f) + { + // node freq is smaller + sum = fb & FREQ_MASK; + STORE_PARENT_DIRECT (b, fb, eHi) + b++; + fb = *b; + if (b == e) + { + if (++pi == n) + break; + sum += f; + fb &= MASK; + fb |= sum; + *e = fb; + f = *pi & FREQ_MASK; + continue; + } + } + else if (++pi == n) + { + STORE_PARENT_DIRECT (b, fb, eHi) + b++; + break; + } + else + { + sum = f; + f = *pi & FREQ_MASK; + } + + // (b < e) + + // second child + if (fb < f) + { + sum += fb; + sum &= FREQ_MASK; + STORE_PARENT_DIRECT (b, fb, eHi) + b++; + *e = (*e & MASK) | sum; // set frequency sum + // (b <= e) is possible here + fb = *b; + } + else if (++pi == n) + break; + else + { + sum += f; + f = *pi & FREQ_MASK; + *e = (*e & MASK) | sum; // set frequency sum + } + } + } + + // printf("\nnum-e=%3u, numSymbols=%3u, num=%3u, b=%3u", n - e, numSymbols, n - p, b - p); + { + n -= 2; + *n &= MASK; // root node : we clear high bits (zero bits mean level == 0) + if (n != b) + { + // We go here, if we have some number of non-created nodes up to root. + // We process them in simplified code: + // position of parent for each pair of nodes is known. + // n[-2], n[-1] : current pair of child nodes + // (p1) : parent node for current pair. + UInt32 *p1 = n; + do + { + const unsigned len = LOAD_PARENT(p1) + 1; + p1--; + (lenCounters )[len] -= 2; // we remove 2 leaves from level (len) + (lenCounters + 1)[len] += 2 * 2; // we add 4 leaves at level (len + 1) + n -= 2; + STORE_PARENT (n , n[0], len) + STORE_PARENT (n + 1, n[1], len) + } + while (n != b); + } + } + + if (b != p) + { + // we detect level of each node (realtive to root), + // and update lenCounters[]. + // We process only intermediate nodes and we don't process leaves. + do + { + // if (ii < b) : parent_bits_of (p[ii]) == index of parent node : ii < (p[ii]) + // if (ii >= b) : parent_bits_of (p[ii]) == level of this (ii) node in tree + unsigned len; + b--; + len = (unsigned)LOAD_PARENT(p + LOAD_PARENT(b)) + 1; + STORE_PARENT (b, *b, len) + if (len >= maxLen) + { + // We are not allowed to create node at level (maxLen) and higher, + // because all leaves must be placed to level (maxLen) or lower. + // We find nearest allowed leaf and place current node to level of that leaf: + for (len = maxLen - 1; lenCounters[len] == 0; len--) {} + } + lenCounters[len]--; // we remove 1 leaf from level (len) + (lenCounters + 1)[len] += 2; // we add 2 leaves at level (len + 1) + } + while (b != p); + } + } + { + { + unsigned len = maxLen; + const UInt32 *p2 = p; + do + { + unsigned k = lenCounters[len]; + if (k) + do + lens[(unsigned)*p2++ & MASK] = (Byte)len; + while (--k); + } + while (--len); + } + codes[0] = 0; // we don't want garbage values to be written to p[] array. + // codes[1] = 0; + { + UInt32 code = 0; + unsigned len; + for (len = 0; len < kMaxLen; len++) + (codes + 1)[len] = code = (code + lenCounters[len]) << 1; + } + /* if (code + lenCounters[kMaxLen] - 1 != (1 << kMaxLen) - 1) throw 1; */ + { + const Byte * const limit = lens + numSymbols; + do + { + unsigned len; + UInt32 c; + len = lens[0]; c = codes[len]; p[0] = c; codes[len] = c + 1; + // len = lens[1]; c = codes[len]; p[1] = c; codes[len] = c + 1; + p += 1; + lens += 1; + } + while (lens != limit); + } + } +} + +#undef kMaxLen +#undef NUM_BITS +#undef MASK +#undef FREQ_MASK +#undef NUM_COUNTERS +#undef CTR_ITEM_FOR_FREQ +#undef LOAD_PARENT +#undef STORE_PARENT +#undef STORE_PARENT_DIRECT +#undef UPDATE_E +#undef HI_HALF_OFFSET +#undef NUM_UNROLLS +#undef lenCounters +#undef codes diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/HuffEnc.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/HuffEnc.h new file mode 100644 index 0000000..2217f55 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/HuffEnc.h @@ -0,0 +1,23 @@ +/* HuffEnc.h -- Huffman encoding +Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_HUFF_ENC_H +#define ZIP7_INC_HUFF_ENC_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define Z7_HUFFMAN_LEN_MAX 16 +/* +Conditions: + 2 <= num <= 1024 = 2 ^ NUM_BITS + Sum(freqs) < 4M = 2 ^ (32 - NUM_BITS) + 1 <= maxLen <= 16 = Z7_HUFFMAN_LEN_MAX + Num_Items(p) >= HUFFMAN_TEMP_SIZE(num) +*/ +void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, UInt32 num, UInt32 maxLen); + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Md5.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/Md5.h new file mode 100644 index 0000000..49c0741 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Md5.h @@ -0,0 +1,34 @@ +/* Md5.h -- MD5 Hash +: Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_MD5_H +#define ZIP7_INC_MD5_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define MD5_NUM_BLOCK_WORDS 16 +#define MD5_NUM_DIGEST_WORDS 4 + +#define MD5_BLOCK_SIZE (MD5_NUM_BLOCK_WORDS * 4) +#define MD5_DIGEST_SIZE (MD5_NUM_DIGEST_WORDS * 4) + +typedef struct +{ + UInt64 count; + UInt64 _pad_1; + // we want 16-bytes alignment here + UInt32 state[MD5_NUM_DIGEST_WORDS]; + UInt64 _pad_2[4]; + // we want 64-bytes alignment here + Byte buffer[MD5_BLOCK_SIZE]; +} CMd5; + +void Md5_Init(CMd5 *p); +void Md5_Update(CMd5 *p, const Byte *data, size_t size); +void Md5_Final(CMd5 *p, Byte *digest); + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8.c b/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8.c new file mode 100644 index 0000000..c6bdd86 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8.c @@ -0,0 +1,1574 @@ +/* Ppmd8.c -- PPMdI codec +2023-09-07 : Igor Pavlov : Public domain +This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ + +#include "Precomp.h" + +#include + +#include "Ppmd8.h" + + + + +MY_ALIGN(16) +static const Byte PPMD8_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; +MY_ALIGN(16) +static const UInt16 PPMD8_kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; + +#define MAX_FREQ 124 +#define UNIT_SIZE 12 + +#define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) +#define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) +#define I2U(indx) ((unsigned)p->Indx2Units[indx]) + + +#define REF(ptr) Ppmd_GetRef(p, ptr) + +#define STATS_REF(ptr) ((CPpmd_State_Ref)REF(ptr)) + +#define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref)) +#define STATS(ctx) Ppmd8_GetStats(p, ctx) +#define ONE_STATE(ctx) Ppmd8Context_OneState(ctx) +#define SUFFIX(ctx) CTX((ctx)->Suffix) + +typedef CPpmd8_Context * PPMD8_CTX_PTR; + +struct CPpmd8_Node_; + +typedef Ppmd_Ref_Type(struct CPpmd8_Node_) CPpmd8_Node_Ref; + +typedef struct CPpmd8_Node_ +{ + UInt32 Stamp; + + CPpmd8_Node_Ref Next; + UInt32 NU; +} CPpmd8_Node; + +#define NODE(r) Ppmd_GetPtr_Type(p, r, CPpmd8_Node) + +void Ppmd8_Construct(CPpmd8 *p) +{ + unsigned i, k, m; + + p->Base = NULL; + + for (i = 0, k = 0; i < PPMD_NUM_INDEXES; i++) + { + unsigned step = (i >= 12 ? 4 : (i >> 2) + 1); + do { p->Units2Indx[k++] = (Byte)i; } while (--step); + p->Indx2Units[i] = (Byte)k; + } + + p->NS2BSIndx[0] = (0 << 1); + p->NS2BSIndx[1] = (1 << 1); + memset(p->NS2BSIndx + 2, (2 << 1), 9); + memset(p->NS2BSIndx + 11, (3 << 1), 256 - 11); + + for (i = 0; i < 5; i++) + p->NS2Indx[i] = (Byte)i; + + for (m = i, k = 1; i < 260; i++) + { + p->NS2Indx[i] = (Byte)m; + if (--k == 0) + k = (++m) - 4; + } + + memcpy(p->ExpEscape, PPMD8_kExpEscape, 16); +} + + +void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->Base); + p->Size = 0; + p->Base = NULL; +} + + +BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc) +{ + if (!p->Base || p->Size != size) + { + Ppmd8_Free(p, alloc); + p->AlignOffset = (4 - size) & 3; + if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size)) == NULL) + return False; + p->Size = size; + } + return True; +} + + + +// ---------- Internal Memory Allocator ---------- + + + + + + +#define EMPTY_NODE 0xFFFFFFFF + + +static void Ppmd8_InsertNode(CPpmd8 *p, void *node, unsigned indx) +{ + ((CPpmd8_Node *)node)->Stamp = EMPTY_NODE; + ((CPpmd8_Node *)node)->Next = (CPpmd8_Node_Ref)p->FreeList[indx]; + ((CPpmd8_Node *)node)->NU = I2U(indx); + p->FreeList[indx] = REF(node); + p->Stamps[indx]++; +} + + +static void *Ppmd8_RemoveNode(CPpmd8 *p, unsigned indx) +{ + CPpmd8_Node *node = NODE((CPpmd8_Node_Ref)p->FreeList[indx]); + p->FreeList[indx] = node->Next; + p->Stamps[indx]--; + + return node; +} + + +static void Ppmd8_SplitBlock(CPpmd8 *p, void *ptr, unsigned oldIndx, unsigned newIndx) +{ + unsigned i, nu = I2U(oldIndx) - I2U(newIndx); + ptr = (Byte *)ptr + U2B(I2U(newIndx)); + if (I2U(i = U2I(nu)) != nu) + { + unsigned k = I2U(--i); + Ppmd8_InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1); + } + Ppmd8_InsertNode(p, ptr, i); +} + + + + + + + + + + + + + + +static void Ppmd8_GlueFreeBlocks(CPpmd8 *p) +{ + /* + we use first UInt32 field of 12-bytes UNITs as record type stamp + CPpmd_State { Byte Symbol; Byte Freq; : Freq != 0xFF + CPpmd8_Context { Byte NumStats; Byte Flags; UInt16 SummFreq; : Flags != 0xFF ??? + CPpmd8_Node { UInt32 Stamp : Stamp == 0xFFFFFFFF for free record + : Stamp == 0 for guard + Last 12-bytes UNIT in array is always contains 12-bytes order-0 CPpmd8_Context record + */ + CPpmd8_Node_Ref n; + + p->GlueCount = 1 << 13; + memset(p->Stamps, 0, sizeof(p->Stamps)); + + /* we set guard NODE at LoUnit */ + if (p->LoUnit != p->HiUnit) + ((CPpmd8_Node *)(void *)p->LoUnit)->Stamp = 0; + + { + /* Glue free blocks */ + CPpmd8_Node_Ref *prev = &n; + unsigned i; + for (i = 0; i < PPMD_NUM_INDEXES; i++) + { + + CPpmd8_Node_Ref next = (CPpmd8_Node_Ref)p->FreeList[i]; + p->FreeList[i] = 0; + while (next != 0) + { + CPpmd8_Node *node = NODE(next); + UInt32 nu = node->NU; + *prev = next; + next = node->Next; + if (nu != 0) + { + CPpmd8_Node *node2; + prev = &(node->Next); + while ((node2 = node + nu)->Stamp == EMPTY_NODE) + { + nu += node2->NU; + node2->NU = 0; + node->NU = nu; + } + } + } + } + + *prev = 0; + } + + + + + + + + + + + + + + + + + + + + + /* Fill lists of free blocks */ + while (n != 0) + { + CPpmd8_Node *node = NODE(n); + UInt32 nu = node->NU; + unsigned i; + n = node->Next; + if (nu == 0) + continue; + for (; nu > 128; nu -= 128, node += 128) + Ppmd8_InsertNode(p, node, PPMD_NUM_INDEXES - 1); + if (I2U(i = U2I(nu)) != nu) + { + unsigned k = I2U(--i); + Ppmd8_InsertNode(p, node + k, (unsigned)nu - k - 1); + } + Ppmd8_InsertNode(p, node, i); + } +} + + +Z7_NO_INLINE +static void *Ppmd8_AllocUnitsRare(CPpmd8 *p, unsigned indx) +{ + unsigned i; + + if (p->GlueCount == 0) + { + Ppmd8_GlueFreeBlocks(p); + if (p->FreeList[indx] != 0) + return Ppmd8_RemoveNode(p, indx); + } + + i = indx; + + do + { + if (++i == PPMD_NUM_INDEXES) + { + UInt32 numBytes = U2B(I2U(indx)); + Byte *us = p->UnitsStart; + p->GlueCount--; + return ((UInt32)(us - p->Text) > numBytes) ? (p->UnitsStart = us - numBytes) : (NULL); + } + } + while (p->FreeList[i] == 0); + + { + void *block = Ppmd8_RemoveNode(p, i); + Ppmd8_SplitBlock(p, block, i, indx); + return block; + } +} + + +static void *Ppmd8_AllocUnits(CPpmd8 *p, unsigned indx) +{ + if (p->FreeList[indx] != 0) + return Ppmd8_RemoveNode(p, indx); + { + UInt32 numBytes = U2B(I2U(indx)); + Byte *lo = p->LoUnit; + if ((UInt32)(p->HiUnit - lo) >= numBytes) + { + p->LoUnit = lo + numBytes; + return lo; + } + } + return Ppmd8_AllocUnitsRare(p, indx); +} + + +#define MEM_12_CPY(dest, src, num) \ + { UInt32 *d = (UInt32 *)(dest); \ + const UInt32 *z = (const UInt32 *)(src); \ + unsigned n = (num); \ + do { \ + d[0] = z[0]; \ + d[1] = z[1]; \ + d[2] = z[2]; \ + z += 3; \ + d += 3; \ + } while (--n); \ + } + + + +static void *ShrinkUnits(CPpmd8 *p, void *oldPtr, unsigned oldNU, unsigned newNU) +{ + unsigned i0 = U2I(oldNU); + unsigned i1 = U2I(newNU); + if (i0 == i1) + return oldPtr; + if (p->FreeList[i1] != 0) + { + void *ptr = Ppmd8_RemoveNode(p, i1); + MEM_12_CPY(ptr, oldPtr, newNU) + Ppmd8_InsertNode(p, oldPtr, i0); + return ptr; + } + Ppmd8_SplitBlock(p, oldPtr, i0, i1); + return oldPtr; +} + + +static void FreeUnits(CPpmd8 *p, void *ptr, unsigned nu) +{ + Ppmd8_InsertNode(p, ptr, U2I(nu)); +} + + +static void SpecialFreeUnit(CPpmd8 *p, void *ptr) +{ + if ((Byte *)ptr != p->UnitsStart) + Ppmd8_InsertNode(p, ptr, 0); + else + { + #ifdef PPMD8_FREEZE_SUPPORT + *(UInt32 *)ptr = EMPTY_NODE; /* it's used for (Flags == 0xFF) check in RemoveBinContexts() */ + #endif + p->UnitsStart += UNIT_SIZE; + } +} + + +/* +static void *MoveUnitsUp(CPpmd8 *p, void *oldPtr, unsigned nu) +{ + unsigned indx = U2I(nu); + void *ptr; + if ((Byte *)oldPtr > p->UnitsStart + (1 << 14) || REF(oldPtr) > p->FreeList[indx]) + return oldPtr; + ptr = Ppmd8_RemoveNode(p, indx); + MEM_12_CPY(ptr, oldPtr, nu) + if ((Byte *)oldPtr != p->UnitsStart) + Ppmd8_InsertNode(p, oldPtr, indx); + else + p->UnitsStart += U2B(I2U(indx)); + return ptr; +} +*/ + +static void ExpandTextArea(CPpmd8 *p) +{ + UInt32 count[PPMD_NUM_INDEXES]; + unsigned i; + + memset(count, 0, sizeof(count)); + if (p->LoUnit != p->HiUnit) + ((CPpmd8_Node *)(void *)p->LoUnit)->Stamp = 0; + + { + CPpmd8_Node *node = (CPpmd8_Node *)(void *)p->UnitsStart; + while (node->Stamp == EMPTY_NODE) + { + UInt32 nu = node->NU; + node->Stamp = 0; + count[U2I(nu)]++; + node += nu; + } + p->UnitsStart = (Byte *)node; + } + + for (i = 0; i < PPMD_NUM_INDEXES; i++) + { + UInt32 cnt = count[i]; + if (cnt == 0) + continue; + { + CPpmd8_Node_Ref *prev = (CPpmd8_Node_Ref *)&p->FreeList[i]; + CPpmd8_Node_Ref n = *prev; + p->Stamps[i] -= cnt; + for (;;) + { + CPpmd8_Node *node = NODE(n); + n = node->Next; + if (node->Stamp != 0) + { + prev = &node->Next; + continue; + } + *prev = n; + if (--cnt == 0) + break; + } + } + } +} + + +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) +static void Ppmd8State_SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) +{ + Ppmd_SET_SUCCESSOR(p, v) +} + +#define RESET_TEXT(offs) { p->Text = p->Base + p->AlignOffset + (offs); } + +Z7_NO_INLINE +static +void Ppmd8_RestartModel(CPpmd8 *p) +{ + unsigned i, k, m; + + memset(p->FreeList, 0, sizeof(p->FreeList)); + memset(p->Stamps, 0, sizeof(p->Stamps)); + RESET_TEXT(0) + p->HiUnit = p->Text + p->Size; + p->LoUnit = p->UnitsStart = p->HiUnit - p->Size / 8 / UNIT_SIZE * 7 * UNIT_SIZE; + p->GlueCount = 0; + + p->OrderFall = p->MaxOrder; + p->RunLength = p->InitRL = -(Int32)((p->MaxOrder < 12) ? p->MaxOrder : 12) - 1; + p->PrevSuccess = 0; + + { + CPpmd8_Context *mc = (PPMD8_CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ + CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* Ppmd8_AllocUnits(p, PPMD_NUM_INDEXES - 1); */ + + p->LoUnit += U2B(256 / 2); + p->MaxContext = p->MinContext = mc; + p->FoundState = s; + mc->Flags = 0; + mc->NumStats = 256 - 1; + mc->Union2.SummFreq = 256 + 1; + mc->Union4.Stats = REF(s); + mc->Suffix = 0; + + for (i = 0; i < 256; i++, s++) + { + s->Symbol = (Byte)i; + s->Freq = 1; + Ppmd8State_SetSuccessor(s, 0); + } + } + + + + + + + + + + + + + for (i = m = 0; m < 25; m++) + { + while (p->NS2Indx[i] == m) + i++; + for (k = 0; k < 8; k++) + { + unsigned r; + UInt16 *dest = p->BinSumm[m] + k; + const UInt16 val = (UInt16)(PPMD_BIN_SCALE - PPMD8_kInitBinEsc[k] / (i + 1)); + for (r = 0; r < 64; r += 8) + dest[r] = val; + } + } + + for (i = m = 0; m < 24; m++) + { + unsigned summ; + CPpmd_See *s; + while (p->NS2Indx[(size_t)i + 3] == m + 3) + i++; + s = p->See[m]; + summ = ((2 * i + 5) << (PPMD_PERIOD_BITS - 4)); + for (k = 0; k < 32; k++, s++) + { + s->Summ = (UInt16)summ; + s->Shift = (PPMD_PERIOD_BITS - 4); + s->Count = 7; + } + } + + p->DummySee.Summ = 0; /* unused */ + p->DummySee.Shift = PPMD_PERIOD_BITS; + p->DummySee.Count = 64; /* unused */ +} + + +void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod) +{ + p->MaxOrder = maxOrder; + p->RestoreMethod = restoreMethod; + Ppmd8_RestartModel(p); +} + + +#define FLAG_RESCALED (1 << 2) +// #define FLAG_SYM_HIGH (1 << 3) +#define FLAG_PREV_HIGH (1 << 4) + +#define HiBits_Prepare(sym) ((unsigned)(sym) + 0xC0) + +#define HiBits_Convert_3(flags) (((flags) >> (8 - 3)) & (1 << 3)) +#define HiBits_Convert_4(flags) (((flags) >> (8 - 4)) & (1 << 4)) + +#define PPMD8_HiBitsFlag_3(sym) HiBits_Convert_3(HiBits_Prepare(sym)) +#define PPMD8_HiBitsFlag_4(sym) HiBits_Convert_4(HiBits_Prepare(sym)) + +// #define PPMD8_HiBitsFlag_3(sym) (0x08 * ((sym) >= 0x40)) +// #define PPMD8_HiBitsFlag_4(sym) (0x10 * ((sym) >= 0x40)) + +/* +Refresh() is called when we remove some symbols (successors) in context. +It increases Escape_Freq for sum of all removed symbols. +*/ + +static void Refresh(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned oldNU, unsigned scale) +{ + unsigned i = ctx->NumStats, escFreq, sumFreq, flags; + CPpmd_State *s = (CPpmd_State *)ShrinkUnits(p, STATS(ctx), oldNU, (i + 2) >> 1); + ctx->Union4.Stats = REF(s); + + // #ifdef PPMD8_FREEZE_SUPPORT + /* + (ctx->Union2.SummFreq >= ((UInt32)1 << 15)) can be in FREEZE mode for some files. + It's not good for range coder. So new versions of support fix: + - original PPMdI code rev.1 + + original PPMdI code rev.2 + - 7-Zip default ((PPMD8_FREEZE_SUPPORT is not defined) + + 7-Zip (p->RestoreMethod >= PPMD8_RESTORE_METHOD_FREEZE) + if we use that fixed line, we can lose compatibility with some files created before fix + if we don't use that fixed line, the program can work incorrectly in FREEZE mode in rare case. + */ + // if (p->RestoreMethod >= PPMD8_RESTORE_METHOD_FREEZE) + { + scale |= (ctx->Union2.SummFreq >= ((UInt32)1 << 15)); + } + // #endif + + + + flags = HiBits_Prepare(s->Symbol); + { + unsigned freq = s->Freq; + escFreq = ctx->Union2.SummFreq - freq; + freq = (freq + scale) >> scale; + sumFreq = freq; + s->Freq = (Byte)freq; + } + + do + { + unsigned freq = (++s)->Freq; + escFreq -= freq; + freq = (freq + scale) >> scale; + sumFreq += freq; + s->Freq = (Byte)freq; + flags |= HiBits_Prepare(s->Symbol); + } + while (--i); + + ctx->Union2.SummFreq = (UInt16)(sumFreq + ((escFreq + scale) >> scale)); + ctx->Flags = (Byte)((ctx->Flags & (FLAG_PREV_HIGH + FLAG_RESCALED * scale)) + HiBits_Convert_3(flags)); +} + + +static void SWAP_STATES(CPpmd_State *t1, CPpmd_State *t2) +{ + CPpmd_State tmp = *t1; + *t1 = *t2; + *t2 = tmp; +} + + +/* +CutOff() reduces contexts: + It conversts Successors at MaxOrder to another Contexts to NULL-Successors + It removes RAW-Successors and NULL-Successors that are not Order-0 + and it removes contexts when it has no Successors. + if the (Union4.Stats) is close to (UnitsStart), it moves it up. +*/ + +static CPpmd_Void_Ref CutOff(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned order) +{ + int ns = ctx->NumStats; + unsigned nu; + CPpmd_State *stats; + + if (ns == 0) + { + CPpmd_State *s = ONE_STATE(ctx); + CPpmd_Void_Ref successor = SUCCESSOR(s); + if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart) + { + if (order < p->MaxOrder) + successor = CutOff(p, CTX(successor), order + 1); + else + successor = 0; + Ppmd8State_SetSuccessor(s, successor); + if (successor || order <= 9) /* O_BOUND */ + return REF(ctx); + } + SpecialFreeUnit(p, ctx); + return 0; + } + + nu = ((unsigned)ns + 2) >> 1; + // ctx->Union4.Stats = STATS_REF(MoveUnitsUp(p, STATS(ctx), nu)); + { + unsigned indx = U2I(nu); + stats = STATS(ctx); + + if ((UInt32)((Byte *)stats - p->UnitsStart) <= (1 << 14) + && (CPpmd_Void_Ref)ctx->Union4.Stats <= p->FreeList[indx]) + { + void *ptr = Ppmd8_RemoveNode(p, indx); + ctx->Union4.Stats = STATS_REF(ptr); + MEM_12_CPY(ptr, (const void *)stats, nu) + if ((Byte *)stats != p->UnitsStart) + Ppmd8_InsertNode(p, stats, indx); + else + p->UnitsStart += U2B(I2U(indx)); + stats = ptr; + } + } + + { + CPpmd_State *s = stats + (unsigned)ns; + do + { + CPpmd_Void_Ref successor = SUCCESSOR(s); + if ((Byte *)Ppmd8_GetPtr(p, successor) < p->UnitsStart) + { + CPpmd_State *s2 = stats + (unsigned)(ns--); + if (order) + { + if (s != s2) + *s = *s2; + } + else + { + SWAP_STATES(s, s2); + Ppmd8State_SetSuccessor(s2, 0); + } + } + else + { + if (order < p->MaxOrder) + Ppmd8State_SetSuccessor(s, CutOff(p, CTX(successor), order + 1)); + else + Ppmd8State_SetSuccessor(s, 0); + } + } + while (--s >= stats); + } + + if (ns != ctx->NumStats && order) + { + if (ns < 0) + { + FreeUnits(p, stats, nu); + SpecialFreeUnit(p, ctx); + return 0; + } + ctx->NumStats = (Byte)ns; + if (ns == 0) + { + const Byte sym = stats->Symbol; + ctx->Flags = (Byte)((ctx->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(sym)); + // *ONE_STATE(ctx) = *stats; + ctx->Union2.State2.Symbol = sym; + ctx->Union2.State2.Freq = (Byte)(((unsigned)stats->Freq + 11) >> 3); + ctx->Union4.State4.Successor_0 = stats->Successor_0; + ctx->Union4.State4.Successor_1 = stats->Successor_1; + FreeUnits(p, stats, nu); + } + else + { + Refresh(p, ctx, nu, ctx->Union2.SummFreq > 16 * (unsigned)ns); + } + } + + return REF(ctx); +} + + + +#ifdef PPMD8_FREEZE_SUPPORT + +/* +RemoveBinContexts() + It conversts Successors at MaxOrder to another Contexts to NULL-Successors + It changes RAW-Successors to NULL-Successors + removes Bin Context without Successor, if suffix of that context is also binary. +*/ + +static CPpmd_Void_Ref RemoveBinContexts(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned order) +{ + if (!ctx->NumStats) + { + CPpmd_State *s = ONE_STATE(ctx); + CPpmd_Void_Ref successor = SUCCESSOR(s); + if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder) + successor = RemoveBinContexts(p, CTX(successor), order + 1); + else + successor = 0; + Ppmd8State_SetSuccessor(s, successor); + /* Suffix context can be removed already, since different (high-order) + Successors may refer to same context. So we check Flags == 0xFF (Stamp == EMPTY_NODE) */ + if (!successor && (!SUFFIX(ctx)->NumStats || SUFFIX(ctx)->Flags == 0xFF)) + { + FreeUnits(p, ctx, 1); + return 0; + } + } + else + { + CPpmd_State *s = STATS(ctx) + ctx->NumStats; + do + { + CPpmd_Void_Ref successor = SUCCESSOR(s); + if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder) + Ppmd8State_SetSuccessor(s, RemoveBinContexts(p, CTX(successor), order + 1)); + else + Ppmd8State_SetSuccessor(s, 0); + } + while (--s >= STATS(ctx)); + } + + return REF(ctx); +} + +#endif + + + +static UInt32 GetUsedMemory(const CPpmd8 *p) +{ + UInt32 v = 0; + unsigned i; + for (i = 0; i < PPMD_NUM_INDEXES; i++) + v += p->Stamps[i] * I2U(i); + return p->Size - (UInt32)(p->HiUnit - p->LoUnit) - (UInt32)(p->UnitsStart - p->Text) - U2B(v); +} + +#ifdef PPMD8_FREEZE_SUPPORT + #define RESTORE_MODEL(c1, fSuccessor) RestoreModel(p, c1, fSuccessor) +#else + #define RESTORE_MODEL(c1, fSuccessor) RestoreModel(p, c1) +#endif + + +static void RestoreModel(CPpmd8 *p, PPMD8_CTX_PTR ctxError + #ifdef PPMD8_FREEZE_SUPPORT + , PPMD8_CTX_PTR fSuccessor + #endif + ) +{ + PPMD8_CTX_PTR c; + CPpmd_State *s; + RESET_TEXT(0) + + // we go here in cases of error of allocation for context (c1) + // Order(MinContext) < Order(ctxError) <= Order(MaxContext) + + // We remove last symbol from each of contexts [p->MaxContext ... ctxError) contexts + // So we rollback all created (symbols) before error. + for (c = p->MaxContext; c != ctxError; c = SUFFIX(c)) + if (--(c->NumStats) == 0) + { + s = STATS(c); + c->Flags = (Byte)((c->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(s->Symbol)); + // *ONE_STATE(c) = *s; + c->Union2.State2.Symbol = s->Symbol; + c->Union2.State2.Freq = (Byte)(((unsigned)s->Freq + 11) >> 3); + c->Union4.State4.Successor_0 = s->Successor_0; + c->Union4.State4.Successor_1 = s->Successor_1; + + SpecialFreeUnit(p, s); + } + else + { + /* Refresh() can increase Escape_Freq on value of Freq of last symbol, that was added before error. + so the largest possible increase for Escape_Freq is (8) from value before ModelUpoadet() */ + Refresh(p, c, ((unsigned)c->NumStats + 3) >> 1, 0); + } + + // increase Escape Freq for context [ctxError ... p->MinContext) + for (; c != p->MinContext; c = SUFFIX(c)) + if (c->NumStats == 0) + { + // ONE_STATE(c) + c->Union2.State2.Freq = (Byte)(((unsigned)c->Union2.State2.Freq + 1) >> 1); + } + else if ((c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 4)) > 128 + 4 * c->NumStats) + Refresh(p, c, ((unsigned)c->NumStats + 2) >> 1, 1); + + #ifdef PPMD8_FREEZE_SUPPORT + if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) + { + p->MaxContext = fSuccessor; + p->GlueCount += !(p->Stamps[1] & 1); // why? + } + else if (p->RestoreMethod == PPMD8_RESTORE_METHOD_FREEZE) + { + while (p->MaxContext->Suffix) + p->MaxContext = SUFFIX(p->MaxContext); + RemoveBinContexts(p, p->MaxContext, 0); + // we change the current mode to (PPMD8_RESTORE_METHOD_FREEZE + 1) + p->RestoreMethod = PPMD8_RESTORE_METHOD_FREEZE + 1; + p->GlueCount = 0; + p->OrderFall = p->MaxOrder; + } + else + #endif + if (p->RestoreMethod == PPMD8_RESTORE_METHOD_RESTART || GetUsedMemory(p) < (p->Size >> 1)) + Ppmd8_RestartModel(p); + else + { + while (p->MaxContext->Suffix) + p->MaxContext = SUFFIX(p->MaxContext); + do + { + CutOff(p, p->MaxContext, 0); + ExpandTextArea(p); + } + while (GetUsedMemory(p) > 3 * (p->Size >> 2)); + p->GlueCount = 0; + p->OrderFall = p->MaxOrder; + } + p->MinContext = p->MaxContext; +} + + + +Z7_NO_INLINE +static PPMD8_CTX_PTR Ppmd8_CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, PPMD8_CTX_PTR c) +{ + + CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); + Byte newSym, newFreq, flags; + unsigned numPs = 0; + CPpmd_State *ps[PPMD8_MAX_ORDER + 1]; /* fixed over Shkarin's code. Maybe it could work without + 1 too. */ + + if (!skip) + ps[numPs++] = p->FoundState; + + while (c->Suffix) + { + CPpmd_Void_Ref successor; + CPpmd_State *s; + c = SUFFIX(c); + + if (s1) { s = s1; s1 = NULL; } + else if (c->NumStats != 0) + { + Byte sym = p->FoundState->Symbol; + for (s = STATS(c); s->Symbol != sym; s++); + if (s->Freq < MAX_FREQ - 9) { s->Freq++; c->Union2.SummFreq++; } + } + else + { + s = ONE_STATE(c); + s->Freq = (Byte)(s->Freq + (!SUFFIX(c)->NumStats & (s->Freq < 24))); + } + successor = SUCCESSOR(s); + if (successor != upBranch) + { + + c = CTX(successor); + if (numPs == 0) + { + + + return c; + } + break; + } + ps[numPs++] = s; + } + + + + + + newSym = *(const Byte *)Ppmd8_GetPtr(p, upBranch); + upBranch++; + flags = (Byte)(PPMD8_HiBitsFlag_4(p->FoundState->Symbol) + PPMD8_HiBitsFlag_3(newSym)); + + if (c->NumStats == 0) + newFreq = c->Union2.State2.Freq; + else + { + UInt32 cf, s0; + CPpmd_State *s; + for (s = STATS(c); s->Symbol != newSym; s++); + cf = (UInt32)s->Freq - 1; + s0 = (UInt32)c->Union2.SummFreq - c->NumStats - cf; + /* + + + max(newFreq)= (s->Freq - 1), when (s0 == 1) + + + */ + newFreq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((cf + 2 * s0 - 3) / s0))); + } + + + + do + { + PPMD8_CTX_PTR c1; + /* = AllocContext(p); */ + if (p->HiUnit != p->LoUnit) + c1 = (PPMD8_CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); + else if (p->FreeList[0] != 0) + c1 = (PPMD8_CTX_PTR)Ppmd8_RemoveNode(p, 0); + else + { + c1 = (PPMD8_CTX_PTR)Ppmd8_AllocUnitsRare(p, 0); + if (!c1) + return NULL; + } + c1->Flags = flags; + c1->NumStats = 0; + c1->Union2.State2.Symbol = newSym; + c1->Union2.State2.Freq = newFreq; + Ppmd8State_SetSuccessor(ONE_STATE(c1), upBranch); + c1->Suffix = REF(c); + Ppmd8State_SetSuccessor(ps[--numPs], REF(c1)); + c = c1; + } + while (numPs != 0); + + return c; +} + + +static PPMD8_CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, PPMD8_CTX_PTR c) +{ + CPpmd_State *s = NULL; + PPMD8_CTX_PTR c1 = c; + CPpmd_Void_Ref upBranch = REF(p->Text); + + #ifdef PPMD8_FREEZE_SUPPORT + /* The BUG in Shkarin's code was fixed: ps could overflow in CUT_OFF mode. */ + CPpmd_State *ps[PPMD8_MAX_ORDER + 1]; + unsigned numPs = 0; + ps[numPs++] = p->FoundState; + #endif + + Ppmd8State_SetSuccessor(p->FoundState, upBranch); + p->OrderFall++; + + for (;;) + { + if (s1) + { + c = SUFFIX(c); + s = s1; + s1 = NULL; + } + else + { + if (!c->Suffix) + { + #ifdef PPMD8_FREEZE_SUPPORT + if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) + { + do { Ppmd8State_SetSuccessor(ps[--numPs], REF(c)); } while (numPs); + RESET_TEXT(1) + p->OrderFall = 1; + } + #endif + return c; + } + c = SUFFIX(c); + if (c->NumStats) + { + if ((s = STATS(c))->Symbol != p->FoundState->Symbol) + do { s++; } while (s->Symbol != p->FoundState->Symbol); + if (s->Freq < MAX_FREQ - 9) + { + s->Freq = (Byte)(s->Freq + 2); + c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2); + } + } + else + { + s = ONE_STATE(c); + s->Freq = (Byte)(s->Freq + (s->Freq < 32)); + } + } + if (SUCCESSOR(s)) + break; + #ifdef PPMD8_FREEZE_SUPPORT + ps[numPs++] = s; + #endif + Ppmd8State_SetSuccessor(s, upBranch); + p->OrderFall++; + } + + #ifdef PPMD8_FREEZE_SUPPORT + if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) + { + c = CTX(SUCCESSOR(s)); + do { Ppmd8State_SetSuccessor(ps[--numPs], REF(c)); } while (numPs); + RESET_TEXT(1) + p->OrderFall = 1; + return c; + } + else + #endif + if (SUCCESSOR(s) <= upBranch) + { + PPMD8_CTX_PTR successor; + CPpmd_State *s2 = p->FoundState; + p->FoundState = s; + + successor = Ppmd8_CreateSuccessors(p, False, NULL, c); + if (!successor) + Ppmd8State_SetSuccessor(s, 0); + else + Ppmd8State_SetSuccessor(s, REF(successor)); + p->FoundState = s2; + } + + { + CPpmd_Void_Ref successor = SUCCESSOR(s); + if (p->OrderFall == 1 && c1 == p->MaxContext) + { + Ppmd8State_SetSuccessor(p->FoundState, successor); + p->Text--; + } + if (successor == 0) + return NULL; + return CTX(successor); + } +} + + + +void Ppmd8_UpdateModel(CPpmd8 *p); +Z7_NO_INLINE +void Ppmd8_UpdateModel(CPpmd8 *p) +{ + CPpmd_Void_Ref maxSuccessor, minSuccessor = SUCCESSOR(p->FoundState); + PPMD8_CTX_PTR c; + unsigned s0, ns, fFreq = p->FoundState->Freq; + Byte flag, fSymbol = p->FoundState->Symbol; + { + CPpmd_State *s = NULL; + if (p->FoundState->Freq < MAX_FREQ / 4 && p->MinContext->Suffix != 0) + { + /* Update Freqs in Suffix Context */ + + c = SUFFIX(p->MinContext); + + if (c->NumStats == 0) + { + s = ONE_STATE(c); + if (s->Freq < 32) + s->Freq++; + } + else + { + Byte sym = p->FoundState->Symbol; + s = STATS(c); + + if (s->Symbol != sym) + { + do + { + + s++; + } + while (s->Symbol != sym); + + if (s[0].Freq >= s[-1].Freq) + { + SWAP_STATES(&s[0], &s[-1]); + s--; + } + } + + if (s->Freq < MAX_FREQ - 9) + { + s->Freq = (Byte)(s->Freq + 2); + c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2); + } + } + } + + c = p->MaxContext; + if (p->OrderFall == 0 && minSuccessor) + { + PPMD8_CTX_PTR cs = Ppmd8_CreateSuccessors(p, True, s, p->MinContext); + if (!cs) + { + Ppmd8State_SetSuccessor(p->FoundState, 0); + RESTORE_MODEL(c, CTX(minSuccessor)); + return; + } + Ppmd8State_SetSuccessor(p->FoundState, REF(cs)); + p->MinContext = p->MaxContext = cs; + return; + } + + + + + { + Byte *text = p->Text; + *text++ = p->FoundState->Symbol; + p->Text = text; + if (text >= p->UnitsStart) + { + RESTORE_MODEL(c, CTX(minSuccessor)); /* check it */ + return; + } + maxSuccessor = REF(text); + } + + if (!minSuccessor) + { + PPMD8_CTX_PTR cs = ReduceOrder(p, s, p->MinContext); + if (!cs) + { + RESTORE_MODEL(c, NULL); + return; + } + minSuccessor = REF(cs); + } + else if ((Byte *)Ppmd8_GetPtr(p, minSuccessor) < p->UnitsStart) + { + PPMD8_CTX_PTR cs = Ppmd8_CreateSuccessors(p, False, s, p->MinContext); + if (!cs) + { + RESTORE_MODEL(c, NULL); + return; + } + minSuccessor = REF(cs); + } + + if (--p->OrderFall == 0) + { + maxSuccessor = minSuccessor; + p->Text -= (p->MaxContext != p->MinContext); + } + #ifdef PPMD8_FREEZE_SUPPORT + else if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) + { + maxSuccessor = minSuccessor; + RESET_TEXT(0) + p->OrderFall = 0; + } + #endif + } + + + + + + + + + + + + + + + + + + + + + + + + + + + + + flag = (Byte)(PPMD8_HiBitsFlag_3(fSymbol)); + s0 = p->MinContext->Union2.SummFreq - (ns = p->MinContext->NumStats) - fFreq; + + for (; c != p->MinContext; c = SUFFIX(c)) + { + unsigned ns1; + UInt32 sum; + + if ((ns1 = c->NumStats) != 0) + { + if ((ns1 & 1) != 0) + { + /* Expand for one UNIT */ + const unsigned oldNU = (ns1 + 1) >> 1; + const unsigned i = U2I(oldNU); + if (i != U2I((size_t)oldNU + 1)) + { + void *ptr = Ppmd8_AllocUnits(p, i + 1); + void *oldPtr; + if (!ptr) + { + RESTORE_MODEL(c, CTX(minSuccessor)); + return; + } + oldPtr = STATS(c); + MEM_12_CPY(ptr, oldPtr, oldNU) + Ppmd8_InsertNode(p, oldPtr, i); + c->Union4.Stats = STATS_REF(ptr); + } + } + sum = c->Union2.SummFreq; + /* max increase of Escape_Freq is 1 here. + an average increase is 1/3 per symbol */ + sum += (UInt32)(unsigned)(3 * ns1 + 1 < ns); + /* original PPMdH uses 16-bit variable for (sum) here. + But (sum < ???). Do we need to truncate (sum) to 16-bit */ + // sum = (UInt16)sum; + } + else + { + + CPpmd_State *s = (CPpmd_State*)Ppmd8_AllocUnits(p, 0); + if (!s) + { + RESTORE_MODEL(c, CTX(minSuccessor)); + return; + } + { + unsigned freq = c->Union2.State2.Freq; + // s = *ONE_STATE(c); + s->Symbol = c->Union2.State2.Symbol; + s->Successor_0 = c->Union4.State4.Successor_0; + s->Successor_1 = c->Union4.State4.Successor_1; + // Ppmd8State_SetSuccessor(s, c->Union4.Stats); // call it only for debug purposes to check the order of + // (Successor_0 and Successor_1) in LE/BE. + c->Union4.Stats = REF(s); + if (freq < MAX_FREQ / 4 - 1) + freq <<= 1; + else + freq = MAX_FREQ - 4; + + s->Freq = (Byte)freq; + + sum = (UInt32)(freq + p->InitEsc + (ns > 2)); // Ppmd8 (> 2) + } + } + + { + CPpmd_State *s = STATS(c) + ns1 + 1; + UInt32 cf = 2 * (sum + 6) * (UInt32)fFreq; + UInt32 sf = (UInt32)s0 + sum; + s->Symbol = fSymbol; + c->NumStats = (Byte)(ns1 + 1); + Ppmd8State_SetSuccessor(s, maxSuccessor); + c->Flags |= flag; + if (cf < 6 * sf) + { + cf = (unsigned)1 + (cf > sf) + (cf >= 4 * sf); + sum += 4; + /* It can add (1, 2, 3) to Escape_Freq */ + } + else + { + cf = (unsigned)4 + (cf > 9 * sf) + (cf > 12 * sf) + (cf > 15 * sf); + sum += cf; + } + + c->Union2.SummFreq = (UInt16)sum; + s->Freq = (Byte)cf; + } + + } + p->MaxContext = p->MinContext = CTX(minSuccessor); +} + + + +Z7_NO_INLINE +static void Ppmd8_Rescale(CPpmd8 *p) +{ + unsigned i, adder, sumFreq, escFreq; + CPpmd_State *stats = STATS(p->MinContext); + CPpmd_State *s = p->FoundState; + + /* Sort the list by Freq */ + if (s != stats) + { + CPpmd_State tmp = *s; + do + s[0] = s[-1]; + while (--s != stats); + *s = tmp; + } + + sumFreq = s->Freq; + escFreq = p->MinContext->Union2.SummFreq - sumFreq; + + + + + + + adder = (p->OrderFall != 0); + + #ifdef PPMD8_FREEZE_SUPPORT + adder |= (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE); + #endif + + sumFreq = (sumFreq + 4 + adder) >> 1; + i = p->MinContext->NumStats; + s->Freq = (Byte)sumFreq; + + do + { + unsigned freq = (++s)->Freq; + escFreq -= freq; + freq = (freq + adder) >> 1; + sumFreq += freq; + s->Freq = (Byte)freq; + if (freq > s[-1].Freq) + { + CPpmd_State tmp = *s; + CPpmd_State *s1 = s; + do + { + s1[0] = s1[-1]; + } + while (--s1 != stats && freq > s1[-1].Freq); + *s1 = tmp; + } + } + while (--i); + + if (s->Freq == 0) + { + /* Remove all items with Freq == 0 */ + CPpmd8_Context *mc; + unsigned numStats, numStatsNew, n0, n1; + + i = 0; do { i++; } while ((--s)->Freq == 0); + + + + + escFreq += i; + mc = p->MinContext; + numStats = mc->NumStats; + numStatsNew = numStats - i; + mc->NumStats = (Byte)(numStatsNew); + n0 = (numStats + 2) >> 1; + + if (numStatsNew == 0) + { + + unsigned freq = (2 * (unsigned)stats->Freq + escFreq - 1) / escFreq; + if (freq > MAX_FREQ / 3) + freq = MAX_FREQ / 3; + mc->Flags = (Byte)((mc->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(stats->Symbol)); + + + + + + s = ONE_STATE(mc); + *s = *stats; + s->Freq = (Byte)freq; + p->FoundState = s; + Ppmd8_InsertNode(p, stats, U2I(n0)); + return; + } + + n1 = (numStatsNew + 2) >> 1; + if (n0 != n1) + mc->Union4.Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); + { + // here we are for max order only. So Ppmd8_MakeEscFreq() doesn't use mc->Flags + // but we still need current (Flags & FLAG_PREV_HIGH), if we will convert context to 1-symbol context later. + /* + unsigned flags = HiBits_Prepare((s = STATS(mc))->Symbol); + i = mc->NumStats; + do { flags |= HiBits_Prepare((++s)->Symbol); } while (--i); + mc->Flags = (Byte)((mc->Flags & ~FLAG_SYM_HIGH) + HiBits_Convert_3(flags)); + */ + } + } + + + + + + + { + CPpmd8_Context *mc = p->MinContext; + mc->Union2.SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); + mc->Flags |= FLAG_RESCALED; + p->FoundState = STATS(mc); + } +} + + +CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked1, UInt32 *escFreq) +{ + CPpmd_See *see; + const CPpmd8_Context *mc = p->MinContext; + unsigned numStats = mc->NumStats; + if (numStats != 0xFF) + { + // (3 <= numStats + 2 <= 256) (3 <= NS2Indx[3] and NS2Indx[256] === 26) + see = p->See[(size_t)(unsigned)p->NS2Indx[(size_t)numStats + 2] - 3] + + (mc->Union2.SummFreq > 11 * (numStats + 1)) + + 2 * (unsigned)(2 * numStats < ((unsigned)SUFFIX(mc)->NumStats + numMasked1)) + + mc->Flags; + + { + // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ + const unsigned summ = (UInt16)see->Summ; // & 0xFFFF + const unsigned r = (summ >> see->Shift); + see->Summ = (UInt16)(summ - r); + *escFreq = (UInt32)(r + (r == 0)); + } + } + else + { + see = &p->DummySee; + *escFreq = 1; + } + return see; +} + + +static void Ppmd8_NextContext(CPpmd8 *p) +{ + PPMD8_CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); + if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart) + p->MaxContext = p->MinContext = c; + else + Ppmd8_UpdateModel(p); +} + + +void Ppmd8_Update1(CPpmd8 *p) +{ + CPpmd_State *s = p->FoundState; + unsigned freq = s->Freq; + freq += 4; + p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); + s->Freq = (Byte)freq; + if (freq > s[-1].Freq) + { + SWAP_STATES(s, &s[-1]); + p->FoundState = --s; + if (freq > MAX_FREQ) + Ppmd8_Rescale(p); + } + Ppmd8_NextContext(p); +} + + +void Ppmd8_Update1_0(CPpmd8 *p) +{ + CPpmd_State *s = p->FoundState; + CPpmd8_Context *mc = p->MinContext; + unsigned freq = s->Freq; + const unsigned summFreq = mc->Union2.SummFreq; + p->PrevSuccess = (2 * freq >= summFreq); // Ppmd8 (>=) + p->RunLength += (Int32)p->PrevSuccess; + mc->Union2.SummFreq = (UInt16)(summFreq + 4); + freq += 4; + s->Freq = (Byte)freq; + if (freq > MAX_FREQ) + Ppmd8_Rescale(p); + Ppmd8_NextContext(p); +} + + +/* +void Ppmd8_UpdateBin(CPpmd8 *p) +{ + unsigned freq = p->FoundState->Freq; + p->FoundState->Freq = (Byte)(freq + (freq < 196)); // Ppmd8 (196) + p->PrevSuccess = 1; + p->RunLength++; + Ppmd8_NextContext(p); +} +*/ + +void Ppmd8_Update2(CPpmd8 *p) +{ + CPpmd_State *s = p->FoundState; + unsigned freq = s->Freq; + freq += 4; + p->RunLength = p->InitRL; + p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); + s->Freq = (Byte)freq; + if (freq > MAX_FREQ) + Ppmd8_Rescale(p); + Ppmd8_UpdateModel(p); +} + +/* H->I changes: + NS2Indx + GlueCount, and Glue method + BinSum + See / EscFreq + Ppmd8_CreateSuccessors updates more suffix contexts + Ppmd8_UpdateModel consts. + PrevSuccess Update + +Flags: + (1 << 2) - the Context was Rescaled + (1 << 3) - there is symbol in Stats with (sym >= 0x40) in + (1 << 4) - main symbol of context is (sym >= 0x40) +*/ + +#undef RESET_TEXT +#undef FLAG_RESCALED +#undef FLAG_PREV_HIGH +#undef HiBits_Prepare +#undef HiBits_Convert_3 +#undef HiBits_Convert_4 +#undef PPMD8_HiBitsFlag_3 +#undef PPMD8_HiBitsFlag_4 +#undef RESTORE_MODEL + +#undef MAX_FREQ +#undef UNIT_SIZE +#undef U2B +#undef U2I +#undef I2U + +#undef REF +#undef STATS_REF +#undef CTX +#undef STATS +#undef ONE_STATE +#undef SUFFIX +#undef NODE +#undef EMPTY_NODE +#undef MEM_12_CPY +#undef SUCCESSOR +#undef SWAP_STATES diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8.h new file mode 100644 index 0000000..d5bb57e --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8.h @@ -0,0 +1,181 @@ +/* Ppmd8.h -- Ppmd8 (PPMdI) compression codec +2023-04-02 : Igor Pavlov : Public domain +This code is based on: + PPMd var.I (2002): Dmitry Shkarin : Public domain + Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ + +#ifndef ZIP7_INC_PPMD8_H +#define ZIP7_INC_PPMD8_H + +#include "Ppmd.h" + +EXTERN_C_BEGIN + +#define PPMD8_MIN_ORDER 2 +#define PPMD8_MAX_ORDER 16 + + + + +struct CPpmd8_Context_; + +typedef Ppmd_Ref_Type(struct CPpmd8_Context_) CPpmd8_Context_Ref; + +// MY_CPU_pragma_pack_push_1 + +typedef struct CPpmd8_Context_ +{ + Byte NumStats; + Byte Flags; + + union + { + UInt16 SummFreq; + CPpmd_State2 State2; + } Union2; + + union + { + CPpmd_State_Ref Stats; + CPpmd_State4 State4; + } Union4; + + CPpmd8_Context_Ref Suffix; +} CPpmd8_Context; + +// MY_CPU_pragma_pop + +#define Ppmd8Context_OneState(p) ((CPpmd_State *)&(p)->Union2) + +/* PPMdI code rev.2 contains the fix over PPMdI code rev.1. + But the code PPMdI.2 is not compatible with PPMdI.1 for some files compressed + in FREEZE mode. So we disable FREEZE mode support. */ + +// #define PPMD8_FREEZE_SUPPORT + +enum +{ + PPMD8_RESTORE_METHOD_RESTART, + PPMD8_RESTORE_METHOD_CUT_OFF + #ifdef PPMD8_FREEZE_SUPPORT + , PPMD8_RESTORE_METHOD_FREEZE + #endif + , PPMD8_RESTORE_METHOD_UNSUPPPORTED +}; + + + + + + + + +typedef struct +{ + CPpmd8_Context *MinContext, *MaxContext; + CPpmd_State *FoundState; + unsigned OrderFall, InitEsc, PrevSuccess, MaxOrder, RestoreMethod; + Int32 RunLength, InitRL; /* must be 32-bit at least */ + + UInt32 Size; + UInt32 GlueCount; + UInt32 AlignOffset; + Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; + + UInt32 Range; + UInt32 Code; + UInt32 Low; + union + { + IByteInPtr In; + IByteOutPtr Out; + } Stream; + + Byte Indx2Units[PPMD_NUM_INDEXES + 2]; // +2 for alignment + Byte Units2Indx[128]; + CPpmd_Void_Ref FreeList[PPMD_NUM_INDEXES]; + UInt32 Stamps[PPMD_NUM_INDEXES]; + Byte NS2BSIndx[256], NS2Indx[260]; + Byte ExpEscape[16]; + CPpmd_See DummySee, See[24][32]; + UInt16 BinSumm[25][64]; + +} CPpmd8; + + +void Ppmd8_Construct(CPpmd8 *p); +BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc); +void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc); +void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod); +#define Ppmd8_WasAllocated(p) ((p)->Base != NULL) + + +/* ---------- Internal Functions ---------- */ + +#define Ppmd8_GetPtr(p, ptr) Ppmd_GetPtr(p, ptr) +#define Ppmd8_GetContext(p, ptr) Ppmd_GetPtr_Type(p, ptr, CPpmd8_Context) +#define Ppmd8_GetStats(p, ctx) Ppmd_GetPtr_Type(p, (ctx)->Union4.Stats, CPpmd_State) + +void Ppmd8_Update1(CPpmd8 *p); +void Ppmd8_Update1_0(CPpmd8 *p); +void Ppmd8_Update2(CPpmd8 *p); + + + + + + +#define Ppmd8_GetBinSumm(p) \ + &p->BinSumm[p->NS2Indx[(size_t)Ppmd8Context_OneState(p->MinContext)->Freq - 1]] \ + [ p->PrevSuccess + ((p->RunLength >> 26) & 0x20) \ + + p->NS2BSIndx[Ppmd8_GetContext(p, p->MinContext->Suffix)->NumStats] + \ + + p->MinContext->Flags ] + + +CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked, UInt32 *scale); + + +/* 20.01: the original PPMdI encoder and decoder probably could work incorrectly in some rare cases, + where the original PPMdI code can give "Divide by Zero" operation. + We use the following fix to allow correct working of encoder and decoder in any cases. + We correct (Escape_Freq) and (_sum_), if (_sum_) is larger than p->Range) */ +#define PPMD8_CORRECT_SUM_RANGE(p, _sum_) if (_sum_ > p->Range /* /1 */) _sum_ = p->Range; + + +/* ---------- Decode ---------- */ + +#define PPMD8_SYM_END (-1) +#define PPMD8_SYM_ERROR (-2) + +/* +You must set (CPpmd8::Stream.In) before Ppmd8_RangeDec_Init() + +Ppmd8_DecodeSymbol() +out: + >= 0 : decoded byte + -1 : PPMD8_SYM_END : End of payload marker + -2 : PPMD8_SYM_ERROR : Data error +*/ + + +BoolInt Ppmd8_Init_RangeDec(CPpmd8 *p); +#define Ppmd8_RangeDec_IsFinishedOK(p) ((p)->Code == 0) +int Ppmd8_DecodeSymbol(CPpmd8 *p); + + + + + + + + +/* ---------- Encode ---------- */ + +#define Ppmd8_Init_RangeEnc(p) { (p)->Low = 0; (p)->Range = 0xFFFFFFFF; } +void Ppmd8_Flush_RangeEnc(CPpmd8 *p); +void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol); + + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8Dec.c b/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8Dec.c new file mode 100644 index 0000000..ff91167 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8Dec.c @@ -0,0 +1,295 @@ +/* Ppmd8Dec.c -- Ppmd8 (PPMdI) Decoder +2023-09-07 : Igor Pavlov : Public domain +This code is based on: + PPMd var.I (2002): Dmitry Shkarin : Public domain + Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ + +#include "Precomp.h" + +#include "Ppmd8.h" + +#define kTop ((UInt32)1 << 24) +#define kBot ((UInt32)1 << 15) + +#define READ_BYTE(p) IByteIn_Read((p)->Stream.In) + +BoolInt Ppmd8_Init_RangeDec(CPpmd8 *p) +{ + unsigned i; + p->Code = 0; + p->Range = 0xFFFFFFFF; + p->Low = 0; + + for (i = 0; i < 4; i++) + p->Code = (p->Code << 8) | READ_BYTE(p); + return (p->Code < 0xFFFFFFFF); +} + +#define RC_NORM(p) \ + while ((p->Low ^ (p->Low + p->Range)) < kTop \ + || (p->Range < kBot && ((p->Range = (0 - p->Low) & (kBot - 1)), 1))) { \ + p->Code = (p->Code << 8) | READ_BYTE(p); \ + p->Range <<= 8; p->Low <<= 8; } + +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) + +#define R p + +Z7_FORCE_INLINE +// Z7_NO_INLINE +static void Ppmd8_RD_Decode(CPpmd8 *p, UInt32 start, UInt32 size) +{ + start *= R->Range; + R->Low += start; + R->Code -= start; + R->Range *= size; + RC_NORM_LOCAL(R) +} + +#define RC_Decode(start, size) Ppmd8_RD_Decode(p, start, size); +#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) +#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) + + +#define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref)) +// typedef CPpmd8_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) +void Ppmd8_UpdateModel(CPpmd8 *p); + +#define MASK(sym) ((Byte *)charMask)[sym] + + +int Ppmd8_DecodeSymbol(CPpmd8 *p) +{ + size_t charMask[256 / sizeof(size_t)]; + + if (p->MinContext->NumStats != 0) + { + CPpmd_State *s = Ppmd8_GetStats(p, p->MinContext); + unsigned i; + UInt32 count, hiCnt; + UInt32 summFreq = p->MinContext->Union2.SummFreq; + + PPMD8_CORRECT_SUM_RANGE(p, summFreq) + + + count = RC_GetThreshold(summFreq); + hiCnt = count; + + if ((Int32)(count -= s->Freq) < 0) + { + Byte sym; + RC_DecodeFinal(0, s->Freq) + p->FoundState = s; + sym = s->Symbol; + Ppmd8_Update1_0(p); + return sym; + } + + p->PrevSuccess = 0; + i = p->MinContext->NumStats; + + do + { + if ((Int32)(count -= (++s)->Freq) < 0) + { + Byte sym; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq) + p->FoundState = s; + sym = s->Symbol; + Ppmd8_Update1(p); + return sym; + } + } + while (--i); + + if (hiCnt >= summFreq) + return PPMD8_SYM_ERROR; + + hiCnt -= count; + RC_Decode(hiCnt, summFreq - hiCnt) + + + PPMD_SetAllBitsIn256Bytes(charMask) + // i = p->MinContext->NumStats - 1; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd8_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } + } + else + { + CPpmd_State *s = Ppmd8Context_OneState(p->MinContext); + UInt16 *prob = Ppmd8_GetBinSumm(p); + UInt32 pr = *prob; + UInt32 size0 = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); + + if (R->Code < size0) + { + Byte sym; + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + + // RangeDec_DecodeBit0(size0); + R->Range = size0; + RC_NORM(R) + + + + // sym = (p->FoundState = Ppmd8Context_OneState(p->MinContext))->Symbol; + // Ppmd8_UpdateBin(p); + { + unsigned freq = s->Freq; + CPpmd8_Context *c = CTX(SUCCESSOR(s)); + sym = s->Symbol; + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 196)); + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart) + p->MaxContext = p->MinContext = c; + else + Ppmd8_UpdateModel(p); + } + return sym; + } + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + + // RangeDec_DecodeBit1(rc2, size0); + R->Low += size0; + R->Code -= size0; + R->Range = (R->Range & ~((UInt32)PPMD_BIN_SCALE - 1)) - size0; + RC_NORM_LOCAL(R) + + PPMD_SetAllBitsIn256Bytes(charMask) + MASK(Ppmd8Context_OneState(p->MinContext)->Symbol) = 0; + p->PrevSuccess = 0; + } + + for (;;) + { + CPpmd_State *s, *s2; + UInt32 freqSum, count, hiCnt; + UInt32 freqSum2; + CPpmd_See *see; + CPpmd8_Context *mc; + unsigned numMasked; + RC_NORM_REMOTE(R) + mc = p->MinContext; + numMasked = mc->NumStats; + + do + { + p->OrderFall++; + if (!mc->Suffix) + return PPMD8_SYM_END; + mc = Ppmd8_GetContext(p, mc->Suffix); + } + while (mc->NumStats == numMasked); + + s = Ppmd8_GetStats(p, mc); + + { + unsigned num = (unsigned)mc->NumStats + 1; + unsigned num2 = num / 2; + + num &= 1; + hiCnt = (s->Freq & (UInt32)(MASK(s->Symbol))) & (0 - (UInt32)num); + s += num; + p->MinContext = mc; + + do + { + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; + s += 2; + hiCnt += (s[-2].Freq & (UInt32)(MASK(sym0))); + hiCnt += (s[-1].Freq & (UInt32)(MASK(sym1))); + } + while (--num2); + } + + see = Ppmd8_MakeEscFreq(p, numMasked, &freqSum); + freqSum += hiCnt; + freqSum2 = freqSum; + PPMD8_CORRECT_SUM_RANGE(R, freqSum2) + + + count = RC_GetThreshold(freqSum2); + + if (count < hiCnt) + { + Byte sym; + // Ppmd_See_UPDATE(see) // new (see->Summ) value can overflow over 16-bits in some rare cases + s = Ppmd8_GetStats(p, p->MinContext); + hiCnt = count; + + + { + for (;;) + { + count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + } + } + s--; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq) + + // new (see->Summ) value can overflow over 16-bits in some rare cases + Ppmd_See_UPDATE(see) + p->FoundState = s; + sym = s->Symbol; + Ppmd8_Update2(p); + return sym; + } + + if (count >= freqSum2) + return PPMD8_SYM_ERROR; + + RC_Decode(hiCnt, freqSum2 - hiCnt) + + // We increase (see->Summ) for sum of Freqs of all non_Masked symbols. + // new (see->Summ) value can overflow over 16-bits in some rare cases + see->Summ = (UInt16)(see->Summ + freqSum); + + s = Ppmd8_GetStats(p, p->MinContext); + s2 = s + p->MinContext->NumStats + 1; + do + { + MASK(s->Symbol) = 0; + s++; + } + while (s != s2); + } +} + +#undef kTop +#undef kBot +#undef READ_BYTE +#undef RC_NORM_BASE +#undef RC_NORM_1 +#undef RC_NORM +#undef RC_NORM_LOCAL +#undef RC_NORM_REMOTE +#undef R +#undef RC_Decode +#undef RC_DecodeFinal +#undef RC_GetThreshold +#undef CTX +#undef SUCCESSOR +#undef MASK diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8Enc.c b/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8Enc.c new file mode 100644 index 0000000..b0e34c4 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Ppmd8Enc.c @@ -0,0 +1,337 @@ +/* Ppmd8Enc.c -- Ppmd8 (PPMdI) Encoder +2023-09-07 : Igor Pavlov : Public domain +This code is based on: + PPMd var.I (2002): Dmitry Shkarin : Public domain + Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ + +#include "Precomp.h" + +#include "Ppmd8.h" + +#define kTop ((UInt32)1 << 24) +#define kBot ((UInt32)1 << 15) + +#define WRITE_BYTE(p) IByteOut_Write(p->Stream.Out, (Byte)(p->Low >> 24)) + +void Ppmd8_Flush_RangeEnc(CPpmd8 *p) +{ + unsigned i; + for (i = 0; i < 4; i++, p->Low <<= 8 ) + WRITE_BYTE(p); +} + + + + + + +#define RC_NORM(p) \ + while ((p->Low ^ (p->Low + p->Range)) < kTop \ + || (p->Range < kBot && ((p->Range = (0 - p->Low) & (kBot - 1)), 1))) \ + { WRITE_BYTE(p); p->Range <<= 8; p->Low <<= 8; } + + + + + + + + + + + + + +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) + +// #define RC_PRE(total) p->Range /= total; +// #define RC_PRE(total) + +#define R p + + + + +Z7_FORCE_INLINE +// Z7_NO_INLINE +static void Ppmd8_RangeEnc_Encode(CPpmd8 *p, UInt32 start, UInt32 size, UInt32 total) +{ + R->Low += start * (R->Range /= total); + R->Range *= size; + RC_NORM_LOCAL(R) +} + + + + + + + + + + +#define RC_Encode(start, size, total) Ppmd8_RangeEnc_Encode(p, start, size, total); +#define RC_EncodeFinal(start, size, total) RC_Encode(start, size, total) RC_NORM_REMOTE(p) + +#define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref)) + +// typedef CPpmd8_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) + +void Ppmd8_UpdateModel(CPpmd8 *p); + +#define MASK(sym) ((Byte *)charMask)[sym] + +// Z7_FORCE_INLINE +// static +void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) +{ + size_t charMask[256 / sizeof(size_t)]; + + if (p->MinContext->NumStats != 0) + { + CPpmd_State *s = Ppmd8_GetStats(p, p->MinContext); + UInt32 sum; + unsigned i; + UInt32 summFreq = p->MinContext->Union2.SummFreq; + + PPMD8_CORRECT_SUM_RANGE(p, summFreq) + + // RC_PRE(summFreq); + + if (s->Symbol == symbol) + { + + RC_EncodeFinal(0, s->Freq, summFreq) + p->FoundState = s; + Ppmd8_Update1_0(p); + return; + } + p->PrevSuccess = 0; + sum = s->Freq; + i = p->MinContext->NumStats; + do + { + if ((++s)->Symbol == symbol) + { + + RC_EncodeFinal(sum, s->Freq, summFreq) + p->FoundState = s; + Ppmd8_Update1(p); + return; + } + sum += s->Freq; + } + while (--i); + + + RC_Encode(sum, summFreq - sum, summFreq) + + + PPMD_SetAllBitsIn256Bytes(charMask) + // MASK(s->Symbol) = 0; + // i = p->MinContext->NumStats; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd8_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } + } + else + { + UInt16 *prob = Ppmd8_GetBinSumm(p); + CPpmd_State *s = Ppmd8Context_OneState(p->MinContext); + UInt32 pr = *prob; + const UInt32 bound = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); + if (s->Symbol == symbol) + { + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + // RangeEnc_EncodeBit_0(p, bound); + R->Range = bound; + RC_NORM(R) + + // p->FoundState = s; + // Ppmd8_UpdateBin(p); + { + const unsigned freq = s->Freq; + CPpmd8_Context *c = CTX(SUCCESSOR(s)); + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 196)); // Ppmd8 (196) + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart) + p->MaxContext = p->MinContext = c; + else + Ppmd8_UpdateModel(p); + } + return; + } + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + // RangeEnc_EncodeBit_1(p, bound); + R->Low += bound; + R->Range = (R->Range & ~((UInt32)PPMD_BIN_SCALE - 1)) - bound; + RC_NORM_LOCAL(R) + + PPMD_SetAllBitsIn256Bytes(charMask) + MASK(s->Symbol) = 0; + p->PrevSuccess = 0; + } + + for (;;) + { + CPpmd_See *see; + CPpmd_State *s; + UInt32 sum, escFreq; + CPpmd8_Context *mc; + unsigned i, numMasked; + + RC_NORM_REMOTE(p) + + mc = p->MinContext; + numMasked = mc->NumStats; + + do + { + p->OrderFall++; + if (!mc->Suffix) + return; /* EndMarker (symbol = -1) */ + mc = Ppmd8_GetContext(p, mc->Suffix); + + } + while (mc->NumStats == numMasked); + + p->MinContext = mc; + + see = Ppmd8_MakeEscFreq(p, numMasked, &escFreq); + + + + + + + + + + + + + + + + + + + + + + + + + s = Ppmd8_GetStats(p, p->MinContext); + sum = 0; + i = (unsigned)p->MinContext->NumStats + 1; + + do + { + const unsigned cur = s->Symbol; + if ((int)cur == symbol) + { + const UInt32 low = sum; + const UInt32 freq = s->Freq; + unsigned num2; + + Ppmd_See_UPDATE(see) + p->FoundState = s; + sum += escFreq; + + num2 = i / 2; + i &= 1; + sum += freq & (0 - (UInt32)i); + if (num2 != 0) + { + s += i; + do + { + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; + s += 2; + sum += (s[-2].Freq & (unsigned)(MASK(sym0))); + sum += (s[-1].Freq & (unsigned)(MASK(sym1))); + } + while (--num2); + } + + PPMD8_CORRECT_SUM_RANGE(p, sum) + + RC_EncodeFinal(low, freq, sum) + Ppmd8_Update2(p); + return; + } + sum += (s->Freq & (unsigned)(MASK(cur))); + s++; + } + while (--i); + + { + UInt32 total = sum + escFreq; + see->Summ = (UInt16)(see->Summ + total); + PPMD8_CORRECT_SUM_RANGE(p, total) + + RC_Encode(sum, total - sum, total) + } + + { + const CPpmd_State *s2 = Ppmd8_GetStats(p, p->MinContext); + s--; + MASK(s->Symbol) = 0; + do + { + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } + } +} + + + + + + + + + +#undef kTop +#undef kBot +#undef WRITE_BYTE +#undef RC_NORM_BASE +#undef RC_NORM_1 +#undef RC_NORM +#undef RC_NORM_LOCAL +#undef RC_NORM_REMOTE +#undef R +#undef RC_Encode +#undef RC_EncodeFinal + +#undef CTX +#undef SUCCESSOR +#undef MASK diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1.c b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1.c new file mode 100644 index 0000000..4ca21d7 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1.c @@ -0,0 +1,441 @@ +/* Sha1.c -- SHA-1 Hash +: Igor Pavlov : Public domain +This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ library. */ + +#include "Precomp.h" + +#include + +#include "Sha1.h" +#include "RotateDefs.h" +#include "CpuArch.h" + +#ifdef MY_CPU_X86_OR_AMD64 + #if defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30800) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 50100) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) \ + || defined(_MSC_VER) && (_MSC_VER >= 1200) + #define Z7_COMPILER_SHA1_SUPPORTED + #endif +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) \ + && (!defined(Z7_MSC_VER_ORIGINAL) || (_MSC_VER >= 1929) && (_MSC_FULL_VER >= 192930037)) + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define Z7_COMPILER_SHA1_SUPPORTED + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) + #define Z7_COMPILER_SHA1_SUPPORTED + #endif + #endif + #endif + #endif +#endif + +void Z7_FASTCALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); + +#ifdef Z7_COMPILER_SHA1_SUPPORTED + void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); + + static SHA1_FUNC_UPDATE_BLOCKS g_SHA1_FUNC_UPDATE_BLOCKS = Sha1_UpdateBlocks; + static SHA1_FUNC_UPDATE_BLOCKS g_SHA1_FUNC_UPDATE_BLOCKS_HW; + + #define SHA1_UPDATE_BLOCKS(p) p->v.vars.func_UpdateBlocks +#else + #define SHA1_UPDATE_BLOCKS(p) Sha1_UpdateBlocks +#endif + + +BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) +{ + SHA1_FUNC_UPDATE_BLOCKS func = Sha1_UpdateBlocks; + + #ifdef Z7_COMPILER_SHA1_SUPPORTED + if (algo != SHA1_ALGO_SW) + { + if (algo == SHA1_ALGO_DEFAULT) + func = g_SHA1_FUNC_UPDATE_BLOCKS; + else + { + if (algo != SHA1_ALGO_HW) + return False; + func = g_SHA1_FUNC_UPDATE_BLOCKS_HW; + if (!func) + return False; + } + } + #else + if (algo > 1) + return False; + #endif + + p->v.vars.func_UpdateBlocks = func; + return True; +} + + +/* define it for speed optimization */ +// #define Z7_SHA1_UNROLL + +// allowed unroll steps: (1, 2, 4, 5, 20) + +#undef Z7_SHA1_BIG_W +#ifdef Z7_SHA1_UNROLL + #define STEP_PRE 20 + #define STEP_MAIN 20 +#else + #define Z7_SHA1_BIG_W + #define STEP_PRE 5 + #define STEP_MAIN 5 +#endif + + +#ifdef Z7_SHA1_BIG_W + #define kNumW 80 + #define w(i) W[i] +#else + #define kNumW 16 + #define w(i) W[(i)&15] +#endif + +#define w0(i) (W[i] = GetBe32(data + (size_t)(i) * 4)) +#define w1(i) (w(i) = rotlFixed(w((size_t)(i)-3) ^ w((size_t)(i)-8) ^ w((size_t)(i)-14) ^ w((size_t)(i)-16), 1)) + +#define f0(x,y,z) ( 0x5a827999 + (z^(x&(y^z))) ) +#define f1(x,y,z) ( 0x6ed9eba1 + (x^y^z) ) +#define f2(x,y,z) ( 0x8f1bbcdc + ((x&y)|(z&(x|y))) ) +#define f3(x,y,z) ( 0xca62c1d6 + (x^y^z) ) + +/* +#define T1(fx, ww) \ + tmp = e + fx(b,c,d) + ww + rotlFixed(a, 5); \ + e = d; \ + d = c; \ + c = rotlFixed(b, 30); \ + b = a; \ + a = tmp; \ +*/ + +#define T5(a,b,c,d,e, fx, ww) \ + e += fx(b,c,d) + ww + rotlFixed(a, 5); \ + b = rotlFixed(b, 30); \ + + +/* +#define R1(i, fx, wx) \ + T1 ( fx, wx(i)); \ + +#define R2(i, fx, wx) \ + R1 ( (i) , fx, wx); \ + R1 ( (i) + 1, fx, wx); \ + +#define R4(i, fx, wx) \ + R2 ( (i) , fx, wx); \ + R2 ( (i) + 2, fx, wx); \ +*/ + +#define M5(i, fx, wx0, wx1) \ + T5 ( a,b,c,d,e, fx, wx0((i) ) ) \ + T5 ( e,a,b,c,d, fx, wx1((i)+1) ) \ + T5 ( d,e,a,b,c, fx, wx1((i)+2) ) \ + T5 ( c,d,e,a,b, fx, wx1((i)+3) ) \ + T5 ( b,c,d,e,a, fx, wx1((i)+4) ) \ + +#define R5(i, fx, wx) \ + M5 ( i, fx, wx, wx) \ + + +#if STEP_PRE > 5 + + #define R20_START \ + R5 ( 0, f0, w0) \ + R5 ( 5, f0, w0) \ + R5 ( 10, f0, w0) \ + M5 ( 15, f0, w0, w1) \ + + #elif STEP_PRE == 5 + + #define R20_START \ + { size_t i; for (i = 0; i < 15; i += STEP_PRE) \ + { R5(i, f0, w0) } } \ + M5 ( 15, f0, w0, w1) \ + +#else + + #if STEP_PRE == 1 + #define R_PRE R1 + #elif STEP_PRE == 2 + #define R_PRE R2 + #elif STEP_PRE == 4 + #define R_PRE R4 + #endif + + #define R20_START \ + { size_t i; for (i = 0; i < 16; i += STEP_PRE) \ + { R_PRE(i, f0, w0) } } \ + R4 ( 16, f0, w1) \ + +#endif + + + +#if STEP_MAIN > 5 + + #define R20(ii, fx) \ + R5 ( (ii) , fx, w1) \ + R5 ( (ii) + 5 , fx, w1) \ + R5 ( (ii) + 10, fx, w1) \ + R5 ( (ii) + 15, fx, w1) \ + +#else + + #if STEP_MAIN == 1 + #define R_MAIN R1 + #elif STEP_MAIN == 2 + #define R_MAIN R2 + #elif STEP_MAIN == 4 + #define R_MAIN R4 + #elif STEP_MAIN == 5 + #define R_MAIN R5 + #endif + + #define R20(ii, fx) \ + { size_t i; for (i = (ii); i < (ii) + 20; i += STEP_MAIN) \ + { R_MAIN(i, fx, w1) } } \ + +#endif + + + +void Sha1_InitState(CSha1 *p) +{ + p->v.vars.count = 0; + p->state[0] = 0x67452301; + p->state[1] = 0xEFCDAB89; + p->state[2] = 0x98BADCFE; + p->state[3] = 0x10325476; + p->state[4] = 0xC3D2E1F0; +} + +void Sha1_Init(CSha1 *p) +{ + p->v.vars.func_UpdateBlocks = + #ifdef Z7_COMPILER_SHA1_SUPPORTED + g_SHA1_FUNC_UPDATE_BLOCKS; + #else + NULL; + #endif + Sha1_InitState(p); +} + + +Z7_NO_INLINE +void Z7_FASTCALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks) +{ + UInt32 a, b, c, d, e; + UInt32 W[kNumW]; + + if (numBlocks == 0) + return; + + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + + do + { + #if STEP_PRE < 5 || STEP_MAIN < 5 + UInt32 tmp; + #endif + + R20_START + R20(20, f1) + R20(40, f2) + R20(60, f3) + + a += state[0]; + b += state[1]; + c += state[2]; + d += state[3]; + e += state[4]; + + state[0] = a; + state[1] = b; + state[2] = c; + state[3] = d; + state[4] = e; + + data += SHA1_BLOCK_SIZE; + } + while (--numBlocks); +} + + +#define Sha1_UpdateBlock(p) SHA1_UPDATE_BLOCKS(p)(p->state, p->buffer, 1) + +void Sha1_Update(CSha1 *p, const Byte *data, size_t size) +{ + if (size == 0) + return; + { + const unsigned pos = (unsigned)p->v.vars.count & (SHA1_BLOCK_SIZE - 1); + const unsigned num = SHA1_BLOCK_SIZE - pos; + p->v.vars.count += size; + if (num > size) + { + memcpy(p->buffer + pos, data, size); + return; + } + if (pos != 0) + { + size -= num; + memcpy(p->buffer + pos, data, num); + data += num; + Sha1_UpdateBlock(p); + } + } + { + const size_t numBlocks = size >> 6; + // if (numBlocks) + SHA1_UPDATE_BLOCKS(p)(p->state, data, numBlocks); + size &= SHA1_BLOCK_SIZE - 1; + if (size == 0) + return; + data += (numBlocks << 6); + memcpy(p->buffer, data, size); + } +} + + +void Sha1_Final(CSha1 *p, Byte *digest) +{ + unsigned pos = (unsigned)p->v.vars.count & (SHA1_BLOCK_SIZE - 1); + p->buffer[pos++] = 0x80; + if (pos > (SHA1_BLOCK_SIZE - 4 * 2)) + { + while (pos != SHA1_BLOCK_SIZE) { p->buffer[pos++] = 0; } + // memset(&p->buf.buffer[pos], 0, SHA1_BLOCK_SIZE - pos); + Sha1_UpdateBlock(p); + pos = 0; + } + memset(&p->buffer[pos], 0, (SHA1_BLOCK_SIZE - 4 * 2) - pos); + { + const UInt64 numBits = p->v.vars.count << 3; + SetBe32(p->buffer + SHA1_BLOCK_SIZE - 4 * 2, (UInt32)(numBits >> 32)) + SetBe32(p->buffer + SHA1_BLOCK_SIZE - 4 * 1, (UInt32)(numBits)) + } + Sha1_UpdateBlock(p); + + SetBe32(digest, p->state[0]) + SetBe32(digest + 4, p->state[1]) + SetBe32(digest + 8, p->state[2]) + SetBe32(digest + 12, p->state[3]) + SetBe32(digest + 16, p->state[4]) + + Sha1_InitState(p); +} + + +void Sha1_PrepareBlock(const CSha1 *p, Byte *block, unsigned size) +{ + const UInt64 numBits = (p->v.vars.count + size) << 3; + SetBe32(&((UInt32 *)(void *)block)[SHA1_NUM_BLOCK_WORDS - 2], (UInt32)(numBits >> 32)) + SetBe32(&((UInt32 *)(void *)block)[SHA1_NUM_BLOCK_WORDS - 1], (UInt32)(numBits)) + // SetBe32((UInt32 *)(block + size), 0x80000000); + SetUi32((UInt32 *)(void *)(block + size), 0x80) + size += 4; + while (size != (SHA1_NUM_BLOCK_WORDS - 2) * 4) + { + *((UInt32 *)(void *)(block + size)) = 0; + size += 4; + } +} + +void Sha1_GetBlockDigest(const CSha1 *p, const Byte *data, Byte *destDigest) +{ + MY_ALIGN (16) + UInt32 st[SHA1_NUM_DIGEST_WORDS]; + + st[0] = p->state[0]; + st[1] = p->state[1]; + st[2] = p->state[2]; + st[3] = p->state[3]; + st[4] = p->state[4]; + + SHA1_UPDATE_BLOCKS(p)(st, data, 1); + + SetBe32(destDigest + 0 , st[0]) + SetBe32(destDigest + 1 * 4, st[1]) + SetBe32(destDigest + 2 * 4, st[2]) + SetBe32(destDigest + 3 * 4, st[3]) + SetBe32(destDigest + 4 * 4, st[4]) +} + + +void Sha1Prepare(void) +{ +#ifdef Z7_COMPILER_SHA1_SUPPORTED + SHA1_FUNC_UPDATE_BLOCKS f, f_hw; + f = Sha1_UpdateBlocks; + f_hw = NULL; +#ifdef MY_CPU_X86_OR_AMD64 + if (CPU_IsSupported_SHA() + && CPU_IsSupported_SSSE3() + ) +#else + if (CPU_IsSupported_SHA1()) +#endif + { + // printf("\n========== HW SHA1 ======== \n"); +#if 1 && defined(MY_CPU_ARM_OR_ARM64) && defined(Z7_MSC_VER_ORIGINAL) && (_MSC_FULL_VER < 192930037) + /* there was bug in MSVC compiler for ARM64 -O2 before version VS2019 16.10 (19.29.30037). + It generated incorrect SHA-1 code. */ + #pragma message("== SHA1 code can work incorrectly with this compiler") + #error Stop_Compiling_MSC_Compiler_BUG_SHA1 +#endif + { + f = f_hw = Sha1_UpdateBlocks_HW; + } + } + g_SHA1_FUNC_UPDATE_BLOCKS = f; + g_SHA1_FUNC_UPDATE_BLOCKS_HW = f_hw; +#endif +} + +#undef kNumW +#undef w +#undef w0 +#undef w1 +#undef f0 +#undef f1 +#undef f2 +#undef f3 +#undef T1 +#undef T5 +#undef M5 +#undef R1 +#undef R2 +#undef R4 +#undef R5 +#undef R20_START +#undef R_PRE +#undef R_MAIN +#undef STEP_PRE +#undef STEP_MAIN +#undef Z7_SHA1_BIG_W +#undef Z7_SHA1_UNROLL +#undef Z7_COMPILER_SHA1_SUPPORTED diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1.h new file mode 100644 index 0000000..529be4d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1.h @@ -0,0 +1,86 @@ +/* Sha1.h -- SHA-1 Hash +: Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_SHA1_H +#define ZIP7_INC_SHA1_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define SHA1_NUM_BLOCK_WORDS 16 +#define SHA1_NUM_DIGEST_WORDS 5 + +#define SHA1_BLOCK_SIZE (SHA1_NUM_BLOCK_WORDS * 4) +#define SHA1_DIGEST_SIZE (SHA1_NUM_DIGEST_WORDS * 4) + + + + +typedef void (Z7_FASTCALL *SHA1_FUNC_UPDATE_BLOCKS)(UInt32 state[5], const Byte *data, size_t numBlocks); + +/* + if (the system supports different SHA1 code implementations) + { + (CSha1::func_UpdateBlocks) will be used + (CSha1::func_UpdateBlocks) can be set by + Sha1_Init() - to default (fastest) + Sha1_SetFunction() - to any algo + } + else + { + (CSha1::func_UpdateBlocks) is ignored. + } +*/ + +typedef struct +{ + union + { + struct + { + SHA1_FUNC_UPDATE_BLOCKS func_UpdateBlocks; + UInt64 count; + } vars; + UInt64 _pad_64bit[4]; + void *_pad_align_ptr[2]; + } v; + UInt32 state[SHA1_NUM_DIGEST_WORDS]; + UInt32 _pad_3[3]; + Byte buffer[SHA1_BLOCK_SIZE]; +} CSha1; + + +#define SHA1_ALGO_DEFAULT 0 +#define SHA1_ALGO_SW 1 +#define SHA1_ALGO_HW 2 + +/* +Sha1_SetFunction() +return: + 0 - (algo) value is not supported, and func_UpdateBlocks was not changed + 1 - func_UpdateBlocks was set according (algo) value. +*/ + +BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo); + +void Sha1_InitState(CSha1 *p); +void Sha1_Init(CSha1 *p); +void Sha1_Update(CSha1 *p, const Byte *data, size_t size); +void Sha1_Final(CSha1 *p, Byte *digest); + +void Sha1_PrepareBlock(const CSha1 *p, Byte *block, unsigned size); +void Sha1_GetBlockDigest(const CSha1 *p, const Byte *data, Byte *destDigest); + +// void Z7_FASTCALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); + +/* +call Sha1Prepare() once at program start. +It prepares all supported implementations, and detects the fastest implementation. +*/ + +void Sha1Prepare(void); + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1Opt.c b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1Opt.c new file mode 100644 index 0000000..8738b94 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha1Opt.c @@ -0,0 +1,424 @@ +/* Sha1Opt.c -- SHA-1 optimized code for SHA-1 hardware instructions +: Igor Pavlov : Public domain */ + +#include "Precomp.h" +#include "Compiler.h" +#include "CpuArch.h" + +// #define Z7_USE_HW_SHA_STUB // for debug +#ifdef MY_CPU_X86_OR_AMD64 + #if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) // fix that check + #define USE_HW_SHA + #elif defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30800) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 50100) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) + #define USE_HW_SHA + #if !defined(__INTEL_COMPILER) + // icc defines __GNUC__, but icc doesn't support __attribute__(__target__) + #if !defined(__SHA__) || !defined(__SSSE3__) + #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) + #endif + #endif + #elif defined(_MSC_VER) + #if (_MSC_VER >= 1900) + #define USE_HW_SHA + #else + #define Z7_USE_HW_SHA_STUB + #endif + #endif +// #endif // MY_CPU_X86_OR_AMD64 +#ifndef USE_HW_SHA + // #define Z7_USE_HW_SHA_STUB // for debug +#endif + +#ifdef USE_HW_SHA + +// #pragma message("Sha1 HW") + + + + +// sse/sse2/ssse3: +#include +// sha*: +#include + +#if defined (__clang__) && defined(_MSC_VER) + #if !defined(__SHA__) + #include + #endif +#else + +#endif + +/* +SHA1 uses: +SSE2: + _mm_loadu_si128 + _mm_storeu_si128 + _mm_set_epi32 + _mm_add_epi32 + _mm_shuffle_epi32 / pshufd + _mm_xor_si128 + _mm_cvtsi128_si32 + _mm_cvtsi32_si128 +SSSE3: + _mm_shuffle_epi8 / pshufb + +SHA: + _mm_sha1* +*/ + +#define XOR_SI128(dest, src) dest = _mm_xor_si128(dest, src); +#define SHUFFLE_EPI8(dest, mask) dest = _mm_shuffle_epi8(dest, mask); +#define SHUFFLE_EPI32(dest, mask) dest = _mm_shuffle_epi32(dest, mask); +#ifdef __clang__ +#define SHA1_RNDS4_RET_TYPE_CAST (__m128i) +#else +#define SHA1_RNDS4_RET_TYPE_CAST +#endif +#define SHA1_RND4(abcd, e0, f) abcd = SHA1_RNDS4_RET_TYPE_CAST _mm_sha1rnds4_epu32(abcd, e0, f); +#define SHA1_NEXTE(e, m) e = _mm_sha1nexte_epu32(e, m); +#define ADD_EPI32(dest, src) dest = _mm_add_epi32(dest, src); +#define SHA1_MSG1(dest, src) dest = _mm_sha1msg1_epu32(dest, src); +#define SHA1_MSG2(dest, src) dest = _mm_sha1msg2_epu32(dest, src); + +#define LOAD_SHUFFLE(m, k) \ + m = _mm_loadu_si128((const __m128i *)(const void *)(data + (k) * 16)); \ + SHUFFLE_EPI8(m, mask) \ + +#define NNN(m0, m1, m2, m3) + +#define SM1(m0, m1, m2, m3) \ + SHA1_MSG1(m0, m1) \ + +#define SM2(m0, m1, m2, m3) \ + XOR_SI128(m3, m1) \ + SHA1_MSG2(m3, m2) \ + +#define SM3(m0, m1, m2, m3) \ + XOR_SI128(m3, m1) \ + SM1(m0, m1, m2, m3) \ + SHA1_MSG2(m3, m2) \ + +#define R4(k, m0, m1, m2, m3, e0, e1, OP) \ + e1 = abcd; \ + SHA1_RND4(abcd, e0, (k) / 5) \ + SHA1_NEXTE(e1, m1) \ + OP(m0, m1, m2, m3) \ + + + +#define R16(k, mx, OP0, OP1, OP2, OP3) \ + R4 ( (k)*4+0, m0,m1,m2,m3, e0,e1, OP0 ) \ + R4 ( (k)*4+1, m1,m2,m3,m0, e1,e0, OP1 ) \ + R4 ( (k)*4+2, m2,m3,m0,m1, e0,e1, OP2 ) \ + R4 ( (k)*4+3, m3,mx,m1,m2, e1,e0, OP3 ) \ + +#define PREPARE_STATE \ + SHUFFLE_EPI32 (abcd, 0x1B) \ + SHUFFLE_EPI32 (e0, 0x1B) \ + + + + + +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); +#ifdef ATTRIB_SHA +ATTRIB_SHA +#endif +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) +{ + const __m128i mask = _mm_set_epi32(0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f); + + + __m128i abcd, e0; + + if (numBlocks == 0) + return; + + abcd = _mm_loadu_si128((const __m128i *) (const void *) &state[0]); // dbca + e0 = _mm_cvtsi32_si128((int)state[4]); // 000e + + PREPARE_STATE + + do + { + __m128i abcd_save, e2; + __m128i m0, m1, m2, m3; + __m128i e1; + + + abcd_save = abcd; + e2 = e0; + + LOAD_SHUFFLE (m0, 0) + LOAD_SHUFFLE (m1, 1) + LOAD_SHUFFLE (m2, 2) + LOAD_SHUFFLE (m3, 3) + + ADD_EPI32(e0, m0) + + R16 ( 0, m0, SM1, SM3, SM3, SM3 ) + R16 ( 1, m0, SM3, SM3, SM3, SM3 ) + R16 ( 2, m0, SM3, SM3, SM3, SM3 ) + R16 ( 3, m0, SM3, SM3, SM3, SM3 ) + R16 ( 4, e2, SM2, NNN, NNN, NNN ) + + ADD_EPI32(abcd, abcd_save) + + data += 64; + } + while (--numBlocks); + + PREPARE_STATE + + _mm_storeu_si128((__m128i *) (void *) state, abcd); + *(state + 4) = (UInt32)_mm_cvtsi128_si32(e0); +} + +#endif // USE_HW_SHA + +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) \ + && (!defined(Z7_MSC_VER_ORIGINAL) || (_MSC_VER >= 1929) && (_MSC_FULL_VER >= 192930037)) + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_SHA + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) + #define USE_HW_SHA + #endif + #endif + #endif + #endif + +#ifdef USE_HW_SHA + +// #pragma message("=== Sha1 HW === ") +// __ARM_FEATURE_CRYPTO macro is deprecated in favor of the finer grained feature macro __ARM_FEATURE_SHA2 + +#if defined(__clang__) || defined(__GNUC__) +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) + #ifdef MY_CPU_ARM64 +#if defined(__clang__) + #define ATTRIB_SHA __attribute__((__target__("crypto"))) +#else + #define ATTRIB_SHA __attribute__((__target__("+crypto"))) +#endif + #else +#if defined(__clang__) && (__clang_major__ >= 1) + #define ATTRIB_SHA __attribute__((__target__("armv8-a,sha2"))) +#else + #define ATTRIB_SHA __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) +#endif + #endif +#endif +#else + // _MSC_VER + // for arm32 + #define _ARM_USE_NEW_NEON_INTRINSICS +#endif + +#if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) +#include +#else + +#if defined(__clang__) && __clang_major__ < 16 +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) +// #pragma message("=== we set __ARM_FEATURE_CRYPTO 1 === ") + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRYPTO_WAS_SET 1 +// #if defined(__clang__) && __clang_major__ < 13 + #define __ARM_FEATURE_CRYPTO 1 +// #else + #define __ARM_FEATURE_SHA2 1 +// #endif + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif // clang + +#if defined(__clang__) + +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") + #undef __ARM_ARCH + #define __ARM_ARCH 8 + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // clang + +#include + +#if defined(Z7_ARM_FEATURE_CRYPTO_WAS_SET) && \ + defined(__ARM_FEATURE_CRYPTO) && \ + defined(__ARM_FEATURE_SHA2) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #undef __ARM_FEATURE_CRYPTO + #undef __ARM_FEATURE_SHA2 + #undef Z7_ARM_FEATURE_CRYPTO_WAS_SET +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +// #pragma message("=== we undefine __ARM_FEATURE_CRYPTO === ") +#endif + +#endif // Z7_MSC_VER_ORIGINAL + +typedef uint32x4_t v128; +// typedef __n128 v128; // MSVC +// the bug in clang 3.8.1: +// __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); +#if defined(__clang__) && (__clang_major__ <= 9) +#pragma GCC diagnostic ignored "-Wvector-conversion" +#endif + +#ifdef MY_CPU_BE + #define MY_rev32_for_LE(x) x +#else + #define MY_rev32_for_LE(x) vrev32q_u8(x) +#endif + +#define LOAD_128_32(_p) vld1q_u32(_p) +#define LOAD_128_8(_p) vld1q_u8 (_p) +#define STORE_128_32(_p, _v) vst1q_u32(_p, _v) + +#define LOAD_SHUFFLE(m, k) \ + m = vreinterpretq_u32_u8( \ + MY_rev32_for_LE( \ + LOAD_128_8(data + (k) * 16))); \ + +#define N0(dest, src2, src3) +#define N1(dest, src) +#define U0(dest, src2, src3) dest = vsha1su0q_u32(dest, src2, src3); +#define U1(dest, src) dest = vsha1su1q_u32(dest, src); +#define C(e) abcd = vsha1cq_u32(abcd, e, t) +#define P(e) abcd = vsha1pq_u32(abcd, e, t) +#define M(e) abcd = vsha1mq_u32(abcd, e, t) +#define H(e) e = vsha1h_u32(vgetq_lane_u32(abcd, 0)) +#define T(m, c) t = vaddq_u32(m, c) + +#define R16(d0,d1,d2,d3, f0,z0, f1,z1, f2,z2, f3,z3, w0,w1,w2,w3) \ + T(m0, d0); f0(m3, m0, m1) z0(m2, m1) H(e1); w0(e0); \ + T(m1, d1); f1(m0, m1, m2) z1(m3, m2) H(e0); w1(e1); \ + T(m2, d2); f2(m1, m2, m3) z2(m0, m3) H(e1); w2(e0); \ + T(m3, d3); f3(m2, m3, m0) z3(m1, m0) H(e0); w3(e1); \ + + +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +#ifdef ATTRIB_SHA +ATTRIB_SHA +#endif +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + v128 abcd; + v128 c0, c1, c2, c3; + uint32_t e0; + + if (numBlocks == 0) + return; + + c0 = vdupq_n_u32(0x5a827999); + c1 = vdupq_n_u32(0x6ed9eba1); + c2 = vdupq_n_u32(0x8f1bbcdc); + c3 = vdupq_n_u32(0xca62c1d6); + + abcd = LOAD_128_32(&state[0]); + e0 = state[4]; + + do + { + v128 abcd_save; + v128 m0, m1, m2, m3; + v128 t; + uint32_t e0_save, e1; + + abcd_save = abcd; + e0_save = e0; + + LOAD_SHUFFLE (m0, 0) + LOAD_SHUFFLE (m1, 1) + LOAD_SHUFFLE (m2, 2) + LOAD_SHUFFLE (m3, 3) + + R16 ( c0,c0,c0,c0, N0,N1, U0,N1, U0,U1, U0,U1, C,C,C,C ) + R16 ( c0,c1,c1,c1, U0,U1, U0,U1, U0,U1, U0,U1, C,P,P,P ) + R16 ( c1,c1,c2,c2, U0,U1, U0,U1, U0,U1, U0,U1, P,P,M,M ) + R16 ( c2,c2,c2,c3, U0,U1, U0,U1, U0,U1, U0,U1, M,M,M,P ) + R16 ( c3,c3,c3,c3, U0,U1, N0,U1, N0,N1, N0,N1, P,P,P,P ) + + abcd = vaddq_u32(abcd, abcd_save); + e0 += e0_save; + + data += 64; + } + while (--numBlocks); + + STORE_128_32(&state[0], abcd); + state[4] = e0; +} + +#endif // USE_HW_SHA + +#endif // MY_CPU_ARM_OR_ARM64 + +#if !defined(USE_HW_SHA) && defined(Z7_USE_HW_SHA_STUB) +// #error Stop_Compiling_UNSUPPORTED_SHA +// #include +// #include "Sha1.h" +// #if defined(_MSC_VER) +#pragma message("Sha1 HW-SW stub was used") +// #endif +void Z7_FASTCALL Sha1_UpdateBlocks (UInt32 state[5], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) +{ + Sha1_UpdateBlocks(state, data, numBlocks); + /* + UNUSED_VAR(state); + UNUSED_VAR(data); + UNUSED_VAR(numBlocks); + exit(1); + return; + */ +} +#endif + +#undef U0 +#undef U1 +#undef N0 +#undef N1 +#undef C +#undef P +#undef M +#undef H +#undef T +#undef MY_rev32_for_LE +#undef NNN +#undef LOAD_128 +#undef STORE_128 +#undef LOAD_SHUFFLE +#undef SM1 +#undef SM2 +#undef SM3 +#undef NNN +#undef R4 +#undef R16 +#undef PREPARE_STATE +#undef USE_HW_SHA +#undef ATTRIB_SHA +#undef USE_VER_MIN +#undef Z7_USE_HW_SHA_STUB diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha3.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha3.h new file mode 100644 index 0000000..c5909c9 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha3.h @@ -0,0 +1,36 @@ +/* Sha3.h -- SHA-3 Hash +: Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_MD5_H +#define ZIP7_INC_MD5_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define SHA3_NUM_STATE_WORDS 25 + +#define SHA3_BLOCK_SIZE_FROM_DIGEST_SIZE(digestSize) \ + (SHA3_NUM_STATE_WORDS * 8 - (digestSize) * 2) + +typedef struct +{ + UInt32 count; // < blockSize + UInt32 blockSize; // <= SHA3_NUM_STATE_WORDS * 8 + UInt64 _pad1[3]; + // we want 32-bytes alignment here + UInt64 state[SHA3_NUM_STATE_WORDS]; + UInt64 _pad2[3]; + // we want 64-bytes alignment here + Byte buffer[SHA3_NUM_STATE_WORDS * 8]; // last bytes will be unused with predefined blockSize values +} CSha3; + +#define Sha3_SET_blockSize(p, blockSize) { (p)->blockSize = (blockSize); } + +void Sha3_Init(CSha3 *p); +void Sha3_Update(CSha3 *p, const Byte *data, size_t size); +void Sha3_Final(CSha3 *p, Byte *digest, unsigned digestSize, unsigned shake); + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha512.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha512.h new file mode 100644 index 0000000..1f3a4d1 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Sha512.h @@ -0,0 +1,86 @@ +/* Sha512.h -- SHA-512 Hash +: Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_SHA512_H +#define ZIP7_INC_SHA512_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define SHA512_NUM_BLOCK_WORDS 16 +#define SHA512_NUM_DIGEST_WORDS 8 + +#define SHA512_BLOCK_SIZE (SHA512_NUM_BLOCK_WORDS * 8) +#define SHA512_DIGEST_SIZE (SHA512_NUM_DIGEST_WORDS * 8) +#define SHA512_224_DIGEST_SIZE (224 / 8) +#define SHA512_256_DIGEST_SIZE (256 / 8) +#define SHA512_384_DIGEST_SIZE (384 / 8) + +typedef void (Z7_FASTCALL *SHA512_FUNC_UPDATE_BLOCKS)(UInt64 state[8], const Byte *data, size_t numBlocks); + +/* + if (the system supports different SHA512 code implementations) + { + (CSha512::func_UpdateBlocks) will be used + (CSha512::func_UpdateBlocks) can be set by + Sha512_Init() - to default (fastest) + Sha512_SetFunction() - to any algo + } + else + { + (CSha512::func_UpdateBlocks) is ignored. + } +*/ + +typedef struct +{ + union + { + struct + { + SHA512_FUNC_UPDATE_BLOCKS func_UpdateBlocks; + UInt64 count; + } vars; + UInt64 _pad_64bit[8]; + void *_pad_align_ptr[2]; + } v; + UInt64 state[SHA512_NUM_DIGEST_WORDS]; + + Byte buffer[SHA512_BLOCK_SIZE]; +} CSha512; + + +#define SHA512_ALGO_DEFAULT 0 +#define SHA512_ALGO_SW 1 +#define SHA512_ALGO_HW 2 + +/* +Sha512_SetFunction() +return: + 0 - (algo) value is not supported, and func_UpdateBlocks was not changed + 1 - func_UpdateBlocks was set according (algo) value. +*/ + +BoolInt Sha512_SetFunction(CSha512 *p, unsigned algo); +// we support only these (digestSize) values: 224/8, 256/8, 384/8, 512/8 +void Sha512_InitState(CSha512 *p, unsigned digestSize); +void Sha512_Init(CSha512 *p, unsigned digestSize); +void Sha512_Update(CSha512 *p, const Byte *data, size_t size); +void Sha512_Final(CSha512 *p, Byte *digest, unsigned digestSize); + + + + +// void Z7_FASTCALL Sha512_UpdateBlocks(UInt64 state[8], const Byte *data, size_t numBlocks); + +/* +call Sha512Prepare() once at program start. +It prepares all supported implementations, and detects the fastest implementation. +*/ + +void Sha512Prepare(void); + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/Xxh64.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/Xxh64.h new file mode 100644 index 0000000..efef65e --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/Xxh64.h @@ -0,0 +1,50 @@ +/* Xxh64.h -- XXH64 hash calculation interfaces +2023-08-18 : Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_XXH64_H +#define ZIP7_INC_XXH64_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define Z7_XXH64_BLOCK_SIZE (4 * 8) + +typedef struct +{ + UInt64 v[4]; +} CXxh64State; + +void Xxh64State_Init(CXxh64State *p); + +// end != data && end == data + Z7_XXH64_BLOCK_SIZE * numBlocks +void Z7_FASTCALL Xxh64State_UpdateBlocks(CXxh64State *p, const void *data, const void *end); + +/* +Xxh64State_Digest(): +data: + the function processes only + (totalCount & (Z7_XXH64_BLOCK_SIZE - 1)) bytes in (data): (smaller than 32 bytes). +totalCount: total size of hashed stream: + it includes total size of data processed by previous Xxh64State_UpdateBlocks() calls, + and it also includes current processed size in (data). +*/ +UInt64 Xxh64State_Digest(const CXxh64State *p, const void *data, UInt64 totalCount); + + +typedef struct +{ + CXxh64State state; + UInt64 count; + UInt64 buf64[4]; +} CXxh64; + +void Xxh64_Init(CXxh64 *p); +void Xxh64_Update(CXxh64 *p, const void *data, size_t size); + +#define Xxh64_Digest(p) \ + Xxh64State_Digest(&(p)->state, (p)->buf64, (p)->count) + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/ZstdDec.c b/ext/lzma_sdk_wrapper/lzma_sdk/C/ZstdDec.c new file mode 100644 index 0000000..6ad47eb --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/ZstdDec.c @@ -0,0 +1,4067 @@ +/* ZstdDec.c -- Zstd Decoder +2024-06-18 : the code was developed by Igor Pavlov, using Zstandard format + specification and original zstd decoder code as reference code. +original zstd decoder code: Copyright (c) Facebook, Inc. All rights reserved. +This source code is licensed under BSD 3-Clause License. +*/ + +#include "Precomp.h" + +#include +#include +// #include + +#include "Alloc.h" +#include "Xxh64.h" +#include "ZstdDec.h" +#include "CpuArch.h" + +#if defined(MY_CPU_ARM64) +#include +#endif + +/* original-zstd still doesn't support window larger than 2 GiB. + So we also limit our decoder for 2 GiB window: */ +#if defined(MY_CPU_64BIT) && 0 == 1 + #define MAX_WINDOW_SIZE_LOG 41 +#else + #define MAX_WINDOW_SIZE_LOG 31 +#endif + +typedef + #if MAX_WINDOW_SIZE_LOG < 32 + UInt32 + #else + size_t + #endif + CZstdDecOffset; + +// for debug: simpler and smaller code but slow: +// #define Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + +// #define SHOW_STAT +#ifdef SHOW_STAT +#include +static unsigned g_Num_Blocks_Compressed = 0; +static unsigned g_Num_Blocks_memcpy = 0; +static unsigned g_Num_Wrap_memmove_Num = 0; +static unsigned g_Num_Wrap_memmove_Bytes = 0; +static unsigned g_NumSeqs_total = 0; +// static unsigned g_NumCopy = 0; +static unsigned g_NumOver = 0; +static unsigned g_NumOver2 = 0; +static unsigned g_Num_Match = 0; +static unsigned g_Num_Lits = 0; +static unsigned g_Num_LitsBig = 0; +static unsigned g_Num_Lit0 = 0; +static unsigned g_Num_Rep0 = 0; +static unsigned g_Num_Rep1 = 0; +static unsigned g_Num_Rep2 = 0; +static unsigned g_Num_Rep3 = 0; +static unsigned g_Num_Threshold_0 = 0; +static unsigned g_Num_Threshold_1 = 0; +static unsigned g_Num_Threshold_0sum = 0; +static unsigned g_Num_Threshold_1sum = 0; +#define STAT_UPDATE(v) v +#else +#define STAT_UPDATE(v) +#endif +#define STAT_INC(v) STAT_UPDATE(v++;) + + +typedef struct +{ + const Byte *ptr; + size_t len; +} +CInBufPair; + + +#if defined(MY_CPU_ARM_OR_ARM64) || defined(MY_CPU_X86_OR_AMD64) + #if (defined(__clang__) && (__clang_major__ >= 6)) \ + || (defined(__GNUC__) && (__GNUC__ >= 6)) + // disable for debug: + #define Z7_ZSTD_DEC_USE_BSR + #elif defined(_MSC_VER) && (_MSC_VER >= 1300) + // #if defined(MY_CPU_ARM_OR_ARM64) + #if (_MSC_VER >= 1600) + #include + #endif + // disable for debug: + #define Z7_ZSTD_DEC_USE_BSR + #endif +#endif + +#ifdef Z7_ZSTD_DEC_USE_BSR + #if defined(__clang__) || defined(__GNUC__) + #define MY_clz(x) ((unsigned)__builtin_clz((UInt32)x)) + #else // #if defined(_MSC_VER) + #ifdef MY_CPU_ARM_OR_ARM64 + #define MY_clz _CountLeadingZeros + #endif // MY_CPU_X86_OR_AMD64 + #endif // _MSC_VER +#elif !defined(Z7_ZSTD_DEC_USE_LOG_TABLE) + #define Z7_ZSTD_DEC_USE_LOG_TABLE +#endif + + +static +Z7_FORCE_INLINE +unsigned GetHighestSetBit_32_nonzero_big(UInt32 num) +{ + // (num != 0) + #ifdef MY_clz + return 31 - MY_clz(num); + #elif defined(Z7_ZSTD_DEC_USE_BSR) + { + unsigned long zz; + _BitScanReverse(&zz, num); + return zz; + } + #else + { + int i = -1; + for (;;) + { + i++; + num >>= 1; + if (num == 0) + return (unsigned)i; + } + } + #endif +} + +#ifdef Z7_ZSTD_DEC_USE_LOG_TABLE + +#define R1(a) a, a +#define R2(a) R1(a), R1(a) +#define R3(a) R2(a), R2(a) +#define R4(a) R3(a), R3(a) +#define R5(a) R4(a), R4(a) +#define R6(a) R5(a), R5(a) +#define R7(a) R6(a), R6(a) +#define R8(a) R7(a), R7(a) +#define R9(a) R8(a), R8(a) + +#define Z7_ZSTD_FSE_MAX_ACCURACY 9 +// states[] values in FSE_Generate() can use (Z7_ZSTD_FSE_MAX_ACCURACY + 1) bits. +static const Byte k_zstd_LogTable[2 << Z7_ZSTD_FSE_MAX_ACCURACY] = +{ + R1(0), R1(1), R2(2), R3(3), R4(4), R5(5), R6(6), R7(7), R8(8), R9(9) +}; + +#define GetHighestSetBit_32_nonzero_small(num) (k_zstd_LogTable[num]) +#else +#define GetHighestSetBit_32_nonzero_small GetHighestSetBit_32_nonzero_big +#endif + + +#ifdef MY_clz + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + bitOffset -= (CBitCtr)(MY_clz(b) - 23); +#elif defined(Z7_ZSTD_DEC_USE_BSR) + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + { unsigned long zz; _BitScanReverse(&zz, b); bitOffset -= 8; bitOffset += zz; } +#else + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + for (;;) { bitOffset--; if (b & 0x80) { break; } b <<= 1; } +#endif + +#define SET_bitOffset_TO_PAD(bitOffset, src, srcLen) \ +{ \ + unsigned lastByte = (src)[(size_t)(srcLen) - 1]; \ + if (lastByte == 0) return SZ_ERROR_DATA; \ + bitOffset = (CBitCtr)((srcLen) * 8); \ + UPDATE_BIT_OFFSET_FOR_PADDING(lastByte, bitOffset) \ +} + +#ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + +#define SET_bitOffset_TO_PAD_and_SET_BIT_SIZE(bitOffset, src, srcLen_res) \ +{ \ + unsigned lastByte = (src)[(size_t)(srcLen_res) - 1]; \ + if (lastByte == 0) return SZ_ERROR_DATA; \ + srcLen_res *= 8; \ + bitOffset = (CBitCtr)srcLen_res; \ + UPDATE_BIT_OFFSET_FOR_PADDING(lastByte, bitOffset) \ +} + +#endif + +/* +typedef Int32 CBitCtr_signed; +typedef Int32 CBitCtr; +*/ +// /* +typedef ptrdiff_t CBitCtr_signed; +typedef ptrdiff_t CBitCtr; +// */ + + +#define MATCH_LEN_MIN 3 +#define kBlockSizeMax (1u << 17) + +// #define Z7_ZSTD_DEC_PRINT_TABLE + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE +#define NUM_OFFSET_SYMBOLS_PREDEF 29 +#endif +#define NUM_OFFSET_SYMBOLS_MAX (MAX_WINDOW_SIZE_LOG + 1) // 32 +#define NUM_LL_SYMBOLS 36 +#define NUM_ML_SYMBOLS 53 +#define FSE_NUM_SYMBOLS_MAX 53 // NUM_ML_SYMBOLS + +// /* +#if !defined(MY_CPU_X86) || defined(__PIC__) || defined(MY_CPU_64BIT) +#define Z7_ZSTD_DEC_USE_BASES_IN_OBJECT +#endif +// */ +// for debug: +// #define Z7_ZSTD_DEC_USE_BASES_LOCAL +// #define Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + +#define GLOBAL_TABLE(n) k_ ## n + +#if defined(Z7_ZSTD_DEC_USE_BASES_LOCAL) + #define BASES_TABLE(n) a_ ## n +#elif defined(Z7_ZSTD_DEC_USE_BASES_IN_OBJECT) + #define BASES_TABLE(n) p->m_ ## n +#else + #define BASES_TABLE(n) GLOBAL_TABLE(n) +#endif + +#define Z7_ZSTD_DEC_USE_ML_PLUS3 + +#if defined(Z7_ZSTD_DEC_USE_BASES_LOCAL) || \ + defined(Z7_ZSTD_DEC_USE_BASES_IN_OBJECT) + +#define SEQ_EXTRA_TABLES(n) \ + Byte n ## SEQ_LL_EXTRA [NUM_LL_SYMBOLS]; \ + Byte n ## SEQ_ML_EXTRA [NUM_ML_SYMBOLS]; \ + UInt32 n ## SEQ_LL_BASES [NUM_LL_SYMBOLS]; \ + UInt32 n ## SEQ_ML_BASES [NUM_ML_SYMBOLS]; \ + +#define Z7_ZSTD_DEC_USE_BASES_CALC + +#ifdef Z7_ZSTD_DEC_USE_BASES_CALC + + #define FILL_LOC_BASES(n, startSum) \ + { unsigned i; UInt32 sum = startSum; \ + for (i = 0; i != Z7_ARRAY_SIZE(GLOBAL_TABLE(n ## _EXTRA)); i++) \ + { const unsigned a = GLOBAL_TABLE(n ## _EXTRA)[i]; \ + BASES_TABLE(n ## _BASES)[i] = sum; \ + /* if (sum != GLOBAL_TABLE(n ## _BASES)[i]) exit(1); */ \ + sum += (UInt32)1 << a; \ + BASES_TABLE(n ## _EXTRA)[i] = (Byte)a; }} + + #define FILL_LOC_BASES_ALL \ + FILL_LOC_BASES (SEQ_LL, 0) \ + FILL_LOC_BASES (SEQ_ML, MATCH_LEN_MIN) \ + +#else + #define COPY_GLOBAL_ARR(n) \ + memcpy(BASES_TABLE(n), GLOBAL_TABLE(n), sizeof(GLOBAL_TABLE(n))); + #define FILL_LOC_BASES_ALL \ + COPY_GLOBAL_ARR (SEQ_LL_EXTRA) \ + COPY_GLOBAL_ARR (SEQ_ML_EXTRA) \ + COPY_GLOBAL_ARR (SEQ_LL_BASES) \ + COPY_GLOBAL_ARR (SEQ_ML_BASES) \ + +#endif + +#endif + + + +/// The sequence decoding baseline and number of additional bits to read/add +#if !defined(Z7_ZSTD_DEC_USE_BASES_CALC) +static const UInt32 GLOBAL_TABLE(SEQ_LL_BASES) [NUM_LL_SYMBOLS] = +{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 +}; +#endif + +static const Byte GLOBAL_TABLE(SEQ_LL_EXTRA) [NUM_LL_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16 +}; + +#if !defined(Z7_ZSTD_DEC_USE_BASES_CALC) +static const UInt32 GLOBAL_TABLE(SEQ_ML_BASES) [NUM_ML_SYMBOLS] = +{ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 +}; +#endif + +static const Byte GLOBAL_TABLE(SEQ_ML_EXTRA) [NUM_ML_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16 +}; + + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE + +static const Int16 SEQ_LL_PREDEF_DIST [NUM_LL_SYMBOLS] = +{ + 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 +}; +static const Int16 SEQ_OFFSET_PREDEF_DIST [NUM_OFFSET_SYMBOLS_PREDEF] = +{ + 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 +}; +static const Int16 SEQ_ML_PREDEF_DIST [NUM_ML_SYMBOLS] = +{ + 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 +}; + +#endif + +// typedef int FastInt; +// typedef Int32 FastInt32; +typedef unsigned FastInt; +typedef UInt32 FastInt32; +typedef FastInt32 CFseRecord; + + +#define FSE_REC_LEN_OFFSET 8 +#define FSE_REC_STATE_OFFSET 16 +#define GET_FSE_REC_SYM(st) ((Byte)(st)) +#define GET_FSE_REC_LEN(st) ((Byte)((st) >> FSE_REC_LEN_OFFSET)) +#define GET_FSE_REC_STATE(st) ((st) >> FSE_REC_STATE_OFFSET) + +// #define FSE_REC_SYM_MASK (0xff) +// #define GET_FSE_REC_SYM(st) (st & FSE_REC_SYM_MASK) + +#define W_BASE(state, len, sym) \ + (((UInt32)state << (4 + FSE_REC_STATE_OFFSET)) + \ + (len << FSE_REC_LEN_OFFSET) + (sym)) +#define W(state, len, sym) W_BASE(state, len, sym) +static const CFseRecord k_PredefRecords_LL[1 << 6] = { +W(0,4, 0),W(1,4, 0),W(2,5, 1),W(0,5, 3),W(0,5, 4),W(0,5, 6),W(0,5, 7),W(0,5, 9), +W(0,5,10),W(0,5,12),W(0,6,14),W(0,5,16),W(0,5,18),W(0,5,19),W(0,5,21),W(0,5,22), +W(0,5,24),W(2,5,25),W(0,5,26),W(0,6,27),W(0,6,29),W(0,6,31),W(2,4, 0),W(0,4, 1), +W(0,5, 2),W(2,5, 4),W(0,5, 5),W(2,5, 7),W(0,5, 8),W(2,5,10),W(0,5,11),W(0,6,13), +W(2,5,16),W(0,5,17),W(2,5,19),W(0,5,20),W(2,5,22),W(0,5,23),W(0,4,25),W(1,4,25), +W(2,5,26),W(0,6,28),W(0,6,30),W(3,4, 0),W(1,4, 1),W(2,5, 2),W(2,5, 3),W(2,5, 5), +W(2,5, 6),W(2,5, 8),W(2,5, 9),W(2,5,11),W(2,5,12),W(0,6,15),W(2,5,17),W(2,5,18), +W(2,5,20),W(2,5,21),W(2,5,23),W(2,5,24),W(0,6,35),W(0,6,34),W(0,6,33),W(0,6,32) +}; +static const CFseRecord k_PredefRecords_OF[1 << 5] = { +W(0,5, 0),W(0,4, 6),W(0,5, 9),W(0,5,15),W(0,5,21),W(0,5, 3),W(0,4, 7),W(0,5,12), +W(0,5,18),W(0,5,23),W(0,5, 5),W(0,4, 8),W(0,5,14),W(0,5,20),W(0,5, 2),W(1,4, 7), +W(0,5,11),W(0,5,17),W(0,5,22),W(0,5, 4),W(1,4, 8),W(0,5,13),W(0,5,19),W(0,5, 1), +W(1,4, 6),W(0,5,10),W(0,5,16),W(0,5,28),W(0,5,27),W(0,5,26),W(0,5,25),W(0,5,24) +}; +#if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) +#undef W +#define W(state, len, sym) W_BASE(state, len, (sym + MATCH_LEN_MIN)) +#endif +static const CFseRecord k_PredefRecords_ML[1 << 6] = { +W(0,6, 0),W(0,4, 1),W(2,5, 2),W(0,5, 3),W(0,5, 5),W(0,5, 6),W(0,5, 8),W(0,6,10), +W(0,6,13),W(0,6,16),W(0,6,19),W(0,6,22),W(0,6,25),W(0,6,28),W(0,6,31),W(0,6,33), +W(0,6,35),W(0,6,37),W(0,6,39),W(0,6,41),W(0,6,43),W(0,6,45),W(1,4, 1),W(0,4, 2), +W(2,5, 3),W(0,5, 4),W(2,5, 6),W(0,5, 7),W(0,6, 9),W(0,6,12),W(0,6,15),W(0,6,18), +W(0,6,21),W(0,6,24),W(0,6,27),W(0,6,30),W(0,6,32),W(0,6,34),W(0,6,36),W(0,6,38), +W(0,6,40),W(0,6,42),W(0,6,44),W(2,4, 1),W(3,4, 1),W(1,4, 2),W(2,5, 4),W(2,5, 5), +W(2,5, 7),W(2,5, 8),W(0,6,11),W(0,6,14),W(0,6,17),W(0,6,20),W(0,6,23),W(0,6,26), +W(0,6,29),W(0,6,52),W(0,6,51),W(0,6,50),W(0,6,49),W(0,6,48),W(0,6,47),W(0,6,46) +}; + + +// sum of freqs[] must be correct +// (numSyms != 0) +// (accuracy >= 5) +static +Z7_NO_INLINE +// Z7_FORCE_INLINE +void FSE_Generate(CFseRecord *table, + const Int16 *const freqs, const size_t numSyms, + const unsigned accuracy, UInt32 delta) +{ + size_t size = (size_t)1 << accuracy; + // max value in states[x] is ((1 << accuracy) * 2) + UInt16 states[FSE_NUM_SYMBOLS_MAX]; + { + /* Symbols with "less than 1" probability get a single cell, + starting from the end of the table. + These symbols define a full state reset, reading (accuracy) bits. */ + size_t threshold = size; + { + size_t s = 0; + do + if (freqs[s] == -1) + { + table[--threshold] = (CFseRecord)s; + states[s] = 1; + } + while (++s != numSyms); + } + + #ifdef SHOW_STAT + if (threshold == size) + { + STAT_INC(g_Num_Threshold_0) + STAT_UPDATE(g_Num_Threshold_0sum += (unsigned)size;) + } + else + { + STAT_INC(g_Num_Threshold_1) + STAT_UPDATE(g_Num_Threshold_1sum += (unsigned)size;) + } + #endif + + // { unsigned uuu; for (uuu = 0; uuu < 400; uuu++) + { + // Each (symbol) gets freqs[symbol] cells. + // Cell allocation is spread, not linear. + const size_t step = (size >> 1) + (size >> 3) + 3; + size_t pos = 0; + // const unsigned mask = size - 1; + /* + if (threshold == size) + { + size_t s = 0; + size--; + do + { + int freq = freqs[s]; + if (freq <= 0) + continue; + states[s] = (UInt16)freq; + do + { + table[pos] (CFseRecord)s; + pos = (pos + step) & size; // & mask; + } + while (--freq); + } + while (++s != numSyms); + } + else + */ + { + size_t s = 0; + size--; + do + { + int freq = freqs[s]; + if (freq <= 0) + continue; + states[s] = (UInt16)freq; + do + { + table[pos] = (CFseRecord)s; + // we skip position, if it's already occupied by a "less than 1" probability symbol. + // (step) is coprime to table size, so the cycle will visit each position exactly once + do + pos = (pos + step) & size; // & mask; + while (pos >= threshold); + } + while (--freq); + } + while (++s != numSyms); + } + size++; + // (pos != 0) is unexpected case that means that freqs[] are not correct. + // so it's some failure in code (for example, incorrect predefined freq[] table) + // if (pos != 0) return SZ_ERROR_FAIL; + } + // } + } + { + const CFseRecord * const limit = table + size; + delta = ((UInt32)size << FSE_REC_STATE_OFFSET) - delta; + /* State increases by symbol over time, decreasing number of bits. + Baseline increases until the bit threshold is passed, at which point it resets to 0 */ + do + { + #define TABLE_ITER(a) \ + { \ + const FastInt sym = (FastInt)table[a]; \ + const unsigned nextState = states[sym]; \ + unsigned nb; \ + states[sym] = (UInt16)(nextState + 1); \ + nb = accuracy - GetHighestSetBit_32_nonzero_small(nextState); \ + table[a] = (CFseRecord)(sym - delta \ + + ((UInt32)nb << FSE_REC_LEN_OFFSET) \ + + ((UInt32)nextState << FSE_REC_STATE_OFFSET << nb)); \ + } + TABLE_ITER(0) + TABLE_ITER(1) + table += 2; + } + while (table != limit); + } +} + + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE + +static void Print_Predef(unsigned predefAccuracy, + const unsigned numSymsPredef, + const Int16 * const predefFreqs, + const CFseRecord *checkTable) +{ + CFseRecord table[1 << 6]; + unsigned i; + FSE_Generate(table, predefFreqs, numSymsPredef, predefAccuracy, + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + numSymsPredef == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : + #endif + 0 + ); + if (memcmp(table, checkTable, sizeof(UInt32) << predefAccuracy) != 0) + exit(1); + for (i = 0; i < (1u << predefAccuracy); i++) + { + const UInt32 v = table[i]; + const unsigned state = (unsigned)(GET_FSE_REC_STATE(v)); + if (state & 0xf) + exit(1); + if (i != 0) + { + printf(","); + if (i % 8 == 0) + printf("\n"); + } + printf("W(%d,%d,%2d)", + (unsigned)(state >> 4), + (unsigned)((v >> FSE_REC_LEN_OFFSET) & 0xff), + (unsigned)GET_FSE_REC_SYM(v)); + } + printf("\n\n"); +} + +#endif + + +#define GET16(dest, p) { const Byte *ptr = p; dest = GetUi16(ptr); } +#define GET32(dest, p) { const Byte *ptr = p; dest = GetUi32(ptr); } + +// (1 <= numBits <= 9) +#define FORWARD_READ_BITS(destVal, numBits, mask) \ + { const CBitCtr_signed bos3 = (bitOffset) >> 3; \ + if (bos3 >= 0) return SZ_ERROR_DATA; \ + GET16(destVal, src + bos3) \ + destVal >>= (bitOffset) & 7; \ + bitOffset += (CBitCtr_signed)(numBits); \ + mask = (1u << (numBits)) - 1; \ + destVal &= mask; \ + } + +#define FORWARD_READ_1BIT(destVal) \ + { const CBitCtr_signed bos3 = (bitOffset) >> 3; \ + if (bos3 >= 0) return SZ_ERROR_DATA; \ + destVal = *(src + bos3); \ + destVal >>= (bitOffset) & 7; \ + (bitOffset)++; \ + destVal &= 1; \ + } + + +// in: (accuracyMax <= 9) +// at least 2 bytes will be processed from (in) stream. +// at return: (in->len > 0) +static +Z7_NO_INLINE +SRes FSE_DecodeHeader(CFseRecord *const table, + CInBufPair *const in, + const unsigned accuracyMax, + Byte *const accuracyRes, + unsigned numSymbolsMax) +{ + unsigned accuracy; + unsigned remain1; + unsigned syms; + Int16 freqs[FSE_NUM_SYMBOLS_MAX + 3]; // +3 for overwrite (repeat) + const Byte *src = in->ptr; + CBitCtr_signed bitOffset = (CBitCtr_signed)in->len - 1; + if (bitOffset <= 0) + return SZ_ERROR_DATA; + accuracy = *src & 0xf; + accuracy += 5; + if (accuracy > accuracyMax) + return SZ_ERROR_DATA; + *accuracyRes = (Byte)accuracy; + remain1 = (1u << accuracy) + 1; // (it's remain_freqs_sum + 1) + syms = 0; + src += bitOffset; // src points to last byte + bitOffset = 4 - (bitOffset << 3); + + for (;;) + { + // (2 <= remain1) + const unsigned bits = GetHighestSetBit_32_nonzero_small((unsigned)remain1); + // (1 <= bits <= accuracy) + unsigned val; // it must be unsigned or int + unsigned mask; + FORWARD_READ_BITS(val, bits, mask) + { + const unsigned val2 = remain1 + val - mask; + if (val2 > mask) + { + unsigned bit; + FORWARD_READ_1BIT(bit) + if (bit) + val = val2; + } + } + { + // (remain1 >= 2) + // (0 <= (int)val <= remain1) + val = (unsigned)((int)val - 1); + // val now is "probability" of symbol + // (probability == -1) means "less than 1" frequency. + // (-1 <= (int)val <= remain1 - 1) + freqs[syms++] = (Int16)(int)val; + if (val != 0) + { + remain1 -= (int)val < 0 ? 1u : (unsigned)val; + // remain1 -= val; + // val >>= (sizeof(val) * 8 - 2); + // remain1 -= val & 2; + // freqs[syms++] = (Int16)(int)val; + // syms++; + if (remain1 == 1) + break; + if (syms >= FSE_NUM_SYMBOLS_MAX) + return SZ_ERROR_DATA; + } + else // if (val == 0) + { + // freqs[syms++] = 0; + // syms++; + for (;;) + { + unsigned repeat; + FORWARD_READ_BITS(repeat, 2, mask) + freqs[syms ] = 0; + freqs[syms + 1] = 0; + freqs[syms + 2] = 0; + syms += repeat; + if (syms >= FSE_NUM_SYMBOLS_MAX) + return SZ_ERROR_DATA; + if (repeat != 3) + break; + } + } + } + } + + if (syms > numSymbolsMax) + return SZ_ERROR_DATA; + bitOffset += 7; + bitOffset >>= 3; + if (bitOffset > 0) + return SZ_ERROR_DATA; + in->ptr = src + bitOffset; + in->len = (size_t)(1 - bitOffset); + { + // unsigned uuu; for (uuu = 0; uuu < 50; uuu++) + FSE_Generate(table, freqs, syms, accuracy, + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + numSymbolsMax == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : + #endif + 0 + ); + } + return SZ_OK; +} + + +// ---------- HUFFMAN ---------- + +#define HUF_MAX_BITS 12 +#define HUF_MAX_SYMBS 256 +#define HUF_DUMMY_SIZE (128 + 8 * 2) // it must multiple of 8 +// #define HUF_DUMMY_SIZE 0 +#define HUF_TABLE_SIZE ((2 << HUF_MAX_BITS) + HUF_DUMMY_SIZE) +#define HUF_GET_SYMBOLS(table) ((table) + (1 << HUF_MAX_BITS) + HUF_DUMMY_SIZE) +// #define HUF_GET_LENS(table) (table) + +typedef struct +{ + // Byte table[HUF_TABLE_SIZE]; + UInt64 table64[HUF_TABLE_SIZE / sizeof(UInt64)]; +} +CZstdDecHufTable; + +/* +Input: + numSyms != 0 + (bits) array size must be aligned for 2 + if (numSyms & 1), then bits[numSyms] == 0, + Huffman tree must be correct before Huf_Build() call: + (sum (1/2^bits[i]) == 1). + && (bits[i] <= HUF_MAX_BITS) +*/ +static +Z7_FORCE_INLINE +void Huf_Build(Byte * const table, + const Byte *bits, const unsigned numSyms) +{ + unsigned counts0[HUF_MAX_BITS + 1]; + unsigned counts1[HUF_MAX_BITS + 1]; + const Byte * const bitsEnd = bits + numSyms; + // /* + { + unsigned t; + for (t = 0; t < Z7_ARRAY_SIZE(counts0); t++) counts0[t] = 0; + for (t = 0; t < Z7_ARRAY_SIZE(counts1); t++) counts1[t] = 0; + } + // */ + // memset(counts0, 0, sizeof(counts0)); + // memset(counts1, 0, sizeof(counts1)); + { + const Byte *bits2 = bits; + // we access additional bits[symbol] if (numSyms & 1) + do + { + counts0[bits2[0]]++; + counts1[bits2[1]]++; + } + while ((bits2 += 2) < bitsEnd); + } + { + unsigned r = 0; + unsigned i = HUF_MAX_BITS; + // Byte *lens = HUF_GET_LENS(symbols); + do + { + const unsigned num = (counts0[i] + counts1[i]) << (HUF_MAX_BITS - i); + counts0[i] = r; + if (num) + { + Byte *lens = &table[r]; + r += num; + memset(lens, (int)i, num); + } + } + while (--i); + counts0[0] = 0; // for speculated loads + // no need for check: + // if (r != (UInt32)1 << HUF_MAX_BITS) exit(0); + } + { + #ifdef MY_CPU_64BIT + UInt64 + #else + UInt32 + #endif + v = 0; + Byte *symbols = HUF_GET_SYMBOLS(table); + do + { + const unsigned nb = *bits++; + if (nb) + { + const unsigned code = counts0[nb]; + const unsigned num = (1u << HUF_MAX_BITS) >> nb; + counts0[nb] = code + num; + // memset(&symbols[code], i, num); + // /* + { + Byte *s2 = &symbols[code]; + if (num <= 2) + { + s2[0] = (Byte)v; + s2[(size_t)num - 1] = (Byte)v; + } + else if (num <= 8) + { + *(UInt32 *)(void *)s2 = (UInt32)v; + *(UInt32 *)(void *)(s2 + (size_t)num - 4) = (UInt32)v; + } + else + { + #ifdef MY_CPU_64BIT + UInt64 *s = (UInt64 *)(void *)s2; + const UInt64 *lim = (UInt64 *)(void *)(s2 + num); + do + { + s[0] = v; s[1] = v; s += 2; + } + while (s != lim); + #else + UInt32 *s = (UInt32 *)(void *)s2; + const UInt32 *lim = (const UInt32 *)(const void *)(s2 + num); + do + { + s[0] = v; s[1] = v; s += 2; + s[0] = v; s[1] = v; s += 2; + } + while (s != lim); + #endif + } + } + // */ + } + v += + #ifdef MY_CPU_64BIT + 0x0101010101010101; + #else + 0x01010101; + #endif + } + while (bits != bitsEnd); + } +} + + + +// how many bytes (src) was moved back from original value. +// we need (HUF_SRC_OFFSET == 3) for optimized 32-bit memory access +#define HUF_SRC_OFFSET 3 + +// v <<= 8 - (bitOffset & 7) + numBits; +// v >>= 32 - HUF_MAX_BITS; +#define HUF_GET_STATE(v, bitOffset, numBits) \ + GET32(v, src + (HUF_SRC_OFFSET - 3) + ((CBitCtr_signed)bitOffset >> 3)) \ + v >>= 32 - HUF_MAX_BITS - 8 + ((unsigned)bitOffset & 7) - numBits; \ + v &= (1u << HUF_MAX_BITS) - 1; \ + + +#ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS +#if defined(MY_CPU_AMD64) && defined(_MSC_VER) && _MSC_VER == 1400 \ + || !defined(MY_CPU_X86_OR_AMD64) \ + // || 1 == 1 /* for debug : to force STREAM4_PRELOAD mode */ + // we need big number (>=16) of registers for PRELOAD4 + #define Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4 + // #define Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2 // for debug +#endif +#endif + +// for debug: simpler and smaller code but slow: +// #define Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE + +#if defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE) || \ + !defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS) + +#define HUF_DECODE(bitOffset, dest) \ +{ \ + UInt32 v; \ + HUF_GET_STATE(v, bitOffset, 0) \ + bitOffset -= table[v]; \ + *(dest) = symbols[v]; \ + if ((CBitCtr_signed)bitOffset < 0) return SZ_ERROR_DATA; \ +} + +#endif + +#if !defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE) || \ + defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4) || \ + defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2) \ + +#define HUF_DECODE_2_INIT(v, bitOffset) \ + HUF_GET_STATE(v, bitOffset, 0) + +#define HUF_DECODE_2(v, bitOffset, dest) \ +{ \ + unsigned numBits; \ + numBits = table[v]; \ + *(dest) = symbols[v]; \ + HUF_GET_STATE(v, bitOffset, numBits) \ + bitOffset -= (CBitCtr)numBits; \ + if ((CBitCtr_signed)bitOffset < 0) return SZ_ERROR_DATA; \ +} + +#endif + + +// src == ptr - HUF_SRC_OFFSET +// we are allowed to access 3 bytes before start of input buffer +static +Z7_NO_INLINE +SRes Huf_Decompress_1stream(const Byte * const table, + const Byte *src, const size_t srcLen, + Byte *dest, const size_t destLen) +{ + CBitCtr bitOffset; + if (srcLen == 0) + return SZ_ERROR_DATA; + SET_bitOffset_TO_PAD (bitOffset, src + HUF_SRC_OFFSET, srcLen) + if (destLen) + { + const Byte *symbols = HUF_GET_SYMBOLS(table); + const Byte *destLim = dest + destLen; + #ifdef Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE + { + do + { + HUF_DECODE (bitOffset, dest) + } + while (++dest != destLim); + } + #else + { + UInt32 v; + HUF_DECODE_2_INIT (v, bitOffset) + do + { + HUF_DECODE_2 (v, bitOffset, dest) + } + while (++dest != destLim); + } + #endif + } + return bitOffset == 0 ? SZ_OK : SZ_ERROR_DATA; +} + + +// for debug : it reduces register pressure : by array copy can be slow : +// #define Z7_ZSTD_DEC_USE_HUF_LOCAL + +// src == ptr + (6 - HUF_SRC_OFFSET) +// srcLen >= 10 +// we are allowed to access 3 bytes before start of input buffer +static +Z7_NO_INLINE +SRes Huf_Decompress_4stream(const Byte * const + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + table2, + #else + table, + #endif + const Byte *src, size_t srcLen, + Byte *dest, size_t destLen) +{ + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + Byte table[HUF_TABLE_SIZE]; + #endif + UInt32 sizes[3]; + const size_t delta = (destLen + 3) / 4; + if ((sizes[0] = GetUi16(src + (0 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + if ((sizes[1] = GetUi16(src + (2 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + sizes[1] += sizes[0]; + if ((sizes[2] = GetUi16(src + (4 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + sizes[2] += sizes[1]; + srcLen -= 6; + if (srcLen <= sizes[2]) + return SZ_ERROR_DATA; + + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + { + // unsigned i = 0; for(; i < 1000; i++) + memcpy(table, table2, HUF_TABLE_SIZE); + } + #endif + + #ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + { + CBitCtr bitOffset_0, + bitOffset_1, + bitOffset_2, + bitOffset_3; + { + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_0, src + HUF_SRC_OFFSET, sizes[0]) + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_1, src + HUF_SRC_OFFSET, sizes[1]) + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_2, src + HUF_SRC_OFFSET, sizes[2]) + SET_bitOffset_TO_PAD (bitOffset_3, src + HUF_SRC_OFFSET, srcLen) + } + { + const Byte * const symbols = HUF_GET_SYMBOLS(table); + Byte *destLim = dest + destLen - delta * 3; + + if (dest != destLim) + #ifdef Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4 + { + UInt32 v_0, v_1, v_2, v_3; + HUF_DECODE_2_INIT (v_0, bitOffset_0) + HUF_DECODE_2_INIT (v_1, bitOffset_1) + HUF_DECODE_2_INIT (v_2, bitOffset_2) + HUF_DECODE_2_INIT (v_3, bitOffset_3) + // #define HUF_DELTA (1 << 17) / 4 + do + { + HUF_DECODE_2 (v_3, bitOffset_3, dest + delta * 3) + HUF_DECODE_2 (v_2, bitOffset_2, dest + delta * 2) + HUF_DECODE_2 (v_1, bitOffset_1, dest + delta) + HUF_DECODE_2 (v_0, bitOffset_0, dest) + } + while (++dest != destLim); + /* + {// unsigned y = 0; for (;y < 1; y++) + { + const size_t num = destLen - delta * 3; + Byte *orig = dest - num; + memmove (orig + delta , orig + HUF_DELTA, num); + memmove (orig + delta * 2, orig + HUF_DELTA * 2, num); + memmove (orig + delta * 3, orig + HUF_DELTA * 3, num); + }} + */ + } + #elif defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2) + { + UInt32 v_0, v_1, v_2, v_3; + HUF_DECODE_2_INIT (v_0, bitOffset_0) + HUF_DECODE_2_INIT (v_1, bitOffset_1) + do + { + HUF_DECODE_2 (v_0, bitOffset_0, dest) + HUF_DECODE_2 (v_1, bitOffset_1, dest + delta) + } + while (++dest != destLim); + dest = destLim - (destLen - delta * 3); + dest += delta * 2; + destLim += delta * 2; + HUF_DECODE_2_INIT (v_2, bitOffset_2) + HUF_DECODE_2_INIT (v_3, bitOffset_3) + do + { + HUF_DECODE_2 (v_2, bitOffset_2, dest) + HUF_DECODE_2 (v_3, bitOffset_3, dest + delta) + } + while (++dest != destLim); + dest -= delta * 2; + destLim -= delta * 2; + } + #else + { + do + { + HUF_DECODE (bitOffset_3, dest + delta * 3) + HUF_DECODE (bitOffset_2, dest + delta * 2) + HUF_DECODE (bitOffset_1, dest + delta) + HUF_DECODE (bitOffset_0, dest) + } + while (++dest != destLim); + } + #endif + + if (bitOffset_3 != (CBitCtr)sizes[2]) + return SZ_ERROR_DATA; + if (destLen &= 3) + { + destLim = dest + 4 - destLen; + do + { + HUF_DECODE (bitOffset_2, dest + delta * 2) + HUF_DECODE (bitOffset_1, dest + delta) + HUF_DECODE (bitOffset_0, dest) + } + while (++dest != destLim); + } + if ( bitOffset_0 != 0 + || bitOffset_1 != (CBitCtr)sizes[0] + || bitOffset_2 != (CBitCtr)sizes[1]) + return SZ_ERROR_DATA; + } + } + #else // Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + { + unsigned i; + for (i = 0; i < 4; i++) + { + size_t d = destLen; + size_t size = srcLen; + if (i != 3) + { + d = delta; + size = sizes[i]; + } + if (i != 0) + size -= sizes[i - 1]; + destLen -= d; + RINOK(Huf_Decompress_1stream(table, src, size, dest, d)) + dest += d; + src += size; + } + } + #endif + + return SZ_OK; +} + + + +// (in->len != 0) +// we are allowed to access in->ptr[-3] +// at least 2 bytes in (in->ptr) will be processed +static SRes Huf_DecodeTable(CZstdDecHufTable *const p, CInBufPair *const in) +{ + Byte weights[HUF_MAX_SYMBS + 1]; // +1 for extra write for loop unroll + unsigned numSyms; + const unsigned header = *(in->ptr)++; + in->len--; + // memset(weights, 0, sizeof(weights)); + if (header >= 128) + { + // direct representation: 4 bits field (0-15) per weight + numSyms = header - 127; + // numSyms != 0 + { + const size_t numBytes = (numSyms + 1) / 2; + const Byte *const ws = in->ptr; + size_t i = 0; + if (in->len < numBytes) + return SZ_ERROR_DATA; + in->ptr += numBytes; + in->len -= numBytes; + do + { + const unsigned b = ws[i]; + weights[i * 2 ] = (Byte)(b >> 4); + weights[i * 2 + 1] = (Byte)(b & 0xf); + } + while (++i != numBytes); + /* 7ZIP: we can restore correct zero value for weights[numSyms], + if we want to use zero values starting from numSyms in code below. */ + // weights[numSyms] = 0; + } + } + else + { + #define MAX_ACCURACY_LOG_FOR_WEIGHTS 6 + CFseRecord table[1 << MAX_ACCURACY_LOG_FOR_WEIGHTS]; + + Byte accuracy; + const Byte *src; + size_t srcLen; + if (in->len < header) + return SZ_ERROR_DATA; + { + CInBufPair fse_stream; + fse_stream.len = header; + fse_stream.ptr = in->ptr; + in->ptr += header; + in->len -= header; + RINOK(FSE_DecodeHeader(table, &fse_stream, + MAX_ACCURACY_LOG_FOR_WEIGHTS, + &accuracy, + 16 // num weight symbols max (max-symbol is 15) + )) + // at least 2 bytes were processed in fse_stream. + // (srcLen > 0) after FSE_DecodeHeader() + // if (srcLen == 0) return SZ_ERROR_DATA; + src = fse_stream.ptr; + srcLen = fse_stream.len; + } + // we are allowed to access src[-5] + { + // unsigned yyy = 200; do { + CBitCtr bitOffset; + FastInt32 state1, state2; + SET_bitOffset_TO_PAD (bitOffset, src, srcLen) + state1 = accuracy; + src -= state1 >> 2; // src -= 1; // for GET16() optimization + state1 <<= FSE_REC_LEN_OFFSET; + state2 = state1; + numSyms = 0; + for (;;) + { + #define FSE_WEIGHT_DECODE(st) \ + { \ + const unsigned bits = GET_FSE_REC_LEN(st); \ + FastInt r; \ + GET16(r, src + (bitOffset >> 3)) \ + r >>= (unsigned)bitOffset & 7; \ + if ((CBitCtr_signed)(bitOffset -= (CBitCtr)bits) < 0) \ + { if (bitOffset + (CBitCtr)bits != 0) \ + return SZ_ERROR_DATA; \ + break; } \ + r &= 0xff; \ + r >>= 8 - bits; \ + st = table[GET_FSE_REC_STATE(st) + r]; \ + weights[numSyms++] = (Byte)GET_FSE_REC_SYM(st); \ + } + FSE_WEIGHT_DECODE (state1) + FSE_WEIGHT_DECODE (state2) + if (numSyms == HUF_MAX_SYMBS) + return SZ_ERROR_DATA; + } + // src += (unsigned)accuracy >> 2; } while (--yyy); + } + } + + // Build using weights: + { + UInt32 sum = 0; + { + // numSyms >= 1 + unsigned i = 0; + weights[numSyms] = 0; + do + { + sum += ((UInt32)1 << weights[i ]) & ~(UInt32)1; + sum += ((UInt32)1 << weights[i + 1]) & ~(UInt32)1; + i += 2; + } + while (i < numSyms); + if (sum == 0) + return SZ_ERROR_DATA; + } + { + const unsigned maxBits = GetHighestSetBit_32_nonzero_big(sum) + 1; + { + const UInt32 left = ((UInt32)1 << maxBits) - sum; + // (left != 0) + // (left) must be power of 2 in correct stream + if (left & (left - 1)) + return SZ_ERROR_DATA; + weights[numSyms++] = (Byte)GetHighestSetBit_32_nonzero_big(left); + } + // if (numSyms & 1) + weights[numSyms] = 0; // for loop unroll + // numSyms >= 2 + { + unsigned i = 0; + do + { + /* + #define WEIGHT_ITER(a) \ + { unsigned w = weights[i + (a)]; \ + const unsigned t = maxBits - w; \ + w = w ? t: w; \ + if (w > HUF_MAX_BITS) return SZ_ERROR_DATA; \ + weights[i + (a)] = (Byte)w; } + */ + // /* + #define WEIGHT_ITER(a) \ + { unsigned w = weights[i + (a)]; \ + if (w) { \ + w = maxBits - w; \ + if (w > HUF_MAX_BITS) return SZ_ERROR_DATA; \ + weights[i + (a)] = (Byte)w; }} + // */ + WEIGHT_ITER(0) + // WEIGHT_ITER(1) + // i += 2; + } + while (++i != numSyms); + } + } + } + { + // unsigned yyy; for (yyy = 0; yyy < 100; yyy++) + Huf_Build((Byte *)(void *)p->table64, weights, numSyms); + } + return SZ_OK; +} + + +typedef enum +{ + k_SeqMode_Predef = 0, + k_SeqMode_RLE = 1, + k_SeqMode_FSE = 2, + k_SeqMode_Repeat = 3 +} +z7_zstd_enum_SeqMode; + +// predefAccuracy == 5 for OFFSET symbols +// predefAccuracy == 6 for MATCH/LIT LEN symbols +static +SRes +Z7_NO_INLINE +// Z7_FORCE_INLINE +FSE_Decode_SeqTable(CFseRecord * const table, + CInBufPair * const in, + unsigned predefAccuracy, + Byte * const accuracyRes, + unsigned numSymbolsMax, + const CFseRecord * const predefs, + const unsigned seqMode) +{ + // UNUSED_VAR(numSymsPredef) + // UNUSED_VAR(predefFreqs) + if (seqMode == k_SeqMode_FSE) + { + // unsigned y = 50; CInBufPair in2 = *in; do { *in = in2; RINOK( + return + FSE_DecodeHeader(table, in, + predefAccuracy + 3, // accuracyMax + accuracyRes, + numSymbolsMax) + ; + // )} while (--y); return SZ_OK; + } + // numSymsMax = numSymsPredef + ((predefAccuracy & 1) * (32 - 29))); // numSymsMax + // numSymsMax == 32 for offsets + + if (seqMode == k_SeqMode_Predef) + { + *accuracyRes = (Byte)predefAccuracy; + memcpy(table, predefs, sizeof(UInt32) << predefAccuracy); + return SZ_OK; + } + + // (seqMode == k_SeqMode_RLE) + if (in->len == 0) + return SZ_ERROR_DATA; + in->len--; + { + const Byte *ptr = in->ptr; + const unsigned sym = ptr[0]; + in->ptr = ptr + 1; + if (sym >= numSymbolsMax) + return SZ_ERROR_DATA; + table[0] = (FastInt32)sym + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + + (numSymbolsMax == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : 0) + #endif + ; + *accuracyRes = 0; + } + return SZ_OK; +} + + +typedef struct +{ + CFseRecord of[1 << 8]; + CFseRecord ll[1 << 9]; + CFseRecord ml[1 << 9]; +} +CZstdDecFseTables; + + +typedef struct +{ + Byte *win; + SizeT cycSize; + /* + if (outBuf_fromCaller) : cycSize = outBufSize_fromCaller + else { + if ( isCyclicMode) : cycSize = cyclic_buffer_size = (winSize + extra_space) + if (!isCyclicMode) : cycSize = ContentSize, + (isCyclicMode == true) if (ContetSize >= winSize) or ContetSize is unknown + } + */ + SizeT winPos; + + CZstdDecOffset reps[3]; + + Byte ll_accuracy; + Byte of_accuracy; + Byte ml_accuracy; + // Byte seqTables_wereSet; + Byte litHuf_wasSet; + + Byte *literalsBase; + + size_t winSize; // from header + size_t totalOutCheck; // totalOutCheck <= winSize + + #ifdef Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + SEQ_EXTRA_TABLES(m_) + #endif + // UInt64 _pad_Alignment; // is not required now + CZstdDecFseTables fse; + CZstdDecHufTable huf; +} +CZstdDec1; + +#define ZstdDec1_GET_BLOCK_SIZE_LIMIT(p) \ + ((p)->winSize < kBlockSizeMax ? (UInt32)(p)->winSize : kBlockSizeMax) + +#define SEQ_TABLES_WERE_NOT_SET_ml_accuracy 1 // accuracy=1 is not used by zstd +#define IS_SEQ_TABLES_WERE_SET(p) (((p)->ml_accuracy != SEQ_TABLES_WERE_NOT_SET_ml_accuracy)) +// #define IS_SEQ_TABLES_WERE_SET(p) ((p)->seqTables_wereSet) + + +static void ZstdDec1_Construct(CZstdDec1 *p) +{ + #ifdef Z7_ZSTD_DEC_PRINT_TABLE + Print_Predef(6, NUM_LL_SYMBOLS, SEQ_LL_PREDEF_DIST, k_PredefRecords_LL); + Print_Predef(5, NUM_OFFSET_SYMBOLS_PREDEF, SEQ_OFFSET_PREDEF_DIST, k_PredefRecords_OF); + Print_Predef(6, NUM_ML_SYMBOLS, SEQ_ML_PREDEF_DIST, k_PredefRecords_ML); + #endif + + p->win = NULL; + p->cycSize = 0; + p->literalsBase = NULL; + #ifdef Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + FILL_LOC_BASES_ALL + #endif +} + + +static void ZstdDec1_Init(CZstdDec1 *p) +{ + p->reps[0] = 1; + p->reps[1] = 4; + p->reps[2] = 8; + // p->seqTables_wereSet = False; + p->ml_accuracy = SEQ_TABLES_WERE_NOT_SET_ml_accuracy; + p->litHuf_wasSet = False; + p->totalOutCheck = 0; +} + + + +#ifdef MY_CPU_LE_UNALIGN + #define Z7_ZSTD_DEC_USE_UNALIGNED_COPY +#endif + +#ifdef Z7_ZSTD_DEC_USE_UNALIGNED_COPY + + #define COPY_CHUNK_SIZE 16 + + #define COPY_CHUNK_4_2(dest, src) \ + { \ + ((UInt32 *)(void *)dest)[0] = ((const UInt32 *)(const void *)src)[0]; \ + ((UInt32 *)(void *)dest)[1] = ((const UInt32 *)(const void *)src)[1]; \ + src += 4 * 2; \ + dest += 4 * 2; \ + } + + /* sse2 doesn't help here in GCC and CLANG. + so we disabled sse2 here */ + /* + #if defined(MY_CPU_AMD64) + #define Z7_ZSTD_DEC_USE_SSE2 + #elif defined(MY_CPU_X86) + #if defined(_MSC_VER) && _MSC_VER >= 1300 && defined(_M_IX86_FP) && (_M_IX86_FP >= 2) \ + || defined(__SSE2__) \ + // || 1 == 1 // for debug only + #define Z7_ZSTD_DEC_USE_SSE2 + #endif + #endif + */ + + #if defined(MY_CPU_ARM64) + #define COPY_OFFSET_MIN 16 + #define COPY_CHUNK1(dest, src) \ + { \ + vst1q_u8((uint8_t *)(void *)dest, \ + vld1q_u8((const uint8_t *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if ((len -= COPY_CHUNK_SIZE) == 0) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(Z7_ZSTD_DEC_USE_SSE2) + #include // sse2 + #define COPY_OFFSET_MIN 16 + + #define COPY_CHUNK1(dest, src) \ + { \ + _mm_storeu_si128((__m128i *)(void *)dest, \ + _mm_loadu_si128((const __m128i *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if ((len -= COPY_CHUNK_SIZE) == 0) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(MY_CPU_64BIT) + #define COPY_OFFSET_MIN 8 + + #define COPY_CHUNK(dest, src) \ + { \ + ((UInt64 *)(void *)dest)[0] = ((const UInt64 *)(const void *)src)[0]; \ + ((UInt64 *)(void *)dest)[1] = ((const UInt64 *)(const void *)src)[1]; \ + src += 8 * 2; \ + dest += 8 * 2; \ + } + + #else + #define COPY_OFFSET_MIN 4 + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_4_2(dest, src); \ + COPY_CHUNK_4_2(dest, src); \ + } + + #endif +#endif + + +#ifndef COPY_CHUNK_SIZE + #define COPY_OFFSET_MIN 4 + #define COPY_CHUNK_SIZE 8 + #define COPY_CHUNK_2(dest, src) \ + { \ + const Byte a0 = src[0]; \ + const Byte a1 = src[1]; \ + dest[0] = a0; \ + dest[1] = a1; \ + src += 2; \ + dest += 2; \ + } + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + } +#endif + + +#define COPY_PREPARE \ + len += (COPY_CHUNK_SIZE - 1); \ + len &= ~(size_t)(COPY_CHUNK_SIZE - 1); \ + { if (len > rem) \ + { len = rem; \ + rem &= (COPY_CHUNK_SIZE - 1); \ + if (rem) { \ + len -= rem; \ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + do *dest++ = *src++; while (--rem); \ + if (len == 0) return; }}} + +#define COPY_CHUNKS \ +{ \ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + do { COPY_CHUNK(dest, src) } \ + while (len -= COPY_CHUNK_SIZE); \ +} + +// (len != 0) +// (len <= rem) +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyLiterals(Byte *dest, Byte const *src, size_t len, size_t rem) +{ + COPY_PREPARE + COPY_CHUNKS +} + + +/* we can define Z7_STD_DEC_USE_AFTER_CYC_BUF, if we want to use additional + space after cycSize that can be used to reduce the code in CopyMatch(): */ +// for debug: +// #define Z7_STD_DEC_USE_AFTER_CYC_BUF + +/* +CopyMatch() +if wrap (offset > winPos) +{ + then we have at least (COPY_CHUNK_SIZE) avail in (dest) before we will overwrite (src): + (cycSize >= offset + COPY_CHUNK_SIZE) + if defined(Z7_STD_DEC_USE_AFTER_CYC_BUF) + we are allowed to read win[cycSize + COPY_CHUNK_SIZE - 1], +} +(len != 0) +*/ +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyMatch(size_t offset, size_t len, + Byte *win, size_t winPos, size_t rem, const size_t cycSize) +{ + Byte *dest = win + winPos; + const Byte *src; + // STAT_INC(g_NumCopy) + + if (offset > winPos) + { + size_t back = offset - winPos; + // src = win + cycSize - back; + // cycSize -= offset; + STAT_INC(g_NumOver) + src = dest + (cycSize - offset); + // (src >= dest) here + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + if (back < len) + { + #else + if (back < len + (COPY_CHUNK_SIZE - 1)) + { + if (back >= len) + { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + *dest++ = *src++; + while (--len); + return; + } + #endif + // back < len + STAT_INC(g_NumOver2) + len -= back; + rem -= back; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + *dest++ = *src++; + while (--back); + src = dest - offset; + // src = win; + // we go to MAIN-COPY + } + } + else + src = dest - offset; + + // len != 0 + // do *dest++ = *src++; while (--len); return; + + // --- MAIN COPY --- + // if (src >= dest), then ((size_t)(src - dest) >= COPY_CHUNK_SIZE) + // so we have at least COPY_CHUNK_SIZE space before overlap for writing. + COPY_PREPARE + + /* now (len == COPY_CHUNK_SIZE * x) + so we can unroll for aligned copy */ + { + // const unsigned b0 = src[0]; + // (COPY_OFFSET_MIN >= 4) + + if (offset >= COPY_OFFSET_MIN) + { + COPY_CHUNKS + // return; + } + else + #if (COPY_OFFSET_MIN > 4) + #if COPY_CHUNK_SIZE < 8 + #error Stop_Compiling_Bad_COPY_CHUNK_SIZE + #endif + if (offset >= 4) + { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + COPY_CHUNK_4_2(dest, src) + #if COPY_CHUNK_SIZE != 16 + if (len == 8) break; + #endif + COPY_CHUNK_4_2(dest, src) + } + while (len -= 16); + // return; + } + else + #endif + { + // (offset < 4) + const unsigned b0 = src[0]; + if (offset < 2) + { + #if defined(Z7_ZSTD_DEC_USE_UNALIGNED_COPY) && (COPY_CHUNK_SIZE == 16) + #if defined(MY_CPU_64BIT) + { + const UInt64 v64 = (UInt64)b0 * 0x0101010101010101; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + ((UInt64 *)(void *)dest)[0] = v64; + ((UInt64 *)(void *)dest)[1] = v64; + dest += 16; + } + while (len -= 16); + } + #else + { + UInt32 v = b0; + v |= v << 8; + v |= v << 16; + do + { + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + } + while (len -= 16); + } + #endif + #else + do + { + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + } + while (len -= 4); + #endif + } + else if (offset == 2) + { + const Byte b1 = src[1]; + { + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest += 2; + } + while (len -= 2); + } + } + else // (offset == 3) + { + const Byte *lim = dest + len - 2; + const Byte b1 = src[1]; + const Byte b2 = src[2]; + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest[2] = b2; + dest += 3; + } + while (dest < lim); + lim++; // points to last byte that must be written + if (dest <= lim) + { + *dest = (Byte)b0; + if (dest != lim) + dest[1] = b1; + } + } + } + } +} + + + +#define UPDATE_TOTAL_OUT(p, size) \ +{ \ + size_t _toc = (p)->totalOutCheck + (size); \ + const size_t _ws = (p)->winSize; \ + if (_toc >= _ws) _toc = _ws; \ + (p)->totalOutCheck = _toc; \ +} + + +#if defined(MY_CPU_64BIT) && defined(MY_CPU_LE_UNALIGN) +// we can disable it for debug: +#define Z7_ZSTD_DEC_USE_64BIT_LOADS +#endif +// #define Z7_ZSTD_DEC_USE_64BIT_LOADS // for debug : slow in 32-bit + +// SEQ_SRC_OFFSET: how many bytes (src) (seqSrc) was moved back from original value. +// we need (SEQ_SRC_OFFSET != 0) for optimized memory access +#ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define SEQ_SRC_OFFSET 7 +#else + #define SEQ_SRC_OFFSET 3 +#endif +#define SRC_PLUS_FOR_4BYTES(bitOffset) (SEQ_SRC_OFFSET - 3) + ((CBitCtr_signed)(bitOffset) >> 3) +#define BIT_OFFSET_7BITS(bitOffset) ((unsigned)(bitOffset) & 7) +/* + if (BIT_OFFSET_DELTA_BITS == 0) : bitOffset == number_of_unprocessed_bits + if (BIT_OFFSET_DELTA_BITS == 1) : bitOffset == number_of_unprocessed_bits - 1 + and we can read 1 bit more in that mode : (8 * n + 1). +*/ +// #define BIT_OFFSET_DELTA_BITS 0 +#define BIT_OFFSET_DELTA_BITS 1 +#if BIT_OFFSET_DELTA_BITS == 1 + #define GET_SHIFT_FROM_BOFFS7(boff7) (7 ^ (boff7)) +#else + #define GET_SHIFT_FROM_BOFFS7(boff7) (8 - BIT_OFFSET_DELTA_BITS - (boff7)) +#endif + +#define UPDATE_BIT_OFFSET(bitOffset, numBits) \ + (bitOffset) -= (CBitCtr)(numBits); + +#define GET_SHIFT(bitOffset) GET_SHIFT_FROM_BOFFS7(BIT_OFFSET_7BITS(bitOffset)) + + +#if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) + #if (NUM_OFFSET_SYMBOLS_MAX - BIT_OFFSET_DELTA_BITS < 32) + /* if (NUM_OFFSET_SYMBOLS_MAX == 32 && BIT_OFFSET_DELTA_BITS == 1), + we have depth 31 + 9 + 9 + 8 = 57 bits that can b read with single read. */ + #define Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + #endif + #ifndef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + #if (BIT_OFFSET_DELTA_BITS == 1) + /* if (winLimit - winPos <= (kBlockSizeMax = (1 << 17))) + { + the case (16 bits literal extra + 16 match extra) is not possible + in correct stream. So error will be detected for (16 + 16) case. + And longest correct sequence after offset reading is (31 + 9 + 9 + 8 = 57 bits). + So we can use just one 64-bit load here in that case. + } + */ + #define Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML + #endif + #endif +#endif + + +#if !defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) || \ + (!defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML)) +// in : (0 < bits <= (24 or 25)): +#define STREAM_READ_BITS(dest, bits) \ +{ \ + GET32(dest, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + dest <<= GET_SHIFT(bitOffset); \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + dest >>= 32 - bits; \ +} +#endif + + +#define FSE_Peek_1(table, state) table[state] + +#define STATE_VAR(name) state_ ## name + +// in : (0 <= accuracy <= (24 or 25)) +#define FSE_INIT_STATE(name, cond) \ +{ \ + UInt32 r; \ + const unsigned bits = p->name ## _accuracy; \ + GET32(r, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + r <<= GET_SHIFT(bitOffset); \ + r >>= 1; \ + r >>= 31 ^ bits; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + STATE_VAR(name) = FSE_Peek_1(FSE_TABLE(name), r); \ + /* STATE_VAR(name) = dest << 16; */ \ +} + + +#define FSE_Peek_Plus(name, r) \ + STATE_VAR(name) = FSE_Peek_1(FSE_TABLE(name), \ + GET_FSE_REC_STATE(STATE_VAR(name)) + r); + +#define LZ_LOOP_ERROR_EXIT { return SZ_ERROR_DATA; } + +#define BO_OVERFLOW_CHECK \ + { if ((CBitCtr_signed)bitOffset < 0) LZ_LOOP_ERROR_EXIT } + + +#ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + +#define GET64(dest, p) { const Byte *ptr = p; dest = GetUi64(ptr); } + +#define FSE_PRELOAD \ +{ \ + GET64(v, src - 4 + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + v <<= GET_SHIFT(bitOffset); \ +} + +#define FSE_UPDATE_STATE_2(name, cond) \ +{ \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + UInt64 r = v; \ + v <<= bits; \ + r >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + r >>= 63 ^ bits; \ + FSE_Peek_Plus(name, r); \ +} + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_2 (ll, {} ) \ + FSE_UPDATE_STATE_2 (ml, {} ) \ + FSE_UPDATE_STATE_2 (of, BO_OVERFLOW_CHECK) \ + +#else // Z7_ZSTD_DEC_USE_64BIT_LOADS + +// it supports 8 bits accuracy for any code +// it supports 9 bits accuracy, if (BIT_OFFSET_DELTA_BITS == 1) +#define FSE_UPDATE_STATE_0(name, cond) \ +{ \ + UInt32 r; \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + GET16(r, src + 2 + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + r >>= (bitOffset & 7); \ + r &= (1 << (8 + BIT_OFFSET_DELTA_BITS)) - 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + r >>= (8 + BIT_OFFSET_DELTA_BITS) - bits; \ + FSE_Peek_Plus(name, r); \ +} + +// for debug (slow): +// #define Z7_ZSTD_DEC_USE_FSE_FUSION_FORCE +#if BIT_OFFSET_DELTA_BITS == 0 || defined(Z7_ZSTD_DEC_USE_FSE_FUSION_FORCE) + #define Z7_ZSTD_DEC_USE_FSE_FUSION +#endif + +#ifdef Z7_ZSTD_DEC_USE_FSE_FUSION +#define FSE_UPDATE_STATE_1(name) \ +{ UInt32 rest2; \ +{ \ + UInt32 r; \ + unsigned bits; \ + GET32(r, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + r <<= GET_SHIFT(bitOffset); \ + rest2 = r << bits; \ + r >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + r >>= 31 ^ bits; \ + FSE_Peek_Plus(name, r); \ +} + +#define FSE_UPDATE_STATE_3(name) \ +{ \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + rest2 >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + rest2 >>= 31 ^ bits; \ + FSE_Peek_Plus(name, rest2); \ +}} + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_1 (ll) \ + FSE_UPDATE_STATE_3 (ml) \ + FSE_UPDATE_STATE_0 (of, BO_OVERFLOW_CHECK) \ + +#else // Z7_ZSTD_DEC_USE_64BIT_LOADS + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_0 (ll, {} ) \ + FSE_UPDATE_STATE_0 (ml, {} ) \ + FSE_UPDATE_STATE_0 (of, BO_OVERFLOW_CHECK) \ + +#endif // Z7_ZSTD_DEC_USE_FSE_FUSION +#endif // Z7_ZSTD_DEC_USE_64BIT_LOADS + + + +typedef struct +{ + UInt32 numSeqs; + UInt32 literalsLen; + const Byte *literals; +} +CZstdDec1_Vars; + + +// if (BIT_OFFSET_DELTA_BITS != 0), we need (BIT_OFFSET_DELTA_BYTES > 0) +#define BIT_OFFSET_DELTA_BYTES BIT_OFFSET_DELTA_BITS + +/* if (NUM_OFFSET_SYMBOLS_MAX == 32) + max_seq_bit_length = (31) + 16 + 16 + 9 + 8 + 9 = 89 bits + if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) we have longest backward + lookahead offset, and we read UInt64 after literal_len reading. + if (BIT_OFFSET_DELTA_BITS == 1 && NUM_OFFSET_SYMBOLS_MAX == 32) + MAX_BACKWARD_DEPTH = 16 bytes +*/ +#define MAX_BACKWARD_DEPTH \ + ((NUM_OFFSET_SYMBOLS_MAX - 1 + 16 + 16 + 7) / 8 + 7 + BIT_OFFSET_DELTA_BYTES) + +/* srcLen != 0 + src == real_data_ptr - SEQ_SRC_OFFSET - BIT_OFFSET_DELTA_BYTES + if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) then + (winLimit - p->winPos <= (1 << 17)) is required +*/ +static +Z7_NO_INLINE +// Z7_ATTRIB_NO_VECTOR +SRes Decompress_Sequences(CZstdDec1 * const p, + const Byte *src, const size_t srcLen, + const size_t winLimit, + const CZstdDec1_Vars * const vars) +{ +#ifdef Z7_ZSTD_DEC_USE_BASES_LOCAL + SEQ_EXTRA_TABLES(a_) +#endif + + // for debug: + // #define Z7_ZSTD_DEC_USE_LOCAL_FSE_TABLES +#ifdef Z7_ZSTD_DEC_USE_LOCAL_FSE_TABLES + #define FSE_TABLE(n) fse. n + const CZstdDecFseTables fse = p->fse; + /* + CZstdDecFseTables fse; + #define COPY_FSE_TABLE(n) \ + memcpy(fse. n, p->fse. n, (size_t)4 << p-> n ## _accuracy); + COPY_FSE_TABLE(of) + COPY_FSE_TABLE(ll) + COPY_FSE_TABLE(ml) + */ +#else + #define FSE_TABLE(n) (p->fse. n) +#endif + +#ifdef Z7_ZSTD_DEC_USE_BASES_LOCAL + FILL_LOC_BASES_ALL +#endif + + { + unsigned numSeqs = vars->numSeqs; + const Byte *literals = vars->literals; + ptrdiff_t literalsLen = (ptrdiff_t)vars->literalsLen; + Byte * const win = p->win; + size_t winPos = p->winPos; + const size_t cycSize = p->cycSize; + size_t totalOutCheck = p->totalOutCheck; + const size_t winSize = p->winSize; + size_t reps_0 = p->reps[0]; + size_t reps_1 = p->reps[1]; + size_t reps_2 = p->reps[2]; + UInt32 STATE_VAR(ll), STATE_VAR(of), STATE_VAR(ml); + CBitCtr bitOffset; + + SET_bitOffset_TO_PAD (bitOffset, src + SEQ_SRC_OFFSET, srcLen + BIT_OFFSET_DELTA_BYTES) + + bitOffset -= BIT_OFFSET_DELTA_BITS; + + FSE_INIT_STATE(ll, {} ) + FSE_INIT_STATE(of, {} ) + FSE_INIT_STATE(ml, BO_OVERFLOW_CHECK) + + for (;;) + { + size_t matchLen; + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + UInt64 v; + #endif + + #ifdef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + FSE_PRELOAD + #endif + + // if (of_code == 0) + if ((Byte)STATE_VAR(of) == 0) + { + if (GET_FSE_REC_SYM(STATE_VAR(ll)) == 0) + { + const size_t offset = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep1) + } + STAT_UPDATE(else g_Num_Rep0++;) + } + else + { + const unsigned of_code = (Byte)STATE_VAR(of); + + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #if !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #endif + #else + UInt32 v; + { + const Byte *src4 = src + SRC_PLUS_FOR_4BYTES(bitOffset); + const unsigned skip = GET_SHIFT(bitOffset); + GET32(v, src4) + v <<= skip; + v |= (UInt32)src4[-1] >> (8 - skip); + } + #endif + + UPDATE_BIT_OFFSET(bitOffset, of_code) + + if (of_code == 1) + { + // read 1 bit + #if defined(Z7_MSC_VER_ORIGINAL) || defined(MY_CPU_X86_OR_AMD64) + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define CHECK_HIGH_BIT_64(a) ((Int64)(UInt64)(a) < 0) + #else + #define CHECK_HIGH_BIT_32(a) ((Int32)(UInt32)(a) < 0) + #endif + #else + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define CHECK_HIGH_BIT_64(a) ((UInt64)(a) & ((UInt64)1 << 63)) + #else + #define CHECK_HIGH_BIT_32(a) ((UInt32)(a) & ((UInt32)1 << 31)) + #endif + #endif + + if + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + CHECK_HIGH_BIT_64 (((UInt64)GET_FSE_REC_SYM(STATE_VAR(ll)) - 1) ^ v) + #else + CHECK_HIGH_BIT_32 (((UInt32)GET_FSE_REC_SYM(STATE_VAR(ll)) - 1) ^ v) + #endif + { + v <<= 1; + { + const size_t offset = reps_2; + reps_2 = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep2) + } + } + else + { + if (GET_FSE_REC_SYM(STATE_VAR(ll)) == 0) + { + // litLen == 0 && bit == 1 + STAT_INC(g_Num_Rep3) + v <<= 1; + reps_2 = reps_1; + reps_1 = reps_0; + if (--reps_0 == 0) + { + // LZ_LOOP_ERROR_EXIT + // original-zstd decoder : input is corrupted; force offset to 1 + // reps_0 = 1; + reps_0++; + } + } + else + { + // litLen != 0 && bit == 0 + v <<= 1; + { + const size_t offset = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep1) + } + } + } + } + else + { + // (2 <= of_code) + // if (of_code >= 32) LZ_LOOP_ERROR_EXIT // optional check + // we don't allow (of_code >= 32) cases in another code + reps_2 = reps_1; + reps_1 = reps_0; + reps_0 = ((size_t)1 << of_code) - 3 + (size_t) + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + (v >> (64 - of_code)); + v <<= of_code; + #else + (v >> (32 - of_code)); + #endif + } + } + + #ifdef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML + FSE_PRELOAD + #endif + + matchLen = (size_t)GET_FSE_REC_SYM(STATE_VAR(ml)) + #ifndef Z7_ZSTD_DEC_USE_ML_PLUS3 + + MATCH_LEN_MIN + #endif + ; + { + { + if (matchLen >= 32 + MATCH_LEN_MIN) // if (state_ml & 0x20) + { + const unsigned extra = BASES_TABLE(SEQ_ML_EXTRA) [(size_t)matchLen - MATCH_LEN_MIN]; + matchLen = BASES_TABLE(SEQ_ML_BASES) [(size_t)matchLen - MATCH_LEN_MIN]; + #if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) && \ + (defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) || \ + defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF)) + { + UPDATE_BIT_OFFSET(bitOffset, extra) + matchLen += (size_t)(v >> (64 - extra)); + #if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #else + v <<= extra; + #endif + } + #else + { + UInt32 v32; + STREAM_READ_BITS(v32, extra) + matchLen += v32; + } + #endif + STAT_INC(g_Num_Match) + } + } + } + + #if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) + FSE_PRELOAD + #endif + + { + size_t litLen = GET_FSE_REC_SYM(STATE_VAR(ll)); + if (litLen) + { + // if (STATE_VAR(ll) & 0x70) + if (litLen >= 16) + { + const unsigned extra = BASES_TABLE(SEQ_LL_EXTRA) [litLen]; + litLen = BASES_TABLE(SEQ_LL_BASES) [litLen]; + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + { + UPDATE_BIT_OFFSET(bitOffset, extra) + litLen += (size_t)(v >> (64 - extra)); + #if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #else + v <<= extra; + #endif + } + #else + { + UInt32 v32; + STREAM_READ_BITS(v32, extra) + litLen += v32; + } + #endif + STAT_INC(g_Num_LitsBig) + } + + if ((literalsLen -= (ptrdiff_t)litLen) < 0) + LZ_LOOP_ERROR_EXIT + totalOutCheck += litLen; + { + const size_t rem = winLimit - winPos; + if (litLen > rem) + LZ_LOOP_ERROR_EXIT + { + const Byte *literals_temp = literals; + Byte *d = win + winPos; + literals += litLen; + winPos += litLen; + CopyLiterals(d, literals_temp, litLen, rem); + } + } + } + STAT_UPDATE(else g_Num_Lit0++;) + } + + #define COPY_MATCH \ + { if (reps_0 > winSize || reps_0 > totalOutCheck) LZ_LOOP_ERROR_EXIT \ + totalOutCheck += matchLen; \ + { const size_t rem = winLimit - winPos; \ + if (matchLen > rem) LZ_LOOP_ERROR_EXIT \ + { const size_t winPos_temp = winPos; \ + winPos += matchLen; \ + CopyMatch(reps_0, matchLen, win, winPos_temp, rem, cycSize); }}} + + if (--numSeqs == 0) + { + COPY_MATCH + break; + } + FSE_UPDATE_STATES + COPY_MATCH + } // for + + if ((CBitCtr_signed)bitOffset != BIT_OFFSET_DELTA_BYTES * 8 - BIT_OFFSET_DELTA_BITS) + return SZ_ERROR_DATA; + + if (literalsLen) + { + const size_t rem = winLimit - winPos; + if ((size_t)literalsLen > rem) + return SZ_ERROR_DATA; + { + Byte *d = win + winPos; + winPos += (size_t)literalsLen; + totalOutCheck += (size_t)literalsLen; + CopyLiterals + // memcpy + (d, literals, (size_t)literalsLen, rem); + } + } + if (totalOutCheck >= winSize) + totalOutCheck = winSize; + p->totalOutCheck = totalOutCheck; + p->winPos = winPos; + p->reps[0] = (CZstdDecOffset)reps_0; + p->reps[1] = (CZstdDecOffset)reps_1; + p->reps[2] = (CZstdDecOffset)reps_2; + } + return SZ_OK; +} + + +// for debug: define to check that ZstdDec1_NeedTempBufferForInput() works correctly: +// #define Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP // define it for debug only +#ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP +static unsigned g_numSeqs; +#endif + + +#define k_LitBlockType_Flag_RLE_or_Treeless 1 +#define k_LitBlockType_Flag_Compressed 2 + +// outLimit : is strong limit +// outLimit <= ZstdDec1_GET_BLOCK_SIZE_LIMIT(p) +// inSize != 0 +static +Z7_NO_INLINE +SRes ZstdDec1_DecodeBlock(CZstdDec1 *p, + const Byte *src, SizeT inSize, SizeT afterAvail, + const size_t outLimit) +{ + CZstdDec1_Vars vars; + vars.literals = p->literalsBase; + { + const unsigned b0 = *src++; + UInt32 numLits, compressedSize; + const Byte *litStream; + Byte *literalsDest; + inSize--; + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + // we need at least one additional byte for (numSeqs). + // so we check for that additional byte in conditions. + numLits = b0 >> 3; + if (b0 & 4) + { + UInt32 v; + if (inSize < 1 + 1) // we need at least 1 byte here and 1 byte for (numSeqs). + return SZ_ERROR_DATA; + numLits >>= 1; + v = GetUi16(src); + src += 2; + inSize -= 2; + if ((b0 & 8) == 0) + { + src--; + inSize++; + v = (Byte)v; + } + numLits += v << 4; + } + compressedSize = 1; + if ((b0 & k_LitBlockType_Flag_RLE_or_Treeless) == 0) + compressedSize = numLits; + } + else if (inSize < 4) + return SZ_ERROR_DATA; + else + { + const unsigned mode4Streams = b0 & 0xc; + const unsigned numBytes = (3 * mode4Streams + 32) >> 4; + const unsigned numBits = 4 * numBytes - 2; + const UInt32 mask = ((UInt32)16 << numBits) - 1; + compressedSize = GetUi32(src); + numLits = (( + #ifdef MY_CPU_LE_UNALIGN + GetUi32(src - 1) + #else + ((compressedSize << 8) + b0) + #endif + ) >> 4) & mask; + src += numBytes; + inSize -= numBytes; + compressedSize >>= numBits; + compressedSize &= mask; + /* + if (numLits != 0) printf("inSize = %7u num_lits=%7u compressed=%7u ratio = %u ratio2 = %u\n", + i1, numLits, (unsigned)compressedSize * 1, (unsigned)compressedSize * 100 / numLits, + (unsigned)numLits * 100 / (unsigned)inSize); + } + */ + if (compressedSize == 0) + return SZ_ERROR_DATA; // (compressedSize == 0) is not allowed + } + + STAT_UPDATE(g_Num_Lits += numLits;) + + vars.literalsLen = numLits; + + if (compressedSize >= inSize) + return SZ_ERROR_DATA; + litStream = src; + src += compressedSize; + inSize -= compressedSize; + // inSize != 0 + { + UInt32 numSeqs = *src++; + inSize--; + if (numSeqs > 127) + { + UInt32 b1; + if (inSize == 0) + return SZ_ERROR_DATA; + numSeqs -= 128; + b1 = *src++; + inSize--; + if (numSeqs == 127) + { + if (inSize == 0) + return SZ_ERROR_DATA; + numSeqs = (UInt32)(*src++) + 127; + inSize--; + } + numSeqs = (numSeqs << 8) + b1; + } + if (numSeqs * MATCH_LEN_MIN + numLits > outLimit) + return SZ_ERROR_DATA; + vars.numSeqs = numSeqs; + + STAT_UPDATE(g_NumSeqs_total += numSeqs;) + /* + #ifdef SHOW_STAT + printf("\n %5u : %8u, %8u : %5u", (int)g_Num_Blocks_Compressed, (int)numSeqs, (int)g_NumSeqs_total, + (int)g_NumSeqs_total / g_Num_Blocks_Compressed); + #endif + // printf("\nnumSeqs2 = %d", numSeqs); + */ + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + if (numSeqs != g_numSeqs) return SZ_ERROR_DATA; // for debug + #endif + if (numSeqs == 0) + { + if (inSize != 0) + return SZ_ERROR_DATA; + literalsDest = p->win + p->winPos; + } + else + literalsDest = p->literalsBase; + } + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + if (b0 & k_LitBlockType_Flag_RLE_or_Treeless) + { + memset(literalsDest, litStream[0], numLits); + if (vars.numSeqs) + { + // literalsDest == p->literalsBase == vars.literals + #if COPY_CHUNK_SIZE > 1 + memset(p->literalsBase + numLits, 0, COPY_CHUNK_SIZE); + #endif + } + } + else + { + // unsigned y; + // for (y = 0; y < 10000; y++) + memcpy(literalsDest, litStream, numLits); + if (vars.numSeqs) + { + /* we need up to (15 == COPY_CHUNK_SIZE - 1) space for optimized CopyLiterals(). + If we have additional space in input stream after literals stream, + we use direct copy of rar literals in input stream */ + if ((size_t)(src + inSize - litStream) - numLits + afterAvail >= (COPY_CHUNK_SIZE - 1)) + vars.literals = litStream; + else + { + // literalsDest == p->literalsBase == vars.literals + #if COPY_CHUNK_SIZE > 1 + /* CopyLiterals(): + 1) we don't want reading non-initialized data + 2) we will copy only zero byte after literals buffer */ + memset(p->literalsBase + numLits, 0, COPY_CHUNK_SIZE); + #endif + } + } + } + } + else + { + CInBufPair hufStream; + hufStream.ptr = litStream; + hufStream.len = compressedSize; + + if ((b0 & k_LitBlockType_Flag_RLE_or_Treeless) == 0) + { + // unsigned y = 100; CInBufPair hs2 = hufStream; do { hufStream = hs2; + RINOK(Huf_DecodeTable(&p->huf, &hufStream)) + p->litHuf_wasSet = True; + // } while (--y); + } + else if (!p->litHuf_wasSet) + return SZ_ERROR_DATA; + + { + // int yyy; for (yyy = 0; yyy < 34; yyy++) { + SRes sres; + if ((b0 & 0xc) == 0) // mode4Streams + sres = Huf_Decompress_1stream((const Byte *)(const void *)p->huf.table64, + hufStream.ptr - HUF_SRC_OFFSET, hufStream.len, literalsDest, numLits); + else + { + // 6 bytes for the jump table + 4x1 bytes of end-padding Bytes) + if (hufStream.len < 6 + 4) + return SZ_ERROR_DATA; + // the condition from original-zstd decoder: + #define Z7_ZSTD_MIN_LITERALS_FOR_4_STREAMS 6 + if (numLits < Z7_ZSTD_MIN_LITERALS_FOR_4_STREAMS) + return SZ_ERROR_DATA; + sres = Huf_Decompress_4stream((const Byte *)(const void *)p->huf.table64, + hufStream.ptr + (6 - HUF_SRC_OFFSET), hufStream.len, literalsDest, numLits); + } + RINOK(sres) + // } + } + } + + if (vars.numSeqs == 0) + { + p->winPos += numLits; + UPDATE_TOTAL_OUT(p, numLits) + return SZ_OK; + } + } + { + CInBufPair in; + unsigned mode; + unsigned seqMode; + + in.ptr = src; + in.len = inSize; + if (in.len == 0) + return SZ_ERROR_DATA; + in.len--; + mode = *in.ptr++; + if (mode & 3) // Reserved bits + return SZ_ERROR_DATA; + + seqMode = (mode >> 6); + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else RINOK(FSE_Decode_SeqTable( + p->fse.ll, + &in, + 6, // predefAccuracy + &p->ll_accuracy, + NUM_LL_SYMBOLS, + k_PredefRecords_LL, + seqMode)) + + seqMode = (mode >> 4) & 3; + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else RINOK(FSE_Decode_SeqTable( + p->fse.of, + &in, + 5, // predefAccuracy + &p->of_accuracy, + NUM_OFFSET_SYMBOLS_MAX, + k_PredefRecords_OF, + seqMode)) + + seqMode = (mode >> 2) & 3; + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else + { + RINOK(FSE_Decode_SeqTable( + p->fse.ml, + &in, + 6, // predefAccuracy + &p->ml_accuracy, + NUM_ML_SYMBOLS, + k_PredefRecords_ML, + seqMode)) + /* + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + // { unsigned y = 1 << 10; do + { + const unsigned accuracy = p->ml_accuracy; + if (accuracy == 0) + p->fse.ml[0] += 3; + else + #ifdef MY_CPU_64BIT + { + // alignemt (UInt64 _pad_Alignment) in fse.ml is required for that code + UInt64 *table = (UInt64 *)(void *)p->fse.ml; + const UInt64 *end = (const UInt64 *)(const void *) + ((const Byte *)(const void *)table + ((size_t)sizeof(CFseRecord) << accuracy)); + do + { + table[0] += ((UInt64)MATCH_LEN_MIN << 32) + MATCH_LEN_MIN; + table[1] += ((UInt64)MATCH_LEN_MIN << 32) + MATCH_LEN_MIN; + table += 2; + } + while (table != end); + } + #else + { + UInt32 *table = p->fse.ml; + const UInt32 *end = (const UInt32 *)(const void *) + ((const Byte *)(const void *)table + ((size_t)sizeof(CFseRecord) << accuracy)); + do + { + table[0] += MATCH_LEN_MIN; + table[1] += MATCH_LEN_MIN; + table += 2; + table[0] += MATCH_LEN_MIN; + table[1] += MATCH_LEN_MIN; + table += 2; + } + while (table != end); + } + #endif + } + // while (--y); } + #endif + */ + } + + // p->seqTables_wereSet = True; + if (in.len == 0) + return SZ_ERROR_DATA; + return Decompress_Sequences(p, + in.ptr - SEQ_SRC_OFFSET - BIT_OFFSET_DELTA_BYTES, in.len, + p->winPos + outLimit, &vars); + } +} + + + + +// inSize != 0 +// it must do similar to ZstdDec1_DecodeBlock() +static size_t ZstdDec1_NeedTempBufferForInput( + const SizeT beforeSize, const Byte * const src, const SizeT inSize) +{ + unsigned b0; + UInt32 pos; + + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + g_numSeqs = 1 << 24; + #else + // we have at least 3 bytes before seq data: litBlockType, numSeqs, seqMode + #define MIN_BLOCK_LZ_HEADERS_SIZE 3 + if (beforeSize >= MAX_BACKWARD_DEPTH - MIN_BLOCK_LZ_HEADERS_SIZE) + return 0; + #endif + + b0 = src[0]; + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + UInt32 numLits = b0 >> 3; + pos = 1; + if (b0 & 4) + { + UInt32 v; + if (inSize < 3) + return 0; + numLits >>= 1; + v = GetUi16(src + 1); + pos = 3; + if ((b0 & 8) == 0) + { + pos = 2; + v = (Byte)v; + } + numLits += v << 4; + } + if (b0 & k_LitBlockType_Flag_RLE_or_Treeless) + numLits = 1; + pos += numLits; + } + else if (inSize < 5) + return 0; + else + { + const unsigned mode4Streams = b0 & 0xc; + const unsigned numBytes = (3 * mode4Streams + 48) >> 4; + const unsigned numBits = 4 * numBytes - 6; + UInt32 cs = GetUi32(src + 1); + cs >>= numBits; + cs &= ((UInt32)16 << numBits) - 1; + if (cs == 0) + return 0; + pos = numBytes + cs; + } + + if (pos >= inSize) + return 0; + { + UInt32 numSeqs = src[pos++]; + if (numSeqs > 127) + { + UInt32 b1; + if (pos >= inSize) + return 0; + numSeqs -= 128; + b1 = src[pos++]; + if (numSeqs == 127) + { + if (pos >= inSize) + return 0; + numSeqs = (UInt32)(src[pos++]) + 127; + } + numSeqs = (numSeqs << 8) + b1; + } + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + g_numSeqs = numSeqs; // for debug + #endif + if (numSeqs == 0) + return 0; + } + /* + if (pos >= inSize) + return 0; + pos++; + */ + // we will have one additional byte for seqMode: + if (beforeSize + pos >= MAX_BACKWARD_DEPTH - 1) + return 0; + return 1; +} + + + +// ---------- ZSTD FRAME ---------- + +#define kBlockType_Raw 0 +#define kBlockType_RLE 1 +#define kBlockType_Compressed 2 +#define kBlockType_Reserved 3 + +typedef enum +{ + // begin: states that require 4 bytes: + ZSTD2_STATE_SIGNATURE, + ZSTD2_STATE_HASH, + ZSTD2_STATE_SKIP_HEADER, + // end of states that require 4 bytes + + ZSTD2_STATE_SKIP_DATA, + ZSTD2_STATE_FRAME_HEADER, + ZSTD2_STATE_AFTER_HEADER, + ZSTD2_STATE_BLOCK, + ZSTD2_STATE_DATA, + ZSTD2_STATE_FINISHED +} EZstd2State; + + +struct CZstdDec +{ + EZstd2State frameState; + unsigned tempSize; + + Byte temp[14]; // 14 is required + + Byte descriptor; + Byte windowDescriptor; + Byte isLastBlock; + Byte blockType; + Byte isErrorState; + Byte hashError; + Byte disableHash; + Byte isCyclicMode; + + UInt32 blockSize; + UInt32 dictionaryId; + UInt32 curBlockUnpackRem; // for compressed blocks only + UInt32 inTempPos; + + UInt64 contentSize; + UInt64 contentProcessed; + CXxh64State xxh64; + + Byte *inTemp; + SizeT winBufSize_Allocated; + Byte *win_Base; + + ISzAllocPtr alloc_Small; + ISzAllocPtr alloc_Big; + + CZstdDec1 decoder; +}; + +#define ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p) \ + ((unsigned)(p)->contentProcessed & (Z7_XXH64_BLOCK_SIZE - 1)) + +#define ZSTD_DEC_IS_LAST_BLOCK(p) ((p)->isLastBlock) + + +static void ZstdDec_FreeWindow(CZstdDec * const p) +{ + if (p->win_Base) + { + ISzAlloc_Free(p->alloc_Big, p->win_Base); + p->win_Base = NULL; + // p->decoder.win = NULL; + p->winBufSize_Allocated = 0; + } +} + + +CZstdDecHandle ZstdDec_Create(ISzAllocPtr alloc_Small, ISzAllocPtr alloc_Big) +{ + CZstdDec *p = (CZstdDec *)ISzAlloc_Alloc(alloc_Small, sizeof(CZstdDec)); + if (!p) + return NULL; + p->alloc_Small = alloc_Small; + p->alloc_Big = alloc_Big; + // ZstdDec_CONSTRUCT(p) + p->inTemp = NULL; + p->win_Base = NULL; + p->winBufSize_Allocated = 0; + p->disableHash = False; + ZstdDec1_Construct(&p->decoder); + return p; +} + +void ZstdDec_Destroy(CZstdDecHandle p) +{ + #ifdef SHOW_STAT + #define PRINT_STAT1(name, v) \ + printf("\n%25s = %9u", name, v); + PRINT_STAT1("g_Num_Blocks_Compressed", g_Num_Blocks_Compressed) + PRINT_STAT1("g_Num_Blocks_memcpy", g_Num_Blocks_memcpy) + PRINT_STAT1("g_Num_Wrap_memmove_Num", g_Num_Wrap_memmove_Num) + PRINT_STAT1("g_Num_Wrap_memmove_Bytes", g_Num_Wrap_memmove_Bytes) + if (g_Num_Blocks_Compressed) + { + #define PRINT_STAT(name, v) \ + printf("\n%17s = %9u, per_block = %8u", name, v, v / g_Num_Blocks_Compressed); + PRINT_STAT("g_NumSeqs", g_NumSeqs_total) + // PRINT_STAT("g_NumCopy", g_NumCopy) + PRINT_STAT("g_NumOver", g_NumOver) + PRINT_STAT("g_NumOver2", g_NumOver2) + PRINT_STAT("g_Num_Match", g_Num_Match) + PRINT_STAT("g_Num_Lits", g_Num_Lits) + PRINT_STAT("g_Num_LitsBig", g_Num_LitsBig) + PRINT_STAT("g_Num_Lit0", g_Num_Lit0) + PRINT_STAT("g_Num_Rep_0", g_Num_Rep0) + PRINT_STAT("g_Num_Rep_1", g_Num_Rep1) + PRINT_STAT("g_Num_Rep_2", g_Num_Rep2) + PRINT_STAT("g_Num_Rep_3", g_Num_Rep3) + PRINT_STAT("g_Num_Threshold_0", g_Num_Threshold_0) + PRINT_STAT("g_Num_Threshold_1", g_Num_Threshold_1) + PRINT_STAT("g_Num_Threshold_0sum", g_Num_Threshold_0sum) + PRINT_STAT("g_Num_Threshold_1sum", g_Num_Threshold_1sum) + } + printf("\n"); + #endif + + ISzAlloc_Free(p->alloc_Small, p->decoder.literalsBase); + // p->->decoder.literalsBase = NULL; + ISzAlloc_Free(p->alloc_Small, p->inTemp); + // p->inTemp = NULL; + ZstdDec_FreeWindow(p); + ISzAlloc_Free(p->alloc_Small, p); +} + + + +#define kTempBuffer_PreSize (1u << 6) +#if kTempBuffer_PreSize < MAX_BACKWARD_DEPTH + #error Stop_Compiling_Bad_kTempBuffer_PreSize +#endif + +static SRes ZstdDec_AllocateMisc(CZstdDec *p) +{ + #define k_Lit_AfterAvail (1u << 6) + #if k_Lit_AfterAvail < (COPY_CHUNK_SIZE - 1) + #error Stop_Compiling_Bad_k_Lit_AfterAvail + #endif + // return ZstdDec1_Allocate(&p->decoder, p->alloc_Small); + if (!p->decoder.literalsBase) + { + p->decoder.literalsBase = (Byte *)ISzAlloc_Alloc(p->alloc_Small, + kBlockSizeMax + k_Lit_AfterAvail); + if (!p->decoder.literalsBase) + return SZ_ERROR_MEM; + } + if (!p->inTemp) + { + // we need k_Lit_AfterAvail here for owerread from raw literals stream + p->inTemp = (Byte *)ISzAlloc_Alloc(p->alloc_Small, + kBlockSizeMax + kTempBuffer_PreSize + k_Lit_AfterAvail); + if (!p->inTemp) + return SZ_ERROR_MEM; + } + return SZ_OK; +} + + +static void ZstdDec_Init_ForNewFrame(CZstdDec *p) +{ + p->frameState = ZSTD2_STATE_SIGNATURE; + p->tempSize = 0; + + p->isErrorState = False; + p->hashError = False; + p->isCyclicMode = False; + p->contentProcessed = 0; + Xxh64State_Init(&p->xxh64); + ZstdDec1_Init(&p->decoder); +} + + +void ZstdDec_Init(CZstdDec *p) +{ + ZstdDec_Init_ForNewFrame(p); + p->decoder.winPos = 0; + memset(p->temp, 0, sizeof(p->temp)); +} + + +#define DESCRIPTOR_Get_DictionaryId_Flag(d) ((d) & 3) +#define DESCRIPTOR_FLAG_CHECKSUM (1 << 2) +#define DESCRIPTOR_FLAG_RESERVED (1 << 3) +// #define DESCRIPTOR_FLAG_UNUSED (1 << 4) +#define DESCRIPTOR_FLAG_SINGLE (1 << 5) +#define DESCRIPTOR_Get_ContentSize_Flag3(d) ((d) >> 5) +#define DESCRIPTOR_Is_ContentSize_Defined(d) (((d) & 0xe0) != 0) + + +static EZstd2State ZstdDec_UpdateState(CZstdDec * const p, const Byte b, CZstdDecInfo * const info) +{ + unsigned tempSize = p->tempSize; + p->temp[tempSize++] = b; + p->tempSize = tempSize; + + if (p->frameState == ZSTD2_STATE_BLOCK) + { + if (tempSize < 3) + return ZSTD2_STATE_BLOCK; + { + UInt32 b0 = GetUi32(p->temp); + const unsigned type = ((unsigned)b0 >> 1) & 3; + if (type == kBlockType_RLE && tempSize == 3) + return ZSTD2_STATE_BLOCK; + // info->num_Blocks_forType[type]++; + info->num_Blocks++; + if (type == kBlockType_Reserved) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED + return ZSTD2_STATE_BLOCK; + } + p->blockType = (Byte)type; + p->isLastBlock = (Byte)(b0 & 1); + p->inTempPos = 0; + p->tempSize = 0; + b0 >>= 3; + b0 &= 0x1fffff; + // info->num_BlockBytes_forType[type] += b0; + if (b0 == 0) + { + // empty RAW/RLE blocks are allowed in original-zstd decoder + if (type == kBlockType_Compressed) + { + p->isErrorState = True; + return ZSTD2_STATE_BLOCK; + } + if (!ZSTD_DEC_IS_LAST_BLOCK(p)) + return ZSTD2_STATE_BLOCK; + if (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) + return ZSTD2_STATE_HASH; + return ZSTD2_STATE_FINISHED; + } + p->blockSize = b0; + { + UInt32 blockLim = ZstdDec1_GET_BLOCK_SIZE_LIMIT(&p->decoder); + // compressed and uncompressed block sizes cannot be larger than min(kBlockSizeMax, window_size) + if (b0 > blockLim) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED; + return ZSTD2_STATE_BLOCK; + } + if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor)) + { + const UInt64 rem = p->contentSize - p->contentProcessed; + if (blockLim > rem) + blockLim = (UInt32)rem; + } + p->curBlockUnpackRem = blockLim; + // uncompressed block size cannot be larger than remain data size: + if (type != kBlockType_Compressed) + { + if (b0 > blockLim) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED; + return ZSTD2_STATE_BLOCK; + } + } + } + } + return ZSTD2_STATE_DATA; + } + + if ((unsigned)p->frameState < ZSTD2_STATE_SKIP_DATA) + { + UInt32 v; + if (tempSize != 4) + return p->frameState; + v = GetUi32(p->temp); + if ((unsigned)p->frameState < ZSTD2_STATE_HASH) // == ZSTD2_STATE_SIGNATURE + { + if (v == 0xfd2fb528) + { + p->tempSize = 0; + info->num_DataFrames++; + return ZSTD2_STATE_FRAME_HEADER; + } + if ((v & 0xfffffff0) == 0x184d2a50) + { + p->tempSize = 0; + info->num_SkipFrames++; + return ZSTD2_STATE_SKIP_HEADER; + } + p->isErrorState = True; + return ZSTD2_STATE_SIGNATURE; + // return ZSTD2_STATE_ERROR; // is not ZSTD stream + } + if (p->frameState == ZSTD2_STATE_HASH) + { + info->checksum_Defined = True; + info->checksum = v; + // #ifndef DISABLE_XXH_CHECK + if (!p->disableHash) + { + if (p->decoder.winPos < ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p)) + { + // unexpected code failure + p->isErrorState = True; + // SZ_ERROR_FAIL; + } + else + if ((UInt32)Xxh64State_Digest(&p->xxh64, + p->decoder.win + (p->decoder.winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p)), + p->contentProcessed) != v) + { + p->hashError = True; + // return ZSTD2_STATE_ERROR; // hash error + } + } + // #endif + return ZSTD2_STATE_FINISHED; + } + // (p->frameState == ZSTD2_STATE_SKIP_HEADER) + { + p->blockSize = v; + info->skipFrames_Size += v; + p->tempSize = 0; + /* we want the caller could know that there was finished frame + finished frame. So we allow the case where + we have ZSTD2_STATE_SKIP_DATA state with (blockSize == 0). + */ + // if (v == 0) return ZSTD2_STATE_SIGNATURE; + return ZSTD2_STATE_SKIP_DATA; + } + } + + // if (p->frameState == ZSTD2_STATE_FRAME_HEADER) + { + unsigned descriptor; + const Byte *h; + descriptor = p->temp[0]; + p->descriptor = (Byte)descriptor; + if (descriptor & DESCRIPTOR_FLAG_RESERVED) // reserved bit + { + p->isErrorState = True; + return ZSTD2_STATE_FRAME_HEADER; + // return ZSTD2_STATE_ERROR; + } + { + const unsigned n = DESCRIPTOR_Get_ContentSize_Flag3(descriptor); + // tempSize -= 1 + ((1u << (n >> 1)) | ((n + 1) & 1)); + tempSize -= (0x9a563422u >> (n * 4)) & 0xf; + } + if (tempSize != (4u >> (3 - DESCRIPTOR_Get_DictionaryId_Flag(descriptor)))) + return ZSTD2_STATE_FRAME_HEADER; + + info->descriptor_OR = (Byte)(info->descriptor_OR | descriptor); + info->descriptor_NOT_OR = (Byte)(info->descriptor_NOT_OR | ~descriptor); + + h = &p->temp[1]; + { + Byte w = 0; + if ((descriptor & DESCRIPTOR_FLAG_SINGLE) == 0) + { + w = *h++; + if (info->windowDescriptor_MAX < w) + info->windowDescriptor_MAX = w; + // info->are_WindowDescriptors = True; + // info->num_WindowDescriptors++; + } + else + { + // info->are_SingleSegments = True; + // info->num_SingleSegments++; + } + p->windowDescriptor = w; + } + { + unsigned n = DESCRIPTOR_Get_DictionaryId_Flag(descriptor); + UInt32 d = 0; + if (n) + { + n = 1u << (n - 1); + d = GetUi32(h) & ((UInt32)(Int32)-1 >> (32 - 8u * n)); + h += n; + } + p->dictionaryId = d; + // info->dictionaryId_Cur = d; + if (d != 0) + { + if (info->dictionaryId == 0) + info->dictionaryId = d; + else if (info->dictionaryId != d) + info->are_DictionaryId_Different = True; + } + } + { + unsigned n = DESCRIPTOR_Get_ContentSize_Flag3(descriptor); + UInt64 v = 0; + if (n) + { + n >>= 1; + if (n == 1) + v = 256; + v += GetUi64(h) & ((UInt64)(Int64)-1 >> (64 - (8u << n))); + // info->are_ContentSize_Known = True; + // info->num_Frames_with_ContentSize++; + if (info->contentSize_MAX < v) + info->contentSize_MAX = v; + info->contentSize_Total += v; + } + else + { + info->are_ContentSize_Unknown = True; + // info->num_Frames_without_ContentSize++; + } + p->contentSize = v; + } + // if ((size_t)(h - p->temp) != headerSize) return ZSTD2_STATE_ERROR; // it's unexpected internal code failure + p->tempSize = 0; + + info->checksum_Defined = False; + /* + if (descriptor & DESCRIPTOR_FLAG_CHECKSUM) + info->are_Checksums = True; + else + info->are_Non_Checksums = True; + */ + + return ZSTD2_STATE_AFTER_HEADER; // ZSTD2_STATE_BLOCK; + } +} + + +static void ZstdDec_Update_XXH(CZstdDec * const p, size_t xxh64_winPos) +{ + /* + #ifdef DISABLE_XXH_CHECK + UNUSED_VAR(data) + #else + */ + if (!p->disableHash && (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM)) + { + // const size_t pos = p->xxh64_winPos; + const size_t size = (p->decoder.winPos - xxh64_winPos) & ~(size_t)31; + if (size) + { + // p->xxh64_winPos = pos + size; + Xxh64State_UpdateBlocks(&p->xxh64, + p->decoder.win + xxh64_winPos, + p->decoder.win + xxh64_winPos + size); + } + } +} + + +/* +in: + (winLimit) : is relaxed limit, where this function is allowed to stop writing of decoded data (if possible). + - this function uses (winLimit) for RAW/RLE blocks only, + because this function can decode single RAW/RLE block in several different calls. + - this function DOESN'T use (winLimit) for Compressed blocks, + because this function decodes full compressed block in single call. + (CZstdDec1::winPos <= winLimit) + (winLimit <= CZstdDec1::cycSize). + Note: if (ds->outBuf_fromCaller) mode is used, then + { + (strong_limit) is stored in CZstdDec1::cycSize. + So (winLimit) is more strong than (strong_limit). + } + +exit: + Note: (CZstdDecState::winPos) will be set by caller after exit of this function. + + This function can exit for any of these conditions: + - (frameState == ZSTD2_STATE_AFTER_HEADER) + - (frameState == ZSTD2_STATE_FINISHED) : frame was finished : (status == ZSTD_STATUS_FINISHED_FRAME) is set + - finished non-empty non-last block. So (CZstdDec1::winPos_atExit != winPos_atFuncStart). + - ZSTD_STATUS_NEEDS_MORE_INPUT in src + - (CZstdDec1::winPos) have reached (winLimit) in non-finished RAW/RLE block + + This function decodes no more than one non-empty block. + So it fulfills the condition at exit: + (CZstdDec1::winPos_atExit - winPos_atFuncStart <= block_size_max) + Note: (winPos_atExit > winLimit) is possible in some cases after compressed block decoding. + + if (ds->outBuf_fromCaller) mode (useAdditionalWinLimit medo) + { + then this function uses additional strong limit from (CZstdDec1::cycSize). + So this function will not write any data after (CZstdDec1::cycSize) + And it fulfills the condition at exit: + (CZstdDec1::winPos_atExit <= CZstdDec1::cycSize) + } +*/ +static SRes ZstdDec_DecodeBlock(CZstdDec * const p, CZstdDecState * const ds, + SizeT winLimitAdd) +{ + const Byte *src = ds->inBuf; + SizeT * const srcLen = &ds->inPos; + const SizeT inSize = ds->inLim; + // const int useAdditionalWinLimit = ds->outBuf_fromCaller ? 1 : 0; + enum_ZstdStatus * const status = &ds->status; + CZstdDecInfo * const info = &ds->info; + SizeT winLimit; + + const SizeT winPos_atFuncStart = p->decoder.winPos; + src += *srcLen; + *status = ZSTD_STATUS_NOT_SPECIFIED; + + // finishMode = ZSTD_FINISH_ANY; + if (ds->outSize_Defined) + { + if (ds->outSize < ds->outProcessed) + { + // p->isAfterSizeMode = 2; // we have extra bytes already + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + // size = 0; + } + else + { + // p->outSize >= p->outProcessed + const UInt64 rem = ds->outSize - ds->outProcessed; + /* + if (rem == 0) + p->isAfterSizeMode = 1; // we have reached exact required size + */ + if (winLimitAdd >= rem) + { + winLimitAdd = (SizeT)rem; + // if (p->finishMode) finishMode = ZSTD_FINISH_END; + } + } + } + + winLimit = p->decoder.winPos + winLimitAdd; + // (p->decoder.winPos <= winLimit) + + // while (p->frameState != ZSTD2_STATE_ERROR) + while (!p->isErrorState) + { + SizeT inCur = inSize - *srcLen; + + if (p->frameState == ZSTD2_STATE_DATA) + { + /* (p->decoder.winPos == winPos_atFuncStart) is expected, + because this function doesn't start new block. + if it have finished some non-empty block in this call. */ + if (p->decoder.winPos != winPos_atFuncStart) + return SZ_ERROR_FAIL; // it's unexpected + + /* + if (p->decoder.winPos > winLimit) + { + // we can be here, if in this function call + // - we have extracted non-empty compressed block, and (winPos > winLimit) after that. + // - we have started new block decoding after that. + // It's unexpected case, because we exit after non-empty non-last block. + *status = (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED; + return SZ_OK; + } + */ + // p->decoder.winPos <= winLimit + + if (p->blockType != kBlockType_Compressed) + { + // it's RLE or RAW block. + // p->BlockSize != 0_ + // winLimit <= p->decoder.cycSize + /* So here we use more strong (winLimit), even for + (ds->outBuf_fromCaller) mode. */ + SizeT outCur = winLimit - p->decoder.winPos; + { + const UInt32 rem = p->blockSize; + if (outCur > rem) + outCur = rem; + } + if (p->blockType == kBlockType_Raw) + { + if (outCur > inCur) + outCur = inCur; + /* output buffer is better aligned for XXH code. + So we use hash for output buffer data */ + // ZstdDec_Update_XXH(p, src, outCur); // for debug: + memcpy(p->decoder.win + p->decoder.winPos, src, outCur); + src += outCur; + *srcLen += outCur; + } + else // kBlockType_RLE + { + #define RLE_BYTE_INDEX_IN_temp 3 + memset(p->decoder.win + p->decoder.winPos, + p->temp[RLE_BYTE_INDEX_IN_temp], outCur); + } + { + const SizeT xxh64_winPos = p->decoder.winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p); + p->decoder.winPos += outCur; + UPDATE_TOTAL_OUT(&p->decoder, outCur) + p->contentProcessed += outCur; + ZstdDec_Update_XXH(p, xxh64_winPos); + } + // ds->winPos = p->decoder.winPos; // the caller does it instead. for debug: + ds->outProcessed += outCur; + if (p->blockSize -= (UInt32)outCur) + { + /* + if (ds->outSize_Defined) + { + if (ds->outSize <= ds->outProcessed) ds->isAfterSizeMode = (enum_ZstdStatus) + (ds->outSize == ds->outProcessed ? 1u: 2u); + } + */ + *status = (enum_ZstdStatus) + (ds->outSize_Defined && ds->outSize <= ds->outProcessed ? + ZSTD_STATUS_OUT_REACHED : (p->blockType == kBlockType_Raw && inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED); + return SZ_OK; + } + } + else // kBlockType_Compressed + { + // p->blockSize != 0 + // (uncompressed_size_of_block == 0) is allowed + // (p->curBlockUnpackRem == 0) is allowed + /* + if (p->decoder.winPos >= winLimit) + { + if (p->decoder.winPos != winPos_atFuncStart) + { + // it's unexpected case + // We already have some data in finished blocks in this function call. + // So we don't decompress new block after (>=winLimit), + // even if it's empty block. + *status = (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED; + return SZ_OK; + } + // (p->decoder.winPos == winLimit == winPos_atFuncStart) + // we will decode current block, because that current + // block can be empty block and we want to make some visible + // change of (src) stream after function start. + } + */ + /* + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + // we don't want to start new block, if we have more extra decoded bytes already + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + } + */ + { + const Byte *comprStream; + size_t afterAvail; + UInt32 inTempPos = p->inTempPos; + const UInt32 rem = p->blockSize - inTempPos; + // rem != 0 + if (inTempPos != 0 // (inTemp) buffer already contains some input data + || inCur < rem // available input data size is smaller than compressed block size + || ZstdDec1_NeedTempBufferForInput(*srcLen, src, rem)) + { + if (inCur > rem) + inCur = rem; + if (inCur) + { + STAT_INC(g_Num_Blocks_memcpy) + // we clear data for backward lookahead reading + if (inTempPos == 0) + memset(p->inTemp + kTempBuffer_PreSize - MAX_BACKWARD_DEPTH, 0, MAX_BACKWARD_DEPTH); + // { unsigned y = 0; for(;y < 1000; y++) + memcpy(p->inTemp + inTempPos + kTempBuffer_PreSize, src, inCur); + // } + src += inCur; + *srcLen += inCur; + inTempPos += (UInt32)inCur; + p->inTempPos = inTempPos; + } + if (inTempPos != p->blockSize) + { + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + #if COPY_CHUNK_SIZE > 1 + memset(p->inTemp + kTempBuffer_PreSize + inTempPos, 0, COPY_CHUNK_SIZE); + #endif + comprStream = p->inTemp + kTempBuffer_PreSize; + afterAvail = k_Lit_AfterAvail; + // we don't want to read non-initialized data or junk in CopyMatch(): + } + else + { + // inCur >= rem + // we use direct decoding from (src) buffer: + afterAvail = inCur - rem; + comprStream = src; + src += rem; + *srcLen += rem; + } + + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + ZstdDec1_NeedTempBufferForInput(*srcLen, comprStream, p->blockSize); + #endif + // printf("\nblockSize=%u", p->blockSize); + // printf("%x\n", (unsigned)p->contentProcessed); + STAT_INC(g_Num_Blocks_Compressed) + { + SRes sres; + const size_t winPos = p->decoder.winPos; + /* + if ( useAdditionalWinLimit), we use strong unpack limit: smallest from + - limit from stream : (curBlockUnpackRem) + - limit from caller : (cycSize - winPos) + if (!useAdditionalWinLimit), we use only relaxed limit: + - limit from stream : (curBlockUnpackRem) + */ + SizeT outLimit = p->curBlockUnpackRem; + if (ds->outBuf_fromCaller) + // if (useAdditionalWinLimit) + { + const size_t limit = p->decoder.cycSize - winPos; + if (outLimit > limit) + outLimit = limit; + } + sres = ZstdDec1_DecodeBlock(&p->decoder, + comprStream, p->blockSize, afterAvail, outLimit); + // ds->winPos = p->decoder.winPos; // the caller does it instead. for debug: + if (sres) + { + p->isErrorState = True; + return sres; + } + { + const SizeT xxh64_winPos = winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p); + const size_t num = p->decoder.winPos - winPos; + ds->outProcessed += num; + p->contentProcessed += num; + ZstdDec_Update_XXH(p, xxh64_winPos); + } + } + // printf("\nwinPos=%x", (int)(unsigned)p->decoder.winPos); + } + } + + /* + if (ds->outSize_Defined) + { + if (ds->outSize <= ds->outProcessed) ds->isAfterSizeMode = (enum_ZstdStatus) + (ds->outSize == ds->outProcessed ? 1u: 2u); + } + */ + + if (!ZSTD_DEC_IS_LAST_BLOCK(p)) + { + p->frameState = ZSTD2_STATE_BLOCK; + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + } + // we exit only if (winPos) was changed in this function call: + if (p->decoder.winPos != winPos_atFuncStart) + { + // decoded block was not empty. So we exit: + *status = (enum_ZstdStatus)( + (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED); + return SZ_OK; + } + // (p->decoder.winPos == winPos_atFuncStart) + // so current decoded block was empty. + // we will try to decode more blocks in this function. + continue; + } + + // decoded block was last in frame + if (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) + { + p->frameState = ZSTD2_STATE_HASH; + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; // disable if want to + /* We want to get same return codes for any input buffer sizes. + We want to get faster ZSTD_STATUS_OUT_REACHED status. + So we exit with ZSTD_STATUS_OUT_REACHED here, + instead of ZSTD2_STATE_HASH and ZSTD2_STATE_FINISHED processing. + that depends from input buffer size and that can set + ZSTD_STATUS_NEEDS_MORE_INPUT or return SZ_ERROR_DATA or SZ_ERROR_CRC. + */ + } + } + else + { + /* ZSTD2_STATE_FINISHED proccesing doesn't depend from input buffer */ + p->frameState = ZSTD2_STATE_FINISHED; + } + /* + p->frameState = (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) ? + ZSTD2_STATE_HASH : + ZSTD2_STATE_FINISHED; + */ + /* it's required to process ZSTD2_STATE_FINISHED state in this function call, + because we must check contentSize and hashError in ZSTD2_STATE_FINISHED code, + while the caller can reinit full state for ZSTD2_STATE_FINISHED + So we can't exit from function here. */ + continue; + } + + if (p->frameState == ZSTD2_STATE_FINISHED) + { + *status = ZSTD_STATUS_FINISHED_FRAME; + if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor) + && p->contentSize != p->contentProcessed) + return SZ_ERROR_DATA; + if (p->hashError) // for debug + return SZ_ERROR_CRC; + return SZ_OK; + // p->frameState = ZSTD2_STATE_SIGNATURE; + // continue; + } + + if (p->frameState == ZSTD2_STATE_AFTER_HEADER) + return SZ_OK; // we need memory allocation for that state + + if (p->frameState == ZSTD2_STATE_SKIP_DATA) + { + UInt32 blockSize = p->blockSize; + // (blockSize == 0) is possible + if (inCur > blockSize) + inCur = blockSize; + src += inCur; + *srcLen += inCur; + blockSize -= (UInt32)inCur; + p->blockSize = blockSize; + if (blockSize == 0) + { + p->frameState = ZSTD2_STATE_SIGNATURE; + // continue; // for debug: we can continue without return to caller. + // we notify the caller that skip frame was finished: + *status = ZSTD_STATUS_FINISHED_FRAME; + return SZ_OK; + } + // blockSize != 0 + // (inCur) was smaller than previous value of p->blockSize. + // (inSize == *srcLen) now + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + if (inCur == 0) + { + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + { + (*srcLen)++; + p->frameState = ZstdDec_UpdateState(p, *src++, info); + } + } + + *status = ZSTD_STATUS_NOT_SPECIFIED; + p->isErrorState = True; + // p->frameState = ZSTD2_STATE_ERROR; + // if (p->frameState = ZSTD2_STATE_SIGNATURE) return SZ_ERROR_NO_ARCHIVE + return SZ_ERROR_DATA; +} + + + + +SRes ZstdDec_Decode(CZstdDecHandle dec, CZstdDecState *p) +{ + p->needWrite_Size = 0; + p->status = ZSTD_STATUS_NOT_SPECIFIED; + dec->disableHash = p->disableHash; + + if (p->outBuf_fromCaller) + { + dec->decoder.win = p->outBuf_fromCaller; + dec->decoder.cycSize = p->outBufSize_fromCaller; + } + + // p->winPos = dec->decoder.winPos; + + for (;;) + { + SizeT winPos, size; + // SizeT outProcessed; + SRes res; + + if (p->wrPos > dec->decoder.winPos) + return SZ_ERROR_FAIL; + + if (dec->frameState == ZSTD2_STATE_FINISHED) + { + if (!p->outBuf_fromCaller) + { + // we need to set positions to zero for new frame. + if (p->wrPos != dec->decoder.winPos) + { + /* We have already asked the caller to flush all data + with (p->needWrite_Size) and (ZSTD_STATUS_FINISHED_FRAME) status. + So it's unexpected case */ + // p->winPos = dec->decoder.winPos; + // p->needWrite_Size = dec->decoder.winPos - p->wrPos; // flush size asking + // return SZ_OK; // ask to flush again + return SZ_ERROR_FAIL; + } + // (p->wrPos == dec->decoder.winPos), and we wrap to zero: + dec->decoder.winPos = 0; + p->winPos = 0; + p->wrPos = 0; + } + ZstdDec_Init_ForNewFrame(dec); + // continue; + } + + winPos = dec->decoder.winPos; + { + SizeT next = dec->decoder.cycSize; + /* cycSize == 0, if no buffer was allocated still, + or, if (outBuf_fromCaller) mode and (outBufSize_fromCaller == 0) */ + if (!p->outBuf_fromCaller + && next + && next <= winPos + && dec->isCyclicMode) + { + // (0 < decoder.cycSize <= winPos) in isCyclicMode. + // so we need to wrap (winPos) and (wrPos) over (cycSize). + const size_t delta = next; + // (delta) is how many bytes we remove from buffer. + /* + // we don't need data older than last (cycSize) bytes. + size_t delta = winPos - next; // num bytes after (cycSize) + if (delta <= next) // it's expected case + delta = next; + // delta == Max(cycSize, winPos - cycSize) + */ + if (p->wrPos < delta) + { + // (wrPos < decoder.cycSize) + // We have asked already the caller to flush required data + // p->status = ZSTD_STATUS_NOT_SPECIFIED; + // p->winPos = winPos; + // p->needWrite_Size = delta - p->wrPos; // flush size asking + // return SZ_OK; // ask to flush again + return SZ_ERROR_FAIL; + } + // p->wrPos >= decoder.cycSize + // we move extra data after (decoder.cycSize) to start of cyclic buffer: + winPos -= delta; + if (winPos) + { + if (winPos >= delta) + return SZ_ERROR_FAIL; + memmove(dec->decoder.win, dec->decoder.win + delta, winPos); + // printf("\nmemmove processed=%8x winPos=%8x\n", (unsigned)p->outProcessed, (unsigned)dec->decoder.winPos); + STAT_INC(g_Num_Wrap_memmove_Num) + STAT_UPDATE(g_Num_Wrap_memmove_Bytes += (unsigned)winPos;) + } + dec->decoder.winPos = winPos; + p->winPos = winPos; + p->wrPos -= delta; + // dec->xxh64_winPos -= delta; + + // (winPos < delta) + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + /* we set the data after cycSize, because + we don't want to read non-initialized data or junk in CopyMatch(). */ + memset(dec->decoder.win + next, 0, COPY_CHUNK_SIZE); + #endif + + /* + if (winPos == next) + { + if (winPos != p->wrPos) + { + // we already requested before to flush full data for that case. + // but we give the caller a second chance to flush data: + p->needWrite_Size = winPos - p->wrPos; + return SZ_OK; + } + // (decoder.cycSize == winPos == p->wrPos) + // so we do second wrapping to zero: + winPos = 0; + dec->decoder.winPos = 0; + p->winPos = 0; + p->wrPos = 0; + } + */ + // (winPos < next) + } + + if (winPos > next) + return SZ_ERROR_FAIL; // it's unexpected case + /* + if (!outBuf_fromCaller && isCyclicMode && cycSize != 0) + then (winPos < cycSize) + else (winPos <= cycSize) + */ + if (!p->outBuf_fromCaller) + { + // that code is optional. We try to optimize write chunk sizes. + /* (next2) is expected next write position in the caller, + if the caller writes by kBlockSizeMax chunks. + */ + /* + const size_t next2 = (winPos + kBlockSizeMax) & (kBlockSizeMax - 1); + if (winPos < next2 && next2 < next) + next = next2; + */ + } + size = next - winPos; + } + + // note: ZstdDec_DecodeBlock() uses (winLimit = winPos + size) only for RLE and RAW blocks + res = ZstdDec_DecodeBlock(dec, p, size); + /* + after one block decoding: + if (!outBuf_fromCaller && isCyclicMode && cycSize != 0) + then (winPos < cycSize + max_block_size) + else (winPos <= cycSize) + */ + + if (!p->outBuf_fromCaller) + p->win = dec->decoder.win; + p->winPos = dec->decoder.winPos; + + // outProcessed = dec->decoder.winPos - winPos; + // p->outProcessed += outProcessed; + + if (res != SZ_OK) + return res; + + if (dec->frameState != ZSTD2_STATE_AFTER_HEADER) + { + if (p->outBuf_fromCaller) + return SZ_OK; + { + // !p->outBuf_fromCaller + /* + if (ZSTD_STATUS_FINISHED_FRAME), we request full flushing here because + 1) it's simpler to work with allocation and extracting of next frame, + 2) it's better to start writing to next new frame with aligned memory + for faster xxh 64-bit reads. + */ + size_t end = dec->decoder.winPos; // end pos for all data flushing + if (p->status != ZSTD_STATUS_FINISHED_FRAME) + { + // we will request flush here only for cases when wrap in cyclic buffer can be required in next call. + if (!dec->isCyclicMode) + return SZ_OK; + // isCyclicMode + { + const size_t delta = dec->decoder.cycSize; + if (end < delta) + return SZ_OK; // (winPos < cycSize). no need for flush + // cycSize <= winPos + // So we ask the caller to flush of (cycSize - wrPos) bytes, + // and then we will wrap cylicBuffer in next call + end = delta; + } + } + p->needWrite_Size = end - p->wrPos; + } + return SZ_OK; + } + + // ZSTD2_STATE_AFTER_HEADER + { + BoolInt useCyclic = False; + size_t cycSize; + + // p->status = ZSTD_STATUS_NOT_FINISHED; + if (dec->dictionaryId != 0) + { + /* actually we can try to decode some data, + because it's possible that some data doesn't use dictionary */ + // p->status = ZSTD_STATUS_NOT_SPECIFIED; + return SZ_ERROR_UNSUPPORTED; + } + + { + UInt64 winSize = dec->contentSize; + UInt64 winSize_Allocate = winSize; + const unsigned descriptor = dec->descriptor; + + if ((descriptor & DESCRIPTOR_FLAG_SINGLE) == 0) + { + const Byte wd = dec->windowDescriptor; + winSize = (UInt64)(8 + (wd & 7)) << ((wd >> 3) + 10 - 3); + if (!DESCRIPTOR_Is_ContentSize_Defined(descriptor) + || winSize_Allocate > winSize) + { + winSize_Allocate = winSize; + useCyclic = True; + } + } + /* + else + { + if (p->info.singleSegment_ContentSize_MAX < winSize) + p->info.singleSegment_ContentSize_MAX = winSize; + // p->info.num_SingleSegments++; + } + */ + if (p->info.windowSize_MAX < winSize) + p->info.windowSize_MAX = winSize; + if (p->info.windowSize_Allocate_MAX < winSize_Allocate) + p->info.windowSize_Allocate_MAX = winSize_Allocate; + /* + winSize_Allocate is MIN(content_size, window_size_from_descriptor). + Wven if (content_size < (window_size_from_descriptor)) + original-zstd still uses (window_size_from_descriptor) to check that decoding is allowed. + We try to follow original-zstd, and here we check (winSize) instead of (winSize_Allocate)) + */ + if ( + // winSize_Allocate // it's relaxed check + winSize // it's more strict check to be compatible with original-zstd + > ((UInt64)1 << MAX_WINDOW_SIZE_LOG)) + return SZ_ERROR_UNSUPPORTED; // SZ_ERROR_MEM + cycSize = (size_t)winSize_Allocate; + if (cycSize != winSize_Allocate) + return SZ_ERROR_MEM; + // cycSize <= winSize + /* later we will use (CZstdDec1::winSize) to check match offsets and check block sizes. + if (there is window descriptor) + { + We will check block size with (window_size_from_descriptor) instead of (winSize_Allocate). + Does original-zstd do it that way also? + } + Here we must reduce full real 64-bit (winSize) to size_t for (CZstdDec1::winSize). + Also we don't want too big values for (CZstdDec1::winSize). + our (CZstdDec1::winSize) will meet the condition: + (CZstdDec1::winSize < kBlockSizeMax || CZstdDec1::winSize <= cycSize). + */ + dec->decoder.winSize = (winSize < kBlockSizeMax) ? (size_t)winSize: cycSize; + // note: (CZstdDec1::winSize > cycSize) is possible, if (!useCyclic) + } + + RINOK(ZstdDec_AllocateMisc(dec)) + + if (p->outBuf_fromCaller) + dec->isCyclicMode = False; + else + { + size_t d = cycSize; + + if (dec->decoder.winPos != p->wrPos) + return SZ_ERROR_FAIL; + + dec->decoder.winPos = 0; + p->wrPos = 0; + p->winPos = dec->decoder.winPos; + + /* + const size_t needWrite = dec->decoder.winPos - p->wrPos; + if (!needWrite) + { + dec->decoder.winPos = 0; + p->wrPos = 0; + p->winPos = dec->decoder.winPos; + } + */ + /* if (!useCyclic) we allocate only cycSize = ContentSize. + But if we want to support the case where new frame starts with winPos != 0, + then we will wrap over zero, and we still need + to set (useCyclic) and allocate additional buffer spaces. + Now we don't allow new frame starting with (winPos != 0). + so (dec->decoder->winPos == 0) + can use (!useCyclic) with reduced buffer sizes. + */ + /* + if (dec->decoder->winPos != 0) + useCyclic = True; + */ + + if (useCyclic) + { + /* cyclyc buffer size must be at least (COPY_CHUNK_SIZE - 1) bytes + larger than window size, because CopyMatch() can write additional + (COPY_CHUNK_SIZE - 1) bytes and overwrite oldests data in cyclyc buffer. + But for performance reasons we align (cycSize) for (kBlockSizeMax). + also we must provide (cycSize >= max_decoded_data_after_cycSize), + because after data move wrapping over zero we must provide (winPos < cycSize). + */ + const size_t alignSize = kBlockSizeMax; + /* here we add (1 << 7) instead of (COPY_CHUNK_SIZE - 1), because + we want to get same (cycSize) for different COPY_CHUNK_SIZE values. */ + // cycSize += (COPY_CHUNK_SIZE - 1) + (alignSize - 1); // for debug : we can get smallest (cycSize) + cycSize += (1 << 7) + alignSize; + cycSize &= ~(size_t)(alignSize - 1); + // cycSize must be aligned for 32, because xxh requires 32-bytes blocks. + // cycSize += 12345; // for debug + // cycSize += 1 << 10; // for debug + // cycSize += 32; // for debug + // cycSize += kBlockSizeMax; // for debug + if (cycSize < d) + return SZ_ERROR_MEM; + /* + in cyclic buffer mode we allow to decode one additional block + that exceeds (cycSize). + So we must allocate additional (kBlockSizeMax) bytes after (cycSize). + if defined(Z7_STD_DEC_USE_AFTER_CYC_BUF) + { + we can read (COPY_CHUNK_SIZE - 1) bytes after (cycSize) + but we aready allocate additional kBlockSizeMax that + is larger than COPY_CHUNK_SIZE. + So we don't need additional space of COPY_CHUNK_SIZE after (cycSize). + } + */ + /* + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + d = cycSize + (1 << 7); // we must add at least (COPY_CHUNK_SIZE - 1) + #endif + */ + d = cycSize + kBlockSizeMax; + if (d < cycSize) + return SZ_ERROR_MEM; + } + + { + const size_t kMinWinAllocSize = 1 << 12; + if (d < kMinWinAllocSize) + d = kMinWinAllocSize; + } + + if (d > dec->winBufSize_Allocated) + { + /* + if (needWrite) + { + p->needWrite_Size = needWrite; + return SZ_OK; + // return SZ_ERROR_FAIL; + } + */ + + if (dec->winBufSize_Allocated != 0) + { + const size_t k_extra = (useCyclic || d >= (1u << 20)) ? + 2 * kBlockSizeMax : 0; + unsigned i = useCyclic ? 17 : 12; + for (; i < sizeof(size_t) * 8; i++) + { + const size_t d2 = ((size_t)1 << i) + k_extra; + if (d2 >= d) + { + d = d2; + break; + } + } + } + // RINOK(ZstdDec_AllocateWindow(dec, d)) + ZstdDec_FreeWindow(dec); + dec->win_Base = (Byte *)ISzAlloc_Alloc(dec->alloc_Big, d); + if (!dec->win_Base) + return SZ_ERROR_MEM; + dec->decoder.win = dec->win_Base; + dec->winBufSize_Allocated = d; + } + /* + else + { + // for non-cyclycMode we want flush data, and set winPos = 0 + if (needWrite) + { + if (!useCyclic || dec->decoder.winPos >= cycSize) + { + p->needWrite_Size = needWrite; + return SZ_OK; + // return SZ_ERROR_FAIL; + } + } + } + */ + + dec->decoder.cycSize = cycSize; + p->win = dec->decoder.win; + // p->cycSize = dec->decoder.cycSize; + dec->isCyclicMode = (Byte)useCyclic; + } // (!p->outBuf_fromCaller) end + + // p->winPos = dec->decoder.winPos; + dec->frameState = ZSTD2_STATE_BLOCK; + // continue; + } // ZSTD2_STATE_AFTER_HEADER end + } +} + + +void ZstdDec_GetResInfo(const CZstdDec *dec, + const CZstdDecState *p, + SRes res, + CZstdDecResInfo *stat) +{ + // ZstdDecInfo_CLEAR(stat); + stat->extraSize = 0; + stat->is_NonFinishedFrame = False; + if (dec->frameState != ZSTD2_STATE_FINISHED) + { + if (dec->frameState == ZSTD2_STATE_SIGNATURE) + { + stat->extraSize = (Byte)dec->tempSize; + if (ZstdDecInfo_GET_NUM_FRAMES(&p->info) == 0) + res = SZ_ERROR_NO_ARCHIVE; + } + else + { + stat->is_NonFinishedFrame = True; + if (res == SZ_OK && p->status == ZSTD_STATUS_NEEDS_MORE_INPUT) + res = SZ_ERROR_INPUT_EOF; + } + } + stat->decode_SRes = res; +} + + +size_t ZstdDec_ReadUnusedFromInBuf( + CZstdDecHandle dec, + size_t afterDecoding_tempPos, + void *data, size_t size) +{ + size_t processed = 0; + if (dec->frameState == ZSTD2_STATE_SIGNATURE) + { + Byte *dest = (Byte *)data; + const size_t tempSize = dec->tempSize; + while (afterDecoding_tempPos < tempSize) + { + if (size == 0) + break; + size--; + *dest++ = dec->temp[afterDecoding_tempPos++]; + processed++; + } + } + return processed; +} + + +void ZstdDecState_Clear(CZstdDecState *p) +{ + memset(p, 0 , sizeof(*p)); +} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/C/ZstdDec.h b/ext/lzma_sdk_wrapper/lzma_sdk/C/ZstdDec.h new file mode 100644 index 0000000..cd26131 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/C/ZstdDec.h @@ -0,0 +1,173 @@ +/* ZstdDec.h -- Zstd Decoder interfaces +2024-01-21 : Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_ZSTD_DEC_H +#define ZIP7_INC_ZSTD_DEC_H + +EXTERN_C_BEGIN + +typedef struct CZstdDec CZstdDec; +typedef CZstdDec * CZstdDecHandle; + +CZstdDecHandle ZstdDec_Create(ISzAllocPtr alloc_Small, ISzAllocPtr alloc_Big); +void ZstdDec_Destroy(CZstdDecHandle p); + +typedef enum +{ + ZSTD_STATUS_NOT_SPECIFIED, /* use main error code instead */ + ZSTD_STATUS_FINISHED_FRAME, /* data frame or skip frame was finished */ + ZSTD_STATUS_NOT_FINISHED, /* just finished non-empty block or unfinished RAW/RLE block */ + ZSTD_STATUS_NEEDS_MORE_INPUT, /* the callee needs more input bytes. It has more priority over ZSTD_STATUS_NOT_FINISHED */ + ZSTD_STATUS_OUT_REACHED /* is not finihed frame and ((outProcessed > outSize) || (outProcessed == outSize && unfinished RAW/RLE block) */ +} enum_ZstdStatus_Dummy; + +#define ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(p) \ + ((p)->status & ZSTD_STATUS_FINISHED_FRAME) +/* + ((p)->status == ZSTD_STATUS_NEEDS_MORE_INPUT || \ + (p)->status == ZSTD_STATUS_FINISHED_FRAME) +*/ + +typedef Byte enum_ZstdStatus; + + +void ZstdDec_Init(CZstdDecHandle p); + +typedef struct +{ + UInt64 num_Blocks; + Byte descriptor_OR; + Byte descriptor_NOT_OR; + Byte are_ContentSize_Unknown; + Byte windowDescriptor_MAX; + + // Byte are_ContentSize_Known; + // Byte are_SingleSegments; + // Byte are_WindowDescriptors; + Byte checksum_Defined; + // Byte are_Checksums; + // Byte are_Non_Checksums; + + // Byte are_DictionaryId; + Byte are_DictionaryId_Different; + + // Byte reserved[3]; + + UInt32 checksum; // checksum of last data frame + /// UInt32 dictionaryId_Cur; + UInt32 dictionaryId; // if there are non-zero dictionary IDs, then it's first dictionaryId + + UInt64 num_DataFrames; + UInt64 num_SkipFrames; + UInt64 skipFrames_Size; + UInt64 contentSize_Total; + UInt64 contentSize_MAX; + // UInt64 num_Checksums; + // UInt64 num_Non_Checksums; // frames without checksum + // UInt64 num_WindowDescriptors; + // UInt64 num_SingleSegments; + // UInt64 num_Frames_with_ContentSize; + // UInt64 num_Frames_without_ContentSize; + UInt64 windowSize_MAX; + UInt64 windowSize_Allocate_MAX; + // UInt64 num_DictionaryIds; + // UInt64 num_Blocks_forType[4]; + // UInt64 num_BlockBytes_forType[4]; + // UInt64 num_SingleSegments; + // UInt64 singleSegment_ContentSize_MAX; +} CZstdDecInfo; + +#define ZstdDecInfo_CLEAR(p) { memset(p, 0, sizeof(*(p))); } + +#define ZstdDecInfo_GET_NUM_FRAMES(p) ((p)->num_DataFrames + (p)->num_SkipFrames) + + +typedef struct CZstdDecState +{ + enum_ZstdStatus status; // out + Byte disableHash; + // Byte mustBeFinished; + Byte outSize_Defined; + // Byte isAfterSizeMode; + // UInt64 inProcessed; + // SRes codeRes; + // Byte needWrite_IsStrong; + + const Byte *inBuf; + size_t inPos; // in/out + size_t inLim; + + const Byte *win; // out + size_t winPos; // out + size_t wrPos; // in/out + // size_t cycSize; // out : if (!outBuf_fromCaller) + size_t needWrite_Size; // out + + Byte *outBuf_fromCaller; + size_t outBufSize_fromCaller; + /* (outBufSize_fromCaller >= full_uncompressed_size_of_all_frames) is required + for success decoding. + If outBufSize_fromCaller < full_uncompressed_size_of_all_frames), + decoding can give error message, because we decode per block basis. + */ + + // size_t outStep; + UInt64 outSize; // total in all frames + UInt64 outProcessed; // out decoded in all frames (it can be >= outSize) + + CZstdDecInfo info; +} CZstdDecState; + +void ZstdDecState_Clear(CZstdDecState *p); + +/* +ZstdDec_Decode() +return: + SZ_OK - no error + SZ_ERROR_DATA - Data Error + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported method or method properties + SZ_ERROR_CRC - XXH hash Error + // SZ_ERROR_ARCHIVE - Headers error (not used now) +*/ +SRes ZstdDec_Decode(CZstdDecHandle dec, CZstdDecState *p); + +/* +ZstdDec_ReadUnusedFromInBuf(): +returns: the number of bytes that were read from InBuf +(*afterDecoding_tempPos) must be set to zero before first call of ZstdDec_ReadUnusedFromInBuf() +*/ +size_t ZstdDec_ReadUnusedFromInBuf( + CZstdDecHandle dec, + size_t afterDecoding_tempPos, // in/out + void *data, size_t size); + +typedef struct +{ + SRes decode_SRes; // error code of data decoding + Byte is_NonFinishedFrame; // there is unfinished decoding for data frame or skip frame + Byte extraSize; +} CZstdDecResInfo; + +/* +#define ZstdDecResInfo_CLEAR(p) \ +{ (p)->decode_SRes = 0; \ + (p)->is_NonFinishedFrame; \ + (p)->extraSize = 0; \ +} +// memset(p, 0, sizeof(*p)); +*/ + +/* +additional error codes for CZstdDecResInfo::decode_SRes: + SZ_ERROR_NO_ARCHIVE - is not zstd stream (no frames) + SZ_ERROR_INPUT_EOF - need more data in input stream +*/ +void ZstdDec_GetResInfo(const CZstdDec *dec, + const CZstdDecState *p, + SRes res, // it's result from ZstdDec_Decode() + CZstdDecResInfo *info); + +EXTERN_C_END + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/StdAfx.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/StdAfx.h new file mode 100644 index 0000000..035267c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/StdAfx.h @@ -0,0 +1,11 @@ +// StdAfx.h + +#ifndef ZIP7_INC_STDAFX_H +#define ZIP7_INC_STDAFX_H + +#if defined(_MSC_VER) && _MSC_VER >= 1800 +#pragma warning(disable : 4464) // relative include path contains '..' +#endif +#include "../../../Common/Common.h" + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandler.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandler.cpp new file mode 100644 index 0000000..29f28e8 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandler.cpp @@ -0,0 +1,1094 @@ +// TarHandler.cpp + +#include "StdAfx.h" + +#include "../../../Common/ComTry.h" +#include "../../../Common/IntToString.h" +#include "../../../Common/StringConvert.h" +#include "../../../Common/UTFConvert.h" + +#include "../../../Windows/TimeUtils.h" + +#include "../../Common/LimitedStreams.h" +#include "../../Common/MethodProps.h" +#include "../../Common/ProgressUtils.h" +#include "../../Common/StreamObjects.h" +#include "../../Common/StreamUtils.h" + +#include "../Common/ItemNameUtils.h" + +#include "TarHandler.h" + +using namespace NWindows; + +namespace NArchive { +namespace NTar { + +// 21.02: we use UTF8 code page by default, even if some files show error +// before 21.02 : CP_OEMCP; +// static const UINT k_DefaultCodePage = CP_UTF8; + + +static const Byte kProps[] = +{ + kpidPath, + kpidIsDir, + kpidSize, + kpidPackSize, + kpidMTime, + kpidCTime, + kpidATime, + kpidPosixAttrib, +#if 0 + kpidAttrib, +#endif + kpidUser, + kpidGroup, + kpidUserId, + kpidGroupId, + kpidSymLink, + kpidHardLink, + kpidCharacts, + kpidComment + , kpidDeviceMajor + , kpidDeviceMinor + // , kpidDevice + // , kpidHeadersSize // for debug + // , kpidOffset // for debug +}; + +static const Byte kArcProps[] = +{ + kpidHeadersSize, + kpidCodePage, + kpidCharacts, + kpidComment +}; + +static const char *k_Characts_Prefix = "PREFIX"; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPhySize: if (_arc._phySize_Defined) prop = _arc._phySize; break; + case kpidHeadersSize: if (_arc._phySize_Defined) prop = _arc._headersSize; break; + case kpidErrorFlags: + { + UInt32 flags = 0; + if (!_isArc) + flags |= kpv_ErrorFlags_IsNotArc; + else switch ((int)_arc._error) + { + case k_ErrorType_UnexpectedEnd: flags = kpv_ErrorFlags_UnexpectedEnd; break; + case k_ErrorType_Corrupted: flags = kpv_ErrorFlags_HeadersError; break; + // case k_ErrorType_OK: break; + // case k_ErrorType_Warning: break; + // case k_ErrorType_OK: + default: break; + } + if (flags != 0) + prop = flags; + break; + } + + case kpidWarningFlags: + { + if (_arc._is_Warning) + prop = kpv_ErrorFlags_HeadersError; + break; + } + + case kpidCodePage: + { + char sz[16]; + const char *name = NULL; + switch (_openCodePage) + { + case CP_OEMCP: name = "OEM"; break; + case CP_UTF8: name = "UTF-8"; break; + default: break; + } + if (!name) + { + ConvertUInt32ToString(_openCodePage, sz); + name = sz; + } + prop = name; + break; + } + + case kpidCharacts: + { + AString s; + if (_arc._are_Gnu) s.Add_OptSpaced("GNU"); + if (_arc._are_Posix) s.Add_OptSpaced("POSIX"); + if (_arc._are_Pax_Items) s.Add_OptSpaced("PAX_ITEM"); + if (_arc._pathPrefix_WasUsed) s.Add_OptSpaced(k_Characts_Prefix); + if (_arc._are_LongName) s.Add_OptSpaced("LongName"); + if (_arc._are_LongLink) s.Add_OptSpaced("LongLink"); + if (_arc._are_Pax) s.Add_OptSpaced("PAX"); + if (_arc._are_pax_path) s.Add_OptSpaced("path"); + if (_arc._are_pax_link) s.Add_OptSpaced("linkpath"); + if (_arc._are_mtime) s.Add_OptSpaced("mtime"); + if (_arc._are_atime) s.Add_OptSpaced("atime"); + if (_arc._are_ctime) s.Add_OptSpaced("ctime"); + if (_arc._are_SCHILY_fflags) s.Add_OptSpaced("SCHILY.fflags"); + if (_arc._is_PaxGlobal_Error) s.Add_OptSpaced("PAX_GLOBAL_ERROR"); + s.Add_OptSpaced(_encodingCharacts.GetCharactsString()); + prop = s; + break; + } + + case kpidComment: + { + if (_arc.PaxGlobal_Defined) + { + AString s; + _arc.PaxGlobal.Print_To_String(s); + if (!s.IsEmpty()) + prop = s; + } + break; + } + default: break; + } + prop.Detach(value); + return S_OK; +} + + +void CEncodingCharacts::Check(const AString &s) +{ + IsAscii = s.IsAscii(); + if (!IsAscii) + { + /* + { + Oem_Checked = true; + UString u; + MultiByteToUnicodeString2(u, s, CP_OEMCP); + Oem_Ok = (u.Find((wchar_t)0xfffd) <= 0); + } + Utf_Checked = true; + */ + UtfCheck.Check_AString(s); + } +} + + +AString CEncodingCharacts::GetCharactsString() const +{ + AString s; + if (IsAscii) + { + s += "ASCII"; + } + /* + if (Oem_Checked) + { + s.Add_Space_if_NotEmpty(); + s += (Oem_Ok ? "oem-ok" : "oem-error"); + } + if (Utf_Checked) + */ + else + { + s.Add_Space_if_NotEmpty(); + s += (UtfCheck.IsOK() ? "UTF8" : "UTF8-ERROR"); // "UTF8-error" + { + AString s2; + UtfCheck.PrintStatus(s2); + s.Add_Space_if_NotEmpty(); + s += s2; + } + } + return s; +} + + +HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) +{ + UInt64 endPos; + { + RINOK(InStream_AtBegin_GetSize(stream, endPos)) + } + + _arc._phySize_Defined = true; + + // bool utf8_OK = true; + + _arc.SeqStream = stream; + _arc.InStream = stream; + _arc.OpenCallback = callback; + + CItemEx item; + for (;;) + { + _arc.NumFiles = _items.Size(); + RINOK(_arc.ReadItem(item)) + if (!_arc.filled) + break; + + _isArc = true; + + /* + if (!_forceCodePage) + { + if (utf8_OK) utf8_OK = CheckUTF8(item.Name, item.NameCouldBeReduced); + if (utf8_OK) utf8_OK = CheckUTF8(item.LinkName, item.LinkNameCouldBeReduced); + if (utf8_OK) utf8_OK = CheckUTF8(item.User); + if (utf8_OK) utf8_OK = CheckUTF8(item.Group); + } + */ + + item.EncodingCharacts.Check(item.Name); + _encodingCharacts.Update(item.EncodingCharacts); + + _items.Add(item); + + RINOK(stream->Seek((Int64)item.Get_PackSize_Aligned(), STREAM_SEEK_CUR, &_arc._phySize)) + if (_arc._phySize > endPos) + { + _arc._error = k_ErrorType_UnexpectedEnd; + break; + } + /* + if (_phySize == endPos) + { + _errorMessage = "There are no trailing zero-filled records"; + break; + } + */ + /* + if (callback) + { + if (_items.Size() == 1) + { + RINOK(callback->SetTotal(NULL, &endPos)); + } + if ((_items.Size() & 0x3FF) == 0) + { + const UInt64 numFiles = _items.Size(); + RINOK(callback->SetCompleted(&numFiles, &_phySize)); + } + } + */ + } + + /* + if (!_forceCodePage) + { + if (!utf8_OK) + _curCodePage = k_DefaultCodePage; + } + */ + _openCodePage = _curCodePage; + + if (_items.Size() == 0) + { + if (_arc._error != k_ErrorType_OK) + { + _isArc = false; + return S_FALSE; + } + if (!callback) + return S_FALSE; + Z7_DECL_CMyComPtr_QI_FROM( + IArchiveOpenVolumeCallback, + openVolumeCallback, callback) + if (!openVolumeCallback) + return S_FALSE; + NCOM::CPropVariant prop; + if (openVolumeCallback->GetProperty(kpidName, &prop) != S_OK) + return S_FALSE; + if (prop.vt != VT_BSTR) + return S_FALSE; + unsigned len = MyStringLen(prop.bstrVal); + if (len < 4 || MyStringCompareNoCase(prop.bstrVal + len - 4, L".tar") != 0) + return S_FALSE; + } + + _isArc = true; + return S_OK; +} + +Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *openArchiveCallback)) +{ + COM_TRY_BEGIN + // for (int i = 0; i < 10; i++) // for debug + { + Close(); + RINOK(Open2(stream, openArchiveCallback)) + _stream = stream; + } + return S_OK; + COM_TRY_END +} + +Z7_COM7F_IMF(CHandler::OpenSeq(ISequentialInStream *stream)) +{ + Close(); + _seqStream = stream; + _isArc = true; + return S_OK; +} + +Z7_COM7F_IMF(CHandler::Close()) +{ + _isArc = false; + + _arc.Clear(); + + _curIndex = 0; + _latestIsRead = false; + _encodingCharacts.Clear(); + _items.Clear(); + _seqStream.Release(); + _stream.Release(); + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) +{ + *numItems = (_stream ? _items.Size() : (UInt32)(Int32)-1); + return S_OK; +} + +CHandler::CHandler() +{ + // copyCoder = new NCompress::CCopyCoder(); + // copyCoder = copyCoder; + _openCodePage = CP_UTF8; + Init(); +} + +HRESULT CHandler::SkipTo(UInt32 index) +{ + while (_curIndex < index || !_latestIsRead) + { + if (_latestIsRead) + { + const UInt64 packSize = _latestItem.Get_PackSize_Aligned(); + RINOK(copyCoder.Interface()->Code(_seqStream, NULL, &packSize, &packSize, NULL)) + _arc._phySize += copyCoder->TotalSize; + if (copyCoder->TotalSize != packSize) + { + _arc._error = k_ErrorType_UnexpectedEnd; + return S_FALSE; + } + _latestIsRead = false; + _curIndex++; + } + else + { + _arc.SeqStream = _seqStream; + _arc.InStream = NULL; + RINOK(_arc.ReadItem(_latestItem)) + if (!_arc.filled) + { + _arc._phySize_Defined = true; + return E_INVALIDARG; + } + _latestIsRead = true; + } + } + return S_OK; +} + +void CHandler::TarStringToUnicode(const AString &s, NWindows::NCOM::CPropVariant &prop, bool toOs) const +{ + UString dest; + if (_curCodePage == CP_UTF8) + ConvertUTF8ToUnicode(s, dest); + else + MultiByteToUnicodeString2(dest, s, _curCodePage); + if (toOs) + NItemName::ReplaceToOsSlashes_Remove_TailSlash(dest, + true); // useBackslashReplacement + prop = dest; +} + + +// CPaxTime is defined (NumDigits >= 0) +static void PaxTimeToProp(const CPaxTime &pt, NWindows::NCOM::CPropVariant &prop) +{ + UInt64 v; + if (!NTime::UnixTime64_To_FileTime64(pt.Sec, v)) + return; + if (pt.Ns != 0) + v += pt.Ns / 100; + FILETIME ft; + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); + prop.SetAsTimeFrom_FT_Prec_Ns100(ft, + k_PropVar_TimePrec_Base + (unsigned)pt.NumDigits, pt.Ns % 100); +} + + +static void AddSpecCharToString(const char c, AString &s) +{ + if ((Byte)c <= 0x20 || (Byte)c > 127) + { + s.Add_Char('['); + s.Add_Char(GET_HEX_CHAR_LOWER((Byte)c >> 4)); + s.Add_Char(GET_HEX_CHAR_LOWER(c & 15)); + s.Add_Char(']'); + } + else + s.Add_Char(c); +} + +static void AddSpecUInt64(AString &s, const char *name, UInt64 v) +{ + if (v != 0) + { + s.Add_OptSpaced(name); + if (v > 1) + { + s.Add_Colon(); + s.Add_UInt64(v); + } + } +} + +static void AddSpecBools(AString &s, const char *name, bool b1, bool b2) +{ + if (b1) + { + s.Add_OptSpaced(name); + if (b2) + s.Add_Char('*'); + } +} + + +#if 0 +static bool Parse_Attrib_from_SCHILY_fflags(const AString &s, UInt32 &attribRes) +{ + UInt32 attrib = 0; + attribRes = attrib; + unsigned pos = 0; + while (pos < s.Len()) + { + int pos2 = s.Find(',', pos); + if (pos2 < 0) + pos2 = (int)s.Len(); + const AString str = s.Mid(pos, (unsigned)pos2 - pos); + if (str.IsEqualTo("hidden")) attrib |= FILE_ATTRIBUTE_HIDDEN; + else if (str.IsEqualTo("rdonly")) attrib |= FILE_ATTRIBUTE_READONLY; + else if (str.IsEqualTo("system")) attrib |= FILE_ATTRIBUTE_SYSTEM; + else + return false; + pos = (unsigned)pos2 + 1; + } + attribRes = attrib; + return true; +} +#endif + + +Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value)) +{ + COM_TRY_BEGIN + NCOM::CPropVariant prop; + + const CItemEx *item; + if (_stream) + item = &_items[index]; + else + { + if (index < _curIndex) + return E_INVALIDARG; + else + { + RINOK(SkipTo(index)) + item = &_latestItem; + } + } + + switch (propID) + { + case kpidPath: TarStringToUnicode(item->Name, prop, true); break; + case kpidIsDir: prop = item->IsDir(); break; + case kpidSize: prop = item->Get_UnpackSize(); break; + case kpidPackSize: prop = item->Get_PackSize_Aligned(); break; + case kpidMTime: + { + /* + // for debug: + PropVariant_SetFrom_UnixTime(prop, 1 << 30); + prop.wReserved1 = k_PropVar_TimePrec_Base + 1; + prop.wReserved2 = 12; + break; + */ + + if (item->PaxTimes.MTime.IsDefined()) + PaxTimeToProp(item->PaxTimes.MTime, prop); + else + // if (item->MTime != 0) + { + // we allow (item->MTime == 0) + FILETIME ft; + if (NTime::UnixTime64_To_FileTime(item->MTime, ft)) + { + unsigned prec = k_PropVar_TimePrec_Unix; + if (item->MTime_IsBin) + { + /* we report here that it's Int64-UnixTime + instead of basic UInt32-UnixTime range */ + prec = k_PropVar_TimePrec_Base; + } + prop.SetAsTimeFrom_FT_Prec(ft, prec); + } + } + break; + } + case kpidATime: + if (item->PaxTimes.ATime.IsDefined()) + PaxTimeToProp(item->PaxTimes.ATime, prop); + break; + case kpidCTime: + if (item->PaxTimes.CTime.IsDefined()) + PaxTimeToProp(item->PaxTimes.CTime, prop); + break; + case kpidPosixAttrib: prop = item->Get_Combined_Mode(); break; + + // kpidAttrib has priority over kpidPosixAttrib in 7-Zip. + // but if we want kpidPosixAttrib priority for TAR, we disable kpidAttrib. +#if 0 + case kpidAttrib: + { + if (!item->SCHILY_fflags.IsEmpty()) + { + UInt32 attrib = 0; + if (Parse_Attrib_from_SCHILY_fflags(item->SCHILY_fflags, attrib)) + { + if (attrib != 0) + { + if (item->IsDir()) + attrib |= FILE_ATTRIBUTE_DIRECTORY; + attrib |= ((UInt32)item->Get_Combined_Mode() << 16) | 0x8000; // FILE_ATTRIBUTE_UNIX_EXTENSION; + prop = attrib; + } + } + } + break; + } +#endif + + case kpidUser: + if (!item->User.IsEmpty()) + TarStringToUnicode(item->User, prop); + break; + case kpidGroup: + if (!item->Group.IsEmpty()) + TarStringToUnicode(item->Group, prop); + break; + + case kpidUserId: + // if (item->UID != 0) + prop = (UInt32)item->UID; + break; + case kpidGroupId: + // if (item->GID != 0) + prop = (UInt32)item->GID; + break; + + case kpidDeviceMajor: + if (item->DeviceMajor_Defined) + // if (item->DeviceMajor != 0) + prop = (UInt32)item->DeviceMajor; + break; + + case kpidDeviceMinor: + if (item->DeviceMinor_Defined) + // if (item->DeviceMinor != 0) + prop = (UInt32)item->DeviceMinor; + break; + /* + case kpidDevice: + if (item->DeviceMajor_Defined) + if (item->DeviceMinor_Defined) + prop = (UInt64)MY_dev_makedev(item->DeviceMajor, item->DeviceMinor); + break; + */ + + case kpidSymLink: + if (item->Is_SymLink()) + if (!item->LinkName.IsEmpty()) + TarStringToUnicode(item->LinkName, prop); + break; + case kpidHardLink: + if (item->Is_HardLink()) + if (!item->LinkName.IsEmpty()) + TarStringToUnicode(item->LinkName, prop); + break; + + case kpidCharacts: + { + AString s; + { + s.Add_Space_if_NotEmpty(); + AddSpecCharToString(item->LinkFlag, s); + } + if (item->IsMagic_GNU()) + s.Add_OptSpaced("GNU"); + else if (item->IsMagic_Posix_ustar_00()) + s.Add_OptSpaced("POSIX"); + else + { + s.Add_Space_if_NotEmpty(); + for (unsigned i = 0; i < sizeof(item->Magic); i++) + AddSpecCharToString(item->Magic[i], s); + } + + if (item->IsSignedChecksum) + s.Add_OptSpaced("SignedChecksum"); + + if (item->Prefix_WasUsed) + s.Add_OptSpaced(k_Characts_Prefix); + + s.Add_OptSpaced(item->EncodingCharacts.GetCharactsString()); + + // AddSpecUInt64(s, "LongName", item->Num_LongName_Records); + // AddSpecUInt64(s, "LongLink", item->Num_LongLink_Records); + AddSpecBools(s, "LongName", item->LongName_WasUsed, item->LongName_WasUsed_2); + AddSpecBools(s, "LongLink", item->LongLink_WasUsed, item->LongLink_WasUsed_2); + + if (item->MTime_IsBin) + s.Add_OptSpaced("bin_mtime"); + if (item->PackSize_IsBin) + s.Add_OptSpaced("bin_psize"); + if (item->Size_IsBin) + s.Add_OptSpaced("bin_size"); + + AddSpecUInt64(s, "PAX", item->Num_Pax_Records); + + if (item->PaxTimes.MTime.IsDefined()) s.Add_OptSpaced("mtime"); + if (item->PaxTimes.ATime.IsDefined()) s.Add_OptSpaced("atime"); + if (item->PaxTimes.CTime.IsDefined()) s.Add_OptSpaced("ctime"); + + if (item->pax_path_WasUsed) + s.Add_OptSpaced("pax_path"); + if (item->pax_link_WasUsed) + s.Add_OptSpaced("pax_linkpath"); + if (item->pax_size_WasUsed) + s.Add_OptSpaced("pax_size"); + if (!item->SCHILY_fflags.IsEmpty()) + { + s.Add_OptSpaced("SCHILY.fflags="); + s += item->SCHILY_fflags; + } + if (item->IsThereWarning()) + s.Add_OptSpaced("WARNING"); + if (item->HeaderError) + s.Add_OptSpaced("ERROR"); + if (item->Pax_Error) + s.Add_OptSpaced("PAX_error"); + if (!item->PaxExtra.RawLines.IsEmpty()) + s.Add_OptSpaced("PAX_unsupported_line"); + if (item->Pax_Overflow) + s.Add_OptSpaced("PAX_overflow"); + if (!s.IsEmpty()) + prop = s; + break; + } + case kpidComment: + { + AString s; + item->PaxExtra.Print_To_String(s); + if (!s.IsEmpty()) + prop = s; + break; + } + // case kpidHeadersSize: prop = item->HeaderSize; break; // for debug + // case kpidOffset: prop = item->HeaderPos; break; // for debug + default: break; + } + prop.Detach(value); + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback)) +{ + COM_TRY_BEGIN + ISequentialInStream *stream = _seqStream; + const bool seqMode = (_stream == NULL); + if (!seqMode) + stream = _stream; + + const bool allFilesMode = (numItems == (UInt32)(Int32)-1); + if (allFilesMode) + numItems = _items.Size(); + if (_stream && numItems == 0) + return S_OK; + UInt64 totalSize = 0; + UInt32 i; + for (i = 0; i < numItems; i++) + totalSize += _items[allFilesMode ? i : indices[i]].Get_UnpackSize(); + RINOK(extractCallback->SetTotal(totalSize)) + + UInt64 totalPackSize; + totalSize = totalPackSize = 0; + + CMyComPtr2_Create lps; + lps->Init(extractCallback, false); + CMyComPtr2_Create inStream; + inStream->SetStream(stream); + CMyComPtr2_Create outStreamSpec; + + for (i = 0; ; i++) + { + lps->InSize = totalPackSize; + lps->OutSize = totalSize; + RINOK(lps->SetCur()) + if (i >= numItems && !seqMode) + break; + const UInt32 index = allFilesMode ? i : indices[i]; + const CItemEx *item; + if (seqMode) + { + const HRESULT res = SkipTo(index); + if (res == E_INVALIDARG) + break; + RINOK(res) + item = &_latestItem; + } + else + item = &_items[index]; + + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + CMyComPtr realOutStream; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) + const UInt64 unpackSize = item->Get_UnpackSize(); + totalSize += unpackSize; + totalPackSize += item->Get_PackSize_Aligned(); + if (item->IsDir()) + { + RINOK(extractCallback->PrepareOperation(askMode)) + // realOutStream.Release(); + RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) + continue; + } + bool skipMode = false; + if (!testMode && !realOutStream) + { + if (!seqMode) + { + /* + // GetStream() creates link. + // so we can show extracting info in GetStream() instead + if (item->Is_HardLink() || + item->Is_SymLink()) + { + RINOK(extractCallback->PrepareOperation(askMode)) + RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) + } + */ + continue; + } + skipMode = true; + askMode = NExtract::NAskMode::kSkip; + } + RINOK(extractCallback->PrepareOperation(askMode)) + + outStreamSpec->SetStream(realOutStream); + realOutStream.Release(); + outStreamSpec->Init(skipMode ? 0 : unpackSize, true); + + Int32 opRes = NExtract::NOperationResult::kOK; + CMyComPtr inStream2; + if (!item->Is_Sparse()) + inStream2 = inStream; + else + { + GetStream(index, &inStream2); + if (!inStream2) + return E_FAIL; + } + + { + if (item->Is_SymLink()) + { + RINOK(WriteStream(outStreamSpec, (const char *)item->LinkName, item->LinkName.Len())) + } + else + { + if (!seqMode) + { + RINOK(InStream_SeekSet(_stream, item->Get_DataPos())) + } + inStream->Init(item->Get_PackSize_Aligned()); + RINOK(copyCoder.Interface()->Code(inStream2, outStreamSpec, NULL, NULL, lps)) + } + if (outStreamSpec->GetRem() != 0) + opRes = NExtract::NOperationResult::kDataError; + } + if (seqMode) + { + _latestIsRead = false; + _curIndex++; + } + outStreamSpec->ReleaseStream(); + RINOK(extractCallback->SetOperationResult(opRes)) + } + return S_OK; + COM_TRY_END +} + + +Z7_CLASS_IMP_IInStream( + CSparseStream +) + UInt64 _phyPos; + UInt64 _virtPos; + bool _needStartSeek; + +public: + CHandler *Handler; + CMyComPtr HandlerRef; + unsigned ItemIndex; + CRecordVector PhyOffsets; + + void Init() + { + _virtPos = 0; + _phyPos = 0; + _needStartSeek = true; + } +}; + + +Z7_COM7F_IMF(CSparseStream::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + if (processedSize) + *processedSize = 0; + if (size == 0) + return S_OK; + const CItemEx &item = Handler->_items[ItemIndex]; + if (_virtPos >= item.Size) + return S_OK; + { + UInt64 rem = item.Size - _virtPos; + if (size > rem) + size = (UInt32)rem; + } + + HRESULT res = S_OK; + + if (item.SparseBlocks.IsEmpty()) + memset(data, 0, size); + else + { + unsigned left = 0, right = item.SparseBlocks.Size(); + for (;;) + { + const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + if (mid == left) + break; + if (_virtPos < item.SparseBlocks[mid].Offset) + right = mid; + else + left = mid; + } + + const CSparseBlock &sb = item.SparseBlocks[left]; + UInt64 relat = _virtPos - sb.Offset; + + if (_virtPos >= sb.Offset && relat < sb.Size) + { + UInt64 rem = sb.Size - relat; + if (size > rem) + size = (UInt32)rem; + UInt64 phyPos = PhyOffsets[left] + relat; + if (_needStartSeek || _phyPos != phyPos) + { + RINOK(InStream_SeekSet(Handler->_stream, (item.Get_DataPos() + phyPos))) + _needStartSeek = false; + _phyPos = phyPos; + } + res = Handler->_stream->Read(data, size, &size); + _phyPos += size; + } + else + { + UInt64 next = item.Size; + if (_virtPos < sb.Offset) + next = sb.Offset; + else if (left + 1 < item.SparseBlocks.Size()) + next = item.SparseBlocks[left + 1].Offset; + UInt64 rem = next - _virtPos; + if (size > rem) + size = (UInt32)rem; + memset(data, 0, size); + } + } + + _virtPos += size; + if (processedSize) + *processedSize = size; + return res; +} + +Z7_COM7F_IMF(CSparseStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) +{ + switch (seekOrigin) + { + case STREAM_SEEK_SET: break; + case STREAM_SEEK_CUR: offset += _virtPos; break; + case STREAM_SEEK_END: offset += Handler->_items[ItemIndex].Size; break; + default: return STG_E_INVALIDFUNCTION; + } + if (offset < 0) + return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; + _virtPos = (UInt64)offset; + if (newPosition) + *newPosition = _virtPos; + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) +{ + COM_TRY_BEGIN + + const CItemEx &item = _items[index]; + + if (item.Is_Sparse()) + { + CSparseStream *streamSpec = new CSparseStream; + CMyComPtr streamTemp = streamSpec; + streamSpec->Init(); + streamSpec->Handler = this; + streamSpec->HandlerRef = (IInArchive *)this; + streamSpec->ItemIndex = index; + streamSpec->PhyOffsets.Reserve(item.SparseBlocks.Size()); + UInt64 offs = 0; + FOR_VECTOR(i, item.SparseBlocks) + { + const CSparseBlock &sb = item.SparseBlocks[i]; + streamSpec->PhyOffsets.AddInReserved(offs); + offs += sb.Size; + } + *stream = streamTemp.Detach(); + return S_OK; + } + + if (item.Is_SymLink()) + { + Create_BufInStream_WithReference((const Byte *)(const char *)item.LinkName, item.LinkName.Len(), (IInArchive *)this, stream); + return S_OK; + } + + return CreateLimitedInStream(_stream, item.Get_DataPos(), item.PackSize, stream); + + COM_TRY_END +} + + +void CHandler::Init() +{ + _forceCodePage = false; + _curCodePage = _specifiedCodePage = CP_UTF8; // CP_OEMCP; + _posixMode = false; + _posixMode_WasForced = false; + // TimeOptions.Clear(); + _handlerTimeOptions.Init(); + // _handlerTimeOptions.Write_MTime.Val = true; // it's default already +} + + +Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)) +{ + Init(); + + for (UInt32 i = 0; i < numProps; i++) + { + UString name = names[i]; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + const PROPVARIANT &prop = values[i]; + + if (name[0] == L'x') + { + // some clients write 'x' property. So we support it + UInt32 level = 0; + RINOK(ParsePropToUInt32(name.Ptr(1), prop, level)) + } + else if (name.IsEqualTo("cp")) + { + UInt32 cp = CP_OEMCP; + RINOK(ParsePropToUInt32(L"", prop, cp)) + _forceCodePage = true; + _curCodePage = _specifiedCodePage = cp; + } + else if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + } + else if (name.IsPrefixedBy_Ascii_NoCase("memuse")) + { + } + else if (name.IsEqualTo("m")) + { + if (prop.vt != VT_BSTR) + return E_INVALIDARG; + const UString s = prop.bstrVal; + if (s.IsEqualTo_Ascii_NoCase("pax") || + s.IsEqualTo_Ascii_NoCase("posix")) + _posixMode = true; + else if (s.IsEqualTo_Ascii_NoCase("gnu")) + _posixMode = false; + else + return E_INVALIDARG; + _posixMode_WasForced = true; + } + else + { + /* + if (name.IsPrefixedBy_Ascii_NoCase("td")) + { + name.Delete(0, 3); + if (prop.vt == VT_EMPTY) + { + if (name.IsEqualTo_Ascii_NoCase("n")) + { + // TimeOptions.UseNativeDigits = true; + } + else if (name.IsEqualTo_Ascii_NoCase("r")) + { + // TimeOptions.RemoveZeroDigits = true; + } + else + return E_INVALIDARG; + } + else + { + UInt32 numTimeDigits = 0; + RINOK(ParsePropToUInt32(name, prop, numTimeDigits)); + TimeOptions.NumDigits_WasForced = true; + TimeOptions.NumDigits = numTimeDigits; + } + } + */ + bool processed = false; + RINOK(_handlerTimeOptions.Parse(name, prop, processed)) + if (processed) + continue; + return E_INVALIDARG; + } + } + return S_OK; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandler.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandler.h new file mode 100644 index 0000000..f1ef3b6 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandler.h @@ -0,0 +1,57 @@ +// TarHandler.h + +#ifndef ZIP7_INC_TAR_HANDLER_H +#define ZIP7_INC_TAR_HANDLER_H + +#include "../../../Common/MyCom.h" + +#include "../../Compress/CopyCoder.h" + +#include "../Common/HandlerOut.h" + +#include "TarIn.h" + +namespace NArchive { +namespace NTar { + +Z7_CLASS_IMP_CHandler_IInArchive_4( + IArchiveOpenSeq + , IInArchiveGetStream + , ISetProperties + , IOutArchive +) +public: + CObjectVector _items; + CMyComPtr _stream; + CMyComPtr _seqStream; +private: + bool _isArc; + bool _posixMode_WasForced; + bool _posixMode; + bool _forceCodePage; + UInt32 _specifiedCodePage; + UInt32 _curCodePage; + UInt32 _openCodePage; + // CTimeOptions TimeOptions; + CHandlerTimeOptions _handlerTimeOptions; + CEncodingCharacts _encodingCharacts; + + UInt32 _curIndex; + bool _latestIsRead; + CItemEx _latestItem; + + CArchive _arc; + + CMyComPtr2_Create copyCoder; + + HRESULT Open2(IInStream *stream, IArchiveOpenCallback *callback); + HRESULT SkipTo(UInt32 index); + void TarStringToUnicode(const AString &s, NWindows::NCOM::CPropVariant &prop, bool toOs = false) const; +public: + void Init(); + CHandler(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandlerOut.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandlerOut.cpp new file mode 100644 index 0000000..bcbb35d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHandlerOut.cpp @@ -0,0 +1,332 @@ +// TarHandlerOut.cpp + +#include "StdAfx.h" + +// #include + +#include "../../../Common/ComTry.h" +#include "../../../Common/MyLinux.h" +#include "../../../Common/StringConvert.h" + +#include "../../../Windows/TimeUtils.h" + +#include "../Common/ItemNameUtils.h" + +#include "TarHandler.h" +#include "TarUpdate.h" + +using namespace NWindows; + +namespace NArchive { +namespace NTar { + +Z7_COM7F_IMF(CHandler::GetFileTimeType(UInt32 *type)) +{ + UInt32 t = NFileTimeType::kUnix; + const UInt32 prec = _handlerTimeOptions.Prec; + if (prec != (UInt32)(Int32)-1) + { + t = NFileTimeType::kWindows; + if (prec == k_PropVar_TimePrec_0 || + prec == k_PropVar_TimePrec_100ns) + t = NFileTimeType::kWindows; + else if (prec == k_PropVar_TimePrec_HighPrec) + t = k_PropVar_TimePrec_1ns; + else if (prec >= k_PropVar_TimePrec_Base) + t = prec; + } + // 7-Zip before 22.00 fails, if unknown typeType. + *type = t; + return S_OK; +} + + +void Get_AString_From_UString(const UString &s, AString &res, + UINT codePage, unsigned utfFlags) +{ + if (codePage == CP_UTF8) + ConvertUnicodeToUTF8_Flags(s, res, utfFlags); + else + UnicodeStringToMultiByte2(res, s, codePage); +} + + +HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, AString &res, + UINT codePage, unsigned utfFlags, bool convertSlash) +{ + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(index, propId, &prop)) + + if (prop.vt == VT_BSTR) + { + UString s = prop.bstrVal; + if (convertSlash) + NItemName::ReplaceSlashes_OsToUnix(s); + Get_AString_From_UString(s, res, codePage, utfFlags); + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + + return S_OK; +} + + +// sort old files with original order. + +static int CompareUpdateItems(void *const *p1, void *const *p2, void *) +{ + const CUpdateItem &u1 = *(*((const CUpdateItem *const *)p1)); + const CUpdateItem &u2 = *(*((const CUpdateItem *const *)p2)); + if (!u1.NewProps) + { + if (u2.NewProps) + return -1; + return MyCompare(u1.IndexInArc, u2.IndexInArc); + } + if (!u2.NewProps) + return 1; + return MyCompare(u1.IndexInClient, u2.IndexInClient); +} + + +static HRESULT GetTime(UInt32 i, UInt32 pid, IArchiveUpdateCallback *callback, + CPaxTime &pt) +{ + pt.Clear(); + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, pid, &prop)) + return Prop_To_PaxTime(prop, pt); +} + + +/* +static HRESULT GetDevice(IArchiveUpdateCallback *callback, UInt32 i, + UInt32 &majo, UInt32 &mino, bool &majo_defined, bool &mino_defined) +{ + NWindows::NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidDevice, &prop)); + if (prop.vt == VT_EMPTY) + return S_OK; + if (prop.vt != VT_UI8) + return E_INVALIDARG; + { + const UInt64 v = prop.uhVal.QuadPart; + majo = MY_dev_major(v); + mino = MY_dev_minor(v); + majo_defined = true; + mino_defined = true; + } + return S_OK; +} +*/ + +static HRESULT GetDevice(IArchiveUpdateCallback *callback, UInt32 i, + UInt32 pid, UInt32 &id, bool &defined) +{ + defined = false; + NWindows::NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, pid, &prop)) + if (prop.vt == VT_EMPTY) + return S_OK; + if (prop.vt == VT_UI4) + { + id = prop.ulVal; + defined = true; + return S_OK; + } + return E_INVALIDARG; +} + + +static HRESULT GetUser(IArchiveUpdateCallback *callback, UInt32 i, + UInt32 pidName, UInt32 pidId, AString &name, UInt32 &id, + UINT codePage, unsigned utfFlags) +{ + // printf("\ncallback->GetProperty(i, pidId, &prop))\n"); + + bool isSet = false; + { + NWindows::NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, pidId, &prop)) + if (prop.vt == VT_UI4) + { + isSet = true; + id = prop.ulVal; + // printf("\ncallback->GetProperty(i, pidId, &prop)); = %d \n", (unsigned)id); + name.Empty(); + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + } + { + NWindows::NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, pidName, &prop)) + if (prop.vt == VT_BSTR) + { + const UString s = prop.bstrVal; + Get_AString_From_UString(s, name, codePage, utfFlags); + if (!isSet) + id = 0; + } + else if (prop.vt == VT_UI4) + { + id = prop.ulVal; + name.Empty(); + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + } + return S_OK; +} + + + +Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *callback)) +{ + COM_TRY_BEGIN + + if ((_stream && (_arc._error != k_ErrorType_OK || _arc._is_Warning + /* || _isSparse */ + )) || _seqStream) + return E_NOTIMPL; + CObjectVector updateItems; + const UINT codePage = (_forceCodePage ? _specifiedCodePage : _openCodePage); + const unsigned utfFlags = g_Unicode_To_UTF8_Flags; + /* + // for debug only: + unsigned utfFlags = 0; + utfFlags |= Z7_UTF_FLAG_TO_UTF8_EXTRACT_BMP_ESCAPE; + utfFlags |= Z7_UTF_FLAG_TO_UTF8_SURROGATE_ERROR; + */ + + for (UInt32 i = 0; i < numItems; i++) + { + CUpdateItem ui; + Int32 newData; + Int32 newProps; + UInt32 indexInArc; + + if (!callback) + return E_FAIL; + + RINOK(callback->GetUpdateItemInfo(i, &newData, &newProps, &indexInArc)) + + ui.NewProps = IntToBool(newProps); + ui.NewData = IntToBool(newData); + ui.IndexInArc = (int)indexInArc; + ui.IndexInClient = i; + + if (IntToBool(newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidIsDir, &prop)) + if (prop.vt == VT_EMPTY) + ui.IsDir = false; + else if (prop.vt != VT_BOOL) + return E_INVALIDARG; + else + ui.IsDir = (prop.boolVal != VARIANT_FALSE); + } + + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidPosixAttrib, &prop)) + if (prop.vt == VT_EMPTY) + ui.Mode = + MY_LIN_S_IRWXO + | MY_LIN_S_IRWXG + | MY_LIN_S_IRWXU + | (ui.IsDir ? MY_LIN_S_IFDIR : MY_LIN_S_IFREG); + else if (prop.vt != VT_UI4) + return E_INVALIDARG; + else + ui.Mode = prop.ulVal; + // 21.07 : we clear high file type bits as GNU TAR. + // we will clear it later + // ui.Mode &= ~(UInt32)MY_LIN_S_IFMT; + } + + if (_handlerTimeOptions.Write_MTime.Val) + RINOK(GetTime(i, kpidMTime, callback, ui.PaxTimes.MTime)) + if (_handlerTimeOptions.Write_ATime.Val) + RINOK(GetTime(i, kpidATime, callback, ui.PaxTimes.ATime)) + if (_handlerTimeOptions.Write_CTime.Val) + RINOK(GetTime(i, kpidCTime, callback, ui.PaxTimes.CTime)) + + RINOK(GetPropString(callback, i, kpidPath, ui.Name, codePage, utfFlags, true)) + if (ui.IsDir && !ui.Name.IsEmpty() && ui.Name.Back() != '/') + ui.Name.Add_Slash(); + // ui.Name.Add_Slash(); // for debug + + if (_posixMode) + { + RINOK(GetDevice(callback, i, kpidDeviceMajor, ui.DeviceMajor, ui.DeviceMajor_Defined)) + RINOK(GetDevice(callback, i, kpidDeviceMinor, ui.DeviceMinor, ui.DeviceMinor_Defined)) + } + + RINOK(GetUser(callback, i, kpidUser, kpidUserId, ui.User, ui.UID, codePage, utfFlags)) + RINOK(GetUser(callback, i, kpidGroup, kpidGroupId, ui.Group, ui.GID, codePage, utfFlags)) + } + + if (IntToBool(newData)) + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidSize, &prop)) + if (prop.vt != VT_UI8) + return E_INVALIDARG; + ui.Size = prop.uhVal.QuadPart; + /* + // now we support GNU extension for big files + if (ui.Size >= ((UInt64)1 << 33)) + return E_INVALIDARG; + */ + } + + updateItems.Add(ui); + } + + if (_arc._are_Pax_Items) + { + // we restore original order of files, if there are pax items + updateItems.Sort(CompareUpdateItems, NULL); + } + + CUpdateOptions options; + + options.CodePage = codePage; + options.UtfFlags = utfFlags; + options.PosixMode = _posixMode; + + options.Write_MTime = _handlerTimeOptions.Write_MTime; + options.Write_ATime = _handlerTimeOptions.Write_ATime; + options.Write_CTime = _handlerTimeOptions.Write_CTime; + + // options.TimeOptions = TimeOptions; + + const UInt32 prec = _handlerTimeOptions.Prec; + if (prec != (UInt32)(Int32)-1) + { + unsigned numDigits = 0; + if (prec == 0) + numDigits = 7; + else if (prec == k_PropVar_TimePrec_HighPrec + || prec >= k_PropVar_TimePrec_1ns) + numDigits = 9; + else if (prec >= k_PropVar_TimePrec_Base) + numDigits = prec - k_PropVar_TimePrec_Base; + options.TimeOptions.NumDigitsMax = numDigits; + // options.TimeOptions.RemoveZeroMode = + // k_PaxTimeMode_DontRemoveZero; // pure for debug + // k_PaxTimeMode_RemoveZero_if_PureSecondOnly; // optimized code + // k_PaxTimeMode_RemoveZero_Always; // original pax code + } + + return UpdateArchive(_stream, outStream, _items, updateItems, + options, callback); + + COM_TRY_END +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHeader.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHeader.cpp new file mode 100644 index 0000000..deae357 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHeader.cpp @@ -0,0 +1,99 @@ +// Archive/TarHeader.cpp + +#include "StdAfx.h" + +#include "TarHeader.h" + +namespace NArchive { +namespace NTar { +namespace NFileHeader { + + const char * const kLongLink = "././@LongLink"; + const char * const kLongLink2 = "@LongLink"; + + // The magic field is filled with this if uname and gname are valid. + namespace NMagic + { + // const char * const kUsTar = "ustar"; // 5 chars + // const char * const kGNUTar = "GNUtar "; // 7 chars and a null + // const char * const kEmpty = "\0\0\0\0\0\0\0\0"; + // 7-Zip used kUsTar_00 before 21.07: + const char k_Posix_ustar_00[8] = { 'u', 's', 't', 'a', 'r', 0, '0', '0' } ; + // GNU TAR uses such header: + const char k_GNU_ustar[8] = { 'u', 's', 't', 'a', 'r', ' ', ' ', 0 } ; + } + +/* +pre-POSIX.1-1988 (i.e. v7) tar header: +----- +Link indicator: +'0' or 0 : Normal file +'1' : Hard link +'2' : Symbolic link +Some pre-POSIX.1-1988 tar implementations indicated a directory by having +a trailing slash (/) in the name. + +Numeric values : octal with leading zeroes. +For historical reasons, a final NUL or space character should also be used. +Thus only 11 octal digits can be stored from 12 bytes field. + +2001 star : introduced a base-256 coding that is indicated by +setting the high-order bit of the leftmost byte of a numeric field. +GNU-tar and BSD-tar followed this idea. + +versions of tar from before the first POSIX standard from 1988 +pad the values with spaces instead of zeroes. + +UStar +----- +UStar (Unix Standard TAR) : POSIX IEEE P1003.1 : 1988. + 257 signature: "ustar", 0, "00" + 265 32 Owner user name + 297 32 Owner group name + 329 8 Device major number + 337 8 Device minor number + 345 155 Filename prefix + +POSIX.1-2001/pax +---- +format is known as extended tar format or pax format +vendor-tagged vendor-specific enhancements. +tags Defined by the POSIX standard: + atime, mtime, path, linkpath, uname, gname, size, uid, gid, ... + + +PAX EXTENSION +----------- +Hard links +A further difference from the ustar header block is that data blocks +for files of typeflag 1 (hard link) may be included, +which means that the size field may be greater than zero. +Archives created by pax -o linkdata shall include these data +blocks with the hard links. +* + +compatiblity +------------ + 7-Zip 16.03 supports "PaxHeader/" + 7-Zip 20.01 supports "PaxHeaders.X/" with optional "./" + 7-Zip 21.02 supports "@PaxHeader" with optional "./" "./" + + GNU tar --format=posix uses "PaxHeaders/" in folder of file + + +GNU TAR format +============== +v7 - Unix V7 +oldgnu - GNU tar <=1.12 : writes zero in last character in name +gnu - GNU tar 1.13 : doesn't write zero in last character in name + as 7-zip 21.07 +ustar - POSIX.1-1988 +posix (pax) - POSIX.1-2001 + + gnu tar: + if (S_ISCHR (st->stat.st_mode) || S_ISBLK (st->stat.st_mode)) { + major_t devmajor = major (st->stat.st_rdev); + minor_t devminor = minor (st->stat.st_rdev); } +*/ + +}}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHeader.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHeader.h new file mode 100644 index 0000000..aeccd28 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarHeader.h @@ -0,0 +1,90 @@ +// Archive/TarHeader.h + +#ifndef ZIP7_INC_ARCHIVE_TAR_HEADER_H +#define ZIP7_INC_ARCHIVE_TAR_HEADER_H + +#include "../../../Common/MyTypes.h" + +namespace NArchive { +namespace NTar { + +namespace NFileHeader +{ + const unsigned kRecordSize = 512; + const unsigned kNameSize = 100; + const unsigned kUserNameSize = 32; + const unsigned kGroupNameSize = 32; + const unsigned kPrefixSize = 155; + + const unsigned kUstarMagic_Offset = 257; + + /* + struct CHeader + { + char Name[kNameSize]; + char Mode[8]; + char UID[8]; + char GID[8]; + char Size[12]; + char ModificationTime[12]; + char CheckSum[8]; + char LinkFlag; + char LinkName[kNameSize]; + char Magic[8]; + char UserName[kUserNameSize]; + char GroupName[kGroupNameSize]; + char DeviceMajor[8]; + char DeviceMinor[8]; + char Prefix[155]; + }; + union CRecord + { + CHeader Header; + Byte Padding[kRecordSize]; + }; + */ + + namespace NLinkFlag + { + const char kOldNormal = 0; // Normal disk file, Unix compatible + const char kNormal = '0'; // Normal disk file + const char kHardLink = '1'; // Link to previously dumped file + const char kSymLink = '2'; // Symbolic link + const char kCharacter = '3'; // Character special file + const char kBlock = '4'; // Block special file + const char kDirectory = '5'; // Directory + const char kFIFO = '6'; // FIFO special file + const char kContiguous = '7'; // Contiguous file + const char kGnu_LongLink = 'K'; + const char kGnu_LongName = 'L'; + const char kSparse = 'S'; + const char kLabel = 'V'; + const char kPax = 'x'; // Extended header with meta data for the next file in the archive (POSIX.1-2001) + const char kPax_2 = 'X'; + const char kGlobal = 'g'; // Global extended header with meta data (POSIX.1-2001) + const char kDumpDir = 'D'; /* GNUTYPE_DUMPDIR. + data: list of files created by the --incremental (-G) option + Each file name is preceded by either + - 'Y' (file should be in this archive) + - 'N' (file is a directory, or is not stored in the archive.) + Each file name is terminated by a null + an additional null after + the last file name. */ + // 'A'-'Z' Vendor specific extensions (POSIX.1-1988) + } + + extern const char * const kLongLink; // = "././@LongLink"; + extern const char * const kLongLink2; // = "@LongLink"; + + namespace NMagic + { + // extern const char * const kUsTar; // = "ustar"; // 5 chars + // extern const char * const kGNUTar; // = "GNUtar "; // 7 chars and a null + // extern const char * const kEmpty; // = "\0\0\0\0\0\0\0\0" + extern const char k_Posix_ustar_00[8]; + extern const char k_GNU_ustar[8]; + } +} + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarIn.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarIn.cpp new file mode 100644 index 0000000..22b8902 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarIn.cpp @@ -0,0 +1,1150 @@ +// TarIn.cpp + +#include "StdAfx.h" + +#include "../../../../C/CpuArch.h" + +#include "../../../Common/StringToInt.h" + +#include "../../Common/StreamUtils.h" + +#include "../IArchive.h" + +#include "TarIn.h" + +#define NUM_UNROLL_BYTES (8 * 4) + +Z7_NO_INLINE static bool IsBufNonZero(const void *data, size_t size); +Z7_NO_INLINE static bool IsBufNonZero(const void *data, size_t size) +{ + const Byte *p = (const Byte *)data; + + for (; size != 0 && ((unsigned)(ptrdiff_t)p & (NUM_UNROLL_BYTES - 1)) != 0; size--) + if (*p++ != 0) + return true; + + if (size >= NUM_UNROLL_BYTES) + { + const Byte *lim = p + size; + size &= (NUM_UNROLL_BYTES - 1); + lim -= size; + do + { + if (*(const UInt64 *)(const void *)(p ) != 0) return true; + if (*(const UInt64 *)(const void *)(p + 8 * 1) != 0) return true; + if (*(const UInt64 *)(const void *)(p + 8 * 2) != 0) return true; + if (*(const UInt64 *)(const void *)(p + 8 * 3) != 0) return true; + // if (*(const UInt32 *)(const void *)(p ) != 0) return true; + // if (*(const UInt32 *)(const void *)(p + 4 * 1) != 0) return true; + // if (*(const UInt32 *)(const void *)(p + 4 * 2) != 0) return true; + // if (*(const UInt32 *)(const void *)(p + 4 * 3) != 0) return true; + p += NUM_UNROLL_BYTES; + } + while (p != lim); + } + + for (; size != 0; size--) + if (*p++ != 0) + return true; + + return false; +} + + +namespace NArchive { +namespace NTar { + +static void MyStrNCpy(char *dest, const char *src, unsigned size) +{ + for (unsigned i = 0; i < size; i++) + { + char c = src[i]; + dest[i] = c; + if (c == 0) + break; + } +} + +static bool OctalToNumber(const char *srcString, unsigned size, UInt64 &res, bool allowEmpty = false) +{ + res = 0; + char sz[32]; + MyStrNCpy(sz, srcString, size); + sz[size] = 0; + const char *end; + unsigned i; + for (i = 0; sz[i] == ' '; i++); + if (sz[i] == 0) + return allowEmpty; + res = ConvertOctStringToUInt64(sz + i, &end); + return (*end == ' ' || *end == 0); +} + +static bool OctalToNumber32(const char *srcString, UInt32 &res, bool allowEmpty = false) +{ + const unsigned kSize = 8; + UInt64 res64; + if (!OctalToNumber(srcString, kSize, res64, allowEmpty)) + return false; + res = (UInt32)res64; + return (res64 <= 0xFFFFFFFF); +} + +#define RIF(x) { if (!(x)) return S_OK; } + +static void ReadString(const char *s, unsigned size, AString &result) +{ + result.SetFrom_CalcLen(s, size); +} + +static bool ParseInt64(const char *p, Int64 &val, bool &isBin) +{ + const UInt32 h = GetBe32(p); + val = (Int64)GetBe64(p + 4); + isBin = true; + if (h == (UInt32)1 << 31) + return ((val >> 63) & 1) == 0; + if (h == (UInt32)(Int32)-1) + return ((val >> 63) & 1) != 0; + isBin = false; + UInt64 u; + const bool res = OctalToNumber(p, 12, u); + val = (Int64)u; + return res; +} + +static bool ParseInt64_MTime(const char *p, Int64 &val, bool &isBin) +{ + // rare case tar : ZEROs in Docker-Windows TARs + // rare case tar : spaces + isBin = false; + if (GetUi32(p) != 0) + for (unsigned i = 0; i < 12; i++) + if (p[i] != ' ') + return ParseInt64(p, val, isBin); + val = 0; + return true; +} + +static bool ParseSize(const char *p, UInt64 &val, bool &isBin) +{ + if (GetBe32(p) == (UInt32)1 << 31) + { + // GNU extension + isBin = true; + val = GetBe64(p + 4); + return ((val >> 63) & 1) == 0; + } + isBin = false; + return OctalToNumber(p, 12, val, + true // 20.03: allow empty size for 'V' Label entry + ); +} + +static bool ParseSize(const char *p, UInt64 &val) +{ + bool isBin; + return ParseSize(p, val, isBin); +} + +#define CHECK(x) { if (!(x)) return k_IsArc_Res_NO; } + +API_FUNC_IsArc IsArc_Tar(const Byte *p2, size_t size) +{ + if (size < NFileHeader::kRecordSize) + return k_IsArc_Res_NEED_MORE; + + const char *p = (const char *)p2; + p += NFileHeader::kNameSize; + + UInt32 mode; + // we allow empty Mode value for LongName prefix items + CHECK(OctalToNumber32(p, mode, true)) p += 8; + + // if (!OctalToNumber32(p, item.UID)) item.UID = 0; + p += 8; + // if (!OctalToNumber32(p, item.GID)) item.GID = 0; + p += 8; + + UInt64 packSize; + Int64 time; + UInt32 checkSum; + bool isBin; + CHECK(ParseSize(p, packSize, isBin)) p += 12; + CHECK(ParseInt64_MTime(p, time, isBin)) p += 12; + CHECK(OctalToNumber32(p, checkSum)) + return k_IsArc_Res_YES; +} + + +HRESULT CArchive::GetNextItemReal(CItemEx &item) +{ + char buf[NFileHeader::kRecordSize]; + + error = k_ErrorType_OK; + filled = false; + + bool thereAreEmptyRecords = false; + for (;;) + { + size_t processedSize = NFileHeader::kRecordSize; + RINOK(ReadStream(SeqStream, buf, &processedSize)) + if (processedSize == 0) + { + if (!thereAreEmptyRecords) + error = k_ErrorType_UnexpectedEnd; // "There are no trailing zero-filled records"; + return S_OK; + } + if (processedSize != NFileHeader::kRecordSize) + { + if (!thereAreEmptyRecords) + error = k_ErrorType_UnexpectedEnd; // error = "There is no correct record at the end of archive"; + else + { + /* + if (IsEmptyData(buf, processedSize)) + error = k_ErrorType_UnexpectedEnd; + else + { + // extraReadSize = processedSize; + // error = k_ErrorType_Corrupted; // some data after the end tail zeros + } + */ + } + + return S_OK; + } + if (IsBufNonZero(buf, NFileHeader::kRecordSize)) + break; + item.HeaderSize += NFileHeader::kRecordSize; + thereAreEmptyRecords = true; + if (OpenCallback) + { + RINOK(Progress(item, 0)) + } + } + if (thereAreEmptyRecords) + { + // error = "There are data after end of archive"; + return S_OK; + } + + char *p = buf; + + error = k_ErrorType_Corrupted; + + // ReadString(p, NFileHeader::kNameSize, item.Name); + p += NFileHeader::kNameSize; + + /* + item.Name_CouldBeReduced = + (item.Name.Len() == NFileHeader::kNameSize || + item.Name.Len() == NFileHeader::kNameSize - 1); + */ + + // we allow empty Mode value for LongName prefix items + RIF(OctalToNumber32(p, item.Mode, true)) p += 8; + + if (!OctalToNumber32(p, item.UID)) { item.UID = 0; } p += 8; + if (!OctalToNumber32(p, item.GID)) { item.GID = 0; } p += 8; + + RIF(ParseSize(p, item.PackSize, item.PackSize_IsBin)) + item.Size = item.PackSize; + item.Size_IsBin = item.PackSize_IsBin; + p += 12; + RIF(ParseInt64_MTime(p, item.MTime, item.MTime_IsBin)) p += 12; + + UInt32 checkSum; + RIF(OctalToNumber32(p, checkSum)) + memset(p, ' ', 8); p += 8; + + item.LinkFlag = *p++; + + ReadString(p, NFileHeader::kNameSize, item.LinkName); p += NFileHeader::kNameSize; + + /* + item.LinkName_CouldBeReduced = + (item.LinkName.Len() == NFileHeader::kNameSize || + item.LinkName.Len() == NFileHeader::kNameSize - 1); + */ + + memcpy(item.Magic, p, 8); p += 8; + + ReadString(p, NFileHeader::kUserNameSize, item.User); p += NFileHeader::kUserNameSize; + ReadString(p, NFileHeader::kGroupNameSize, item.Group); p += NFileHeader::kGroupNameSize; + + item.DeviceMajor_Defined = (p[0] != 0); if (item.DeviceMajor_Defined) { RIF(OctalToNumber32(p, item.DeviceMajor)) } p += 8; + item.DeviceMinor_Defined = (p[0] != 0); if (item.DeviceMinor_Defined) { RIF(OctalToNumber32(p, item.DeviceMinor)) } p += 8; + + if (p[0] != 0 + && item.IsMagic_ustar_5chars() + && (item.LinkFlag != 'L' )) + { + item.Prefix_WasUsed = true; + ReadString(p, NFileHeader::kPrefixSize, item.Name); + item.Name.Add_Slash(); + unsigned i; + for (i = 0; i < NFileHeader::kNameSize; i++) + if (buf[i] == 0) + break; + item.Name.AddFrom(buf, i); + } + else + ReadString(buf, NFileHeader::kNameSize, item.Name); + + p += NFileHeader::kPrefixSize; + + if (item.LinkFlag == NFileHeader::NLinkFlag::kHardLink) + { + item.PackSize = 0; + item.Size = 0; + } + + if (item.LinkFlag == NFileHeader::NLinkFlag::kDirectory) + { + // GNU tar ignores Size field, if LinkFlag is kDirectory + // 21.02 : we set PackSize = 0 to be more compatible with GNU tar + item.PackSize = 0; + // item.Size = 0; + } + + /* + TAR standard requires sum of unsigned byte values. + But some old TAR programs use sum of signed byte values. + So we check both values. + */ + // for (int y = 0; y < 100; y++) // for debug + { + UInt32 sum0 = 0; + { + for (unsigned i = 0; i < NFileHeader::kRecordSize; i++) + sum0 += (Byte)buf[i]; + } + if (sum0 != checkSum) + { + Int32 sum = 0; + for (unsigned i = 0; i < NFileHeader::kRecordSize; i++) + sum += (signed char)buf[i]; + if ((UInt32)sum != checkSum) + return S_OK; + item.IsSignedChecksum = true; + } + } + + item.HeaderSize += NFileHeader::kRecordSize; + + if (item.LinkFlag == NFileHeader::NLinkFlag::kSparse) + { + Byte isExtended = (Byte)buf[482]; + if (isExtended != 0 && isExtended != 1) + return S_OK; + RIF(ParseSize(buf + 483, item.Size, item.Size_IsBin)) + UInt64 min = 0; + for (unsigned i = 0; i < 4; i++) + { + p = buf + 386 + 24 * i; + if (GetBe32(p) == 0) + { + if (isExtended != 0) + return S_OK; + break; + } + CSparseBlock sb; + RIF(ParseSize(p, sb.Offset)) + RIF(ParseSize(p + 12, sb.Size)) + item.SparseBlocks.Add(sb); + if (sb.Offset < min || sb.Offset > item.Size) + return S_OK; + if ((sb.Offset & 0x1FF) != 0 || (sb.Size & 0x1FF) != 0) + return S_OK; + min = sb.Offset + sb.Size; + if (min < sb.Offset) + return S_OK; + } + if (min > item.Size) + return S_OK; + + while (isExtended != 0) + { + size_t processedSize = NFileHeader::kRecordSize; + RINOK(ReadStream(SeqStream, buf, &processedSize)) + if (processedSize != NFileHeader::kRecordSize) + { + error = k_ErrorType_UnexpectedEnd; + return S_OK; + } + + item.HeaderSize += NFileHeader::kRecordSize; + + if (OpenCallback) + { + RINOK(Progress(item, 0)) + } + + isExtended = (Byte)buf[21 * 24]; + if (isExtended != 0 && isExtended != 1) + return S_OK; + for (unsigned i = 0; i < 21; i++) + { + p = buf + 24 * i; + if (GetBe32(p) == 0) + { + if (isExtended != 0) + return S_OK; + break; + } + CSparseBlock sb; + RIF(ParseSize(p, sb.Offset)) + RIF(ParseSize(p + 12, sb.Size)) + item.SparseBlocks.Add(sb); + if (sb.Offset < min || sb.Offset > item.Size) + return S_OK; + if ((sb.Offset & 0x1FF) != 0 || (sb.Size & 0x1FF) != 0) + return S_OK; + min = sb.Offset + sb.Size; + if (min < sb.Offset) + return S_OK; + } + } + if (min > item.Size) + return S_OK; + } + + if (item.PackSize >= (UInt64)1 << 63) + return S_OK; + + filled = true; + error = k_ErrorType_OK; + return S_OK; +} + + +HRESULT CArchive::Progress(const CItemEx &item, UInt64 posOffset) +{ + const UInt64 pos = item.Get_DataPos() + posOffset; + if (NumFiles - NumFiles_Prev < (1 << 16) + // && NumRecords - NumRecords_Prev < (1 << 16) + && pos - Pos_Prev < ((UInt32)1 << 28)) + return S_OK; + { + Pos_Prev = pos; + NumFiles_Prev = NumFiles; + // NumRecords_Prev = NumRecords; + // Sleep(100); // for debug + return OpenCallback->SetCompleted(&NumFiles, &pos); + } +} + + +HRESULT CArchive::ReadDataToBuffer(const CItemEx &item, + CTempBuffer &tb, size_t stringLimit) +{ + tb.Init(); + UInt64 packSize = item.Get_PackSize_Aligned(); + if (packSize == 0) + return S_OK; + + UInt64 pos; + + { + size_t size = stringLimit; + if (size > packSize) + size = (size_t)packSize; + tb.Buffer.AllocAtLeast(size); + size_t processedSize = size; + const HRESULT res = ReadStream(SeqStream, tb.Buffer, &processedSize); + pos = processedSize; + if (processedSize != size) + { + error = k_ErrorType_UnexpectedEnd; + return res; + } + RINOK(res) + + packSize -= size; + + size_t i; + const Byte *p = tb.Buffer; + for (i = 0; i < size; i++) + if (p[i] == 0) + break; + if (i >= item.PackSize) + tb.StringSize_IsConfirmed = true; + if (i > item.PackSize) + { + tb.StringSize = (size_t)item.PackSize; + tb.IsNonZeroTail = true; + } + else + { + tb.StringSize = i; + if (i != size) + { + tb.StringSize_IsConfirmed = true; + if (IsBufNonZero(p + i, size - i)) + tb.IsNonZeroTail = true; + } + } + + if (packSize == 0) + return S_OK; + } + + if (InStream) + { + RINOK(InStream->Seek((Int64)packSize, STREAM_SEEK_CUR, NULL)) + return S_OK; + } + const unsigned kBufSize = 1 << 15; + Buffer.AllocAtLeast(kBufSize); + + do + { + if (OpenCallback) + { + RINOK(Progress(item, pos)) + } + + unsigned size = kBufSize; + if (size > packSize) + size = (unsigned)packSize; + size_t processedSize = size; + const HRESULT res = ReadStream(SeqStream, Buffer, &processedSize); + if (processedSize != size) + { + error = k_ErrorType_UnexpectedEnd; + return res; + } + if (!tb.IsNonZeroTail) + { + if (IsBufNonZero(Buffer, size)) + tb.IsNonZeroTail = true; + } + packSize -= size; + pos += size; + } + while (packSize != 0); + return S_OK; +} + + + +struct CPaxInfo: public CPaxTimes +{ + bool DoubleTagError; + bool TagParsingError; + bool UnknownLines_Overflow; + bool Size_Defined; + bool UID_Defined; + bool GID_Defined; + bool Path_Defined; + bool Link_Defined; + bool User_Defined; + bool Group_Defined; + bool SCHILY_fflags_Defined; + + UInt64 Size; + UInt32 UID; + UInt32 GID; + + AString Path; + AString Link; + AString User; + AString Group; + AString UnknownLines; + AString SCHILY_fflags; + + bool ParseID(const AString &val, bool &defined, UInt32 &res) + { + if (defined) + DoubleTagError = true; + if (val.IsEmpty()) + return false; + const char *end2; + res = ConvertStringToUInt32(val.Ptr(), &end2); + if (*end2 != 0) + return false; + defined = true; + return true; + } + + bool ParsePax(const CTempBuffer &tb, bool isFile); +}; + + +static bool ParsePaxTime(const AString &src, CPaxTime &pt, bool &doubleTagError) +{ + if (pt.IsDefined()) + doubleTagError = true; + pt.Clear(); + const char *s = src.Ptr(); + bool isNegative = false; + if (*s == '-') + { + isNegative = true; + s++; + } + const char *end; + { + UInt64 sec = ConvertStringToUInt64(s, &end); + if (s == end) + return false; + if (sec >= ((UInt64)1 << 63)) + return false; + if (isNegative) + sec = (UInt64)-(Int64)sec; + pt.Sec = (Int64)sec; + } + if (*end == 0) + { + pt.Ns = 0; + pt.NumDigits = 0; + return true; + } + if (*end != '.') + return false; + s = end + 1; + + UInt32 ns = 0; + unsigned i; + const unsigned kNsDigits = 9; + for (i = 0;; i++) + { + const char c = s[i]; + if (c == 0) + break; + if (c < '0' || c > '9') + return false; + // we ignore digits after 9 digits as GNU TAR + if (i < kNsDigits) + { + ns *= 10; + ns += (unsigned)(c - '0'); + } + } + pt.NumDigits = (int)(i < kNsDigits ? i : kNsDigits); + while (i < kNsDigits) + { + ns *= 10; + i++; + } + if (isNegative && ns != 0) + { + pt.Sec--; + ns = (UInt32)1000 * 1000 * 1000 - ns; + } + pt.Ns = ns; + return true; +} + + +bool CPaxInfo::ParsePax(const CTempBuffer &tb, bool isFile) +{ + DoubleTagError = false; + TagParsingError = false; + UnknownLines_Overflow = false; + Size_Defined = false; + UID_Defined = false; + GID_Defined = false; + Path_Defined = false; + Link_Defined = false; + User_Defined = false; + Group_Defined = false; + SCHILY_fflags_Defined = false; + + // CPaxTimes::Clear(); + + const char *s = (const char *)(const void *)(const Byte *)tb.Buffer; + size_t rem = tb.StringSize; + + Clear(); + + AString name, val; + + while (rem != 0) + { + unsigned i; + for (i = 0;; i++) + { + if (i > 24 || i >= rem) // we use limitation for size of (size) field + return false; + if (s[i] == ' ') + break; + } + if (i == 0) + return false; + const char *end; + const UInt32 size = ConvertStringToUInt32(s, &end); + const unsigned offset = (unsigned)(end - s) + 1; + if (size > rem + || size <= offset + 1 + || offset != i + 1 + || s[size - 1] != '\n') + return false; + + for (i = offset; i < size; i++) + if (s[i] == 0) + return false; + + for (i = offset; i < size - 1; i++) + if (s[i] == '=') + break; + if (i == size - 1) + return false; + + name.SetFrom(s + offset, i - offset); + val.SetFrom(s + i + 1, (unsigned)(size - 1 - (i + 1))); + + bool parsed = false; + if (isFile) + { + bool isDetectedName = true; + // only lower case (name) is supported + if (name.IsEqualTo("path")) + { + if (Path_Defined) + DoubleTagError = true; + Path = val; + Path_Defined = true; + parsed = true; + } + else if (name.IsEqualTo("linkpath")) + { + if (Link_Defined) + DoubleTagError = true; + Link = val; + Link_Defined = true; + parsed = true; + } + else if (name.IsEqualTo("uname")) + { + if (User_Defined) + DoubleTagError = true; + User = val; + User_Defined = true; + parsed = true; + } + else if (name.IsEqualTo("gname")) + { + if (Group_Defined) + DoubleTagError = true; + Group = val; + Group_Defined = true; + parsed = true; + } + else if (name.IsEqualTo("uid")) + { + parsed = ParseID(val, UID_Defined, UID); + } + else if (name.IsEqualTo("gid")) + { + parsed = ParseID(val, GID_Defined, GID); + } + else if (name.IsEqualTo("size")) + { + if (Size_Defined) + DoubleTagError = true; + Size_Defined = false; + if (!val.IsEmpty()) + { + const char *end2; + Size = ConvertStringToUInt64(val.Ptr(), &end2); + if (*end2 == 0) + { + Size_Defined = true; + parsed = true; + } + } + } + else if (name.IsEqualTo("mtime")) + { parsed = ParsePaxTime(val, MTime, DoubleTagError); } + else if (name.IsEqualTo("atime")) + { parsed = ParsePaxTime(val, ATime, DoubleTagError); } + else if (name.IsEqualTo("ctime")) + { parsed = ParsePaxTime(val, CTime, DoubleTagError); } + else if (name.IsEqualTo("SCHILY.fflags")) + { + if (SCHILY_fflags_Defined) + DoubleTagError = true; + SCHILY_fflags = val; + SCHILY_fflags_Defined = true; + parsed = true; + } + else + isDetectedName = false; + if (isDetectedName && !parsed) + TagParsingError = true; + } + if (!parsed) + { + if (!UnknownLines_Overflow) + { + const unsigned addSize = size - offset; + if (UnknownLines.Len() + addSize < (1 << 16)) + UnknownLines.AddFrom(s + offset, addSize); + else + UnknownLines_Overflow = true; + } + } + + s += size; + rem -= size; + } + return true; +} + + +HRESULT CArchive::ReadItem2(CItemEx &item) +{ + // CItem + + item.SparseBlocks.Clear(); + item.PaxTimes.Clear(); + + // CItemEx + + item.HeaderSize = 0; + item.Num_Pax_Records = 0; + + item.LongName_WasUsed = false; + item.LongName_WasUsed_2 = false; + + item.LongLink_WasUsed = false; + item.LongLink_WasUsed_2 = false; + + item.HeaderError = false; + item.IsSignedChecksum = false; + item.Prefix_WasUsed = false; + + item.Pax_Error = false; + item.Pax_Overflow = false; + item.pax_path_WasUsed = false; + item.pax_link_WasUsed = false; + item.pax_size_WasUsed = false; + + item.PaxExtra.Clear(); + item.SCHILY_fflags.Empty(); + + item.EncodingCharacts.Clear(); + + // CArchive temp variable + + NameBuf.Init(); + LinkBuf.Init(); + PaxBuf.Init(); + PaxBuf_global.Init(); + + UInt64 numExtraRecords = 0; + + for (;;) + { + if (OpenCallback) + { + RINOK(Progress(item, 0)) + } + + RINOK(GetNextItemReal(item)) + + // NumRecords++; + + if (!filled) + { + if (error == k_ErrorType_OK) + if (numExtraRecords != 0 + || item.LongName_WasUsed + || item.LongLink_WasUsed + || item.Num_Pax_Records != 0) + error = k_ErrorType_Corrupted; + return S_OK; + } + if (error != k_ErrorType_OK) + return S_OK; + + numExtraRecords++; + + const char lf = item.LinkFlag; + if (lf == NFileHeader::NLinkFlag::kGnu_LongName || + lf == NFileHeader::NLinkFlag::kGnu_LongLink) + { + // GNU tar ignores item.Name after LinkFlag test + // 22.00 : now we also ignore item.Name here + /* + if (item.Name != NFileHeader::kLongLink && + item.Name != NFileHeader::kLongLink2) + { + break; + // return S_OK; + } + */ + + CTempBuffer *tb = + lf == NFileHeader::NLinkFlag::kGnu_LongName ? + &NameBuf : + &LinkBuf; + + /* + if (item.PackSize > (1 << 29)) + { + // break; + return S_OK; + } + */ + + const unsigned kLongNameSizeMax = (unsigned)1 << 14; + RINOK(ReadDataToBuffer(item, *tb, kLongNameSizeMax)) + if (error != k_ErrorType_OK) + return S_OK; + + if (lf == NFileHeader::NLinkFlag::kGnu_LongName) + { + item.LongName_WasUsed_2 = + item.LongName_WasUsed; + item.LongName_WasUsed = true; + } + else + { + item.LongLink_WasUsed_2 = + item.LongLink_WasUsed; + item.LongLink_WasUsed = true; + } + + if (!tb->StringSize_IsConfirmed) + tb->StringSize = 0; + item.HeaderSize += item.Get_PackSize_Aligned(); + if (tb->StringSize == 0 || + tb->StringSize + 1 != item.PackSize) + item.HeaderError = true; + if (tb->IsNonZeroTail) + item.HeaderError = true; + continue; + } + + if (lf == NFileHeader::NLinkFlag::kGlobal || + lf == NFileHeader::NLinkFlag::kPax || + lf == NFileHeader::NLinkFlag::kPax_2) + { + // GNU tar ignores item.Name after LinkFlag test + // 22.00 : now we also ignore item.Name here + /* + if (item.PackSize > (UInt32)1 << 26) + { + break; // we don't want big PaxBuf files + // return S_OK; + } + */ + const unsigned kParsingPaxSizeMax = (unsigned)1 << 26; + + const bool isStartHeader = (item.HeaderSize == NFileHeader::kRecordSize); + + CTempBuffer *tb = (lf == NFileHeader::NLinkFlag::kGlobal ? &PaxBuf_global : &PaxBuf); + + RINOK(ReadDataToBuffer(item, *tb, kParsingPaxSizeMax)) + if (error != k_ErrorType_OK) + return S_OK; + + item.HeaderSize += item.Get_PackSize_Aligned(); + + if (tb->StringSize != item.PackSize + || tb->StringSize == 0 + || tb->IsNonZeroTail) + item.Pax_Error = true; + + item.Num_Pax_Records++; + if (lf != NFileHeader::NLinkFlag::kGlobal) + { + item.PaxExtra.RecordPath = item.Name; + continue; + } + // break; // for debug + { + if (PaxGlobal_Defined) + _is_PaxGlobal_Error = true; + CPaxInfo paxInfo; + if (paxInfo.ParsePax(PaxBuf_global, false)) + { + PaxGlobal.RawLines = paxInfo.UnknownLines; + PaxGlobal.RecordPath = item.Name; + PaxGlobal_Defined = true; + } + else + _is_PaxGlobal_Error = true; + + if (isStartHeader + && item.Num_Pax_Records == 1 + && numExtraRecords == 1) + { + // we skip global pax header info after parsing + item.HeaderPos += item.HeaderSize; + item.HeaderSize = 0; + item.Num_Pax_Records = 0; + numExtraRecords = 0; + } + else + _is_PaxGlobal_Error = true; + } + continue; + } + + /* + if (lf == NFileHeader::NLinkFlag::kDumpDir || + lf == NFileHeader::NLinkFlag::kSparse) + { + // GNU Extensions to the Archive Format + break; + } + if (lf > '7' || (lf < '0' && lf != 0)) + { + break; + // return S_OK; + } + */ + break; + } + + // we still use name from main header, if long_name is bad + if (item.LongName_WasUsed && NameBuf.StringSize != 0) + { + NameBuf.CopyToString(item.Name); + // item.Name_CouldBeReduced = false; + } + + if (item.LongLink_WasUsed) + { + // we use empty link, if long_link is bad + LinkBuf.CopyToString(item.LinkName); + // item.LinkName_CouldBeReduced = false; + } + + error = k_ErrorType_OK; + + if (PaxBuf.StringSize != 0) + { + CPaxInfo paxInfo; + if (!paxInfo.ParsePax(PaxBuf, true)) + item.Pax_Error = true; + else + { + if (paxInfo.Path_Defined) // if (!paxInfo.Path.IsEmpty()) + { + item.Name = paxInfo.Path; + item.pax_path_WasUsed = true; + } + if (paxInfo.Link_Defined) // (!paxInfo.Link.IsEmpty()) + { + item.LinkName = paxInfo.Link; + item.pax_link_WasUsed = true; + } + if (paxInfo.User_Defined) + { + item.User = paxInfo.User; + // item.pax_uname_WasUsed = true; + } + if (paxInfo.Group_Defined) + { + item.Group = paxInfo.Group; + // item.pax_gname_WasUsed = true; + } + if (paxInfo.SCHILY_fflags_Defined) + { + item.SCHILY_fflags = paxInfo.SCHILY_fflags; + // item.SCHILY_fflags_WasUsed = true; + } + if (paxInfo.UID_Defined) + { + item.UID = (UInt32)paxInfo.UID; + } + if (paxInfo.GID_Defined) + { + item.GID = (UInt32)paxInfo.GID; + } + + if (paxInfo.Size_Defined) + { + const UInt64 piSize = paxInfo.Size; + // GNU TAR ignores (item.Size) in that case + if (item.Size != 0 && item.Size != piSize) + item.Pax_Error = true; + item.Size = piSize; + item.PackSize = piSize; + item.pax_size_WasUsed = true; + } + + item.PaxTimes = paxInfo; + item.PaxExtra.RawLines = paxInfo.UnknownLines; + if (paxInfo.UnknownLines_Overflow) + item.Pax_Overflow = true; + if (paxInfo.TagParsingError) + item.Pax_Error = true; + if (paxInfo.DoubleTagError) + item.Pax_Error = true; + } + } + + return S_OK; +} + + + +HRESULT CArchive::ReadItem(CItemEx &item) +{ + item.HeaderPos = _phySize; + + const HRESULT res = ReadItem2(item); + + /* + if (error == k_ErrorType_Warning) + _is_Warning = true; + else + */ + + if (error != k_ErrorType_OK) + _error = error; + + RINOK(res) + + if (filled) + { + if (item.IsMagic_GNU()) + _are_Gnu = true; + else if (item.IsMagic_Posix_ustar_00()) + _are_Posix = true; + + if (item.Num_Pax_Records != 0) + _are_Pax = true; + + if (item.PaxTimes.MTime.IsDefined()) _are_mtime = true; + if (item.PaxTimes.ATime.IsDefined()) _are_atime = true; + if (item.PaxTimes.CTime.IsDefined()) _are_ctime = true; + if (!item.SCHILY_fflags.IsEmpty()) _are_SCHILY_fflags = true; + + if (item.pax_path_WasUsed) + _are_pax_path = true; + if (item.pax_link_WasUsed) + _are_pax_link = true; + if (item.LongName_WasUsed) + _are_LongName = true; + if (item.LongLink_WasUsed) + _are_LongLink = true; + if (item.Prefix_WasUsed) + _pathPrefix_WasUsed = true; + /* + if (item.IsSparse()) + _isSparse = true; + */ + if (item.Is_PaxExtendedHeader()) + _are_Pax_Items = true; + if (item.IsThereWarning() + || item.HeaderError + || item.Pax_Error) + _is_Warning = true; + } + + const UInt64 headerEnd = item.HeaderPos + item.HeaderSize; + // _headersSize += headerEnd - _phySize; + // we don't count skipped records + _headersSize += item.HeaderSize; + _phySize = headerEnd; + return S_OK; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarIn.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarIn.h new file mode 100644 index 0000000..edfc94a --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarIn.h @@ -0,0 +1,151 @@ +// TarIn.h + +#ifndef ZIP7_INC_ARCHIVE_TAR_IN_H +#define ZIP7_INC_ARCHIVE_TAR_IN_H + +#include "../IArchive.h" + +#include "TarItem.h" + +namespace NArchive { +namespace NTar { + +enum EErrorType +{ + k_ErrorType_OK, + k_ErrorType_Corrupted, + k_ErrorType_UnexpectedEnd + // , k_ErrorType_Warning +}; + + +struct CTempBuffer +{ + CByteBuffer Buffer; + size_t StringSize; // num characters before zero Byte (StringSize <= item.PackSize) + bool IsNonZeroTail; + bool StringSize_IsConfirmed; + + void CopyToString(AString &s) + { + s.Empty(); + if (StringSize != 0) + s.SetFrom((const char *)(const void *)(const Byte *)Buffer, (unsigned)StringSize); + } + + void Init() + { + StringSize = 0; + IsNonZeroTail = false; + StringSize_IsConfirmed = false; + } +}; + + +class CArchive +{ +public: + bool _phySize_Defined; + bool _is_Warning; + bool PaxGlobal_Defined; + bool _is_PaxGlobal_Error; + bool _are_Pax_Items; + bool _are_Gnu; + bool _are_Posix; + bool _are_Pax; + bool _are_mtime; + bool _are_atime; + bool _are_ctime; + bool _are_pax_path; + bool _are_pax_link; + bool _are_LongName; + bool _are_LongLink; + bool _pathPrefix_WasUsed; + bool _are_SCHILY_fflags; + // bool _isSparse; + + // temp internal vars for ReadItem(): + bool filled; +private: + EErrorType error; + +public: + UInt64 _phySize; + UInt64 _headersSize; + EErrorType _error; + + ISequentialInStream *SeqStream; + IInStream *InStream; + IArchiveOpenCallback *OpenCallback; + UInt64 NumFiles; + UInt64 NumFiles_Prev; + UInt64 Pos_Prev; + // UInt64 NumRecords; + // UInt64 NumRecords_Prev; + + CPaxExtra PaxGlobal; + + void Clear() + { + SeqStream = NULL; + InStream = NULL; + OpenCallback = NULL; + NumFiles = 0; + NumFiles_Prev = 0; + Pos_Prev = 0; + // NumRecords = 0; + // NumRecords_Prev = 0; + + PaxGlobal.Clear(); + PaxGlobal_Defined = false; + _is_PaxGlobal_Error = false; + _are_Pax_Items = false; // if there are final paxItems + _are_Gnu = false; + _are_Posix = false; + _are_Pax = false; + _are_mtime = false; + _are_atime = false; + _are_ctime = false; + _are_pax_path = false; + _are_pax_link = false; + _are_LongName = false; + _are_LongLink = false; + _pathPrefix_WasUsed = false; + _are_SCHILY_fflags = false; + // _isSparse = false; + + _is_Warning = false; + _error = k_ErrorType_OK; + + _phySize_Defined = false; + _phySize = 0; + _headersSize = 0; + } + +private: + CTempBuffer NameBuf; + CTempBuffer LinkBuf; + CTempBuffer PaxBuf; + CTempBuffer PaxBuf_global; + + CByteBuffer Buffer; + + HRESULT ReadDataToBuffer(const CItemEx &item, CTempBuffer &tb, size_t stringLimit); + HRESULT Progress(const CItemEx &item, UInt64 posOffset); + HRESULT GetNextItemReal(CItemEx &item); + HRESULT ReadItem2(CItemEx &itemInfo); +public: + CArchive() + { + // we will call Clear() in CHandler::Close(). + // Clear(); // it's not required here + } + HRESULT ReadItem(CItemEx &itemInfo); +}; + + +API_FUNC_IsArc IsArc_Tar(const Byte *p, size_t size); + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarItem.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarItem.h new file mode 100644 index 0000000..112f38d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarItem.h @@ -0,0 +1,364 @@ +// TarItem.h + +#ifndef ZIP7_INC_ARCHIVE_TAR_ITEM_H +#define ZIP7_INC_ARCHIVE_TAR_ITEM_H + +#include "../../../Common/MyLinux.h" +#include "../../../Common/UTFConvert.h" + +#include "TarHeader.h" + +namespace NArchive { +namespace NTar { + +struct CSparseBlock +{ + UInt64 Offset; + UInt64 Size; +}; + + +enum EPaxTimeRemoveZeroMode +{ + k_PaxTimeMode_DontRemoveZero, + k_PaxTimeMode_RemoveZero_if_PureSecondOnly, + k_PaxTimeMode_RemoveZero_Always +}; + +struct CTimeOptions +{ + EPaxTimeRemoveZeroMode RemoveZeroMode; + unsigned NumDigitsMax; + + void Init() + { + RemoveZeroMode = k_PaxTimeMode_RemoveZero_if_PureSecondOnly; + NumDigitsMax = 0; + } + CTimeOptions() { Init(); } +}; + + +struct CPaxTime +{ + Int32 NumDigits; // -1 means undefined + UInt32 Ns; // it's smaller than 1G. Even if (Sec < 0), larger (Ns) value means newer files. + Int64 Sec; // can be negative + + Int64 GetSec() const { return NumDigits != -1 ? Sec : 0; } + + bool IsDefined() const { return NumDigits != -1; } + // bool IsDefined_And_nonZero() const { return NumDigits != -1 && (Sec != 0 || Ns != 0); } + + void Clear() + { + NumDigits = -1; + Ns = 0; + Sec = 0; + } + CPaxTime() { Clear(); } + + /* + void ReducePrecison(int numDigits) + { + // we don't use this->NumDigits here + if (numDigits > 0) + { + if (numDigits >= 9) + return; + UInt32 r = 1; + for (unsigned i = numDigits; i < 9; i++) + r *= 10; + Ns /= r; + Ns *= r; + return; + } + Ns = 0; + if (numDigits == 0) + return; + UInt32 r; + if (numDigits == -1) r = 60; + else if (numDigits == -2) r = 60 * 60; + else if (numDigits == -3) r = 60 * 60 * 24; + else return; + Sec /= r; + Sec *= r; + } + */ +}; + + +struct CPaxTimes +{ + CPaxTime MTime; + CPaxTime ATime; + CPaxTime CTime; + + void Clear() + { + MTime.Clear(); + ATime.Clear(); + CTime.Clear(); + } + + /* + void ReducePrecison(int numDigits) + { + MTime.ReducePrecison(numDigits); + CTime.ReducePrecison(numDigits); + ATime.ReducePrecison(numDigits); + } + */ +}; + + +struct CItem +{ + UInt64 PackSize; + UInt64 Size; + Int64 MTime; + + char LinkFlag; + bool DeviceMajor_Defined; + bool DeviceMinor_Defined; + + UInt32 Mode; + UInt32 UID; + UInt32 GID; + UInt32 DeviceMajor; + UInt32 DeviceMinor; + + AString Name; + AString LinkName; + AString User; + AString Group; + + char Magic[8]; + + CPaxTimes PaxTimes; + + CRecordVector SparseBlocks; + + void SetMagic_Posix(bool posixMode) + { + memcpy(Magic, posixMode ? + NFileHeader::NMagic::k_Posix_ustar_00 : + NFileHeader::NMagic::k_GNU_ustar, + 8); + } + + bool Is_SymLink() const { return LinkFlag == NFileHeader::NLinkFlag::kSymLink && (Size == 0); } + bool Is_HardLink() const { return LinkFlag == NFileHeader::NLinkFlag::kHardLink; } + bool Is_Sparse() const { return LinkFlag == NFileHeader::NLinkFlag::kSparse; } + + UInt64 Get_UnpackSize() const { return Is_SymLink() ? LinkName.Len() : Size; } + + bool Is_PaxExtendedHeader() const + { + switch (LinkFlag) + { + case NFileHeader::NLinkFlag::kPax: + case NFileHeader::NLinkFlag::kPax_2: + case NFileHeader::NLinkFlag::kGlobal: + return true; + default: break; + } + return false; + } + + UInt32 Get_Combined_Mode() const + { + return (Mode & ~(UInt32)MY_LIN_S_IFMT) | Get_FileTypeMode_from_LinkFlag(); + } + + void Set_LinkFlag_for_File(UInt32 mode) + { + char lf = NFileHeader::NLinkFlag::kNormal; + if (MY_LIN_S_ISCHR(mode)) lf = NFileHeader::NLinkFlag::kCharacter; + else if (MY_LIN_S_ISBLK(mode)) lf = NFileHeader::NLinkFlag::kBlock; + else if (MY_LIN_S_ISFIFO(mode)) lf = NFileHeader::NLinkFlag::kFIFO; + // else if (MY_LIN_S_ISDIR(mode)) lf = NFileHeader::NLinkFlag::kDirectory; + // else if (MY_LIN_S_ISLNK(mode)) lf = NFileHeader::NLinkFlag::kSymLink; + LinkFlag = lf; + } + + UInt32 Get_FileTypeMode_from_LinkFlag() const + { + switch (LinkFlag) + { + /* + case NFileHeader::NLinkFlag::kDirectory: + case NFileHeader::NLinkFlag::kDumpDir: + return MY_LIN_S_IFDIR; + */ + case NFileHeader::NLinkFlag::kSymLink: return MY_LIN_S_IFLNK; + case NFileHeader::NLinkFlag::kBlock: return MY_LIN_S_IFBLK; + case NFileHeader::NLinkFlag::kCharacter: return MY_LIN_S_IFCHR; + case NFileHeader::NLinkFlag::kFIFO: return MY_LIN_S_IFIFO; + // case return MY_LIN_S_IFSOCK; + default: break; + } + + if (IsDir()) + return MY_LIN_S_IFDIR; + return MY_LIN_S_IFREG; + } + + bool IsDir() const + { + switch (LinkFlag) + { + case NFileHeader::NLinkFlag::kDirectory: + case NFileHeader::NLinkFlag::kDumpDir: + return true; + case NFileHeader::NLinkFlag::kOldNormal: + case NFileHeader::NLinkFlag::kNormal: + case NFileHeader::NLinkFlag::kSymLink: + if (Name.IsEmpty()) + return false; + // GNU TAR uses last character as directory marker + // we also do it + return Name.Back() == '/'; + // return NItemName::HasTailSlash(Name, CP_OEMCP); + default: break; + } + return false; + } + + bool IsMagic_ustar_5chars() const + { + for (unsigned i = 0; i < 5; i++) + if (Magic[i] != NFileHeader::NMagic::k_GNU_ustar[i]) + return false; + return true; + } + + bool IsMagic_Posix_ustar_00() const + { + for (unsigned i = 0; i < 8; i++) + if (Magic[i] != NFileHeader::NMagic::k_Posix_ustar_00[i]) + return false; + return true; + } + + bool IsMagic_GNU() const + { + for (unsigned i = 0; i < 8; i++) + if (Magic[i] != NFileHeader::NMagic::k_GNU_ustar[i]) + return false; + return true; + } + + UInt64 Get_PackSize_Aligned() const { return (PackSize + 0x1FF) & (~((UInt64)0x1FF)); } + + bool IsThereWarning() const + { + // that Header Warning is possible if (Size != 0) for dir item + return (PackSize < Size) && (LinkFlag == NFileHeader::NLinkFlag::kDirectory); + } +}; + + + +struct CEncodingCharacts +{ + bool IsAscii; + // bool Oem_Checked; + // bool Oem_Ok; + // bool Utf_Checked; + CUtf8Check UtfCheck; + + void Clear() + { + IsAscii = true; + // Oem_Checked = false; + // Oem_Ok = false; + // Utf_Checked = false; + UtfCheck.Clear(); + } + + void Update(const CEncodingCharacts &ec) + { + if (!ec.IsAscii) + IsAscii = false; + + // if (ec.Utf_Checked) + { + UtfCheck.Update(ec.UtfCheck); + // Utf_Checked = true; + } + } + + CEncodingCharacts() { Clear(); } + void Check(const AString &s); + AString GetCharactsString() const; +}; + + +struct CPaxExtra +{ + AString RecordPath; + AString RawLines; + + void Clear() + { + RecordPath.Empty(); + RawLines.Empty(); + } + + void Print_To_String(AString &s) const + { + if (!RecordPath.IsEmpty()) + { + s += RecordPath; + s.Add_LF(); + } + if (!RawLines.IsEmpty()) + s += RawLines; + } +}; + + +struct CItemEx: public CItem +{ + bool HeaderError; + + bool IsSignedChecksum; + bool Prefix_WasUsed; + + bool Pax_Error; + bool Pax_Overflow; + bool pax_path_WasUsed; + bool pax_link_WasUsed; + bool pax_size_WasUsed; + + bool MTime_IsBin; + bool PackSize_IsBin; + bool Size_IsBin; + + bool LongName_WasUsed; + bool LongName_WasUsed_2; + + bool LongLink_WasUsed; + bool LongLink_WasUsed_2; + + // bool Name_CouldBeReduced; + // bool LinkName_CouldBeReduced; + + UInt64 HeaderPos; + UInt64 HeaderSize; + + UInt64 Num_Pax_Records; + CPaxExtra PaxExtra; + AString SCHILY_fflags; + + CEncodingCharacts EncodingCharacts; + + UInt64 Get_DataPos() const { return HeaderPos + HeaderSize; } + // UInt64 GetFullSize() const { return HeaderSize + PackSize; } + UInt64 Get_FullSize_Aligned() const { return HeaderSize + Get_PackSize_Aligned(); } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarOut.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarOut.cpp new file mode 100644 index 0000000..8a80c0c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarOut.cpp @@ -0,0 +1,644 @@ +// TarOut.cpp + +#include "StdAfx.h" + +#include "../../../../C/7zCrc.h" + +#include "../../../Common/IntToString.h" + +#include "../../Common/StreamUtils.h" + +#include "TarOut.h" + +namespace NArchive { +namespace NTar { + +using namespace NFileHeader; + +// it's path prefix assigned by 7-Zip to show that file path was cut +#define K_PREFIX_PATH_CUT "@PathCut" + +static const UInt32 k_7_oct_digits_Val_Max = ((UInt32)1 << (7 * 3)) - 1; + +static void WriteOctal_8(char *s, UInt32 val) +{ + const unsigned kNumDigits = 8 - 1; + if (val >= ((UInt32)1 << (kNumDigits * 3))) + { + val = 0; + // return false; + } + for (unsigned i = 0; i < kNumDigits; i++) + { + s[kNumDigits - 1 - i] = (char)('0' + (val & 7)); + val >>= 3; + } + // return true; +} + +static void WriteBin_64bit(char *s, UInt64 val) +{ + for (unsigned i = 0; i < 8; i++, val <<= 8) + s[i] = (char)(val >> 56); +} + +static void WriteOctal_12(char *s, UInt64 val) +{ + const unsigned kNumDigits = 12 - 1; + if (val >= ((UInt64)1 << (kNumDigits * 3))) + { + // GNU extension; + s[0] = (char)(Byte)0x80; + s[1] = s[2] = s[3] = 0; + WriteBin_64bit(s + 4, val); + return; + } + for (unsigned i = 0; i < kNumDigits; i++) + { + s[kNumDigits - 1 - i] = (char)('0' + (val & 7)); + val >>= 3; + } +} + +static void WriteOctal_12_Signed(char *s, const Int64 val) +{ + if (val >= 0) + { + WriteOctal_12(s, (UInt64)val); + return; + } + s[0] = s[1] = s[2] = s[3] = (char)(Byte)0xFF; + WriteBin_64bit(s + 4, (UInt64)val); +} + +static void CopyString(char *dest, const AString &src, const unsigned maxSize) +{ + unsigned len = src.Len(); + if (len == 0) + return; + // 21.07: new gnu : we don't require additional 0 character at the end + // if (len >= maxSize) + if (len > maxSize) + { + len = maxSize; + /* + // oldgnu needs 0 character at the end + len = maxSize - 1; + dest[len] = 0; + */ + } + memcpy(dest, src.Ptr(), len); +} + +// #define RETURN_IF_NOT_TRUE(x) { if (!(x)) return E_INVALIDARG; } +#define RETURN_IF_NOT_TRUE(x) { x; } + +#define COPY_STRING_CHECK(dest, src, size) \ + CopyString(dest, src, size); dest += (size); + +#define WRITE_OCTAL_8_CHECK(dest, src) \ + RETURN_IF_NOT_TRUE(WriteOctal_8(dest, src)) + + +HRESULT COutArchive::WriteHeaderReal(const CItem &item, bool isPax + // , bool zero_PackSize + // , bool zero_MTime + ) +{ + /* + if (isPax) { we don't use Glob_Name and Prefix } + if (!isPax) + { + we use Glob_Name if it's not empty + we use Prefix if it's not empty + } + */ + char record[kRecordSize]; + memset(record, 0, kRecordSize); + char *cur = record; + + COPY_STRING_CHECK (cur, + (!isPax && !Glob_Name.IsEmpty()) ? Glob_Name : item.Name, + kNameSize) + + WRITE_OCTAL_8_CHECK (cur, item.Mode) cur += 8; // & k_7_oct_digits_Val_Max + WRITE_OCTAL_8_CHECK (cur, item.UID) cur += 8; + WRITE_OCTAL_8_CHECK (cur, item.GID) cur += 8; + + WriteOctal_12 (cur, /* zero_PackSize ? 0 : */ item.PackSize); cur += 12; + WriteOctal_12_Signed (cur, /* zero_MTime ? 0 : */ item.MTime); cur += 12; + + // we will use binary init for checksum instead of memset + // checksum field: + // memset(cur, ' ', 8); + cur += 8; + + *cur++ = item.LinkFlag; + + COPY_STRING_CHECK (cur, item.LinkName, kNameSize) + + memcpy(cur, item.Magic, 8); + cur += 8; + + COPY_STRING_CHECK (cur, item.User, kUserNameSize) + COPY_STRING_CHECK (cur, item.Group, kGroupNameSize) + + const bool needDevice = (IsPosixMode && !isPax); + + if (item.DeviceMajor_Defined) + WRITE_OCTAL_8_CHECK (cur, item.DeviceMajor) + else if (needDevice) + WRITE_OCTAL_8_CHECK (cur, 0) + cur += 8; + + if (item.DeviceMinor_Defined) + WRITE_OCTAL_8_CHECK (cur, item.DeviceMinor) + else if (needDevice) + WRITE_OCTAL_8_CHECK (cur, 0) + cur += 8; + + if (!isPax && !Prefix.IsEmpty()) + { + COPY_STRING_CHECK (cur, Prefix, kPrefixSize) + } + + if (item.Is_Sparse()) + { + record[482] = (char)(item.SparseBlocks.Size() > 4 ? 1 : 0); + WriteOctal_12(record + 483, item.Size); + for (unsigned i = 0; i < item.SparseBlocks.Size() && i < 4; i++) + { + const CSparseBlock &sb = item.SparseBlocks[i]; + char *p = record + 386 + 24 * i; + WriteOctal_12(p, sb.Offset); + WriteOctal_12(p + 12, sb.Size); + } + } + + { + UInt32 sum = (unsigned)(' ') * 8; // we use binary init + { + for (unsigned i = 0; i < kRecordSize; i++) + sum += (Byte)record[i]; + } + /* checksum field is formatted differently from the + other fields: it has [6] digits, a null, then a space. */ + // WRITE_OCTAL_8_CHECK(record + 148, sum); + const unsigned kNumDigits = 6; + for (unsigned i = 0; i < kNumDigits; i++) + { + record[148 + kNumDigits - 1 - i] = (char)('0' + (sum & 7)); + sum >>= 3; + } + // record[148 + 6] = 0; // we need it, if we use memset(' ') init + record[148 + 7] = ' '; // we need it, if we use binary init + } + + RINOK(Write_Data(record, kRecordSize)) + + if (item.Is_Sparse()) + { + for (unsigned i = 4; i < item.SparseBlocks.Size();) + { + memset(record, 0, kRecordSize); + for (unsigned t = 0; t < 21 && i < item.SparseBlocks.Size(); t++, i++) + { + const CSparseBlock &sb = item.SparseBlocks[i]; + char *p = record + 24 * t; + WriteOctal_12(p, sb.Offset); + WriteOctal_12(p + 12, sb.Size); + } + record[21 * 24] = (char)(i < item.SparseBlocks.Size() ? 1 : 0); + RINOK(Write_Data(record, kRecordSize)) + } + } + + return S_OK; +} + + +static void AddPaxLine(AString &s, const char *name, const AString &val) +{ + // s.Add_LF(); // for debug + const unsigned len = 3 + (unsigned)strlen(name) + val.Len(); + AString n; + for (unsigned numDigits = 1;; numDigits++) + { + n.Empty(); + n.Add_UInt32(numDigits + len); + if (numDigits == n.Len()) + break; + } + s += n; + s.Add_Space(); + s += name; + s.Add_Char('='); + s += val; + s.Add_LF(); +} + +// pt is defined : (pt.NumDigits >= 0) +static void AddPaxTime(AString &s, const char *name, const CPaxTime &pt, + const CTimeOptions &options) +{ + unsigned numDigits = (unsigned)pt.NumDigits; + if (numDigits > options.NumDigitsMax) + numDigits = options.NumDigitsMax; + + bool needNs = false; + UInt32 ns = 0; + if (numDigits != 0) + { + ns = pt.Ns; + // if (ns != 0) before reduction, we show all digits after digits reduction + needNs = (ns != 0 || options.RemoveZeroMode == k_PaxTimeMode_DontRemoveZero); + UInt32 d = 1; + for (unsigned k = numDigits; k < 9; k++) + d *= 10; + ns /= d; + ns *= d; + } + + AString v; + { + Int64 sec = pt.Sec; + if (pt.Sec < 0) + { + sec = -sec; + v.Add_Minus(); + if (ns != 0) + { + ns = 1000*1000*1000 - ns; + sec--; + } + } + v.Add_UInt64((UInt64)sec); + } + + if (needNs) + { + AString d; + d.Add_UInt32(ns); + while (d.Len() < 9) + d.InsertAtFront('0'); + // here we have precision + while (d.Len() > (unsigned)numDigits) + d.DeleteBack(); + // GNU TAR reduces '0' digits. + if (options.RemoveZeroMode == k_PaxTimeMode_RemoveZero_Always) + while (!d.IsEmpty() && d.Back() == '0') + d.DeleteBack(); + + if (!d.IsEmpty()) + { + v.Add_Dot(); + v += d; + // v += "1234567009999"; // for debug + // for (int y = 0; y < 1000; y++) v += '8'; // for debug + } + } + + AddPaxLine(s, name, v); +} + + +static void AddPax_UInt32_ifBig(AString &s, const char *name, const UInt32 &v) +{ + if (v > k_7_oct_digits_Val_Max) + { + AString s2; + s2.Add_UInt32(v); + AddPaxLine(s, name, s2); + } +} + + +/* OLD_GNU_TAR: writes name with zero at the end + NEW_GNU_TAR: can write name filled with all kNameSize characters */ + +static const unsigned kNameSize_Max = + kNameSize; // NEW_GNU_TAR / 7-Zip 21.07 + // kNameSize - 1; // OLD_GNU_TAR / old 7-Zip + +#define DOES_NAME_FIT_IN_FIELD(name) ((name).Len() <= kNameSize_Max) + + +HRESULT COutArchive::WriteHeader(const CItem &item) +{ + Glob_Name.Empty(); + Prefix.Empty(); + + unsigned namePos = 0; + bool needPathCut = false; + bool allowPrefix = false; + + if (!DOES_NAME_FIT_IN_FIELD(item.Name)) + { + const char *s = item.Name; + const char *p = s + item.Name.Len() - 1; + for (; *p == '/' && p != s; p--) + {} + for (; p != s && p[-1] != '/'; p--) + {} + namePos = (unsigned)(p - s); + needPathCut = true; + } + + if (IsPosixMode) + { + AString s; + + if (needPathCut) + { + const unsigned nameLen = item.Name.Len() - namePos; + if ( item.LinkFlag >= NLinkFlag::kNormal + && item.LinkFlag <= NLinkFlag::kDirectory + && namePos > 1 + && nameLen != 0 + // && IsPrefixAllowed + && item.IsMagic_Posix_ustar_00()) + { + /* GNU TAR decoder supports prefix field, only if (magic) + signature matches 6-bytes "ustar\0". + so here we use prefix field only in posix mode with posix signature */ + + allowPrefix = true; + // allowPrefix = false; // for debug + if (namePos <= kPrefixSize + 1 && nameLen <= kNameSize_Max) + { + needPathCut = false; + /* we will set Prefix and Glob_Name later, for such conditions: + if (!DOES_NAME_FIT_IN_FIELD(item.Name) && !needPathCut) */ + } + } + + if (needPathCut) + AddPaxLine(s, "path", item.Name); + } + + // AddPaxLine(s, "testname", AString("testval")); // for debug + + if (item.LinkName.Len() > kNameSize_Max) + AddPaxLine(s, "linkpath", item.LinkName); + + const UInt64 kPaxSize_Limit = ((UInt64)1 << 33); + // const UInt64 kPaxSize_Limit = ((UInt64)1 << 1); // for debug + // bool zero_PackSize = false; + if (item.PackSize >= kPaxSize_Limit) + { + /* GNU TAR in pax mode sets PackSize = 0 in main record, if pack_size >= 8 GiB + But old 7-Zip doesn't detect "size" property from pax header. + So we write real size (>= 8 GiB) to main record in binary format, + and old 7-Zip can decode size correctly */ + // zero_PackSize = true; + AString v; + v.Add_UInt64(item.PackSize); + AddPaxLine(s, "size", v); + } + + /* GNU TAR encoder can set "devmajor" / "devminor" attributes, + but GNU TAR decoder doesn't parse "devmajor" / "devminor" */ + if (item.DeviceMajor_Defined) + AddPax_UInt32_ifBig(s, "devmajor", item.DeviceMajor); + if (item.DeviceMinor_Defined) + AddPax_UInt32_ifBig(s, "devminor", item.DeviceMinor); + + AddPax_UInt32_ifBig(s, "uid", item.UID); + AddPax_UInt32_ifBig(s, "gid", item.GID); + + const UInt64 kPax_MTime_Limit = ((UInt64)1 << 33); + const bool zero_MTime = ( + item.MTime < 0 || + item.MTime >= (Int64)kPax_MTime_Limit); + + const CPaxTime &mtime = item.PaxTimes.MTime; + if (mtime.IsDefined()) + { + bool needPax = false; + if (zero_MTime) + needPax = true; + else if (TimeOptions.NumDigitsMax > 0) + if (mtime.Ns != 0 || + (mtime.NumDigits != 0 && + TimeOptions.RemoveZeroMode == k_PaxTimeMode_DontRemoveZero)) + needPax = true; + if (needPax) + AddPaxTime(s, "mtime", mtime, TimeOptions); + } + + if (item.PaxTimes.ATime.IsDefined()) + AddPaxTime(s, "atime", item.PaxTimes.ATime, TimeOptions); + if (item.PaxTimes.CTime.IsDefined()) + AddPaxTime(s, "ctime", item.PaxTimes.CTime, TimeOptions); + + if (item.User.Len() > kUserNameSize) + AddPaxLine(s, "uname", item.User); + if (item.Group.Len() > kGroupNameSize) + AddPaxLine(s, "gname", item.Group); + + /* + // for debug + AString a ("11"); for (int y = 0; y < (1 << 24); y++) AddPaxLine(s, "temp", a); + */ + + const unsigned paxSize = s.Len(); + if (paxSize != 0) + { + CItem mi = item; + mi.LinkName.Empty(); + // SparseBlocks will be ignored by Is_Sparse() + // mi.SparseBlocks.Clear(); + // we use "PaxHeader/*" for compatibility with previous 7-Zip decoder + + // GNU TAR writes empty for these fields; + mi.User.Empty(); + mi.Group.Empty(); + mi.UID = 0; + mi.GID = 0; + + mi.DeviceMajor_Defined = false; + mi.DeviceMinor_Defined = false; + + mi.Name = "PaxHeader/@PaxHeader"; + mi.Mode = 0644; // octal + if (zero_MTime) + mi.MTime = 0; + mi.LinkFlag = NLinkFlag::kPax; + // mi.LinkFlag = 'Z'; // for debug + mi.PackSize = paxSize; + // for (unsigned y = 0; y < 1; y++) { // for debug + RINOK(WriteHeaderReal(mi, true)) // isPax + RINOK(Write_Data_And_Residual(s, paxSize)) + // } // for debug + /* + we can send (zero_MTime) for compatibility with gnu tar output. + we can send (zero_MTime = false) for better compatibility with old 7-Zip + */ + // return WriteHeaderReal(item); + /* + false, // isPax + false, // zero_PackSize + false); // zero_MTime + */ + } + } + else // !PosixMode + if (!DOES_NAME_FIT_IN_FIELD(item.Name) || + !DOES_NAME_FIT_IN_FIELD(item.LinkName)) + { + // here we can get all fields from main (item) or create new empty item + /* + CItem mi; + mi.SetDefaultWriteFields(); + */ + CItem mi = item; + mi.LinkName.Empty(); + // SparseBlocks will be ignored by Is_Sparse() + // mi.SparseBlocks.Clear(); + mi.Name = kLongLink; + // mi.Name = "././@BAD_LONG_LINK_TEST"; // for debug + // 21.07 : we set Mode and MTime props as in GNU TAR: + mi.Mode = 0644; // octal + mi.MTime = 0; + + mi.User.Empty(); + mi.Group.Empty(); + /* + gnu tar sets "root" for such items: + uid_to_uname (0, &uname); + gid_to_gname (0, &gname); + */ + /* + mi.User = "root"; + mi.Group = "root"; + */ + mi.UID = 0; + mi.GID = 0; + mi.DeviceMajor_Defined = false; + mi.DeviceMinor_Defined = false; + + + for (unsigned i = 0; i < 2; i++) + { + const AString *name; + // We suppose that GNU TAR also writes item for long link before item for LongName? + if (i == 0) + { + mi.LinkFlag = NLinkFlag::kGnu_LongLink; + name = &item.LinkName; + } + else + { + mi.LinkFlag = NLinkFlag::kGnu_LongName; + name = &item.Name; + } + if (DOES_NAME_FIT_IN_FIELD(*name)) + continue; + // GNU TAR writes null character after NAME to file. We do same here: + const unsigned nameStreamSize = name->Len() + 1; + mi.PackSize = nameStreamSize; + // for (unsigned y = 0; y < 3; y++) { // for debug + RINOK(WriteHeaderReal(mi)) + RINOK(Write_Data_And_Residual(name->Ptr(), nameStreamSize)) + // } + + // for debug + /* + const unsigned kSize = (1 << 29) + 16; + CByteBuffer buf; + buf.Alloc(kSize); + memset(buf, 0, kSize); + memcpy(buf, name->Ptr(), name->Len()); + const unsigned nameStreamSize = kSize; + mi.PackSize = nameStreamSize; + // for (unsigned y = 0; y < 3; y++) { // for debug + RINOK(WriteHeaderReal(mi)); + RINOK(WriteBytes(buf, nameStreamSize)); + RINOK(FillDataResidual(nameStreamSize)); + */ + } + } + + // bool fals = false; if (fals) // for debug: for bit-to-bit output compatibility with GNU TAR + + if (!DOES_NAME_FIT_IN_FIELD(item.Name)) + { + const unsigned nameLen = item.Name.Len() - namePos; + if (!needPathCut) + Prefix.SetFrom(item.Name, namePos - 1); + else + { + Glob_Name = K_PREFIX_PATH_CUT "/_pc_"; + + if (namePos == 0) + Glob_Name += "root"; + else + { + Glob_Name += "crc32/"; + char temp[12]; + ConvertUInt32ToHex8Digits(CrcCalc(item.Name, namePos - 1), temp); + Glob_Name += temp; + } + + if (!allowPrefix || Glob_Name.Len() + 1 + nameLen <= kNameSize_Max) + Glob_Name.Add_Slash(); + else + { + Prefix = Glob_Name; + Glob_Name.Empty(); + } + } + Glob_Name.AddFrom(item.Name.Ptr(namePos), nameLen); + } + + return WriteHeaderReal(item); +} + + +HRESULT COutArchive::Write_Data(const void *data, unsigned size) +{ + Pos += size; + return WriteStream(Stream, data, size); +} + +HRESULT COutArchive::Write_AfterDataResidual(UInt64 dataSize) +{ + const unsigned v = ((unsigned)dataSize & (kRecordSize - 1)); + if (v == 0) + return S_OK; + const unsigned rem = kRecordSize - v; + Byte buf[kRecordSize]; + memset(buf, 0, rem); + return Write_Data(buf, rem); +} + + +HRESULT COutArchive::Write_Data_And_Residual(const void *data, unsigned size) +{ + RINOK(Write_Data(data, size)) + return Write_AfterDataResidual(size); +} + + +HRESULT COutArchive::WriteFinishHeader() +{ + Byte record[kRecordSize]; + memset(record, 0, kRecordSize); + + const unsigned kNumFinishRecords = 2; + + /* GNU TAR by default uses --blocking-factor=20 (512 * 20 = 10 KiB) + we also can use cluster alignment: + const unsigned numBlocks = (unsigned)(Pos / kRecordSize) + kNumFinishRecords; + const unsigned kNumClusterBlocks = (1 << 3); // 8 blocks = 4 KiB + const unsigned numFinishRecords = kNumFinishRecords + ((kNumClusterBlocks - numBlocks) & (kNumClusterBlocks - 1)); + */ + + for (unsigned i = 0; i < kNumFinishRecords; i++) + { + RINOK(Write_Data(record, kRecordSize)) + } + return S_OK; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarOut.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarOut.h new file mode 100644 index 0000000..7b99c26 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarOut.h @@ -0,0 +1,53 @@ +// Archive/TarOut.h + +#ifndef ZIP7_INC_ARCHIVE_TAR_OUT_H +#define ZIP7_INC_ARCHIVE_TAR_OUT_H + +#include "../../../Common/MyCom.h" + +#include "../../IStream.h" + +#include "TarItem.h" + +namespace NArchive { +namespace NTar { + +class COutArchive +{ + CMyComPtr Stream; + + AString Glob_Name; + AString Prefix; + + HRESULT WriteHeaderReal(const CItem &item, bool isPax = false + // , bool zero_PackSize = false + // , bool zero_MTime = false + ); + + HRESULT Write_Data(const void *data, unsigned size); + HRESULT Write_Data_And_Residual(const void *data, unsigned size); + +public: + UInt64 Pos; + bool IsPosixMode; + // bool IsPrefixAllowed; // it's used only if (IsPosixMode == true) + CTimeOptions TimeOptions; + + void Create(ISequentialOutStream *outStream) + { + Stream = outStream; + } + HRESULT WriteHeader(const CItem &item); + HRESULT Write_AfterDataResidual(UInt64 dataSize); + HRESULT WriteFinishHeader(); + + COutArchive(): + Pos(0), + IsPosixMode(false) + // , IsPrefixAllowed(true) + {} +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarRegister.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarRegister.cpp new file mode 100644 index 0000000..709c191 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarRegister.cpp @@ -0,0 +1,31 @@ +// TarRegister.cpp + +#include "StdAfx.h" + +#include "../../Common/RegisterArc.h" + +#include "TarHandler.h" + +namespace NArchive { +namespace NTar { + +static const Byte k_Signature[] = { 'u', 's', 't', 'a', 'r' }; + +REGISTER_ARC_IO( + "tar", "tar ova", NULL, 0xEE, + k_Signature, + NFileHeader::kUstarMagic_Offset, + NArcInfoFlags::kStartOpen + | NArcInfoFlags::kSymLinks + | NArcInfoFlags::kHardLinks + | NArcInfoFlags::kMTime + | NArcInfoFlags::kMTime_Default + // | NArcInfoTimeFlags::kCTime + // | NArcInfoTimeFlags::kATime + , TIME_PREC_TO_ARC_FLAGS_MASK (NFileTimeType::kWindows) + | TIME_PREC_TO_ARC_FLAGS_MASK (NFileTimeType::kUnix) + | TIME_PREC_TO_ARC_FLAGS_MASK (NFileTimeType::k1ns) + | TIME_PREC_TO_ARC_FLAGS_TIME_DEFAULT (NFileTimeType::kUnix) + , IsArc_Tar) + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarUpdate.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarUpdate.cpp new file mode 100644 index 0000000..d4ed772 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarUpdate.cpp @@ -0,0 +1,563 @@ +// TarUpdate.cpp + +#include "StdAfx.h" + +// #include + +#include "../../../Windows/TimeUtils.h" + +#include "../../Common/LimitedStreams.h" +#include "../../Common/ProgressUtils.h" +#include "../../Common/StreamUtils.h" + +#include "../../Compress/CopyCoder.h" + +#include "TarOut.h" +#include "TarUpdate.h" + +namespace NArchive { +namespace NTar { + +static void FILETIME_To_PaxTime(const FILETIME &ft, CPaxTime &pt) +{ + UInt32 ns; + pt.Sec = NWindows::NTime::FileTime_To_UnixTime64_and_Quantums(ft, ns); + pt.Ns = ns * 100; + pt.NumDigits = 7; +} + + +HRESULT Prop_To_PaxTime(const NWindows::NCOM::CPropVariant &prop, CPaxTime &pt) +{ + pt.Clear(); + if (prop.vt == VT_EMPTY) + { + // pt.Sec = 0; + return S_OK; + } + if (prop.vt != VT_FILETIME) + return E_INVALIDARG; + { + UInt32 ns; + pt.Sec = NWindows::NTime::FileTime_To_UnixTime64_and_Quantums(prop.filetime, ns); + ns *= 100; + pt.NumDigits = 7; + const unsigned prec = prop.wReserved1; + if (prec >= k_PropVar_TimePrec_Base) + { + pt.NumDigits = (int)(prec - k_PropVar_TimePrec_Base); + if (prop.wReserved2 < 100) + ns += prop.wReserved2; + } + pt.Ns = ns; + return S_OK; + } +} + + +static HRESULT GetTime(IStreamGetProp *getProp, UInt32 pid, CPaxTime &pt) +{ + pt.Clear(); + NWindows::NCOM::CPropVariant prop; + RINOK(getProp->GetProperty(pid, &prop)) + return Prop_To_PaxTime(prop, pt); +} + + +static HRESULT GetUser(IStreamGetProp *getProp, + UInt32 pidName, UInt32 pidId, AString &name, UInt32 &id, + UINT codePage, unsigned utfFlags) +{ + // printf("\nGetUser\n"); + // we keep old values, if both GetProperty() return VT_EMPTY + // we clear old values, if any of GetProperty() returns non-VT_EMPTY; + bool isSet = false; + { + NWindows::NCOM::CPropVariant prop; + RINOK(getProp->GetProperty(pidId, &prop)) + if (prop.vt == VT_UI4) + { + isSet = true; + id = prop.ulVal; + name.Empty(); + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + } + { + NWindows::NCOM::CPropVariant prop; + RINOK(getProp->GetProperty(pidName, &prop)) + if (prop.vt == VT_BSTR) + { + const UString s = prop.bstrVal; + Get_AString_From_UString(s, name, codePage, utfFlags); + // printf("\ngetProp->GetProperty(pidName, &prop) : %s" , name.Ptr()); + if (!isSet) + id = 0; + } + else if (prop.vt == VT_UI4) + { + id = prop.ulVal; + name.Empty(); + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + } + return S_OK; +} + + +/* +static HRESULT GetDevice(IStreamGetProp *getProp, + UInt32 &majo, UInt32 &mino, bool &majo_defined, bool &mino_defined) +{ + NWindows::NCOM::CPropVariant prop; + RINOK(getProp->GetProperty(kpidDevice, &prop)); + if (prop.vt == VT_EMPTY) + return S_OK; + if (prop.vt != VT_UI8) + return E_INVALIDARG; + { + printf("\nTarUpdate.cpp :: GetDevice()\n"); + const UInt64 v = prop.uhVal.QuadPart; + majo = MY_dev_major(v); + mino = MY_dev_minor(v); + majo_defined = true; + mino_defined = true; + } + return S_OK; +} +*/ + +static HRESULT GetDevice(IStreamGetProp *getProp, + UInt32 pid, UInt32 &id, bool &defined) +{ + defined = false; + NWindows::NCOM::CPropVariant prop; + RINOK(getProp->GetProperty(pid, &prop)) + if (prop.vt == VT_EMPTY) + return S_OK; + if (prop.vt == VT_UI4) + { + id = prop.ulVal; + defined = true; + return S_OK; + } + return E_INVALIDARG; +} + + +HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, + const CObjectVector &inputItems, + const CObjectVector &updateItems, + const CUpdateOptions &options, + IArchiveUpdateCallback *updateCallback) +{ + COutArchive outArchive; + outArchive.Create(outStream); + outArchive.Pos = 0; + outArchive.IsPosixMode = options.PosixMode; + outArchive.TimeOptions = options.TimeOptions; + + Z7_DECL_CMyComPtr_QI_FROM(IOutStream, outSeekStream, outStream) + Z7_DECL_CMyComPtr_QI_FROM(IStreamSetRestriction, setRestriction, outStream) + Z7_DECL_CMyComPtr_QI_FROM(IArchiveUpdateCallbackFile, opCallback, outStream) + + if (outSeekStream) + { + /* + // for debug + Byte buf[1 << 14]; + memset (buf, 0, sizeof(buf)); + RINOK(outStream->Write(buf, sizeof(buf), NULL)); + */ + // we need real outArchive.Pos, if outSeekStream->SetSize() will be used. + RINOK(outSeekStream->Seek(0, STREAM_SEEK_CUR, &outArchive.Pos)) + } + if (setRestriction) + RINOK(setRestriction->SetRestriction(0, 0)) + + UInt64 complexity = 0; + + unsigned i; + for (i = 0; i < updateItems.Size(); i++) + { + const CUpdateItem &ui = updateItems[i]; + if (ui.NewData) + { + if (ui.Size == (UInt64)(Int64)-1) + break; + complexity += ui.Size; + } + else + complexity += inputItems[(unsigned)ui.IndexInArc].Get_FullSize_Aligned(); + } + + if (i == updateItems.Size()) + RINOK(updateCallback->SetTotal(complexity)) + + CMyComPtr2_Create lps; + lps->Init(updateCallback, true); + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create inStreamLimited; + inStreamLimited->SetStream(inStream); + + complexity = 0; + + // const int kNumReduceDigits = -1; // for debug + + for (i = 0;; i++) + { + lps->InSize = lps->OutSize = complexity; + RINOK(lps->SetCur()) + + if (i == updateItems.Size()) + { + if (outSeekStream && setRestriction) + RINOK(setRestriction->SetRestriction(0, 0)) + return outArchive.WriteFinishHeader(); + } + + const CUpdateItem &ui = updateItems[i]; + CItem item; + + if (ui.NewProps) + { + item.SetMagic_Posix(options.PosixMode); + item.Name = ui.Name; + item.User = ui.User; + item.Group = ui.Group; + item.UID = ui.UID; + item.GID = ui.GID; + item.DeviceMajor = ui.DeviceMajor; + item.DeviceMinor = ui.DeviceMinor; + item.DeviceMajor_Defined = ui.DeviceMajor_Defined; + item.DeviceMinor_Defined = ui.DeviceMinor_Defined; + + if (ui.IsDir) + { + item.LinkFlag = NFileHeader::NLinkFlag::kDirectory; + item.PackSize = 0; + } + else + { + item.PackSize = ui.Size; + item.Set_LinkFlag_for_File(ui.Mode); + } + + // 22.00 + item.Mode = ui.Mode & ~(UInt32)MY_LIN_S_IFMT; + item.PaxTimes = ui.PaxTimes; + // item.PaxTimes.ReducePrecison(kNumReduceDigits); // for debug + item.MTime = ui.PaxTimes.MTime.GetSec(); + } + else + item = inputItems[(unsigned)ui.IndexInArc]; + + AString symLink; + if (ui.NewData || ui.NewProps) + { + RINOK(GetPropString(updateCallback, ui.IndexInClient, kpidSymLink, symLink, + options.CodePage, options.UtfFlags, true)) + if (!symLink.IsEmpty()) + { + item.LinkFlag = NFileHeader::NLinkFlag::kSymLink; + item.LinkName = symLink; + } + } + + if (ui.NewData) + { + item.SparseBlocks.Clear(); + item.PackSize = ui.Size; + item.Size = ui.Size; +#if 0 + if (ui.Size == (UInt64)(Int64)-1) + return E_INVALIDARG; +#endif + CMyComPtr fileInStream; + + bool needWrite = true; + + if (!symLink.IsEmpty()) + { + item.PackSize = 0; + item.Size = 0; + } + else + { + const HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream); + + if (res == S_FALSE) + needWrite = false; + else + { + RINOK(res) + + if (!fileInStream) + { + item.PackSize = 0; + item.Size = 0; + } + else + { + Z7_DECL_CMyComPtr_QI_FROM(IStreamGetProp, getProp, fileInStream) + if (getProp) + { + if (options.Write_MTime.Val) RINOK(GetTime(getProp, kpidMTime, item.PaxTimes.MTime)) + if (options.Write_ATime.Val) RINOK(GetTime(getProp, kpidATime, item.PaxTimes.ATime)) + if (options.Write_CTime.Val) RINOK(GetTime(getProp, kpidCTime, item.PaxTimes.CTime)) + + if (options.PosixMode) + { + /* + RINOK(GetDevice(getProp, item.DeviceMajor, item.DeviceMinor, + item.DeviceMajor_Defined, item.DeviceMinor_Defined)); + */ + bool defined = false; + UInt32 val = 0; + RINOK(GetDevice(getProp, kpidDeviceMajor, val, defined)) + if (defined) + { + item.DeviceMajor = val; + item.DeviceMajor_Defined = true; + item.DeviceMinor = 0; + item.DeviceMinor_Defined = false; + RINOK(GetDevice(getProp, kpidDeviceMinor, item.DeviceMinor, item.DeviceMinor_Defined)) + } + } + + RINOK(GetUser(getProp, kpidUser, kpidUserId, item.User, item.UID, options.CodePage, options.UtfFlags)) + RINOK(GetUser(getProp, kpidGroup, kpidGroupId, item.Group, item.GID, options.CodePage, options.UtfFlags)) + + { + NWindows::NCOM::CPropVariant prop; + RINOK(getProp->GetProperty(kpidPosixAttrib, &prop)) + if (prop.vt == VT_EMPTY) + item.Mode = + MY_LIN_S_IRWXO + | MY_LIN_S_IRWXG + | MY_LIN_S_IRWXU + | (ui.IsDir ? MY_LIN_S_IFDIR : MY_LIN_S_IFREG); + else if (prop.vt != VT_UI4) + return E_INVALIDARG; + else + item.Mode = prop.ulVal; + // 21.07 : we clear high file type bits as GNU TAR. + item.Set_LinkFlag_for_File(item.Mode); + item.Mode &= ~(UInt32)MY_LIN_S_IFMT; + } + + { + NWindows::NCOM::CPropVariant prop; + RINOK(getProp->GetProperty(kpidSize, &prop)) + if (prop.vt != VT_UI8) + return E_INVALIDARG; + const UInt64 size = prop.uhVal.QuadPart; + // printf("\nTAR after GetProperty(kpidSize size = %8d\n", (unsigned)size); + item.PackSize = size; + item.Size = size; + } + /* + printf("\nNum digits = %d %d\n", + (int)item.PaxTimes.MTime.NumDigits, + (int)item.PaxTimes.MTime.Ns); + */ + } + else + { + Z7_DECL_CMyComPtr_QI_FROM(IStreamGetProps, getProps, fileInStream) + if (getProps) + { + FILETIME mTime, aTime, cTime; + UInt64 size2; + if (getProps->GetProps(&size2, + options.Write_CTime.Val ? &cTime : NULL, + options.Write_ATime.Val ? &aTime : NULL, + options.Write_MTime.Val ? &mTime : NULL, + NULL) == S_OK) + { + item.PackSize = size2; + item.Size = size2; + if (options.Write_MTime.Val) FILETIME_To_PaxTime(mTime, item.PaxTimes.MTime); + if (options.Write_ATime.Val) FILETIME_To_PaxTime(aTime, item.PaxTimes.ATime); + if (options.Write_CTime.Val) FILETIME_To_PaxTime(cTime, item.PaxTimes.CTime); + } + } + } + } + + { + // we must request kpidHardLink after updateCallback->GetStream() + AString hardLink; + RINOK(GetPropString(updateCallback, ui.IndexInClient, kpidHardLink, hardLink, + options.CodePage, options.UtfFlags, true)) + if (!hardLink.IsEmpty()) + { + item.LinkFlag = NFileHeader::NLinkFlag::kHardLink; + item.LinkName = hardLink; + item.PackSize = 0; + item.Size = 0; + fileInStream.Release(); + } + } + } + } + + // item.PaxTimes.ReducePrecison(kNumReduceDigits); // for debug + + if (ui.NewProps) + item.MTime = item.PaxTimes.MTime.GetSec(); + + if (needWrite) + { + if (fileInStream) + // if (item.PackSize == (UInt64)(Int64)-1) + if (item.Size == (UInt64)(Int64)-1) + return E_INVALIDARG; + + const UInt64 headerPos = outArchive.Pos; + // item.PackSize = ((UInt64)1 << 33); // for debug + + if (outSeekStream && setRestriction) + RINOK(setRestriction->SetRestriction(outArchive.Pos, (UInt64)(Int64)-1)) + + RINOK(outArchive.WriteHeader(item)) + if (fileInStream) + { + for (unsigned numPasses = 0;; numPasses++) + { + // printf("\nTAR numPasses = %d" " old size = %8d\n", numPasses, (unsigned)item.PackSize); + /* we support 2 attempts to write header: + pass-0: main pass: + pass-1: additional pass, if size_of_file and size_of_header are changed */ + if (numPasses >= 2) + { + // opRes = NArchive::NUpdate::NOperationResult::kError_FileChanged; + // break; + return E_FAIL; + } + + const UInt64 dataPos = outArchive.Pos; + RINOK(copyCoder.Interface()->Code(fileInStream, outStream, NULL, NULL, lps)) + outArchive.Pos += copyCoder->TotalSize; + RINOK(outArchive.Write_AfterDataResidual(copyCoder->TotalSize)) + // printf("\nTAR after Code old size = %8d copyCoder->TotalSize = %8d \n", (unsigned)item.PackSize, (unsigned)copyCoder->TotalSize); + // if (numPasses >= 10) // for debug + if (copyCoder->TotalSize == item.PackSize) + break; + + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kOutArcIndex, (UInt32)ui.IndexInClient, + NUpdateNotifyOp::kInFileChanged)) + } + + if (!outSeekStream) + return E_FAIL; + const UInt64 nextPos = outArchive.Pos; + RINOK(outSeekStream->Seek(-(Int64)(nextPos - headerPos), STREAM_SEEK_CUR, NULL)) + outArchive.Pos = headerPos; + item.PackSize = copyCoder->TotalSize; + + RINOK(outArchive.WriteHeader(item)) + + // if (numPasses >= 10) // for debug + if (outArchive.Pos == dataPos) + { + const UInt64 alignedSize = nextPos - dataPos; + if (alignedSize != 0) + { + RINOK(outSeekStream->Seek((Int64)alignedSize, STREAM_SEEK_CUR, NULL)) + outArchive.Pos += alignedSize; + } + break; + } + + // size of header was changed. + // we remove data after header and try new attempt, if required + Z7_DECL_CMyComPtr_QI_FROM(IInStream, fileSeekStream, fileInStream) + if (!fileSeekStream) + return E_FAIL; + RINOK(InStream_SeekToBegin(fileSeekStream)) + RINOK(outSeekStream->SetSize(outArchive.Pos)) + if (item.PackSize == 0) + break; + } + } + } + + complexity += item.PackSize; + fileInStream.Release(); + RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)) + } + else + { + // (ui.NewData == false) + + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kInArcIndex, (UInt32)ui.IndexInArc, + NUpdateNotifyOp::kReplicate)) + } + + const CItemEx &existItem = inputItems[(unsigned)ui.IndexInArc]; + UInt64 size, pos; + + if (ui.NewProps) + { + // memcpy(item.Magic, NFileHeader::NMagic::kEmpty, 8); + + if (!symLink.IsEmpty()) + { + item.PackSize = 0; + item.Size = 0; + } + else + { + if (ui.IsDir == existItem.IsDir()) + item.LinkFlag = existItem.LinkFlag; + + item.SparseBlocks = existItem.SparseBlocks; + item.Size = existItem.Size; + item.PackSize = existItem.PackSize; + } + + item.DeviceMajor_Defined = existItem.DeviceMajor_Defined; + item.DeviceMinor_Defined = existItem.DeviceMinor_Defined; + item.DeviceMajor = existItem.DeviceMajor; + item.DeviceMinor = existItem.DeviceMinor; + item.UID = existItem.UID; + item.GID = existItem.GID; + + RINOK(outArchive.WriteHeader(item)) + size = existItem.Get_PackSize_Aligned(); + pos = existItem.Get_DataPos(); + } + else + { + size = existItem.Get_FullSize_Aligned(); + pos = existItem.HeaderPos; + } + + if (size != 0) + { + RINOK(InStream_SeekSet(inStream, pos)) + inStreamLimited->Init(size); + if (outSeekStream && setRestriction) + RINOK(setRestriction->SetRestriction(0, 0)) + // 22.00 : we copy Residual data from old archive to new archive instead of zeroing + RINOK(copyCoder.Interface()->Code(inStreamLimited, outStream, NULL, NULL, lps)) + if (copyCoder->TotalSize != size) + return E_FAIL; + outArchive.Pos += size; + // RINOK(outArchive.Write_AfterDataResidual(existItem.PackSize)); + complexity += size; + } + } + } +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarUpdate.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarUpdate.h new file mode 100644 index 0000000..f6c2e77 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Tar/TarUpdate.h @@ -0,0 +1,74 @@ +// TarUpdate.h + +#ifndef ZIP7_INC_TAR_UPDATE_H +#define ZIP7_INC_TAR_UPDATE_H + +#include "../IArchive.h" + +#include "TarItem.h" + +namespace NArchive { +namespace NTar { + +struct CUpdateItem +{ + int IndexInArc; + unsigned IndexInClient; + UInt64 Size; + // Int64 MTime; + UInt32 Mode; + bool NewData; + bool NewProps; + bool IsDir; + bool DeviceMajor_Defined; + bool DeviceMinor_Defined; + UInt32 UID; + UInt32 GID; + UInt32 DeviceMajor; + UInt32 DeviceMinor; + AString Name; + AString User; + AString Group; + + CPaxTimes PaxTimes; + + CUpdateItem(): + Size(0), + IsDir(false), + DeviceMajor_Defined(false), + DeviceMinor_Defined(false), + UID(0), + GID(0) + {} +}; + + +struct CUpdateOptions +{ + UINT CodePage; + unsigned UtfFlags; + bool PosixMode; + CBoolPair Write_MTime; + CBoolPair Write_ATime; + CBoolPair Write_CTime; + CTimeOptions TimeOptions; +}; + + +HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, + const CObjectVector &inputItems, + const CObjectVector &updateItems, + const CUpdateOptions &options, + IArchiveUpdateCallback *updateCallback); + +HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, AString &res, + UINT codePage, unsigned utfFlags, bool convertSlash); + +HRESULT Prop_To_PaxTime(const NWindows::NCOM::CPropVariant &prop, CPaxTime &pt); + +void Get_AString_From_UString(const UString &s, AString &res, + UINT codePage, unsigned utfFlags); + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/StdAfx.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/StdAfx.h new file mode 100644 index 0000000..035267c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/StdAfx.h @@ -0,0 +1,11 @@ +// StdAfx.h + +#ifndef ZIP7_INC_STDAFX_H +#define ZIP7_INC_STDAFX_H + +#if defined(_MSC_VER) && _MSC_VER >= 1800 +#pragma warning(disable : 4464) // relative include path contains '..' +#endif +#include "../../../Common/Common.h" + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipAddCommon.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipAddCommon.cpp new file mode 100644 index 0000000..43d68c7 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipAddCommon.cpp @@ -0,0 +1,484 @@ +// ZipAddCommon.cpp + +#include "StdAfx.h" + +#include "../../../../C/7zCrc.h" +#include "../../../../C/Alloc.h" + +#include "../../../Windows/PropVariant.h" + +#include "../../ICoder.h" +#include "../../IPassword.h" +#include "../../MyVersion.h" + +#include "../../Common/CreateCoder.h" +#include "../../Common/StreamObjects.h" +#include "../../Common/StreamUtils.h" + +#include "../../Compress/LzmaEncoder.h" +#include "../../Compress/PpmdZip.h" +#include "../../Compress/XzEncoder.h" + +#include "../Common/InStreamWithCRC.h" + +#include "ZipAddCommon.h" +#include "ZipHeader.h" + +namespace NArchive { +namespace NZip { + +using namespace NFileHeader; + + +static const unsigned kLzmaPropsSize = 5; +static const unsigned kLzmaHeaderSize = 4 + kLzmaPropsSize; + +Z7_CLASS_IMP_NOQIB_3( + CLzmaEncoder + , ICompressCoder + , ICompressSetCoderProperties + , ICompressSetCoderPropertiesOpt +) +public: + CMyComPtr2 Encoder; + Byte Header[kLzmaHeaderSize]; +}; + +Z7_COM7F_IMF(CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)) +{ + Encoder.Create_if_Empty(); + CMyComPtr2_Create outStream; + outStream->Init(Header + 4, kLzmaPropsSize); + RINOK(Encoder->SetCoderProperties(propIDs, props, numProps)) + RINOK(Encoder->WriteCoderProperties(outStream)) + if (outStream->GetPos() != kLzmaPropsSize) + return E_FAIL; + Header[0] = MY_VER_MAJOR; + Header[1] = MY_VER_MINOR; + Header[2] = kLzmaPropsSize; + Header[3] = 0; + return S_OK; +} + +Z7_COM7F_IMF(CLzmaEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)) +{ + return Encoder->SetCoderPropertiesOpt(propIDs, props, numProps); +} + +Z7_COM7F_IMF(CLzmaEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + RINOK(WriteStream(outStream, Header, kLzmaHeaderSize)) + return Encoder.Interface()->Code(inStream, outStream, inSize, outSize, progress); +} + + +CAddCommon::CAddCommon(): + _isLzmaEos(false), + _buf(NULL) + {} + +void CAddCommon::SetOptions(const CCompressionMethodMode &options) +{ + _options = options; +} + +CAddCommon::~CAddCommon() +{ + MidFree(_buf); +} + +static const UInt32 kBufSize = ((UInt32)1 << 16); + +HRESULT CAddCommon::CalcStreamCRC(ISequentialInStream *inStream, UInt32 &resultCRC) +{ + if (!_buf) + { + _buf = (Byte *)MidAlloc(kBufSize); + if (!_buf) + return E_OUTOFMEMORY; + } + + UInt32 crc = CRC_INIT_VAL; + for (;;) + { + UInt32 processed; + RINOK(inStream->Read(_buf, kBufSize, &processed)) + if (processed == 0) + { + resultCRC = CRC_GET_DIGEST(crc); + return S_OK; + } + crc = CrcUpdate(crc, _buf, (size_t)processed); + } +} + + +HRESULT CAddCommon::Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize, + CCompressingResult &opRes) const +{ + // We use Zip64, if unPackSize size is larger than 0xF8000000 to support + // cases when compressed size can be about 3% larger than uncompressed size + + const UInt32 kUnpackZip64Limit = 0xF8000000; + + opRes.UnpackSize = unpackSize; + opRes.PackSize = (UInt64)1 << 60; // we use big value to force Zip64 mode. + + if (unpackSize < kUnpackZip64Limit) + opRes.PackSize = (UInt32)0xFFFFFFFF - 1; // it will not use Zip64 for that size + + if (opRes.PackSize < unpackSize) + opRes.PackSize = unpackSize; + + const Byte method = _options.MethodSequence[0]; + + if (method == NCompressionMethod::kStore && !_options.Password_Defined) + opRes.PackSize = unpackSize; + + opRes.CRC = 0; + + opRes.LzmaEos = false; + + opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default; + opRes.DescriptorMode = outSeqMode; + + if (_options.Password_Defined) + { + opRes.ExtractVersion = NCompressionMethod::kExtractVersion_ZipCrypto; + if (_options.IsAesMode) + opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Aes; + else + { + if (inSeqMode) + opRes.DescriptorMode = true; + } + } + + opRes.Method = method; + Byte ver = 0; + + switch (method) + { + case NCompressionMethod::kStore: break; + case NCompressionMethod::kDeflate: ver = NCompressionMethod::kExtractVersion_Deflate; break; + case NCompressionMethod::kDeflate64: ver = NCompressionMethod::kExtractVersion_Deflate64; break; + case NCompressionMethod::kXz : ver = NCompressionMethod::kExtractVersion_Xz; break; + case NCompressionMethod::kPPMd : ver = NCompressionMethod::kExtractVersion_PPMd; break; + case NCompressionMethod::kBZip2: ver = NCompressionMethod::kExtractVersion_BZip2; break; + case NCompressionMethod::kLZMA : + { + ver = NCompressionMethod::kExtractVersion_LZMA; + const COneMethodInfo *oneMethodMain = &_options._methods[0]; + opRes.LzmaEos = oneMethodMain->Get_Lzma_Eos(); + break; + } + default: break; + } + if (opRes.ExtractVersion < ver) + opRes.ExtractVersion = ver; + + return S_OK; +} + + +HRESULT CAddCommon::Compress( + DECL_EXTERNAL_CODECS_LOC_VARS + ISequentialInStream *inStream, IOutStream *outStream, + bool inSeqMode, bool outSeqMode, + UInt32 fileTime, + UInt64 expectedDataSize, bool expectedDataSize_IsConfirmed, + ICompressProgressInfo *progress, CCompressingResult &opRes) +{ + // opRes.LzmaEos = false; + + if (!inStream) + { + // We can create empty stream here. But it was already implemented in caller code in 9.33+ + return E_INVALIDARG; + } + + CMyComPtr2_Create inCrcStream; + + CMyComPtr inStream2; + if (!inSeqMode) + { + inStream->QueryInterface(IID_IInStream, (void **)&inStream2); + if (!inStream2) + { + // inSeqMode = true; + // inSeqMode must be correct before + return E_FAIL; + } + } + + inCrcStream->SetStream(inStream); + inCrcStream->SetFullSize(expectedDataSize_IsConfirmed ? expectedDataSize : (UInt64)(Int64)-1); + // inCrcStream->Init(); + + unsigned numTestMethods = _options.MethodSequence.Size(); + // numTestMethods != 0 + + bool descriptorMode = outSeqMode; + + // ZipCrypto without descriptor requires additional reading pass for + // inStream to calculate CRC for password check field. + // The descriptor allows to use ZipCrypto check field without CRC (InfoZip's modification). + + if (!outSeqMode) + if (inSeqMode && _options.Password_Defined && !_options.IsAesMode) + descriptorMode = true; + opRes.DescriptorMode = descriptorMode; + + if (numTestMethods > 1) + if (inSeqMode || outSeqMode || !inStream2) + numTestMethods = 1; + + UInt32 crc = 0; + bool crc_IsCalculated = false; + + CFilterCoder::C_OutStream_Releaser outStreamReleaser; + // opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default; + + for (unsigned i = 0; i < numTestMethods; i++) + { + inCrcStream->Init(); + + if (i != 0) + { + // if (inStream2) + { + RINOK(InStream_SeekToBegin(inStream2)) + } + RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL)) + RINOK(outStream->SetSize(0)) + } + + opRes.LzmaEos = false; + opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default; + + const Byte method = _options.MethodSequence[i]; + if (method == NCompressionMethod::kStore && descriptorMode) + { + // we still can create descriptor_mode archives with "Store" method, but they are not good for 100% + return E_NOTIMPL; + } + + bool needCode = true; + + if (_options.Password_Defined) + { + opRes.ExtractVersion = NCompressionMethod::kExtractVersion_ZipCrypto; + + if (!_cryptoStream.IsDefined()) + _cryptoStream.SetFromCls(new CFilterCoder(true)); + + if (_options.IsAesMode) + { + opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Aes; + if (!_cryptoStream->Filter) + { + _cryptoStream->Filter = _filterAesSpec = new NCrypto::NWzAes::CEncoder; + _filterAesSpec->SetKeyMode(_options.AesKeyMode); + RINOK(_filterAesSpec->CryptoSetPassword((const Byte *)(const char *)_options.Password, _options.Password.Len())) + } + RINOK(_filterAesSpec->WriteHeader(outStream)) + } + else + { + if (!_cryptoStream->Filter) + { + _cryptoStream->Filter = _filterSpec = new NCrypto::NZip::CEncoder; + _filterSpec->CryptoSetPassword((const Byte *)(const char *)_options.Password, _options.Password.Len()); + } + + UInt32 check; + + if (descriptorMode) + { + // it's Info-ZIP modification for stream_mode descriptor_mode (bit 3 of the general purpose bit flag is set) + check = (fileTime & 0xFFFF); + } + else + { + if (!crc_IsCalculated) + { + RINOK(CalcStreamCRC(inStream, crc)) + crc_IsCalculated = true; + RINOK(InStream_SeekToBegin(inStream2)) + inCrcStream->Init(); + } + check = (crc >> 16); + } + + RINOK(_filterSpec->WriteHeader_Check16(outStream, (UInt16)check)) + } + + if (method == NCompressionMethod::kStore) + { + needCode = false; + RINOK(_cryptoStream->Code(inCrcStream, outStream, NULL, NULL, progress)) + } + else + { + RINOK(_cryptoStream->SetOutStream(outStream)) + RINOK(_cryptoStream->InitEncoder()) + outStreamReleaser.FilterCoder = _cryptoStream.ClsPtr(); + } + } + + if (needCode) + { + switch (method) + { + case NCompressionMethod::kStore: + { + _copyCoder.Create_if_Empty(); + CMyComPtr outStreamNew; + if (_options.Password_Defined) + outStreamNew = _cryptoStream; + else + outStreamNew = outStream; + RINOK(_copyCoder.Interface()->Code(inCrcStream, outStreamNew, NULL, NULL, progress)) + break; + } + + default: + { + if (!_compressEncoder) + { + CLzmaEncoder *_lzmaEncoder = NULL; + if (method == NCompressionMethod::kLZMA) + { + _compressExtractVersion = NCompressionMethod::kExtractVersion_LZMA; + _lzmaEncoder = new CLzmaEncoder(); + _compressEncoder = _lzmaEncoder; + } + else if (method == NCompressionMethod::kXz) + { + _compressExtractVersion = NCompressionMethod::kExtractVersion_Xz; + NCompress::NXz::CEncoder *encoder = new NCompress::NXz::CEncoder(); + _compressEncoder = encoder; + } + else if (method == NCompressionMethod::kPPMd) + { + _compressExtractVersion = NCompressionMethod::kExtractVersion_PPMd; + NCompress::NPpmdZip::CEncoder *encoder = new NCompress::NPpmdZip::CEncoder(); + _compressEncoder = encoder; + } + else + { + CMethodId methodId; + switch (method) + { + case NCompressionMethod::kBZip2: + methodId = kMethodId_BZip2; + _compressExtractVersion = NCompressionMethod::kExtractVersion_BZip2; + break; + default: + _compressExtractVersion = ((method == NCompressionMethod::kDeflate64) ? + NCompressionMethod::kExtractVersion_Deflate64 : + NCompressionMethod::kExtractVersion_Deflate); + methodId = kMethodId_ZipBase + method; + break; + } + RINOK(CreateCoder_Id( + EXTERNAL_CODECS_LOC_VARS + methodId, true, _compressEncoder)) + if (!_compressEncoder) + return E_NOTIMPL; + + if (method == NCompressionMethod::kDeflate || + method == NCompressionMethod::kDeflate64) + { + } + else if (method == NCompressionMethod::kBZip2) + { + } + } + { + CMyComPtr setCoderProps; + _compressEncoder.QueryInterface(IID_ICompressSetCoderProperties, &setCoderProps); + if (setCoderProps) + { + if (!_options._methods.IsEmpty()) + { + COneMethodInfo *oneMethodMain = &_options._methods[0]; + + RINOK(oneMethodMain->SetCoderProps(setCoderProps, + _options.DataSizeReduce_Defined ? &_options.DataSizeReduce : NULL)) + } + } + } + if (method == NCompressionMethod::kLZMA) + _isLzmaEos = _lzmaEncoder->Encoder->IsWriteEndMark(); + } + + if (method == NCompressionMethod::kLZMA) + opRes.LzmaEos = _isLzmaEos; + + CMyComPtr outStreamNew; + if (_options.Password_Defined) + outStreamNew = _cryptoStream; + else + outStreamNew = outStream; + if (_compressExtractVersion > opRes.ExtractVersion) + opRes.ExtractVersion = _compressExtractVersion; + + { + CMyComPtr optProps; + _compressEncoder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void **)&optProps); + if (optProps) + { + const PROPID propID = NCoderPropID::kExpectedDataSize; + NWindows::NCOM::CPropVariant prop = (UInt64)expectedDataSize; + RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1)) + } + } + + try { + RINOK(_compressEncoder->Code(inCrcStream, outStreamNew, NULL, NULL, progress)) + } catch (...) { return E_FAIL; } + break; + } + } // switch end + + if (_options.Password_Defined) + { + RINOK(_cryptoStream->OutStreamFinish()) + } + } + + if (_options.Password_Defined) + { + if (_options.IsAesMode) + { + RINOK(_filterAesSpec->WriteFooter(outStream)) + } + } + + RINOK(outStream->Seek(0, STREAM_SEEK_CUR, &opRes.PackSize)) + + { + opRes.CRC = inCrcStream->GetCRC(); + opRes.UnpackSize = inCrcStream->GetSize(); + opRes.Method = method; + } + + if (!inCrcStream->WasFinished()) + return E_FAIL; + + if (_options.Password_Defined) + { + if (opRes.PackSize < opRes.UnpackSize + + (_options.IsAesMode ? _filterAesSpec->GetAddPackSize() : NCrypto::NZip::kHeaderSize)) + break; + } + else if (opRes.PackSize < opRes.UnpackSize) + break; + } + + return S_OK; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipAddCommon.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipAddCommon.h new file mode 100644 index 0000000..e72ec7b --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipAddCommon.h @@ -0,0 +1,76 @@ +// ZipAddCommon.h + +#ifndef ZIP7_INC_ZIP_ADD_COMMON_H +#define ZIP7_INC_ZIP_ADD_COMMON_H + +#include "../../ICoder.h" +#include "../../IProgress.h" + +#include "../../Common/CreateCoder.h" +#include "../../Common/FilterCoder.h" + +#include "../../Compress/CopyCoder.h" + +#include "../../Crypto/ZipCrypto.h" +#include "../../Crypto/WzAes.h" + +#include "ZipCompressionMode.h" + +namespace NArchive { +namespace NZip { + +struct CCompressingResult +{ + UInt64 UnpackSize; + UInt64 PackSize; + UInt32 CRC; + UInt16 Method; + Byte ExtractVersion; + bool DescriptorMode; + bool LzmaEos; + + CCompressingResult() + { + // for GCC: + UnpackSize = 0; + } +}; + +class CAddCommon MY_UNCOPYABLE +{ + CCompressionMethodMode _options; + CMyComPtr2 _copyCoder; + + CMyComPtr _compressEncoder; + Byte _compressExtractVersion; + bool _isLzmaEos; + + CMyComPtr2 _cryptoStream; + + NCrypto::NZip::CEncoder *_filterSpec; + NCrypto::NWzAes::CEncoder *_filterAesSpec; + + Byte *_buf; + + HRESULT CalcStreamCRC(ISequentialInStream *inStream, UInt32 &resultCRC); +public: + // CAddCommon(const CCompressionMethodMode &options); + CAddCommon(); + void SetOptions(const CCompressionMethodMode &options); + ~CAddCommon(); + + HRESULT Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize, + CCompressingResult &opRes) const; + + HRESULT Compress( + DECL_EXTERNAL_CODECS_LOC_VARS + ISequentialInStream *inStream, IOutStream *outStream, + bool inSeqMode, bool outSeqMode, + UInt32 fileTime, + UInt64 expectedDataSize, bool expectedDataSize_IsConfirmed, + ICompressProgressInfo *progress, CCompressingResult &opRes); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipCompressionMode.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipCompressionMode.h new file mode 100644 index 0000000..8974261 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipCompressionMode.h @@ -0,0 +1,62 @@ +// CompressionMode.h + +#ifndef ZIP7_INC_ZIP_COMPRESSION_MODE_H +#define ZIP7_INC_ZIP_COMPRESSION_MODE_H + +#include "../../../Common/MyString.h" + +#ifndef Z7_ST +#include "../../../Windows/System.h" +#endif + +#include "../Common/HandlerOut.h" + +namespace NArchive { +namespace NZip { + +const CMethodId kMethodId_ZipBase = 0x040100; +const CMethodId kMethodId_BZip2 = 0x040202; + +struct CBaseProps: public CMultiMethodProps +{ + bool IsAesMode; + Byte AesKeyMode; + + void Init() + { + CMultiMethodProps::Init(); + + IsAesMode = false; + AesKeyMode = 3; + } +}; + +struct CCompressionMethodMode: public CBaseProps +{ + CRecordVector MethodSequence; + AString Password; // _Wipe + bool Password_Defined; + bool Force_SeqOutMode; + bool DataSizeReduce_Defined; + UInt64 DataSizeReduce; + + bool IsRealAesMode() const { return Password_Defined && IsAesMode; } + + CCompressionMethodMode() + { + Password_Defined = false; + Force_SeqOutMode = false; + DataSizeReduce_Defined = false; + DataSizeReduce = 0; + } + +#ifdef Z7_CPP_IS_SUPPORTED_default + CCompressionMethodMode(const CCompressionMethodMode &) = default; + CCompressionMethodMode& operator =(const CCompressionMethodMode &) = default; +#endif + ~CCompressionMethodMode() { Password.Wipe_and_Empty(); } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandler.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandler.cpp new file mode 100644 index 0000000..c6b6e39 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -0,0 +1,1679 @@ +// ZipHandler.cpp + +#include "StdAfx.h" + +#include "../../../Common/ComTry.h" +#include "../../../Common/StringConvert.h" + +#include "../../../Windows/PropVariant.h" +#include "../../../Windows/PropVariantUtils.h" +#include "../../../Windows/TimeUtils.h" + +#include "../../IPassword.h" + +#include "../../Common/FilterCoder.h" +#include "../../Common/LimitedStreams.h" +#include "../../Common/ProgressUtils.h" +#include "../../Common/StreamObjects.h" +#include "../../Common/StreamUtils.h" + +#include "../../Compress/CopyCoder.h" +#ifndef Z7_ZIP_LZFSE_DISABLE +#include "../../Compress/LzfseDecoder.h" +#endif +#include "../../Compress/LzmaDecoder.h" +#include "../../Compress/ImplodeDecoder.h" +#include "../../Compress/PpmdZip.h" +#include "../../Compress/ShrinkDecoder.h" +#include "../../Compress/XzDecoder.h" +#include "../../Compress/ZstdDecoder.h" + +#include "../../Crypto/WzAes.h" +#include "../../Crypto/ZipCrypto.h" +#include "../../Crypto/ZipStrong.h" + +#include "../Common/ItemNameUtils.h" +#include "../Common/OutStreamWithCRC.h" + + +#include "ZipHandler.h" + +using namespace NWindows; + +namespace NArchive { +namespace NZip { + +static const char * const kHostOS[] = +{ + "FAT" + , "AMIGA" + , "VMS" + , "Unix" + , "VM/CMS" + , "Atari" + , "HPFS" + , "Macintosh" + , "Z-System" + , "CP/M" + , "TOPS-20" + , "NTFS" + , "SMS/QDOS" + , "Acorn" + , "VFAT" + , "MVS" + , "BeOS" + , "Tandem" + , "OS/400" + , "OS/X" +}; + + +const char * const kMethodNames1[kNumMethodNames1] = +{ + "Store" + , "Shrink" + , "Reduce1" + , "Reduce2" + , "Reduce3" + , "Reduce4" + , "Implode" + , NULL // "Tokenize" + , "Deflate" + , "Deflate64" + , "PKImploding" + , NULL + , "BZip2" + , NULL + , "LZMA" + /* + , NULL + , NULL + , NULL + , NULL + , NULL + , "zstd-pk" // deprecated + */ +}; + + +const char * const kMethodNames2[kNumMethodNames2] = +{ + "zstd" + , "MP3" + , "xz" + , "Jpeg" + , "WavPack" + , "PPMd" + , "LZFSE" // , "WzAES" +}; + +#define kMethod_AES "AES" +#define kMethod_ZipCrypto "ZipCrypto" +#define kMethod_StrongCrypto "StrongCrypto" + +static const char * const kDeflateLevels[4] = +{ + "Normal" + , "Maximum" + , "Fast" + , "Fastest" +}; + + +static const CUInt32PCharPair g_HeaderCharacts[] = +{ + { 0, "Encrypt" }, + { 3, "Descriptor" }, + // { 4, "Enhanced" }, + // { 5, "Patched" }, + { 6, kMethod_StrongCrypto }, + { 11, "UTF8" }, + { 14, "Alt" } +}; + +struct CIdToNamePair +{ + unsigned Id; + const char *Name; +}; + + +static const CIdToNamePair k_StrongCryptoPairs[] = +{ + { NStrongCrypto_AlgId::kDES, "DES" }, + { NStrongCrypto_AlgId::kRC2old, "RC2a" }, + { NStrongCrypto_AlgId::k3DES168, "3DES-168" }, + { NStrongCrypto_AlgId::k3DES112, "3DES-112" }, + { NStrongCrypto_AlgId::kAES128, "pkAES-128" }, + { NStrongCrypto_AlgId::kAES192, "pkAES-192" }, + { NStrongCrypto_AlgId::kAES256, "pkAES-256" }, + { NStrongCrypto_AlgId::kRC2, "RC2" }, + { NStrongCrypto_AlgId::kBlowfish, "Blowfish" }, + { NStrongCrypto_AlgId::kTwofish, "Twofish" }, + { NStrongCrypto_AlgId::kRC4, "RC4" } +}; + +static const char *FindNameForId(const CIdToNamePair *pairs, unsigned num, unsigned id) +{ + for (unsigned i = 0; i < num; i++) + { + const CIdToNamePair &pair = pairs[i]; + if (id == pair.Id) + return pair.Name; + } + return NULL; +} + + +static const Byte kProps[] = +{ + kpidPath, + kpidIsDir, + kpidSize, + kpidPackSize, + kpidMTime, + kpidCTime, + kpidATime, + kpidAttrib, + // kpidPosixAttrib, + kpidEncrypted, + kpidComment, + kpidCRC, + kpidMethod, + kpidCharacts, + kpidHostOS, + kpidUnpackVer, + kpidVolumeIndex, + kpidOffset + // kpidIsAltStream + // , kpidChangeTime // for debug + // , 255 // for debug +}; + +static const Byte kArcProps[] = +{ + kpidEmbeddedStubSize, + kpidBit64, + kpidComment, + kpidCharacts, + kpidTotalPhySize, + kpidIsVolume, + kpidVolumeIndex, + kpidNumVolumes +}; + +CHandler::CHandler() +{ + InitMethodProps(); +} + +static AString BytesToString(const CByteBuffer &data) +{ + AString s; + s.SetFrom_CalcLen((const char *)(const Byte *)data, (unsigned)data.Size()); + return s; +} + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) +{ + COM_TRY_BEGIN + NWindows::NCOM::CPropVariant prop; + switch (propID) + { + case kpidBit64: if (m_Archive.IsZip64) prop = m_Archive.IsZip64; break; + case kpidComment: if (m_Archive.ArcInfo.Comment.Size() != 0) prop = MultiByteToUnicodeString(BytesToString(m_Archive.ArcInfo.Comment), CP_ACP); break; + + case kpidPhySize: prop = m_Archive.GetPhySize(); break; + case kpidOffset: prop = m_Archive.GetOffset(); break; + + case kpidEmbeddedStubSize: + { + UInt64 stubSize = m_Archive.GetEmbeddedStubSize(); + if (stubSize != 0) + prop = stubSize; + break; + } + + case kpidTotalPhySize: if (m_Archive.IsMultiVol) prop = m_Archive.Vols.TotalBytesSize; break; + case kpidVolumeIndex: if (m_Archive.IsMultiVol) prop = (UInt32)m_Archive.Vols.StartVolIndex; break; + case kpidIsVolume: if (m_Archive.IsMultiVol) prop = true; break; + case kpidNumVolumes: if (m_Archive.IsMultiVol) prop = (UInt32)m_Archive.Vols.Streams.Size(); break; + + case kpidCharacts: + { + AString s; + + if (m_Archive.LocalsWereRead) + { + s.Add_OptSpaced("Local"); + + if (m_Archive.LocalsCenterMerged) + s.Add_OptSpaced("Central"); + } + + if (m_Archive.IsZip64) + s.Add_OptSpaced("Zip64"); + + if (m_Archive.IsCdUnsorted) + s.Add_OptSpaced("Unsorted_CD"); + + if (m_Archive.IsApk) + s.Add_OptSpaced("apk"); + + if (m_Archive.ExtraMinorError) + s.Add_OptSpaced("Minor_Extra_ERROR"); + + if (!s.IsEmpty()) + prop = s; + break; + } + + case kpidWarningFlags: + { + UInt32 v = 0; + // if (m_Archive.ExtraMinorError) v |= kpv_ErrorFlags_HeadersError; + if (m_Archive.HeadersWarning) v |= kpv_ErrorFlags_HeadersError; + if (v != 0) + prop = v; + break; + } + + case kpidWarning: + { + AString s; + if (m_Archive.Overflow32bit) + s.Add_OptSpaced("32-bit overflow in headers"); + if (m_Archive.Cd_NumEntries_Overflow_16bit) + s.Add_OptSpaced("16-bit overflow for number of files in headers"); + if (!s.IsEmpty()) + prop = s; + break; + } + + case kpidError: + { + if (!m_Archive.Vols.MissingName.IsEmpty()) + { + UString s("Missing volume : "); + s += m_Archive.Vols.MissingName; + prop = s; + } + break; + } + + case kpidErrorFlags: + { + UInt32 v = 0; + if (!m_Archive.IsArc) v |= kpv_ErrorFlags_IsNotArc; + if (m_Archive.HeadersError) v |= kpv_ErrorFlags_HeadersError; + if (m_Archive.UnexpectedEnd) v |= kpv_ErrorFlags_UnexpectedEnd; + if (m_Archive.ArcInfo.Base < 0) + { + /* We try to support case when we have sfx-zip with embedded stub, + but the stream has access only to zip part. + In that case we ignore UnavailableStart error. + maybe we must show warning in that case. */ + UInt64 stubSize = m_Archive.GetEmbeddedStubSize(); + if (stubSize < (UInt64)-m_Archive.ArcInfo.Base) + v |= kpv_ErrorFlags_UnavailableStart; + } + if (m_Archive.NoCentralDir) v |= kpv_ErrorFlags_UnconfirmedStart; + prop = v; + break; + } + + case kpidReadOnly: + { + if (m_Archive.IsOpen()) + if (!m_Archive.CanUpdate()) + prop = true; + break; + } + + // case kpidIsAltStream: prop = true; break; + default: break; + } + return prop.Detach(value); + COM_TRY_END +} + +Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) +{ + *numItems = m_Items.Size(); + return S_OK; +} + + +static bool NtfsUnixTimeToProp(bool fromCentral, + const CExtraBlock &extra, + unsigned ntfsIndex, unsigned unixIndex, NWindows::NCOM::CPropVariant &prop) +{ + { + FILETIME ft; + if (extra.GetNtfsTime(ntfsIndex, ft)) + { + PropVariant_SetFrom_NtfsTime(prop, ft); + return true; + } + } + { + UInt32 unixTime = 0; + if (!extra.GetUnixTime(fromCentral, unixIndex, unixTime)) + return false; + /* + // we allow unixTime == 0 + if (unixTime == 0) + return false; + */ + PropVariant_SetFrom_UnixTime(prop, unixTime); + return true; + } +} + + +Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value)) +{ + COM_TRY_BEGIN + NWindows::NCOM::CPropVariant prop; + const CItemEx &item = m_Items[index]; + const CExtraBlock &extra = item.GetMainExtra(); + + switch (propID) + { + case kpidPath: + { + UString res; + item.GetUnicodeString(res, item.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(res, + item.Is_MadeBy_Unix() // useBackslashReplacement + ); + /* + if (item.ParentOfAltStream >= 0) + { + const CItemEx &prevItem = m_Items[item.ParentOfAltStream]; + UString prevName; + prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName); + if (res.IsPrefixedBy(prevName)) + if (IsString1PrefixedByString2(res.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM)) + { + res.Delete(prevName.Len(), (unsigned)strlen(k_SpecName_NTFS_STREAM)); + res.Insert(prevName.Len(), L":"); + } + } + */ + prop = res; + break; + } + + case kpidIsDir: prop = item.IsDir(); break; + case kpidSize: + { + if (!item.IsBadDescriptor()) + prop = item.Size; + break; + } + + case kpidPackSize: prop = item.PackSize; break; + + case kpidCTime: + NtfsUnixTimeToProp(item.FromCentral, extra, + NFileHeader::NNtfsExtra::kCTime, + NFileHeader::NUnixTime::kCTime, prop); + break; + + case kpidATime: + NtfsUnixTimeToProp(item.FromCentral, extra, + NFileHeader::NNtfsExtra::kATime, + NFileHeader::NUnixTime::kATime, prop); + break; + + case kpidMTime: + { + if (!NtfsUnixTimeToProp(item.FromCentral, extra, + NFileHeader::NNtfsExtra::kMTime, + NFileHeader::NUnixTime::kMTime, prop)) + { + if (item.Time != 0) + PropVariant_SetFrom_DosTime(prop, item.Time); + } + break; + } + + case kpidTimeType: + { + FILETIME ft; + UInt32 unixTime; + UInt32 type; + if (extra.GetNtfsTime(NFileHeader::NNtfsExtra::kMTime, ft)) + type = NFileTimeType::kWindows; + else if (extra.GetUnixTime(item.FromCentral, NFileHeader::NUnixTime::kMTime, unixTime)) + type = NFileTimeType::kUnix; + else + type = NFileTimeType::kDOS; + prop = type; + break; + } + + /* + // for debug to get Dos time values: + case kpidChangeTime: if (item.Time != 0) PropVariant_SetFrom_DosTime(prop, item.Time); break; + // for debug + // time difference (dos - utc) + case 255: + { + if (NtfsUnixTimeToProp(item.FromCentral, extra, + NFileHeader::NNtfsExtra::kMTime, + NFileHeader::NUnixTime::kMTime, prop)) + { + FILETIME localFileTime; + if (item.Time != 0 && NTime::DosTime_To_FileTime(item.Time, localFileTime)) + { + UInt64 t1 = FILETIME_To_UInt64(prop.filetime); + UInt64 t2 = FILETIME_To_UInt64(localFileTime); + prop.Set_Int64(t2 - t1); + } + } + break; + } + */ + + case kpidAttrib: prop = item.GetWinAttrib(); break; + + case kpidPosixAttrib: + { + UInt32 attrib; + if (item.GetPosixAttrib(attrib)) + prop = attrib; + break; + } + + case kpidEncrypted: prop = item.IsEncrypted(); break; + + case kpidComment: + { + if (item.Comment.Size() != 0) + { + UString res; + item.GetUnicodeString(res, BytesToString(item.Comment), true, _forceCodePage, _specifiedCodePage); + prop = res; + } + break; + } + + case kpidCRC: if (item.IsThereCrc()) prop = item.Crc; break; + + case kpidMethod: + { + AString m; + bool isWzAes = false; + unsigned id = item.Method; + + if (id == NFileHeader::NCompressionMethod::kWzAES) + { + CWzAesExtra aesField; + if (extra.GetWzAes(aesField)) + { + m += kMethod_AES; + m.Add_Minus(); + m.Add_UInt32(((unsigned)aesField.Strength + 1) * 64); + id = aesField.Method; + isWzAes = true; + } + } + + if (item.IsEncrypted()) + if (!isWzAes) + { + if (item.IsStrongEncrypted()) + { + CStrongCryptoExtra f; + f.AlgId = 0; + if (extra.GetStrongCrypto(f)) + { + const char *s = FindNameForId(k_StrongCryptoPairs, Z7_ARRAY_SIZE(k_StrongCryptoPairs), f.AlgId); + if (s) + m += s; + else + { + m += kMethod_StrongCrypto; + m.Add_Colon(); + m.Add_UInt32(f.AlgId); + } + if (f.CertificateIsUsed()) + m += "-Cert"; + } + else + m += kMethod_StrongCrypto; + } + else + m += kMethod_ZipCrypto; + } + + m.Add_Space_if_NotEmpty(); + + { + const char *s = NULL; + if (id < kNumMethodNames1) + s = kMethodNames1[id]; + else + { + const int id2 = (int)id - (int)kMethodNames2Start; + if (id2 >= 0 && (unsigned)id2 < kNumMethodNames2) + s = kMethodNames2[id2]; + } + if (s) + m += s; + else + m.Add_UInt32(id); + } + { + unsigned level = item.GetDeflateLevel(); + if (level != 0) + { + if (id == NFileHeader::NCompressionMethod::kLZMA) + { + if (level & 1) + m += ":eos"; + level &= ~(unsigned)1; + } + else if (id == NFileHeader::NCompressionMethod::kDeflate) + { + m.Add_Colon(); + m += kDeflateLevels[level]; + level = 0; + } + + if (level != 0) + { + m += ":v"; + m.Add_UInt32(level); + } + } + } + + prop = m; + break; + } + + case kpidCharacts: + { + AString s; + + if (item.FromLocal) + { + s.Add_OptSpaced("Local"); + + item.LocalExtra.PrintInfo(s); + + if (item.FromCentral) + { + s.Add_OptSpaced(":"); + s.Add_OptSpaced("Central"); + } + } + + if (item.FromCentral) + { + item.CentralExtra.PrintInfo(s); + } + + UInt32 flags = item.Flags; + flags &= ~(unsigned)6; // we don't need compression related bits here. + + if (flags != 0) + { + const AString s2 = FlagsToString(g_HeaderCharacts, Z7_ARRAY_SIZE(g_HeaderCharacts), flags); + if (!s2.IsEmpty()) + { + if (!s.IsEmpty()) + s.Add_OptSpaced(":"); + s.Add_OptSpaced(s2); + } + } + + if (item.IsBadDescriptor()) + s.Add_OptSpaced("Descriptor_ERROR"); + + if (!s.IsEmpty()) + prop = s; + break; + } + + case kpidHostOS: + { + if (item.FromCentral) + { + // 18.06: now we use HostOS only from Central::MadeByVersion + const Byte hostOS = item.MadeByVersion.HostOS; + TYPE_TO_PROP(kHostOS, hostOS, prop); + } + break; + } + + case kpidUnpackVer: + prop = (UInt32)item.ExtractVersion.Version; + break; + + case kpidVolumeIndex: + prop = item.Disk; + break; + + case kpidOffset: + prop = item.LocalHeaderPos; + break; + + /* + case kpidIsAltStream: + prop = (bool)(item.ParentOfAltStream >= 0); // item.IsAltStream(); + break; + + case kpidName: + if (item.ParentOfAltStream >= 0) + { + // extract name of stream here + } + break; + */ + default: break; + } + + return prop.Detach(value); + COM_TRY_END +} + + + +/* +Z7_COM7F_IMF(CHandler::GetNumRawProps(UInt32 *numProps) +{ + *numProps = 0; + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID) +{ + UNUSED_VAR(index); + *propID = 0; + *name = 0; + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetParent(UInt32 index, UInt32 *parent, UInt32 *parentType) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; + if (index >= m_Items.Size()) + return S_OK; + const CItemEx &item = m_Items[index]; + + if (item.ParentOfAltStream >= 0) + { + *parentType = NParentType::kAltStream; + *parent = item.ParentOfAltStream; + } + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType) +{ + UNUSED_VAR(index); + UNUSED_VAR(propID); + *data = NULL; + *dataSize = 0; + *propType = 0; + return S_OK; +} + + +void CHandler::MarkAltStreams(CObjectVector &items) +{ + int prevIndex = -1; + UString prevName; + UString name; + + for (unsigned i = 0; i < items.Size(); i++) + { + CItemEx &item = m_Items[i]; + if (item.IsAltStream()) + { + if (prevIndex == -1) + continue; + if (prevName.IsEmpty()) + { + const CItemEx &prevItem = m_Items[prevIndex]; + prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName); + } + name.Empty(); + item.GetUnicodeString(name, item.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(name); + + if (name.IsPrefixedBy(prevName)) + if (IsString1PrefixedByString2(name.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM)) + item.ParentOfAltStream = prevIndex; + } + else + { + prevIndex = i; + prevName.Empty(); + } + } +} +*/ + +Z7_COM7F_IMF(CHandler::Open(IInStream *inStream, + const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *callback)) +{ + COM_TRY_BEGIN + try + { + Close(); + m_Archive.Force_ReadLocals_Mode = _force_OpenSeq; + // m_Archive.Disable_VolsRead = _force_OpenSeq; + // m_Archive.Disable_FindMarker = _force_OpenSeq; + HRESULT res = m_Archive.Open(inStream, maxCheckStartPosition, callback, m_Items); + if (res != S_OK) + { + m_Items.Clear(); + m_Archive.ClearRefs(); // we don't want to clear error flags + } + // MarkAltStreams(m_Items); + return res; + } + catch(...) { Close(); throw; } + COM_TRY_END +} + +Z7_COM7F_IMF(CHandler::Close()) +{ + m_Items.Clear(); + m_Archive.Close(); + return S_OK; +} + + +Z7_CLASS_IMP_NOQIB_3( + CLzmaDecoder + , ICompressCoder + , ICompressSetFinishMode + , ICompressGetInStreamProcessedSize +) +public: + CMyComPtr2_Create Decoder; +}; + +static const unsigned kZipLzmaPropsSize = 4 + LZMA_PROPS_SIZE; + +Z7_COM7F_IMF(CLzmaDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + Byte buf[kZipLzmaPropsSize]; + RINOK(ReadStream_FALSE(inStream, buf, kZipLzmaPropsSize)) + if (buf[2] != LZMA_PROPS_SIZE || buf[3] != 0) + return E_NOTIMPL; + RINOK(Decoder->SetDecoderProperties2(buf + 4, LZMA_PROPS_SIZE)) + UInt64 inSize2 = 0; + if (inSize) + { + inSize2 = *inSize; + if (inSize2 < kZipLzmaPropsSize) + return S_FALSE; + inSize2 -= kZipLzmaPropsSize; + } + return Decoder.Interface()->Code(inStream, outStream, inSize ? &inSize2 : NULL, outSize, progress); +} + +Z7_COM7F_IMF(CLzmaDecoder::SetFinishMode(UInt32 finishMode)) +{ + Decoder->FinishStream = (finishMode != 0); + return S_OK; +} + +Z7_COM7F_IMF(CLzmaDecoder::GetInStreamProcessedSize(UInt64 *value)) +{ + *value = Decoder->GetInputProcessedSize() + kZipLzmaPropsSize; + return S_OK; +} + + + + + + + +struct CMethodItem +{ + unsigned ZipMethod; + CMyComPtr Coder; +}; + + + +class CZipDecoder +{ + CMyComPtr2 _zipCryptoDecoder; + CMyComPtr2 _pkAesDecoder; + CMyComPtr2 _wzAesDecoder; + + CMyComPtr2 filterStream; + CMyComPtr getTextPassword; + CObjectVector methodItems; + + CLzmaDecoder *lzmaDecoderSpec; +public: + CZipDecoder(): + lzmaDecoderSpec(NULL) + {} + + HRESULT Decode( + DECL_EXTERNAL_CODECS_LOC_VARS + CInArchive &archive, const CItemEx &item, + ISequentialOutStream *realOutStream, + IArchiveExtractCallback *extractCallback, + ICompressProgressInfo *compressProgress, + #ifndef Z7_ST + UInt32 numThreads, UInt64 memUsage, + #endif + Int32 &res); +}; + + +static HRESULT SkipStreamData(ISequentialInStream *stream, + ICompressProgressInfo *progress, UInt64 packSize, UInt64 unpackSize, + bool &thereAreData) +{ + thereAreData = false; + const size_t kBufSize = 1 << 12; + Byte buf[kBufSize]; + UInt64 prev = packSize; + for (;;) + { + size_t size = kBufSize; + RINOK(ReadStream(stream, buf, &size)) + if (size == 0) + return S_OK; + thereAreData = true; + packSize += size; + if ((packSize - prev) >= (1 << 22)) + { + prev = packSize; + RINOK(progress->SetRatioInfo(&packSize, &unpackSize)) + } + } +} + + + +Z7_CLASS_IMP_NOQIB_1( + COutStreamWithPadPKCS7 + , ISequentialOutStream +) + CMyComPtr _stream; + UInt64 _size; + UInt64 _padPos; + UInt32 _padSize; + bool _padFailure; +public: + void SetStream(ISequentialOutStream *stream) { _stream = stream; } + void ReleaseStream() { _stream.Release(); } + + // padSize == 0 means (no_pad Mode) + void Init(UInt64 padPos, UInt32 padSize) + { + _padPos = padPos; + _padSize = padSize; + _size = 0; + _padFailure = false; + } + UInt64 GetSize() const { return _size; } + bool WasPadFailure() const { return _padFailure; } +}; + + +Z7_COM7F_IMF(COutStreamWithPadPKCS7::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + UInt32 written = 0; + HRESULT result = S_OK; + if (_size < _padPos) + { + const UInt64 rem = _padPos - _size; + UInt32 num = size; + if (num > rem) + num = (UInt32)rem; + result = _stream->Write(data, num, &written); + _size += written; + if (processedSize) + *processedSize = written; + if (_size != _padPos || result != S_OK) + return result; + size -= written; + data = ((const Byte *)data) + written; + } + _size += size; + written += size; + if (processedSize) + *processedSize = written; + if (_padSize != 0) + for (; size != 0; size--) + { + if (*(const Byte *)data != _padSize) + _padFailure = true; + data = ((const Byte *)data) + 1; + } + return result; +} + + + +HRESULT CZipDecoder::Decode( + DECL_EXTERNAL_CODECS_LOC_VARS + CInArchive &archive, const CItemEx &item, + ISequentialOutStream *realOutStream, + IArchiveExtractCallback *extractCallback, + ICompressProgressInfo *compressProgress, + #ifndef Z7_ST + UInt32 numThreads, UInt64 memUsage, + #endif + Int32 &res) +{ + res = NExtract::NOperationResult::kHeadersError; + + CFilterCoder::C_InStream_Releaser inStreamReleaser; + CFilterCoder::C_Filter_Releaser filterReleaser; + + bool needCRC = true; + bool wzAesMode = false; + bool pkAesMode = false; + + bool badDescriptor = item.IsBadDescriptor(); + if (badDescriptor) + needCRC = false; + + + unsigned id = item.Method; + + CWzAesExtra aesField; + // LZFSE and WinZip's AES use same id - kWzAES. + + if (id == NFileHeader::NCompressionMethod::kWzAES) + { + if (item.GetMainExtra().GetWzAes(aesField)) + { + if (!item.IsEncrypted()) + { + res = NExtract::NOperationResult::kUnsupportedMethod; + return S_OK; + } + wzAesMode = true; + needCRC = aesField.NeedCrc(); + } + } + + if (!wzAesMode) + if (item.IsEncrypted()) + { + if (item.IsStrongEncrypted()) + { + CStrongCryptoExtra f; + if (!item.CentralExtra.GetStrongCrypto(f)) + { + res = NExtract::NOperationResult::kUnsupportedMethod; + return S_OK; + } + pkAesMode = true; + } + } + + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + outStream->Init(needCRC); + + CMyComPtr packStream; + CMyComPtr2_Create inStream; + + { + UInt64 packSize = item.PackSize; + if (wzAesMode) + { + if (packSize < NCrypto::NWzAes::kMacSize) + return S_OK; + packSize -= NCrypto::NWzAes::kMacSize; + } + RINOK(archive.GetItemStream(item, true, packStream)) + if (!packStream) + { + res = NExtract::NOperationResult::kUnavailable; + return S_OK; + } + inStream->SetStream(packStream); + inStream->Init(packSize); + } + + + res = NExtract::NOperationResult::kDataError; + + CMyComPtr cryptoFilter; + + if (item.IsEncrypted()) + { + if (wzAesMode) + { + id = aesField.Method; + _wzAesDecoder.Create_if_Empty(); + cryptoFilter = _wzAesDecoder; + if (!_wzAesDecoder->SetKeyMode(aesField.Strength)) + { + res = NExtract::NOperationResult::kUnsupportedMethod; + return S_OK; + } + } + else if (pkAesMode) + { + _pkAesDecoder.Create_if_Empty(); + cryptoFilter = _pkAesDecoder; + } + else + { + _zipCryptoDecoder.Create_if_Empty(); + cryptoFilter = _zipCryptoDecoder; + } + + CMyComPtr cryptoSetPassword; + RINOK(cryptoFilter.QueryInterface(IID_ICryptoSetPassword, &cryptoSetPassword)) + if (!cryptoSetPassword) + return E_FAIL; + + if (!getTextPassword) + extractCallback->QueryInterface(IID_ICryptoGetTextPassword, (void **)&getTextPassword); + + if (getTextPassword) + { + CMyComBSTR_Wipe password; + RINOK(getTextPassword->CryptoGetTextPassword(&password)) + AString_Wipe charPassword; + if (password) + { +#if 0 && defined(_WIN32) + // do we need UTF-8 passwords here ? + if (item.GetHostOS() == NFileHeader::NHostOS::kUnix // 24.05 + // || item.IsUtf8() // 22.00 + ) + { + // throw 1; + ConvertUnicodeToUTF8((LPCOLESTR)password, charPassword); + } + else +#endif + { + UnicodeStringToMultiByte2(charPassword, (LPCOLESTR)password, CP_ACP); + } + /* + if (wzAesMode || pkAesMode) + { + } + else + { + // PASSWORD encoding for ZipCrypto: + // pkzip25 / WinZip / Windows probably use ANSI + // 7-Zip < 4.43 creates ZIP archives with OEM encoding in password + // 7-Zip >= 4.43 creates ZIP archives only with ASCII characters in password + // 7-Zip < 17.00 uses CP_OEMCP for password decoding + // 7-Zip >= 17.00 uses CP_ACP for password decoding + } + */ + } + HRESULT result = cryptoSetPassword->CryptoSetPassword( + (const Byte *)(const char *)charPassword, charPassword.Len()); + if (result != S_OK) + { + res = NExtract::NOperationResult::kWrongPassword; + return S_OK; + } + } + else + { + res = NExtract::NOperationResult::kWrongPassword; + return S_OK; + // RINOK(cryptoSetPassword->CryptoSetPassword(NULL, 0)); + } + } + + unsigned m; + for (m = 0; m < methodItems.Size(); m++) + if (methodItems[m].ZipMethod == id) + break; + + if (m == methodItems.Size()) + { + CMethodItem mi; + mi.ZipMethod = id; + if (id == NFileHeader::NCompressionMethod::kStore) + mi.Coder = new NCompress::CCopyCoder; + else if (id == NFileHeader::NCompressionMethod::kShrink) + mi.Coder = new NCompress::NShrink::CDecoder; + else if (id == NFileHeader::NCompressionMethod::kImplode) + mi.Coder = new NCompress::NImplode::NDecoder::CCoder; + else if (id == NFileHeader::NCompressionMethod::kLZMA) + { + lzmaDecoderSpec = new CLzmaDecoder; + mi.Coder = lzmaDecoderSpec; + } + else if (id == NFileHeader::NCompressionMethod::kXz) + mi.Coder = new NCompress::NXz::CComDecoder; + else if (id == NFileHeader::NCompressionMethod::kPPMd) + mi.Coder = new NCompress::NPpmdZip::CDecoder(true); + else if (id == NFileHeader::NCompressionMethod::kZstdWz) + mi.Coder = new NCompress::NZstd::CDecoder(); +#ifndef Z7_ZIP_LZFSE_DISABLE + else if (id == NFileHeader::NCompressionMethod::kWzAES) + mi.Coder = new NCompress::NLzfse::CDecoder; +#endif + else + { + CMethodId szMethodID; + if (id == NFileHeader::NCompressionMethod::kBZip2) + szMethodID = kMethodId_BZip2; + else + { + if (id > 0xFF) + { + res = NExtract::NOperationResult::kUnsupportedMethod; + return S_OK; + } + szMethodID = kMethodId_ZipBase + (Byte)id; + } + + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder)) + + if (!mi.Coder) + { + res = NExtract::NOperationResult::kUnsupportedMethod; + return S_OK; + } + } + m = methodItems.Add(mi); + } + + const CMethodItem &mi = methodItems[m]; + ICompressCoder *coder = mi.Coder; + + + #ifndef Z7_ST + { + CMyComPtr setCoderMt; + coder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); + if (setCoderMt) + { + RINOK(setCoderMt->SetNumberOfThreads(numThreads)) + } + } + // if (memUsage != 0) + { + CMyComPtr setMemLimit; + coder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit); + if (setMemLimit) + { + RINOK(setMemLimit->SetMemLimit(memUsage)) + } + } + #endif + + { + CMyComPtr setDecoderProperties; + coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); + if (setDecoderProperties) + { + Byte properties = (Byte)item.Flags; + RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1)) + } + } + + + bool isFullStreamExpected = (!item.HasDescriptor() || item.PackSize != 0); + bool needReminderCheck = false; + + bool dataAfterEnd = false; + bool truncatedError = false; + bool lzmaEosError = false; + bool headersError = false; + bool padError = false; + bool readFromFilter = false; + + const bool useUnpackLimit = (id == NFileHeader::NCompressionMethod::kStore + || !item.HasDescriptor() + || item.Size >= ((UInt64)1 << 32) + || item.LocalExtra.IsZip64 + || item.CentralExtra.IsZip64 + ); + + { + HRESULT result = S_OK; + if (item.IsEncrypted()) + { + if (!filterStream.IsDefined()) + filterStream.SetFromCls(new CFilterCoder(false)); + + filterReleaser.FilterCoder = filterStream.ClsPtr(); + filterStream->Filter = cryptoFilter; + + if (wzAesMode) + { + result = _wzAesDecoder->ReadHeader(inStream); + if (result == S_OK) + { + if (!_wzAesDecoder->Init_and_CheckPassword()) + { + res = NExtract::NOperationResult::kWrongPassword; + return S_OK; + } + } + } + else if (pkAesMode) + { + isFullStreamExpected = false; + result = _pkAesDecoder->ReadHeader(inStream, item.Crc, item.Size); + if (result == S_OK) + { + bool passwOK; + result = _pkAesDecoder->Init_and_CheckPassword(passwOK); + if (result == S_OK && !passwOK) + { + res = NExtract::NOperationResult::kWrongPassword; + return S_OK; + } + } + } + else + { + result = _zipCryptoDecoder->ReadHeader(inStream); + if (result == S_OK) + { + _zipCryptoDecoder->Init_BeforeDecode(); + + /* Info-ZIP modification to ZipCrypto format: + if bit 3 of the general purpose bit flag is set, + it uses high byte of 16-bit File Time. + Info-ZIP code probably writes 2 bytes of File Time. + We check only 1 byte. */ + + // UInt32 v1 = GetUi16(_zipCryptoDecoder->_header + NCrypto::NZip::kHeaderSize - 2); + // UInt32 v2 = (item.HasDescriptor() ? (item.Time & 0xFFFF) : (item.Crc >> 16)); + + Byte v1 = _zipCryptoDecoder->_header[NCrypto::NZip::kHeaderSize - 1]; + Byte v2 = (Byte)(item.HasDescriptor() ? (item.Time >> 8) : (item.Crc >> 24)); + + if (v1 != v2) + { + res = NExtract::NOperationResult::kWrongPassword; + return S_OK; + } + } + } + } + + if (result == S_OK) + { + CMyComPtr setFinishMode; + coder->QueryInterface(IID_ICompressSetFinishMode, (void **)&setFinishMode); + if (setFinishMode) + { + RINOK(setFinishMode->SetFinishMode(BoolToUInt(true))) + } + + const UInt64 coderPackSize = inStream->GetRem(); + + if (id == NFileHeader::NCompressionMethod::kStore && item.IsEncrypted()) + { + // for debug : we can disable this code (kStore + 50), if we want to test CopyCoder+Filter + // here we use filter without CopyCoder + readFromFilter = false; + + COutStreamWithPadPKCS7 *padStreamSpec = NULL; + CMyComPtr padStream; + UInt32 padSize = 0; + + if (pkAesMode) + { + padStreamSpec = new COutStreamWithPadPKCS7; + padStream = padStreamSpec; + padSize = _pkAesDecoder->GetPadSize((UInt32)item.Size); + padStreamSpec->SetStream(outStream); + padStreamSpec->Init(item.Size, padSize); + } + + // Here we decode minimal required size, including padding + const UInt64 expectedSize = item.Size + padSize; + UInt64 size = coderPackSize; + if (item.Size > coderPackSize) + headersError = true; + else if (expectedSize != coderPackSize) + { + headersError = true; + if (coderPackSize > expectedSize) + size = expectedSize; + } + + result = filterStream->Code(inStream, padStream ? + padStream.Interface() : + outStream.Interface(), + NULL, &size, compressProgress); + + if (outStream->GetSize() != item.Size) + truncatedError = true; + + if (pkAesMode) + { + if (padStreamSpec->GetSize() != size) + truncatedError = true; + if (padStreamSpec->WasPadFailure()) + padError = true; + } + } + else + { + if (item.IsEncrypted()) + { + readFromFilter = true; + inStreamReleaser.FilterCoder = filterStream.ClsPtr(); + RINOK(filterStream->SetInStream(inStream)) + + /* IFilter::Init() does nothing in all zip crypto filters. + So we can call any Initialize function in CFilterCoder. */ + + RINOK(filterStream->Init_NoSubFilterInit()) + // RINOK(filterStream->SetOutStreamSize(NULL)); + } + + try { + result = coder->Code(readFromFilter ? + filterStream.Interface() : + inStream.Interface(), + outStream, + isFullStreamExpected ? &coderPackSize : NULL, + // NULL, + useUnpackLimit ? &item.Size : NULL, + compressProgress); + } catch (...) { return E_FAIL; } + + if (result == S_OK) + { + CMyComPtr getInStreamProcessedSize; + coder->QueryInterface(IID_ICompressGetInStreamProcessedSize, (void **)&getInStreamProcessedSize); + if (getInStreamProcessedSize && setFinishMode) + { + UInt64 processed; + RINOK(getInStreamProcessedSize->GetInStreamProcessedSize(&processed)) + if (processed != (UInt64)(Int64)-1) + { + if (pkAesMode) + { + const UInt32 padSize = _pkAesDecoder->GetPadSize((UInt32)processed); + if (processed + padSize > coderPackSize) + truncatedError = true; + else if (processed + padSize < coderPackSize) + dataAfterEnd = true; + else + { + { + // here we check PKCS7 padding data from reminder (it can be inside stream buffer in coder). + CMyComPtr readInStream; + coder->QueryInterface(IID_ICompressReadUnusedFromInBuf, (void **)&readInStream); + // CCopyCoder() for kStore doesn't read data outside of (item.Size) + if (readInStream || id == NFileHeader::NCompressionMethod::kStore) + { + // change pad size, if we support another block size in ZipStrong. + // here we request more data to detect error with data after end. + const UInt32 kBufSize = NCrypto::NZipStrong::kAesPadAllign + 16; + Byte buf[kBufSize]; + UInt32 processedSize = 0; + if (readInStream) + { + RINOK(readInStream->ReadUnusedFromInBuf(buf, kBufSize, &processedSize)) + } + if (processedSize > padSize) + dataAfterEnd = true; + else + { + size_t processedSize2 = kBufSize - processedSize; + result = ReadStream(filterStream, buf + processedSize, &processedSize2); + if (result == S_OK) + { + processedSize2 += processedSize; + if (processedSize2 > padSize) + dataAfterEnd = true; + else if (processedSize2 < padSize) + truncatedError = true; + else + for (unsigned i = 0; i < padSize; i++) + if (buf[i] != padSize) + padError = true; + } + } + } + } + } + } + else + { + if (processed < coderPackSize) + { + if (isFullStreamExpected) + dataAfterEnd = true; + } + else if (processed > coderPackSize) + { + // that case is additional check, that can show the bugs in code (coder) + truncatedError = true; + } + needReminderCheck = isFullStreamExpected; + } + } + } + } + } + + if (result == S_OK && id == NFileHeader::NCompressionMethod::kLZMA) + if (!lzmaDecoderSpec->Decoder->CheckFinishStatus(item.IsLzmaEOS())) + lzmaEosError = true; + } + + if (result == S_FALSE) + return S_OK; + + if (result == E_NOTIMPL) + { + res = NExtract::NOperationResult::kUnsupportedMethod; + return S_OK; + } + + RINOK(result) + } + + bool crcOK = true; + bool authOk = true; + if (needCRC) + crcOK = (outStream->GetCRC() == item.Crc); + + if (useUnpackLimit) + if (outStream->GetSize() != item.Size) + truncatedError = true; + + if (wzAesMode) + { + const UInt64 unpackSize = outStream->GetSize(); + const UInt64 packSize = inStream->GetSize(); + bool thereAreData = false; + // read to the end from filter or from packed stream + if (SkipStreamData(readFromFilter ? + filterStream.Interface() : + inStream.Interface(), + compressProgress, packSize, unpackSize, thereAreData) != S_OK) + authOk = false; + if (needReminderCheck && thereAreData) + dataAfterEnd = true; + + if (inStream->GetRem() != 0) + truncatedError = true; + else + { + inStream->Init(NCrypto::NWzAes::kMacSize); + if (_wzAesDecoder->CheckMac(inStream, authOk) != S_OK) + authOk = false; + } + } + + res = NExtract::NOperationResult::kCRCError; + + if (crcOK && authOk) + { + res = NExtract::NOperationResult::kOK; + + if (dataAfterEnd) + res = NExtract::NOperationResult::kDataAfterEnd; + else if (padError) + res = NExtract::NOperationResult::kCRCError; + else if (truncatedError) + res = NExtract::NOperationResult::kUnexpectedEnd; + else if (headersError) + res = NExtract::NOperationResult::kHeadersError; + else if (lzmaEosError) + res = NExtract::NOperationResult::kHeadersError; + else if (badDescriptor) + res = NExtract::NOperationResult::kUnexpectedEnd; + + // CheckDescriptor() supports only data descriptor with signature and + // it doesn't support "old" pkzip's data descriptor without signature. + // So we disable that check. + /* + if (item.HasDescriptor() && archive.CheckDescriptor(item) != S_OK) + res = NExtract::NOperationResult::kHeadersError; + */ + } + + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback)) +{ + COM_TRY_BEGIN + const bool allFilesMode = (numItems == (UInt32)(Int32)-1); + if (allFilesMode) + numItems = m_Items.Size(); + if (numItems == 0) + return S_OK; + UInt64 total = 0; // , totalPacked = 0; + UInt32 i; + for (i = 0; i < numItems; i++) + { + const CItemEx &item = m_Items[allFilesMode ? i : indices[i]]; + total += item.Size; + // totalPacked += item.PackSize; + } + RINOK(extractCallback->SetTotal(total)) + + CZipDecoder myDecoder; + UInt64 cur_Unpacked, cur_Packed; + + CMyComPtr2_Create lps; + lps->Init(extractCallback, false); + + for (i = 0;; i++, + lps->OutSize += cur_Unpacked, + lps->InSize += cur_Packed) + { + RINOK(lps->SetCur()) + if (i >= numItems) + return S_OK; + const UInt32 index = allFilesMode ? i : indices[i]; + CItemEx item = m_Items[index]; + cur_Unpacked = item.Size; + cur_Packed = item.PackSize; + + const bool isLocalOffsetOK = m_Archive.IsLocalOffsetOK(item); + const bool skip = !isLocalOffsetOK && !item.IsDir(); + const Int32 askMode = skip ? + NExtract::NAskMode::kSkip : testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + + Int32 opRes; + { + CMyComPtr realOutStream; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) + + if (!isLocalOffsetOK) + { + RINOK(extractCallback->PrepareOperation(askMode)) + realOutStream.Release(); + RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kUnavailable)) + continue; + } + + bool headersError = false; + + if (!item.FromLocal) + { + bool isAvail = true; + const HRESULT hres = m_Archive.Read_LocalItem_After_CdItem(item, isAvail, headersError); + if (hres == S_FALSE) + { + if (item.IsDir() || realOutStream || testMode) + { + RINOK(extractCallback->PrepareOperation(askMode)) + realOutStream.Release(); + RINOK(extractCallback->SetOperationResult( + isAvail ? + NExtract::NOperationResult::kHeadersError : + NExtract::NOperationResult::kUnavailable)) + } + continue; + } + RINOK(hres) + } + + if (item.IsDir()) + { + // if (!testMode) + { + RINOK(extractCallback->PrepareOperation(askMode)) + realOutStream.Release(); + RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) + } + continue; + } + + if (!testMode && !realOutStream) + continue; + + RINOK(extractCallback->PrepareOperation(askMode)) + + const HRESULT hres = myDecoder.Decode( + EXTERNAL_CODECS_VARS + m_Archive, item, realOutStream, extractCallback, + lps, + #ifndef Z7_ST + _props._numThreads, _props._memUsage_Decompress, + #endif + opRes); + + RINOK(hres) + // realOutStream.Release(); + + if (opRes == NExtract::NOperationResult::kOK && headersError) + opRes = NExtract::NOperationResult::kHeadersError; + } + RINOK(extractCallback->SetOperationResult(opRes)) + } + + COM_TRY_END +} + +IMPL_ISetCompressCodecsInfo + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandler.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandler.h new file mode 100644 index 0000000..1f38bff --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandler.h @@ -0,0 +1,94 @@ +// Zip/Handler.h + +#ifndef ZIP7_INC_ZIP_HANDLER_H +#define ZIP7_INC_ZIP_HANDLER_H + +#include "../../../Common/DynamicBuffer.h" +#include "../../ICoder.h" +#include "../IArchive.h" + +#include "../../Common/CreateCoder.h" + +#include "ZipCompressionMode.h" +#include "ZipIn.h" + +namespace NArchive { +namespace NZip { + +const unsigned kNumMethodNames1 = NFileHeader::NCompressionMethod::kZstdPk + 1; +const unsigned kMethodNames2Start = NFileHeader::NCompressionMethod::kZstdWz; +const unsigned kNumMethodNames2 = NFileHeader::NCompressionMethod::kWzAES + 1 - kMethodNames2Start; + +extern const char * const kMethodNames1[kNumMethodNames1]; +extern const char * const kMethodNames2[kNumMethodNames2]; + + +class CHandler Z7_final: + public IInArchive, + // public IArchiveGetRawProps, + public IOutArchive, + public ISetProperties, + Z7_PUBLIC_ISetCompressCodecsInfo_IFEC + public CMyUnknownImp +{ + Z7_COM_QI_BEGIN2(IInArchive) + // Z7_COM_QI_ENTRY(IArchiveGetRawProps) + Z7_COM_QI_ENTRY(IOutArchive) + Z7_COM_QI_ENTRY(ISetProperties) + Z7_COM_QI_ENTRY_ISetCompressCodecsInfo_IFEC + Z7_COM_QI_END + Z7_COM_ADDREF_RELEASE + + Z7_IFACE_COM7_IMP(IInArchive) + // Z7_IFACE_COM7_IMP(IArchiveGetRawProps) + Z7_IFACE_COM7_IMP(IOutArchive) + Z7_IFACE_COM7_IMP(ISetProperties) + DECL_ISetCompressCodecsInfo + +private: + CObjectVector m_Items; + CInArchive m_Archive; + + CBaseProps _props; + CHandlerTimeOptions TimeOptions; + + int m_MainMethod; + bool m_ForceAesMode; + + bool _removeSfxBlock; + bool m_ForceLocal; + bool m_ForceUtf8; + bool _force_SeqOutMode; // for creation + bool _force_OpenSeq; + bool _forceCodePage; + UInt32 _specifiedCodePage; + + DECL_EXTERNAL_CODECS_VARS + + void InitMethodProps() + { + _props.Init(); + TimeOptions.Init(); + TimeOptions.Prec = k_PropVar_TimePrec_0; + m_MainMethod = -1; + m_ForceAesMode = false; + _removeSfxBlock = false; + m_ForceLocal = false; + m_ForceUtf8 = false; + _force_SeqOutMode = false; + _force_OpenSeq = false; + _forceCodePage = false; + _specifiedCodePage = CP_OEMCP; + } + + // void MarkAltStreams(CObjectVector &items); + + HRESULT GetOutProperty(IArchiveUpdateCallback *callback, UInt32 callbackIndex, Int32 arcIndex, PROPID propID, PROPVARIANT *value); + +public: + CHandler(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp new file mode 100644 index 0000000..46ccf3d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp @@ -0,0 +1,626 @@ +// ZipHandlerOut.cpp + +#include "StdAfx.h" + +#include "../../../Common/ComTry.h" +#include "../../../Common/StringConvert.h" +#include "../../../Common/StringToInt.h" + +#include "../../../Windows/PropVariant.h" +#include "../../../Windows/TimeUtils.h" + +#include "../../IPassword.h" + +#include "../../Common/OutBuffer.h" + +#include "../../Crypto/WzAes.h" + +#include "../Common/ItemNameUtils.h" +#include "../Common/ParseProperties.h" + +#include "ZipHandler.h" +#include "ZipUpdate.h" + +using namespace NWindows; +using namespace NCOM; +using namespace NTime; + +namespace NArchive { +namespace NZip { + +Z7_COM7F_IMF(CHandler::GetFileTimeType(UInt32 *timeType)) +{ + *timeType = TimeOptions.Prec; + return S_OK; +} + +static bool IsSimpleAsciiString(const wchar_t *s) +{ + for (;;) + { + wchar_t c = *s++; + if (c == 0) + return true; + if (c < 0x20 || c > 0x7F) + return false; + } +} + + +static int FindZipMethod(const char *s, const char * const *names, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + { + const char *name = names[i]; + if (name && StringsAreEqualNoCase_Ascii(s, name)) + return (int)i; + } + return -1; +} + +static int FindZipMethod(const char *s) +{ + int k = FindZipMethod(s, kMethodNames1, kNumMethodNames1); + if (k >= 0) + return k; + k = FindZipMethod(s, kMethodNames2, kNumMethodNames2); + if (k >= 0) + return (int)kMethodNames2Start + k; + return -1; +} + + +#define COM_TRY_BEGIN2 try { +#define COM_TRY_END2 } \ +catch(const CSystemException &e) { return e.ErrorCode; } \ +catch(...) { return E_OUTOFMEMORY; } + +static HRESULT GetTime(IArchiveUpdateCallback *callback, unsigned index, PROPID propID, FILETIME &filetime) +{ + filetime.dwHighDateTime = filetime.dwLowDateTime = 0; + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(index, propID, &prop)) + if (prop.vt == VT_FILETIME) + filetime = prop.filetime; + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *callback)) +{ + COM_TRY_BEGIN2 + + if (m_Archive.IsOpen()) + { + if (!m_Archive.CanUpdate()) + return E_NOTIMPL; + } + + CObjectVector updateItems; + updateItems.ClearAndReserve(numItems); + + bool thereAreAesUpdates = false; + UInt64 largestSize = 0; + bool largestSizeDefined = false; + + #ifdef _WIN32 + const UINT oemCP = GetOEMCP(); + #endif + + UString name; + CUpdateItem ui; + + for (UInt32 i = 0; i < numItems; i++) + { + Int32 newData; + Int32 newProps; + UInt32 indexInArc; + + if (!callback) + return E_FAIL; + + RINOK(callback->GetUpdateItemInfo(i, &newData, &newProps, &indexInArc)) + + name.Empty(); + ui.Clear(); + + ui.NewProps = IntToBool(newProps); + ui.NewData = IntToBool(newData); + ui.IndexInArc = (int)indexInArc; + ui.IndexInClient = i; + + bool existInArchive = (indexInArc != (UInt32)(Int32)-1); + if (existInArchive) + { + const CItemEx &inputItem = m_Items[indexInArc]; + if (inputItem.IsAesEncrypted()) + thereAreAesUpdates = true; + if (!IntToBool(newProps)) + ui.IsDir = inputItem.IsDir(); + // ui.IsAltStream = inputItem.IsAltStream(); + } + + if (IntToBool(newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidAttrib, &prop)) + if (prop.vt == VT_EMPTY) + ui.Attrib = 0; + else if (prop.vt != VT_UI4) + return E_INVALIDARG; + else + ui.Attrib = prop.ulVal; + } + + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidPath, &prop)) + if (prop.vt == VT_EMPTY) + { + // name.Empty(); + } + else if (prop.vt != VT_BSTR) + return E_INVALIDARG; + else + name = prop.bstrVal; + } + + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidIsDir, &prop)) + if (prop.vt == VT_EMPTY) + ui.IsDir = false; + else if (prop.vt != VT_BOOL) + return E_INVALIDARG; + else + ui.IsDir = (prop.boolVal != VARIANT_FALSE); + } + + /* + { + bool isAltStream = false; + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidIsAltStream, &prop)); + if (prop.vt == VT_BOOL) + isAltStream = (prop.boolVal != VARIANT_FALSE); + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + } + + if (isAltStream) + { + if (ui.IsDir) + return E_INVALIDARG; + int delim = name.ReverseFind(L':'); + if (delim >= 0) + { + name.Delete(delim, 1); + name.Insert(delim, UString(k_SpecName_NTFS_STREAM)); + ui.IsAltStream = true; + } + } + } + */ + + // 22.00 : kpidTimeType is useless here : the code was disabled + /* + { + CPropVariant prop; + RINOK(callback->GetProperty(i, kpidTimeType, &prop)); + if (prop.vt == VT_UI4) + ui.NtfsTime_IsDefined = (prop.ulVal == NFileTimeType::kWindows); + else + ui.NtfsTime_IsDefined = _Write_NtfsTime; + } + */ + + if (TimeOptions.Write_MTime.Val) RINOK (GetTime (callback, i, kpidMTime, ui.Ntfs_MTime)) + if (TimeOptions.Write_ATime.Val) RINOK (GetTime (callback, i, kpidATime, ui.Ntfs_ATime)) + if (TimeOptions.Write_CTime.Val) RINOK (GetTime (callback, i, kpidCTime, ui.Ntfs_CTime)) + + if (TimeOptions.Prec != k_PropVar_TimePrec_DOS) + { + if (TimeOptions.Prec == k_PropVar_TimePrec_Unix || + TimeOptions.Prec == k_PropVar_TimePrec_Base) + ui.Write_UnixTime = ! FILETIME_IsZero (ui.Ntfs_MTime); + else + { + /* + // if we want to store zero timestamps as zero timestamp, use the following: + ui.Write_NtfsTime = + _Write_MTime || + _Write_ATime || + _Write_CTime; + */ + + // We treat zero timestamp as no timestamp + ui.Write_NtfsTime = + ! FILETIME_IsZero (ui.Ntfs_MTime) || + ! FILETIME_IsZero (ui.Ntfs_ATime) || + ! FILETIME_IsZero (ui.Ntfs_CTime); + } + } + + /* + how 0 in dos time works: + win10 explorer extract : some random date 1601-04-25. + winrar 6.10 : write time. + 7zip : MTime of archive is used + how 0 in tar works: + winrar 6.10 : 1970 + 0 in dos field can show that there is no timestamp. + we write correct 1970-01-01 in dos field, to support correct extraction in Win10. + */ + + UtcFileTime_To_LocalDosTime(ui.Ntfs_MTime, ui.Time); + + NItemName::ReplaceSlashes_OsToUnix(name); + + bool needSlash = ui.IsDir; + const wchar_t kSlash = L'/'; + if (!name.IsEmpty()) + { + if (name.Back() == kSlash) + { + if (!ui.IsDir) + return E_INVALIDARG; + needSlash = false; + } + } + if (needSlash) + name += kSlash; + + const UINT codePage = _forceCodePage ? _specifiedCodePage : CP_OEMCP; + bool tryUtf8 = true; + + /* + Windows 10 allows users to set UTF-8 in Region Settings via option: + "Beta: Use Unicode UTF-8 for worldwide language support" + In that case Windows uses CP_UTF8 when we use CP_OEMCP. + 21.02 fixed: + we set UTF-8 mark for non-latin files for such UTF-8 mode in Windows. + we write additional Info-Zip Utf-8 FileName Extra for non-latin names/ + */ + + if ((codePage != CP_UTF8) && + #ifdef _WIN32 + (m_ForceLocal || !m_ForceUtf8) && (oemCP != CP_UTF8) + #else + (m_ForceLocal && !m_ForceUtf8) + #endif + ) + { + bool defaultCharWasUsed; + ui.Name = UnicodeStringToMultiByte(name, codePage, '_', defaultCharWasUsed); + tryUtf8 = (!m_ForceLocal && (defaultCharWasUsed || + MultiByteToUnicodeString(ui.Name, codePage) != name)); + } + + const bool isNonLatin = !name.IsAscii(); + + if (tryUtf8) + { + ui.IsUtf8 = isNonLatin; + ConvertUnicodeToUTF8(name, ui.Name); + + #ifndef _WIN32 + if (ui.IsUtf8 && !CheckUTF8_AString(ui.Name)) + { + // if it's non-Windows and there are non-UTF8 characters we clear UTF8-flag + ui.IsUtf8 = false; + } + #endif + } + else if (isNonLatin) + Convert_Unicode_To_UTF8_Buf(name, ui.Name_Utf); + + if (ui.Name.Len() >= (1 << 16) + || ui.Name_Utf.Size() >= (1 << 16) - 128) + return E_INVALIDARG; + + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidComment, &prop)) + if (prop.vt == VT_EMPTY) + { + // ui.Comment.Free(); + } + else if (prop.vt != VT_BSTR) + return E_INVALIDARG; + else + { + UString s = prop.bstrVal; + AString a; + if (ui.IsUtf8) + ConvertUnicodeToUTF8(s, a); + else + { + bool defaultCharWasUsed; + a = UnicodeStringToMultiByte(s, codePage, '_', defaultCharWasUsed); + } + if (a.Len() >= (1 << 16)) + return E_INVALIDARG; + ui.Comment.CopyFrom((const Byte *)(const char *)a, a.Len()); + } + } + + + /* + if (existInArchive) + { + const CItemEx &itemInfo = m_Items[indexInArc]; + // ui.Commented = itemInfo.IsCommented(); + ui.Commented = false; + if (ui.Commented) + { + ui.CommentRange.Position = itemInfo.GetCommentPosition(); + ui.CommentRange.Size = itemInfo.CommentSize; + } + } + else + ui.Commented = false; + */ + } + + + if (IntToBool(newData)) + { + UInt64 size = 0; + if (!ui.IsDir) + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidSize, &prop)) + if (prop.vt != VT_UI8) + return E_INVALIDARG; + size = prop.uhVal.QuadPart; + if (largestSize < size) + largestSize = size; + largestSizeDefined = true; + } + ui.Size = size; + } + + updateItems.Add(ui); + } + + + CMyComPtr getTextPassword; + { + CMyComPtr udateCallBack2(callback); + udateCallBack2.QueryInterface(IID_ICryptoGetTextPassword2, &getTextPassword); + } + CCompressionMethodMode options; + (CBaseProps &)options = _props; + options.DataSizeReduce = largestSize; + options.DataSizeReduce_Defined = largestSizeDefined; + + options.Password_Defined = false; + options.Password.Wipe_and_Empty(); + if (getTextPassword) + { + CMyComBSTR_Wipe password; + Int32 passwordIsDefined; + RINOK(getTextPassword->CryptoGetTextPassword2(&passwordIsDefined, &password)) + options.Password_Defined = IntToBool(passwordIsDefined); + if (options.Password_Defined) + { + if (!m_ForceAesMode) + options.IsAesMode = thereAreAesUpdates; + + if (!IsSimpleAsciiString(password)) + return E_INVALIDARG; + if (password) + UnicodeStringToMultiByte2(options.Password, (LPCOLESTR)password, CP_OEMCP); + if (options.IsAesMode) + { + if (options.Password.Len() > NCrypto::NWzAes::kPasswordSizeMax) + return E_INVALIDARG; + } + } + } + + + int mainMethod = m_MainMethod; + + if (mainMethod < 0) + { + if (!_props._methods.IsEmpty()) + { + const AString &methodName = _props._methods.Front().MethodName; + if (!methodName.IsEmpty()) + { + mainMethod = FindZipMethod(methodName); + if (mainMethod < 0) + { + CMethodId methodId; + UInt32 numStreams; + bool isFilter; + if (FindMethod_Index(EXTERNAL_CODECS_VARS methodName, true, + methodId, numStreams, isFilter) < 0) + return E_NOTIMPL; + if (numStreams != 1) + return E_NOTIMPL; + if (methodId == kMethodId_BZip2) + mainMethod = NFileHeader::NCompressionMethod::kBZip2; + else + { + if (methodId < kMethodId_ZipBase) + return E_NOTIMPL; + methodId -= kMethodId_ZipBase; + if (methodId > 0xFF) + return E_NOTIMPL; + mainMethod = (int)methodId; + } + } + } + } + } + + if (mainMethod < 0) + mainMethod = (Byte)(((_props.GetLevel() == 0) ? + NFileHeader::NCompressionMethod::kStore : + NFileHeader::NCompressionMethod::kDeflate)); + else + mainMethod = (Byte)mainMethod; + + options.MethodSequence.Add((Byte)mainMethod); + + if (mainMethod != NFileHeader::NCompressionMethod::kStore) + options.MethodSequence.Add(NFileHeader::NCompressionMethod::kStore); + + options.Force_SeqOutMode = _force_SeqOutMode; + + CUpdateOptions uo; + uo.Write_MTime = TimeOptions.Write_MTime.Val; + uo.Write_ATime = TimeOptions.Write_ATime.Val; + uo.Write_CTime = TimeOptions.Write_CTime.Val; + /* + uo.Write_NtfsTime = _Write_NtfsTime && + (_Write_MTime || _Write_ATime || _Write_CTime); + uo.Write_UnixTime = _Write_UnixTime; + */ + + return Update( + EXTERNAL_CODECS_VARS + m_Items, updateItems, outStream, + m_Archive.IsOpen() ? &m_Archive : NULL, _removeSfxBlock, + uo, options, callback); + + COM_TRY_END2 +} + + + +Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)) +{ + InitMethodProps(); + + for (UInt32 i = 0; i < numProps; i++) + { + UString name = names[i]; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + const PROPVARIANT &prop = values[i]; + + if (name.IsEqualTo_Ascii_NoCase("em")) + { + if (prop.vt != VT_BSTR) + return E_INVALIDARG; + { + const wchar_t *m = prop.bstrVal; + if (IsString1PrefixedByString2_NoCase_Ascii(m, "AES")) + { + m += 3; + UInt32 v = 3; + if (*m != 0) + { + if (*m == '-') + m++; + const wchar_t *end; + v = ConvertStringToUInt32(m, &end); + if (*end != 0 || v % 64 != 0) + return E_INVALIDARG; + v /= 64; + v -= 2; + if (v >= 3) + return E_INVALIDARG; + v++; + } + _props.AesKeyMode = (Byte)v; + _props.IsAesMode = true; + m_ForceAesMode = true; + } + else if (StringsAreEqualNoCase_Ascii(m, "ZipCrypto")) + { + _props.IsAesMode = false; + m_ForceAesMode = true; + } + else + return E_INVALIDARG; + } + } + + + + else if (name.IsEqualTo("cl")) + { + RINOK(PROPVARIANT_to_bool(prop, m_ForceLocal)) + if (m_ForceLocal) + m_ForceUtf8 = false; + } + else if (name.IsEqualTo("cu")) + { + RINOK(PROPVARIANT_to_bool(prop, m_ForceUtf8)) + if (m_ForceUtf8) + m_ForceLocal = false; + } + else if (name.IsEqualTo("cp")) + { + UInt32 cp = CP_OEMCP; + RINOK(ParsePropToUInt32(L"", prop, cp)) + _forceCodePage = true; + _specifiedCodePage = cp; + } + else if (name.IsEqualTo("rsfx")) + { + RINOK(PROPVARIANT_to_bool(prop, _removeSfxBlock)) + } + else if (name.IsEqualTo("rws")) + { + RINOK(PROPVARIANT_to_bool(prop, _force_SeqOutMode)) + } + else if (name.IsEqualTo("ros")) + { + RINOK(PROPVARIANT_to_bool(prop, _force_OpenSeq)) + } + else + { + if (name.IsEqualTo_Ascii_NoCase("m") && prop.vt == VT_UI4) + { + UInt32 id = prop.ulVal; + if (id > 0xFF) + return E_INVALIDARG; + m_MainMethod = (int)id; + } + else + { + bool processed = false; + RINOK(TimeOptions.Parse(name, prop, processed)) + if (!processed) + { + RINOK(_props.SetProperty(name, prop)) + } + } + // RINOK(_props.MethodInfo.ParseParamsFromPROPVARIANT(name, prop)); + } + } + + _props._methods.DeleteFrontal(_props.GetNumEmptyMethods()); + if (_props._methods.Size() > 1) + return E_INVALIDARG; + if (_props._methods.Size() == 1) + { + const AString &methodName = _props._methods[0].MethodName; + + if (!methodName.IsEmpty()) + { + const char *end; + UInt32 id = ConvertStringToUInt32(methodName, &end); + if (*end == 0 && id <= 0xFF) + m_MainMethod = (int)id; + else if (methodName.IsEqualTo_Ascii_NoCase("Copy")) // it's alias for "Store" + m_MainMethod = 0; + } + } + + return S_OK; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHeader.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHeader.h new file mode 100644 index 0000000..038d49d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipHeader.h @@ -0,0 +1,201 @@ +// ZipHeader.h + +#ifndef ZIP7_INC_ARCHIVE_ZIP_HEADER_H +#define ZIP7_INC_ARCHIVE_ZIP_HEADER_H + +#include "../../../Common/MyTypes.h" + +namespace NArchive { +namespace NZip { + +const unsigned kMarkerSize = 4; + +namespace NSignature +{ + const UInt32 kLocalFileHeader = 0x04034B50; + const UInt32 kDataDescriptor = 0x08074B50; + const UInt32 kCentralFileHeader = 0x02014B50; + const UInt32 kEcd = 0x06054B50; + const UInt32 kEcd64 = 0x06064B50; + const UInt32 kEcd64Locator = 0x07064B50; + const UInt32 kSpan = 0x08074B50; + const UInt32 kNoSpan = 0x30304B50; // PK00, replaces kSpan, if there is only 1 segment +} + +const unsigned kLocalHeaderSize = 4 + 26; // including signature +const unsigned kDataDescriptorSize32 = 4 + 4 + 4 * 2; // including signature +const unsigned kDataDescriptorSize64 = 4 + 4 + 8 * 2; // including signature +const unsigned kCentralHeaderSize = 4 + 42; // including signature + +const unsigned kEcdSize = 22; // including signature +const unsigned kEcd64_MainSize = 44; +const unsigned kEcd64_FullSize = 12 + kEcd64_MainSize; +const unsigned kEcd64Locator_Size = 20; + +namespace NFileHeader +{ + namespace NCompressionMethod + { + enum EType + { + kStore = 0, + kShrink = 1, + kReduce1 = 2, + kReduce2 = 3, + kReduce3 = 4, + kReduce4 = 5, + kImplode = 6, + kTokenize = 7, + kDeflate = 8, + kDeflate64 = 9, + kPKImploding = 10, + + kBZip2 = 12, + + kLZMA = 14, + + kTerse = 18, + kLz77 = 19, + kZstdPk = 20, + + kZstdWz = 93, + kMP3 = 94, + kXz = 95, + kJpeg = 96, + kWavPack = 97, + kPPMd = 98, + kWzAES = 99 + }; + + const Byte kMadeByProgramVersion = 63; + + const Byte kExtractVersion_Default = 10; + const Byte kExtractVersion_Dir = 20; + const Byte kExtractVersion_ZipCrypto = 20; + const Byte kExtractVersion_Deflate = 20; + const Byte kExtractVersion_Deflate64 = 21; + const Byte kExtractVersion_Zip64 = 45; + const Byte kExtractVersion_BZip2 = 46; + const Byte kExtractVersion_Aes = 51; + const Byte kExtractVersion_LZMA = 63; + const Byte kExtractVersion_PPMd = 63; + const Byte kExtractVersion_Xz = 20; // test it + } + + namespace NExtraID + { + enum + { + kZip64 = 0x01, + kNTFS = 0x0A, + kUnix0 = 0x0D, // Info-ZIP : (UNIX) PK + kStrongEncrypt = 0x17, + kIzNtSecurityDescriptor = 0x4453, + kUnixTime = 0x5455, // "UT" (time) Info-ZIP + kUnix1 = 0x5855, // Info-ZIP + kIzUnicodeComment = 0x6375, + kIzUnicodeName = 0x7075, + kUnix2 = 0x7855, // Info-ZIP + kUnixN = 0x7875, // Info-ZIP + kWzAES = 0x9901, + kApkAlign = 0xD935 + }; + } + + namespace NNtfsExtra + { + const UInt16 kTagTime = 1; + enum + { + kMTime = 0, + kATime, + kCTime + }; + } + + namespace NUnixTime + { + enum + { + kMTime = 0, + kATime, + kCTime + }; + } + + namespace NUnixExtra + { + enum + { + kATime = 0, + kMTime + }; + } + + namespace NFlags + { + const unsigned kEncrypted = 1 << 0; + const unsigned kLzmaEOS = 1 << 1; + const unsigned kDescriptorUsedMask = 1 << 3; + const unsigned kStrongEncrypted = 1 << 6; + const unsigned kUtf8 = 1 << 11; + const unsigned kAltStream = 1 << 14; + + const unsigned kImplodeDictionarySizeMask = 1 << 1; + const unsigned kImplodeLiteralsOnMask = 1 << 2; + + /* + const unsigned kDeflateTypeBitStart = 1; + const unsigned kNumDeflateTypeBits = 2; + const unsigned kNumDeflateTypes = (1 << kNumDeflateTypeBits); + const unsigned kDeflateTypeMask = (1 << kNumDeflateTypeBits) - 1; + */ + } + + namespace NHostOS + { + enum EEnum + { + kFAT = 0, + kAMIGA = 1, + kVMS = 2, // VAX/VMS + kUnix = 3, + kVM_CMS = 4, + kAtari = 5, // what if it's a minix filesystem? [cjh] + kHPFS = 6, // filesystem used by OS/2 (and NT 3.x) + kMac = 7, + kZ_System = 8, + kCPM = 9, + kTOPS20 = 10, // pkzip 2.50 NTFS + kNTFS = 11, // filesystem used by Windows NT + kQDOS = 12, // SMS/QDOS + kAcorn = 13, // Archimedes Acorn RISC OS + kVFAT = 14, // filesystem used by Windows 95, NT + kMVS = 15, + kBeOS = 16, // hybrid POSIX/database filesystem + kTandem = 17, + kOS400 = 18, + kOSX = 19 + }; + } + + + namespace NAmigaAttrib + { + const UInt32 kIFMT = 06000; // Amiga file type mask + const UInt32 kIFDIR = 04000; // Amiga directory + const UInt32 kIFREG = 02000; // Amiga regular file + const UInt32 kIHIDDEN = 00200; // to be supported in AmigaDOS 3.x + const UInt32 kISCRIPT = 00100; // executable script (text command file) + const UInt32 kIPURE = 00040; // allow loading into resident memory + const UInt32 kIARCHIVE = 00020; // not modified since bit was last set + const UInt32 kIREAD = 00010; // can be opened for reading + const UInt32 kIWRITE = 00004; // can be opened for writing + const UInt32 kIEXECUTE = 00002; // executable image, a loadable runfile + const UInt32 kIDELETE = 00001; // can be deleted + } +} + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipIn.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipIn.cpp new file mode 100644 index 0000000..788810f --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -0,0 +1,3485 @@ +// Archive/ZipIn.cpp + +#include "StdAfx.h" + +// #include + +#include "../../../Common/DynamicBuffer.h" +#include "../../../Common/IntToString.h" +#include "../../../Common/MyException.h" +#include "../../../Common/StringToInt.h" + +#include "../../../Windows/PropVariant.h" + +#include "../IArchive.h" + +#include "ZipIn.h" + +#define Get16(p) GetUi16(p) +#define Get32(p) GetUi32(p) +#define Get64(p) GetUi64(p) + +#define G16(offs, v) v = Get16(p + (offs)) +#define G32(offs, v) v = Get32(p + (offs)) +#define G64(offs, v) v = Get64(p + (offs)) + +namespace NArchive { +namespace NZip { + +/* we try to use same size of Buffer (1 << 17) for all tasks. + it allow to avoid reallocations and cache clearing. */ + +static const size_t kSeqBufferSize = (size_t)1 << 17; + +/* +Open() +{ + _inBufMode = false; + ReadVols() + FindCd(); + TryEcd64() + SeekToVol() + FindMarker() + _inBufMode = true; + ReadHeaders() + _inBufMode = false; + ReadCd() + FindCd() + TryEcd64() + TryReadCd() + { + SeekToVol(); + _inBufMode = true; + } + _inBufMode = true; + ReadLocals() + ReadCdItem() + .... +} +FindCd() writes to Buffer without touching (_inBufMode) +*/ + +/* + if (not defined ZIP_SELF_CHECK) : it reads CD and if error in first pass CD reading, it reads LOCALS-CD-MODE + if ( defined ZIP_SELF_CHECK) : it always reads CD and LOCALS-CD-MODE + use ZIP_SELF_CHECK to check LOCALS-CD-MODE for any zip archive +*/ + +// #define ZIP_SELF_CHECK + + +struct CEcd +{ + UInt16 ThisDisk; + UInt16 CdDisk; + UInt16 NumEntries_in_ThisDisk; + UInt16 NumEntries; + UInt32 Size; + UInt32 Offset; + UInt16 CommentSize; + + bool IsEmptyArc() const + { + return ThisDisk == 0 + && CdDisk == 0 + && NumEntries_in_ThisDisk == 0 + && NumEntries == 0 + && Size == 0 + && Offset == 0 // test it + ; + } + + void Parse(const Byte *p); // (p) doesn't include signature +}; + +void CEcd::Parse(const Byte *p) +{ + // (p) doesn't include signature + G16(0, ThisDisk); + G16(2, CdDisk); + G16(4, NumEntries_in_ThisDisk); + G16(6, NumEntries); + G32(8, Size); + G32(12, Offset); + G16(16, CommentSize); +} + + +void CCdInfo::ParseEcd32(const Byte *p) +{ + IsFromEcd64 = false; + // (p) includes signature + p += 4; + G16(0, ThisDisk); + G16(2, CdDisk); + G16(4, NumEntries_in_ThisDisk); + G16(6, NumEntries); + G32(8, Size); + G32(12, Offset); + G16(16, CommentSize); +} + +void CCdInfo::ParseEcd64e(const Byte *p) +{ + IsFromEcd64 = true; + // (p) exclude signature + G16(0, VersionMade); + G16(2, VersionNeedExtract); + G32(4, ThisDisk); + G32(8, CdDisk); + + G64(12, NumEntries_in_ThisDisk); + G64(20, NumEntries); + G64(28, Size); + G64(36, Offset); +} + + +struct CLocator +{ + UInt32 Ecd64Disk; + UInt32 NumDisks; + UInt64 Ecd64Offset; + + CLocator(): Ecd64Disk(0), NumDisks(0), Ecd64Offset(0) {} + + void Parse(const Byte *p) + { + G32(0, Ecd64Disk); + G64(4, Ecd64Offset); + G32(12, NumDisks); + } + + bool IsEmptyArc() const + { + return Ecd64Disk == 0 && NumDisks == 0 && Ecd64Offset == 0; + } +}; + + + + +void CInArchive::ClearRefs() +{ + StreamRef.Release(); + Stream = NULL; + StartStream = NULL; + Callback = NULL; + + Vols.Clear(); +} + +void CInArchive::Close() +{ + _cnt = 0; + DisableBufMode(); + + IsArcOpen = false; + + IsArc = false; + IsZip64 = false; + + IsApk = false; + IsCdUnsorted = false; + + HeadersError = false; + HeadersWarning = false; + ExtraMinorError = false; + + UnexpectedEnd = false; + LocalsWereRead = false; + LocalsCenterMerged = false; + NoCentralDir = false; + Overflow32bit = false; + Cd_NumEntries_Overflow_16bit = false; + + MarkerIsFound = false; + MarkerIsSafe = false; + + IsMultiVol = false; + UseDisk_in_SingleVol = false; + EcdVolIndex = 0; + + ArcInfo.Clear(); + + ClearRefs(); +} + + + +HRESULT CInArchive::Seek_SavePos(UInt64 offset) +{ + // InitBuf(); + // if (!Stream) return S_FALSE; + return Stream->Seek((Int64)offset, STREAM_SEEK_SET, &_streamPos); +} + + +/* SeekToVol() will keep the cached mode, if new volIndex is + same Vols.StreamIndex volume, and offset doesn't go out of cached region */ + +HRESULT CInArchive::SeekToVol(int volIndex, UInt64 offset) +{ + if (volIndex != Vols.StreamIndex) + { + if (IsMultiVol && volIndex >= 0) + { + if ((unsigned)volIndex >= Vols.Streams.Size()) + return S_FALSE; + if (!Vols.Streams[(unsigned)volIndex].Stream) + return S_FALSE; + Stream = Vols.Streams[(unsigned)volIndex].Stream; + } + else if (volIndex == -2) + { + if (!Vols.ZipStream) + return S_FALSE; + Stream = Vols.ZipStream; + } + else + Stream = StartStream; + Vols.StreamIndex = volIndex; + } + else + { + if (offset <= _streamPos) + { + const UInt64 back = _streamPos - offset; + if (back <= _bufCached) + { + _bufPos = _bufCached - (size_t)back; + return S_OK; + } + } + } + InitBuf(); + return Seek_SavePos(offset); +} + + +HRESULT CInArchive::AllocateBuffer(size_t size) +{ + if (size <= Buffer.Size()) + return S_OK; + /* in cached mode virtual_pos is not equal to phy_pos (_streamPos) + so we change _streamPos and do Seek() to virtual_pos before cache clearing */ + if (_bufPos != _bufCached) + { + RINOK(Seek_SavePos(GetVirtStreamPos())) + } + InitBuf(); + Buffer.AllocAtLeast(size); + if (!Buffer.IsAllocated()) + return E_OUTOFMEMORY; + return S_OK; +} + +// ---------- ReadFromCache ---------- +// reads from cache and from Stream +// move to next volume can be allowed if (CanStartNewVol) and only before first byte reading + +HRESULT CInArchive::ReadFromCache(Byte *data, unsigned size, unsigned &processed) +{ + HRESULT result = S_OK; + processed = 0; + + for (;;) + { + if (size == 0) + return S_OK; + + const size_t avail = GetAvail(); + + if (avail != 0) + { + unsigned cur = size; + if (cur > avail) + cur = (unsigned)avail; + memcpy(data, (const Byte *)Buffer + _bufPos, cur); + + data += cur; + size -= cur; + processed += cur; + + _bufPos += cur; + _cnt += cur; + + CanStartNewVol = false; + + continue; + } + + InitBuf(); + + if (_inBufMode) + { + UInt32 cur = 0; + result = Stream->Read(Buffer, (UInt32)Buffer.Size(), &cur); + _bufPos = 0; + _bufCached = cur; + _streamPos += cur; + if (cur != 0) + CanStartNewVol = false; + if (result != S_OK) + break; + if (cur != 0) + continue; + } + else + { + size_t cur = size; + result = ReadStream(Stream, data, &cur); + data += cur; + size -= (unsigned)cur; + processed += (unsigned)cur; + _streamPos += cur; + _cnt += cur; + if (cur != 0) + { + CanStartNewVol = false; + break; + } + if (result != S_OK) + break; + } + + if ( !IsMultiVol + || !CanStartNewVol + || Vols.StreamIndex < 0 + || (unsigned)Vols.StreamIndex + 1 >= Vols.Streams.Size()) + break; + + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; + if (!s.Stream) + break; + result = s.SeekToStart(); + if (result != S_OK) + break; + Vols.StreamIndex++; + _streamPos = 0; + // Vols.NeedSeek = false; + + Stream = s.Stream; + } + + return result; +} + + +HRESULT CInArchive::ReadFromCache_FALSE(Byte *data, unsigned size) +{ + unsigned processed; + HRESULT res = ReadFromCache(data, size, processed); + if (res == S_OK && size != processed) + return S_FALSE; + return res; +} + + +static bool CheckDosTime(UInt32 dosTime) +{ + if (dosTime == 0) + return true; + unsigned month = (dosTime >> 21) & 0xF; + unsigned day = (dosTime >> 16) & 0x1F; + unsigned hour = (dosTime >> 11) & 0x1F; + unsigned min = (dosTime >> 5) & 0x3F; + unsigned sec = (dosTime & 0x1F) * 2; + if (month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || min > 59 || sec > 59) + return false; + return true; +} + +API_FUNC_IsArc IsArc_Zip(const Byte *p, size_t size) +{ + if (size < 8) + return k_IsArc_Res_NEED_MORE; + if (p[0] != 'P') + return k_IsArc_Res_NO; + + UInt32 sig = Get32(p); + + if (sig == NSignature::kNoSpan || sig == NSignature::kSpan) + { + p += 4; + size -= 4; + } + + sig = Get32(p); + + if (sig == NSignature::kEcd64) + { + if (size < kEcd64_FullSize) + return k_IsArc_Res_NEED_MORE; + + const UInt64 recordSize = Get64(p + 4); + if ( recordSize < kEcd64_MainSize + || recordSize > kEcd64_MainSize + (1 << 20)) + return k_IsArc_Res_NO; + CCdInfo cdInfo; + cdInfo.ParseEcd64e(p + 12); + if (!cdInfo.IsEmptyArc()) + return k_IsArc_Res_NO; + return k_IsArc_Res_YES; // k_IsArc_Res_YES_2; + } + + if (sig == NSignature::kEcd) + { + if (size < kEcdSize) + return k_IsArc_Res_NEED_MORE; + CEcd ecd; + ecd.Parse(p + 4); + // if (ecd.cdSize != 0) + if (!ecd.IsEmptyArc()) + return k_IsArc_Res_NO; + return k_IsArc_Res_YES; // k_IsArc_Res_YES_2; + } + + if (sig != NSignature::kLocalFileHeader) + return k_IsArc_Res_NO; + + if (size < kLocalHeaderSize) + return k_IsArc_Res_NEED_MORE; + + p += 4; + + { + const unsigned kPureHeaderSize = kLocalHeaderSize - 4; + unsigned i; + for (i = 0; i < kPureHeaderSize && p[i] == 0; i++); + if (i == kPureHeaderSize) + return k_IsArc_Res_NEED_MORE; + } + + /* + if (p[0] >= 128) // ExtractVersion.Version; + return k_IsArc_Res_NO; + */ + + // ExtractVersion.Version = p[0]; + // ExtractVersion.HostOS = p[1]; + // Flags = Get16(p + 2); + // Method = Get16(p + 4); + /* + // 9.33: some zip archives contain incorrect value in timestamp. So we don't check it now + UInt32 dosTime = Get32(p + 6); + if (!CheckDosTime(dosTime)) + return k_IsArc_Res_NO; + */ + // Crc = Get32(p + 10); + // PackSize = Get32(p + 14); + // Size = Get32(p + 18); + const unsigned nameSize = Get16(p + 22); + unsigned extraSize = Get16(p + 24); + const UInt32 extraOffset = kLocalHeaderSize + (UInt32)nameSize; + + /* + // 21.02: fixed. we don't use the following check + if (extraOffset + extraSize > (1 << 16)) + return k_IsArc_Res_NO; + */ + + p -= 4; + + { + size_t rem = size - kLocalHeaderSize; + if (rem > nameSize) + rem = nameSize; + const Byte *p2 = p + kLocalHeaderSize; + for (size_t i = 0; i < rem; i++) + if (p2[i] == 0) + { + // we support some "bad" zip archives that contain zeros after name + for (size_t k = i + 1; k < rem; k++) + if (p2[k] != 0) + return k_IsArc_Res_NO; + break; + /* + if (i != nameSize - 1) + return k_IsArc_Res_NO; + */ + } + } + + if (size < extraOffset) + return k_IsArc_Res_NEED_MORE; + + if (extraSize > 0) + { + p += extraOffset; + size -= extraOffset; + while (extraSize != 0) + { + if (extraSize < 4) + { + // 7-Zip before 9.31 created incorrect WzAES Extra in folder's local headers. + // so we return k_IsArc_Res_YES to support such archives. + // return k_IsArc_Res_NO; // do we need to support such extra ? + return k_IsArc_Res_YES; + } + if (size < 4) + return k_IsArc_Res_NEED_MORE; + unsigned dataSize = Get16(p + 2); + size -= 4; + extraSize -= 4; + p += 4; + if (dataSize > extraSize) + { + // It can be error on header. + // We want to support such rare case bad archives. + // We use additional checks to reduce false-positive probability. + if (nameSize == 0 + || nameSize > (1 << 9) + || extraSize > (1 << 9)) + return k_IsArc_Res_NO; + return k_IsArc_Res_YES; + } + if (dataSize > size) + return k_IsArc_Res_NEED_MORE; + size -= dataSize; + extraSize -= dataSize; + p += dataSize; + } + } + + return k_IsArc_Res_YES; +} + +static UInt32 IsArc_Zip_2(const Byte *p, size_t size, bool isFinal) +{ + UInt32 res = IsArc_Zip(p, size); + if (res == k_IsArc_Res_NEED_MORE && isFinal) + return k_IsArc_Res_NO; + return res; +} + + + +/* FindPK_4() is allowed to access data up to and including &limit[3]. + limit[4] access is not allowed. + return: + (return_ptr < limit) : "PK" was found at (return_ptr) + (return_ptr >= limit) : limit was reached or crossed. So no "PK" found before limit +*/ +Z7_NO_INLINE +static const Byte *FindPK_4(const Byte *p, const Byte *limit) +{ + for (;;) + { + for (;;) + { + if (p >= limit) + return limit; + Byte b = p[1]; + if (b == 0x4B) { if (p[0] == 0x50) { return p; } p += 1; break; } + if (b == 0x50) { if (p[2] == 0x4B) { return p + 1; } p += 2; break; } + b = p[3]; + p += 4; + if (b == 0x4B) { if (p[-2]== 0x50) { return p - 2; } p -= 1; break; } + if (b == 0x50) { if (p[0] == 0x4B) { return p - 1; } break; } + } + } + /* + for (;;) + { + for (;;) + { + if (p >= limit) + return limit; + if (*p++ == 0x50) break; + if (*p++ == 0x50) break; + if (*p++ == 0x50) break; + if (*p++ == 0x50) break; + } + if (*p == 0x4B) + return p - 1; + } + */ +} + + +/* +---------- FindMarker ---------- +returns: + S_OK: + ArcInfo.MarkerVolIndex : volume of marker + ArcInfo.MarkerPos : Pos of first signature + ArcInfo.MarkerPos2 : Pos of main signature (local item signature in most cases) + _streamPos : stream pos + _cnt : The number of virtal Bytes after start of search to offset after signature + _signature : main signature + + S_FALSE: can't find marker, or there is some non-zip data after marker + + Error code: stream reading error. +*/ + +HRESULT CInArchive::FindMarker(const UInt64 *searchLimit) +{ + ArcInfo.MarkerPos = GetVirtStreamPos(); + ArcInfo.MarkerPos2 = ArcInfo.MarkerPos; + ArcInfo.MarkerVolIndex = Vols.StreamIndex; + + _cnt = 0; + + CanStartNewVol = false; + + if (searchLimit && *searchLimit == 0) + { + Byte startBuf[kMarkerSize]; + RINOK(ReadFromCache_FALSE(startBuf, kMarkerSize)) + + UInt32 marker = Get32(startBuf); + _signature = marker; + + if ( marker == NSignature::kNoSpan + || marker == NSignature::kSpan) + { + RINOK(ReadFromCache_FALSE(startBuf, kMarkerSize)) + _signature = Get32(startBuf); + } + + if ( _signature != NSignature::kEcd + && _signature != NSignature::kEcd64 + && _signature != NSignature::kLocalFileHeader) + return S_FALSE; + + ArcInfo.MarkerPos2 = GetVirtStreamPos() - 4; + ArcInfo.IsSpanMode = (marker == NSignature::kSpan); + + // we use weak test in case of (*searchLimit == 0) + // since error will be detected later in Open function + return S_OK; + } + + // zip specification: (_zip_header_size < (1 << 16)) + // so we need such size to check header + const size_t kCheckSize = (size_t)1 << 16; + const size_t kBufSize = (size_t)1 << 17; // (kBufSize must be > kCheckSize) + + RINOK(AllocateBuffer(kBufSize)) + + _inBufMode = true; + + UInt64 progressPrev = 0; + + for (;;) + { + RINOK(LookAhead(kBufSize)) + + const size_t avail = GetAvail(); + + size_t limitPos; + // (avail > kBufSize) is possible, if (Buffer.Size() > kBufSize) + const bool isFinished = (avail < kBufSize); + if (isFinished) + { + const unsigned kMinAllowed = 4; + if (avail <= kMinAllowed) + { + if ( !IsMultiVol + || Vols.StreamIndex < 0 + || (unsigned)Vols.StreamIndex + 1 >= Vols.Streams.Size()) + break; + + SkipLookahed(avail); + + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; + if (!s.Stream) + break; + + RINOK(s.SeekToStart()) + + InitBuf(); + Vols.StreamIndex++; + _streamPos = 0; + Stream = s.Stream; + continue; + } + limitPos = avail - kMinAllowed; + } + else + limitPos = (avail - kCheckSize); + + // we don't check at (limitPos) for good fast aligned operations + + if (searchLimit) + { + if (_cnt > *searchLimit) + break; + UInt64 rem = *searchLimit - _cnt; + if (limitPos > rem) + limitPos = (size_t)rem + 1; + } + + if (limitPos == 0) + break; + + const Byte * const pStart = Buffer + _bufPos; + const Byte * p = pStart; + const Byte * const limit = pStart + limitPos; + + for (;; p++) + { + p = FindPK_4(p, limit); + if (p >= limit) + break; + size_t rem = (size_t)(pStart + avail - p); + /* 22.02 : we limit check size with kCheckSize to be consistent for + any different combination of _bufPos in Buffer and size of Buffer. */ + if (rem > kCheckSize) + rem = kCheckSize; + const UInt32 res = IsArc_Zip_2(p, rem, isFinished); + if (res != k_IsArc_Res_NO) + { + if (rem < kMarkerSize) + return S_FALSE; + _signature = Get32(p); + SkipLookahed((size_t)(p - pStart)); + ArcInfo.MarkerVolIndex = Vols.StreamIndex; + ArcInfo.MarkerPos = GetVirtStreamPos(); + ArcInfo.MarkerPos2 = ArcInfo.MarkerPos; + SkipLookahed(4); + if ( _signature == NSignature::kNoSpan + || _signature == NSignature::kSpan) + { + if (rem < kMarkerSize * 2) + return S_FALSE; + ArcInfo.IsSpanMode = (_signature == NSignature::kSpan); + _signature = Get32(p + 4); + ArcInfo.MarkerPos2 += 4; + SkipLookahed(4); + } + return S_OK; + } + } + + if (!IsMultiVol && isFinished) + break; + + SkipLookahed((size_t)(p - pStart)); + + if (Callback && (_cnt - progressPrev) >= ((UInt32)1 << 23)) + { + progressPrev = _cnt; + // const UInt64 numFiles64 = 0; + RINOK(Callback->SetCompleted(NULL, &_cnt)) + } + } + + return S_FALSE; +} + + +/* +---------- IncreaseRealPosition ---------- +moves virtual offset in virtual stream. +changing to new volumes is allowed +*/ + +HRESULT CInArchive::IncreaseRealPosition(UInt64 offset, bool &isFinished) +{ + isFinished = false; + + for (;;) + { + const size_t avail = GetAvail(); + + if (offset <= avail) + { + _bufPos += (size_t)offset; + _cnt += offset; + return S_OK; + } + + _cnt += avail; + offset -= avail; + + _bufCached = 0; + _bufPos = 0; + + if (!_inBufMode) + break; + + CanStartNewVol = true; + LookAhead(1); + + if (GetAvail() == 0) + return S_OK; + } + + // cache is empty + + if (!IsMultiVol) + { + _cnt += offset; + return Stream->Seek((Int64)offset, STREAM_SEEK_CUR, &_streamPos); + } + + for (;;) + { + if (offset == 0) + return S_OK; + + if (Vols.StreamIndex < 0) + return S_FALSE; + if ((unsigned)Vols.StreamIndex >= Vols.Streams.Size()) + { + isFinished = true; + return S_OK; + } + { + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex]; + if (!s.Stream) + { + isFinished = true; + return S_OK; + } + if (_streamPos > s.Size) + return S_FALSE; + const UInt64 rem = s.Size - _streamPos; + if ((UInt64)offset <= rem) + { + _cnt += offset; + return Stream->Seek((Int64)offset, STREAM_SEEK_CUR, &_streamPos); + } + RINOK(Seek_SavePos(s.Size)) + offset -= rem; + _cnt += rem; + } + + Stream = NULL; + _streamPos = 0; + Vols.StreamIndex++; + if ((unsigned)Vols.StreamIndex >= Vols.Streams.Size()) + { + isFinished = true; + return S_OK; + } + const CVols::CSubStreamInfo &s2 = Vols.Streams[(unsigned)Vols.StreamIndex]; + if (!s2.Stream) + { + isFinished = true; + return S_OK; + } + Stream = s2.Stream; + RINOK(Seek_SavePos(0)) + } +} + + + +/* +---------- LookAhead ---------- +Reads data to buffer, if required. + +It can read from volumes as long as Buffer.Size(). +But it moves to new volume, only if it's required to provide minRequired bytes in buffer. + +in: + (minRequired <= Buffer.Size()) + +return: + S_OK : if (GetAvail() < minRequired) after function return, it's end of stream(s) data, or no new volume stream. + Error codes: IInStream::Read() error or IInStream::Seek() error for multivol +*/ + +HRESULT CInArchive::LookAhead(size_t minRequired) +{ + for (;;) + { + const size_t avail = GetAvail(); + + if (minRequired <= avail) + return S_OK; + + if (_bufPos != 0) + { + if (avail != 0) + memmove(Buffer, Buffer + _bufPos, avail); + _bufPos = 0; + _bufCached = avail; + } + + const size_t pos = _bufCached; + UInt32 processed = 0; + HRESULT res = Stream->Read(Buffer + pos, (UInt32)(Buffer.Size() - pos), &processed); + _streamPos += processed; + _bufCached += processed; + + if (res != S_OK) + return res; + + if (processed != 0) + continue; + + if ( !IsMultiVol + || !CanStartNewVol + || Vols.StreamIndex < 0 + || (unsigned)Vols.StreamIndex + 1 >= Vols.Streams.Size()) + return S_OK; + + const CVols::CSubStreamInfo &s = Vols.Streams[(unsigned)Vols.StreamIndex + 1]; + if (!s.Stream) + return S_OK; + + RINOK(s.SeekToStart()) + + Vols.StreamIndex++; + _streamPos = 0; + Stream = s.Stream; + // Vols.NeedSeek = false; + } +} + + +class CUnexpectEnd {}; + + +/* +---------- SafeRead ---------- + +reads data of exact size from stream(s) + +in: + _inBufMode + if (CanStartNewVol) it can go to next volume before first byte reading, if there is end of volume data. + +in, out: + _streamPos : position in Stream + Stream + Vols : if (IsMultiVol) + _cnt + +out: + (CanStartNewVol == false), if some data was read + +return: + S_OK : success reading of requested data + +exceptions: + CSystemException() - stream reading error + CUnexpectEnd() : could not read data of requested size +*/ + +void CInArchive::SafeRead(Byte *data, unsigned size) +{ + unsigned processed; + HRESULT result = ReadFromCache(data, size, processed); + if (result != S_OK) + throw CSystemException(result); + if (size != processed) + throw CUnexpectEnd(); +} + +void CInArchive::ReadBuffer(CByteBuffer &buffer, unsigned size) +{ + buffer.Alloc(size); + if (size != 0) + SafeRead(buffer, size); +} + +// Byte CInArchive::ReadByte () { Byte b; SafeRead(&b, 1); return b; } +// UInt16 CInArchive::ReadUInt16() { Byte buf[2]; SafeRead(buf, 2); return Get16(buf); } +UInt32 CInArchive::ReadUInt32() { Byte buf[4]; SafeRead(buf, 4); return Get32(buf); } +UInt64 CInArchive::ReadUInt64() { Byte buf[8]; SafeRead(buf, 8); return Get64(buf); } + +void CInArchive::ReadSignature() +{ + CanStartNewVol = true; + _signature = ReadUInt32(); + // CanStartNewVol = false; // it's already changed in SafeRead +} + + +// we Skip() inside headers only, so no need for stream change in multivol. + +void CInArchive::Skip(size_t num) +{ + while (num != 0) + { + const unsigned kBufSize = (size_t)1 << 10; + Byte buf[kBufSize]; + unsigned step = kBufSize; + if (step > num) + step = (unsigned)num; + SafeRead(buf, step); + num -= step; + } +} + +/* +HRESULT CInArchive::Callback_Completed(unsigned numFiles) +{ + const UInt64 numFiles64 = numFiles; + return Callback->SetCompleted(&numFiles64, &_cnt); +} +*/ + +HRESULT CInArchive::Skip64(UInt64 num, unsigned numFiles) +{ + if (num == 0) + return S_OK; + + for (;;) + { + size_t step = (size_t)1 << 24; + if (step > num) + step = (size_t)num; + Skip(step); + num -= step; + if (num == 0) + return S_OK; + if (Callback) + { + const UInt64 numFiles64 = numFiles; + RINOK(Callback->SetCompleted(&numFiles64, &_cnt)) + } + } +} + + +bool CInArchive::ReadFileName(unsigned size, AString &s) +{ + if (size == 0) + { + s.Empty(); + return true; + } + char *p = s.GetBuf(size); + SafeRead((Byte *)p, size); + unsigned i = size; + do + { + if (p[i - 1] != 0) + break; + } + while (--i); + s.ReleaseBuf_CalcLen(size); + return s.Len() == i; +} + + +#define ZIP64_IS_32_MAX(n) ((n) == 0xFFFFFFFF) +#define ZIP64_IS_16_MAX(n) ((n) == 0xFFFF) + + +bool CInArchive::ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, + UInt64 &unpackSize, UInt64 &packSize, + CItem *cdItem) +{ + extra.Clear(); + + while (extraSize >= 4) + { + CExtraSubBlock subBlock; + const UInt32 pair = ReadUInt32(); + subBlock.ID = (pair & 0xFFFF); + unsigned size = (unsigned)(pair >> 16); + // const unsigned origSize = size; + + extraSize -= 4; + + if (size > extraSize) + { + // it's error in extra + HeadersWarning = true; + extra.Error = true; + Skip(extraSize); + return false; + } + + extraSize -= size; + + if (subBlock.ID == NFileHeader::NExtraID::kZip64) + { + extra.IsZip64 = true; + bool isOK = true; + + if (!cdItem + && size == 16 + && !ZIP64_IS_32_MAX(unpackSize) + && !ZIP64_IS_32_MAX(packSize)) + { + /* Win10 Explorer's "Send to Zip" for big (3500 MiB) files + creates Zip64 Extra in local file header. + But if both uncompressed and compressed sizes are smaller than 4 GiB, + Win10 doesn't store 0xFFFFFFFF in 32-bit fields as expected by zip specification. + 21.04: we ignore these minor errors in Win10 zip archives. */ + if (ReadUInt64() != unpackSize) + isOK = false; + if (ReadUInt64() != packSize) + isOK = false; + size = 0; + } + else + { + if (ZIP64_IS_32_MAX(unpackSize)) + { if (size < 8) isOK = false; else { size -= 8; unpackSize = ReadUInt64(); }} + + if (isOK && ZIP64_IS_32_MAX(packSize)) + { if (size < 8) isOK = false; else { size -= 8; packSize = ReadUInt64(); }} + + if (cdItem) + { + if (isOK) + { + if (ZIP64_IS_32_MAX(cdItem->LocalHeaderPos)) + { if (size < 8) isOK = false; else { size -= 8; cdItem->LocalHeaderPos = ReadUInt64(); }} + /* + else if (size == 8) + { + size -= 8; + const UInt64 v = ReadUInt64(); + // soong_zip, an AOSP tool (written in the Go) writes incorrect value. + // we can ignore that minor error here + if (v != cdItem->LocalHeaderPos) + isOK = false; // ignore error + // isOK = false; // force error + } + */ + } + + if (isOK && ZIP64_IS_16_MAX(cdItem->Disk)) + { if (size < 4) isOK = false; else { size -= 4; cdItem->Disk = ReadUInt32(); }} + } + } + + // we can ignore errors, when some zip archiver still write all fields to zip64 extra in local header + // if (&& (cdItem || !isOK || origSize != 8 * 3 + 4 || size != 8 * 1 + 4)) + if (!isOK || size != 0) + { + HeadersWarning = true; + extra.Error = true; + extra.IsZip64_Error = true; + } + Skip(size); + } + else + { + ReadBuffer(subBlock.Data, size); + extra.SubBlocks.Add(subBlock); + if (subBlock.ID == NFileHeader::NExtraID::kIzUnicodeName) + { + if (!subBlock.CheckIzUnicode(item.Name)) + extra.Error = true; + } + } + } + + if (extraSize != 0) + { + ExtraMinorError = true; + extra.MinorError = true; + // 7-Zip before 9.31 created incorrect WzAES Extra in folder's local headers. + // so we don't return false, but just set warning flag + // return false; + Skip(extraSize); + } + + return true; +} + + +bool CInArchive::ReadLocalItem(CItemEx &item) +{ + item.Disk = 0; + if (IsMultiVol && Vols.StreamIndex >= 0) + item.Disk = (UInt32)Vols.StreamIndex; + const unsigned kPureHeaderSize = kLocalHeaderSize - 4; + Byte p[kPureHeaderSize]; + SafeRead(p, kPureHeaderSize); + { + unsigned i; + for (i = 0; i < kPureHeaderSize && p[i] == 0; i++); + if (i == kPureHeaderSize) + return false; + } + + item.ExtractVersion.Version = p[0]; + item.ExtractVersion.HostOS = p[1]; + G16(2, item.Flags); + G16(4, item.Method); + G32(6, item.Time); + G32(10, item.Crc); + G32(14, item.PackSize); + G32(18, item.Size); + const unsigned nameSize = Get16(p + 22); + const unsigned extraSize = Get16(p + 24); + bool isOkName = ReadFileName(nameSize, item.Name); + item.LocalFullHeaderSize = kLocalHeaderSize + (UInt32)nameSize + extraSize; + item.DescriptorWasRead = false; + + /* + if (item.IsDir()) + item.Size = 0; // check It + */ + + if (extraSize > 0) + { + if (!ReadExtra(item, extraSize, item.LocalExtra, item.Size, item.PackSize, NULL)) + { + /* Most of archives are OK for Extra. But there are some rare cases + that have error. And if error in first item, it can't open archive. + So we ignore that error */ + // return false; + } + } + + if (!CheckDosTime(item.Time)) + { + HeadersWarning = true; + // return false; + } + + if (item.Name.Len() != nameSize) + { + // we support some "bad" zip archives that contain zeros after name + if (!isOkName) + return false; + HeadersWarning = true; + } + + // return item.LocalFullHeaderSize <= ((UInt32)1 << 16); + return true; +} + + +static bool FlagsAreSame(const CItem &i1, const CItem &i2_cd) +{ + if (i1.Method != i2_cd.Method) + return false; + + UInt32 mask = i1.Flags ^ i2_cd.Flags; + if (mask == 0) + return true; + switch (i1.Method) + { + case NFileHeader::NCompressionMethod::kDeflate: + mask &= 0x7FF9; + break; + default: + if (i1.Method <= NFileHeader::NCompressionMethod::kImplode) + mask &= 0x7FFF; + } + + // we can ignore utf8 flag, if name is ascii, or if only cdItem has utf8 flag + if (mask & NFileHeader::NFlags::kUtf8) + if ((i1.Name.IsAscii() && i2_cd.Name.IsAscii()) + || (i2_cd.Flags & NFileHeader::NFlags::kUtf8)) + mask &= ~NFileHeader::NFlags::kUtf8; + + // some bad archive in rare case can use descriptor without descriptor flag in Central Dir + // if (i1.HasDescriptor()) + mask &= ~NFileHeader::NFlags::kDescriptorUsedMask; + + return (mask == 0); +} + + +// #ifdef _WIN32 +static bool AreEqualPaths_IgnoreSlashes(const char *s1, const char *s2) +{ + for (;;) + { + char c1 = *s1++; + char c2 = *s2++; + if (c1 == c2) + { + if (c1 == 0) + return true; + } + else + { + if (c1 == '\\') c1 = '/'; + if (c2 == '\\') c2 = '/'; + if (c1 != c2) + return false; + } + } +} +// #endif + + +static bool AreItemsEqual(const CItemEx &localItem, const CItemEx &cdItem) +{ + if (!FlagsAreSame(localItem, cdItem)) + return false; + if (!localItem.HasDescriptor()) + { + if (cdItem.PackSize != localItem.PackSize + || cdItem.Size != localItem.Size + || (cdItem.Crc != localItem.Crc && cdItem.Crc != 0)) // some program writes 0 to crc field in central directory + return false; + } + /* pkzip 2.50 creates incorrect archives. It uses + - WIN encoding for name in local header + - OEM encoding for name in central header + We don't support these strange items. */ + + /* if (cdItem.Name.Len() != localItem.Name.Len()) + return false; + */ + if (cdItem.Name != localItem.Name) + { + // #ifdef _WIN32 + // some xap files use backslash in central dir items. + // we can ignore such errors in windows, where all slashes are converted to backslashes + unsigned hostOs = cdItem.GetHostOS(); + + if (hostOs == NFileHeader::NHostOS::kFAT || + hostOs == NFileHeader::NHostOS::kNTFS) + { + if (!AreEqualPaths_IgnoreSlashes(cdItem.Name, localItem.Name)) + { + // pkzip 2.50 uses DOS encoding in central dir and WIN encoding in local header. + // so we ignore that error + if (hostOs != NFileHeader::NHostOS::kFAT + || cdItem.MadeByVersion.Version < 25 + || cdItem.MadeByVersion.Version > 40) + return false; + } + } + /* + else + #endif + return false; + */ + } + return true; +} + + +HRESULT CInArchive::Read_LocalItem_After_CdItem(CItemEx &item, bool &isAvail, bool &headersError) +{ + isAvail = true; + headersError = false; + if (item.FromLocal) + return S_OK; + try + { + UInt64 offset = item.LocalHeaderPos; + + if (IsMultiVol) + { + if (item.Disk >= Vols.Streams.Size()) + { + isAvail = false; + return S_FALSE; + } + Stream = Vols.Streams[item.Disk].Stream; + Vols.StreamIndex = (int)item.Disk; + if (!Stream) + { + isAvail = false; + return S_FALSE; + } + } + else + { + if (UseDisk_in_SingleVol && item.Disk != EcdVolIndex) + { + isAvail = false; + return S_FALSE; + } + Stream = StreamRef; + + offset = (UInt64)((Int64)offset + ArcInfo.Base); + if (ArcInfo.Base < 0 && (Int64)offset < 0) + { + isAvail = false; + return S_FALSE; + } + } + + _inBufMode = false; + RINOK(Seek_SavePos(offset)) + InitBuf(); + /* + // we can use buf mode with small buffer to reduce + // the number of Read() calls in ReadLocalItem() + _inBufMode = true; + Buffer.Alloc(1 << 10); + if (!Buffer.IsAllocated()) + return E_OUTOFMEMORY; + */ + + CItemEx localItem; + if (ReadUInt32() != NSignature::kLocalFileHeader) + return S_FALSE; + ReadLocalItem(localItem); + if (!AreItemsEqual(localItem, item)) + return S_FALSE; + item.LocalFullHeaderSize = localItem.LocalFullHeaderSize; + item.LocalExtra = localItem.LocalExtra; + if (item.Crc != localItem.Crc && !localItem.HasDescriptor()) + { + item.Crc = localItem.Crc; + headersError = true; + } + if ((item.Flags ^ localItem.Flags) & NFileHeader::NFlags::kDescriptorUsedMask) + { + item.Flags = (UInt16)(item.Flags ^ NFileHeader::NFlags::kDescriptorUsedMask); + headersError = true; + } + item.FromLocal = true; + } + catch(...) { return S_FALSE; } + return S_OK; +} + + +/* +---------- FindDescriptor ---------- + +in: + _streamPos : position in Stream + Stream : + Vols : if (IsMultiVol) + +action: + searches descriptor in input stream(s). + sets + item.DescriptorWasRead = true; + item.Size + item.PackSize + item.Crc + if descriptor was found + +out: + S_OK: + if ( item.DescriptorWasRead) : if descriptor was found + if (!item.DescriptorWasRead) : if descriptor was not found : unexpected end of stream(s) + + S_FALSE: if no items or there is just one item with strange properies that doesn't look like real archive. + + another error code: Callback error. + +exceptions : + CSystemException() : stream reading error +*/ + +HRESULT CInArchive::FindDescriptor(CItemEx &item, unsigned numFiles) +{ + // const size_t kBufSize = (size_t)1 << 5; // don't increase it too much. It reads data look ahead. + + // Buffer.Alloc(kBufSize); + // Byte *buf = Buffer; + + UInt64 packedSize = 0; + + UInt64 progressPrev = _cnt; + + for (;;) + { + /* appnote specification claims that we must use 64-bit descriptor, if there is zip64 extra. + But some old third-party xps archives used 64-bit descriptor without zip64 extra. */ + // unsigned descriptorSize = kDataDescriptorSize64 + kNextSignatureSize; + + // const unsigned kNextSignatureSize = 0; // we can disable check for next signatuire + const unsigned kNextSignatureSize = 4; // we check also for signature for next File headear + + const unsigned descriptorSize4 = item.GetDescriptorSize() + kNextSignatureSize; + + if (descriptorSize4 > Buffer.Size()) return E_FAIL; + + // size_t processedSize; + CanStartNewVol = true; + RINOK(LookAhead(descriptorSize4)) + const size_t avail = GetAvail(); + + if (avail < descriptorSize4) + { + // we write to packSize all these available bytes. + // later it's simpler to work with such value than with 0 + // if (item.PackSize == 0) + item.PackSize = packedSize + avail; + if (item.Method == 0) + item.Size = item.PackSize; + SkipLookahed(avail); + return S_OK; + } + + const Byte * const pStart = Buffer + _bufPos; + const Byte * p = pStart; + const Byte * const limit = pStart + (avail - descriptorSize4); + + for (; p <= limit; p++) + { + // descriptor signature field is Info-ZIP's extension to pkware Zip specification. + // New ZIP specification also allows descriptorSignature. + + p = FindPK_4(p, limit + 1); + if (p > limit) + break; + + /* + if (*p != 0x50) + continue; + */ + + if (Get32(p) != NSignature::kDataDescriptor) + continue; + + // we check next signatuire after descriptor + // maybe we need check only 2 bytes "PK" instead of 4 bytes, if some another type of header is possible after descriptor + const UInt32 sig = Get32(p + descriptorSize4 - kNextSignatureSize); + if ( sig != NSignature::kLocalFileHeader + && sig != NSignature::kCentralFileHeader) + continue; + + const UInt64 packSizeCur = packedSize + (size_t)(p - pStart); + if (descriptorSize4 == kDataDescriptorSize64 + kNextSignatureSize) // if (item.LocalExtra.IsZip64) + { + const UInt64 descriptorPackSize = Get64(p + 8); + if (descriptorPackSize != packSizeCur) + continue; + item.Size = Get64(p + 16); + } + else + { + const UInt32 descriptorPackSize = Get32(p + 8); + if (descriptorPackSize != (UInt32)packSizeCur) + continue; + item.Size = Get32(p + 12); + // that item.Size can be truncated to 32-bit value here + } + // We write calculated 64-bit packSize, even if descriptor64 was not used + item.PackSize = packSizeCur; + + item.DescriptorWasRead = true; + item.Crc = Get32(p + 4); + + const size_t skip = (size_t)(p - pStart) + descriptorSize4 - kNextSignatureSize; + + SkipLookahed(skip); + + return S_OK; + } + + const size_t skip = (size_t)(p - pStart); + SkipLookahed(skip); + + packedSize += skip; + + if (Callback) + if (_cnt - progressPrev >= ((UInt32)1 << 22)) + { + progressPrev = _cnt; + const UInt64 numFiles64 = numFiles; + RINOK(Callback->SetCompleted(&numFiles64, &_cnt)) + } + } +} + + +HRESULT CInArchive::CheckDescriptor(const CItemEx &item) +{ + if (!item.HasDescriptor()) + return S_OK; + + // pkzip's version without descriptor signature is not supported + + bool isFinished = false; + RINOK(IncreaseRealPosition(item.PackSize, isFinished)) + if (isFinished) + return S_FALSE; + + /* + if (!IsMultiVol) + { + RINOK(Seek_SavePos(ArcInfo.Base + item.GetDataPosition() + item.PackSize)); + } + */ + + Byte buf[kDataDescriptorSize64]; + try + { + CanStartNewVol = true; + SafeRead(buf, item.GetDescriptorSize()); + } + catch (const CSystemException &e) { return e.ErrorCode; } + // catch (const CUnexpectEnd &) + catch(...) + { + return S_FALSE; + } + // RINOK(ReadStream_FALSE(Stream, buf, item.GetDescriptorSize())); + + if (Get32(buf) != NSignature::kDataDescriptor) + return S_FALSE; + UInt32 crc = Get32(buf + 4); + UInt64 packSize, unpackSize; + + if (item.LocalExtra.IsZip64) + { + packSize = Get64(buf + 8); + unpackSize = Get64(buf + 16); + } + else + { + packSize = Get32(buf + 8); + unpackSize = Get32(buf + 12); + } + + if (crc != item.Crc || item.PackSize != packSize || item.Size != unpackSize) + return S_FALSE; + return S_OK; +} + + +HRESULT CInArchive::Read_LocalItem_After_CdItem_Full(CItemEx &item) +{ + if (item.FromLocal) + return S_OK; + try + { + bool isAvail = true; + bool headersError = false; + RINOK(Read_LocalItem_After_CdItem(item, isAvail, headersError)) + if (headersError) + return S_FALSE; + if (item.HasDescriptor()) + return CheckDescriptor(item); + } + catch(...) { return S_FALSE; } + return S_OK; +} + + +HRESULT CInArchive::ReadCdItem(CItemEx &item) +{ + item.FromCentral = true; + Byte p[kCentralHeaderSize - 4]; + SafeRead(p, kCentralHeaderSize - 4); + + item.MadeByVersion.Version = p[0]; + item.MadeByVersion.HostOS = p[1]; + item.ExtractVersion.Version = p[2]; + item.ExtractVersion.HostOS = p[3]; + G16(4, item.Flags); + G16(6, item.Method); + G32(8, item.Time); + G32(12, item.Crc); + G32(16, item.PackSize); + G32(20, item.Size); + const unsigned nameSize = Get16(p + 24); + const unsigned extraSize = Get16(p + 26); + const unsigned commentSize = Get16(p + 28); + G16(30, item.Disk); + G16(32, item.InternalAttrib); + G32(34, item.ExternalAttrib); + G32(38, item.LocalHeaderPos); + ReadFileName(nameSize, item.Name); + + if (extraSize > 0) + ReadExtra(item, extraSize, item.CentralExtra, item.Size, item.PackSize, &item); + + // May be these strings must be deleted + /* + if (item.IsDir()) + item.Size = 0; + */ + + ReadBuffer(item.Comment, commentSize); + return S_OK; +} + + +/* +TryEcd64() + (_inBufMode == false) is expected here + so TryEcd64() can't change the Buffer. + if (Ecd64 is not covered by cached region), + TryEcd64() can change cached region ranges (_bufCached, _bufPos) and _streamPos. +*/ + +HRESULT CInArchive::TryEcd64(UInt64 offset, CCdInfo &cdInfo) +{ + if (offset >= ((UInt64)1 << 63)) + return S_FALSE; + Byte buf[kEcd64_FullSize]; + + RINOK(SeekToVol(Vols.StreamIndex, offset)) + RINOK(ReadFromCache_FALSE(buf, kEcd64_FullSize)) + + if (Get32(buf) != NSignature::kEcd64) + return S_FALSE; + UInt64 mainSize = Get64(buf + 4); + if (mainSize < kEcd64_MainSize || mainSize > ((UInt64)1 << 40)) + return S_FALSE; + cdInfo.ParseEcd64e(buf + 12); + return S_OK; +} + + +/* FindCd() doesn't use previous cached region, + but it uses Buffer. So it sets new cached region */ + +HRESULT CInArchive::FindCd(bool checkOffsetMode) +{ + CCdInfo &cdInfo = Vols.ecd; + + UInt64 endPos; + + // There are no useful data in cache in most cases here. + // So here we don't use cache data from previous operations . + + InitBuf(); + RINOK(InStream_GetSize_SeekToEnd(Stream, endPos)) + _streamPos = endPos; + + // const UInt32 kBufSizeMax2 = ((UInt32)1 << 16) + kEcdSize + kEcd64Locator_Size + kEcd64_FullSize; + const size_t kBufSizeMax = ((size_t)1 << 17); // must be larger than kBufSizeMax2 + + const size_t bufSize = (endPos < kBufSizeMax) ? (size_t)endPos : kBufSizeMax; + if (bufSize < kEcdSize) + return S_FALSE; + // CByteArr byteBuffer(bufSize); + + RINOK(AllocateBuffer(kBufSizeMax)) + + RINOK(Seek_SavePos(endPos - bufSize)) + + size_t processed = bufSize; + HRESULT res = ReadStream(Stream, Buffer, &processed); + _streamPos += processed; + _bufCached = processed; + _bufPos = 0; + _cnt += processed; + if (res != S_OK) + return res; + if (processed != bufSize) + return S_FALSE; + + + for (size_t i = bufSize - kEcdSize + 1;;) + { + if (i == 0) + return S_FALSE; + + const Byte *buf = Buffer; + + for (;;) + { + i--; + if (buf[i] == 0x50) + break; + if (i == 0) + return S_FALSE; + } + + if (Get32(buf + i) != NSignature::kEcd) + continue; + + cdInfo.ParseEcd32(buf + i); + + if (i >= kEcd64Locator_Size) + { + const size_t locatorIndex = i - kEcd64Locator_Size; + if (Get32(buf + locatorIndex) == NSignature::kEcd64Locator) + { + CLocator locator; + locator.Parse(buf + locatorIndex + 4); + UInt32 numDisks = locator.NumDisks; + // we ignore the error, where some zip creators use (NumDisks == 0) + if (numDisks == 0) + numDisks = 1; + if ((cdInfo.ThisDisk == numDisks - 1 || ZIP64_IS_16_MAX(cdInfo.ThisDisk)) + && locator.Ecd64Disk < numDisks) + { + if (locator.Ecd64Disk != cdInfo.ThisDisk && !ZIP64_IS_16_MAX(cdInfo.ThisDisk)) + return E_NOTIMPL; + + // Most of the zip64 use fixed size Zip64 ECD + // we try relative backward reading. + + UInt64 absEcd64 = endPos - bufSize + i - (kEcd64Locator_Size + kEcd64_FullSize); + + if (locatorIndex >= kEcd64_FullSize) + if (checkOffsetMode || absEcd64 == locator.Ecd64Offset) + { + const Byte *ecd64 = buf + locatorIndex - kEcd64_FullSize; + if (Get32(ecd64) == NSignature::kEcd64) + { + UInt64 mainEcd64Size = Get64(ecd64 + 4); + if (mainEcd64Size == kEcd64_MainSize) + { + cdInfo.ParseEcd64e(ecd64 + 12); + ArcInfo.Base = (Int64)(absEcd64 - locator.Ecd64Offset); + // ArcInfo.BaseVolIndex = cdInfo.ThisDisk; + return S_OK; + } + } + } + + // some zip64 use variable size Zip64 ECD. + // we try to use absolute offset from locator. + + if (absEcd64 != locator.Ecd64Offset) + { + if (TryEcd64(locator.Ecd64Offset, cdInfo) == S_OK) + { + ArcInfo.Base = 0; + // ArcInfo.BaseVolIndex = cdInfo.ThisDisk; + return S_OK; + } + } + + // for variable Zip64 ECD with for archives with offset != 0. + + if (checkOffsetMode + && ArcInfo.MarkerPos != 0 + && ArcInfo.MarkerPos + locator.Ecd64Offset != absEcd64) + { + if (TryEcd64(ArcInfo.MarkerPos + locator.Ecd64Offset, cdInfo) == S_OK) + { + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; + // ArcInfo.BaseVolIndex = cdInfo.ThisDisk; + return S_OK; + } + } + } + } + } + + // bool isVolMode = (Vols.EndVolIndex != -1); + // UInt32 searchDisk = (isVolMode ? Vols.EndVolIndex : 0); + + if (/* searchDisk == thisDisk && */ cdInfo.CdDisk <= cdInfo.ThisDisk) + { + // if (isVolMode) + { + if (cdInfo.CdDisk != cdInfo.ThisDisk) + return S_OK; + } + + UInt64 absEcdPos = endPos - bufSize + i; + UInt64 cdEnd = cdInfo.Size + cdInfo.Offset; + ArcInfo.Base = 0; + // ArcInfo.BaseVolIndex = cdInfo.ThisDisk; + if (absEcdPos != cdEnd) + { + /* + if (cdInfo.Offset <= 16 && cdInfo.Size != 0) + { + // here we support some rare ZIP files with Central directory at the start + ArcInfo.Base = 0; + } + else + */ + ArcInfo.Base = (Int64)(absEcdPos - cdEnd); + } + return S_OK; + } + } +} + + +HRESULT CInArchive::TryReadCd(CObjectVector &items, const CCdInfo &cdInfo, UInt64 cdOffset, UInt64 cdSize) +{ + items.Clear(); + IsCdUnsorted = false; + + // _startLocalFromCd_Disk = (UInt32)(Int32)-1; + // _startLocalFromCd_Offset = (UInt64)(Int64)-1; + + RINOK(SeekToVol(IsMultiVol ? (int)cdInfo.CdDisk : -1, cdOffset)) + + _inBufMode = true; + _cnt = 0; + + if (Callback) + { + RINOK(Callback->SetTotal(&cdInfo.NumEntries, IsMultiVol ? &Vols.TotalBytesSize : NULL)) + } + UInt64 numFileExpected = cdInfo.NumEntries; + const UInt64 *totalFilesPtr = &numFileExpected; + bool isCorrect_NumEntries = (cdInfo.IsFromEcd64 || numFileExpected >= ((UInt32)1 << 16)); + + while (_cnt < cdSize) + { + CanStartNewVol = true; + if (ReadUInt32() != NSignature::kCentralFileHeader) + return S_FALSE; + CanStartNewVol = false; + { + CItemEx cdItem; + RINOK(ReadCdItem(cdItem)) + + /* + if (cdItem.Disk < _startLocalFromCd_Disk || + cdItem.Disk == _startLocalFromCd_Disk && + cdItem.LocalHeaderPos < _startLocalFromCd_Offset) + { + _startLocalFromCd_Disk = cdItem.Disk; + _startLocalFromCd_Offset = cdItem.LocalHeaderPos; + } + */ + + if (items.Size() > 0 && !IsCdUnsorted) + { + const CItemEx &prev = items.Back(); + if (cdItem.Disk < prev.Disk + || (cdItem.Disk == prev.Disk && + cdItem.LocalHeaderPos < prev.LocalHeaderPos)) + IsCdUnsorted = true; + } + + items.Add(cdItem); + } + if (Callback && (items.Size() & 0xFFF) == 0) + { + const UInt64 numFiles = items.Size(); + + if (numFiles > numFileExpected && totalFilesPtr) + { + if (isCorrect_NumEntries) + totalFilesPtr = NULL; + else + while (numFiles > numFileExpected) + numFileExpected += (UInt32)1 << 16; + RINOK(Callback->SetTotal(totalFilesPtr, NULL)) + } + + RINOK(Callback->SetCompleted(&numFiles, &_cnt)) + } + } + + CanStartNewVol = true; + + return (_cnt == cdSize) ? S_OK : S_FALSE; +} + + +/* +static int CompareCdItems(void *const *elem1, void *const *elem2, void *) +{ + const CItemEx *i1 = *(const CItemEx **)elem1; + const CItemEx *i2 = *(const CItemEx **)elem2; + + if (i1->Disk < i2->Disk) return -1; + if (i1->Disk > i2->Disk) return 1; + if (i1->LocalHeaderPos < i2->LocalHeaderPos) return -1; + if (i1->LocalHeaderPos > i2->LocalHeaderPos) return 1; + if (i1 < i2) return -1; + if (i1 > i2) return 1; + return 0; +} +*/ + +HRESULT CInArchive::ReadCd(CObjectVector &items, UInt32 &cdDisk, UInt64 &cdOffset, UInt64 &cdSize) +{ + bool checkOffsetMode = true; + + if (IsMultiVol) + { + if (Vols.EndVolIndex == -1) + return S_FALSE; + Stream = Vols.Streams[(unsigned)Vols.EndVolIndex].Stream; + if (!Vols.StartIsZip) + checkOffsetMode = false; + } + else + Stream = StartStream; + + if (!Vols.ecd_wasRead) + { + RINOK(FindCd(checkOffsetMode)) + } + + CCdInfo &cdInfo = Vols.ecd; + + HRESULT res = S_FALSE; + + cdSize = cdInfo.Size; + cdOffset = cdInfo.Offset; + cdDisk = cdInfo.CdDisk; + + if (!IsMultiVol) + { + if (cdInfo.ThisDisk != cdInfo.CdDisk) + return S_FALSE; + } + + const UInt64 base = (IsMultiVol ? 0 : (UInt64)ArcInfo.Base); + res = TryReadCd(items, cdInfo, base + cdOffset, cdSize); + + if (res == S_FALSE && !IsMultiVol && base != ArcInfo.MarkerPos) + { + // do we need that additional attempt to read cd? + res = TryReadCd(items, cdInfo, ArcInfo.MarkerPos + cdOffset, cdSize); + if (res == S_OK) + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; + } + + // Some rare case files are unsorted + // items.Sort(CompareCdItems, NULL); + return res; +} + + +static int FindItem(const CObjectVector &items, const CItemEx &item) +{ + unsigned left = 0, right = items.Size(); + for (;;) + { + if (left >= right) + return -1; + const unsigned index = (unsigned)(((size_t)left + (size_t)right) / 2); + const CItemEx &item2 = items[index]; + if (item.Disk < item2.Disk) + right = index; + else if (item.Disk > item2.Disk) + left = index + 1; + else if (item.LocalHeaderPos == item2.LocalHeaderPos) + return (int)index; + else if (item.LocalHeaderPos < item2.LocalHeaderPos) + right = index; + else + left = index + 1; + } +} + +static bool IsStrangeItem(const CItem &item) +{ + return item.Name.Len() > (1 << 14) || item.Method > (1 << 8); +} + + + +/* + ---------- ReadLocals ---------- + +in: + (_signature == NSignature::kLocalFileHeader) + VirtStreamPos : after _signature : position in Stream + Stream : + Vols : if (IsMultiVol) + (_inBufMode == false) + +action: + it parses local items. + + if ( IsMultiVol) it writes absolute offsets to CItemEx::LocalHeaderPos + if (!IsMultiVol) it writes relative (from ArcInfo.Base) offsets to CItemEx::LocalHeaderPos + later we can correct CItemEx::LocalHeaderPos values, if + some new value for ArcInfo.Base will be detected +out: + S_OK: + (_signature != NSignature::kLocalFileHeade) + _streamPos : after _signature + + S_FALSE: if no items or there is just one item with strange properies that doesn't look like real archive. + + another error code: stream reading error or Callback error. + + CUnexpectEnd() exception : it's not fatal exception here. + It means that reading was interrupted by unexpected end of input stream, + but some CItemEx items were parsed OK. + We can stop further archive parsing. + But we can use all filled CItemEx items. +*/ + +HRESULT CInArchive::ReadLocals(CObjectVector &items) +{ + items.Clear(); + + UInt64 progressPrev = _cnt; + + if (Callback) + { + RINOK(Callback->SetTotal(NULL, IsMultiVol ? &Vols.TotalBytesSize : NULL)) + } + + while (_signature == NSignature::kLocalFileHeader) + { + CItemEx item; + + item.LocalHeaderPos = GetVirtStreamPos() - 4; + if (!IsMultiVol) + item.LocalHeaderPos = (UInt64)((Int64)item.LocalHeaderPos - ArcInfo.Base); + + try + { + ReadLocalItem(item); + item.FromLocal = true; + bool isFinished = false; + + if (item.HasDescriptor()) + { + RINOK(FindDescriptor(item, items.Size())) + isFinished = !item.DescriptorWasRead; + } + else + { + if (item.PackSize >= ((UInt64)1 << 62)) + throw CUnexpectEnd(); + RINOK(IncreaseRealPosition(item.PackSize, isFinished)) + } + + items.Add(item); + + if (isFinished) + throw CUnexpectEnd(); + + ReadSignature(); + } + catch (CUnexpectEnd &) + { + if (items.IsEmpty() || (items.Size() == 1 && IsStrangeItem(items[0]))) + return S_FALSE; + throw; + } + + + if (Callback) + if ((items.Size() & 0xFF) == 0 + || _cnt - progressPrev >= ((UInt32)1 << 22)) + { + progressPrev = _cnt; + const UInt64 numFiles = items.Size(); + RINOK(Callback->SetCompleted(&numFiles, &_cnt)) + } + } + + if (items.Size() == 1 && _signature != NSignature::kCentralFileHeader) + if (IsStrangeItem(items[0])) + return S_FALSE; + + return S_OK; +} + + + +HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) +{ + UString name; + { + NWindows::NCOM::CPropVariant prop; + RINOK(volCallback->GetProperty(kpidName, &prop)) + if (prop.vt != VT_BSTR) + return S_OK; + name = prop.bstrVal; + } + + const int dotPos = name.ReverseFind_Dot(); + if (dotPos < 0) + return S_OK; + const UString ext = name.Ptr((unsigned)(dotPos + 1)); + name.DeleteFrom((unsigned)(dotPos + 1)); + + StartVolIndex = (Int32)(-1); + + if (ext.IsEmpty()) + return S_OK; + { + wchar_t c = ext[0]; + IsUpperCase = (c >= 'A' && c <= 'Z'); + if (ext.IsEqualTo_Ascii_NoCase("zip")) + { + BaseName = name; + StartIsZ = true; + StartIsZip = true; + return S_OK; + } + else if (ext.IsEqualTo_Ascii_NoCase("exe")) + { + /* possible cases: + - exe with zip inside + - sfx: a.exe, a.z02, a.z03,... , a.zip + a.exe is start volume. + - zip renamed to exe + */ + + StartIsExe = true; + BaseName = name; + StartVolIndex = 0; + /* sfx-zip can use both arc.exe and arc.zip + We can open arc.zip, if it was requesed to open arc.exe. + But it's possible that arc.exe and arc.zip are not parts of same archive. + So we can disable such operation */ + + // 18.04: we still want to open zip renamed to exe. + /* + { + UString volName = name; + volName += IsUpperCase ? "Z01" : "z01"; + { + CMyComPtr stream; + HRESULT res2 = volCallback->GetStream(volName, &stream); + if (res2 == S_OK) + DisableVolsSearch = true; + } + } + */ + DisableVolsSearch = true; + return S_OK; + } + else if (ext[0] == 'z' || ext[0] == 'Z') + { + if (ext.Len() < 3) + return S_OK; + const wchar_t *end = NULL; + UInt32 volNum = ConvertStringToUInt32(ext.Ptr(1), &end); + if (*end != 0 || volNum < 1 || volNum > ((UInt32)1 << 30)) + return S_OK; + StartVolIndex = (Int32)(volNum - 1); + BaseName = name; + StartIsZ = true; + } + else + return S_OK; + } + + UString volName = BaseName; + volName += (IsUpperCase ? "ZIP" : "zip"); + + HRESULT res = volCallback->GetStream(volName, &ZipStream); + + if (res == S_FALSE || !ZipStream) + { + if (MissingName.IsEmpty()) + { + MissingZip = true; + MissingName = volName; + } + return S_OK; + } + + return res; +} + + +HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, + unsigned start, int lastDisk, int zipDisk, unsigned numMissingVolsMax, unsigned &numMissingVols) +{ + if (Vols.DisableVolsSearch) + return S_OK; + + numMissingVols = 0; + + for (unsigned i = start;; i++) + { + if (lastDisk >= 0 && i >= (unsigned)lastDisk) + break; + + if (i < Vols.Streams.Size()) + if (Vols.Streams[i].Stream) + continue; + + CMyComPtr stream; + + if ((int)i == zipDisk) + { + stream = Vols.ZipStream; + } + else if ((int)i == Vols.StartVolIndex) + { + stream = StartStream; + } + else + { + UString volName = Vols.BaseName; + { + volName.Add_Char(Vols.IsUpperCase ? 'Z' : 'z'); + const unsigned v = i + 1; + if (v < 10) + volName.Add_Char('0'); + volName.Add_UInt32(v); + } + + HRESULT res = volCallback->GetStream(volName, &stream); + if (res != S_OK && res != S_FALSE) + return res; + if (res == S_FALSE || !stream) + { + if (i == 0) + { + UString volName_exe = Vols.BaseName; + volName_exe += (Vols.IsUpperCase ? "EXE" : "exe"); + + HRESULT res2 = volCallback->GetStream(volName_exe, &stream); + if (res2 != S_OK && res2 != S_FALSE) + return res2; + res = res2; + } + } + if (res == S_FALSE || !stream) + { + if (i == 1 && Vols.StartIsExe) + return S_OK; + if (Vols.MissingName.IsEmpty()) + Vols.MissingName = volName; + numMissingVols++; + if (numMissingVols > numMissingVolsMax) + return S_OK; + if (lastDisk == -1 && numMissingVols != 0) + return S_OK; + continue; + } + } + + UInt64 pos, size; + RINOK(InStream_GetPos_GetSize(stream, pos, size)) + + while (i >= Vols.Streams.Size()) + Vols.Streams.AddNew(); + + CVols::CSubStreamInfo &ss = Vols.Streams[i]; + Vols.NumVols++; + Vols.TotalBytesSize += size; + + ss.Stream = stream; + ss.Size = size; + + if ((int)i == zipDisk) + { + Vols.EndVolIndex = (int)(Vols.Streams.Size() - 1); + break; + } + } + + return S_OK; +} + + +HRESULT CInArchive::ReadVols() +{ + CMyComPtr volCallback; + + Callback->QueryInterface(IID_IArchiveOpenVolumeCallback, (void **)&volCallback); + if (!volCallback) + return S_OK; + + RINOK(Vols.ParseArcName(volCallback)) + + // const int startZIndex = Vols.StartVolIndex; + + if (!Vols.StartIsZ) + { + if (!Vols.StartIsExe) + return S_OK; + } + + int zipDisk = -1; + int cdDisk = -1; + + if (Vols.StartIsZip) + Vols.ZipStream = StartStream; + + if (Vols.ZipStream) + { + Stream = Vols.ZipStream; + + if (Vols.StartIsZip) + Vols.StreamIndex = -1; + else + { + Vols.StreamIndex = -2; + InitBuf(); + } + + HRESULT res = FindCd(true); + + CCdInfo &ecd = Vols.ecd; + if (res == S_OK) + { + zipDisk = (int)ecd.ThisDisk; + Vols.ecd_wasRead = true; + + // if is not multivol or bad multivol, we return to main single stream code + if (ecd.ThisDisk == 0 + || ecd.ThisDisk >= ((UInt32)1 << 30) + || ecd.ThisDisk < ecd.CdDisk) + return S_OK; + + cdDisk = (int)ecd.CdDisk; + if (Vols.StartVolIndex < 0) + Vols.StartVolIndex = (Int32)ecd.ThisDisk; + else if ((UInt32)Vols.StartVolIndex >= ecd.ThisDisk) + return S_OK; + + // Vols.StartVolIndex = ecd.ThisDisk; + // Vols.EndVolIndex = ecd.ThisDisk; + unsigned numMissingVols; + if (cdDisk != zipDisk) + { + // get volumes required for cd. + RINOK(ReadVols2(volCallback, (unsigned)cdDisk, zipDisk, zipDisk, 0, numMissingVols)) + if (numMissingVols != 0) + { + // cdOK = false; + } + } + } + else if (res != S_FALSE) + return res; + } + + if (Vols.StartVolIndex < 0) + { + // is not mutivol; + return S_OK; + } + + /* + if (!Vols.Streams.IsEmpty()) + IsMultiVol = true; + */ + + unsigned numMissingVols; + + if (cdDisk != 0) + { + // get volumes that were no requested still + const unsigned kNumMissingVolsMax = 1 << 12; + RINOK(ReadVols2(volCallback, 0, cdDisk < 0 ? -1 : cdDisk, zipDisk, kNumMissingVolsMax, numMissingVols)) + } + + // if (Vols.StartVolIndex >= 0) + { + if (Vols.Streams.IsEmpty()) + if (Vols.StartVolIndex > (1 << 20)) + return S_OK; + if ((unsigned)Vols.StartVolIndex >= Vols.Streams.Size() + || !Vols.Streams[(unsigned)Vols.StartVolIndex].Stream) + { + // we get volumes starting from StartVolIndex, if they we not requested before know the volume index (if FindCd() was ok) + RINOK(ReadVols2(volCallback, (unsigned)Vols.StartVolIndex, zipDisk, zipDisk, 0, numMissingVols)) + } + } + + if (Vols.ZipStream) + { + // if there is no another volumes and volumeIndex is too big, we don't use multivol mode + if (Vols.Streams.IsEmpty()) + if (zipDisk > (1 << 10)) + return S_OK; + if (zipDisk >= 0) + { + // we create item in Streams for ZipStream, if we know the volume index (if FindCd() was ok) + RINOK(ReadVols2(volCallback, (unsigned)zipDisk, zipDisk + 1, zipDisk, 0, numMissingVols)) + } + } + + if (!Vols.Streams.IsEmpty()) + { + IsMultiVol = true; + /* + if (cdDisk) + IsMultiVol = true; + */ + const int startZIndex = Vols.StartVolIndex; + if (startZIndex >= 0) + { + // if all volumes before start volume are OK, we can start parsing from 0 + // if there are missing volumes before startZIndex, we start parsing in current startZIndex + if ((unsigned)startZIndex < Vols.Streams.Size()) + { + for (unsigned i = 0; i <= (unsigned)startZIndex; i++) + if (!Vols.Streams[i].Stream) + { + Vols.StartParsingVol = startZIndex; + break; + } + } + } + } + + return S_OK; +} + + + +HRESULT CVols::Read(void *data, UInt32 size, UInt32 *processedSize) +{ + if (processedSize) + *processedSize = 0; + if (size == 0) + return S_OK; + + for (;;) + { + if (StreamIndex < 0) + return S_OK; + if ((unsigned)StreamIndex >= Streams.Size()) + return S_OK; + const CVols::CSubStreamInfo &s = Streams[(unsigned)StreamIndex]; + if (!s.Stream) + return S_FALSE; + if (NeedSeek) + { + RINOK(s.SeekToStart()) + NeedSeek = false; + } + UInt32 realProcessedSize = 0; + HRESULT res = s.Stream->Read(data, size, &realProcessedSize); + if (processedSize) + *processedSize = realProcessedSize; + if (res != S_OK) + return res; + if (realProcessedSize != 0) + return res; + StreamIndex++; + NeedSeek = true; + } +} + +Z7_COM7F_IMF(CVolStream::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + return Vols->Read(data, size, processedSize); +} + + + + +#define COPY_ECD_ITEM_16(n) if (!isZip64 || !ZIP64_IS_16_MAX(ecd. n)) cdInfo. n = ecd. n; +#define COPY_ECD_ITEM_32(n) if (!isZip64 || !ZIP64_IS_32_MAX(ecd. n)) cdInfo. n = ecd. n; + + +HRESULT CInArchive::ReadHeaders(CObjectVector &items) +{ + // buffer that can be used for cd reading + RINOK(AllocateBuffer(kSeqBufferSize)) + + // here we can read small records. So we switch off _inBufMode. + _inBufMode = false; + + HRESULT res = S_OK; + + bool localsWereRead = false; + + /* we try to open archive with the following modes: + 1) CD-MODE : fast mode : we read backward ECD and CD, compare CD items with first Local item. + 2) LOCALS-CD-MODE : slow mode, if CD-MODE fails : we sequentially read all Locals and then CD. + Then we read sequentially ECD64, Locator, ECD again at the end. + + - in LOCALS-CD-MODE we use use the following + variables (with real cd properties) to set Base archive offset + and check real cd properties with values from ECD/ECD64. + */ + + UInt64 cdSize = 0; + UInt64 cdRelatOffset = 0; + UInt32 cdDisk = 0; + + UInt64 cdAbsOffset = 0; // absolute cd offset, for LOCALS-CD-MODE only. + +if (Force_ReadLocals_Mode) +{ + IsArc = true; + res = S_FALSE; // we will use LOCALS-CD-MODE mode +} +else +{ + if (!MarkerIsFound || !MarkerIsSafe) + { + IsArc = true; + res = ReadCd(items, cdDisk, cdRelatOffset, cdSize); + if (res == S_OK) + ReadSignature(); + else if (res != S_FALSE) + return res; + } + else // (MarkerIsFound && MarkerIsSafe) + { + + // _signature must be kLocalFileHeader or kEcd or kEcd64 + + SeekToVol(ArcInfo.MarkerVolIndex, ArcInfo.MarkerPos2 + 4); + + CanStartNewVol = false; + + if (_signature == NSignature::kEcd64) + { + // UInt64 ecd64Offset = GetVirtStreamPos() - 4; + IsZip64 = true; + + { + const UInt64 recordSize = ReadUInt64(); + if (recordSize < kEcd64_MainSize) + return S_FALSE; + if (recordSize >= ((UInt64)1 << 62)) + return S_FALSE; + + { + const unsigned kBufSize = kEcd64_MainSize; + Byte buf[kBufSize]; + SafeRead(buf, kBufSize); + CCdInfo cdInfo; + cdInfo.ParseEcd64e(buf); + if (!cdInfo.IsEmptyArc()) + return S_FALSE; + } + + RINOK(Skip64(recordSize - kEcd64_MainSize, 0)) + } + + ReadSignature(); + if (_signature != NSignature::kEcd64Locator) + return S_FALSE; + + { + const unsigned kBufSize = 16; + Byte buf[kBufSize]; + SafeRead(buf, kBufSize); + CLocator locator; + locator.Parse(buf); + if (!locator.IsEmptyArc()) + return S_FALSE; + } + + ReadSignature(); + if (_signature != NSignature::kEcd) + return S_FALSE; + } + + if (_signature == NSignature::kEcd) + { + // It must be empty archive or backware archive + // we don't support backware archive still + + const unsigned kBufSize = kEcdSize - 4; + Byte buf[kBufSize]; + SafeRead(buf, kBufSize); + CEcd ecd; + ecd.Parse(buf); + // if (ecd.cdSize != 0) + // Do we need also to support the case where empty zip archive with PK00 uses cdOffset = 4 ?? + if (!ecd.IsEmptyArc()) + return S_FALSE; + + ArcInfo.Base = (Int64)ArcInfo.MarkerPos; + IsArc = true; // check it: we need more tests? + + RINOK(SeekToVol(ArcInfo.MarkerVolIndex, ArcInfo.MarkerPos2)) + ReadSignature(); + } + else + { + CItemEx firstItem; + try + { + try + { + if (!ReadLocalItem(firstItem)) + return S_FALSE; + } + catch(CUnexpectEnd &) + { + return S_FALSE; + } + + IsArc = true; + res = ReadCd(items, cdDisk, cdRelatOffset, cdSize); + if (res == S_OK) + ReadSignature(); + } + catch(CUnexpectEnd &) { res = S_FALSE; } + + if (res != S_FALSE && res != S_OK) + return res; + + if (res == S_OK && items.Size() == 0) + res = S_FALSE; + + if (res == S_OK) + { + // we can't read local items here to keep _inBufMode state + if ((Int64)ArcInfo.MarkerPos2 < ArcInfo.Base) + res = S_FALSE; + else + { + firstItem.LocalHeaderPos = (UInt64)((Int64)ArcInfo.MarkerPos2 - ArcInfo.Base); + int index = -1; + + UInt32 min_Disk = (UInt32)(Int32)-1; + UInt64 min_LocalHeaderPos = (UInt64)(Int64)-1; + + if (!IsCdUnsorted) + index = FindItem(items, firstItem); + else + { + FOR_VECTOR (i, items) + { + const CItemEx &cdItem = items[i]; + if (cdItem.Disk == firstItem.Disk + && (cdItem.LocalHeaderPos == firstItem.LocalHeaderPos)) + index = (int)i; + + if (i == 0 + || cdItem.Disk < min_Disk + || (cdItem.Disk == min_Disk && cdItem.LocalHeaderPos < min_LocalHeaderPos)) + { + min_Disk = cdItem.Disk; + min_LocalHeaderPos = cdItem.LocalHeaderPos; + } + } + } + + if (index == -1) + res = S_FALSE; + else if (!AreItemsEqual(firstItem, items[(unsigned)index])) + res = S_FALSE; + else + { + ArcInfo.CdWasRead = true; + if (IsCdUnsorted) + ArcInfo.FirstItemRelatOffset = min_LocalHeaderPos; + else + ArcInfo.FirstItemRelatOffset = items[0].LocalHeaderPos; + + // ArcInfo.FirstItemRelatOffset = _startLocalFromCd_Offset; + } + } + } + } + } // (MarkerIsFound && MarkerIsSafe) + +} // (!onlyLocalsMode) + + + CObjectVector cdItems; + + bool needSetBase = false; // we set needSetBase only for LOCALS_CD_MODE + unsigned numCdItems = items.Size(); + + #ifdef ZIP_SELF_CHECK + res = S_FALSE; // if uncommented, it uses additional LOCALS-CD-MODE mode to check the code + #endif + + if (res != S_OK) + { + // ---------- LOCALS-CD-MODE ---------- + // CD doesn't match firstItem, + // so we clear items and read Locals and CD. + + items.Clear(); + localsWereRead = true; + + HeadersError = false; + HeadersWarning = false; + ExtraMinorError = false; + + /* we can use any mode: with buffer and without buffer + without buffer : skips packed data : fast for big files : slow for small files + with buffer : reads packed data : slow for big files : fast for small files + Buffer mode is more effective. */ + // _inBufMode = false; + _inBufMode = true; + // we could change the buffer size here, if we want smaller Buffer. + // RINOK(ReAllocateBuffer(kSeqBufferSize)); + // InitBuf() + + ArcInfo.Base = 0; + + if (!Disable_FindMarker) + { + if (!MarkerIsFound) + { + if (!IsMultiVol) + return S_FALSE; + if (Vols.StartParsingVol != 0) + return S_FALSE; + // if (StartParsingVol == 0) and we didn't find marker, we use default zero marker. + // so we suppose that there is no sfx stub + RINOK(SeekToVol(0, ArcInfo.MarkerPos2)) + } + else + { + if (ArcInfo.MarkerPos != 0) + { + /* + If multi-vol or there is (No)Span-marker at start of stream, we set (Base) as 0. + In another caes: + (No)Span-marker is supposed as false positive. So we set (Base) as main marker (MarkerPos2). + The (Base) can be corrected later after ECD reading. + But sfx volume with stub and (No)Span-marker in (!IsMultiVol) mode will have incorrect (Base) here. + */ + ArcInfo.Base = (Int64)ArcInfo.MarkerPos2; + } + RINOK(SeekToVol(ArcInfo.MarkerVolIndex, ArcInfo.MarkerPos2)) + } + } + _cnt = 0; + + ReadSignature(); + + LocalsWereRead = true; + + RINOK(ReadLocals(items)) + + if (_signature != NSignature::kCentralFileHeader) + { + // GetVirtStreamPos() - 4 + if (items.IsEmpty()) + return S_FALSE; + + bool isError = true; + + const UInt32 apkSize = _signature; + const unsigned kApkFooterSize = 16 + 8; + if (apkSize >= kApkFooterSize && apkSize <= (1 << 20)) + { + if (ReadUInt32() == 0) + { + CByteBuffer apk; + apk.Alloc(apkSize); + SafeRead(apk, apkSize); + ReadSignature(); + const Byte *footer = apk + apkSize - kApkFooterSize; + if (_signature == NSignature::kCentralFileHeader) + if (GetUi64(footer) == apkSize) + if (memcmp(footer + 8, "APK Sig Block 42", 16) == 0) + { + isError = false; + IsApk = true; + } + } + } + + if (isError) + { + NoCentralDir = true; + HeadersError = true; + return S_OK; + } + } + + _inBufMode = true; + + cdAbsOffset = GetVirtStreamPos() - 4; + cdDisk = (UInt32)Vols.StreamIndex; + + #ifdef ZIP_SELF_CHECK + if (!IsMultiVol && _cnt != GetVirtStreamPos() - ArcInfo.MarkerPos2) + return E_FAIL; + #endif + + const UInt64 processedCnt_start = _cnt; + + for (;;) + { + CItemEx cdItem; + + RINOK(ReadCdItem(cdItem)) + + cdItems.Add(cdItem); + if (Callback && (cdItems.Size() & 0xFFF) == 0) + { + const UInt64 numFiles = items.Size(); + const UInt64 numBytes = _cnt; + RINOK(Callback->SetCompleted(&numFiles, &numBytes)) + } + ReadSignature(); + if (_signature != NSignature::kCentralFileHeader) + break; + } + + cdSize = _cnt - processedCnt_start; + + #ifdef ZIP_SELF_CHECK + if (!IsMultiVol) + { + if (_cnt != GetVirtStreamPos() - ArcInfo.MarkerPos2) + return E_FAIL; + if (cdSize != (GetVirtStreamPos() - 4) - cdAbsOffset) + return E_FAIL; + } + #endif + + needSetBase = true; + numCdItems = cdItems.Size(); + cdRelatOffset = (UInt64)((Int64)cdAbsOffset - ArcInfo.Base); + + if (!cdItems.IsEmpty()) + { + ArcInfo.CdWasRead = true; + ArcInfo.FirstItemRelatOffset = cdItems[0].LocalHeaderPos; + } + } + + + + CCdInfo cdInfo; + CLocator locator; + bool isZip64 = false; + const UInt64 ecd64AbsOffset = GetVirtStreamPos() - 4; + int ecd64Disk = -1; + + if (_signature == NSignature::kEcd64) + { + ecd64Disk = Vols.StreamIndex; + + IsZip64 = isZip64 = true; + + { + const UInt64 recordSize = ReadUInt64(); + if (recordSize < kEcd64_MainSize + || recordSize >= ((UInt64)1 << 62)) + { + HeadersError = true; + return S_OK; + } + + { + const unsigned kBufSize = kEcd64_MainSize; + Byte buf[kBufSize]; + SafeRead(buf, kBufSize); + cdInfo.ParseEcd64e(buf); + } + + RINOK(Skip64(recordSize - kEcd64_MainSize, items.Size())) + } + + + ReadSignature(); + + if (_signature != NSignature::kEcd64Locator) + { + HeadersError = true; + return S_OK; + } + + { + const unsigned kBufSize = 16; + Byte buf[kBufSize]; + SafeRead(buf, kBufSize); + locator.Parse(buf); + // we ignore the error, where some zip creators use (NumDisks == 0) + // if (locator.NumDisks == 0) HeadersWarning = true; + } + + ReadSignature(); + } + + + if (_signature != NSignature::kEcd) + { + HeadersError = true; + return S_OK; + } + + + CanStartNewVol = false; + + // ---------- ECD ---------- + + CEcd ecd; + { + const unsigned kBufSize = kEcdSize - 4; + Byte buf[kBufSize]; + SafeRead(buf, kBufSize); + ecd.Parse(buf); + } + + COPY_ECD_ITEM_16(ThisDisk) + COPY_ECD_ITEM_16(CdDisk) + COPY_ECD_ITEM_16(NumEntries_in_ThisDisk) + COPY_ECD_ITEM_16(NumEntries) + COPY_ECD_ITEM_32(Size) + COPY_ECD_ITEM_32(Offset) + + bool cdOK = true; + + if ((UInt32)cdInfo.Size != (UInt32)cdSize) + { + // return S_FALSE; + cdOK = false; + } + + if (isZip64) + { + if (cdInfo.NumEntries != numCdItems + || cdInfo.Size != cdSize) + { + cdOK = false; + } + } + + + if (IsMultiVol) + { + if (cdDisk != cdInfo.CdDisk) + HeadersError = true; + } + else if (needSetBase && cdOK) + { + const UInt64 oldBase = (UInt64)ArcInfo.Base; + // localsWereRead == true + // ArcInfo.Base == ArcInfo.MarkerPos2 + // cdRelatOffset == (cdAbsOffset - ArcInfo.Base) + + if (isZip64) + { + if (ecd64Disk == Vols.StartVolIndex) + { + const Int64 newBase = (Int64)ecd64AbsOffset - (Int64)locator.Ecd64Offset; + if (newBase <= (Int64)ecd64AbsOffset) + { + if (!localsWereRead || newBase <= (Int64)ArcInfo.MarkerPos2) + { + ArcInfo.Base = newBase; + cdRelatOffset = (UInt64)((Int64)cdAbsOffset - newBase); + } + else + cdOK = false; + } + } + } + else if (numCdItems != 0) // we can't use ecd.Offset in empty archive? + { + if ((int)cdDisk == Vols.StartVolIndex) + { + const Int64 newBase = (Int64)cdAbsOffset - (Int64)cdInfo.Offset; + if (newBase <= (Int64)cdAbsOffset) + { + if (!localsWereRead || newBase <= (Int64)ArcInfo.MarkerPos2) + { + // cd can be more accurate, when it points before Locals + // so we change Base and cdRelatOffset + ArcInfo.Base = newBase; + cdRelatOffset = cdInfo.Offset; + } + else + { + // const UInt64 delta = ((UInt64)cdRelatOffset - cdInfo.Offset); + const UInt64 delta = ((UInt64)(newBase - ArcInfo.Base)); + if ((UInt32)delta == 0) + { + // we set Overflow32bit mode, only if there is (x<<32) offset + // between real_CD_offset_from_MarkerPos and CD_Offset_in_ECD. + // Base and cdRelatOffset unchanged + Overflow32bit = true; + } + else + cdOK = false; + } + } + else + cdOK = false; + } + } + // cdRelatOffset = cdAbsOffset - ArcInfo.Base; + + if (localsWereRead) + { + const UInt64 delta = (UInt64)((Int64)oldBase - ArcInfo.Base); + if (delta != 0) + { + FOR_VECTOR (i, items) + items[i].LocalHeaderPos += delta; + } + } + } + + if (!cdOK) + HeadersError = true; + + EcdVolIndex = cdInfo.ThisDisk; + + if (!IsMultiVol) + { + if (EcdVolIndex == 0 && Vols.MissingZip && Vols.StartIsExe) + { + Vols.MissingName.Empty(); + Vols.MissingZip = false; + } + + if (localsWereRead) + { + if (EcdVolIndex != 0) + { + FOR_VECTOR (i, items) + items[i].Disk = EcdVolIndex; + } + } + + UseDisk_in_SingleVol = true; + } + + if (isZip64) + { + if ((cdInfo.ThisDisk == 0 && ecd64AbsOffset != (UInt64)(ArcInfo.Base + (Int64)locator.Ecd64Offset)) + // || cdInfo.NumEntries_in_ThisDisk != numCdItems + || cdInfo.NumEntries != numCdItems + || cdInfo.Size != cdSize + || (cdInfo.Offset != cdRelatOffset && !items.IsEmpty())) + { + HeadersError = true; + return S_OK; + } + } + + if (cdOK && !cdItems.IsEmpty()) + { + // ---------- merge Central Directory Items ---------- + + CRecordVector items2; + + int nextLocalIndex = 0; + + LocalsCenterMerged = true; + + FOR_VECTOR (i, cdItems) + { + if (Callback) + if ((i & 0x3FFF) == 0) + { + const UInt64 numFiles64 = items.Size() + items2.Size(); + RINOK(Callback->SetCompleted(&numFiles64, &_cnt)) + } + + const CItemEx &cdItem = cdItems[i]; + + int index = -1; + + if (nextLocalIndex != -1) + { + if ((unsigned)nextLocalIndex < items.Size()) + { + CItemEx &item = items[(unsigned)nextLocalIndex]; + if (item.Disk == cdItem.Disk && + (item.LocalHeaderPos == cdItem.LocalHeaderPos + || (Overflow32bit && (UInt32)item.LocalHeaderPos == cdItem.LocalHeaderPos))) + index = nextLocalIndex++; + else + nextLocalIndex = -1; + } + } + + if (index == -1) + index = FindItem(items, cdItem); + + // index = -1; + + if (index == -1) + { + items2.Add(i); + HeadersError = true; + continue; + } + + CItemEx &item = items[(unsigned)index]; + if (item.Name != cdItem.Name + // || item.Name.Len() != cdItem.Name.Len() + || item.PackSize != cdItem.PackSize + || item.Size != cdItem.Size + // item.ExtractVersion != cdItem.ExtractVersion + || !FlagsAreSame(item, cdItem) + || item.Crc != cdItem.Crc) + { + HeadersError = true; + continue; + } + + // item.Name = cdItem.Name; + item.MadeByVersion = cdItem.MadeByVersion; + item.CentralExtra = cdItem.CentralExtra; + item.InternalAttrib = cdItem.InternalAttrib; + item.ExternalAttrib = cdItem.ExternalAttrib; + item.Comment = cdItem.Comment; + item.FromCentral = cdItem.FromCentral; + // 22.02: we force utf8 flag, if central header has utf8 flag + if (cdItem.Flags & NFileHeader::NFlags::kUtf8) + item.Flags |= NFileHeader::NFlags::kUtf8; + } + + FOR_VECTOR (k, items2) + items.Add(cdItems[items2[k]]); + } + + if (ecd.NumEntries < ecd.NumEntries_in_ThisDisk) + HeadersError = true; + + if (ecd.ThisDisk == 0) + { + // if (isZip64) + { + if (ecd.NumEntries != ecd.NumEntries_in_ThisDisk) + HeadersError = true; + } + } + + if (isZip64) + { + if (cdInfo.NumEntries != items.Size() + || (ecd.NumEntries != items.Size() && ecd.NumEntries != 0xFFFF)) + HeadersError = true; + } + else + { + // old 7-zip could store 32-bit number of CD items to 16-bit field. + // if (ecd.NumEntries != items.Size()) + if (ecd.NumEntries > items.Size()) + HeadersError = true; + + if (cdInfo.NumEntries != numCdItems) + { + if ((UInt16)cdInfo.NumEntries != (UInt16)numCdItems) + HeadersError = true; + else + Cd_NumEntries_Overflow_16bit = true; + } + } + + ReadBuffer(ArcInfo.Comment, ecd.CommentSize); + + _inBufMode = false; + + // DisableBufMode(); + // Buffer.Free(); + /* we can't clear buf varibles. we need them to calculate PhySize of archive */ + + if ((UInt16)cdInfo.NumEntries != (UInt16)numCdItems + || (UInt32)cdInfo.Size != (UInt32)cdSize + || ((UInt32)cdInfo.Offset != (UInt32)cdRelatOffset && !items.IsEmpty())) + { + // return S_FALSE; + HeadersError = true; + } + + #ifdef ZIP_SELF_CHECK + if (localsWereRead) + { + const UInt64 endPos = ArcInfo.MarkerPos2 + _cnt; + if (endPos != (IsMultiVol ? Vols.TotalBytesSize : ArcInfo.FileEndPos)) + { + // there are some data after the end of archive or error in code; + return E_FAIL; + } + } + #endif + + // printf("\nOpen OK"); + return S_OK; +} + + + +HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchLimit, + IArchiveOpenCallback *callback, CObjectVector &items) +{ + items.Clear(); + + Close(); + + UInt64 startPos; + RINOK(InStream_GetPos(stream, startPos)) + RINOK(InStream_GetSize_SeekToEnd(stream, ArcInfo.FileEndPos)) + _streamPos = ArcInfo.FileEndPos; + + StartStream = stream; + Stream = stream; + Callback = callback; + + DisableBufMode(); + + bool volWasRequested = false; + + if (!Disable_VolsRead) + if (callback + && (startPos == 0 || !searchLimit || *searchLimit != 0)) + { + // we try to read volumes only if it's first call (offset == 0) or scan is allowed. + volWasRequested = true; + RINOK(ReadVols()) + } + + if (Disable_FindMarker) + { + RINOK(SeekToVol(-1, startPos)) + StreamRef = stream; + Stream = stream; + MarkerIsFound = true; + MarkerIsSafe = true; + ArcInfo.MarkerPos = startPos; + ArcInfo.MarkerPos2 = startPos; + } + else + if (IsMultiVol && Vols.StartParsingVol == 0 && (unsigned)Vols.StartParsingVol < Vols.Streams.Size()) + { + // only StartParsingVol = 0 is safe search. + RINOK(SeekToVol(0, 0)) + // if (Stream) + { + // UInt64 limit = 1 << 22; // for sfx + UInt64 limit = 0; // without sfx + + HRESULT res = FindMarker(&limit); + + if (res == S_OK) + { + MarkerIsFound = true; + MarkerIsSafe = true; + } + else if (res != S_FALSE) + return res; + } + } + else + { + // printf("\nOpen offset = %u\n", (unsigned)startPos); + if (IsMultiVol + && (unsigned)Vols.StartParsingVol < Vols.Streams.Size() + && Vols.Streams[(unsigned)Vols.StartParsingVol].Stream) + { + RINOK(SeekToVol(Vols.StartParsingVol, Vols.StreamIndex == Vols.StartVolIndex ? startPos : 0)) + } + else + { + RINOK(SeekToVol(-1, startPos)) + } + + // UInt64 limit = 1 << 22; + // HRESULT res = FindMarker(&limit); + + HRESULT res = FindMarker(searchLimit); + + // const UInt64 curPos = GetVirtStreamPos(); + const UInt64 curPos = ArcInfo.MarkerPos2 + 4; + + if (res == S_OK) + MarkerIsFound = true; + else if (!IsMultiVol) + { + /* + // if (startPos != 0), probably CD could be already tested with another call with (startPos == 0). + // so we don't want to try to open CD again in that case. + if (startPos != 0) + return res; + // we can try to open CD, if there is no Marker and (startPos == 0). + // is it OK to open such files as ZIP, or big number of false positive, when CD can be find in end of file ? + */ + return res; + } + + if (ArcInfo.IsSpanMode && !volWasRequested) + { + RINOK(ReadVols()) + if (IsMultiVol && MarkerIsFound && ArcInfo.MarkerVolIndex < 0) + ArcInfo.MarkerVolIndex = Vols.StartVolIndex; + } + + MarkerIsSafe = !IsMultiVol + || (ArcInfo.MarkerVolIndex == 0 && ArcInfo.MarkerPos == 0) + ; + + + if (IsMultiVol) + { + if ((unsigned)Vols.StartVolIndex < Vols.Streams.Size()) + { + Stream = Vols.Streams[(unsigned)Vols.StartVolIndex].Stream; + if (Stream) + { + RINOK(Seek_SavePos(curPos)) + } + else + IsMultiVol = false; + } + else + IsMultiVol = false; + } + + if (!IsMultiVol) + { + if (Vols.StreamIndex != -1) + { + Stream = StartStream; + Vols.StreamIndex = -1; + InitBuf(); + RINOK(Seek_SavePos(curPos)) + } + + ArcInfo.MarkerVolIndex = -1; + StreamRef = stream; + Stream = stream; + } + } + + + if (!IsMultiVol) + Vols.ClearRefs(); + + { + HRESULT res; + try + { + res = ReadHeaders(items); + } + catch (const CSystemException &e) { res = e.ErrorCode; } + catch (const CUnexpectEnd &) + { + if (items.IsEmpty()) + return S_FALSE; + UnexpectedEnd = true; + res = S_OK; + } + catch (...) + { + DisableBufMode(); + throw; + } + + if (IsMultiVol) + { + ArcInfo.FinishPos = ArcInfo.FileEndPos; + if ((unsigned)Vols.StreamIndex < Vols.Streams.Size()) + if (GetVirtStreamPos() < Vols.Streams[(unsigned)Vols.StreamIndex].Size) + ArcInfo.ThereIsTail = true; + } + else + { + ArcInfo.FinishPos = GetVirtStreamPos(); + ArcInfo.ThereIsTail = (ArcInfo.FileEndPos > ArcInfo.FinishPos); + } + + DisableBufMode(); + + IsArcOpen = true; + if (!IsMultiVol) + Vols.Streams.Clear(); + return res; + } +} + + +HRESULT CInArchive::GetItemStream(const CItemEx &item, bool seekPackData, CMyComPtr &stream) +{ + stream.Release(); + + UInt64 pos = item.LocalHeaderPos; + if (seekPackData) + pos += item.LocalFullHeaderSize; + + if (!IsMultiVol) + { + if (UseDisk_in_SingleVol && item.Disk != EcdVolIndex) + return S_OK; + pos = (UInt64)((Int64)pos + ArcInfo.Base); + RINOK(InStream_SeekSet(StreamRef, pos)) + stream = StreamRef; + return S_OK; + } + + if (item.Disk >= Vols.Streams.Size()) + return S_OK; + + IInStream *str2 = Vols.Streams[item.Disk].Stream; + if (!str2) + return S_OK; + RINOK(InStream_SeekSet(str2, pos)) + + Vols.NeedSeek = false; + Vols.StreamIndex = (int)item.Disk; + + CVolStream *volsStreamSpec = new CVolStream; + volsStreamSpec->Vols = &Vols; + stream = volsStreamSpec; + + return S_OK; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipIn.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipIn.h new file mode 100644 index 0000000..bea26dc --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipIn.h @@ -0,0 +1,449 @@ +// Archive/ZipIn.h + +#ifndef ZIP7_INC_ZIP_IN_H +#define ZIP7_INC_ZIP_IN_H + +#include "../../../Common/MyBuffer2.h" +#include "../../../Common/MyCom.h" + +#include "../../Common/StreamUtils.h" +#include "../../IStream.h" + +#include "ZipHeader.h" +#include "ZipItem.h" + +API_FUNC_IsArc IsArc_Zip(const Byte *p, size_t size); + +namespace NArchive { +namespace NZip { + +class CItemEx: public CItem +{ +public: + UInt32 LocalFullHeaderSize; // including Name and Extra + // int ParentOfAltStream; // -1, if not AltStream + + bool DescriptorWasRead; + + CItemEx(): + // ParentOfAltStream(-1), + DescriptorWasRead(false) {} + + UInt64 GetLocalFullSize() const + { return LocalFullHeaderSize + GetPackSizeWithDescriptor(); } + UInt64 GetDataPosition() const + { return LocalHeaderPos + LocalFullHeaderSize; } + + bool IsBadDescriptor() const + { + return !FromCentral && FromLocal && HasDescriptor() && !DescriptorWasRead; + } +}; + + +struct CInArchiveInfo +{ + Int64 Base; /* Base offset of start of archive in stream. + Offsets in headers must be calculated from that Base. + Base is equal to MarkerPos for normal ZIPs. + Base can point to PE stub for some ZIP SFXs. + if CentralDir was read, + Base can be negative, if start of data is not available, + if CentralDirs was not read, + Base = ArcInfo.MarkerPos; */ + + /* The following *Pos variables contain absolute offsets in Stream */ + + UInt64 MarkerPos; /* Pos of first signature, it can point to kSpan/kNoSpan signature + = MarkerPos2 in most archives + = MarkerPos2 - 4 if there is kSpan/kNoSpan signature */ + UInt64 MarkerPos2; // Pos of first local item signature in stream + UInt64 FinishPos; // Finish pos of archive data in starting volume + UInt64 FileEndPos; // Finish pos of stream + + UInt64 FirstItemRelatOffset; /* Relative offset of first local (read from cd) (relative to Base). + = 0 in most archives + = size of stub for some SFXs */ + + + int MarkerVolIndex; + + bool CdWasRead; + bool IsSpanMode; + bool ThereIsTail; + + // UInt32 BaseVolIndex; + + CByteBuffer Comment; + + + CInArchiveInfo(): + Base(0), + MarkerPos(0), + MarkerPos2(0), + FinishPos(0), + FileEndPos(0), + FirstItemRelatOffset(0), + MarkerVolIndex(-1), + CdWasRead(false), + IsSpanMode(false), + ThereIsTail(false) + // BaseVolIndex(0) + {} + + void Clear() + { + // BaseVolIndex = 0; + Base = 0; + MarkerPos = 0; + MarkerPos2 = 0; + FinishPos = 0; + FileEndPos = 0; + MarkerVolIndex = -1; + ThereIsTail = false; + + FirstItemRelatOffset = 0; + + CdWasRead = false; + IsSpanMode = false; + + Comment.Free(); + } +}; + + +struct CCdInfo +{ + bool IsFromEcd64; + + UInt16 CommentSize; + + // 64 + UInt16 VersionMade; + UInt16 VersionNeedExtract; + + // old zip + UInt32 ThisDisk; + UInt32 CdDisk; + UInt64 NumEntries_in_ThisDisk; + UInt64 NumEntries; + UInt64 Size; + UInt64 Offset; + + CCdInfo() { memset(this, 0, sizeof(*this)); IsFromEcd64 = false; } + + void ParseEcd32(const Byte *p); // (p) includes signature + void ParseEcd64e(const Byte *p); // (p) exclude signature + + bool IsEmptyArc() const + { + return ThisDisk == 0 + && CdDisk == 0 + && NumEntries_in_ThisDisk == 0 + && NumEntries == 0 + && Size == 0 + && Offset == 0 // test it + ; + } +}; + + +struct CVols +{ + struct CSubStreamInfo + { + CMyComPtr Stream; + UInt64 Size; + + HRESULT SeekToStart() const { return InStream_SeekToBegin(Stream); } + + CSubStreamInfo(): Size(0) {} + }; + + CObjectVector Streams; + + int StreamIndex; // -1 for StartStream + // -2 for ZipStream at multivol detection code + // >=0 volume index in multivol + + bool NeedSeek; + + bool DisableVolsSearch; + bool StartIsExe; // is .exe + bool StartIsZ; // is .zip or .zNN + bool StartIsZip; // is .zip + bool IsUpperCase; + bool MissingZip; + + bool ecd_wasRead; + + Int32 StartVolIndex; // -1, if unknown vol index + // = (NN - 1), if StartStream is .zNN + // = 0, if start vol is exe + + Int32 StartParsingVol; // if we need local parsing, we must use that stream + unsigned NumVols; + + int EndVolIndex; // index of last volume (ecd volume), + // -1, if is not multivol + + UString BaseName; // name of archive including '.' + UString MissingName; + + CMyComPtr ZipStream; + + CCdInfo ecd; + + UInt64 TotalBytesSize; // for MultiVol only + + void ClearRefs() + { + Streams.Clear(); + ZipStream.Release(); + TotalBytesSize = 0; + } + + void Clear() + { + StreamIndex = -1; + NeedSeek = false; + + DisableVolsSearch = false; + StartIsExe = false; + StartIsZ = false; + StartIsZip = false; + IsUpperCase = false; + + StartVolIndex = -1; + StartParsingVol = 0; + NumVols = 0; + EndVolIndex = -1; + + BaseName.Empty(); + MissingName.Empty(); + + MissingZip = false; + ecd_wasRead = false; + + ClearRefs(); + } + + HRESULT ParseArcName(IArchiveOpenVolumeCallback *volCallback); + + HRESULT Read(void *data, UInt32 size, UInt32 *processedSize); +}; + + +Z7_CLASS_IMP_COM_1( + CVolStream + , ISequentialInStream +) +public: + CVols *Vols; +}; + + +class CInArchive +{ + CMidBuffer Buffer; + size_t _bufPos; + size_t _bufCached; + + UInt64 _streamPos; + UInt64 _cnt; + + // UInt32 _startLocalFromCd_Disk; + // UInt64 _startLocalFromCd_Offset; + + size_t GetAvail() const { return _bufCached - _bufPos; } + + void InitBuf() { _bufPos = 0; _bufCached = 0; } + void DisableBufMode() { InitBuf(); _inBufMode = false; } + + void SkipLookahed(size_t skip) + { + _bufPos += skip; + _cnt += skip; + } + + HRESULT AllocateBuffer(size_t size); + + UInt64 GetVirtStreamPos() { return _streamPos - _bufCached + _bufPos; } + + bool _inBufMode; + + bool IsArcOpen; + bool CanStartNewVol; + + UInt32 _signature; + + CMyComPtr StreamRef; + IInStream *Stream; + IInStream *StartStream; + IArchiveOpenCallback *Callback; + + HRESULT Seek_SavePos(UInt64 offset); + HRESULT SeekToVol(int volIndex, UInt64 offset); + + HRESULT ReadFromCache(Byte *data, unsigned size, unsigned &processed); + HRESULT ReadFromCache_FALSE(Byte *data, unsigned size); + + HRESULT ReadVols2(IArchiveOpenVolumeCallback *volCallback, + unsigned start, int lastDisk, int zipDisk, unsigned numMissingVolsMax, unsigned &numMissingVols); + HRESULT ReadVols(); + + HRESULT FindMarker(const UInt64 *searchLimit); + HRESULT IncreaseRealPosition(UInt64 addValue, bool &isFinished); + + HRESULT LookAhead(size_t minRequiredInBuffer); + void SafeRead(Byte *data, unsigned size); + void ReadBuffer(CByteBuffer &buffer, unsigned size); + // Byte ReadByte(); + // UInt16 ReadUInt16(); + UInt32 ReadUInt32(); + UInt64 ReadUInt64(); + + void ReadSignature(); + + void Skip(size_t num); + HRESULT Skip64(UInt64 num, unsigned numFiles); + + bool ReadFileName(unsigned nameSize, AString &dest); + + bool ReadExtra(const CLocalItem &item, unsigned extraSize, CExtraBlock &extra, + UInt64 &unpackSize, UInt64 &packSize, CItem *cdItem); + bool ReadLocalItem(CItemEx &item); + HRESULT FindDescriptor(CItemEx &item, unsigned numFiles); + HRESULT ReadCdItem(CItemEx &item); + HRESULT TryEcd64(UInt64 offset, CCdInfo &cdInfo); + HRESULT FindCd(bool checkOffsetMode); + HRESULT TryReadCd(CObjectVector &items, const CCdInfo &cdInfo, UInt64 cdOffset, UInt64 cdSize); + HRESULT ReadCd(CObjectVector &items, UInt32 &cdDisk, UInt64 &cdOffset, UInt64 &cdSize); + HRESULT ReadLocals(CObjectVector &localItems); + + HRESULT ReadHeaders(CObjectVector &items); + + HRESULT GetVolStream(unsigned vol, UInt64 pos, CMyComPtr &stream); + +public: + CInArchiveInfo ArcInfo; + + bool IsArc; + bool IsZip64; + + bool IsApk; + bool IsCdUnsorted; + + bool HeadersError; + bool HeadersWarning; + bool ExtraMinorError; + bool UnexpectedEnd; + bool LocalsWereRead; + bool LocalsCenterMerged; + bool NoCentralDir; + bool Overflow32bit; // = true, if zip without Zip64 extension support and it has some fields values truncated to 32-bits. + bool Cd_NumEntries_Overflow_16bit; // = true, if no Zip64 and 16-bit ecd:NumEntries was overflowed. + + bool MarkerIsFound; + bool MarkerIsSafe; + + bool IsMultiVol; + bool UseDisk_in_SingleVol; + UInt32 EcdVolIndex; + + CVols Vols; + + bool Force_ReadLocals_Mode; + bool Disable_VolsRead; + bool Disable_FindMarker; + + CInArchive(): + IsArcOpen(false), + Stream(NULL), + StartStream(NULL), + Callback(NULL), + Force_ReadLocals_Mode(false), + Disable_VolsRead(false), + Disable_FindMarker(false) + {} + + UInt64 GetPhySize() const + { + if (IsMultiVol) + return ArcInfo.FinishPos; + else + return (UInt64)((Int64)ArcInfo.FinishPos - ArcInfo.Base); + } + + UInt64 GetOffset() const + { + if (IsMultiVol) + return 0; + else + return (UInt64)ArcInfo.Base; + } + + + void ClearRefs(); + void Close(); + HRESULT Open(IInStream *stream, const UInt64 *searchLimit, IArchiveOpenCallback *callback, CObjectVector &items); + + bool IsOpen() const { return IsArcOpen; } + + bool AreThereErrors() const + { + return HeadersError + || UnexpectedEnd + || !Vols.MissingName.IsEmpty(); + } + + bool IsLocalOffsetOK(const CItemEx &item) const + { + if (item.FromLocal) + return true; + return (Int64)GetOffset() + (Int64)item.LocalHeaderPos >= 0; + } + + UInt64 GetEmbeddedStubSize() const + { + // it's possible that first item in CD doesn refers to first local item + // so FirstItemRelatOffset is not first local item + + if (ArcInfo.CdWasRead) + return ArcInfo.FirstItemRelatOffset; + if (IsMultiVol) + return 0; + return (UInt64)((Int64)ArcInfo.MarkerPos2 - ArcInfo.Base); + } + + + HRESULT CheckDescriptor(const CItemEx &item); + HRESULT Read_LocalItem_After_CdItem(CItemEx &item, bool &isAvail, bool &headersError); + HRESULT Read_LocalItem_After_CdItem_Full(CItemEx &item); + + HRESULT GetItemStream(const CItemEx &item, bool seekPackData, CMyComPtr &stream); + + IInStream *GetBaseStream() { return StreamRef; } + + bool CanUpdate() const + { + if (AreThereErrors() + || IsMultiVol + || ArcInfo.Base < 0 + || (Int64)ArcInfo.MarkerPos2 < ArcInfo.Base + || ArcInfo.ThereIsTail + || GetEmbeddedStubSize() != 0 + || IsApk + || IsCdUnsorted) + return false; + + // 7-zip probably can update archives with embedded stubs. + // we just disable that feature for more safety. + + return true; + } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipItem.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipItem.cpp new file mode 100644 index 0000000..5684cac --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipItem.cpp @@ -0,0 +1,464 @@ +// Archive/ZipItem.cpp + +#include "StdAfx.h" + +#include "../../../../C/CpuArch.h" +#include "../../../../C/7zCrc.h" + +#include "../../../Common/IntToString.h" +#include "../../../Common/MyLinux.h" +#include "../../../Common/StringConvert.h" + +#include "../../../Windows/PropVariantUtils.h" + +#include "../Common/ItemNameUtils.h" + +#include "ZipItem.h" + +namespace NArchive { +namespace NZip { + +using namespace NFileHeader; + + +/* +const char *k_SpecName_NTFS_STREAM = "@@NTFS@STREAM@"; +const char *k_SpecName_MAC_RESOURCE_FORK = "@@MAC@RESOURCE-FORK@"; +*/ + +static const CUInt32PCharPair g_ExtraTypes[] = +{ + { NExtraID::kZip64, "Zip64" }, + { NExtraID::kNTFS, "NTFS" }, + { NExtraID::kUnix0, "UNIX" }, + { NExtraID::kStrongEncrypt, "StrongCrypto" }, + { NExtraID::kUnixTime, "UT" }, + { NExtraID::kUnix1, "UX" }, + { NExtraID::kUnix2, "Ux" }, + { NExtraID::kUnixN, "ux" }, + { NExtraID::kIzUnicodeComment, "uc" }, + { NExtraID::kIzUnicodeName, "up" }, + { NExtraID::kIzNtSecurityDescriptor, "SD" }, + { NExtraID::kWzAES, "WzAES" }, + { NExtraID::kApkAlign, "ApkAlign" } +}; + +void CExtraSubBlock::PrintInfo(AString &s) const +{ + for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_ExtraTypes); i++) + { + const CUInt32PCharPair &pair = g_ExtraTypes[i]; + if (pair.Value == ID) + { + s += pair.Name; + if (ID == NExtraID::kUnixTime) + { + if (Data.Size() >= 1) + { + s.Add_Colon(); + const Byte flags = Data[0]; + if (flags & 1) s.Add_Char('M'); + if (flags & 2) s.Add_Char('A'); + if (flags & 4) s.Add_Char('C'); + const UInt32 size = (UInt32)(Data.Size()) - 1; + if (size % 4 == 0) + { + s.Add_Colon(); + s.Add_UInt32(size / 4); + } + } + } + /* + if (ID == NExtraID::kApkAlign && Data.Size() >= 2) + { + char sz[32]; + sz[0] = ':'; + ConvertUInt32ToHex(GetUi16(Data), sz + 1); + s += sz; + for (unsigned j = 2; j < Data.Size(); j++) + { + char sz[32]; + sz[0] = '-'; + ConvertUInt32ToHex(Data[j], sz + 1); + s += sz; + } + } + */ + return; + } + } + { + char sz[16]; + sz[0] = '0'; + sz[1] = 'x'; + ConvertUInt32ToHex(ID, sz + 2); + s += sz; + } +} + + +void CExtraBlock::PrintInfo(AString &s) const +{ + if (Error) + s.Add_OptSpaced("Extra_ERROR"); + + if (MinorError) + s.Add_OptSpaced("Minor_Extra_ERROR"); + + if (IsZip64 || IsZip64_Error) + { + s.Add_OptSpaced("Zip64"); + if (IsZip64_Error) + s += "_ERROR"; + } + + FOR_VECTOR (i, SubBlocks) + { + s.Add_Space_if_NotEmpty(); + SubBlocks[i].PrintInfo(s); + } +} + + +bool CExtraSubBlock::ExtractNtfsTime(unsigned index, FILETIME &ft) const +{ + ft.dwHighDateTime = ft.dwLowDateTime = 0; + UInt32 size = (UInt32)Data.Size(); + if (ID != NExtraID::kNTFS || size < 32) + return false; + const Byte *p = (const Byte *)Data; + p += 4; // for reserved + size -= 4; + while (size > 4) + { + UInt16 tag = GetUi16(p); + unsigned attrSize = GetUi16(p + 2); + p += 4; + size -= 4; + if (attrSize > size) + attrSize = size; + + if (tag == NNtfsExtra::kTagTime && attrSize >= 24) + { + p += 8 * index; + ft.dwLowDateTime = GetUi32(p); + ft.dwHighDateTime = GetUi32(p + 4); + return true; + } + p += attrSize; + size -= attrSize; + } + return false; +} + +bool CExtraSubBlock::Extract_UnixTime(bool isCentral, unsigned index, UInt32 &res) const +{ + /* Info-Zip : + The central-header extra field contains the modification + time only, or no timestamp at all. + Size of Data is used to flag its presence or absence + If "Flags" indicates that Modtime is present in the local header + field, it MUST be present in the central header field, too + */ + + res = 0; + UInt32 size = (UInt32)Data.Size(); + if (ID != NExtraID::kUnixTime || size < 5) + return false; + const Byte *p = (const Byte *)Data; + const Byte flags = *p++; + size--; + if (isCentral) + { + if (index != NUnixTime::kMTime || + (flags & (1 << NUnixTime::kMTime)) == 0 || + size < 4) + return false; + res = GetUi32(p); + return true; + } + for (unsigned i = 0; i < 3; i++) + if ((flags & (1 << i)) != 0) + { + if (size < 4) + return false; + if (index == i) + { + res = GetUi32(p); + return true; + } + p += 4; + size -= 4; + } + return false; +} + + +// Info-ZIP's abandoned "Unix1 timestamps & owner ID info" + +bool CExtraSubBlock::Extract_Unix01_Time(unsigned index, UInt32 &res) const +{ + res = 0; + const unsigned offset = index * 4; + if (Data.Size() < offset + 4) + return false; + if (ID != NExtraID::kUnix0 && + ID != NExtraID::kUnix1) + return false; + const Byte *p = (const Byte *)Data + offset; + res = GetUi32(p); + return true; +} + +/* +// PKWARE's Unix "extra" is similar to Info-ZIP's abandoned "Unix1 timestamps" +bool CExtraSubBlock::Extract_Unix_Time(unsigned index, UInt32 &res) const +{ + res = 0; + const unsigned offset = index * 4; + if (ID != NExtraID::kUnix0 || Data.Size() < offset) + return false; + const Byte *p = (const Byte *)Data + offset; + res = GetUi32(p); + return true; +} +*/ + +bool CExtraBlock::GetNtfsTime(unsigned index, FILETIME &ft) const +{ + FOR_VECTOR (i, SubBlocks) + { + const CExtraSubBlock &sb = SubBlocks[i]; + if (sb.ID == NFileHeader::NExtraID::kNTFS) + return sb.ExtractNtfsTime(index, ft); + } + return false; +} + +bool CExtraBlock::GetUnixTime(bool isCentral, unsigned index, UInt32 &res) const +{ + { + FOR_VECTOR (i, SubBlocks) + { + const CExtraSubBlock &sb = SubBlocks[i]; + if (sb.ID == NFileHeader::NExtraID::kUnixTime) + return sb.Extract_UnixTime(isCentral, index, res); + } + } + + switch (index) + { + case NUnixTime::kMTime: index = NUnixExtra::kMTime; break; + case NUnixTime::kATime: index = NUnixExtra::kATime; break; + default: return false; + } + + { + FOR_VECTOR (i, SubBlocks) + { + const CExtraSubBlock &sb = SubBlocks[i]; + if (sb.ID == NFileHeader::NExtraID::kUnix0 || + sb.ID == NFileHeader::NExtraID::kUnix1) + return sb.Extract_Unix01_Time(index, res); + } + } + return false; +} + + +bool CLocalItem::IsDir() const +{ + return NItemName::HasTailSlash(Name, GetCodePage()); +} + +bool CItem::IsDir() const +{ + // FIXME: we can check InfoZip UTF-8 name at first. + if (NItemName::HasTailSlash(Name, GetCodePage())) + return true; + + Byte hostOS = GetHostOS(); + + if (Size == 0 && PackSize == 0 && !Name.IsEmpty() && Name.Back() == '\\') + { + // do we need to use CharPrevExA? + // .NET Framework 4.5 : System.IO.Compression::CreateFromDirectory() probably writes backslashes to headers? + // so we support that case + switch (hostOS) + { + case NHostOS::kFAT: + case NHostOS::kNTFS: + case NHostOS::kHPFS: + case NHostOS::kVFAT: + return true; + default: break; + } + } + + if (!FromCentral) + return false; + + UInt16 highAttrib = (UInt16)((ExternalAttrib >> 16 ) & 0xFFFF); + + switch (hostOS) + { + case NHostOS::kAMIGA: + switch (highAttrib & NAmigaAttrib::kIFMT) + { + case NAmigaAttrib::kIFDIR: return true; + case NAmigaAttrib::kIFREG: return false; + default: return false; // change it throw kUnknownAttributes; + } + case NHostOS::kFAT: + case NHostOS::kNTFS: + case NHostOS::kHPFS: + case NHostOS::kVFAT: + return ((ExternalAttrib & FILE_ATTRIBUTE_DIRECTORY) != 0); + case NHostOS::kAtari: + case NHostOS::kMac: + case NHostOS::kVMS: + case NHostOS::kVM_CMS: + case NHostOS::kAcorn: + case NHostOS::kMVS: + return false; // change it throw kUnknownAttributes; + case NHostOS::kUnix: + return MY_LIN_S_ISDIR(highAttrib); + default: + return false; + } +} + +UInt32 CItem::GetWinAttrib() const +{ + UInt32 winAttrib = 0; + switch (GetHostOS()) + { + case NHostOS::kFAT: + case NHostOS::kNTFS: + if (FromCentral) + winAttrib = ExternalAttrib; + break; + case NHostOS::kUnix: + // do we need to clear 16 low bits in this case? + if (FromCentral) + { + /* + Some programs write posix attributes in high 16 bits of ExternalAttrib + Also some programs can write additional marker flag: + 0x8000 - p7zip + 0x4000 - Zip in MacOS + no marker - Info-Zip + + Client code has two options to detect posix field: + 1) check 0x8000 marker. In that case we must add 0x8000 marker here. + 2) check that high 4 bits (file type bits in posix field) of attributes are not zero. + */ + + winAttrib = ExternalAttrib & 0xFFFF0000; + + // #ifndef _WIN32 + winAttrib |= 0x8000; // add posix mode marker + // #endif + } + break; + default: break; + } + if (IsDir()) // test it; + winAttrib |= FILE_ATTRIBUTE_DIRECTORY; + return winAttrib; +} + +bool CItem::GetPosixAttrib(UInt32 &attrib) const +{ + // some archivers can store PosixAttrib in high 16 bits even with HostOS=FAT. + if (FromCentral && GetHostOS() == NHostOS::kUnix) + { + attrib = ExternalAttrib >> 16; + return (attrib != 0); + } + attrib = 0; + if (IsDir()) + attrib = MY_LIN_S_IFDIR; + return false; +} + + +bool CExtraSubBlock::CheckIzUnicode(const AString &s) const +{ + size_t size = Data.Size(); + if (size < 1 + 4) + return false; + const Byte *p = (const Byte *)Data; + if (p[0] > 1) + return false; + if (CrcCalc(s, s.Len()) != GetUi32(p + 1)) + return false; + size -= 5; + p += 5; + for (size_t i = 0; i < size; i++) + if (p[i] == 0) + return false; + return Check_UTF8_Buf((const char *)(const void *)p, size, false); +} + + +void CItem::GetUnicodeString(UString &res, const AString &s, bool isComment, bool useSpecifiedCodePage, UINT codePage) const +{ + bool isUtf8 = IsUtf8(); + // bool ignore_Utf8_Errors = true; + + if (!isUtf8) + { + { + const unsigned id = isComment ? + NFileHeader::NExtraID::kIzUnicodeComment: + NFileHeader::NExtraID::kIzUnicodeName; + const CObjectVector &subBlocks = GetMainExtra().SubBlocks; + + FOR_VECTOR (i, subBlocks) + { + const CExtraSubBlock &sb = subBlocks[i]; + if (sb.ID == id) + { + if (sb.CheckIzUnicode(s)) + { + // const unsigned kIzUnicodeHeaderSize = 5; + if (Convert_UTF8_Buf_To_Unicode( + (const char *)(const void *)(const Byte *)sb.Data + 5, + sb.Data.Size() - 5, res)) + return; + } + break; + } + } + } + + if (useSpecifiedCodePage) + isUtf8 = (codePage == CP_UTF8); + #ifdef _WIN32 + else if (GetHostOS() == NFileHeader::NHostOS::kUnix) + { + /* Some ZIP archives in Unix use UTF-8 encoding without Utf8 flag in header. + We try to get name as UTF-8. + Do we need to do it in POSIX version also? */ + isUtf8 = true; + + /* 21.02: we want to ignore UTF-8 errors to support file paths that are mixed + of UTF-8 and non-UTF-8 characters. */ + // ignore_Utf8_Errors = false; + // ignore_Utf8_Errors = true; + } + #endif + } + + + if (isUtf8) + { + ConvertUTF8ToUnicode(s, res); + return; + } + + MultiByteToUnicodeString2(res, s, useSpecifiedCodePage ? codePage : GetCodePage()); +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipItem.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipItem.h new file mode 100644 index 0000000..4a25de3 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipItem.h @@ -0,0 +1,356 @@ +// Archive/ZipItem.h + +#ifndef ZIP7_INC_ARCHIVE_ZIP_ITEM_H +#define ZIP7_INC_ARCHIVE_ZIP_ITEM_H + +#include "../../../../C/CpuArch.h" + +#include "../../../Common/MyBuffer.h" +#include "../../../Common/MyString.h" +#include "../../../Common/UTFConvert.h" + +#include "ZipHeader.h" + +namespace NArchive { +namespace NZip { + +/* +extern const char *k_SpecName_NTFS_STREAM; +extern const char *k_SpecName_MAC_RESOURCE_FORK; +*/ + +struct CVersion +{ + Byte Version; + Byte HostOS; +}; + +struct CExtraSubBlock +{ + UInt32 ID; + CByteBuffer Data; + + bool ExtractNtfsTime(unsigned index, FILETIME &ft) const; + bool Extract_UnixTime(bool isCentral, unsigned index, UInt32 &res) const; + bool Extract_Unix01_Time(unsigned index, UInt32 &res) const; + // bool Extract_Unix_Time(unsigned index, UInt32 &res) const; + + bool CheckIzUnicode(const AString &s) const; + + void PrintInfo(AString &s) const; +}; + +const unsigned k_WzAesExtra_Size = 7; + +struct CWzAesExtra +{ + UInt16 VendorVersion; // 1: AE-1, 2: AE-2, + // UInt16 VendorId; // 'A' 'E' + Byte Strength; // 1: 128-bit, 2: 192-bit, 3: 256-bit + UInt16 Method; + + CWzAesExtra(): VendorVersion(2), Strength(3), Method(0) {} + + bool NeedCrc() const { return (VendorVersion == 1); } + + bool ParseFromSubBlock(const CExtraSubBlock &sb) + { + if (sb.ID != NFileHeader::NExtraID::kWzAES) + return false; + if (sb.Data.Size() < k_WzAesExtra_Size) + return false; + const Byte *p = (const Byte *)sb.Data; + VendorVersion = GetUi16(p); + if (p[2] != 'A' || p[3] != 'E') + return false; + Strength = p[4]; + // 9.31: The BUG was fixed: + Method = GetUi16(p + 5); + return true; + } + + void SetSubBlock(CExtraSubBlock &sb) const + { + sb.Data.Alloc(k_WzAesExtra_Size); + sb.ID = NFileHeader::NExtraID::kWzAES; + Byte *p = (Byte *)sb.Data; + p[0] = (Byte)VendorVersion; + p[1] = (Byte)(VendorVersion >> 8); + p[2] = 'A'; + p[3] = 'E'; + p[4] = Strength; + p[5] = (Byte)Method; + p[6] = (Byte)(Method >> 8); + } +}; + +namespace NStrongCrypto_AlgId +{ + const UInt16 kDES = 0x6601; + const UInt16 kRC2old = 0x6602; + const UInt16 k3DES168 = 0x6603; + const UInt16 k3DES112 = 0x6609; + const UInt16 kAES128 = 0x660E; + const UInt16 kAES192 = 0x660F; + const UInt16 kAES256 = 0x6610; + const UInt16 kRC2 = 0x6702; + const UInt16 kBlowfish = 0x6720; + const UInt16 kTwofish = 0x6721; + const UInt16 kRC4 = 0x6801; +} + +struct CStrongCryptoExtra +{ + UInt16 Format; + UInt16 AlgId; + UInt16 BitLen; + UInt16 Flags; + + bool ParseFromSubBlock(const CExtraSubBlock &sb) + { + if (sb.ID != NFileHeader::NExtraID::kStrongEncrypt) + return false; + const Byte *p = (const Byte *)sb.Data; + if (sb.Data.Size() < 8) + return false; + Format = GetUi16(p + 0); + AlgId = GetUi16(p + 2); + BitLen = GetUi16(p + 4); + Flags = GetUi16(p + 6); + return (Format == 2); + } + + bool CertificateIsUsed() const { return (Flags > 0x0001); } +}; + + +struct CExtraBlock +{ + CObjectVector SubBlocks; + bool Error; + bool MinorError; + bool IsZip64; + bool IsZip64_Error; + + CExtraBlock(): Error(false), MinorError(false), IsZip64(false), IsZip64_Error(false) {} + + void Clear() + { + SubBlocks.Clear(); + IsZip64 = false; + } + + size_t GetSize() const + { + size_t res = 0; + FOR_VECTOR (i, SubBlocks) + res += SubBlocks[i].Data.Size() + 2 + 2; + return res; + } + + bool GetWzAes(CWzAesExtra &e) const + { + FOR_VECTOR (i, SubBlocks) + if (e.ParseFromSubBlock(SubBlocks[i])) + return true; + return false; + } + + bool HasWzAes() const + { + CWzAesExtra e; + return GetWzAes(e); + } + + bool GetStrongCrypto(CStrongCryptoExtra &e) const + { + FOR_VECTOR (i, SubBlocks) + if (e.ParseFromSubBlock(SubBlocks[i])) + return true; + return false; + } + + /* + bool HasStrongCrypto() const + { + CStrongCryptoExtra e; + return GetStrongCrypto(e); + } + */ + + bool GetNtfsTime(unsigned index, FILETIME &ft) const; + bool GetUnixTime(bool isCentral, unsigned index, UInt32 &res) const; + + void PrintInfo(AString &s) const; + + void RemoveUnknownSubBlocks() + { + for (unsigned i = SubBlocks.Size(); i != 0;) + { + i--; + switch (SubBlocks[i].ID) + { + case NFileHeader::NExtraID::kStrongEncrypt: + case NFileHeader::NExtraID::kWzAES: + break; + default: + SubBlocks.Delete(i); + } + } + } +}; + + +class CLocalItem +{ +public: + UInt16 Flags; + UInt16 Method; + + /* + Zip specification doesn't mention that ExtractVersion field uses HostOS subfield. + 18.06: 7-Zip now doesn't use ExtractVersion::HostOS to detect codePage + */ + + CVersion ExtractVersion; + + UInt64 Size; + UInt64 PackSize; + UInt32 Time; + UInt32 Crc; + + UInt32 Disk; + + AString Name; + + CExtraBlock LocalExtra; + + unsigned GetDescriptorSize() const { return LocalExtra.IsZip64 ? kDataDescriptorSize64 : kDataDescriptorSize32; } + + UInt64 GetPackSizeWithDescriptor() const + { return PackSize + (HasDescriptor() ? GetDescriptorSize() : 0); } + + bool IsUtf8() const { return (Flags & NFileHeader::NFlags::kUtf8) != 0; } + bool IsEncrypted() const { return (Flags & NFileHeader::NFlags::kEncrypted) != 0; } + bool IsStrongEncrypted() const { return IsEncrypted() && (Flags & NFileHeader::NFlags::kStrongEncrypted) != 0; } + bool IsAesEncrypted() const { return IsEncrypted() && (IsStrongEncrypted() || Method == NFileHeader::NCompressionMethod::kWzAES); } + bool IsLzmaEOS() const { return (Flags & NFileHeader::NFlags::kLzmaEOS) != 0; } + bool HasDescriptor() const { return (Flags & NFileHeader::NFlags::kDescriptorUsedMask) != 0; } + // bool IsAltStream() const { return (Flags & NFileHeader::NFlags::kAltStream) != 0; } + + unsigned GetDeflateLevel() const { return (Flags >> 1) & 3; } + + bool IsDir() const; + + /* + void GetUnicodeString(const AString &s, UString &res) const + { + bool isUtf8 = IsUtf8(); + if (isUtf8) + if (ConvertUTF8ToUnicode(s, res)) + return; + MultiByteToUnicodeString2(res, s, GetCodePage()); + } + */ + +private: + + void SetFlag(unsigned bitMask, bool enable) + { + if (enable) + Flags = (UInt16)(Flags | bitMask); + else + Flags = (UInt16)(Flags & ~bitMask); + } + +public: + + void ClearFlags() { Flags = 0; } + void SetEncrypted(bool encrypted) { SetFlag(NFileHeader::NFlags::kEncrypted, encrypted); } + void SetUtf8(bool isUtf8) { SetFlag(NFileHeader::NFlags::kUtf8, isUtf8); } + // void SetFlag_AltStream(bool isAltStream) { SetFlag(NFileHeader::NFlags::kAltStream, isAltStream); } + void SetDescriptorMode(bool useDescriptor) { SetFlag(NFileHeader::NFlags::kDescriptorUsedMask, useDescriptor); } + + UINT GetCodePage() const + { + if (IsUtf8()) + return CP_UTF8; + return CP_OEMCP; + } +}; + + +class CItem: public CLocalItem +{ +public: + CVersion MadeByVersion; + UInt16 InternalAttrib; + UInt32 ExternalAttrib; + + UInt64 LocalHeaderPos; + + CExtraBlock CentralExtra; + CByteBuffer Comment; + + bool FromLocal; + bool FromCentral; + + // CItem can be used as CLocalItem. So we must clear unused fields + CItem(): + InternalAttrib(0), + ExternalAttrib(0), + FromLocal(false), + FromCentral(false) + { + MadeByVersion.Version = 0; + MadeByVersion.HostOS = 0; + } + + const CExtraBlock &GetMainExtra() const { return *(FromCentral ? &CentralExtra : &LocalExtra); } + + bool IsDir() const; + UInt32 GetWinAttrib() const; + bool GetPosixAttrib(UInt32 &attrib) const; + + // 18.06: 0 instead of ExtractVersion.HostOS for local item + Byte GetHostOS() const { return FromCentral ? MadeByVersion.HostOS : (Byte)0; } + + void GetUnicodeString(UString &res, const AString &s, bool isComment, bool useSpecifiedCodePage, UINT codePage) const; + + bool IsThereCrc() const + { + if (Method == NFileHeader::NCompressionMethod::kWzAES) + { + CWzAesExtra aesField; + if (GetMainExtra().GetWzAes(aesField)) + return aesField.NeedCrc(); + } + return (Crc != 0 || !IsDir()); + } + + bool Is_MadeBy_Unix() const + { + if (!FromCentral) + return false; + return (MadeByVersion.HostOS == NFileHeader::NHostOS::kUnix); + } + + UINT GetCodePage() const + { + // 18.06: now we use HostOS only from Central::MadeByVersion + if (IsUtf8()) + return CP_UTF8; + if (!FromCentral) + return CP_OEMCP; + Byte hostOS = MadeByVersion.HostOS; + return (UINT)(( + hostOS == NFileHeader::NHostOS::kFAT + || hostOS == NFileHeader::NHostOS::kNTFS + || hostOS == NFileHeader::NHostOS::kUnix // do we need it? + ) ? CP_OEMCP : CP_ACP); + } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipOut.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipOut.cpp new file mode 100644 index 0000000..63f1a71 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipOut.cpp @@ -0,0 +1,420 @@ +// ZipOut.cpp + +#include "StdAfx.h" + +#include "../../../../C/7zCrc.h" + +#include "../../../Windows/TimeUtils.h" +#include "../../Common/OffsetStream.h" + +#include "ZipOut.h" + +namespace NArchive { +namespace NZip { + +HRESULT COutArchive::ClearRestriction() +{ + if (SetRestriction) + return SetRestriction->SetRestriction(0, 0); + return S_OK; +} + +HRESULT COutArchive::SetRestrictionFromCurrent() +{ + if (SetRestriction) + return SetRestriction->SetRestriction(m_Base + m_CurPos, (UInt64)(Int64)-1); + return S_OK; +} + +HRESULT COutArchive::Create(IOutStream *outStream) +{ + m_CurPos = 0; + if (!m_OutBuffer.Create(1 << 16)) + return E_OUTOFMEMORY; + m_Stream = outStream; + m_OutBuffer.SetStream(outStream); + m_OutBuffer.Init(); + + return m_Stream->Seek(0, STREAM_SEEK_CUR, &m_Base); +} + +void COutArchive::SeekToCurPos() +{ + HRESULT res = m_Stream->Seek((Int64)(m_Base + m_CurPos), STREAM_SEEK_SET, NULL); + if (res != S_OK) + throw CSystemException(res); +} + +#define DOES_NEED_ZIP64(v) (v >= (UInt32)0xFFFFFFFF) +// #define DOES_NEED_ZIP64(v) (v >= 0) + + +void COutArchive::WriteBytes(const void *data, size_t size) +{ + m_OutBuffer.WriteBytes(data, size); + m_CurPos += size; +} + +void COutArchive::Write8(Byte b) +{ + m_OutBuffer.WriteByte(b); + m_CurPos++; +} + +void COutArchive::Write16(UInt16 val) +{ + Write8((Byte)val); + Write8((Byte)(val >> 8)); +} + +void COutArchive::Write32(UInt32 val) +{ + for (int i = 0; i < 4; i++) + { + Write8((Byte)val); + val >>= 8; + } +} + +void COutArchive::Write64(UInt64 val) +{ + for (int i = 0; i < 8; i++) + { + Write8((Byte)val); + val >>= 8; + } +} + +void COutArchive::WriteExtra(const CExtraBlock &extra) +{ + FOR_VECTOR (i, extra.SubBlocks) + { + const CExtraSubBlock &subBlock = extra.SubBlocks[i]; + Write16((UInt16)subBlock.ID); + Write16((UInt16)subBlock.Data.Size()); + WriteBytes(subBlock.Data, (UInt16)subBlock.Data.Size()); + } +} + +void COutArchive::WriteCommonItemInfo(const CLocalItem &item, bool isZip64) +{ + { + Byte ver = item.ExtractVersion.Version; + if (isZip64 && ver < NFileHeader::NCompressionMethod::kExtractVersion_Zip64) + ver = NFileHeader::NCompressionMethod::kExtractVersion_Zip64; + Write8(ver); + } + Write8(item.ExtractVersion.HostOS); + Write16(item.Flags); + Write16(item.Method); + Write32(item.Time); +} + + +#define WRITE_32_VAL_SPEC(_v_, _isZip64_) Write32((_isZip64_) ? 0xFFFFFFFF : (UInt32)(_v_)); + + +void COutArchive::WriteUtfName(const CItemOut &item) +{ + if (item.Name_Utf.Size() == 0) + return; + Write16(NFileHeader::NExtraID::kIzUnicodeName); + Write16((UInt16)(5 + item.Name_Utf.Size())); + Write8(1); // (1 = version) of that extra field + Write32(CrcCalc(item.Name.Ptr(), item.Name.Len())); + WriteBytes(item.Name_Utf, (UInt16)item.Name_Utf.Size()); +} + + +static const unsigned k_Ntfs_ExtraSize = 4 + 2 + 2 + (3 * 8); +static const unsigned k_UnixTime_ExtraSize = 1 + (1 * 4); + +void COutArchive::WriteTimeExtra(const CItemOut &item, bool writeNtfs) +{ + if (writeNtfs) + { + // windows explorer ignores that extra + Write16(NFileHeader::NExtraID::kNTFS); + Write16(k_Ntfs_ExtraSize); + Write32(0); // reserved + Write16(NFileHeader::NNtfsExtra::kTagTime); + Write16(8 * 3); + WriteNtfsTime(item.Ntfs_MTime); + WriteNtfsTime(item.Ntfs_ATime); + WriteNtfsTime(item.Ntfs_CTime); + } + + if (item.Write_UnixTime) + { + // windows explorer ignores that extra + // by specification : should we write to local header also? + Write16(NFileHeader::NExtraID::kUnixTime); + Write16(k_UnixTime_ExtraSize); + const Byte flags = (Byte)((unsigned)1 << NFileHeader::NUnixTime::kMTime); + Write8(flags); + UInt32 unixTime; + NWindows::NTime::FileTime_To_UnixTime(item.Ntfs_MTime, unixTime); + Write32(unixTime); + } +} + + +void COutArchive::WriteLocalHeader(CItemOut &item, bool needCheck) +{ + m_LocalHeaderPos = m_CurPos; + item.LocalHeaderPos = m_CurPos; + + bool isZip64 = + DOES_NEED_ZIP64(item.PackSize) || + DOES_NEED_ZIP64(item.Size); + + if (needCheck && m_IsZip64) + isZip64 = true; + + // Why don't we write NTFS timestamps to local header? + // Probably we want to reduce size of archive? + const bool writeNtfs = false; // do not write NTFS timestamp to local header + // const bool writeNtfs = item.Write_NtfsTime; // write NTFS time to local header + const UInt32 localExtraSize = (UInt32)( + (isZip64 ? (4 + 8 + 8): 0) + + (writeNtfs ? 4 + k_Ntfs_ExtraSize : 0) + + (item.Write_UnixTime ? 4 + k_UnixTime_ExtraSize : 0) + + item.Get_UtfName_ExtraSize() + + item.LocalExtra.GetSize()); + if ((UInt16)localExtraSize != localExtraSize) + throw CSystemException(E_FAIL); + if (needCheck && m_ExtraSize != localExtraSize) + throw CSystemException(E_FAIL); + + m_IsZip64 = isZip64; + m_ExtraSize = localExtraSize; + + item.LocalExtra.IsZip64 = isZip64; + + Write32(NSignature::kLocalFileHeader); + + WriteCommonItemInfo(item, isZip64); + + Write32(item.HasDescriptor() ? 0 : item.Crc); + + UInt64 packSize = item.PackSize; + UInt64 size = item.Size; + + if (item.HasDescriptor()) + { + packSize = 0; + size = 0; + } + + WRITE_32_VAL_SPEC(packSize, isZip64) + WRITE_32_VAL_SPEC(size, isZip64) + + Write16((UInt16)item.Name.Len()); + + Write16((UInt16)localExtraSize); + + WriteBytes((const char *)item.Name, (UInt16)item.Name.Len()); + + if (isZip64) + { + Write16(NFileHeader::NExtraID::kZip64); + Write16(8 + 8); + Write64(size); + Write64(packSize); + } + + WriteTimeExtra(item, writeNtfs); + + WriteUtfName(item); + + WriteExtra(item.LocalExtra); + + const UInt32 localFileHeaderSize = (UInt32)(m_CurPos - m_LocalHeaderPos); + if (needCheck && m_LocalFileHeaderSize != localFileHeaderSize) + throw CSystemException(E_FAIL); + m_LocalFileHeaderSize = localFileHeaderSize; + + m_OutBuffer.FlushWithCheck(); +} + + +void COutArchive::WriteLocalHeader_Replace(CItemOut &item) +{ + m_CurPos = m_LocalHeaderPos + m_LocalFileHeaderSize + item.PackSize; + + if (item.HasDescriptor()) + { + WriteDescriptor(item); + m_OutBuffer.FlushWithCheck(); + return; + // we don't replace local header, if we write Descriptor. + // so local header with Descriptor flag must be written to local header before. + } + + const UInt64 nextPos = m_CurPos; + m_CurPos = m_LocalHeaderPos; + SeekToCurPos(); + WriteLocalHeader(item, true); + m_CurPos = nextPos; + SeekToCurPos(); +} + + +void COutArchive::WriteDescriptor(const CItemOut &item) +{ + Byte buf[kDataDescriptorSize64]; + SetUi32(buf, NSignature::kDataDescriptor) + SetUi32(buf + 4, item.Crc) + unsigned descriptorSize; + if (m_IsZip64) + { + SetUi64(buf + 8, item.PackSize) + SetUi64(buf + 16, item.Size) + descriptorSize = kDataDescriptorSize64; + } + else + { + SetUi32(buf + 8, (UInt32)item.PackSize) + SetUi32(buf + 12, (UInt32)item.Size) + descriptorSize = kDataDescriptorSize32; + } + WriteBytes(buf, descriptorSize); +} + + + +void COutArchive::WriteCentralHeader(const CItemOut &item) +{ + const bool isUnPack64 = DOES_NEED_ZIP64(item.Size); + const bool isPack64 = DOES_NEED_ZIP64(item.PackSize); + const bool isPosition64 = DOES_NEED_ZIP64(item.LocalHeaderPos); + const bool isZip64 = isPack64 || isUnPack64 || isPosition64; + + Write32(NSignature::kCentralFileHeader); + Write8(item.MadeByVersion.Version); + Write8(item.MadeByVersion.HostOS); + + WriteCommonItemInfo(item, isZip64); + Write32(item.Crc); + + WRITE_32_VAL_SPEC(item.PackSize, isPack64) + WRITE_32_VAL_SPEC(item.Size, isUnPack64) + + Write16((UInt16)item.Name.Len()); + + const UInt16 zip64ExtraSize = (UInt16)((isUnPack64 ? 8: 0) + (isPack64 ? 8: 0) + (isPosition64 ? 8: 0)); + const bool writeNtfs = item.Write_NtfsTime; + const size_t centralExtraSize = + (isZip64 ? 4 + zip64ExtraSize : 0) + + (writeNtfs ? 4 + k_Ntfs_ExtraSize : 0) + + (item.Write_UnixTime ? 4 + k_UnixTime_ExtraSize : 0) + + item.Get_UtfName_ExtraSize() + + item.CentralExtra.GetSize(); + + const UInt16 centralExtraSize16 = (UInt16)centralExtraSize; + if (centralExtraSize16 != centralExtraSize) + throw CSystemException(E_FAIL); + + Write16(centralExtraSize16); + + const UInt16 commentSize = (UInt16)item.Comment.Size(); + + Write16(commentSize); + Write16(0); // DiskNumberStart + Write16(item.InternalAttrib); + Write32(item.ExternalAttrib); + WRITE_32_VAL_SPEC(item.LocalHeaderPos, isPosition64) + WriteBytes((const char *)item.Name, item.Name.Len()); + + if (isZip64) + { + Write16(NFileHeader::NExtraID::kZip64); + Write16(zip64ExtraSize); + if (isUnPack64) + Write64(item.Size); + if (isPack64) + Write64(item.PackSize); + if (isPosition64) + Write64(item.LocalHeaderPos); + } + + WriteTimeExtra(item, writeNtfs); + WriteUtfName(item); + + WriteExtra(item.CentralExtra); + if (commentSize != 0) + WriteBytes(item.Comment, commentSize); +} + +HRESULT COutArchive::WriteCentralDir(const CObjectVector &items, const CByteBuffer *comment) +{ + RINOK(ClearRestriction()) + + const UInt64 cdOffset = GetCurPos(); + FOR_VECTOR (i, items) + WriteCentralHeader(items[i]); + const UInt64 cd64EndOffset = GetCurPos(); + const UInt64 cdSize = cd64EndOffset - cdOffset; + const bool cdOffset64 = DOES_NEED_ZIP64(cdOffset); + const bool cdSize64 = DOES_NEED_ZIP64(cdSize); + const bool items64 = items.Size() >= 0xFFFF; + const bool isZip64 = (cdOffset64 || cdSize64 || items64); + + // isZip64 = true; // to test Zip64 + + if (isZip64) + { + Write32(NSignature::kEcd64); + Write64(kEcd64_MainSize); + + // to test extra block: + // const UInt32 extraSize = 1 << 26; + // Write64(kEcd64_MainSize + extraSize); + + Write16(45); // made by version + Write16(45); // extract version + Write32(0); // ThisDiskNumber + Write32(0); // StartCentralDirectoryDiskNumber + Write64((UInt64)items.Size()); + Write64((UInt64)items.Size()); + Write64((UInt64)cdSize); + Write64((UInt64)cdOffset); + + // for (UInt32 iii = 0; iii < extraSize; iii++) Write8(1); + + Write32(NSignature::kEcd64Locator); + Write32(0); // number of the disk with the start of the zip64 end of central directory + Write64(cd64EndOffset); + Write32(1); // total number of disks + } + + Write32(NSignature::kEcd); + Write16(0); // ThisDiskNumber + Write16(0); // StartCentralDirectoryDiskNumber + Write16((UInt16)(items64 ? 0xFFFF: items.Size())); + Write16((UInt16)(items64 ? 0xFFFF: items.Size())); + + WRITE_32_VAL_SPEC(cdSize, cdSize64) + WRITE_32_VAL_SPEC(cdOffset, cdOffset64) + + const UInt16 commentSize = (UInt16)(comment ? comment->Size() : 0); + Write16((UInt16)commentSize); + if (commentSize != 0) + WriteBytes((const Byte *)*comment, commentSize); + m_OutBuffer.FlushWithCheck(); + return S_OK; +} + +void COutArchive::CreateStreamForCompressing(CMyComPtr &outStream) +{ + COffsetOutStream *streamSpec = new COffsetOutStream; + outStream = streamSpec; + streamSpec->Init(m_Stream, m_Base + m_CurPos); +} + +void COutArchive::CreateStreamForCopying(CMyComPtr &outStream) +{ + outStream = m_Stream; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipOut.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipOut.h new file mode 100644 index 0000000..e9b2abb --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipOut.h @@ -0,0 +1,103 @@ +// ZipOut.h + +#ifndef ZIP7_INC_ZIP_OUT_H +#define ZIP7_INC_ZIP_OUT_H + +#include "../../../Common/MyCom.h" + +#include "../../Common/OutBuffer.h" + +#include "ZipItem.h" + +namespace NArchive { +namespace NZip { + +class CItemOut: public CItem +{ +public: + FILETIME Ntfs_MTime; + FILETIME Ntfs_ATime; + FILETIME Ntfs_CTime; + bool Write_NtfsTime; + bool Write_UnixTime; + + // It's possible that NtfsTime is not defined, but there is NtfsTime in Extra. + + CByteBuffer Name_Utf; // for Info-Zip (kIzUnicodeName) Extra + + size_t Get_UtfName_ExtraSize() const + { + const size_t size = Name_Utf.Size(); + if (size == 0) + return 0; + return 4 + 5 + size; + } + + CItemOut(): + Write_NtfsTime(false), + Write_UnixTime(false) + {} +}; + + +// COutArchive can throw CSystemException and COutBufferException + +class COutArchive +{ + COutBuffer m_OutBuffer; + CMyComPtr m_Stream; + + UInt64 m_Base; // Base of archive (offset in output Stream) + UInt64 m_CurPos; // Curent position in archive (relative from m_Base) + UInt64 m_LocalHeaderPos; // LocalHeaderPos (relative from m_Base) for last WriteLocalHeader() call + + UInt32 m_LocalFileHeaderSize; + UInt32 m_ExtraSize; + bool m_IsZip64; + + void WriteBytes(const void *data, size_t size); + void Write8(Byte b); + void Write16(UInt16 val); + void Write32(UInt32 val); + void Write64(UInt64 val); + void WriteNtfsTime(const FILETIME &ft) + { + Write32(ft.dwLowDateTime); + Write32(ft.dwHighDateTime); + } + + void WriteTimeExtra(const CItemOut &item, bool writeNtfs); + void WriteUtfName(const CItemOut &item); + void WriteExtra(const CExtraBlock &extra); + void WriteCommonItemInfo(const CLocalItem &item, bool isZip64); + void WriteCentralHeader(const CItemOut &item); + + void SeekToCurPos(); +public: + CMyComPtr SetRestriction; + + HRESULT ClearRestriction(); + HRESULT SetRestrictionFromCurrent(); + HRESULT Create(IOutStream *outStream); + + UInt64 GetCurPos() const { return m_CurPos; } + + void MoveCurPos(UInt64 distanceToMove) + { + m_CurPos += distanceToMove; + } + + void WriteLocalHeader(CItemOut &item, bool needCheck = false); + void WriteLocalHeader_Replace(CItemOut &item); + + void WriteDescriptor(const CItemOut &item); + + HRESULT WriteCentralDir(const CObjectVector &items, const CByteBuffer *comment); + + void CreateStreamForCompressing(CMyComPtr &outStream); + void CreateStreamForCopying(CMyComPtr &outStream); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipRegister.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipRegister.cpp new file mode 100644 index 0000000..c17a1a3 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipRegister.cpp @@ -0,0 +1,38 @@ +// ZipRegister.cpp + +#include "StdAfx.h" + +#include "../../Common/RegisterArc.h" + +#include "ZipHandler.h" + +namespace NArchive { +namespace NZip { + +static const Byte k_Signature[] = { + 4, 0x50, 0x4B, 0x03, 0x04, // Local + 4, 0x50, 0x4B, 0x05, 0x06, // Ecd + 4, 0x50, 0x4B, 0x06, 0x06, // Ecd64 + 6, 0x50, 0x4B, 0x07, 0x08, 0x50, 0x4B, // Span / Descriptor + 6, 0x50, 0x4B, 0x30, 0x30, 0x50, 0x4B }; // NoSpan + +REGISTER_ARC_IO( + "zip", "zip z01 zipx jar xpi odt ods docx xlsx epub ipa apk appx", NULL, 1, + k_Signature, + 0, + NArcInfoFlags::kFindSignature + | NArcInfoFlags::kMultiSignature + | NArcInfoFlags::kUseGlobalOffset + | NArcInfoFlags::kCTime + // | NArcInfoFlags::kCTime_Default + | NArcInfoFlags::kATime + // | NArcInfoFlags::kATime_Default + | NArcInfoFlags::kMTime + | NArcInfoFlags::kMTime_Default + , TIME_PREC_TO_ARC_FLAGS_MASK (NFileTimeType::kWindows) + | TIME_PREC_TO_ARC_FLAGS_MASK (NFileTimeType::kUnix) + | TIME_PREC_TO_ARC_FLAGS_MASK (NFileTimeType::kDOS) + | TIME_PREC_TO_ARC_FLAGS_TIME_DEFAULT (NFileTimeType::kWindows) + , IsArc_Zip) + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipUpdate.cpp new file mode 100644 index 0000000..b2684dc --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -0,0 +1,2165 @@ +// ZipUpdate.cpp + +#include "StdAfx.h" + +// #define DEBUG_CACHE + +#ifdef DEBUG_CACHE +#include + #define PRF(x) x +#else + #define PRF(x) +#endif + +#include "../../../../C/Alloc.h" + +#include "../../../Common/AutoPtr.h" +#include "../../../Common/Defs.h" +#include "../../../Common/StringConvert.h" + +#include "../../../Windows/TimeUtils.h" +#include "../../../Windows/Thread.h" + +#include "../../Common/CreateCoder.h" +#include "../../Common/LimitedStreams.h" +#include "../../Common/OutMemStream.h" +#include "../../Common/ProgressUtils.h" +#ifndef Z7_ST +#include "../../Common/ProgressMt.h" +#endif +#include "../../Common/StreamUtils.h" + +#include "../../Compress/CopyCoder.h" +// #include "../../Compress/ZstdEncoderProps.h" + +#include "ZipAddCommon.h" +#include "ZipOut.h" +#include "ZipUpdate.h" + +using namespace NWindows; +using namespace NSynchronization; + +namespace NArchive { +namespace NZip { + +static const Byte kHostOS = + #ifdef _WIN32 + NFileHeader::NHostOS::kFAT; + #else + NFileHeader::NHostOS::kUnix; + #endif + +static const Byte kMadeByHostOS = kHostOS; + +// 18.06: now we always write zero to high byte of ExtractVersion field. +// Previous versions of p7zip wrote (NFileHeader::NHostOS::kUnix) there, that is not correct +static const Byte kExtractHostOS = 0; + +static const Byte kMethodForDirectory = NFileHeader::NCompressionMethod::kStore; + + +static void AddAesExtra(CItem &item, Byte aesKeyMode, UInt16 method) +{ + CWzAesExtra wzAesField; + wzAesField.Strength = aesKeyMode; + wzAesField.Method = method; + item.Method = NFileHeader::NCompressionMethod::kWzAES; + item.Crc = 0; + CExtraSubBlock sb; + wzAesField.SetSubBlock(sb); + item.LocalExtra.SubBlocks.Add(sb); + item.CentralExtra.SubBlocks.Add(sb); +} + + +static void Copy_From_UpdateItem_To_ItemOut(const CUpdateItem &ui, CItemOut &item) +{ + item.Name = ui.Name; + item.Name_Utf = ui.Name_Utf; + item.Comment = ui.Comment; + item.SetUtf8(ui.IsUtf8); + // item.SetFlag_AltStream(ui.IsAltStream); + // item.ExternalAttrib = ui.Attrib; + item.Time = ui.Time; + item.Ntfs_MTime = ui.Ntfs_MTime; + item.Ntfs_ATime = ui.Ntfs_ATime; + item.Ntfs_CTime = ui.Ntfs_CTime; + + item.Write_UnixTime = ui.Write_UnixTime; + item.Write_NtfsTime = ui.Write_NtfsTime; +} + +static void SetFileHeader( + const CCompressionMethodMode &options, + const CUpdateItem &ui, + bool useDescriptor, + CItemOut &item) +{ + item.Size = ui.Size; + const bool isDir = ui.IsDir; + + item.ClearFlags(); + + if (ui.NewProps) + { + Copy_From_UpdateItem_To_ItemOut(ui, item); + // item.SetFlag_AltStream(ui.IsAltStream); + item.ExternalAttrib = ui.Attrib; + } + /* + else + isDir = item.IsDir(); + */ + + item.MadeByVersion.HostOS = kMadeByHostOS; + item.MadeByVersion.Version = NFileHeader::NCompressionMethod::kMadeByProgramVersion; + + item.ExtractVersion.HostOS = kExtractHostOS; + + item.InternalAttrib = 0; // test it + item.SetEncrypted(!isDir && options.Password_Defined); + item.SetDescriptorMode(useDescriptor); + + if (isDir) + { + item.ExtractVersion.Version = NFileHeader::NCompressionMethod::kExtractVersion_Dir; + item.Method = kMethodForDirectory; + item.PackSize = 0; + item.Size = 0; + item.Crc = 0; + } + + item.LocalExtra.Clear(); + item.CentralExtra.Clear(); + + if (isDir) + { + item.ExtractVersion.Version = NFileHeader::NCompressionMethod::kExtractVersion_Dir; + item.Method = kMethodForDirectory; + item.PackSize = 0; + item.Size = 0; + item.Crc = 0; + } + else if (options.IsRealAesMode()) + AddAesExtra(item, options.AesKeyMode, (Byte)(options.MethodSequence.IsEmpty() ? 8 : options.MethodSequence[0])); +} + + +// we call SetItemInfoFromCompressingResult() after SetFileHeader() + +static void SetItemInfoFromCompressingResult(const CCompressingResult &compressingResult, + bool isAesMode, Byte aesKeyMode, CItem &item) +{ + item.ExtractVersion.Version = compressingResult.ExtractVersion; + item.Method = compressingResult.Method; + if (compressingResult.Method == NFileHeader::NCompressionMethod::kLZMA && compressingResult.LzmaEos) + item.Flags |= NFileHeader::NFlags::kLzmaEOS; + item.Crc = compressingResult.CRC; + item.Size = compressingResult.UnpackSize; + item.PackSize = compressingResult.PackSize; + + item.LocalExtra.Clear(); + item.CentralExtra.Clear(); + + if (isAesMode) + AddAesExtra(item, aesKeyMode, compressingResult.Method); +} + + +#ifndef Z7_ST + +struct CMtSem +{ + NWindows::NSynchronization::CSemaphore Semaphore; + NWindows::NSynchronization::CCriticalSection CS; + CIntVector Indexes; + int Head; + + void ReleaseItem(unsigned index) + { + { + CCriticalSectionLock lock(CS); + Indexes[index] = Head; + Head = (int)index; + } + Semaphore.Release(); + } + + int GetFreeItem() + { + int i; + { + CCriticalSectionLock lock(CS); + i = Head; + Head = Indexes[(unsigned)i]; + } + return i; + } +}; + +static THREAD_FUNC_DECL CoderThread(void *threadCoderInfo); + +struct CThreadInfo +{ + DECL_EXTERNAL_CODECS_LOC_VARS_DECL + + NWindows::CThread Thread; + NWindows::NSynchronization::CAutoResetEvent CompressEvent; + CMtSem *MtSem; + unsigned ThreadIndex; + + bool ExitThread; + + CMtCompressProgress *ProgressSpec; + CMyComPtr Progress; + + COutMemStream *OutStreamSpec; + CMyComPtr OutStream; + CMyComPtr InStream; + + CAddCommon Coder; + HRESULT Result; + CCompressingResult CompressingResult; + + bool IsFree; + bool InSeqMode; + bool OutSeqMode; + bool ExpectedDataSize_IsConfirmed; + + UInt32 UpdateIndex; + UInt32 FileTime; + UInt64 ExpectedDataSize; + + CThreadInfo(): + MtSem(NULL), + ExitThread(false), + ProgressSpec(NULL), + OutStreamSpec(NULL), + IsFree(true), + InSeqMode(false), + OutSeqMode(false), + ExpectedDataSize_IsConfirmed(false), + FileTime(0), + ExpectedDataSize((UInt64)(Int64)-1) + {} + + void SetOptions(const CCompressionMethodMode &options) + { + Coder.SetOptions(options); + } + + HRESULT CreateEvents() + { + const WRes wres = CompressEvent.CreateIfNotCreated_Reset(); + return HRESULT_FROM_WIN32(wres); + } + + // (group < 0) means no_group. + HRESULT CreateThread_with_group( +#ifdef _WIN32 + int group +#endif + ) + { + // tested in win10: If thread is created by another thread, + // child thread probably uses same group as parent thread. + // So we don't need to send (group) to encoder in created thread. + const WRes wres = +#ifdef _WIN32 + group >= 0 ? + Thread.Create_With_Group(CoderThread, this, (unsigned)group) : +#endif + Thread.Create(CoderThread, this); + return HRESULT_FROM_WIN32(wres); + } + + void WaitAndCode(); + + void StopWait_Close() + { + ExitThread = true; + if (OutStreamSpec) + OutStreamSpec->StopWriting(E_ABORT); + if (CompressEvent.IsCreated()) + CompressEvent.Set(); + Thread.Wait_Close(); + } +}; + +void CThreadInfo::WaitAndCode() +{ + for (;;) + { + CompressEvent.Lock(); + if (ExitThread) + return; + + Result = Coder.Compress( + EXTERNAL_CODECS_LOC_VARS + InStream, OutStream, + InSeqMode, OutSeqMode, FileTime, ExpectedDataSize, + ExpectedDataSize_IsConfirmed, + Progress, CompressingResult); + + if (Result == S_OK && Progress) + Result = Progress->SetRatioInfo(&CompressingResult.UnpackSize, &CompressingResult.PackSize); + + MtSem->ReleaseItem(ThreadIndex); + } +} + +static THREAD_FUNC_DECL CoderThread(void *threadCoderInfo) +{ + ((CThreadInfo *)threadCoderInfo)->WaitAndCode(); + return 0; +} + +class CThreads +{ +public: + CObjectVector Threads; + ~CThreads() + { + FOR_VECTOR (i, Threads) + Threads[i].StopWait_Close(); + } +}; + +struct CMemBlocks2: public CMemLockBlocks +{ + bool Skip; + bool InSeqMode; + bool PreDescriptorMode; + bool Finished; + CCompressingResult CompressingResult; + + CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false), + CompressingResult() {} +}; + +class CMemRefs +{ +public: + CMemBlockManagerMt *Manager; + CObjectVector Refs; + CMemRefs(CMemBlockManagerMt *manager): Manager(manager) {} + ~CMemRefs() + { + FOR_VECTOR (i, Refs) + Refs[i].FreeOpt(Manager); + } +}; + + +Z7_CLASS_IMP_NOQIB_1( + CMtProgressMixer2 + , ICompressProgressInfo +) + UInt64 ProgressOffset; + UInt64 InSizes[2]; + UInt64 OutSizes[2]; + CMyComPtr Progress; + CMyComPtr RatioProgress; + bool _inSizeIsMain; +public: + NWindows::NSynchronization::CCriticalSection CriticalSection; + void Create(IProgress *progress, bool inSizeIsMain); + void SetProgressOffset(UInt64 progressOffset); + void SetProgressOffset_NoLock(UInt64 progressOffset); + HRESULT SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize); +}; + +void CMtProgressMixer2::Create(IProgress *progress, bool inSizeIsMain) +{ + Progress = progress; + Progress.QueryInterface(IID_ICompressProgressInfo, &RatioProgress); + _inSizeIsMain = inSizeIsMain; + ProgressOffset = InSizes[0] = InSizes[1] = OutSizes[0] = OutSizes[1] = 0; +} + +void CMtProgressMixer2::SetProgressOffset_NoLock(UInt64 progressOffset) +{ + InSizes[1] = OutSizes[1] = 0; + ProgressOffset = progressOffset; +} + +void CMtProgressMixer2::SetProgressOffset(UInt64 progressOffset) +{ + CriticalSection.Enter(); + SetProgressOffset_NoLock(progressOffset); + CriticalSection.Leave(); +} + +HRESULT CMtProgressMixer2::SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize) +{ + NWindows::NSynchronization::CCriticalSectionLock lock(CriticalSection); + if (index == 0 && RatioProgress) + { + RINOK(RatioProgress->SetRatioInfo(inSize, outSize)) + } + if (inSize) + InSizes[index] = *inSize; + if (outSize) + OutSizes[index] = *outSize; + UInt64 v = ProgressOffset + (_inSizeIsMain ? + (InSizes[0] + InSizes[1]) : + (OutSizes[0] + OutSizes[1])); + return Progress->SetCompleted(&v); +} + +Z7_COM7F_IMF(CMtProgressMixer2::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize)) +{ + return SetRatioInfo(0, inSize, outSize); +} + + +Z7_CLASS_IMP_NOQIB_1( + CMtProgressMixer + , ICompressProgressInfo +) +public: + CMtProgressMixer2 *Mixer2; + CMyComPtr RatioProgress; + void Create(IProgress *progress, bool inSizeIsMain); +}; + +void CMtProgressMixer::Create(IProgress *progress, bool inSizeIsMain) +{ + Mixer2 = new CMtProgressMixer2; + RatioProgress = Mixer2; + Mixer2->Create(progress, inSizeIsMain); +} + +Z7_COM7F_IMF(CMtProgressMixer::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize)) +{ + return Mixer2->SetRatioInfo(1, inSize, outSize); +} + + +#endif + +static HRESULT UpdateItemOldData( + COutArchive &archive, + CInArchive *inArchive, + const CItemEx &itemEx, + const CUpdateItem &ui, + CItemOut &item, + /* bool izZip64, */ + ICompressProgressInfo *progress, + IArchiveUpdateCallbackFile *opCallback, + UInt64 &complexity) +{ + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kInArcIndex, (UInt32)ui.IndexInArc, + NUpdateNotifyOp::kReplicate)) + } + + UInt64 rangeSize; + + RINOK(archive.ClearRestriction()) + + if (ui.NewProps) + { + if (item.HasDescriptor()) + { + // we know compressed / uncompressed sizes and crc. + // so we remove descriptor here + item.Flags = (UInt16)(item.Flags & ~NFileHeader::NFlags::kDescriptorUsedMask); + // return E_NOTIMPL; + } + // we keep ExternalAttrib and some another properties from old archive + // item.ExternalAttrib = ui.Attrib; + // if we don't change Comment, we keep Comment from OldProperties + Copy_From_UpdateItem_To_ItemOut(ui, item); + // item.SetFlag_AltStream(ui.IsAltStream); + + item.CentralExtra.RemoveUnknownSubBlocks(); + item.LocalExtra.RemoveUnknownSubBlocks(); + + archive.WriteLocalHeader(item); + rangeSize = item.GetPackSizeWithDescriptor(); + } + else + { + item.LocalHeaderPos = archive.GetCurPos(); + rangeSize = itemEx.GetLocalFullSize(); + } + + CMyComPtr packStream; + + RINOK(inArchive->GetItemStream(itemEx, ui.NewProps, packStream)) + if (!packStream) + return E_NOTIMPL; + + complexity += rangeSize; + + CMyComPtr outStream; + archive.CreateStreamForCopying(outStream); + HRESULT res = NCompress::CopyStream_ExactSize(packStream, outStream, rangeSize, progress); + archive.MoveCurPos(rangeSize); + return res; +} + + +static HRESULT WriteDirHeader(COutArchive &archive, const CCompressionMethodMode *options, + const CUpdateItem &ui, CItemOut &item) +{ + SetFileHeader(*options, ui, false, item); + RINOK(archive.ClearRestriction()) + archive.WriteLocalHeader(item); + return S_OK; +} + + +static void UpdatePropsFromStream( + const CUpdateOptions &options, + CUpdateItem &item, ISequentialInStream *fileInStream, + IArchiveUpdateCallback *updateCallback, UInt64 &totalComplexity) +{ + CMyComPtr getProps; + fileInStream->QueryInterface(IID_IStreamGetProps, (void **)&getProps); + UInt64 size = (UInt64)(Int64)-1; + bool size_WasSet = false; + + if (getProps) + { + FILETIME cTime, aTime, mTime; + UInt32 attrib; + if (getProps->GetProps(&size, &cTime, &aTime, &mTime, &attrib) == S_OK) + { + if (options.Write_MTime) + if (!FILETIME_IsZero(mTime)) + { + item.Ntfs_MTime = mTime; + NTime::UtcFileTime_To_LocalDosTime(mTime, item.Time); + } + + if (options.Write_CTime) if (!FILETIME_IsZero(cTime)) item.Ntfs_CTime = cTime; + if (options.Write_ATime) if (!FILETIME_IsZero(aTime)) item.Ntfs_ATime = aTime; + + item.Attrib = attrib; + size_WasSet = true; + } + } + + if (!size_WasSet) + { + CMyComPtr streamGetSize; + fileInStream->QueryInterface(IID_IStreamGetSize, (void **)&streamGetSize); + if (streamGetSize) + { + if (streamGetSize->GetSize(&size) == S_OK) + size_WasSet = true; + } + } + + if (size_WasSet && size != (UInt64)(Int64)-1) + { + item.Size_WasSetFromStream = true; + if (size != item.Size) + { + const Int64 newComplexity = (Int64)totalComplexity + ((Int64)size - (Int64)item.Size); + if (newComplexity > 0) + { + totalComplexity = (UInt64)newComplexity; + updateCallback->SetTotal(totalComplexity); + } + item.Size = size; + } + } +} + + +/* +static HRESULT ReportProps( + IArchiveUpdateCallbackArcProp *reportArcProp, + UInt32 index, + const CItemOut &item, + bool isAesMode) +{ + PROPVARIANT prop; + prop.vt = VT_EMPTY; + prop.wReserved1 = 0; + + NCOM::PropVarEm_Set_UInt64(&prop, item.Size); + RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidSize, &prop)); + + NCOM::PropVarEm_Set_UInt64(&prop, item.PackSize); + RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidPackSize, &prop)); + + if (!isAesMode) + { + NCOM::PropVarEm_Set_UInt32(&prop, item.Crc); + RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidCRC, &prop)); + } + + RINOK(reportArcProp->ReportFinished(NEventIndexType::kOutArcIndex, index, NUpdate::NOperationResult::kOK)); + + // if (opCallback) RINOK(opCallback->ReportOperation(NEventIndexType::kOutArcIndex, index, NUpdateNotifyOp::kOpFinished)) + + return S_OK; +} +*/ + +/* +struct CTotalStats +{ + UInt64 Size; + UInt64 PackSize; + + void UpdateWithItem(const CItemOut &item) + { + Size += item.Size; + PackSize += item.PackSize; + } +}; + +static HRESULT ReportArcProps(IArchiveUpdateCallbackArcProp *reportArcProp, + CTotalStats &st) +{ + PROPVARIANT prop; + prop.vt = VT_EMPTY; + prop.wReserved1 = 0; + { + NWindows::NCOM::PropVarEm_Set_UInt64(&prop, st.Size); + RINOK(reportArcProp->ReportProp( + NEventIndexType::kArcProp, 0, kpidSize, &prop)); + } + { + NWindows::NCOM::PropVarEm_Set_UInt64(&prop, st.PackSize); + RINOK(reportArcProp->ReportProp( + NEventIndexType::kArcProp, 0, kpidPackSize, &prop)); + } + return S_OK; +} +*/ + + +static HRESULT Update2St( + DECL_EXTERNAL_CODECS_LOC_VARS + COutArchive &archive, + CInArchive *inArchive, + const CObjectVector &inputItems, + CObjectVector &updateItems, + const CUpdateOptions &updateOptions, + const CCompressionMethodMode *options, bool outSeqMode, + const CByteBuffer *comment, + IArchiveUpdateCallback *updateCallback, + UInt64 &totalComplexity, + IArchiveUpdateCallbackFile *opCallback + // , IArchiveUpdateCallbackArcProp *reportArcProp + ) +{ + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(updateCallback, true); + + CAddCommon compressor; + compressor.SetOptions(*options); + + CObjectVector items; + UInt64 unpackSizeTotal = 0, packSizeTotal = 0; + + FOR_VECTOR (itemIndex, updateItems) + { + lps->InSize = unpackSizeTotal; + lps->OutSize = packSizeTotal; + RINOK(lps->SetCur()) + CUpdateItem &ui = updateItems[itemIndex]; + CItemEx itemEx; + CItemOut item; + + if (!ui.NewProps || !ui.NewData) + { + // Note: for (ui.NewProps && !ui.NewData) it copies Props from old archive, + // But we will rewrite all important properties later. But we can keep some properties like Comment + itemEx = inputItems[(unsigned)ui.IndexInArc]; + if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK) + return E_NOTIMPL; + (CItem &)item = itemEx; + } + + if (ui.NewData) + { + // bool isDir = ((ui.NewProps) ? ui.IsDir : item.IsDir()); + bool isDir = ui.IsDir; + if (isDir) + { + RINOK(WriteDirHeader(archive, options, ui, item)) + } + else + { + CMyComPtr fileInStream; + { + HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream); + if (res == S_FALSE) + { + lps->ProgressOffset += ui.Size; + RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)) + continue; + } + RINOK(res) + if (!fileInStream) + return E_INVALIDARG; + + bool inSeqMode = false; + if (!inSeqMode) + { + CMyComPtr inStream2; + fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2); + inSeqMode = (inStream2 == NULL); + } + // seqMode = true; // to test seqMode + + UpdatePropsFromStream(updateOptions, ui, fileInStream, updateCallback, totalComplexity); + + CCompressingResult compressingResult; + + RINOK(compressor.Set_Pre_CompressionResult( + inSeqMode, outSeqMode, + ui.Size, + compressingResult)) + + SetFileHeader(*options, ui, compressingResult.DescriptorMode, item); + + // file Size can be 64-bit !!! + + SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item); + + RINOK(archive.SetRestrictionFromCurrent()) + archive.WriteLocalHeader(item); + + CMyComPtr outStream; + archive.CreateStreamForCompressing(outStream); + + RINOK(compressor.Compress( + EXTERNAL_CODECS_LOC_VARS + fileInStream, outStream, + inSeqMode, outSeqMode, + ui.Time, + ui.Size, ui.Size_WasSetFromStream, + progress, compressingResult)) + + if (item.HasDescriptor() != compressingResult.DescriptorMode) + return E_FAIL; + + SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item); + + archive.WriteLocalHeader_Replace(item); + } + // if (reportArcProp) RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options->IsRealAesMode())) + RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)) + unpackSizeTotal += item.Size; + packSizeTotal += item.PackSize; + } + } + else + { + UInt64 complexity = 0; + lps->SendRatio = false; + + RINOK(UpdateItemOldData(archive, inArchive, itemEx, ui, item, progress, opCallback, complexity)) + + lps->SendRatio = true; + lps->ProgressOffset += complexity; + } + + items.Add(item); + lps->ProgressOffset += kLocalHeaderSize; + } + + lps->InSize = unpackSizeTotal; + lps->OutSize = packSizeTotal; + RINOK(lps->SetCur()) + + RINOK(archive.WriteCentralDir(items, comment)) + + /* + CTotalStats stat; + stat.Size = unpackSizeTotal; + stat.PackSize = packSizeTotal; + if (reportArcProp) + RINOK(ReportArcProps(reportArcProp, stat)) + */ + + lps->ProgressOffset += kCentralHeaderSize * updateItems.Size() + 1; + return lps->SetCur(); +} + +#ifndef Z7_ST + + +static const size_t kBlockSize = 1 << 16; +// kMemPerThread must be >= kBlockSize +// +static const size_t kMemPerThread = (size_t)sizeof(size_t) << 23; +// static const size_t kMemPerThread = (size_t)sizeof(size_t) << 16; // for debug +// static const size_t kMemPerThread = (size_t)1 << 16; // for debug + +/* +in: + nt_Zip >= 1: the starting maximum number of ZIP threads for search +out: + nt_Zip: calculated number of ZIP threads + returns: calculated number of ZSTD threads +*/ +/* +static UInt32 CalcThreads_for_ZipZstd(CZstdEncProps *zstdProps, + UInt64 memLimit, UInt32 totalThreads, + UInt32 &nt_Zip) +{ + for (; nt_Zip > 1; nt_Zip--) + { + UInt64 mem1 = memLimit / nt_Zip; + if (mem1 <= kMemPerThread) + continue; + mem1 -= kMemPerThread; + UInt32 n_ZSTD = ZstdEncProps_GetNumThreads_for_MemUsageLimit( + zstdProps, mem1, totalThreads / nt_Zip); + // we don't allow (nbWorkers == 1) here + if (n_ZSTD <= 1) + n_ZSTD = 0; + zstdProps->nbWorkers = n_ZSTD; + mem1 = ZstdEncProps_GetMemUsage(zstdProps); + if ((mem1 + kMemPerThread) * nt_Zip <= memLimit) + return n_ZSTD; + } + return ZstdEncProps_GetNumThreads_for_MemUsageLimit( + zstdProps, memLimit, totalThreads); +} + + +static UInt32 SetZstdThreads( + const CCompressionMethodMode &options, + COneMethodInfo *oneMethodMain, + UInt32 numThreads, + UInt32 numZipThreads_limit, + UInt64 numFilesToCompress, + UInt64 numBytesToCompress) +{ + NCompress::NZstd::CEncoderProps encoderProps; + RINOK(encoderProps.SetFromMethodProps(*oneMethodMain)); + CZstdEncProps &zstdProps = encoderProps.EncProps; + ZstdEncProps_NormalizeFull(&zstdProps); + if (oneMethodMain->FindProp(NCoderPropID::kNumThreads) >= 0) + { + // threads for ZSTD are fixed + if (zstdProps.nbWorkers > 1) + numThreads /= zstdProps.nbWorkers; + if (numThreads > numZipThreads_limit) + numThreads = numZipThreads_limit; + if (options._memUsage_WasSet + && !options._numThreads_WasForced) + { + const UInt64 mem1 = ZstdEncProps_GetMemUsage(&zstdProps); + const UInt64 numZipThreads = options._memUsage_Compress / (mem1 + kMemPerThread); + if (numThreads > numZipThreads) + numThreads = (UInt32)numZipThreads; + } + return numThreads; + } + { + // threads for ZSTD are not fixed + + // calculate estimated required number of ZST threads per file size statistics + UInt32 t = MY_ZSTDMT_NBWORKERS_MAX; + { + UInt64 averageNumberOfBlocks = 0; + const UInt64 averageSize = numBytesToCompress / numFilesToCompress; + const UInt64 jobSize = zstdProps.jobSize; + if (jobSize != 0) + averageNumberOfBlocks = averageSize / jobSize + 0; + if (t > averageNumberOfBlocks) + t = (UInt32)averageNumberOfBlocks; + } + if (t > numThreads) + t = numThreads; + + // calculate the nuber of zip threads + UInt32 numZipThreads = numThreads; + if (t > 1) + numZipThreads = numThreads / t; + if (numZipThreads > numZipThreads_limit) + numZipThreads = numZipThreads_limit; + if (numZipThreads < 1) + numZipThreads = 1; + { + // recalculate the number of ZSTD threads via the number of ZIP threads + const UInt32 t2 = numThreads / numZipThreads; + if (t < t2) + t = t2; + } + + if (options._memUsage_WasSet + && !options._numThreads_WasForced) + { + t = CalcThreads_for_ZipZstd(&zstdProps, + options._memUsage_Compress, numThreads, numZipThreads); + numThreads = numZipThreads; + } + // we don't use (nbWorkers = 1) here + if (t <= 1) + t = 0; + oneMethodMain->AddProp_NumThreads(t); + return numThreads; + } +} +*/ + +#endif + + + + +static HRESULT Update2( + DECL_EXTERNAL_CODECS_LOC_VARS + COutArchive &archive, + CInArchive *inArchive, + const CObjectVector &inputItems, + CObjectVector &updateItems, + const CUpdateOptions &updateOptions, + const CCompressionMethodMode &options, bool outSeqMode, + const CByteBuffer *comment, + IArchiveUpdateCallback *updateCallback) +{ + CMyComPtr opCallback; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback); + + /* + CMyComPtr reportArcProp; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackArcProp, (void **)&reportArcProp); + */ + + bool unknownComplexity = false; + UInt64 complexity = 0; + #ifndef Z7_ST + UInt64 numFilesToCompress = 0; + UInt64 numBytesToCompress = 0; + #endif + + unsigned i; + + for (i = 0; i < updateItems.Size(); i++) + { + const CUpdateItem &ui = updateItems[i]; + if (ui.NewData) + { + if (ui.Size == (UInt64)(Int64)-1) + unknownComplexity = true; + else + complexity += ui.Size; + #ifndef Z7_ST + numBytesToCompress += ui.Size; + numFilesToCompress++; + #endif + /* + if (ui.Commented) + complexity += ui.CommentRange.Size; + */ + } + else + { + CItemEx inputItem = inputItems[(unsigned)ui.IndexInArc]; + if (inArchive->Read_LocalItem_After_CdItem_Full(inputItem) != S_OK) + return E_NOTIMPL; + complexity += inputItem.GetLocalFullSize(); + // complexity += inputItem.GetCentralExtraPlusCommentSize(); + } + complexity += kLocalHeaderSize; + complexity += kCentralHeaderSize; + } + + if (comment) + complexity += comment->Size(); + complexity++; // end of central + + if (!unknownComplexity) + updateCallback->SetTotal(complexity); + + UInt64 totalComplexity = complexity; + + CCompressionMethodMode options2 = options; + + if (options2._methods.IsEmpty()) + { + // we need method item, if default method was used + options2._methods.AddNew(); + } + + CAddCommon compressor; + compressor.SetOptions(options2); + + complexity = 0; + + const Byte method = options.MethodSequence.FrontItem(); + + COneMethodInfo *oneMethodMain = NULL; + if (!options2._methods.IsEmpty()) + oneMethodMain = &options2._methods[0]; + + { + FOR_VECTOR (mi, options2._methods) + { + options2.SetGlobalLevelTo(options2._methods[mi]); + } + } + + if (oneMethodMain) + { + // appnote recommends to use EOS marker for LZMA. + if (method == NFileHeader::NCompressionMethod::kLZMA) + oneMethodMain->AddProp_EndMarker_if_NotFound(true); + } + + + #ifndef Z7_ST + + UInt32 numThreads = options._numThreads; +#ifdef _WIN32 + const UInt32 numThreadGroups = options._numThreadGroups; +#endif + + UInt32 numZipThreads_limit = numThreads; + if (numZipThreads_limit > numFilesToCompress) + numZipThreads_limit = (UInt32)numFilesToCompress; + + if (numZipThreads_limit > 1) + { + const unsigned numFiles_OPEN_MAX = NSystem::Get_File_OPEN_MAX_Reduced_for_3_tasks(); + // printf("\nzip:numFiles_OPEN_MAX =%d\n", (unsigned)numFiles_OPEN_MAX); + if (numZipThreads_limit > numFiles_OPEN_MAX) + numZipThreads_limit = (UInt32)numFiles_OPEN_MAX; + } + + { + // we reduce number of threads for 32-bit to reduce memory usege to 256 MB + const UInt32 kNumMaxThreads = + // _WIN32 (64-bit) supports only 64 threads in one group. + 8 << (sizeof(size_t) / 2); // 32 threads for 32-bit : 128 threads for 64-bit + if (numThreads > kNumMaxThreads) + numThreads = kNumMaxThreads; + } + /* + if (numThreads > MAXIMUM_WAIT_OBJECTS) // is 64 in Windows + numThreads = MAXIMUM_WAIT_OBJECTS; + */ + + + /* + // zstd supports (numThreads == 0); + if (numThreads < 1) + numThreads = 1; + */ + + bool mtMode = (numThreads > 1); + + if (numFilesToCompress <= 1) + mtMode = false; + + // mtMode = true; // debug: to test mtMode + + if (!mtMode) + { + // if (oneMethodMain) { + /* + if (method == NFileHeader::NCompressionMethod::kZstdWz) + { + if (oneMethodMain->FindProp(NCoderPropID::kNumThreads) < 0) + { + // numZstdThreads was not forced in oneMethodMain + if (numThreads >= 1 + && options._memUsage_WasSet + && !options._numThreads_WasForced) + { + NCompress::NZstd::CEncoderProps encoderProps; + RINOK(encoderProps.SetFromMethodProps(*oneMethodMain)) + CZstdEncProps &zstdProps = encoderProps.EncProps; + ZstdEncProps_NormalizeFull(&zstdProps); + numThreads = ZstdEncProps_GetNumThreads_for_MemUsageLimit( + &zstdProps, options._memUsage_Compress, numThreads); + // we allow (nbWorkers = 1) here. + } + oneMethodMain->AddProp_NumThreads(numThreads); + } + } // kZstdWz + */ + // } // oneMethodMain + + FOR_VECTOR (mi, options2._methods) + { + COneMethodInfo &onem = options2._methods[mi]; + + if (onem.FindProp(NCoderPropID::kNumThreads) < 0) + { + // fixme: we should check the number of threads for xz method also + // fixed for 9.31. bzip2 default is just one thread. + onem.AddProp_NumThreads(numThreads); + } + } + } + else // mtMode + { + if (method == NFileHeader::NCompressionMethod::kStore && !options.Password_Defined) + numThreads = 1; + + if (oneMethodMain) + { + + if (method == NFileHeader::NCompressionMethod::kBZip2) + { + bool fixedNumber; + UInt32 numBZip2Threads = oneMethodMain->Get_BZip2_NumThreads(fixedNumber); + if (!fixedNumber) + { + const UInt64 averageSize = numBytesToCompress / numFilesToCompress; + const UInt32 blockSize = oneMethodMain->Get_BZip2_BlockSize(); + const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1; + numBZip2Threads = 64; + if (numBZip2Threads > averageNumberOfBlocks) + numBZip2Threads = (UInt32)averageNumberOfBlocks; + if (numBZip2Threads > numThreads) + numBZip2Threads = numThreads; + oneMethodMain->AddProp_NumThreads(numBZip2Threads); + } + numThreads /= numBZip2Threads; + } + else if (method == NFileHeader::NCompressionMethod::kXz) + { + UInt32 numLzmaThreads = 1; + int numXzThreads = oneMethodMain->Get_Xz_NumThreads(numLzmaThreads); + if (numXzThreads < 0) + { + // numXzThreads is unknown + const UInt64 averageSize = numBytesToCompress / numFilesToCompress; + const UInt64 blockSize = oneMethodMain->Get_Xz_BlockSize(); + UInt64 averageNumberOfBlocks = 1; + if (blockSize != (UInt64)(Int64)-1) + averageNumberOfBlocks = averageSize / blockSize + 1; + UInt32 t = 256; + if (t > averageNumberOfBlocks) + t = (UInt32)averageNumberOfBlocks; + t *= numLzmaThreads; + if (t > numThreads) + t = numThreads; + oneMethodMain->AddProp_NumThreads(t); + numXzThreads = (int)t; + } + numThreads /= (unsigned)numXzThreads; + } + /* + else if (method == NFileHeader::NCompressionMethod::kZstdWz) + { + numThreads = SetZstdThreads(options, + oneMethodMain, numThreads, + numZipThreads_limit, + numFilesToCompress, numBytesToCompress); + } + */ + else if ( + method == NFileHeader::NCompressionMethod::kDeflate + || method == NFileHeader::NCompressionMethod::kDeflate64 + || method == NFileHeader::NCompressionMethod::kPPMd) + { + if (numThreads > 1 + && options._memUsage_WasSet + && !options._numThreads_WasForced) + { + UInt64 methodMemUsage; + if (method == NFileHeader::NCompressionMethod::kPPMd) + methodMemUsage = oneMethodMain->Get_Ppmd_MemSize(); + else + methodMemUsage = (4 << 20); // for deflate + const UInt64 threadMemUsage = kMemPerThread + methodMemUsage; + const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage; + if (numThreads64 < numThreads) + numThreads = (UInt32)numThreads64; + } + } + else if (method == NFileHeader::NCompressionMethod::kLZMA) + { + // we suppose that default LZMA is 2 thread. So we don't change it + const UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads(); + numThreads /= numLZMAThreads; + + if (numThreads > 1 + && options._memUsage_WasSet + && !options._numThreads_WasForced) + { + const UInt64 methodMemUsage = oneMethodMain->Get_Lzma_MemUsage(true); + const UInt64 threadMemUsage = kMemPerThread + methodMemUsage; + const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage; + if (numThreads64 < numThreads) + numThreads = (UInt32)numThreads64; + } + } + } // (oneMethodMain) + + if (numThreads > numZipThreads_limit) + numThreads = numZipThreads_limit; + if (numThreads <= 1) + { + mtMode = false; + numThreads = 1; + } + } + + // mtMode = true; // to test mtMode for seqMode + + if (!mtMode) + #endif + return Update2St( + EXTERNAL_CODECS_LOC_VARS + archive, inArchive, + inputItems, updateItems, + updateOptions, + &options2, outSeqMode, + comment, updateCallback, totalComplexity, + opCallback + // , reportArcProp + ); + + + #ifndef Z7_ST + + /* + CTotalStats stat; + stat.Size = 0; + stat.PackSize = 0; + */ + if (numThreads < 1) + numThreads = 1; + + CObjectVector items; + + CMtProgressMixer *mtProgressMixerSpec = new CMtProgressMixer; + CMyComPtr progress = mtProgressMixerSpec; + mtProgressMixerSpec->Create(updateCallback, true); + + CMtCompressProgressMixer mtCompressProgressMixer; + mtCompressProgressMixer.Init(numThreads, mtProgressMixerSpec->RatioProgress); + + CMemBlockManagerMt memManager(kBlockSize); + CMemRefs refs(&memManager); + + CMtSem mtSem; + CThreads threads; + mtSem.Head = -1; + mtSem.Indexes.ClearAndSetSize(numThreads); + { + WRes wres = mtSem.Semaphore.Create(0, numThreads); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + + CUIntVector threadIndices; // list threads in order of updateItems + + { + RINOK(memManager.AllocateSpaceAlways((size_t)numThreads * (kMemPerThread / kBlockSize))) + for (i = 0; i < updateItems.Size(); i++) + refs.Refs.Add(CMemBlocks2()); + + for (i = 0; i < numThreads; i++) + { + threads.Threads.AddNew(); + // mtSem.Indexes[i] = -1; // actually we don't use these values + } + + for (i = 0; i < numThreads; i++) + { + CThreadInfo &threadInfo = threads.Threads[i]; + threadInfo.ThreadIndex = i; + threadInfo.SetOptions(options2); + #ifdef Z7_EXTERNAL_CODECS + threadInfo._externalCodecs = _externalCodecs; + #endif + RINOK(threadInfo.CreateEvents()) + threadInfo.OutStreamSpec = new COutMemStream(&memManager); + RINOK(threadInfo.OutStreamSpec->CreateEvents(SYNC_WFMO(&memManager.Synchro))) + threadInfo.OutStream = threadInfo.OutStreamSpec; + threadInfo.ProgressSpec = new CMtCompressProgress(); + threadInfo.Progress = threadInfo.ProgressSpec; + threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, i); + threadInfo.MtSem = &mtSem; + const HRESULT hres = + threadInfo.CreateThread_with_group( +#ifdef _WIN32 + (numThreadGroups > 1 && numThreads > 1) ? + (int)(i % numThreadGroups) : -1 +#endif + ); + RINOK(hres) + } + } + + unsigned mtItemIndex = 0; + unsigned itemIndex = 0; + int lastRealStreamItemIndex = -1; + + + while (itemIndex < updateItems.Size()) + { + if (threadIndices.Size() < numThreads && mtItemIndex < updateItems.Size()) + { + // we start ahead the threads for compressing + // also we set refs.Refs[itemIndex].SeqMode that is used later + // don't move that code block + + CUpdateItem &ui = updateItems[mtItemIndex++]; + if (!ui.NewData) + continue; + CItemEx itemEx; + CItemOut item; + + if (ui.NewProps) + { + if (ui.IsDir) + continue; + } + else + { + itemEx = inputItems[(unsigned)ui.IndexInArc]; + if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK) + return E_NOTIMPL; + (CItem &)item = itemEx; + if (item.IsDir() != ui.IsDir) + return E_NOTIMPL; + if (ui.IsDir) + continue; + } + + CMyComPtr fileInStream; + + CMemBlocks2 &memRef2 = refs.Refs[mtItemIndex - 1]; + + { + NWindows::NSynchronization::CCriticalSectionLock lock(mtProgressMixerSpec->Mixer2->CriticalSection); + const HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream); + if (res == S_FALSE) + { + complexity += ui.Size; + complexity += kLocalHeaderSize; + mtProgressMixerSpec->Mixer2->SetProgressOffset_NoLock(complexity); + RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)) + memRef2.Skip = true; + continue; + } + RINOK(res) + if (!fileInStream) + return E_INVALIDARG; + UpdatePropsFromStream(updateOptions, ui, fileInStream, updateCallback, totalComplexity); + RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)) + } + + UInt32 k; + for (k = 0; k < numThreads; k++) + if (threads.Threads[k].IsFree) + break; + + if (k == numThreads) + return E_FAIL; + { + { + CThreadInfo &threadInfo = threads.Threads[k]; + threadInfo.IsFree = false; + threadInfo.InStream = fileInStream; + + bool inSeqMode = false; + + if (!inSeqMode) + { + CMyComPtr inStream2; + fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2); + inSeqMode = (inStream2 == NULL); + } + memRef2.InSeqMode = inSeqMode; + + // !!!!! we must release ref before sending event + // BUG was here in v4.43 and v4.44. It could change ref counter in two threads in same time + fileInStream.Release(); + + threadInfo.OutStreamSpec->Init(); + threadInfo.ProgressSpec->Reinit(); + + threadInfo.UpdateIndex = mtItemIndex - 1; + threadInfo.InSeqMode = inSeqMode; + threadInfo.OutSeqMode = outSeqMode; + threadInfo.FileTime = ui.Time; // FileTime is used for ZipCrypto only in seqMode + threadInfo.ExpectedDataSize = ui.Size; + threadInfo.ExpectedDataSize_IsConfirmed = ui.Size_WasSetFromStream; + + threadInfo.CompressEvent.Set(); + + threadIndices.Add(k); + } + } + + continue; + } + + if (refs.Refs[itemIndex].Skip) + { + itemIndex++; + continue; + } + + const CUpdateItem &ui = updateItems[itemIndex]; + + CItemEx itemEx; + CItemOut item; + + if (!ui.NewProps || !ui.NewData) + { + itemEx = inputItems[(unsigned)ui.IndexInArc]; + if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK) + return E_NOTIMPL; + (CItem &)item = itemEx; + } + + if (ui.NewData) + { + // bool isDir = ((ui.NewProps) ? ui.IsDir : item.IsDir()); + const bool isDir = ui.IsDir; + + if (isDir) + { + RINOK(WriteDirHeader(archive, &options, ui, item)) + } + else + { + CMemBlocks2 &memRef = refs.Refs[itemIndex]; + + if (memRef.Finished) + { + if (lastRealStreamItemIndex < (int)itemIndex) + lastRealStreamItemIndex = (int)itemIndex; + + SetFileHeader(options, ui, memRef.CompressingResult.DescriptorMode, item); + + // the BUG was fixed in 9.26: + // SetItemInfoFromCompressingResult must be after SetFileHeader + // to write correct Size. + + SetItemInfoFromCompressingResult(memRef.CompressingResult, + options.IsRealAesMode(), options.AesKeyMode, item); + RINOK(archive.ClearRestriction()) + archive.WriteLocalHeader(item); + // RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)); + CMyComPtr outStream; + archive.CreateStreamForCopying(outStream); + memRef.WriteToStream(memManager.GetBlockSize(), outStream); + // v23: we fixed the bug: we need to write descriptor also + if (item.HasDescriptor()) + { + /* that function doesn't rewrite local header, if item.HasDescriptor(). + it just writes descriptor */ + archive.WriteLocalHeader_Replace(item); + } + else + archive.MoveCurPos(item.PackSize); + memRef.FreeOpt(&memManager); + /* + if (reportArcProp) + { + stat.UpdateWithItem(item); + RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options.IsRealAesMode())); + } + */ + } + else + { + // current file was not finished + + if (lastRealStreamItemIndex < (int)itemIndex) + { + // LocalHeader was not written for current itemIndex still + + lastRealStreamItemIndex = (int)itemIndex; + + // thread was started before for that item already, and memRef.SeqMode was set + + CCompressingResult compressingResult; + RINOK(compressor.Set_Pre_CompressionResult( + memRef.InSeqMode, outSeqMode, + ui.Size, + compressingResult)) + + memRef.PreDescriptorMode = compressingResult.DescriptorMode; + SetFileHeader(options, ui, compressingResult.DescriptorMode, item); + + SetItemInfoFromCompressingResult(compressingResult, options.IsRealAesMode(), options.AesKeyMode, item); + + // file Size can be 64-bit !!! + RINOK(archive.SetRestrictionFromCurrent()) + archive.WriteLocalHeader(item); + } + + { + CThreadInfo &thread = threads.Threads[threadIndices.FrontItem()]; + if (!thread.OutStreamSpec->WasUnlockEventSent()) + { + CMyComPtr outStream; + archive.CreateStreamForCompressing(outStream); + thread.OutStreamSpec->SetOutStream(outStream); + thread.OutStreamSpec->SetRealStreamMode(); + } + } + + const WRes wres = mtSem.Semaphore.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + + const int ti = mtSem.GetFreeItem(); + if (ti < 0) + return E_FAIL; + + CThreadInfo &threadInfo = threads.Threads[(unsigned)ti]; + threadInfo.InStream.Release(); + threadInfo.IsFree = true; + RINOK(threadInfo.Result) + + unsigned t = 0; + + for (;;) + { + if (t == threadIndices.Size()) + return E_FAIL; + if (threadIndices[t] == (unsigned)ti) + break; + t++; + } + threadIndices.Delete(t); + + if (t == 0) + { + // if thread for current file was finished. + if (threadInfo.UpdateIndex != itemIndex) + return E_FAIL; + + if (memRef.PreDescriptorMode != threadInfo.CompressingResult.DescriptorMode) + return E_FAIL; + + RINOK(threadInfo.OutStreamSpec->WriteToRealStream()) + threadInfo.OutStreamSpec->ReleaseOutStream(); + SetFileHeader(options, ui, threadInfo.CompressingResult.DescriptorMode, item); + SetItemInfoFromCompressingResult(threadInfo.CompressingResult, + options.IsRealAesMode(), options.AesKeyMode, item); + + archive.WriteLocalHeader_Replace(item); + + /* + if (reportArcProp) + { + stat.UpdateWithItem(item); + RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options.IsRealAesMode())); + } + */ + } + else + { + // it's not current file. So we must store information in array + CMemBlocks2 &memRef2 = refs.Refs[threadInfo.UpdateIndex]; + threadInfo.OutStreamSpec->DetachData(memRef2); + memRef2.CompressingResult = threadInfo.CompressingResult; + // memRef2.SeqMode = threadInfo.SeqMode; // it was set before + memRef2.Finished = true; + continue; + } + } + } + } + else + { + RINOK(UpdateItemOldData(archive, inArchive, itemEx, ui, item, progress, opCallback, complexity)) + } + + items.Add(item); + complexity += kLocalHeaderSize; + mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity); + itemIndex++; + } + + RINOK(mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL)) + + RINOK(archive.WriteCentralDir(items, comment)) + + /* + if (reportArcProp) + { + RINOK(ReportArcProps(reportArcProp, stat)); + } + */ + + complexity += kCentralHeaderSize * updateItems.Size() + 1; + mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity); + return mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL); + + #endif +} + +/* +// we need CSeekOutStream, if we need Seek(0, STREAM_SEEK_CUR) for seqential stream +Z7_CLASS_IMP_COM_1( + CSeekOutStream + , IOutStream +) + Z7_IFACE_COM7_IMP(ISequentialOutStream) + + CMyComPtr _seqStream; + UInt64 _size; +public: + void Init(ISequentialOutStream *seqStream) + { + _size = 0; + _seqStream = seqStream; + } +}; + +Z7_COM7F_IMF(CSeekOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + UInt32 realProcessedSize; + const HRESULT result = _seqStream->Write(data, size, &realProcessedSize); + _size += realProcessedSize; + if (processedSize) + *processedSize = realProcessedSize; + return result; +} + +Z7_COM7F_IMF(CSeekOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) +{ + if (seekOrigin != STREAM_SEEK_CUR || offset != 0) + return E_NOTIMPL; + if (newPosition) + *newPosition = (UInt64)_size; + return S_OK; +} + +Z7_COM7F_IMF(CSeekOutStream::SetSize(UInt64 newSize)) +{ + UNUSED_VAR(newSize) + return E_NOTIMPL; +} +*/ + +static const size_t kCacheBlockSize = 1 << 20; +static const size_t kCacheSize = kCacheBlockSize << 2; +static const size_t kCacheMask = kCacheSize - 1; + +Z7_CLASS_IMP_NOQIB_2( + CCacheOutStream + , IOutStream + , IStreamSetRestriction +) + Z7_IFACE_COM7_IMP(ISequentialOutStream) + + HRESULT _hres; + CMyComPtr _seqStream; + CMyComPtr _stream; + CMyComPtr _setRestriction; + Byte *_cache; + size_t _cachedSize; + UInt64 _cachedPos; + UInt64 _virtPos; + UInt64 _virtSize; + UInt64 _phyPos; + UInt64 _phySize; + UInt64 _restrict_begin; + UInt64 _restrict_end; + + HRESULT FlushFromCache(size_t size); + HRESULT FlushNonRestrictedBlocks(); + HRESULT FlushCache(); + HRESULT SetRestriction_ForWrite(size_t writeSize) const; + + HRESULT SeekPhy(UInt64 pos) + { + if (pos == _phyPos) + return S_OK; + if (!_stream) + return E_NOTIMPL; + _hres = _stream->Seek((Int64)pos, STREAM_SEEK_SET, &_phyPos); + if (_hres == S_OK && _phyPos != pos) + _hres = E_FAIL; + return _hres; + } + +public: + CCacheOutStream(): _cache(NULL) {} + ~CCacheOutStream(); + bool Allocate() + { + if (!_cache) + _cache = (Byte *)::MidAlloc(kCacheSize); + return _cache != NULL; + } + HRESULT Init(ISequentialOutStream *seqStream, IOutStream *stream, IStreamSetRestriction *setRestriction); + HRESULT FinalFlush(); +}; + +CCacheOutStream::~CCacheOutStream() +{ + ::MidFree(_cache); +} + + +HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *stream, IStreamSetRestriction *setRestriction) +{ + _hres = S_OK; + _cachedSize = 0; + _cachedPos = 0; + _virtPos = 0; + _virtSize = 0; + // by default we have no restriction + _restrict_begin = 0; + _restrict_end = 0; + _seqStream = seqStream; + _stream = stream; + _setRestriction = setRestriction; + if (_stream) + { + RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos)) + RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize)) + RINOK(_stream->Seek((Int64)_virtPos, STREAM_SEEK_SET, &_virtPos)) + } + _phyPos = _virtPos; + _phySize = _virtSize; + return S_OK; +} + + +/* we call SetRestriction_ForWrite() just before Write() from cache. + (_phyPos == _cachedPos) + (writeSize != 0) +*/ +HRESULT CCacheOutStream::SetRestriction_ForWrite(size_t writeSize) const +{ + if (!_setRestriction) + return S_OK; + PRF(printf("\n-- CCacheOutStream::SetRestriction_ForWrite _cachedPos = 0x%x, writeSize = %d\n", (unsigned)_cachedPos, (unsigned)writeSize)); + UInt64 begin = _restrict_begin; + UInt64 end = _restrict_end; + const UInt64 phyPos = _phyPos; + if (phyPos != _cachedPos) return E_FAIL; + if (phyPos == _phySize) + { + // The writing will be to the end of phy stream. + // So we will try to use non-restricted write, if possible. + if (begin == end) + begin = _virtPos; // _virtSize; // it's supposed that (_virtSize == _virtPos) + if (phyPos + writeSize <= begin) + { + // the write is not restricted + PRF(printf("\n+++ write is not restricted \n")); + begin = 0; + end = 0; + } + else + { + if (begin > phyPos) + begin = phyPos; + end = (UInt64)(Int64)-1; + } + } + else + { + // (phyPos != _phySize) + if (begin == end || begin > phyPos) + begin = phyPos; + end = (UInt64)(Int64)-1; + } + return _setRestriction->SetRestriction(begin, end); +} + + +/* it writes up to (size) bytes from cache. + (size > _cachedSize) is allowed +*/ +HRESULT CCacheOutStream::FlushFromCache(size_t size) +{ + PRF(printf("\n-- CCacheOutStream::FlushFromCache %u\n", (unsigned)size)); + if (_hres != S_OK) + return _hres; + if (size > _cachedSize) + size = _cachedSize; + // (size <= _cachedSize) + if (size == 0) + return S_OK; + RINOK(SeekPhy(_cachedPos)) + for (;;) + { + // (_phyPos == _cachedPos) + const size_t pos = (size_t)_cachedPos & kCacheMask; + const size_t cur = MyMin(kCacheSize - pos, size); + _hres = SetRestriction_ForWrite(cur); + RINOK(_hres) + PRF(printf("\n-- CCacheOutStream::WriteFromCache _phyPos = 0x%x, size = %d\n", (unsigned)_phyPos, (unsigned)cur)); + _hres = WriteStream(_seqStream, _cache + pos, cur); + RINOK(_hres) + _phyPos += cur; + if (_phySize < _phyPos) + _phySize = _phyPos; + _cachedPos += cur; + _cachedSize -= cur; + size -= cur; + if (size == 0) + return S_OK; + } +} + + +HRESULT CCacheOutStream::FlushNonRestrictedBlocks() +{ + for (;;) + { + const size_t size = kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1)); + if (_cachedSize < size) + break; + UInt64 begin = _restrict_begin; + if (begin == _restrict_end) + begin = _virtPos; + // we don't flush the data to restricted area + if (_cachedPos + size > begin) + break; + RINOK(FlushFromCache(size)) + } + return S_OK; +} + + +HRESULT CCacheOutStream::FlushCache() +{ + return FlushFromCache(_cachedSize); +} + +HRESULT CCacheOutStream::FinalFlush() +{ + _restrict_begin = 0; + _restrict_end = 0; + RINOK(FlushCache()) + if (_stream && _hres == S_OK) + { + if (_virtSize != _phySize) + { + // it's unexpected + RINOK(_stream->SetSize(_virtSize)) + _phySize = _virtSize; + } + _hres = SeekPhy(_virtPos); + } + return _hres; +} + + +Z7_COM7F_IMF(CCacheOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + PRF(printf("\n==== CCacheOutStream::Write virtPos=0x%x, %u\n", (unsigned)_virtPos, (unsigned)size)); + + if (processedSize) + *processedSize = 0; + if (size == 0) + return S_OK; + if (_hres != S_OK) + return _hres; + + if (_cachedSize != 0) + if (_virtPos < _cachedPos || + _virtPos > _cachedPos + _cachedSize) + { + RINOK(FlushCache()) + } + + if (_cachedSize == 0) + _cachedPos = _virtPos; + + const size_t pos = (size_t)_virtPos & kCacheMask; + { + const size_t blockRem = kCacheBlockSize - ((size_t)_virtPos & (kCacheBlockSize - 1)); + if (size > blockRem) + size = (UInt32)blockRem; + } + // _cachedPos <= _virtPos <= _cachedPos + _cachedSize + const UInt64 cachedRem = _cachedPos + _cachedSize - _virtPos; + if (cachedRem) + { + // _virtPos < _cachedPos + _cachedSize + // we rewrite only existing data in cache. So _cachedSize will be not changed + if (size > cachedRem) + size = (UInt32)cachedRem; + } + else + { + // _virtPos == _cachedPos + _cachedSize + // so we need to add new data to the end of cache + if (_cachedSize == kCacheSize) + { + // cache is full. So we need to flush some part of cache. + // we flush only one block, but we are allowed to flush any size here + RINOK(FlushFromCache(kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1)))) + } + // _cachedSize != kCacheSize + // so we have some space for new data in cache + if (_cachedSize == 0) + { + /* this code is optional (for optimization): + we write data directly without cache, + if there is no restriction and we have full block. */ + if (_restrict_begin == _restrict_end + && size == kCacheBlockSize) + { + RINOK(SeekPhy(_virtPos)) + if (_setRestriction) + { + _hres = _setRestriction->SetRestriction(_restrict_begin, _restrict_end); + RINOK(_hres) + } + PRF(printf("\n-- CCacheOutStream::WriteDirectly _phyPos = 0x%x, size = %d\n", (unsigned)_phyPos, (unsigned)size)); + _hres = WriteStream(_seqStream, data, size); + RINOK(_hres) + if (processedSize) + *processedSize = size; + _virtPos += size; + if (_virtSize < _virtPos) + _virtSize = _virtPos; + _phyPos += size; + if (_phySize < _phyPos) + _phySize = _phyPos; + return S_OK; + } + } + else // (_cachedSize != 0) + { + const size_t startPos = (size_t)_cachedPos & kCacheMask; + // we don't allow new data to overwrite old start data in cache. + // (startPos == pos) here means that cache is empty. + // (startPos == pos) is not possible here. + if (startPos > pos) + size = (UInt32)MyMin((size_t)size, (size_t)(startPos - pos)); + } + // _virtPos == (_cachedPos + _cachedSize) still + _cachedSize += size; + } + + memcpy(_cache + pos, data, size); + if (processedSize) + *processedSize = size; + _virtPos += size; + if (_virtSize < _virtPos) + _virtSize = _virtPos; + return FlushNonRestrictedBlocks(); +} + + +Z7_COM7F_IMF(CCacheOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) +{ + PRF(printf("\n==== CCacheOutStream::Seek seekOrigin=%d Seek =%u\n", seekOrigin, (unsigned)offset)); + switch (seekOrigin) + { + case STREAM_SEEK_SET: break; + case STREAM_SEEK_CUR: offset += _virtPos; break; + case STREAM_SEEK_END: offset += _virtSize; break; + default: return STG_E_INVALIDFUNCTION; + } + if (offset < 0) + return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; + _virtPos = (UInt64)offset; + if (newPosition) + *newPosition = (UInt64)offset; + return S_OK; +} + + +Z7_COM7F_IMF(CCacheOutStream::SetSize(UInt64 newSize)) +{ + if (_hres != S_OK) + return _hres; + + if (newSize <= _cachedPos || _cachedSize == 0) + { + _cachedSize = 0; + _cachedPos = newSize; + } + else + { + // _cachedSize != 0 + // newSize > _cachedPos + const UInt64 offset = newSize - _cachedPos; + if (offset <= _cachedSize) + { + // newSize is inside cached block or is touching cached block. + // so we reduce cache + _cachedSize = (size_t)offset; + if (_phySize <= newSize) + { + // _phySize will be restored later after cache flush + _virtSize = newSize; + return S_OK; + } + // (_phySize > newSize) + // so we must reduce phyStream size to (newSize) or to (_cachedPos) + // newPhySize = _cachedPos; // optional reduce to _cachedPos + } + else + { + // newSize > _cachedPos + _cachedSize + /* It's possible that we need to write zeros, + if new size is larger than old size. + We don't optimize for possible cases here. + So we just flush the cache. */ + _hres = FlushCache(); + } + } + + _virtSize = newSize; + + if (_hres != S_OK) + return _hres; + + if (newSize != _phySize) + { + if (!_stream) + return E_NOTIMPL; + // if (_phyPos > newSize) + RINOK(SeekPhy(newSize)) + if (_setRestriction) + { + UInt64 begin = _restrict_begin; + UInt64 end = _restrict_end; + if (_cachedSize != 0) + { + if (begin > _cachedPos) + begin = _cachedPos; + end = (UInt64)(Int64)-1; + } + _hres = _setRestriction->SetRestriction(begin, end); + RINOK(_hres) + } + _hres = _stream->SetSize(newSize); + RINOK(_hres) + _phySize = newSize; + } + return S_OK; +} + + +Z7_COM7F_IMF(CCacheOutStream::SetRestriction(UInt64 begin, UInt64 end)) +{ + PRF(printf("\n============ CCacheOutStream::SetRestriction 0x%x, %u\n", (unsigned)begin, (unsigned)end)); + _restrict_begin = begin; + _restrict_end = end; + return FlushNonRestrictedBlocks(); +} + + + +HRESULT Update( + DECL_EXTERNAL_CODECS_LOC_VARS + const CObjectVector &inputItems, + CObjectVector &updateItems, + ISequentialOutStream *seqOutStream, + CInArchive *inArchive, bool removeSfx, + const CUpdateOptions &updateOptions, + const CCompressionMethodMode &compressionMethodMode, + IArchiveUpdateCallback *updateCallback) +{ + /* + // it was tested before + if (inArchive) + { + if (!inArchive->CanUpdate()) + return E_NOTIMPL; + } + */ + + CMyComPtr setRestriction; + seqOutStream->QueryInterface(IID_IStreamSetRestriction, (void **)&setRestriction); + if (setRestriction) + { + RINOK(setRestriction->SetRestriction(0, 0)) + } + + CMyComPtr outStream; + CCacheOutStream *cacheStream; + bool outSeqMode; + + { + CMyComPtr outStreamReal; + + if (!compressionMethodMode.Force_SeqOutMode) + { + seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStreamReal); + /* + if (!outStreamReal) + return E_NOTIMPL; + */ + } + + if (inArchive) + { + if (!inArchive->IsMultiVol && inArchive->ArcInfo.Base > 0 && !removeSfx) + { + IInStream *baseStream = inArchive->GetBaseStream(); + RINOK(InStream_SeekToBegin(baseStream)) + RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, (UInt64)inArchive->ArcInfo.Base, NULL)) + } + } + + outSeqMode = (outStreamReal == NULL); + if (outSeqMode) + setRestriction.Release(); + /* CCacheOutStream works as non-restricted by default. + So we use (setRestriction == NULL) for outSeqMode */ + // bool use_cacheStream = true; + // if (use_cacheStream) + { + cacheStream = new CCacheOutStream(); + outStream = cacheStream; + if (!cacheStream->Allocate()) + return E_OUTOFMEMORY; + RINOK(cacheStream->Init(seqOutStream, outStreamReal, setRestriction)) + setRestriction.Release(); + if (!outSeqMode) + setRestriction = cacheStream; + } + /* + else if (!outStreamReal) + { + CSeekOutStream *seekOutStream = new CSeekOutStream(); + outStream = seekOutStream; + seekOutStream->Init(seqOutStream); + } + else + outStream = outStreamReal; + */ + } + + COutArchive outArchive; + outArchive.SetRestriction = setRestriction; + + RINOK(outArchive.Create(outStream)) + + if (inArchive) + { + if (!inArchive->IsMultiVol && (Int64)inArchive->ArcInfo.MarkerPos2 > inArchive->ArcInfo.Base) + { + IInStream *baseStream = inArchive->GetBaseStream(); + RINOK(InStream_SeekSet(baseStream, (UInt64)inArchive->ArcInfo.Base)) + const UInt64 embStubSize = (UInt64)((Int64)inArchive->ArcInfo.MarkerPos2 - inArchive->ArcInfo.Base); + RINOK(NCompress::CopyStream_ExactSize(baseStream, outStream, embStubSize, NULL)) + outArchive.MoveCurPos(embStubSize); + } + } + + RINOK (Update2( + EXTERNAL_CODECS_LOC_VARS + outArchive, inArchive, + inputItems, updateItems, + updateOptions, + compressionMethodMode, outSeqMode, + inArchive ? &inArchive->ArcInfo.Comment : NULL, + updateCallback)) + + return cacheStream->FinalFlush(); +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipUpdate.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipUpdate.h new file mode 100644 index 0000000..13ecd9c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Archive/Zip/ZipUpdate.h @@ -0,0 +1,107 @@ +// ZipUpdate.h + +#ifndef ZIP7_INC_ZIP_UPDATE_H +#define ZIP7_INC_ZIP_UPDATE_H + +#include "../../ICoder.h" +#include "../IArchive.h" + +#include "../../Common/CreateCoder.h" + +#include "ZipCompressionMode.h" +#include "ZipIn.h" + +namespace NArchive { +namespace NZip { + +/* +struct CUpdateRange +{ + UInt64 Position; + UInt64 Size; + + // CUpdateRange() {} + CUpdateRange(UInt64 position, UInt64 size): Position(position), Size(size) {} +}; +*/ + +struct CUpdateItem +{ + bool NewData; + bool NewProps; + bool IsDir; + bool Write_NtfsTime; + bool Write_UnixTime; + // bool Write_UnixTime_ATime; + bool IsUtf8; + bool Size_WasSetFromStream; + // bool IsAltStream; + int IndexInArc; + unsigned IndexInClient; + UInt32 Attrib; + UInt32 Time; + UInt64 Size; + AString Name; + CByteBuffer Name_Utf; // for Info-Zip (kIzUnicodeName) Extra + CByteBuffer Comment; + // bool Commented; + // CUpdateRange CommentRange; + FILETIME Ntfs_MTime; + FILETIME Ntfs_ATime; + FILETIME Ntfs_CTime; + + void Clear() + { + IsDir = false; + + Write_NtfsTime = false; + Write_UnixTime = false; + + IsUtf8 = false; + Size_WasSetFromStream = false; + // IsAltStream = false; + Time = 0; + Size = 0; + Name.Empty(); + Name_Utf.Free(); + Comment.Free(); + + FILETIME_Clear(Ntfs_MTime); + FILETIME_Clear(Ntfs_ATime); + FILETIME_Clear(Ntfs_CTime); + } + + CUpdateItem(): + IsDir(false), + Write_NtfsTime(false), + Write_UnixTime(false), + IsUtf8(false), + Size_WasSetFromStream(false), + // IsAltStream(false), + Time(0), + Size(0) + {} +}; + + +struct CUpdateOptions +{ + bool Write_MTime; + bool Write_ATime; + bool Write_CTime; +}; + + +HRESULT Update( + DECL_EXTERNAL_CODECS_LOC_VARS + const CObjectVector &inputItems, + CObjectVector &updateItems, + ISequentialOutStream *seqOutStream, + CInArchive *inArchive, bool removeSfx, + const CUpdateOptions &updateOptions, + const CCompressionMethodMode &compressionMethodMode, + IArchiveUpdateCallback *updateCallback); + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/MemBlocks.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/MemBlocks.cpp new file mode 100644 index 0000000..6165c7e --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/MemBlocks.cpp @@ -0,0 +1,215 @@ +// MemBlocks.cpp + +#include "StdAfx.h" + +#include "../../../C/Alloc.h" + +#include "MemBlocks.h" +#include "StreamUtils.h" + +bool CMemBlockManager::AllocateSpace_bool(size_t numBlocks) +{ + FreeSpace(); + if (numBlocks == 0) + { + return true; + // return false; + } + if (_blockSize < sizeof(void *)) + return false; + const size_t totalSize = numBlocks * _blockSize; + if (totalSize / _blockSize != numBlocks) + return false; + _data = ::MidAlloc(totalSize); + if (!_data) + return false; + Byte *p = (Byte *)_data; + for (size_t i = 0; i + 1 < numBlocks; i++, p += _blockSize) + *(Byte **)(void *)p = (p + _blockSize); + *(Byte **)(void *)p = NULL; + _headFree = _data; + return true; +} + +void CMemBlockManager::FreeSpace() +{ + ::MidFree(_data); + _data = NULL; + _headFree= NULL; +} + +void *CMemBlockManager::AllocateBlock() +{ + void *p = _headFree; + if (p) + _headFree = *(void **)p; + return p; +} + +void CMemBlockManager::FreeBlock(void *p) +{ + if (!p) + return; + *(void **)p = _headFree; + _headFree = p; +} + + +// #include + +HRESULT CMemBlockManagerMt::AllocateSpace(size_t numBlocks, size_t numNoLockBlocks) +{ + if (numNoLockBlocks > numBlocks) + return E_INVALIDARG; + const size_t numLockBlocks = numBlocks - numNoLockBlocks; + UInt32 maxCount = (UInt32)numLockBlocks; + if (maxCount != numLockBlocks) + return E_OUTOFMEMORY; + if (!CMemBlockManager::AllocateSpace_bool(numBlocks)) + return E_OUTOFMEMORY; + // we need (maxCount = 1), if we want to create non-use empty Semaphore + if (maxCount == 0) + maxCount = 1; + + // printf("\n Synchro.Create() \n"); + WRes wres; + #ifndef _WIN32 + Semaphore.Close(); + wres = Synchro.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + wres = Semaphore.Create(&Synchro, (UInt32)numLockBlocks, maxCount); + #else + wres = Semaphore.OptCreateInit((UInt32)numLockBlocks, maxCount); + #endif + + return HRESULT_FROM_WIN32(wres); +} + + +HRESULT CMemBlockManagerMt::AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_t numNoLockBlocks) +{ + // desiredNumberOfBlocks = 0; // for debug + if (numNoLockBlocks > desiredNumberOfBlocks) + return E_INVALIDARG; + for (;;) + { + // if (desiredNumberOfBlocks == 0) return E_OUTOFMEMORY; + const HRESULT hres = AllocateSpace(desiredNumberOfBlocks, numNoLockBlocks); + if (hres != E_OUTOFMEMORY) + return hres; + if (desiredNumberOfBlocks == numNoLockBlocks) + return E_OUTOFMEMORY; + desiredNumberOfBlocks = numNoLockBlocks + ((desiredNumberOfBlocks - numNoLockBlocks) >> 1); + } +} + +void CMemBlockManagerMt::FreeSpace() +{ + Semaphore.Close(); + CMemBlockManager::FreeSpace(); +} + +void *CMemBlockManagerMt::AllocateBlock() +{ + // Semaphore.Lock(); + NWindows::NSynchronization::CCriticalSectionLock lock(_criticalSection); + return CMemBlockManager::AllocateBlock(); +} + +void CMemBlockManagerMt::FreeBlock(void *p, bool lockMode) +{ + if (!p) + return; + { + NWindows::NSynchronization::CCriticalSectionLock lock(_criticalSection); + CMemBlockManager::FreeBlock(p); + } + if (lockMode) + Semaphore.Release(); +} + + + +void CMemBlocks::Free(CMemBlockManagerMt *manager) +{ + while (Blocks.Size() > 0) + { + manager->FreeBlock(Blocks.Back()); + Blocks.DeleteBack(); + } + TotalSize = 0; +} + +void CMemBlocks::FreeOpt(CMemBlockManagerMt *manager) +{ + Free(manager); + Blocks.ClearAndFree(); +} + +HRESULT CMemBlocks::WriteToStream(size_t blockSize, ISequentialOutStream *outStream) const +{ + UInt64 totalSize = TotalSize; + for (unsigned blockIndex = 0; totalSize > 0; blockIndex++) + { + size_t curSize = blockSize; + if (curSize > totalSize) + curSize = (size_t)totalSize; + if (blockIndex >= Blocks.Size()) + return E_FAIL; + RINOK(WriteStream(outStream, Blocks[blockIndex], curSize)) + totalSize -= curSize; + } + return S_OK; +} + + +void CMemLockBlocks::FreeBlock(unsigned index, CMemBlockManagerMt *memManager) +{ + memManager->FreeBlock(Blocks[index], LockMode); + Blocks[index] = NULL; +} + +void CMemLockBlocks::Free(CMemBlockManagerMt *memManager) +{ + while (Blocks.Size() > 0) + { + FreeBlock(Blocks.Size() - 1, memManager); + Blocks.DeleteBack(); + } + TotalSize = 0; +} + +/* +HRes CMemLockBlocks::SwitchToNoLockMode(CMemBlockManagerMt *memManager) +{ + if (LockMode) + { + if (Blocks.Size() > 0) + { + RINOK(memManager->ReleaseLockedBlocks(Blocks.Size())); + } + LockMode = false; + } + return 0; +} +*/ + +void CMemLockBlocks::Detach(CMemLockBlocks &blocks, CMemBlockManagerMt *memManager) +{ + blocks.Free(memManager); + blocks.LockMode = LockMode; + UInt64 totalSize = 0; + const size_t blockSize = memManager->GetBlockSize(); + FOR_VECTOR (i, Blocks) + { + if (totalSize < TotalSize) + blocks.Blocks.Add(Blocks[i]); + else + FreeBlock(i, memManager); + Blocks[i] = NULL; + totalSize += blockSize; + } + blocks.TotalSize = TotalSize; + Free(memManager); +} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/MemBlocks.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/MemBlocks.h new file mode 100644 index 0000000..201f3bc --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/MemBlocks.h @@ -0,0 +1,72 @@ +// MemBlocks.h + +#ifndef ZIP7_INC_MEM_BLOCKS_H +#define ZIP7_INC_MEM_BLOCKS_H + +#include "../../Common/MyVector.h" + +#include "../../Windows/Synchronization.h" + +#include "../IStream.h" + +class CMemBlockManager +{ + void *_data; + size_t _blockSize; + void *_headFree; +public: + CMemBlockManager(size_t blockSize = (1 << 20)): _data(NULL), _blockSize(blockSize), _headFree(NULL) {} + ~CMemBlockManager() { FreeSpace(); } + + bool AllocateSpace_bool(size_t numBlocks); + void FreeSpace(); + size_t GetBlockSize() const { return _blockSize; } + void *AllocateBlock(); + void FreeBlock(void *p); +}; + + +class CMemBlockManagerMt: public CMemBlockManager +{ + NWindows::NSynchronization::CCriticalSection _criticalSection; +public: + SYNC_OBJ_DECL(Synchro) + NWindows::NSynchronization::CSemaphore_WFMO Semaphore; + + CMemBlockManagerMt(size_t blockSize = (1 << 20)): CMemBlockManager(blockSize) {} + ~CMemBlockManagerMt() { FreeSpace(); } + + HRESULT AllocateSpace(size_t numBlocks, size_t numNoLockBlocks); + HRESULT AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_t numNoLockBlocks = 0); + void FreeSpace(); + void *AllocateBlock(); + void FreeBlock(void *p, bool lockMode = true); + // WRes ReleaseLockedBlocks_WRes(unsigned number) { return Semaphore.Release(number); } +}; + + +class CMemBlocks +{ + void Free(CMemBlockManagerMt *manager); +public: + CRecordVector Blocks; + UInt64 TotalSize; + + CMemBlocks(): TotalSize(0) {} + + void FreeOpt(CMemBlockManagerMt *manager); + HRESULT WriteToStream(size_t blockSize, ISequentialOutStream *outStream) const; +}; + +struct CMemLockBlocks: public CMemBlocks +{ + bool LockMode; + + CMemLockBlocks(): LockMode(true) {} + void Free(CMemBlockManagerMt *memManager); + void FreeBlock(unsigned index, CMemBlockManagerMt *memManager); + // HRESULT SwitchToNoLockMode(CMemBlockManagerMt *memManager); + void Detach(CMemLockBlocks &blocks, CMemBlockManagerMt *memManager); +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/OutMemStream.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/OutMemStream.cpp new file mode 100644 index 0000000..29a2394 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/OutMemStream.cpp @@ -0,0 +1,158 @@ +// OutMemStream.cpp + +#include "StdAfx.h" + +// #include + +#include "OutMemStream.h" + +void COutMemStream::Free() +{ + Blocks.Free(_memManager); + Blocks.LockMode = true; +} + +void COutMemStream::Init() +{ + WriteToRealStreamEvent.Reset(); + _unlockEventWasSent = false; + _realStreamMode = false; + Free(); + _curBlockPos = 0; + _curBlockIndex = 0; +} + +void COutMemStream::DetachData(CMemLockBlocks &blocks) +{ + Blocks.Detach(blocks, _memManager); + Free(); +} + + +HRESULT COutMemStream::WriteToRealStream() +{ + RINOK(Blocks.WriteToStream(_memManager->GetBlockSize(), OutSeqStream)) + Blocks.Free(_memManager); + return S_OK; +} + + +Z7_COM7F_IMF(COutMemStream::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + if (_realStreamMode) + return OutSeqStream->Write(data, size, processedSize); + if (processedSize) + *processedSize = 0; + while (size != 0) + { + if (_curBlockIndex < Blocks.Blocks.Size()) + { + Byte *p = (Byte *)Blocks.Blocks[_curBlockIndex] + _curBlockPos; + size_t curSize = _memManager->GetBlockSize() - _curBlockPos; + if (size < curSize) + curSize = size; + memcpy(p, data, curSize); + if (processedSize) + *processedSize += (UInt32)curSize; + data = (const void *)((const Byte *)data + curSize); + size -= (UInt32)curSize; + _curBlockPos += curSize; + + const UInt64 pos64 = GetPos(); + if (pos64 > Blocks.TotalSize) + Blocks.TotalSize = pos64; + if (_curBlockPos == _memManager->GetBlockSize()) + { + _curBlockIndex++; + _curBlockPos = 0; + } + continue; + } + + const NWindows::NSynchronization::CHandle_WFMO events[3] = + { StopWritingEvent, WriteToRealStreamEvent, /* NoLockEvent, */ _memManager->Semaphore }; + const DWORD waitResult = NWindows::NSynchronization::WaitForMultiObj_Any_Infinite( + ((Blocks.LockMode /* && _memManager->Semaphore.IsCreated() */) ? 3 : 2), events); + + // printf("\n 1- outMemStream %d\n", waitResult - WAIT_OBJECT_0); + + switch (waitResult) + { + case (WAIT_OBJECT_0 + 0): + return StopWriteResult; + case (WAIT_OBJECT_0 + 1): + { + _realStreamMode = true; + RINOK(WriteToRealStream()) + UInt32 processedSize2; + const HRESULT res = OutSeqStream->Write(data, size, &processedSize2); + if (processedSize) + *processedSize += processedSize2; + return res; + } + case (WAIT_OBJECT_0 + 2): + { + // it has bug: no write. + /* + if (!Blocks.SwitchToNoLockMode(_memManager)) + return E_FAIL; + */ + break; + } + default: + { + if (waitResult == WAIT_FAILED) + { + const DWORD res = ::GetLastError(); + if (res != 0) + return HRESULT_FROM_WIN32(res); + } + return E_FAIL; + } + } + void *p = _memManager->AllocateBlock(); + if (!p) + return E_FAIL; + Blocks.Blocks.Add(p); + } + return S_OK; +} + +Z7_COM7F_IMF(COutMemStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) +{ + if (_realStreamMode) + { + if (!OutStream) + return E_FAIL; + return OutStream->Seek(offset, seekOrigin, newPosition); + } + if (seekOrigin == STREAM_SEEK_CUR) + { + if (offset != 0) + return E_NOTIMPL; + } + else if (seekOrigin == STREAM_SEEK_SET) + { + if (offset != 0) + return E_NOTIMPL; + _curBlockIndex = 0; + _curBlockPos = 0; + } + else + return E_NOTIMPL; + if (newPosition) + *newPosition = GetPos(); + return S_OK; +} + +Z7_COM7F_IMF(COutMemStream::SetSize(UInt64 newSize)) +{ + if (_realStreamMode) + { + if (!OutStream) + return E_FAIL; + return OutStream->SetSize(newSize); + } + Blocks.TotalSize = newSize; + return S_OK; +} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/OutMemStream.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/OutMemStream.h new file mode 100644 index 0000000..a58fb91 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/OutMemStream.h @@ -0,0 +1,104 @@ +// OutMemStream.h + +#ifndef ZIP7_INC_OUT_MEM_STREAM_H +#define ZIP7_INC_OUT_MEM_STREAM_H + +#include "../../Common/MyCom.h" + +#include "MemBlocks.h" + +Z7_CLASS_IMP_NOQIB_1( + COutMemStream + , IOutStream +) + Z7_IFACE_COM7_IMP(ISequentialOutStream) + + CMemBlockManagerMt *_memManager; + size_t _curBlockPos; + unsigned _curBlockIndex; + bool _realStreamMode; + + bool _unlockEventWasSent; + NWindows::NSynchronization::CAutoResetEvent_WFMO StopWritingEvent; + NWindows::NSynchronization::CAutoResetEvent_WFMO WriteToRealStreamEvent; + // NWindows::NSynchronization::CAutoResetEvent NoLockEvent; + + HRESULT StopWriteResult; + CMemLockBlocks Blocks; + + CMyComPtr OutSeqStream; + CMyComPtr OutStream; + + UInt64 GetPos() const { return (UInt64)_curBlockIndex * _memManager->GetBlockSize() + _curBlockPos; } + +public: + + HRESULT CreateEvents(SYNC_PARAM_DECL(synchro)) + { + WRes wres = StopWritingEvent.CreateIfNotCreated_Reset(SYNC_WFMO(synchro)); + if (wres == 0) + wres = WriteToRealStreamEvent.CreateIfNotCreated_Reset(SYNC_WFMO(synchro)); + return HRESULT_FROM_WIN32(wres); + } + + void SetOutStream(IOutStream *outStream) + { + OutStream = outStream; + OutSeqStream = outStream; + } + + void SetSeqOutStream(ISequentialOutStream *outStream) + { + OutStream = NULL; + OutSeqStream = outStream; + } + + void ReleaseOutStream() + { + OutStream.Release(); + OutSeqStream.Release(); + } + + COutMemStream(CMemBlockManagerMt *memManager): + _memManager(memManager) + { + /* + #ifndef _WIN32 + StopWritingEvent._sync = + WriteToRealStreamEvent._sync = &memManager->Synchro; + #endif + */ + } + + ~COutMemStream() { Free(); } + void Free(); + + void Init(); + HRESULT WriteToRealStream(); + + void DetachData(CMemLockBlocks &blocks); + + bool WasUnlockEventSent() const { return _unlockEventWasSent; } + + void SetRealStreamMode() + { + _unlockEventWasSent = true; + WriteToRealStreamEvent.Set(); + } + + /* + void SetNoLockMode() + { + _unlockEventWasSent = true; + NoLockEvent.Set(); + } + */ + + void StopWriting(HRESULT res) + { + StopWriteResult = res; + StopWritingEvent.Set(); + } +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/ProgressMt.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/ProgressMt.h new file mode 100644 index 0000000..78c806c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Common/ProgressMt.h @@ -0,0 +1,43 @@ +// ProgressMt.h + +#ifndef ZIP7_INC_PROGRESSMT_H +#define ZIP7_INC_PROGRESSMT_H + +#include "../../Common/MyCom.h" +#include "../../Common/MyVector.h" +#include "../../Windows/Synchronization.h" + +#include "../ICoder.h" +#include "../IProgress.h" + +class CMtCompressProgressMixer +{ + CMyComPtr _progress; + CRecordVector InSizes; + CRecordVector OutSizes; + UInt64 TotalInSize; + UInt64 TotalOutSize; +public: + NWindows::NSynchronization::CCriticalSection CriticalSection; + void Init(unsigned numItems, ICompressProgressInfo *progress); + void Reinit(unsigned index); + HRESULT SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize); +}; + + +Z7_CLASS_IMP_NOQIB_1( + CMtCompressProgress + , ICompressProgressInfo +) + unsigned _index; + CMtCompressProgressMixer *_progress; +public: + void Init(CMtCompressProgressMixer *progress, unsigned index) + { + _progress = progress; + _index = index; + } + void Reinit() { _progress->Reinit(_index); } +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Const.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Const.h new file mode 100644 index 0000000..3380aaf --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Const.h @@ -0,0 +1,71 @@ +// Compress/BZip2Const.h + +#ifndef ZIP7_INC_COMPRESS_BZIP2_CONST_H +#define ZIP7_INC_COMPRESS_BZIP2_CONST_H + +namespace NCompress { +namespace NBZip2 { + +const Byte kArSig0 = 'B'; +const Byte kArSig1 = 'Z'; +const Byte kArSig2 = 'h'; +const Byte kArSig3 = '0'; + +const Byte kFinSig0 = 0x17; +const Byte kFinSig1 = 0x72; +const Byte kFinSig2 = 0x45; +const Byte kFinSig3 = 0x38; +const Byte kFinSig4 = 0x50; +const Byte kFinSig5 = 0x90; + +const Byte kBlockSig0 = 0x31; +const Byte kBlockSig1 = 0x41; +const Byte kBlockSig2 = 0x59; +const Byte kBlockSig3 = 0x26; +const Byte kBlockSig4 = 0x53; +const Byte kBlockSig5 = 0x59; + +const unsigned kNumOrigBits = 24; + +const unsigned kNumTablesBits = 3; +const unsigned kNumTablesMin = 2; +const unsigned kNumTablesMax = 6; + +const unsigned kNumLevelsBits = 5; + +const unsigned kMaxHuffmanLen = 20; // Check it + +const unsigned kMaxAlphaSize = 258; + +const unsigned kGroupSize = 50; + +const unsigned kBlockSizeMultMin = 1; +const unsigned kBlockSizeMultMax = 9; + +const UInt32 kBlockSizeStep = 100000; +const UInt32 kBlockSizeMax = kBlockSizeMultMax * kBlockSizeStep; + +const unsigned kNumSelectorsBits = 15; +const unsigned kNumSelectorsMax = 2 + kBlockSizeMax / kGroupSize; + +const unsigned kRleModeRepSize = 4; + +/* +The number of selectors stored in bzip2 block: +(numSelectors <= 18001) - must work with any decoder. +(numSelectors == 18002) - works with bzip2 1.0.6 decoder and all derived decoders. +(numSelectors > 18002) + lbzip2 2.5: encoder can write up to (18001 + 7) selectors. + + 7-Zip before 19.03: decoder doesn't support it. + 7-Zip 19.03: decoder allows 8 additional selector records for lbzip2 compatibility. + + bzip2 1.0.6: decoder can overflow selector[18002] arrays. But there are another + arrays after selector arrays. So the compiled code works. + bzip2 1.0.7: decoder doesn't support it. + bzip2 1.0.8: decoder allows additional selector records for lbzip2 compatibility. +*/ + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Crc.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Crc.cpp new file mode 100644 index 0000000..1c62253 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Crc.cpp @@ -0,0 +1,28 @@ +// BZip2Crc.cpp + +#include "StdAfx.h" + +#include "BZip2Crc.h" + +MY_ALIGN(64) +UInt32 CBZip2Crc::Table[256]; + +static const UInt32 kBZip2CrcPoly = 0x04c11db7; /* AUTODIN II, Ethernet, & FDDI */ + +void CBZip2Crc::InitTable() +{ + for (UInt32 i = 0; i < 256; i++) + { + UInt32 r = i << 24; + for (unsigned j = 0; j < 8; j++) + r = (r << 1) ^ (kBZip2CrcPoly & ((UInt32)0 - (r >> 31))); + Table[i] = r; + } +} + +static +class CBZip2CrcTableInit +{ +public: + CBZip2CrcTableInit() { CBZip2Crc::InitTable(); } +} g_BZip2CrcTableInit; diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Crc.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Crc.h new file mode 100644 index 0000000..10e147b --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Crc.h @@ -0,0 +1,31 @@ +// BZip2Crc.h + +#ifndef ZIP7_INC_BZIP2_CRC_H +#define ZIP7_INC_BZIP2_CRC_H + +#include "../../Common/MyTypes.h" + +class CBZip2Crc +{ + UInt32 _value; + static UInt32 Table[256]; +public: + static void InitTable(); + CBZip2Crc(UInt32 initVal = 0xFFFFFFFF): _value(initVal) {} + void Init(UInt32 initVal = 0xFFFFFFFF) { _value = initVal; } + void UpdateByte(Byte b) { _value = Table[(_value >> 24) ^ b] ^ (_value << 8); } + void UpdateByte(unsigned b) { _value = Table[(_value >> 24) ^ b] ^ (_value << 8); } + UInt32 GetDigest() const { return _value ^ 0xFFFFFFFF; } +}; + +class CBZip2CombinedCrc +{ + UInt32 _value; +public: + CBZip2CombinedCrc(): _value(0) {} + void Init() { _value = 0; } + void Update(UInt32 v) { _value = ((_value << 1) | (_value >> 31)) ^ v; } + UInt32 GetDigest() const { return _value ; } +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Decoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Decoder.cpp new file mode 100644 index 0000000..f44faa2 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Decoder.cpp @@ -0,0 +1,1777 @@ +// BZip2Decoder.cpp + +#include "StdAfx.h" + +// #include "CopyCoder.h" + +/* +#include +#include "../../../C/CpuTicks.h" +*/ +#define TICKS_START +#define TICKS_UPDATE(n) + + +/* +#define PRIN(s) printf(s "\n"); fflush(stdout); +#define PRIN_VAL(s, val) printf(s " = %u \n", val); fflush(stdout); +*/ + +#define PRIN(s) +#define PRIN_VAL(s, val) + + +#include "../../../C/Alloc.h" + +#include "../Common/StreamUtils.h" + +#include "BZip2Decoder.h" + + +namespace NCompress { +namespace NBZip2 { + +// #undef NO_INLINE +#define NO_INLINE Z7_NO_INLINE + +#define BZIP2_BYTE_MODE + + +static const UInt32 kInBufSize = (UInt32)1 << 17; +static const size_t kOutBufSize = (size_t)1 << 20; + +static const UInt32 kProgressStep = (UInt32)1 << 16; + +MY_ALIGN(64) +static const UInt16 kRandNums[512] = { + 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, + 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, + 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, + 419, 436, 278, 496, 867, 210, 399, 680, 480, 51, + 878, 465, 811, 169, 869, 675, 611, 697, 867, 561, + 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, + 150, 238, 59, 379, 684, 877, 625, 169, 643, 105, + 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, + 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, + 909, 545, 703, 919, 874, 474, 882, 500, 594, 612, + 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, + 161, 604, 958, 533, 221, 400, 386, 867, 600, 782, + 382, 596, 414, 171, 516, 375, 682, 485, 911, 276, + 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, + 227, 730, 475, 186, 263, 647, 537, 686, 600, 224, + 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, + 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, + 715, 67, 618, 276, 204, 918, 873, 777, 604, 560, + 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, + 652, 934, 970, 447, 318, 353, 859, 672, 112, 785, + 645, 863, 803, 350, 139, 93, 354, 99, 820, 908, + 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, + 653, 282, 762, 623, 680, 81, 927, 626, 789, 125, + 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, + 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, + 857, 956, 358, 619, 580, 124, 737, 594, 701, 612, + 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, + 944, 375, 748, 52, 600, 747, 642, 182, 862, 81, + 344, 805, 988, 739, 511, 655, 814, 334, 249, 515, + 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, + 433, 837, 553, 268, 926, 240, 102, 654, 459, 51, + 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, + 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, + 978, 321, 576, 617, 626, 502, 894, 679, 243, 440, + 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, + 707, 151, 457, 449, 797, 195, 791, 558, 945, 679, + 297, 59, 87, 824, 713, 663, 412, 693, 342, 606, + 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, + 343, 97, 430, 751, 497, 314, 983, 374, 822, 928, + 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, + 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, + 369, 970, 294, 750, 807, 827, 150, 790, 288, 923, + 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, + 896, 831, 547, 261, 524, 462, 293, 465, 502, 56, + 661, 821, 976, 991, 658, 869, 905, 758, 745, 193, + 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, + 61, 688, 793, 644, 986, 403, 106, 366, 905, 644, + 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, + 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, + 920, 176, 193, 713, 857, 265, 203, 50, 668, 108, + 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, + 936, 638 +}; + + + +enum EState +{ + STATE_STREAM_SIGNATURE, + STATE_BLOCK_SIGNATURE, + + STATE_BLOCK_START, + STATE_ORIG_BITS, + STATE_IN_USE, + STATE_IN_USE2, + STATE_NUM_TABLES, + STATE_NUM_SELECTORS, + STATE_SELECTORS, + STATE_LEVELS, + + STATE_BLOCK_SYMBOLS, + + STATE_STREAM_FINISHED +}; + + +#define UPDATE_VAL_2(val, num_bits) { \ + val |= (UInt32)(*_buf) << (24 - num_bits); \ + num_bits += 8; \ + _buf++; \ +} + +#define UPDATE_VAL UPDATE_VAL_2(VAL, NUM_BITS) + +#define READ_BITS(res, num) { \ + while (_numBits < num) { \ + if (_buf == _lim) return SZ_OK; \ + UPDATE_VAL_2(_value, _numBits) } \ + res = _value >> (32 - num); \ + _value <<= num; \ + _numBits -= num; \ +} + +#define READ_BITS_8(res, num) { \ + if (_numBits < num) { \ + if (_buf == _lim) return SZ_OK; \ + UPDATE_VAL_2(_value, _numBits) } \ + res = _value >> (32 - num); \ + _value <<= num; \ + _numBits -= num; \ +} + +#define READ_BIT(res) READ_BITS_8(res, 1) + + + +#define VAL _value2 +// #define NUM_BITS _numBits2 +#define NUM_BITS _numBits +#define BLOCK_SIZE blockSize2 +#define RUN_COUNTER runCounter2 + +#define LOAD_LOCAL \ + UInt32 VAL = this->_value; \ + /* unsigned NUM_BITS = this->_numBits; */ \ + UInt32 BLOCK_SIZE = this->blockSize; \ + UInt32 RUN_COUNTER = this->runCounter; \ + +#define SAVE_LOCAL \ + this->_value = VAL; \ + /* this->_numBits = NUM_BITS; */ \ + this->blockSize = BLOCK_SIZE; \ + this->runCounter = RUN_COUNTER; \ + + + +SRes CBitDecoder::ReadByte(int &b) +{ + b = -1; + READ_BITS_8(b, 8) + return SZ_OK; +} + + +NO_INLINE +SRes CBase::ReadStreamSignature2() +{ + for (;;) + { + unsigned b; + READ_BITS_8(b, 8) + + if ( (state2 == 0 && b != kArSig0) + || (state2 == 1 && b != kArSig1) + || (state2 == 2 && b != kArSig2) + || (state2 == 3 && (b <= kArSig3 || b > kArSig3 + kBlockSizeMultMax))) + return SZ_ERROR_DATA; + state2++; + + if (state2 == 4) + { + blockSizeMax = (UInt32)(b - kArSig3) * kBlockSizeStep; + CombinedCrc.Init(); + state = STATE_BLOCK_SIGNATURE; + state2 = 0; + return SZ_OK; + } + } +} + + +bool IsEndSig(const Byte *p) throw() +{ + return + p[0] == kFinSig0 && + p[1] == kFinSig1 && + p[2] == kFinSig2 && + p[3] == kFinSig3 && + p[4] == kFinSig4 && + p[5] == kFinSig5; +} + +bool IsBlockSig(const Byte *p) throw() +{ + return + p[0] == kBlockSig0 && + p[1] == kBlockSig1 && + p[2] == kBlockSig2 && + p[3] == kBlockSig3 && + p[4] == kBlockSig4 && + p[5] == kBlockSig5; +} + + +NO_INLINE +SRes CBase::ReadBlockSignature2() +{ + while (state2 < 10) + { + unsigned b; + READ_BITS_8(b, 8) + temp[state2] = (Byte)b; + state2++; + } + + crc = 0; + for (unsigned i = 0; i < 4; i++) + { + crc <<= 8; + crc |= temp[6 + i]; + } + + if (IsBlockSig(temp)) + { + if (!IsBz) + NumStreams++; + NumBlocks++; + IsBz = true; + CombinedCrc.Update(crc); + state = STATE_BLOCK_START; + return SZ_OK; + } + + if (!IsEndSig(temp)) + return SZ_ERROR_DATA; + + if (!IsBz) + NumStreams++; + IsBz = true; + + if (_value != 0) + MinorError = true; + + AlignToByte(); + + state = STATE_STREAM_FINISHED; + if (crc != CombinedCrc.GetDigest()) + { + StreamCrcError = true; + return SZ_ERROR_DATA; + } + return SZ_OK; +} + + +NO_INLINE +SRes CBase::ReadBlock2() +{ + if (state != STATE_BLOCK_SYMBOLS) { + PRIN("ReadBlock2") + + if (state == STATE_BLOCK_START) + { + if (Props.randMode) + { + READ_BIT(Props.randMode) + } + state = STATE_ORIG_BITS; + // g_Tick = GetCpuTicks(); + } + + if (state == STATE_ORIG_BITS) + { + READ_BITS(Props.origPtr, kNumOrigBits) + if (Props.origPtr >= blockSizeMax) + return SZ_ERROR_DATA; + state = STATE_IN_USE; + } + + // why original code compares origPtr to (UInt32)(10 + blockSizeMax)) ? + + if (state == STATE_IN_USE) + { + READ_BITS(state2, 16) + state = STATE_IN_USE2; + state3 = 0; + numInUse = 0; + mtf.StartInit(); + } + + if (state == STATE_IN_USE2) + { + for (; state3 < 256; state3++) + if (state2 & ((UInt32)0x8000 >> (state3 >> 4))) + { + unsigned b; + READ_BIT(b) + if (b) + mtf.Add(numInUse++, (Byte)state3); + } + if (numInUse == 0) + return SZ_ERROR_DATA; + state = STATE_NUM_TABLES; + } + + + if (state == STATE_NUM_TABLES) + { + READ_BITS_8(numTables, kNumTablesBits) + state = STATE_NUM_SELECTORS; + if (numTables < kNumTablesMin || numTables > kNumTablesMax) + return SZ_ERROR_DATA; + } + + if (state == STATE_NUM_SELECTORS) + { + READ_BITS(numSelectors, kNumSelectorsBits) + state = STATE_SELECTORS; + state2 = 0x543210; + state3 = 0; + state4 = 0; + // lbzip2 can write small number of additional selectors, + // 20.01: we allow big number of selectors here like bzip2-1.0.8 + if (numSelectors == 0 + // || numSelectors > kNumSelectorsMax_Decoder + ) + return SZ_ERROR_DATA; + } + + if (state == STATE_SELECTORS) + { + const unsigned kMtfBits = 4; + const UInt32 kMtfMask = (1 << kMtfBits) - 1; + do + { + for (;;) + { + unsigned b; + READ_BIT(b) + if (!b) + break; + if (++state4 >= numTables) + return SZ_ERROR_DATA; + } + const UInt32 tmp = (state2 >> (kMtfBits * state4)) & kMtfMask; + const UInt32 mask = ((UInt32)1 << ((state4 + 1) * kMtfBits)) - 1; + state4 = 0; + state2 = ((state2 << kMtfBits) & mask) | (state2 & ~mask) | tmp; + // 20.01: here we keep compatibility with bzip2-1.0.8 decoder: + if (state3 < kNumSelectorsMax) + selectors[state3] = (Byte)tmp; + } + while (++state3 < numSelectors); + + // we allowed additional dummy selector records filled above to support lbzip2's archives. + // but we still don't allow to use these additional dummy selectors in the code bellow + // bzip2 1.0.8 decoder also has similar restriction. + + if (numSelectors > kNumSelectorsMax) + numSelectors = kNumSelectorsMax; + + state = STATE_LEVELS; + state2 = 0; + state3 = 0; + } + + if (state == STATE_LEVELS) + { + do + { + if (state3 == 0) + { + READ_BITS_8(state3, kNumLevelsBits) + state4 = 0; + state5 = 0; + } + const unsigned alphaSize = numInUse + 2; + for (; state4 < alphaSize; state4++) + { + for (;;) + { + if (state3 < 1 || state3 > kMaxHuffmanLen) + return SZ_ERROR_DATA; + + if (state5 == 0) + { + unsigned b; + READ_BIT(b) + if (!b) + break; + } + + state5 = 1; + unsigned b; + READ_BIT(b) + + state5 = 0; + state3++; + state3 -= (b << 1); + } + lens[state4] = (Byte)state3; + state5 = 0; + } + + // 19.03: we use non-full Build() to support lbzip2 archives. + // lbzip2 2.5 can produce dummy tree, where lens[i] = kMaxHuffmanLen + for (unsigned i = state4; i < kMaxAlphaSize; i++) + lens[i] = 0; + if (!huffs[state2].Build(lens)) // k_BuildMode_Partial + return SZ_ERROR_DATA; + state3 = 0; + } + while (++state2 < numTables); + + { + UInt32 *counters = this->Counters; + for (unsigned i = 0; i < 256; i++) + counters[i] = 0; + } + + state = STATE_BLOCK_SYMBOLS; + + groupIndex = 0; + groupSize = kGroupSize; + runPower = 0; + runCounter = 0; + blockSize = 0; + } + + if (state != STATE_BLOCK_SYMBOLS) + return SZ_ERROR_DATA; + + // g_Ticks[3] += GetCpuTicks() - g_Tick; + + } + + { + LOAD_LOCAL + const CHuffmanDecoder *huf = &huffs[selectors[groupIndex]]; + + for (;;) + { + if (groupSize == 0) + { + if (++groupIndex >= numSelectors) + return SZ_ERROR_DATA; + huf = &huffs[selectors[groupIndex]]; + groupSize = kGroupSize; + } + + if (NUM_BITS < kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL + if (NUM_BITS < kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL + if (NUM_BITS < kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL }}} + + unsigned sym; + + #define MOV_POS(bs, len) \ + { \ + if (NUM_BITS < len) \ + { \ + SAVE_LOCAL \ + return SZ_OK; \ + } \ + VAL <<= len; \ + NUM_BITS -= (unsigned)len; \ + } + + Z7_HUFF_DECODE_VAL_IN_HIGH32(sym, huf, kMaxHuffmanLen, kNumTableBits, + VAL, + Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES, + { return SZ_ERROR_DATA; }, + MOV_POS, {}, bs) + + groupSize--; + + if (sym < 2) + { + RUN_COUNTER += (UInt32)(sym + 1) << runPower; + runPower++; + if (blockSizeMax - BLOCK_SIZE < RUN_COUNTER) + return SZ_ERROR_DATA; + continue; + } + + UInt32 *counters = this->Counters; + if (RUN_COUNTER != 0) + { + UInt32 b = (UInt32)(mtf.Buf[0] & 0xFF); + counters[b] += RUN_COUNTER; + runPower = 0; + #ifdef BZIP2_BYTE_MODE + Byte *dest = (Byte *)(&counters[256 + kBlockSizeMax]) + BLOCK_SIZE; + const Byte *limit = dest + RUN_COUNTER; + BLOCK_SIZE += RUN_COUNTER; + RUN_COUNTER = 0; + do + { + dest[0] = (Byte)b; + dest[1] = (Byte)b; + dest[2] = (Byte)b; + dest[3] = (Byte)b; + dest += 4; + } + while (dest < limit); + #else + UInt32 *dest = &counters[256 + BLOCK_SIZE]; + const UInt32 *limit = dest + RUN_COUNTER; + BLOCK_SIZE += RUN_COUNTER; + RUN_COUNTER = 0; + do + { + dest[0] = b; + dest[1] = b; + dest[2] = b; + dest[3] = b; + dest += 4; + } + while (dest < limit); + #endif + } + + sym -= 1; + if (sym < numInUse) + { + if (BLOCK_SIZE >= blockSizeMax) + return SZ_ERROR_DATA; + + // UInt32 b = (UInt32)mtf.GetAndMove((unsigned)sym); + + const unsigned lim = sym >> Z7_MTF_MOVS; + const unsigned pos = (sym & Z7_MTF_MASK) << 3; + CMtfVar next = mtf.Buf[lim]; + CMtfVar prev = (next >> pos) & 0xFF; + + #ifdef BZIP2_BYTE_MODE + ((Byte *)(counters + 256 + kBlockSizeMax))[BLOCK_SIZE++] = (Byte)prev; + #else + (counters + 256)[BLOCK_SIZE++] = (UInt32)prev; + #endif + counters[prev]++; + + CMtfVar *m = mtf.Buf; + CMtfVar *mLim = m + lim; + if (lim != 0) + { + do + { + CMtfVar n0 = *m; + *m = (n0 << 8) | prev; + prev = (n0 >> (Z7_MTF_MASK << 3)); + } + while (++m != mLim); + } + + CMtfVar mask = (((CMtfVar)0x100 << pos) - 1); + *mLim = (next & ~mask) | (((next << 8) | prev) & mask); + continue; + } + + if (sym != numInUse) + return SZ_ERROR_DATA; + break; + } + + // we write additional item that will be read in DecodeBlock1 for prefetching + #ifdef BZIP2_BYTE_MODE + ((Byte *)(Counters + 256 + kBlockSizeMax))[BLOCK_SIZE] = 0; + #else + (counters + 256)[BLOCK_SIZE] = 0; + #endif + + SAVE_LOCAL + Props.blockSize = blockSize; + state = STATE_BLOCK_SIGNATURE; + state2 = 0; + + PRIN_VAL("origPtr", Props.origPtr); + PRIN_VAL("blockSize", Props.blockSize); + + return (Props.origPtr < Props.blockSize) ? SZ_OK : SZ_ERROR_DATA; + } +} + + +NO_INLINE +static void DecodeBlock1(UInt32 *counters, UInt32 blockSize) +{ + { + UInt32 sum = 0; + for (UInt32 i = 0; i < 256; i++) + { + const UInt32 v = counters[i]; + counters[i] = sum; + sum += v; + } + } + + UInt32 *tt = counters + 256; + // Compute the T^(-1) vector + + // blockSize--; + + #ifdef BZIP2_BYTE_MODE + + unsigned c = ((const Byte *)(tt + kBlockSizeMax))[0]; + + for (UInt32 i = 0; i < blockSize; i++) + { + unsigned c1 = c; + const UInt32 pos = counters[c]; + c = ((const Byte *)(tt + kBlockSizeMax))[(size_t)i + 1]; + counters[c1] = pos + 1; + tt[pos] = (i << 8) | ((const Byte *)(tt + kBlockSizeMax))[pos]; + } + + /* + // last iteration without next character prefetching + { + const UInt32 pos = counters[c]; + counters[c] = pos + 1; + tt[pos] = (blockSize << 8) | ((const Byte *)(tt + kBlockSizeMax))[pos]; + } + */ + + #else + + unsigned c = (unsigned)(tt[0] & 0xFF); + + for (UInt32 i = 0; i < blockSize; i++) + { + unsigned c1 = c; + const UInt32 pos = counters[c]; + c = (unsigned)(tt[(size_t)i + 1] & 0xFF); + counters[c1] = pos + 1; + tt[pos] |= (i << 8); + } + + /* + { + const UInt32 pos = counters[c]; + counters[c] = pos + 1; + tt[pos] |= (blockSize << 8); + } + */ + + #endif + + + /* + for (UInt32 i = 0; i < blockSize; i++) + { + #ifdef BZIP2_BYTE_MODE + const unsigned c = ((const Byte *)(tt + kBlockSizeMax))[i]; + const UInt32 pos = counters[c]++; + tt[pos] = (i << 8) | ((const Byte *)(tt + kBlockSizeMax))[pos]; + #else + const unsigned c = (unsigned)(tt[i] & 0xFF); + const UInt32 pos = counters[c]++; + tt[pos] |= (i << 8); + #endif + } + */ +} + + +void CSpecState::Init(UInt32 origPtr, unsigned randMode) throw() +{ + _tPos = _tt[_tt[origPtr] >> 8]; + _prevByte = (unsigned)(_tPos & 0xFF); + _reps = 0; + _randIndex = 0; + _randToGo = -1; + if (randMode) + { + _randIndex = 1; + _randToGo = kRandNums[0] - 2; + } + _crc.Init(); +} + + + +NO_INLINE +Byte * CSpecState::Decode(Byte *data, size_t size) throw() +{ + if (size == 0) + return data; + + unsigned prevByte = _prevByte; + int reps = _reps; + CBZip2Crc crc = _crc; + const Byte *lim = data + size; + + while (reps > 0) + { + reps--; + *data++ = (Byte)prevByte; + crc.UpdateByte(prevByte); + if (data == lim) + break; + } + + UInt32 tPos = _tPos; + UInt32 blockSize = _blockSize; + const UInt32 *tt = _tt; + + if (data != lim && blockSize) + + for (;;) + { + unsigned b = (unsigned)(tPos & 0xFF); + tPos = tt[tPos >> 8]; + blockSize--; + + if (_randToGo >= 0) + { + if (_randToGo == 0) + { + b ^= 1; + _randToGo = kRandNums[_randIndex]; + _randIndex++; + _randIndex &= 0x1FF; + } + _randToGo--; + } + + if (reps != -(int)kRleModeRepSize) + { + if (b != prevByte) + reps = 0; + reps--; + prevByte = b; + *data++ = (Byte)b; + crc.UpdateByte(b); + if (data == lim || blockSize == 0) + break; + continue; + } + + reps = (int)b; + while (reps) + { + reps--; + *data++ = (Byte)prevByte; + crc.UpdateByte(prevByte); + if (data == lim) + break; + } + if (data == lim) + break; + if (blockSize == 0) + break; + } + + if (blockSize == 1 && reps == -(int)kRleModeRepSize) + { + unsigned b = (unsigned)(tPos & 0xFF); + tPos = tt[tPos >> 8]; + blockSize--; + + if (_randToGo >= 0) + { + if (_randToGo == 0) + { + b ^= 1; + _randToGo = kRandNums[_randIndex]; + _randIndex++; + _randIndex &= 0x1FF; + } + _randToGo--; + } + + reps = (int)b; + } + + _tPos = tPos; + _prevByte = prevByte; + _reps = reps; + _crc = crc; + _blockSize = blockSize; + + return data; +} + + +HRESULT CDecoder::Flush() +{ + if (_writeRes == S_OK) + { + _writeRes = WriteStream(_outStream, _outBuf, _outPos); + _outWritten += _outPos; + _outPos = 0; + } + return _writeRes; +} + + +NO_INLINE +HRESULT CDecoder::DecodeBlock(const CBlockProps &props) +{ + _calcedBlockCrc = 0; + _blockFinished = false; + + CSpecState block; + + block._blockSize = props.blockSize; + block._tt = _counters + 256; + + block.Init(props.origPtr, props.randMode); + + for (;;) + { + Byte *data = _outBuf + _outPos; + size_t size = kOutBufSize - _outPos; + + if (_outSizeDefined) + { + const UInt64 rem = _outSize - _outPosTotal; + if (size >= rem) + { + size = (size_t)rem; + if (size == 0) + return FinishMode ? S_FALSE : S_OK; + } + } + + TICKS_START + const size_t processed = (size_t)(block.Decode(data, size) - data); + TICKS_UPDATE(2) + + _outPosTotal += processed; + _outPos += processed; + + if (processed >= size) + { + RINOK(Flush()) + } + + if (block.Finished()) + { + _blockFinished = true; + _calcedBlockCrc = block._crc.GetDigest(); + return S_OK; + } + } +} + + +CDecoder::CDecoder(): + _outBuf(NULL), + FinishMode(false), + _outSizeDefined(false), + _counters(NULL), + _inBuf(NULL), + _inProcessed(0) +{ + #ifndef Z7_ST + MtMode = false; + NeedWaitScout = false; + // ScoutRes = S_OK; + #endif +} + + +CDecoder::~CDecoder() +{ + PRIN("\n~CDecoder()"); + + #ifndef Z7_ST + + if (Thread.IsCreated()) + { + WaitScout(); + + _block.StopScout = true; + + PRIN("\nScoutEvent.Set()"); + ScoutEvent.Set(); + + PRIN("\nThread.Wait()()"); + Thread.Wait_Close(); + PRIN("\n after Thread.Wait()()"); + + // if (ScoutRes != S_OK) throw ScoutRes; + } + + #endif + + BigFree(_counters); + MidFree(_outBuf); + MidFree(_inBuf); +} + + +HRESULT CDecoder::ReadInput() +{ + if (Base._buf != Base._lim || _inputFinished || _inputRes != S_OK) + return _inputRes; + + _inProcessed += (size_t)(Base._buf - _inBuf); + Base._buf = _inBuf; + Base._lim = _inBuf; + UInt32 size = 0; + _inputRes = Base.InStream->Read(_inBuf, kInBufSize, &size); + _inputFinished = (size == 0); + Base._lim = _inBuf + size; + return _inputRes; +} + + +void CDecoder::StartNewStream() +{ + Base.state = STATE_STREAM_SIGNATURE; + Base.state2 = 0; + Base.IsBz = false; +} + + +HRESULT CDecoder::ReadStreamSignature() +{ + for (;;) + { + RINOK(ReadInput()) + SRes res = Base.ReadStreamSignature2(); + if (res != SZ_OK) + return S_FALSE; + if (Base.state == STATE_BLOCK_SIGNATURE) + return S_OK; + if (_inputFinished) + { + Base.NeedMoreInput = true; + return S_FALSE; + } + } +} + + +HRESULT CDecoder::StartRead() +{ + StartNewStream(); + return ReadStreamSignature(); +} + + +HRESULT CDecoder::ReadBlockSignature() +{ + for (;;) + { + RINOK(ReadInput()) + + SRes res = Base.ReadBlockSignature2(); + + if (Base.state == STATE_STREAM_FINISHED) + Base.FinishedPackSize = GetInputProcessedSize(); + if (res != SZ_OK) + return S_FALSE; + if (Base.state != STATE_BLOCK_SIGNATURE) + return S_OK; + if (_inputFinished) + { + Base.NeedMoreInput = true; + return S_FALSE; + } + } +} + + +HRESULT CDecoder::ReadBlock() +{ + for (;;) + { + RINOK(ReadInput()) + + SRes res = Base.ReadBlock2(); + + if (res != SZ_OK) + return S_FALSE; + if (Base.state == STATE_BLOCK_SIGNATURE) + return S_OK; + if (_inputFinished) + { + Base.NeedMoreInput = true; + return S_FALSE; + } + } +} + + + +HRESULT CDecoder::DecodeStreams(ICompressProgressInfo *progress) +{ + { + #ifndef Z7_ST + _block.StopScout = false; + #endif + } + + RINOK(StartRead()) + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + + { + #ifndef Z7_ST + CWaitScout_Releaser waitScout_Releaser(this); + + bool useMt = false; + #endif + + bool wasFinished = false; + + UInt32 crc = 0; + UInt32 nextCrc = 0; + HRESULT nextRes = S_OK; + + UInt64 packPos = 0; + + CBlockProps props; + + props.blockSize = 0; + + for (;;) + { + if (progress) + { + const UInt64 outCur = GetOutProcessedSize(); + if (packPos - inPrev >= kProgressStep || outCur - outPrev >= kProgressStep) + { + RINOK(progress->SetRatioInfo(&packPos, &outCur)) + inPrev = packPos; + outPrev = outCur; + } + } + + if (props.blockSize == 0) + if (wasFinished || nextRes != S_OK) + return nextRes; + + if ( + #ifndef Z7_ST + !useMt && + #endif + !wasFinished && Base.state == STATE_BLOCK_SIGNATURE) + { + nextRes = ReadBlockSignature(); + nextCrc = Base.crc; + packPos = GetInputProcessedSize(); + + wasFinished = true; + + if (nextRes != S_OK) + continue; + + if (Base.state == STATE_STREAM_FINISHED) + { + if (!Base.DecodeAllStreams) + { + wasFinished = true; + continue; + } + + nextRes = StartRead(); + + if (Base.NeedMoreInput) + { + if (Base.state2 == 0) + Base.NeedMoreInput = false; + wasFinished = true; + nextRes = S_OK; + continue; + } + + if (nextRes != S_OK) + continue; + + wasFinished = false; + continue; + } + + wasFinished = false; + + #ifndef Z7_ST + if (MtMode) + if (props.blockSize != 0) + { + // we start multithreading, if next block is big enough. + const UInt32 k_Mt_BlockSize_Threshold = (1 << 12); // (1 << 13) + if (props.blockSize > k_Mt_BlockSize_Threshold) + { + if (!Thread.IsCreated()) + { + PRIN("=== MT_MODE"); + RINOK(CreateThread()) + } + useMt = true; + } + } + #endif + } + + if (props.blockSize == 0) + { + crc = nextCrc; + + #ifndef Z7_ST + if (useMt) + { + PRIN("DecoderEvent.Lock()"); + { + WRes wres = DecoderEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + NeedWaitScout = false; + PRIN("-- DecoderEvent.Lock()"); + props = _block.Props; + nextCrc = _block.NextCrc; + if (_block.Crc_Defined) + crc = _block.Crc; + packPos = _block.PackPos; + wasFinished = _block.WasFinished; + RINOK(_block.Res) + } + else + #endif + { + if (Base.state != STATE_BLOCK_START) + return E_FAIL; + + TICKS_START + Base.Props.randMode = 1; + RINOK(ReadBlock()) + TICKS_UPDATE(0) + + props = Base.Props; + continue; + } + } + + if (props.blockSize != 0) + { + TICKS_START + DecodeBlock1(_counters, props.blockSize); + TICKS_UPDATE(1) + } + + #ifndef Z7_ST + if (useMt && !wasFinished) + { + /* + if (props.blockSize == 0) + { + // this codes switches back to single-threadMode + useMt = false; + PRIN("=== ST_MODE"); + continue; + } + */ + + PRIN("ScoutEvent.Set()"); + { + WRes wres = ScoutEvent.Set(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + NeedWaitScout = true; + } + #endif + + if (props.blockSize == 0) + continue; + + RINOK(DecodeBlock(props)) + + if (!_blockFinished) + return nextRes; + + props.blockSize = 0; + if (_calcedBlockCrc != crc) + { + BlockCrcError = true; + return S_FALSE; + } + } + } +} + + + + +bool CDecoder::CreateInputBufer() +{ + if (!_inBuf) + { + _inBuf = (Byte *)MidAlloc(kInBufSize); + if (!_inBuf) + return false; + Base._buf = _inBuf; + Base._lim = _inBuf; + } + if (!_counters) + { + const size_t size = (256 + kBlockSizeMax) * sizeof(UInt32) + #ifdef BZIP2_BYTE_MODE + + kBlockSizeMax + #endif + + 256; + _counters = (UInt32 *)::BigAlloc(size); + if (!_counters) + return false; + Base.Counters = _counters; + } + return true; +} + + +void CDecoder::InitOutSize(const UInt64 *outSize) +{ + _outPosTotal = 0; + + _outSizeDefined = false; + _outSize = 0; + if (outSize) + { + _outSize = *outSize; + _outSizeDefined = true; + } + + BlockCrcError = false; + + Base.InitNumStreams2(); +} + + +Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + /* + { + RINOK(SetInStream(inStream)); + RINOK(SetOutStreamSize(outSize)); + + RINOK(CopyStream(this, outStream, progress)); + return ReleaseInStream(); + } + */ + + _inputFinished = false; + _inputRes = S_OK; + _writeRes = S_OK; + + try { + + InitOutSize(outSize); + + // we can request data from InputBuffer after Code(). + // so we init InputBuffer before any function return. + + InitInputBuffer(); + + if (!CreateInputBufer()) + return E_OUTOFMEMORY; + + if (!_outBuf) + { + _outBuf = (Byte *)MidAlloc(kOutBufSize); + if (!_outBuf) + return E_OUTOFMEMORY; + } + + Base.InStream = inStream; + + // InitInputBuffer(); + + _outStream = outStream; + _outWritten = 0; + _outPos = 0; + + HRESULT res = DecodeStreams(progress); + + Flush(); + + Base.InStream = NULL; + _outStream = NULL; + + /* + if (res == S_OK) + if (FinishMode && inSize && *inSize != GetInputProcessedSize()) + res = S_FALSE; + */ + + if (res != S_OK) + return res; + + } catch(...) { return E_FAIL; } + + return _writeRes; +} + + +Z7_COM7F_IMF(CDecoder::SetFinishMode(UInt32 finishMode)) +{ + FinishMode = (finishMode != 0); + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::GetInStreamProcessedSize(UInt64 *value)) +{ + *value = GetInStreamSize(); + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::ReadUnusedFromInBuf(void *data, UInt32 size, UInt32 *processedSize)) +{ + Base.AlignToByte(); + UInt32 i; + for (i = 0; i < size; i++) + { + int b; + Base.ReadByte(b); + if (b < 0) + break; + ((Byte *)data)[i] = (Byte)b; + } + if (processedSize) + *processedSize = i; + return S_OK; +} + + +#ifndef Z7_ST + +#define PRIN_MT(s) PRIN(" " s) + +// #define RINOK_THREAD(x) { WRes __result_ = (x); if (__result_ != 0) return __result_; } + +static THREAD_FUNC_DECL RunScout2(void *p) { ((CDecoder *)p)->RunScout(); return 0; } + +HRESULT CDecoder::CreateThread() +{ + WRes wres = DecoderEvent.CreateIfNotCreated_Reset(); + if (wres == 0) { wres = ScoutEvent.CreateIfNotCreated_Reset(); + if (wres == 0) { wres = Thread.Create(RunScout2, this); }} + return HRESULT_FROM_WIN32(wres); +} + +void CDecoder::RunScout() +{ + for (;;) + { + { + PRIN_MT("ScoutEvent.Lock()") + WRes wres = ScoutEvent.Lock(); + PRIN_MT("-- ScoutEvent.Lock()") + if (wres != 0) + { + // ScoutRes = wres; + return; + } + } + + CBlock &block = _block; + + if (block.StopScout) + { + // ScoutRes = S_OK; + return; + } + + block.Res = S_OK; + block.WasFinished = false; + + HRESULT res = S_OK; + + try + { + UInt64 packPos = GetInputProcessedSize(); + + block.Props.blockSize = 0; + block.Crc_Defined = false; + // block.NextCrc_Defined = false; + block.NextCrc = 0; + + for (;;) + { + if (Base.state == STATE_BLOCK_SIGNATURE) + { + res = ReadBlockSignature(); + + if (res != S_OK) + break; + + if (block.Props.blockSize == 0) + { + block.Crc = Base.crc; + block.Crc_Defined = true; + } + else + { + block.NextCrc = Base.crc; + // block.NextCrc_Defined = true; + } + + continue; + } + + if (Base.state == STATE_BLOCK_START) + { + if (block.Props.blockSize != 0) + break; + + Base.Props.randMode = 1; + + res = ReadBlock(); + + PRIN_MT("-- Base.ReadBlock") + if (res != S_OK) + break; + block.Props = Base.Props; + continue; + } + + if (Base.state == STATE_STREAM_FINISHED) + { + if (!Base.DecodeAllStreams) + { + block.WasFinished = true; + break; + } + + res = StartRead(); + + if (Base.NeedMoreInput) + { + if (Base.state2 == 0) + Base.NeedMoreInput = false; + block.WasFinished = true; + res = S_OK; + break; + } + + if (res != S_OK) + break; + + if (GetInputProcessedSize() - packPos > 0) // kProgressStep + break; + continue; + } + + // throw 1; + res = E_FAIL; + break; + } + } + + catch (...) { res = E_FAIL; } + + if (res != S_OK) + { + PRIN_MT("error") + block.Res = res; + block.WasFinished = true; + } + + block.PackPos = GetInputProcessedSize(); + PRIN_MT("DecoderEvent.Set()") + WRes wres = DecoderEvent.Set(); + if (wres != 0) + { + // ScoutRes = wres; + return; + } + } +} + + +Z7_COM7F_IMF(CDecoder::SetNumberOfThreads(UInt32 numThreads)) +{ + MtMode = (numThreads > 1); + + #ifndef BZIP2_BYTE_MODE + MtMode = false; + #endif + + // MtMode = false; + return S_OK; +} + +#endif + + + +#ifndef Z7_NO_READ_FROM_CODER + + +Z7_COM7F_IMF(CDecoder::SetInStream(ISequentialInStream *inStream)) +{ + Base.InStreamRef = inStream; + Base.InStream = inStream; + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::ReleaseInStream()) +{ + Base.InStreamRef.Release(); + Base.InStream = NULL; + return S_OK; +} + + + +Z7_COM7F_IMF(CDecoder::SetOutStreamSize(const UInt64 *outSize)) +{ + InitOutSize(outSize); + + InitInputBuffer(); + + if (!CreateInputBufer()) + return E_OUTOFMEMORY; + + // InitInputBuffer(); + + StartNewStream(); + + _blockFinished = true; + + ErrorResult = S_OK; + + _inputFinished = false; + _inputRes = S_OK; + + return S_OK; +} + + + +Z7_COM7F_IMF(CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + *processedSize = 0; + + try { + + if (ErrorResult != S_OK) + return ErrorResult; + + for (;;) + { + if (Base.state == STATE_STREAM_FINISHED) + { + if (!Base.DecodeAllStreams) + return ErrorResult; + StartNewStream(); + continue; + } + + if (Base.state == STATE_STREAM_SIGNATURE) + { + ErrorResult = ReadStreamSignature(); + + if (Base.NeedMoreInput) + if (Base.state2 == 0 && Base.NumStreams != 0) + { + Base.NeedMoreInput = false; + ErrorResult = S_OK; + return S_OK; + } + if (ErrorResult != S_OK) + return ErrorResult; + continue; + } + + if (_blockFinished && Base.state == STATE_BLOCK_SIGNATURE) + { + ErrorResult = ReadBlockSignature(); + + if (ErrorResult != S_OK) + return ErrorResult; + + continue; + } + + if (_outSizeDefined) + { + const UInt64 rem = _outSize - _outPosTotal; + if (size >= rem) + size = (UInt32)rem; + } + if (size == 0) + return S_OK; + + if (_blockFinished) + { + if (Base.state != STATE_BLOCK_START) + { + ErrorResult = E_FAIL; + return ErrorResult; + } + + Base.Props.randMode = 1; + ErrorResult = ReadBlock(); + + if (ErrorResult != S_OK) + return ErrorResult; + + DecodeBlock1(_counters, Base.Props.blockSize); + + _spec._blockSize = Base.Props.blockSize; + _spec._tt = _counters + 256; + _spec.Init(Base.Props.origPtr, Base.Props.randMode); + + _blockFinished = false; + } + + { + Byte *ptr = _spec.Decode((Byte *)data, size); + + const UInt32 processed = (UInt32)(ptr - (Byte *)data); + data = ptr; + size -= processed; + (*processedSize) += processed; + _outPosTotal += processed; + + if (_spec.Finished()) + { + _blockFinished = true; + if (Base.crc != _spec._crc.GetDigest()) + { + BlockCrcError = true; + ErrorResult = S_FALSE; + return ErrorResult; + } + } + } + } + + } catch(...) { ErrorResult = S_FALSE; return S_FALSE; } +} + + + +// ---------- NSIS ---------- + +Z7_COM7F_IMF(CNsisDecoder::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + *processedSize = 0; + + try { + + if (ErrorResult != S_OK) + return ErrorResult; + + if (Base.state == STATE_STREAM_FINISHED) + return S_OK; + + if (Base.state == STATE_STREAM_SIGNATURE) + { + Base.blockSizeMax = 9 * kBlockSizeStep; + Base.state = STATE_BLOCK_SIGNATURE; + // Base.state2 = 0; + } + + for (;;) + { + if (_blockFinished && Base.state == STATE_BLOCK_SIGNATURE) + { + ErrorResult = ReadInput(); + if (ErrorResult != S_OK) + return ErrorResult; + + int b; + Base.ReadByte(b); + if (b < 0) + { + ErrorResult = S_FALSE; + return ErrorResult; + } + + if (b == kFinSig0) + { + /* + if (!Base.AreRemainByteBitsEmpty()) + ErrorResult = S_FALSE; + */ + Base.state = STATE_STREAM_FINISHED; + return ErrorResult; + } + + if (b != kBlockSig0) + { + ErrorResult = S_FALSE; + return ErrorResult; + } + + Base.state = STATE_BLOCK_START; + } + + if (_outSizeDefined) + { + const UInt64 rem = _outSize - _outPosTotal; + if (size >= rem) + size = (UInt32)rem; + } + if (size == 0) + return S_OK; + + if (_blockFinished) + { + if (Base.state != STATE_BLOCK_START) + { + ErrorResult = E_FAIL; + return ErrorResult; + } + + Base.Props.randMode = 0; + ErrorResult = ReadBlock(); + + if (ErrorResult != S_OK) + return ErrorResult; + + DecodeBlock1(_counters, Base.Props.blockSize); + + _spec._blockSize = Base.Props.blockSize; + _spec._tt = _counters + 256; + _spec.Init(Base.Props.origPtr, Base.Props.randMode); + + _blockFinished = false; + } + + { + Byte *ptr = _spec.Decode((Byte *)data, size); + + const UInt32 processed = (UInt32)(ptr - (Byte *)data); + data = ptr; + size -= processed; + (*processedSize) += processed; + _outPosTotal += processed; + + if (_spec.Finished()) + _blockFinished = true; + } + } + + } catch(...) { ErrorResult = S_FALSE; return S_FALSE; } +} + +#endif + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Decoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Decoder.h new file mode 100644 index 0000000..4adc564 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Decoder.h @@ -0,0 +1,388 @@ +// Compress/BZip2Decoder.h + +#ifndef ZIP7_INC_COMPRESS_BZIP2_DECODER_H +#define ZIP7_INC_COMPRESS_BZIP2_DECODER_H + +#include "../../Common/MyCom.h" + +// #define Z7_NO_READ_FROM_CODER +// #define Z7_ST + +#ifndef Z7_ST +#include "../../Windows/Synchronization.h" +#include "../../Windows/Thread.h" +#endif + +#include "../ICoder.h" + +#include "BZip2Const.h" +#include "BZip2Crc.h" +#include "HuffmanDecoder.h" +#include "Mtf8.h" + +namespace NCompress { +namespace NBZip2 { + +bool IsEndSig(const Byte *p) throw(); +bool IsBlockSig(const Byte *p) throw(); + +const unsigned kNumTableBits = 9; + +typedef NHuffman::CDecoder CHuffmanDecoder; + + +struct CBlockProps +{ + UInt32 blockSize; + UInt32 origPtr; + unsigned randMode; + + CBlockProps(): blockSize(0), origPtr(0), randMode(0) {} +}; + + +struct CBitDecoder +{ + unsigned _numBits; + UInt32 _value; + const Byte *_buf; + const Byte *_lim; + + void InitBitDecoder() + { + _numBits = 0; + _value = 0; + } + + void AlignToByte() + { + unsigned bits = _numBits & 7; + _numBits -= bits; + _value <<= bits; + } + + /* + bool AreRemainByteBitsEmpty() const + { + unsigned bits = _numBits & 7; + if (bits != 0) + return (_value >> (32 - bits)) == 0; + return true; + } + */ + + SRes ReadByte(int &b); + + CBitDecoder(): + _buf(NULL), + _lim(NULL) + { + InitBitDecoder(); + } +}; + + +// 19.03: we allow additional 8 selectors to support files created by lbzip2. +const UInt32 kNumSelectorsMax_Decoder = kNumSelectorsMax + 8; + +struct CBase: public CBitDecoder +{ + unsigned numInUse; + UInt32 groupIndex; + UInt32 groupSize; + unsigned runPower; + UInt32 runCounter; + UInt32 blockSize; + + UInt32 *Counters; + UInt32 blockSizeMax; + + unsigned state; + UInt32 state2; + unsigned state3; + unsigned state4; + unsigned state5; + unsigned numTables; + UInt32 numSelectors; + + CBlockProps Props; + +private: + CMtf8Decoder mtf; + Byte selectors[kNumSelectorsMax_Decoder]; + CHuffmanDecoder huffs[kNumTablesMax]; + + Byte lens[kMaxAlphaSize]; + + Byte temp[10]; + +public: + UInt32 crc; + CBZip2CombinedCrc CombinedCrc; + + bool IsBz; + bool StreamCrcError; + bool MinorError; + bool NeedMoreInput; + + bool DecodeAllStreams; + + UInt64 NumStreams; + UInt64 NumBlocks; + UInt64 FinishedPackSize; + + ISequentialInStream *InStream; + + #ifndef Z7_NO_READ_FROM_CODER + CMyComPtr InStreamRef; + #endif + + CBase(): + StreamCrcError(false), + MinorError(false), + NeedMoreInput(false), + + DecodeAllStreams(false), + + NumStreams(0), + NumBlocks(0), + FinishedPackSize(0) + {} + + void InitNumStreams2() + { + StreamCrcError = false; + MinorError = false; + NeedMoreInput = 0; + NumStreams = 0; + NumBlocks = 0; + FinishedPackSize = 0; + } + + SRes ReadStreamSignature2(); + SRes ReadBlockSignature2(); + + /* ReadBlock2() : Props->randMode: + in: need read randMode bit + out: randMode status */ + SRes ReadBlock2(); +}; + + +class CSpecState +{ + UInt32 _tPos; + unsigned _prevByte; + int _reps; + +public: + CBZip2Crc _crc; + UInt32 _blockSize; + UInt32 *_tt; + + int _randToGo; + unsigned _randIndex; + + void Init(UInt32 origPtr, unsigned randMode) throw(); + + bool Finished() const { return _reps <= 0 && _blockSize == 0; } + + Byte *Decode(Byte *data, size_t size) throw(); +}; + + + + +class CDecoder: + public ICompressCoder, + public ICompressSetFinishMode, + public ICompressGetInStreamProcessedSize, + public ICompressReadUnusedFromInBuf, +#ifndef Z7_NO_READ_FROM_CODER + public ICompressSetInStream, + public ICompressSetOutStreamSize, + public ISequentialInStream, +#endif +#ifndef Z7_ST + public ICompressSetCoderMt, +#endif + public CMyUnknownImp +{ + Z7_COM_QI_BEGIN2(ICompressCoder) + Z7_COM_QI_ENTRY(ICompressSetFinishMode) + Z7_COM_QI_ENTRY(ICompressGetInStreamProcessedSize) + Z7_COM_QI_ENTRY(ICompressReadUnusedFromInBuf) +#ifndef Z7_NO_READ_FROM_CODER + Z7_COM_QI_ENTRY(ICompressSetInStream) + Z7_COM_QI_ENTRY(ICompressSetOutStreamSize) + Z7_COM_QI_ENTRY(ISequentialInStream) +#endif +#ifndef Z7_ST + Z7_COM_QI_ENTRY(ICompressSetCoderMt) +#endif + Z7_COM_QI_END + Z7_COM_ADDREF_RELEASE + + Z7_IFACE_COM7_IMP(ICompressCoder) + Z7_IFACE_COM7_IMP(ICompressSetFinishMode) + Z7_IFACE_COM7_IMP(ICompressGetInStreamProcessedSize) + Z7_IFACE_COM7_IMP(ICompressReadUnusedFromInBuf) +#ifndef Z7_NO_READ_FROM_CODER + Z7_IFACE_COM7_IMP(ICompressSetInStream) + Z7_IFACE_COM7_IMP(ICompressSetOutStreamSize) + Z7_IFACE_COM7_IMP_NONFINAL(ISequentialInStream) +#endif +public: +#ifndef Z7_ST + Z7_IFACE_COM7_IMP(ICompressSetCoderMt) +#endif + +private: + Byte *_outBuf; + size_t _outPos; + UInt64 _outWritten; + ISequentialOutStream *_outStream; + HRESULT _writeRes; + +protected: + HRESULT ErrorResult; // for ISequentialInStream::Read mode only + +public: + + UInt32 _calcedBlockCrc; + bool _blockFinished; + bool BlockCrcError; + + bool FinishMode; + bool _outSizeDefined; + UInt64 _outSize; + UInt64 _outPosTotal; + + CSpecState _spec; + UInt32 *_counters; + + #ifndef Z7_ST + + struct CBlock + { + bool StopScout; + + bool WasFinished; + bool Crc_Defined; + // bool NextCrc_Defined; + + UInt32 Crc; + UInt32 NextCrc; + HRESULT Res; + UInt64 PackPos; + + CBlockProps Props; + }; + + CBlock _block; + + bool NeedWaitScout; + bool MtMode; + + NWindows::CThread Thread; + NWindows::NSynchronization::CAutoResetEvent DecoderEvent; + NWindows::NSynchronization::CAutoResetEvent ScoutEvent; + // HRESULT ScoutRes; + + Byte MtPad[1 << 7]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size. + + + void RunScout(); + + void WaitScout() + { + if (NeedWaitScout) + { + DecoderEvent.Lock(); + NeedWaitScout = false; + } + } + + class CWaitScout_Releaser + { + CDecoder *_decoder; + public: + CWaitScout_Releaser(CDecoder *decoder): _decoder(decoder) {} + ~CWaitScout_Releaser() { _decoder->WaitScout(); } + }; + + HRESULT CreateThread(); + + #endif + + Byte *_inBuf; + UInt64 _inProcessed; + bool _inputFinished; + HRESULT _inputRes; + + CBase Base; + + bool GetCrcError() const { return BlockCrcError || Base.StreamCrcError; } + + void InitOutSize(const UInt64 *outSize); + + bool CreateInputBufer(); + + void InitInputBuffer() + { + // We use InitInputBuffer() before stream init. + // So don't read from stream here + _inProcessed = 0; + Base._buf = _inBuf; + Base._lim = _inBuf; + Base.InitBitDecoder(); + } + + UInt64 GetInputProcessedSize() const + { + // for NSIS case : we need also look the number of bits in bitDecoder + return _inProcessed + (size_t)(Base._buf - _inBuf); + } + + UInt64 GetInStreamSize() const + { + return _inProcessed + (size_t)(Base._buf - _inBuf) - (Base._numBits >> 3); + } + + UInt64 GetOutProcessedSize() const { return _outWritten + _outPos; } + + HRESULT ReadInput(); + + void StartNewStream(); + + HRESULT ReadStreamSignature(); + HRESULT StartRead(); + + HRESULT ReadBlockSignature(); + HRESULT ReadBlock(); + + HRESULT Flush(); + HRESULT DecodeBlock(const CBlockProps &props); + HRESULT DecodeStreams(ICompressProgressInfo *progress); + + UInt64 GetNumStreams() const { return Base.NumStreams; } + UInt64 GetNumBlocks() const { return Base.NumBlocks; } + + CDecoder(); + virtual ~CDecoder(); +}; + + + +#ifndef Z7_NO_READ_FROM_CODER + +class CNsisDecoder Z7_final: public CDecoder +{ + Z7_IFACE_COM7_IMP(ISequentialInStream) +}; + +#endif + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Encoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Encoder.cpp new file mode 100644 index 0000000..af0b312 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Encoder.cpp @@ -0,0 +1,1116 @@ +// BZip2Encoder.cpp + +#include "StdAfx.h" + +#include "../../../C/Alloc.h" +#include "../../../C/BwtSort.h" +#include "../../../C/HuffEnc.h" + +#include "BZip2Encoder.h" + +namespace NCompress { +namespace NBZip2 { + +#define HUFFMAN_LEN 16 +#if HUFFMAN_LEN > Z7_HUFFMAN_LEN_MAX + #error Stop_Compiling_Bad_HUFFMAN_LEN_BZip2Encoder +#endif + +static const size_t kBufferSize = 1 << 17; +static const unsigned kNumHuffPasses = 4; + + +bool CThreadInfo::Alloc() +{ + if (!m_BlockSorterIndex) + { + m_BlockSorterIndex = (UInt32 *)::BigAlloc(BLOCK_SORT_BUF_SIZE(kBlockSizeMax) * sizeof(UInt32)); + if (!m_BlockSorterIndex) + return false; + } + + if (!m_Block_Base) + { + const unsigned kPadSize = 1 << 7; // we need at least 1 byte backward padding, becuase we use (m_Block - 1) pointer; + m_Block_Base = (Byte *)::MidAlloc(kBlockSizeMax * 5 + + kBlockSizeMax / 10 + (20 << 10) + + kPadSize); + if (!m_Block_Base) + return false; + m_Block = m_Block_Base + kPadSize; + m_MtfArray = m_Block + kBlockSizeMax; + m_TempArray = m_MtfArray + kBlockSizeMax * 2 + 2; + } + return true; +} + +void CThreadInfo::Free() +{ + ::BigFree(m_BlockSorterIndex); + m_BlockSorterIndex = NULL; + ::MidFree(m_Block_Base); + m_Block_Base = NULL; +} + +#ifndef Z7_ST + +static THREAD_FUNC_DECL MFThread(void *threadCoderInfo) +{ + return ((CThreadInfo *)threadCoderInfo)->ThreadFunc(); +} + +HRESULT CThreadInfo::Create() +{ + WRes wres = StreamWasFinishedEvent.Create(); + if (wres == 0) { wres = WaitingWasStartedEvent.Create(); + if (wres == 0) { wres = CanWriteEvent.Create(); + if (wres == 0) + { + wres = +#ifdef _WIN32 + Encoder->_props.NumThreadGroups > 1 ? + Thread.Create_With_Group(MFThread, this, ThreadNextGroup_GetNext(&Encoder->ThreadNextGroup), 0) : // affinity +#endif + Encoder->_props.Affinity != 0 ? + Thread.Create_With_Affinity(MFThread, this, (CAffinityMask)Encoder->_props.Affinity) : + Thread.Create(MFThread, this); + }}} + return HRESULT_FROM_WIN32(wres); +} + +void CThreadInfo::FinishStream(bool needLeave) +{ + Encoder->StreamWasFinished = true; + StreamWasFinishedEvent.Set(); + if (needLeave) + Encoder->CS.Leave(); + Encoder->CanStartWaitingEvent.Lock(); + WaitingWasStartedEvent.Set(); +} + +THREAD_FUNC_RET_TYPE CThreadInfo::ThreadFunc() +{ + for (;;) + { + Encoder->CanProcessEvent.Lock(); + Encoder->CS.Enter(); + if (Encoder->CloseThreads) + { + Encoder->CS.Leave(); + return 0; + } + if (Encoder->StreamWasFinished) + { + FinishStream(true); + continue; + } + HRESULT res = S_OK; + bool needLeave = true; + try + { + const UInt32 blockSize = Encoder->ReadRleBlock(m_Block); + m_UnpackSize = Encoder->m_InStream.GetProcessedSize(); + m_BlockIndex = Encoder->NextBlockIndex; + if (++Encoder->NextBlockIndex == Encoder->NumThreads) + Encoder->NextBlockIndex = 0; + if (blockSize == 0) + { + FinishStream(true); + continue; + } + Encoder->CS.Leave(); + needLeave = false; + res = EncodeBlock3(blockSize); + } + catch(const CInBufferException &e) { res = e.ErrorCode; } + catch(const COutBufferException &e) { res = e.ErrorCode; } + catch(...) { res = E_FAIL; } + if (res != S_OK) + { + Encoder->Result = res; + FinishStream(needLeave); + continue; + } + } +} + +#endif + +void CEncProps::Normalize(int level) +{ + if (level < 0) level = 5; + if (level > 9) level = 9; + + if (NumPasses == (UInt32)(Int32)-1) + NumPasses = (level >= 9 ? 7 : (level >= 7 ? 2 : 1)); + if (NumPasses < 1) NumPasses = 1; + if (NumPasses > kNumPassesMax) NumPasses = kNumPassesMax; + + if (BlockSizeMult == (UInt32)(Int32)-1) + BlockSizeMult = (level >= 5 ? 9 : (level >= 1 ? (unsigned)level * 2 - 1: 1)); + if (BlockSizeMult < kBlockSizeMultMin) BlockSizeMult = kBlockSizeMultMin; + if (BlockSizeMult > kBlockSizeMultMax) BlockSizeMult = kBlockSizeMultMax; +} + +CEncoder::CEncoder() +{ + _props.Normalize(-1); + + #ifndef Z7_ST + ThreadsInfo = NULL; + m_NumThreadsPrev = 0; + NumThreads = 1; + #endif +} + +#ifndef Z7_ST +CEncoder::~CEncoder() +{ + Free(); +} + +HRESULT CEncoder::Create() +{ + { + WRes wres = CanProcessEvent.CreateIfNotCreated_Reset(); + if (wres == 0) { wres = CanStartWaitingEvent.CreateIfNotCreated_Reset(); } + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + + if (ThreadsInfo && m_NumThreadsPrev == NumThreads) + return S_OK; + try + { + Free(); + MtMode = (NumThreads > 1); + m_NumThreadsPrev = NumThreads; + ThreadsInfo = new CThreadInfo[NumThreads]; + if (!ThreadsInfo) + return E_OUTOFMEMORY; + } + catch(...) { return E_OUTOFMEMORY; } + for (UInt32 t = 0; t < NumThreads; t++) + { + CThreadInfo &ti = ThreadsInfo[t]; + ti.Encoder = this; + if (MtMode) + { + HRESULT res = ti.Create(); + if (res != S_OK) + { + NumThreads = t; + Free(); + return res; + } + } + } + return S_OK; +} + +void CEncoder::Free() +{ + if (!ThreadsInfo) + return; + CloseThreads = true; + CanProcessEvent.Set(); + for (UInt32 t = 0; t < NumThreads; t++) + { + CThreadInfo &ti = ThreadsInfo[t]; + if (MtMode) + ti.Thread.Wait_Close(); + ti.Free(); + } + delete []ThreadsInfo; + ThreadsInfo = NULL; +} +#endif + +struct CRleEncoder +{ + const Byte *_src; + const Byte *_srcLim; + Byte *_dest; + const Byte *_destLim; + Byte _prevByte; + unsigned _numReps; + + void Encode(); +}; + +Z7_NO_INLINE +void CRleEncoder::Encode() +{ + const Byte *src = _src; + const Byte * const srcLim = _srcLim; + Byte *dest = _dest; + const Byte * const destLim = _destLim; + Byte prev = _prevByte; + unsigned numReps = _numReps; + // (dest < destLim) + // src = srcLim; // for debug + while (dest < destLim) + { + if (src == srcLim) + break; + const Byte b = *src++; + if (b != prev) + { + if (numReps >= kRleModeRepSize) + *dest++ = (Byte)(numReps - kRleModeRepSize); + *dest++ = b; + numReps = 1; + prev = b; + /* + { // speed optimization code: + if (dest >= destLim || src == srcLim) + break; + const Byte b2 = *src++; + *dest++ = b2; + numReps += (prev == b2); + prev = b2; + } + */ + continue; + } + numReps++; + if (numReps <= kRleModeRepSize) + *dest++ = b; + else if (numReps == kRleModeRepSize + 255) + { + *dest++ = (Byte)(numReps - kRleModeRepSize); + numReps = 0; + } + } + _src = src; + _dest = dest; + _prevByte = prev; + _numReps = numReps; + // (dest <= destLim + 1) +} + + +// out: return value is blockSize: size of data filled in buffer[]: +// (returned_blockSize <= _props.BlockSizeMult * kBlockSizeStep) +UInt32 CEncoder::ReadRleBlock(Byte *buffer) +{ + CRleEncoder rle; + UInt32 i = 0; + if (m_InStream.ReadByte(rle._prevByte)) + { + NumBlocks++; + const UInt32 blockSize = _props.BlockSizeMult * kBlockSizeStep - 1; // -1 for RLE + rle._destLim = buffer + blockSize; + rle._numReps = 1; + buffer[i++] = rle._prevByte; + while (i < blockSize) + { + rle._dest = buffer + i; + size_t rem; + const Byte * const ptr = m_InStream.Lookahead(rem); + if (rem == 0) + break; + rle._src = ptr; + rle._srcLim = ptr + rem; + rle.Encode(); + m_InStream.Skip((size_t)(rle._src - ptr)); + i = (UInt32)(size_t)(rle._dest - buffer); + // (i <= blockSize + 1) + } + const int n = (int)rle._numReps - (int)kRleModeRepSize; + if (n >= 0) + buffer[i++] = (Byte)n; + } + return i; +} + + + +Z7_NO_INLINE +void CThreadInfo::WriteBits2(UInt32 value, unsigned numBits) + { m_OutStreamCurrent.WriteBits(value, numBits); } +/* +Z7_NO_INLINE +void CThreadInfo::WriteByte2(unsigned b) + { m_OutStreamCurrent.WriteByte(b); } +*/ +// void CEncoder::WriteBits(UInt32 value, unsigned numBits) { m_OutStream.WriteBits(value, numBits); } +Z7_NO_INLINE +void CEncoder::WriteByte(Byte b) { m_OutStream.WriteByte(b); } + + +#define WRITE_BITS_UPDATE(value, numBits) \ +{ \ + numBits -= _bitPos; \ + const UInt32 hi = value >> numBits; \ + *_buf++ = (Byte)(_curByte | hi); \ + value -= hi << numBits; \ + _bitPos = 8; \ + _curByte = 0; \ +} + +#if HUFFMAN_LEN > 16 + +#define WRITE_BITS_HUFF(value2, numBits2) \ +{ \ + UInt32 value = value2; \ + unsigned numBits = numBits2; \ + while (numBits >= _bitPos) { \ + WRITE_BITS_UPDATE(value, numBits) \ + } \ + _bitPos -= numBits; \ + _curByte |= (value << _bitPos); \ +} + +#else // HUFFMAN_LEN <= 16 + +// numBits2 <= 16 is supported +#define WRITE_BITS_HUFF(value2, numBits2) \ +{ \ + UInt32 value = value2; \ + unsigned numBits = numBits2; \ + if (numBits >= _bitPos) \ + { \ + WRITE_BITS_UPDATE(value, numBits) \ + if (numBits >= _bitPos) \ + { \ + numBits -= _bitPos; \ + const UInt32 hi = value >> numBits; \ + *_buf++ = (Byte)hi; \ + value -= hi << numBits; \ + } \ + } \ + _bitPos -= numBits; \ + _curByte |= (value << _bitPos); \ +} + +#endif + +#define WRITE_BITS_8(value2, numBits2) \ +{ \ + UInt32 value = value2; \ + unsigned numBits = numBits2; \ + if (numBits >= _bitPos) \ + { \ + WRITE_BITS_UPDATE(value, numBits) \ + } \ + _bitPos -= numBits; \ + _curByte |= (value << _bitPos); \ +} + +#define WRITE_BIT_PRE \ + { _bitPos--; } + +#define WRITE_BIT_POST \ +{ \ + if (_bitPos == 0) \ + { \ + *_buf++ = (Byte)_curByte; \ + _curByte = 0; \ + _bitPos = 8; \ + } \ +} + +#define WRITE_BIT_0 \ +{ \ + WRITE_BIT_PRE \ + WRITE_BIT_POST \ +} + +#define WRITE_BIT_1 \ +{ \ + WRITE_BIT_PRE \ + _curByte |= 1u << _bitPos; \ + WRITE_BIT_POST \ +} + + +// blockSize > 0 +void CThreadInfo::EncodeBlock(const Byte *block, UInt32 blockSize) +{ + // WriteBit2(0); // Randomised = false + { + const UInt32 origPtr = BlockSort(m_BlockSorterIndex, block, blockSize); + // if (m_BlockSorterIndex[origPtr] != 0) throw 1; + m_BlockSorterIndex[origPtr] = blockSize; + WriteBits2(origPtr, kNumOrigBits + 1); // + 1 for additional high bit flag (Randomised = false) + } + Byte mtfBuf[256]; + // memset(mtfBuf, 0, sizeof(mtfBuf)); // to disable MSVC warning + unsigned numInUse; + { + Byte inUse[256]; + Byte inUse16[16]; + unsigned i; + for (i = 0; i < 256; i++) + inUse[i] = 0; + for (i = 0; i < 16; i++) + inUse16[i] = 0; + { + const Byte * cur = block; + block = block + (size_t)blockSize - 1; + if (cur != block) + { + do + { + const unsigned b0 = cur[0]; + const unsigned b1 = cur[1]; + cur += 2; + inUse[b0] = 1; + inUse[b1] = 1; + } + while (cur < block); + } + if (cur == block) + inUse[cur[0]] = 1; + block -= blockSize; // block pointer is (original_block - 1) + } + numInUse = 0; + for (i = 0; i < 256; i++) + if (inUse[i]) + { + inUse16[i >> 4] = 1; + mtfBuf[numInUse++] = (Byte)i; + } + for (i = 0; i < 16; i++) + WriteBit2(inUse16[i]); + for (i = 0; i < 256; i++) + if (inUse16[i >> 4]) + WriteBit2(inUse[i]); + } + const unsigned alphaSize = numInUse + 2; + + UInt32 symbolCounts[kMaxAlphaSize]; + { + for (unsigned i = 0; i < kMaxAlphaSize; i++) + symbolCounts[i] = 0; + symbolCounts[(size_t)alphaSize - 1] = 1; + } + + Byte *mtfs = m_MtfArray; + { + const UInt32 *bsIndex = m_BlockSorterIndex; + const UInt32 *bsIndex_rle = bsIndex; + const UInt32 * const bsIndex_end = bsIndex + blockSize; + // block--; // backward fix + // block pointer is (original_block - 1) + do + { + const Byte v = block[*bsIndex++]; + Byte a = mtfBuf[0]; + if (v != a) + { + mtfBuf[0] = v; + { + UInt32 rleSize = (UInt32)(size_t)(bsIndex - bsIndex_rle) - 1; + bsIndex_rle = bsIndex; + while (rleSize) + { + const unsigned sym = (unsigned)(--rleSize & 1); + *mtfs++ = (Byte)sym; + symbolCounts[sym]++; + rleSize >>= 1; + } + } + unsigned pos1 = 2; // = real_pos + 1 + Byte b; + b = mtfBuf[1]; mtfBuf[1] = a; if (v != b) + { a = mtfBuf[2]; mtfBuf[2] = b; if (v == a) pos1 = 3; + else { b = mtfBuf[3]; mtfBuf[3] = a; if (v == b) pos1 = 4; + else + { + Byte *m = mtfBuf + 7; + for (;;) + { + a = m[-3]; m[-3] = b; if (v == a) { pos1 = (unsigned)(size_t)(m - (mtfBuf + 2)); break; } + b = m[-2]; m[-2] = a; if (v == b) { pos1 = (unsigned)(size_t)(m - (mtfBuf + 1)); break; } + a = m[-1]; m[-1] = b; if (v == a) { pos1 = (unsigned)(size_t)(m - (mtfBuf )); break; } + b = m[ 0]; m[ 0] = a; m += 4; if (v == b) { pos1 = (unsigned)(size_t)(m - (mtfBuf + 3)); break; } + } + }}} + symbolCounts[pos1]++; + if (pos1 >= 0xff) + { + *mtfs++ = 0xff; + // pos1 -= 0xff; + pos1++; // we need only low byte + } + *mtfs++ = (Byte)pos1; + } + } + while (bsIndex < bsIndex_end); + + UInt32 rleSize = (UInt32)(size_t)(bsIndex - bsIndex_rle); + while (rleSize) + { + const unsigned sym = (unsigned)(--rleSize & 1); + *mtfs++ = (Byte)sym; + symbolCounts[sym]++; + rleSize >>= 1; + } + + unsigned d = alphaSize - 1; + if (alphaSize >= 256) + { + *mtfs++ = 0xff; + d = alphaSize; // (-256) + } + *mtfs++ = (Byte)d; + } + + const Byte * const mtf_lim = mtfs; + + UInt32 numSymbols = 0; + { + for (unsigned i = 0; i < kMaxAlphaSize; i++) + numSymbols += symbolCounts[i]; + } + + unsigned bestNumTables = kNumTablesMin; + UInt32 bestPrice = 0xFFFFFFFF; + const UInt32 startPos = m_OutStreamCurrent.GetPos(); + const unsigned startCurByte = m_OutStreamCurrent.GetCurByte(); + for (unsigned nt = kNumTablesMin; nt <= kNumTablesMax + 1; nt++) + { + unsigned numTables; + + if (m_OptimizeNumTables) + { + m_OutStreamCurrent.SetPos(startPos); + m_OutStreamCurrent.SetCurState(startPos & 7, startCurByte); + numTables = (nt <= kNumTablesMax ? nt : bestNumTables); + } + else + { + if (numSymbols < 200) numTables = 2; + else if (numSymbols < 600) numTables = 3; + else if (numSymbols < 1200) numTables = 4; + else if (numSymbols < 2400) numTables = 5; + else numTables = 6; + } + + WriteBits2(numTables, kNumTablesBits); + const unsigned numSelectors = (numSymbols + kGroupSize - 1) / kGroupSize; + WriteBits2((UInt32)numSelectors, kNumSelectorsBits); + + { + UInt32 remFreq = numSymbols; + unsigned gs = 0; + unsigned t = numTables; + do + { + UInt32 tFreq = remFreq / t; + unsigned ge = gs; + UInt32 aFreq = 0; + while (aFreq < tFreq) // && ge < alphaSize) + aFreq += symbolCounts[ge++]; + + if (ge > gs + 1 && t != numTables && t != 1 && (((numTables - t) & 1) == 1)) + aFreq -= symbolCounts[--ge]; + + Byte *lens = Lens[(size_t)t - 1]; + unsigned i = 0; + do + lens[i] = (Byte)((i >= gs && i < ge) ? 0 : 1); + while (++i < alphaSize); + gs = ge; + remFreq -= aFreq; + } + while (--t != 0); + } + + + for (unsigned pass = 0; pass < kNumHuffPasses; pass++) + { + memset(Freqs, 0, sizeof(Freqs[0]) * numTables); + // memset(Freqs, 0, sizeof(Freqs)); + { + mtfs = m_MtfArray; + UInt32 g = 0; + do + { + unsigned symbols[kGroupSize]; + unsigned i = 0; + do + { + UInt32 symbol = *mtfs++; + if (symbol >= 0xFF) + symbol += *mtfs++; + symbols[i] = symbol; + } + while (++i < kGroupSize && mtfs < mtf_lim); + + UInt32 bestPrice2 = 0xFFFFFFFF; + unsigned t = 0; + do + { + const Byte *lens = Lens[t]; + UInt32 price = 0; + unsigned j = 0; + do + price += lens[symbols[j]]; + while (++j < i); + if (price < bestPrice2) + { + m_Selectors[g] = (Byte)t; + bestPrice2 = price; + } + } + while (++t < numTables); + UInt32 *freqs = Freqs[m_Selectors[g++]]; + unsigned j = 0; + do + freqs[symbols[j]]++; + while (++j < i); + } + while (mtfs < mtf_lim); + } + + unsigned t = 0; + do + { + UInt32 *freqs = Freqs[t]; + unsigned i = 0; + do + if (freqs[i] == 0) + freqs[i] = 1; + while (++i < alphaSize); + Huffman_Generate(freqs, Codes[t], Lens[t], kMaxAlphaSize, HUFFMAN_LEN); + } + while (++t < numTables); + } + + unsigned _bitPos; // 0 < _bitPos <= 8 : number of non-filled low bits in _curByte + unsigned _curByte; // low (_bitPos) bits are zeros + // high (8 - _bitPos) bits are filled + Byte *_buf; + { + Byte mtfSel[kNumTablesMax]; + { + unsigned t = 0; + do + mtfSel[t] = (Byte)t; + while (++t < numTables); + } + + _bitPos = m_OutStreamCurrent._bitPos; + _curByte = m_OutStreamCurrent._curByte; + _buf = m_OutStreamCurrent._buf; + // stream.Init_from_Global(m_OutStreamCurrent); + + const Byte *selectors = m_Selectors; + const Byte * const selectors_lim = selectors + numSelectors; + Byte prev = 0; // mtfSel[0]; + do + { + const Byte sel = *selectors++; + if (prev != sel) + { + Byte *mtfSel_cur = &mtfSel[1]; + for (;;) + { + WRITE_BIT_1 + const Byte next = *mtfSel_cur; + *mtfSel_cur++ = prev; + prev = next; + if (next == sel) + break; + } + // mtfSel[0] = sel; + } + WRITE_BIT_0 + } + while (selectors != selectors_lim); + } + { + unsigned t = 0; + do + { + const Byte *lens = Lens[t]; + unsigned len = lens[0]; + WRITE_BITS_8(len, kNumLevelsBits) + unsigned i = 0; + do + { + const unsigned level = lens[i]; + while (len != level) + { + WRITE_BIT_1 + if (len < level) + { + len++; + WRITE_BIT_0 + } + else + { + len--; + WRITE_BIT_1 + } + } + WRITE_BIT_0 + } + while (++i < alphaSize); + } + while (++t < numTables); + } + { + UInt32 groupSize = 1; + const Byte *selectors = m_Selectors; + const Byte *lens = NULL; + const UInt32 *codes = NULL; + mtfs = m_MtfArray; + do + { + unsigned symbol = *mtfs++; + if (symbol >= 0xFF) + symbol += *mtfs++; + if (--groupSize == 0) + { + groupSize = kGroupSize; + const unsigned t = *selectors++; + lens = Lens[t]; + codes = Codes[t]; + } + WRITE_BITS_HUFF(codes[symbol], lens[symbol]) + } + while (mtfs < mtf_lim); + } + // Restore_from_Local: + m_OutStreamCurrent._bitPos = _bitPos; + m_OutStreamCurrent._curByte = _curByte; + m_OutStreamCurrent._buf = _buf; + + if (!m_OptimizeNumTables) + break; + const UInt32 price = m_OutStreamCurrent.GetPos() - startPos; + if (price <= bestPrice) + { + if (nt == kNumTablesMax) + break; + bestPrice = price; + bestNumTables = nt; + } + } +} + + +// blockSize > 0 +UInt32 CThreadInfo::EncodeBlockWithHeaders(const Byte *block, UInt32 blockSize) +{ + WriteByte2(kBlockSig0); + WriteByte2(kBlockSig1); + WriteByte2(kBlockSig2); + WriteByte2(kBlockSig3); + WriteByte2(kBlockSig4); + WriteByte2(kBlockSig5); + + CBZip2Crc crc; + const Byte * const lim = block + blockSize; + unsigned b = *block++; + crc.UpdateByte(b); + for (;;) + { + const unsigned prev = b; + if (block >= lim) { break; } b = *block++; crc.UpdateByte(b); if (prev != b) continue; + if (block >= lim) { break; } b = *block++; crc.UpdateByte(b); if (prev != b) continue; + if (block >= lim) { break; } b = *block++; crc.UpdateByte(b); if (prev != b) continue; + if (block >= lim) { break; } b = *block++; if (b) do crc.UpdateByte(prev); while (--b); + if (block >= lim) { break; } b = *block++; crc.UpdateByte(b); + } + const UInt32 crcRes = crc.GetDigest(); + for (int i = 24; i >= 0; i -= 8) + WriteByte2((Byte)(crcRes >> i)); + EncodeBlock(lim - blockSize, blockSize); + return crcRes; +} + + +void CThreadInfo::EncodeBlock2(const Byte *block, UInt32 blockSize, UInt32 numPasses) +{ + const UInt32 numCrcs = m_NumCrcs; + + const UInt32 startBytePos = m_OutStreamCurrent.GetBytePos(); + const UInt32 startPos = m_OutStreamCurrent.GetPos(); + const unsigned startCurByte = m_OutStreamCurrent.GetCurByte(); + unsigned endCurByte = 0; + UInt32 endPos = 0; // 0 means no no additional passes + if (numPasses > 1 && blockSize >= (1 << 10)) + { + UInt32 bs0 = blockSize / 2; + for (; bs0 < blockSize && + (block[ bs0 ] == + block[(size_t)bs0 - 1] || + block[(size_t)bs0 - 1] == + block[(size_t)bs0 - 2]); + bs0++) + {} + + if (bs0 < blockSize) + { + EncodeBlock2(block, bs0, numPasses - 1); + EncodeBlock2(block + bs0, blockSize - bs0, numPasses - 1); + endPos = m_OutStreamCurrent.GetPos(); + endCurByte = m_OutStreamCurrent.GetCurByte(); + // we prepare next byte as identical byte to starting byte for main encoding attempt: + if (endPos & 7) + WriteBits2(0, 8 - (endPos & 7)); + m_OutStreamCurrent.SetCurState((startPos & 7), startCurByte); + } + } + + const UInt32 startBytePos2 = m_OutStreamCurrent.GetBytePos(); + const UInt32 startPos2 = m_OutStreamCurrent.GetPos(); + const UInt32 crcVal = EncodeBlockWithHeaders(block, blockSize); + + if (endPos) + { + const UInt32 size2 = m_OutStreamCurrent.GetPos() - startPos2; + if (size2 >= endPos - startPos) + { + m_OutStreamCurrent.SetPos(endPos); + m_OutStreamCurrent.SetCurState((endPos & 7), endCurByte); + return; + } + const UInt32 numBytes = m_OutStreamCurrent.GetBytePos() - startBytePos2; + Byte * const buffer = m_OutStreamCurrent.GetStream(); + memmove(buffer + startBytePos, buffer + startBytePos2, numBytes); + m_OutStreamCurrent.SetPos(startPos + size2); + // we don't call m_OutStreamCurrent.SetCurState() here because + // m_OutStreamCurrent._curByte is correct already + } + m_CRCs[numCrcs] = crcVal; + m_NumCrcs = numCrcs + 1; +} + + +HRESULT CThreadInfo::EncodeBlock3(UInt32 blockSize) +{ + CMsbfEncoderTemp &outStreamTemp = m_OutStreamCurrent; + outStreamTemp.SetStream(m_TempArray); + outStreamTemp.Init(); + m_NumCrcs = 0; + + EncodeBlock2(m_Block, blockSize, Encoder->_props.NumPasses); + +#ifndef Z7_ST + if (Encoder->MtMode) + Encoder->ThreadsInfo[m_BlockIndex].CanWriteEvent.Lock(); +#endif + + for (UInt32 i = 0; i < m_NumCrcs; i++) + Encoder->CombinedCrc.Update(m_CRCs[i]); + Encoder->WriteBytes(m_TempArray, outStreamTemp.GetPos(), outStreamTemp.GetNonFlushedByteBits()); + HRESULT res = S_OK; + +#ifndef Z7_ST + if (Encoder->MtMode) + { + UInt32 blockIndex = m_BlockIndex + 1; + if (blockIndex == Encoder->NumThreads) + blockIndex = 0; + if (Encoder->Progress) + { + const UInt64 packSize = Encoder->m_OutStream.GetProcessedSize(); + res = Encoder->Progress->SetRatioInfo(&m_UnpackSize, &packSize); + } + Encoder->ThreadsInfo[blockIndex].CanWriteEvent.Set(); + } +#endif + return res; +} + +void CEncoder::WriteBytes(const Byte *data, UInt32 sizeInBits, unsigned lastByteBits) +{ + m_OutStream.WriteBytes(data, sizeInBits >> 3); + sizeInBits &= 7; + if (sizeInBits) + m_OutStream.WriteBits(lastByteBits, sizeInBits); +} + + +HRESULT CEncoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) +{ + NumBlocks = 0; +#ifndef Z7_ST + Progress = progress; + ThreadNextGroup_Init(&ThreadNextGroup, _props.NumThreadGroups, 0); // startGroup + RINOK(Create()) + for (UInt32 t = 0; t < NumThreads; t++) +#endif + { + #ifndef Z7_ST + CThreadInfo &ti = ThreadsInfo[t]; + if (MtMode) + { + WRes wres = ti.StreamWasFinishedEvent.Reset(); + if (wres == 0) { wres = ti.WaitingWasStartedEvent.Reset(); + if (wres == 0) { wres = ti.CanWriteEvent.Reset(); }} + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + #else + CThreadInfo &ti = ThreadsInfo; + ti.Encoder = this; + #endif + + ti.m_OptimizeNumTables = _props.DoOptimizeNumTables(); + + if (!ti.Alloc()) + return E_OUTOFMEMORY; + } + + + if (!m_InStream.Create(kBufferSize)) + return E_OUTOFMEMORY; + if (!m_OutStream.Create(kBufferSize)) + return E_OUTOFMEMORY; + + + m_InStream.SetStream(inStream); + m_InStream.Init(); + + m_OutStream.SetStream(outStream); + m_OutStream.Init(); + + CombinedCrc.Init(); + #ifndef Z7_ST + NextBlockIndex = 0; + StreamWasFinished = false; + CloseThreads = false; + CanStartWaitingEvent.Reset(); + #endif + + WriteByte(kArSig0); + WriteByte(kArSig1); + WriteByte(kArSig2); + WriteByte((Byte)(kArSig3 + _props.BlockSizeMult)); + + #ifndef Z7_ST + + if (MtMode) + { + ThreadsInfo[0].CanWriteEvent.Set(); + Result = S_OK; + CanProcessEvent.Set(); + UInt32 t; + for (t = 0; t < NumThreads; t++) + ThreadsInfo[t].StreamWasFinishedEvent.Lock(); + CanProcessEvent.Reset(); + CanStartWaitingEvent.Set(); + for (t = 0; t < NumThreads; t++) + ThreadsInfo[t].WaitingWasStartedEvent.Lock(); + CanStartWaitingEvent.Reset(); + RINOK(Result) + } + else + #endif + { + for (;;) + { + CThreadInfo &ti = + #ifndef Z7_ST + ThreadsInfo[0]; + #else + ThreadsInfo; + #endif + const UInt32 blockSize = ReadRleBlock(ti.m_Block); + if (blockSize == 0) + break; + RINOK(ti.EncodeBlock3(blockSize)) + if (progress) + { + const UInt64 unpackSize = m_InStream.GetProcessedSize(); + const UInt64 packSize = m_OutStream.GetProcessedSize(); + RINOK(progress->SetRatioInfo(&unpackSize, &packSize)) + } + } + } + WriteByte(kFinSig0); + WriteByte(kFinSig1); + WriteByte(kFinSig2); + WriteByte(kFinSig3); + WriteByte(kFinSig4); + WriteByte(kFinSig5); + { + const UInt32 v = CombinedCrc.GetDigest(); + for (int i = 24; i >= 0; i -= 8) + WriteByte((Byte)(v >> i)); + } + RINOK(Flush()) + if (!m_InStream.WasFinished()) + return E_FAIL; + return S_OK; +} + +Z7_COM7F_IMF(CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + try { return CodeReal(inStream, outStream, inSize, outSize, progress); } + catch(const CInBufferException &e) { return e.ErrorCode; } + catch(const COutBufferException &e) { return e.ErrorCode; } + catch(...) { return S_FALSE; } +} + +Z7_COM7F_IMF(CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps)) +{ + int level = -1; + CEncProps props; + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + const PROPID propID = propIDs[i]; + + if (propID == NCoderPropID::kAffinity) + { + if (prop.vt != VT_UI8) + return E_INVALIDARG; + props.Affinity = prop.uhVal.QuadPart; + continue; + } + + if (propID == NCoderPropID::kNumThreadGroups) + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + props.NumThreadGroups = (UInt32)prop.ulVal; + continue; + } + + if (propID >= NCoderPropID::kReduceSize) + continue; + if (prop.vt != VT_UI4) + return E_INVALIDARG; + const UInt32 v = (UInt32)prop.ulVal; + switch (propID) + { + case NCoderPropID::kNumPasses: props.NumPasses = v; break; + case NCoderPropID::kDictionarySize: props.BlockSizeMult = v / kBlockSizeStep; break; + case NCoderPropID::kLevel: level = (int)v; break; + case NCoderPropID::kNumThreads: + { + #ifndef Z7_ST + SetNumberOfThreads(v); + #endif + break; + } + default: return E_INVALIDARG; + } + } + props.Normalize(level); + _props = props; + return S_OK; +} + +#ifndef Z7_ST +Z7_COM7F_IMF(CEncoder::SetNumberOfThreads(UInt32 numThreads)) +{ + const UInt32 kNumThreadsMax = 64; + if (numThreads < 1) numThreads = 1; + if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; + NumThreads = numThreads; + return S_OK; +} +#endif + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Encoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Encoder.h new file mode 100644 index 0000000..bcb4025 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Encoder.h @@ -0,0 +1,273 @@ +// BZip2Encoder.h + +#ifndef ZIP7_INC_COMPRESS_BZIP2_ENCODER_H +#define ZIP7_INC_COMPRESS_BZIP2_ENCODER_H + +#include "../../Common/MyCom.h" + +#ifndef Z7_ST +#include "../../Windows/Synchronization.h" +#include "../../Windows/Thread.h" +#endif + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" +#include "../Common/OutBuffer.h" + +#include "BitmEncoder.h" +#include "BZip2Const.h" +#include "BZip2Crc.h" + +namespace NCompress { +namespace NBZip2 { + +const unsigned kNumPassesMax = 10; + +struct CMsbfEncoderTemp +{ + unsigned _bitPos; // 0 < _bitPos <= 8 : number of non-filled low bits in _curByte + unsigned _curByte; // low (_bitPos) bits are zeros + // high (8 - _bitPos) bits are filled + Byte *_buf; + Byte *_buf_base; + void SetStream(Byte *buf) { _buf_base = _buf = buf; } + Byte *GetStream() const { return _buf_base; } + + void Init() + { + _bitPos = 8; + _curByte = 0; + _buf = _buf_base; + } + + // required condition: (value >> numBits) == 0 + // numBits == 0 is allowed + void WriteBits(UInt32 value, unsigned numBits) + { + do + { + unsigned bp = _bitPos; + unsigned curByte = _curByte; + if (numBits < bp) + { + bp -= numBits; + _curByte = curByte | (value << bp); + _bitPos = bp; + return; + } + numBits -= bp; + const UInt32 hi = value >> numBits; + value -= (hi << numBits); + Byte *buf = _buf; + _bitPos = 8; + _curByte = 0; + *buf++ = (Byte)(curByte | hi); + _buf = buf; + } + while (numBits); + } + + void WriteBit(unsigned value) + { + const unsigned bp = _bitPos - 1; + const unsigned curByte = _curByte | (value << bp); + _curByte = curByte; + _bitPos = bp; + if (bp == 0) + { + *_buf++ = (Byte)curByte; + _curByte = 0; + _bitPos = 8; + } + } + + void WriteByte(unsigned b) + { + const unsigned bp = _bitPos; + const unsigned a = _curByte | (b >> (8 - bp)); + _curByte = b << bp; + Byte *buf = _buf; + *buf++ = (Byte)a; + _buf = buf; + } + + UInt32 GetBytePos() const { return (UInt32)(size_t)(_buf - _buf_base); } + UInt32 GetPos() const { return GetBytePos() * 8 + 8 - _bitPos; } + unsigned GetCurByte() const { return _curByte; } + unsigned GetNonFlushedByteBits() const { return _curByte >> _bitPos; } + void SetPos(UInt32 bitPos) + { + _buf = _buf_base + (bitPos >> 3); + _bitPos = 8 - ((unsigned)bitPos & 7); + } + void SetCurState(unsigned bitPos, unsigned curByte) + { + _bitPos = 8 - bitPos; + _curByte = curByte; + } +}; + + +class CEncoder; + +class CThreadInfo +{ +private: + CMsbfEncoderTemp m_OutStreamCurrent; +public: + CEncoder *Encoder; + Byte *m_Block; +private: + Byte *m_MtfArray; + Byte *m_TempArray; + UInt32 *m_BlockSorterIndex; + +public: + bool m_OptimizeNumTables; + UInt32 m_NumCrcs; + UInt32 m_BlockIndex; + UInt64 m_UnpackSize; + + Byte *m_Block_Base; + + Byte Lens[kNumTablesMax][kMaxAlphaSize]; + UInt32 Freqs[kNumTablesMax][kMaxAlphaSize]; + UInt32 Codes[kNumTablesMax][kMaxAlphaSize]; + + Byte m_Selectors[kNumSelectorsMax]; + + UInt32 m_CRCs[1 << kNumPassesMax]; + + void WriteBits2(UInt32 value, unsigned numBits); + void WriteByte2(unsigned b) { WriteBits2(b, 8); } + void WriteBit2(unsigned v) { m_OutStreamCurrent.WriteBit(v); } + + void EncodeBlock(const Byte *block, UInt32 blockSize); + UInt32 EncodeBlockWithHeaders(const Byte *block, UInt32 blockSize); + void EncodeBlock2(const Byte *block, UInt32 blockSize, UInt32 numPasses); +public: +#ifndef Z7_ST + NWindows::CThread Thread; + + NWindows::NSynchronization::CAutoResetEvent StreamWasFinishedEvent; + NWindows::NSynchronization::CAutoResetEvent WaitingWasStartedEvent; + + // it's not member of this thread. We just need one event per thread + NWindows::NSynchronization::CAutoResetEvent CanWriteEvent; + +public: + Byte MtPad[1 << 8]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size. + HRESULT Create(); + void FinishStream(bool needLeave); + THREAD_FUNC_RET_TYPE ThreadFunc(); +#endif + + CThreadInfo(): m_BlockSorterIndex(NULL), m_Block_Base(NULL) {} + ~CThreadInfo() { Free(); } + bool Alloc(); + void Free(); + + HRESULT EncodeBlock3(UInt32 blockSize); +}; + + +struct CEncProps +{ + UInt32 BlockSizeMult; + UInt32 NumPasses; + UInt32 NumThreadGroups; + UInt64 Affinity; + + CEncProps() + { + BlockSizeMult = (UInt32)(Int32)-1; + NumPasses = (UInt32)(Int32)-1; + NumThreadGroups = 0; + Affinity = 0; + } + void Normalize(int level); + bool DoOptimizeNumTables() const { return NumPasses > 1; } +}; + +class CEncoder Z7_final: + public ICompressCoder, + public ICompressSetCoderProperties, + #ifndef Z7_ST + public ICompressSetCoderMt, + #endif + public CMyUnknownImp +{ + Z7_COM_QI_BEGIN2(ICompressCoder) + Z7_COM_QI_ENTRY(ICompressSetCoderProperties) + #ifndef Z7_ST + Z7_COM_QI_ENTRY(ICompressSetCoderMt) + #endif + Z7_COM_QI_END + Z7_COM_ADDREF_RELEASE + + Z7_IFACE_COM7_IMP(ICompressCoder) + Z7_IFACE_COM7_IMP(ICompressSetCoderProperties) + #ifndef Z7_ST + Z7_IFACE_COM7_IMP(ICompressSetCoderMt) + #endif + + #ifndef Z7_ST + UInt32 m_NumThreadsPrev; + #endif +public: + CInBuffer m_InStream; + #ifndef Z7_ST + Byte MtPad[1 << 8]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size. + #endif + CBitmEncoder m_OutStream; + CEncProps _props; + CBZip2CombinedCrc CombinedCrc; + + #ifndef Z7_ST + CThreadInfo *ThreadsInfo; + NWindows::NSynchronization::CManualResetEvent CanProcessEvent; + NWindows::NSynchronization::CCriticalSection CS; + UInt32 NumThreads; + bool MtMode; + UInt32 NextBlockIndex; + + bool CloseThreads; + bool StreamWasFinished; + NWindows::NSynchronization::CManualResetEvent CanStartWaitingEvent; + CThreadNextGroup ThreadNextGroup; + + HRESULT Result; + ICompressProgressInfo *Progress; + #else + CThreadInfo ThreadsInfo; + #endif + + UInt64 NumBlocks; + + UInt64 GetInProcessedSize() const { return m_InStream.GetProcessedSize(); } + + UInt32 ReadRleBlock(Byte *buf); + void WriteBytes(const Byte *data, UInt32 sizeInBits, unsigned lastByteBits); + void WriteByte(Byte b); + + #ifndef Z7_ST + HRESULT Create(); + void Free(); + #endif + +public: + CEncoder(); + #ifndef Z7_ST + ~CEncoder(); + #endif + + HRESULT Flush() { return m_OutStream.Flush(); } + + HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Register.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Register.cpp new file mode 100644 index 0000000..83b911d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BZip2Register.cpp @@ -0,0 +1,25 @@ +// BZip2Register.cpp + +#include "StdAfx.h" + +#include "../Common/RegisterCodec.h" + +#include "BZip2Decoder.h" +#if !defined(Z7_EXTRACT_ONLY) && !defined(Z7_BZIP2_EXTRACT_ONLY) +#include "BZip2Encoder.h" +#endif + +namespace NCompress { +namespace NBZip2 { + +REGISTER_CODEC_CREATE(CreateDec, CDecoder) + +#if !defined(Z7_EXTRACT_ONLY) && !defined(Z7_BZIP2_EXTRACT_ONLY) +REGISTER_CODEC_CREATE(CreateEnc, CEncoder) +#else +#define CreateEnc NULL +#endif + +REGISTER_CODEC_2(BZip2, CreateDec, CreateEnc, 0x40202, "BZip2") + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlDecoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlDecoder.cpp new file mode 100644 index 0000000..bf1e5ba --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlDecoder.cpp @@ -0,0 +1,32 @@ +// BitlDecoder.cpp + +#include "StdAfx.h" + +#include "BitlDecoder.h" + +namespace NBitl { + +#if defined(Z7_BITL_USE_REVERSE_BITS_TABLE) + +MY_ALIGN(64) +Byte kReverseTable[256]; + +static +struct CReverseerTableInitializer +{ + CReverseerTableInitializer() + { + for (unsigned i = 0; i < 256; i++) + { + unsigned + x = ((i & 0x55) << 1) | ((i >> 1) & 0x55); + x = ((x & 0x33) << 2) | ((x >> 2) & 0x33); + kReverseTable[i] = (Byte)((x << 4) | (x >> 4)); + } + } +} g_ReverseerTableInitializer; + +#elif 0 +unsigned ReverseBits8test(unsigned i) { return ReverseBits8(i); } +#endif +} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlDecoder.h new file mode 100644 index 0000000..465c4d3 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlDecoder.h @@ -0,0 +1,225 @@ +// BitlDecoder.h -- the Least Significant Bit of byte is First + +#ifndef ZIP7_INC_BITL_DECODER_H +#define ZIP7_INC_BITL_DECODER_H + +#include "../../../C/CpuArch.h" + +#include "../IStream.h" + +namespace NBitl { + +const unsigned kNumBigValueBits = 8 * 4; +const unsigned kNumValueBytes = 3; +const unsigned kNumValueBits = 8 * kNumValueBytes; +const UInt32 kMask = (1 << kNumValueBits) - 1; + +#if !defined(Z7_BITL_USE_REVERSE_BITS_TABLE) +#if 1 && defined(MY_CPU_ARM_OR_ARM64) \ + && (defined(MY_CPU_ARM64) || defined(__ARM_ARCH_6T2__) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 7)) \ + && (defined(__GNUC__) && (__GNUC__ >= 4) \ + || defined(__clang__) && (__clang_major__ >= 4)) + #define Z7_BITL_USE_REVERSE_BITS_INSTRUCTION +#elif 1 + #define Z7_BITL_USE_REVERSE_BITS_TABLE +#endif +#endif + +#if defined(Z7_BITL_USE_REVERSE_BITS_TABLE) +extern Byte kReverseTable[256]; +#endif + +inline unsigned ReverseBits8(unsigned i) +{ +#if defined(Z7_BITL_USE_REVERSE_BITS_TABLE) + return kReverseTable[i]; +#elif defined(Z7_BITL_USE_REVERSE_BITS_INSTRUCTION) + // rbit is available in ARMv6T2 and above + asm ("rbit " +#if defined(MY_CPU_ARM) + "%0,%0" // it uses default register size, + // but we need 32-bit register here. + // we must use it only if default register size is 32-bit. + // it will work incorrectly for ARM64. +#else + "%w0,%w0" // it uses 32-bit registers in ARM64. + // compiler for (MY_CPU_ARM) can't compile it. +#endif + : "+r" (i)); + return i >> 24; +#else + unsigned + x = ((i & 0x55) << 1) | ((i >> 1) & 0x55); + x = ((x & 0x33) << 2) | ((x >> 2) & 0x33); + return ((x & 0x0f) << 4) | (x >> 4); +#endif +} + + +/* TInByte must support "Extra Bytes" (bytes that can be read after the end of stream + TInByte::ReadByte() returns 0xFF after the end of stream + TInByte::NumExtraBytes contains the number "Extra Bytes" + + Bitl decoder can read up to 4 bytes ahead to internal buffer. */ + +template +class CBaseDecoder +{ +protected: + unsigned _bitPos; + UInt32 _value; + TInByte _stream; +public: + bool Create(UInt32 bufSize) { return _stream.Create(bufSize); } + void SetStream(ISequentialInStream *inStream) { _stream.SetStream(inStream); } + void ClearStreamPtr() { _stream.ClearStreamPtr(); } + void Init() + { + _stream.Init(); + _bitPos = kNumBigValueBits; + _value = 0; + } + + // the size of portion data in real stream that was already read from this object. + // it doesn't include unused data in BitStream object buffer (up to 4 bytes) + // it doesn't include unused data in TInByte buffers + // it doesn't include virtual Extra bytes after the end of real stream data + UInt64 GetStreamSize() const + { + return ExtraBitsWereRead() ? + _stream.GetStreamSize(): + GetProcessedSize(); + } + + // the size of virtual data that was read from this object. + UInt64 GetProcessedSize() const { return _stream.GetProcessedSize() - ((kNumBigValueBits - _bitPos) >> 3); } + + bool ThereAreDataInBitsBuffer() const { return this->_bitPos != kNumBigValueBits; } + + Z7_FORCE_INLINE + void Normalize() + { + for (; _bitPos >= 8; _bitPos -= 8) + _value = ((UInt32)_stream.ReadByte() << (kNumBigValueBits - _bitPos)) | _value; + } + + Z7_FORCE_INLINE + UInt32 ReadBits(unsigned numBits) + { + Normalize(); + UInt32 res = _value & ((1 << numBits) - 1); + _bitPos += numBits; + _value >>= numBits; + return res; + } + + bool ExtraBitsWereRead() const + { + return (_stream.NumExtraBytes > 4 || kNumBigValueBits - _bitPos < (_stream.NumExtraBytes << 3)); + } + + bool ExtraBitsWereRead_Fast() const + { + // full version is not inlined in vc6. + // return _stream.NumExtraBytes != 0 && (_stream.NumExtraBytes > 4 || kNumBigValueBits - _bitPos < (_stream.NumExtraBytes << 3)); + + // (_stream.NumExtraBytes > 4) is fast overread detection. It's possible that + // it doesn't return true, if small number of extra bits were read. + return (_stream.NumExtraBytes > 4); + } + + // it must be fixed !!! with extra bits + // UInt32 GetNumExtraBytes() const { return _stream.NumExtraBytes; } +}; + +template +class CDecoder: public CBaseDecoder +{ + UInt32 _normalValue; + +public: + void Init() + { + CBaseDecoder::Init(); + _normalValue = 0; + } + + Z7_FORCE_INLINE + void Normalize() + { + for (; this->_bitPos >= 8; this->_bitPos -= 8) + { + const unsigned b = this->_stream.ReadByte(); + _normalValue = ((UInt32)b << (kNumBigValueBits - this->_bitPos)) | _normalValue; + this->_value = (this->_value << 8) | ReverseBits8(b); + } + } + + Z7_FORCE_INLINE + UInt32 GetValue(unsigned numBits) + { + Normalize(); + return ((this->_value >> (8 - this->_bitPos)) & kMask) >> (kNumValueBits - numBits); + } + + Z7_FORCE_INLINE + UInt32 GetValue_InHigh32bits() + { + Normalize(); + return this->_value << this->_bitPos; + } + + Z7_FORCE_INLINE + void MovePos(size_t numBits) + { + this->_bitPos += (unsigned)numBits; + _normalValue >>= numBits; + } + + Z7_FORCE_INLINE + UInt32 ReadBits(unsigned numBits) + { + Normalize(); + UInt32 res = _normalValue & ((1 << numBits) - 1); + MovePos(numBits); + return res; + } + + void AlignToByte() { MovePos((32 - this->_bitPos) & 7); } + + Z7_FORCE_INLINE + Byte ReadDirectByte() { return this->_stream.ReadByte(); } + + Z7_FORCE_INLINE + size_t ReadDirectBytesPart(Byte *buf, size_t size) { return this->_stream.ReadBytesPart(buf, size); } + + Z7_FORCE_INLINE + Byte ReadAlignedByte() + { + if (this->_bitPos == kNumBigValueBits) + return this->_stream.ReadByte(); + Byte b = (Byte)(_normalValue & 0xFF); + MovePos(8); + return b; + } + + // call it only if the object is aligned for byte. + Z7_FORCE_INLINE + bool ReadAlignedByte_FromBuf(Byte &b) + { + if (this->_stream.NumExtraBytes != 0) + if (this->_stream.NumExtraBytes >= 4 + || kNumBigValueBits - this->_bitPos <= (this->_stream.NumExtraBytes << 3)) + return false; + if (this->_bitPos == kNumBigValueBits) + return this->_stream.ReadByte_FromBuf(b); + b = (Byte)(_normalValue & 0xFF); + MovePos(8); + return true; + } +}; + +} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlEncoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlEncoder.h new file mode 100644 index 0000000..364f84d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitlEncoder.h @@ -0,0 +1,57 @@ +// BitlEncoder.h -- the Least Significant Bit of byte is First + +#ifndef ZIP7_INC_BITL_ENCODER_H +#define ZIP7_INC_BITL_ENCODER_H + +#include "../Common/OutBuffer.h" + +class CBitlEncoder +{ + COutBuffer _stream; + unsigned _bitPos; + Byte _curByte; +public: + bool Create(UInt32 bufSize) { return _stream.Create(bufSize); } + void SetStream(ISequentialOutStream *outStream) { _stream.SetStream(outStream); } + // unsigned GetBitPosition() const { return (8 - _bitPos); } + UInt64 GetProcessedSize() const { return _stream.GetProcessedSize() + ((8 - _bitPos + 7) >> 3); } + void Init() + { + _stream.Init(); + _bitPos = 8; + _curByte = 0; + } + HRESULT Flush() + { + FlushByte(); + return _stream.Flush(); + } + void FlushByte() + { + if (_bitPos < 8) + _stream.WriteByte(_curByte); + _bitPos = 8; + _curByte = 0; + } + Z7_FORCE_INLINE + void WriteBits(UInt32 value, unsigned numBits) + { + while (numBits > 0) + { + if (numBits < _bitPos) + { + _curByte |= (Byte)((value & ((1 << numBits) - 1)) << (8 - _bitPos)); + _bitPos -= numBits; + return; + } + numBits -= _bitPos; + _stream.WriteByte((Byte)(_curByte | (value << (8 - _bitPos)))); + value >>= _bitPos; + _bitPos = 8; + _curByte = 0; + } + } + void WriteByte(Byte b) { _stream.WriteByte(b);} +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitmDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitmDecoder.h new file mode 100644 index 0000000..21c83db --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitmDecoder.h @@ -0,0 +1,106 @@ +// BitmDecoder.h -- the Most Significant Bit of byte is First + +#ifndef ZIP7_INC_BITM_DECODER_H +#define ZIP7_INC_BITM_DECODER_H + +#include "../IStream.h" + +namespace NBitm { + +const unsigned kNumBigValueBits = 8 * 4; +const unsigned kNumValueBytes = 3; +const unsigned kNumValueBits = 8 * kNumValueBytes; + +const UInt32 kMask = (1 << kNumValueBits) - 1; + +// _bitPos - the number of free bits (high bits in _value) +// (kNumBigValueBits - _bitPos) = (32 - _bitPos) == the number of ready to read bits (low bits of _value) + +template +class CDecoder +{ + unsigned _bitPos; + UInt32 _value; + TInByte _stream; +public: + bool Create(UInt32 bufSize) { return _stream.Create(bufSize); } + void SetStream(ISequentialInStream *inStream) { _stream.SetStream(inStream);} + + void Init() + { + _stream.Init(); + _bitPos = kNumBigValueBits; + _value = 0; + Normalize(); + } + + UInt64 GetStreamSize() const { return _stream.GetStreamSize(); } + UInt64 GetProcessedSize() const { return _stream.GetProcessedSize() - ((kNumBigValueBits - _bitPos) >> 3); } + + bool ExtraBitsWereRead() const + { + return (_stream.NumExtraBytes > 4 || kNumBigValueBits - _bitPos < (_stream.NumExtraBytes << 3)); + } + + bool ExtraBitsWereRead_Fast() const + { + return (_stream.NumExtraBytes > 4); + } + + Z7_FORCE_INLINE + void Normalize() + { + for (; _bitPos >= 8; _bitPos -= 8) + _value = (_value << 8) | _stream.ReadByte(); + } + + Z7_FORCE_INLINE + UInt32 GetValue(unsigned numBits) const + { + // return (_value << _bitPos) >> (kNumBigValueBits - numBits); + return ((_value >> (8 - _bitPos)) & kMask) >> (kNumValueBits - numBits); + } + + Z7_FORCE_INLINE + UInt32 GetValue_InHigh32bits() const + { + return this->_value << this->_bitPos; + } + + Z7_FORCE_INLINE + void MovePos(unsigned numBits) + { + _bitPos += numBits; + Normalize(); + } + + Z7_FORCE_INLINE + UInt32 ReadBits(unsigned numBits) + { + UInt32 res = GetValue(numBits); + MovePos(numBits); + return res; + } + + /* + unsigned ReadBit() + { + UInt32 res = ((_value >> (8 - _bitPos)) & kMask) >> (kNumValueBits - 1); + if (++_bitPos >= 8) + { + _value = (_value << 8) | _stream.ReadByte(); + _bitPos -= 8; + } + return (unsigned)res; + } + */ + + void AlignToByte() { MovePos((kNumBigValueBits - _bitPos) & 7); } + + Z7_FORCE_INLINE + UInt32 ReadAlignBits() { return ReadBits((kNumBigValueBits - _bitPos) & 7); } +}; + +} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitmEncoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitmEncoder.h new file mode 100644 index 0000000..f7448cd --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/BitmEncoder.h @@ -0,0 +1,90 @@ +// BitmEncoder.h -- the Most Significant Bit of byte is First + +#ifndef ZIP7_INC_BITM_ENCODER_H +#define ZIP7_INC_BITM_ENCODER_H + +#include "../IStream.h" + +template +class CBitmEncoder +{ + unsigned _bitPos; // 0 < _bitPos <= 8 : number of non-filled low bits in _curByte + unsigned _curByte; // low (_bitPos) bits are zeros + // high (8 - _bitPos) bits are filled + TOutByte _stream; +public: + bool Create(UInt32 bufferSize) { return _stream.Create(bufferSize); } + void SetStream(ISequentialOutStream *outStream) { _stream.SetStream(outStream);} + UInt64 GetProcessedSize() const { return _stream.GetProcessedSize() + ((8 - _bitPos + 7) >> 3); } + void Init() + { + _stream.Init(); + _bitPos = 8; + _curByte = 0; + } + HRESULT Flush() + { + if (_bitPos < 8) + { + _stream.WriteByte((Byte)_curByte); + _bitPos = 8; + _curByte = 0; + } + return _stream.Flush(); + } + + // required condition: (value >> numBits) == 0 + // numBits == 0 is allowed + void WriteBits(UInt32 value, unsigned numBits) + { + do + { + unsigned bp = _bitPos; + unsigned curByte = _curByte; + if (numBits < bp) + { + bp -= numBits; + _curByte = curByte | (value << bp); + _bitPos = bp; + return; + } + numBits -= bp; + const UInt32 hi = (value >> numBits); + value -= (hi << numBits); + _stream.WriteByte((Byte)(curByte | hi)); + _bitPos = 8; + _curByte = 0; + } + while (numBits); + } + void WriteByte(unsigned b) + { + const unsigned bp = _bitPos; + const unsigned a = _curByte | (b >> (8 - bp)); + _curByte = b << bp; + _stream.WriteByte((Byte)a); + } + + void WriteBytes(const Byte *data, size_t num) + { + const unsigned bp = _bitPos; +#if 1 // 1 for optional speed-optimized code branch + if (bp == 8) + { + _stream.WriteBytes(data, num); + return; + } +#endif + unsigned c = _curByte; + const unsigned bp_rev = 8 - bp; + for (size_t i = 0; i < num; i++) + { + const unsigned b = data[i]; + _stream.WriteByte((Byte)(c | (b >> bp_rev))); + c = b << bp; + } + _curByte = c; + } +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Deflate64Register.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Deflate64Register.cpp new file mode 100644 index 0000000..7f6bae0 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Deflate64Register.cpp @@ -0,0 +1,25 @@ +// Deflate64Register.cpp + +#include "StdAfx.h" + +#include "../Common/RegisterCodec.h" + +#include "DeflateDecoder.h" +#if !defined(Z7_EXTRACT_ONLY) && !defined(Z7_DEFLATE_EXTRACT_ONLY) +#include "DeflateEncoder.h" +#endif + +namespace NCompress { +namespace NDeflate { + +REGISTER_CODEC_CREATE(CreateDec, NDecoder::CCOMCoder64()) + +#if !defined(Z7_EXTRACT_ONLY) && !defined(Z7_DEFLATE_EXTRACT_ONLY) +REGISTER_CODEC_CREATE(CreateEnc, NEncoder::CCOMCoder64()) +#else +#define CreateEnc NULL +#endif + +REGISTER_CODEC_2(Deflate64, CreateDec, CreateEnc, 0x40109, "Deflate64") + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateConst.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateConst.h new file mode 100644 index 0000000..a73d8ff --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateConst.h @@ -0,0 +1,131 @@ +// DeflateConst.h + +#ifndef ZIP7_INC_DEFLATE_CONST_H +#define ZIP7_INC_DEFLATE_CONST_H + +namespace NCompress { +namespace NDeflate { + +const unsigned kNumHuffmanBits = 15; + +const UInt32 kHistorySize32 = (1 << 15); +const UInt32 kHistorySize64 = (1 << 16); + +const unsigned kDistTableSize32 = 30; +const unsigned kDistTableSize64 = 32; + +const unsigned kNumLenSymbols32 = 256; +const unsigned kNumLenSymbols64 = 255; // don't change it. It must be <= 255. +const unsigned kNumLenSymbolsMax = kNumLenSymbols32; + +const unsigned kNumLenSlots = 29; + +const unsigned kFixedDistTableSize = 32; +const unsigned kFixedLenTableSize = 31; + +const unsigned kSymbolEndOfBlock = 0x100; +const unsigned kSymbolMatch = kSymbolEndOfBlock + 1; + +const unsigned kMainTableSize = kSymbolMatch + kNumLenSlots; +const unsigned kFixedMainTableSize = kSymbolMatch + kFixedLenTableSize; + +const unsigned kLevelTableSize = 19; + +const unsigned kTableDirectLevels = 16; +const unsigned kTableLevelRepNumber = kTableDirectLevels; +const unsigned kTableLevel0Number = kTableLevelRepNumber + 1; +const unsigned kTableLevel0Number2 = kTableLevel0Number + 1; + +const unsigned kLevelMask = 0xF; + +const Byte kLenStart32[kFixedLenTableSize] = + {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224, 255, 0, 0}; +const Byte kLenStart64[kFixedLenTableSize] = + {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224, 0, 0, 0}; + +const Byte kLenDirectBits32[kFixedLenTableSize] = + {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0}; +const Byte kLenDirectBits64[kFixedLenTableSize] = + {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 16, 0, 0}; + +const UInt32 kDistStart[kDistTableSize64] = + {0,1,2,3,4,6,8,12,16,24,32,48,64,96,128,192,256,384,512,768, + 1024,1536,2048,3072,4096,6144,8192,12288,16384,24576,32768,49152}; +const Byte kDistDirectBits[kDistTableSize64] = + {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14}; + +const Byte kLevelDirectBits[3] = {2, 3, 7}; + +const Byte kCodeLengthAlphabetOrder[kLevelTableSize] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + +const unsigned kMatchMinLen = 3; +const unsigned kMatchMaxLen32 = kNumLenSymbols32 + kMatchMinLen - 1; // 256 + 2 +const unsigned kMatchMaxLen64 = kNumLenSymbols64 + kMatchMinLen - 1; // 255 + 2 +const unsigned kMatchMaxLen = kMatchMaxLen32; + +const unsigned kFinalBlockFieldSize = 1; + +namespace NFinalBlockField +{ + enum + { + kNotFinalBlock = 0, + kFinalBlock = 1 + }; +} + +const unsigned kBlockTypeFieldSize = 2; + +namespace NBlockType +{ + enum + { + kStored = 0, + kFixedHuffman = 1, + kDynamicHuffman = 2 + }; +} + +const unsigned kNumLenCodesFieldSize = 5; +const unsigned kNumDistCodesFieldSize = 5; +const unsigned kNumLevelCodesFieldSize = 4; + +const unsigned kNumLitLenCodesMin = 257; +const unsigned kNumDistCodesMin = 1; +const unsigned kNumLevelCodesMin = 4; + +const unsigned kLevelFieldSize = 3; + +const unsigned kStoredBlockLengthFieldSize = 16; + +struct CLevels +{ + Byte litLenLevels[kFixedMainTableSize]; + Byte distLevels[kFixedDistTableSize]; + + void SubClear() + { + unsigned i; + for (i = kNumLitLenCodesMin; i < kFixedMainTableSize; i++) + litLenLevels[i] = 0; + for (i = 0; i < kFixedDistTableSize; i++) + distLevels[i] = 0; + } + + void SetFixedLevels() + { + unsigned i = 0; + + for (; i < 144; i++) litLenLevels[i] = 8; + for (; i < 256; i++) litLenLevels[i] = 9; + for (; i < 280; i++) litLenLevels[i] = 7; + for (; i < 288; i++) litLenLevels[i] = 8; + + for (i = 0; i < kFixedDistTableSize; i++) // test it: InfoZip only uses kDistTableSize + distLevels[i] = 5; + } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateDecoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateDecoder.cpp new file mode 100644 index 0000000..7993176 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateDecoder.cpp @@ -0,0 +1,566 @@ +// DeflateDecoder.cpp + +#include "StdAfx.h" + +#include "DeflateDecoder.h" + +namespace NCompress { +namespace NDeflate { +namespace NDecoder { + +CCoder::CCoder(bool deflate64Mode): + _deflateNSIS(false), + _deflate64Mode(deflate64Mode), + _keepHistory(false), + _needFinishInput(false), + _needInitInStream(true), + _outSizeDefined(false), + _outStartPos(0) + {} + +UInt32 CCoder::ReadBits(unsigned numBits) +{ + return m_InBitStream.ReadBits(numBits); +} + +Byte CCoder::ReadAlignedByte() +{ + return m_InBitStream.ReadAlignedByte(); +} + +bool CCoder::DecodeLevels(Byte *levels, unsigned numSymbols) +{ + unsigned i = 0; + + do + { + unsigned sym = m_LevelDecoder.Decode(&m_InBitStream); + if (sym < kTableDirectLevels) + levels[i++] = (Byte)sym; + else + { + if (sym >= kLevelTableSize) + return false; + + unsigned num; + unsigned numBits; + Byte symbol; + + if (sym == kTableLevelRepNumber) + { + if (i == 0) + return false; + numBits = 2; + num = 0; + symbol = levels[(size_t)i - 1]; + } + else + { + sym -= kTableLevel0Number; + sym <<= 2; + numBits = 3 + (unsigned)sym; + num = ((unsigned)sym << 1); + symbol = 0; + } + + num += i + 3 + ReadBits(numBits); + if (num > numSymbols) + return false; + do + levels[i++] = symbol; + while (i < num); + } + } + while (i < numSymbols); + + return true; +} + +#define RIF(x) { if (!(x)) return false; } + +bool CCoder::ReadTables(void) +{ + m_FinalBlock = (ReadBits(kFinalBlockFieldSize) == NFinalBlockField::kFinalBlock); + if (m_InBitStream.ExtraBitsWereRead()) + return false; + const UInt32 blockType = ReadBits(kBlockTypeFieldSize); + if (blockType > NBlockType::kDynamicHuffman) + return false; + if (m_InBitStream.ExtraBitsWereRead()) + return false; + + if (blockType == NBlockType::kStored) + { + m_StoredMode = true; + m_InBitStream.AlignToByte(); + m_StoredBlockSize = ReadAligned_UInt16(); // ReadBits(kStoredBlockLengthFieldSize) + if (_deflateNSIS) + return true; + return (m_StoredBlockSize == (UInt16)~ReadAligned_UInt16()); + } + + m_StoredMode = false; + + CLevels levels; + if (blockType == NBlockType::kFixedHuffman) + { + levels.SetFixedLevels(); + _numDistLevels = _deflate64Mode ? kDistTableSize64 : kDistTableSize32; + } + else + { + const unsigned numLitLenLevels = ReadBits(kNumLenCodesFieldSize) + kNumLitLenCodesMin; + _numDistLevels = (unsigned)ReadBits(kNumDistCodesFieldSize) + kNumDistCodesMin; + const unsigned numLevelCodes = ReadBits(kNumLevelCodesFieldSize) + kNumLevelCodesMin; + + if (!_deflate64Mode) + if (_numDistLevels > kDistTableSize32) + return false; + + const unsigned kLevelTableSize_aligned4 = kLevelTableSize + 1; + Byte levelLevels[kLevelTableSize_aligned4]; + memset (levelLevels, 0, sizeof(levelLevels)); + unsigned i = 0; + do + levelLevels[kCodeLengthAlphabetOrder[i++]] = (Byte)ReadBits(kLevelFieldSize); + while (i != numLevelCodes); + + if (m_InBitStream.ExtraBitsWereRead()) + return false; + + RIF(m_LevelDecoder.Build(levelLevels, false)) // full + + Byte tmpLevels[kFixedMainTableSize + kFixedDistTableSize]; + if (!DecodeLevels(tmpLevels, numLitLenLevels + _numDistLevels)) + return false; + + if (m_InBitStream.ExtraBitsWereRead()) + return false; + + levels.SubClear(); + memcpy(levels.litLenLevels, tmpLevels, numLitLenLevels); + memcpy(levels.distLevels, tmpLevels + numLitLenLevels, _numDistLevels); + } + RIF(m_MainDecoder.Build(levels.litLenLevels)) + return m_DistDecoder.Build(levels.distLevels); +} + + +HRESULT CCoder::InitInStream(bool needInit) +{ + if (needInit) + { + // for HDD-Windows: + // (1 << 15) - best for reading only prefetch + // (1 << 22) - best for real reading / writing + if (!m_InBitStream.Create(1 << 20)) + return E_OUTOFMEMORY; + m_InBitStream.Init(); + _needInitInStream = false; + } + return S_OK; +} + + +HRESULT CCoder::CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputProgressLimit) +{ + if (_remainLen == kLenIdFinished) + return S_OK; + + if (_remainLen == kLenIdNeedInit) + { + if (!_keepHistory) + if (!m_OutWindowStream.Create(_deflate64Mode ? kHistorySize64: kHistorySize32)) + return E_OUTOFMEMORY; + RINOK(InitInStream(_needInitInStream)) + m_OutWindowStream.Init(_keepHistory); + + m_FinalBlock = false; + _remainLen = 0; + _needReadTable = true; + } + + // _remainLen >= 0 + while (_remainLen && curSize) + { + _remainLen--; + const Byte b = m_OutWindowStream.GetByte(_rep0); + m_OutWindowStream.PutByte(b); + curSize--; + } + + UInt64 inputStart = 0; + if (inputProgressLimit != 0) + inputStart = m_InBitStream.GetProcessedSize(); + + while (curSize || finishInputStream) + { + if (m_InBitStream.ExtraBitsWereRead()) + return S_FALSE; + + if (_needReadTable) + { + if (m_FinalBlock) + { + _remainLen = kLenIdFinished; + break; + } + + if (inputProgressLimit != 0) + if (m_InBitStream.GetProcessedSize() - inputStart >= inputProgressLimit) + return S_OK; + + if (!ReadTables()) + return S_FALSE; + if (m_InBitStream.ExtraBitsWereRead()) + return S_FALSE; + _needReadTable = false; + } + + if (m_StoredMode) + { + if (finishInputStream && curSize == 0 && m_StoredBlockSize != 0) + return S_FALSE; + /* NSIS version contains some bits in bitl bits buffer. + So we must read some first bytes via ReadAlignedByte */ + UInt32 num = m_StoredBlockSize; + if (num > curSize) + num = curSize; + m_StoredBlockSize -= num; + curSize -= num; + for (; num && m_InBitStream.ThereAreDataInBitsBuffer(); num--) + m_OutWindowStream.PutByte(ReadAlignedByte()); + if (num) + { +#if 1 + // fast code + do + { + size_t a; + Byte *buf = m_OutWindowStream.GetOutBuffer(a); + // a != 0 + if (a > num) + a = num; + // a != 0 + a = m_InBitStream.ReadDirectBytesPart(buf, a); + if (a == 0) + return S_FALSE; + m_OutWindowStream.SkipWrittenBytes(a); + num -= (UInt32)a; + } + while (num); +#else + // slow code: + do + m_OutWindowStream.PutByte(m_InBitStream.ReadDirectByte()); + while (--num); +#endif + } + _needReadTable = (m_StoredBlockSize == 0); + continue; + } + + while (curSize) + { + if (m_InBitStream.ExtraBitsWereRead_Fast()) + return S_FALSE; + unsigned sym; +#if 0 + sym = m_MainDecoder.Decode(&m_InBitStream); +#else + Z7_HUFF_DECODE_CHECK(sym, &m_MainDecoder, kNumHuffmanBits, kNumTableBits_Main, &m_InBitStream, { return S_FALSE; }) +#endif + + if (sym < 0x100) + { + m_OutWindowStream.PutByte((Byte)sym); + curSize--; + continue; + } + if (sym == kSymbolEndOfBlock) + { + _needReadTable = true; + break; + } +#if 0 + if (sym >= kMainTableSize) + return S_FALSE; +#endif + { + sym -= kSymbolMatch; + UInt32 len; + { + unsigned numBits; + if (_deflate64Mode) + { + len = kLenStart64[sym]; + numBits = kLenDirectBits64[sym]; + } + else + { + len = kLenStart32[sym]; + numBits = kLenDirectBits32[sym]; + } + len += kMatchMinLen + m_InBitStream.ReadBits(numBits); + } + +#if 0 + sym = m_DistDecoder.Decode(&m_InBitStream); + if (sym >= _numDistLevels) + return S_FALSE; +#else + Z7_HUFF_DECODE_CHECK(sym, &m_DistDecoder, kNumHuffmanBits, kNumTableBits_Dist, &m_InBitStream, { return S_FALSE; }) +#endif + +#if 1 + sym = kDistStart[sym] + m_InBitStream.ReadBits(kDistDirectBits[sym]); +#else + if (sym >= 4) + { + // sym &= 31; + const unsigned numDirectBits = (sym - 2) >> 1; + sym = (2u | (sym & 1)) << numDirectBits; + sym += m_InBitStream.ReadBits(numDirectBits); + } +#endif + UInt32 locLen = len; + if (locLen > curSize) + locLen = (UInt32)curSize; + if (!m_OutWindowStream.CopyBlock(sym, locLen)) + return S_FALSE; + curSize -= locLen; + len -= locLen; + if (len != 0) + { + _remainLen = (Int32)len; + _rep0 = sym; + break; + } + } + } + + if (finishInputStream && curSize == 0) + { + if (m_MainDecoder.Decode(&m_InBitStream) != kSymbolEndOfBlock) + return S_FALSE; + _needReadTable = true; + } + } + + if (m_InBitStream.ExtraBitsWereRead()) + return S_FALSE; + + return S_OK; +} + + +#ifdef Z7_NO_EXCEPTIONS + +#define DEFLATE_TRY_BEGIN +#define DEFLATE_TRY_END(res) + +#else + +#define DEFLATE_TRY_BEGIN try { +#define DEFLATE_TRY_END(res) } \ + catch(const CSystemException &e) { res = e.ErrorCode; } \ + catch(...) { res = S_FALSE; } + + // catch(const CInBufferException &e) { res = e.ErrorCode; } + // catch(const CLzOutWindowException &e) { res = e.ErrorCode; } + +#endif + + +HRESULT CCoder::CodeReal(ISequentialOutStream *outStream, ICompressProgressInfo *progress) +{ + HRESULT res; + + DEFLATE_TRY_BEGIN + + m_OutWindowStream.SetStream(outStream); + CCoderReleaser flusher(this); + + const UInt64 inStart = _needInitInStream ? 0 : m_InBitStream.GetProcessedSize(); + + for (;;) + { + const UInt32 kInputProgressLimit = 1 << 21; + UInt32 curSize = 1 << 20; + bool finishInputStream = false; + if (_outSizeDefined) + { + const UInt64 rem = _outSize - GetOutProcessedCur(); + if (curSize >= rem) + { + curSize = (UInt32)rem; + if (_needFinishInput) + finishInputStream = true; + else if (curSize == 0) + break; + } + } + + RINOK(CodeSpec(curSize, finishInputStream, progress ? kInputProgressLimit : 0)) + + if (_remainLen == kLenIdFinished) + break; + + if (progress) + { + const UInt64 inSize = m_InBitStream.GetProcessedSize() - inStart; + const UInt64 nowPos64 = GetOutProcessedCur(); + RINOK(progress->SetRatioInfo(&inSize, &nowPos64)) + } + } + + flusher.NeedFlush = false; + res = Flush(); + if (res == S_OK && _remainLen != kLenIdNeedInit && InputEofError()) + return S_FALSE; + + DEFLATE_TRY_END(res) + + return res; +} + + +Z7_COM7F_IMF(CCoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + SetInStream(inStream); + SetOutStreamSize(outSize); + const HRESULT res = CodeReal(outStream, progress); + ReleaseInStream(); + /* + if (res == S_OK) + if (_needFinishInput && inSize && *inSize != m_InBitStream.GetProcessedSize()) + res = S_FALSE; + */ + return res; +} + + +Z7_COM7F_IMF(CCoder::SetFinishMode(UInt32 finishMode)) +{ + Set_NeedFinishInput(finishMode != 0); + return S_OK; +} + + +Z7_COM7F_IMF(CCoder::GetInStreamProcessedSize(UInt64 *value)) +{ + *value = m_InBitStream.GetStreamSize(); + return S_OK; +} + + +Z7_COM7F_IMF(CCoder::ReadUnusedFromInBuf(void *data, UInt32 size, UInt32 *processedSize)) +{ + AlignToByte(); + UInt32 i = 0; + { + for (i = 0; i < size; i++) + { + if (!m_InBitStream.ReadAlignedByte_FromBuf(((Byte *)data)[i])) + break; + } + } + if (processedSize) + *processedSize = i; + return S_OK; +} + + +Z7_COM7F_IMF(CCoder::SetInStream(ISequentialInStream *inStream)) +{ + m_InStreamRef = inStream; + m_InBitStream.SetStream(inStream); + return S_OK; +} + + +Z7_COM7F_IMF(CCoder::ReleaseInStream()) +{ + m_InStreamRef.Release(); + m_InBitStream.ClearStreamPtr(); + return S_OK; +} + + +void CCoder::SetOutStreamSizeResume(const UInt64 *outSize) +{ + _outSizeDefined = (outSize != NULL); + _outSize = 0; + if (_outSizeDefined) + _outSize = *outSize; + m_OutWindowStream.Init(_keepHistory); + _outStartPos = m_OutWindowStream.GetProcessedSize(); + _remainLen = kLenIdNeedInit; +} + + +Z7_COM7F_IMF(CCoder::SetOutStreamSize(const UInt64 *outSize)) +{ + /* + 18.06: + We want to support GetInputProcessedSize() before CCoder::Read() + So we call m_InBitStream.Init() even before buffer allocations + m_InBitStream.Init() just sets variables to default values + But later we will call m_InBitStream.Init() again with real buffer pointers + */ + m_InBitStream.Init(); + _needInitInStream = true; + SetOutStreamSizeResume(outSize); + return S_OK; +} + + +#ifndef Z7_NO_READ_FROM_CODER + +Z7_COM7F_IMF(CCoder::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + if (processedSize) + *processedSize = 0; + const UInt64 outPos = GetOutProcessedCur(); + + bool finishInputStream = false; + if (_outSizeDefined) + { + const UInt64 rem = _outSize - outPos; + if (size >= rem) + { + size = (UInt32)rem; + if (_needFinishInput) + finishInputStream = true; + } + } + if (!finishInputStream && size == 0) + return S_OK; + + HRESULT res; + DEFLATE_TRY_BEGIN + m_OutWindowStream.SetMemStream((Byte *)data); + res = CodeSpec(size, finishInputStream); + DEFLATE_TRY_END(res) + { + const HRESULT res2 = Flush(); + if (res2 != S_OK) + res = res2; + } + if (processedSize) + *processedSize = (UInt32)(GetOutProcessedCur() - outPos); + m_OutWindowStream.SetMemStream(NULL); + return res; +} + +#endif + + +HRESULT CCoder::CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + SetOutStreamSizeResume(outSize); + return CodeReal(outStream, progress); +} + +}}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateDecoder.h new file mode 100644 index 0000000..4e0ae61 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateDecoder.h @@ -0,0 +1,154 @@ +// DeflateDecoder.h + +#ifndef ZIP7_INC_DEFLATE_DECODER_H +#define ZIP7_INC_DEFLATE_DECODER_H + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" + +#include "BitlDecoder.h" +#include "DeflateConst.h" +#include "HuffmanDecoder.h" +#include "LzOutWindow.h" + +namespace NCompress { +namespace NDeflate { +namespace NDecoder { + +const int kLenIdFinished = -1; +const int kLenIdNeedInit = -2; + +const unsigned kNumTableBits_Main = 10; +const unsigned kNumTableBits_Dist = 6; + +class CCoder: + public ICompressCoder, + public ICompressSetFinishMode, + public ICompressGetInStreamProcessedSize, + public ICompressReadUnusedFromInBuf, + public ICompressSetInStream, + public ICompressSetOutStreamSize, +#ifndef Z7_NO_READ_FROM_CODER + public ISequentialInStream, +#endif + public CMyUnknownImp +{ + Z7_COM_QI_BEGIN2(ICompressCoder) + Z7_COM_QI_ENTRY(ICompressSetFinishMode) + Z7_COM_QI_ENTRY(ICompressGetInStreamProcessedSize) + Z7_COM_QI_ENTRY(ICompressReadUnusedFromInBuf) + Z7_COM_QI_ENTRY(ICompressSetInStream) + Z7_COM_QI_ENTRY(ICompressSetOutStreamSize) +#ifndef Z7_NO_READ_FROM_CODER + Z7_COM_QI_ENTRY(ISequentialInStream) +#endif + Z7_COM_QI_END + Z7_COM_ADDREF_RELEASE + + Z7_IFACE_COM7_IMP(ICompressCoder) + Z7_IFACE_COM7_IMP(ICompressSetFinishMode) + Z7_IFACE_COM7_IMP(ICompressGetInStreamProcessedSize) +public: + Z7_IFACE_COM7_IMP(ICompressReadUnusedFromInBuf) + Z7_IFACE_COM7_IMP(ICompressSetInStream) +private: + Z7_IFACE_COM7_IMP(ICompressSetOutStreamSize) +#ifndef Z7_NO_READ_FROM_CODER + Z7_IFACE_COM7_IMP(ISequentialInStream) +#endif + + CLzOutWindow m_OutWindowStream; + NBitl::CDecoder m_InBitStream; + NCompress::NHuffman::CDecoder m_MainDecoder; + NCompress::NHuffman::CDecoder256 m_DistDecoder; + NCompress::NHuffman::CDecoder7b m_LevelDecoder; + + UInt32 m_StoredBlockSize; + + unsigned _numDistLevels; + bool m_FinalBlock; + bool m_StoredMode; + + bool _deflateNSIS; + bool _deflate64Mode; + bool _keepHistory; + bool _needFinishInput; + + bool _needInitInStream; + bool _needReadTable; + Int32 _remainLen; + UInt32 _rep0; + + bool _outSizeDefined; + CMyComPtr m_InStreamRef; + UInt64 _outSize; + UInt64 _outStartPos; + + void SetOutStreamSizeResume(const UInt64 *outSize); + UInt64 GetOutProcessedCur() const { return m_OutWindowStream.GetProcessedSize() - _outStartPos; } + + UInt32 ReadBits(unsigned numBits); + + bool DecodeLevels(Byte *levels, unsigned numSymbols); + bool ReadTables(); + + HRESULT Flush() { return m_OutWindowStream.Flush(); } + class CCoderReleaser + { + CCoder *_coder; + public: + bool NeedFlush; + CCoderReleaser(CCoder *coder): _coder(coder), NeedFlush(true) {} + ~CCoderReleaser() + { + if (NeedFlush) + _coder->Flush(); + } + }; + friend class CCoderReleaser; + + HRESULT CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputProgressLimit = 0); +public: + + CCoder(bool deflate64Mode); + virtual ~CCoder() {} + + void SetNsisMode(bool nsisMode) { _deflateNSIS = nsisMode; } + + void Set_KeepHistory(bool keepHistory) { _keepHistory = keepHistory; } + void Set_NeedFinishInput(bool needFinishInput) { _needFinishInput = needFinishInput; } + + bool IsFinished() const { return _remainLen == kLenIdFinished; } + bool IsFinalBlock() const { return m_FinalBlock; } + + HRESULT CodeReal(ISequentialOutStream *outStream, ICompressProgressInfo *progress); + +public: + HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress); + HRESULT InitInStream(bool needInit); + + void AlignToByte() { m_InBitStream.AlignToByte(); } + Byte ReadAlignedByte(); + UInt32 ReadAligned_UInt16() // aligned for Byte range + { + const UInt32 v = m_InBitStream.ReadAlignedByte(); + return v | ((UInt32)m_InBitStream.ReadAlignedByte() << 8); + } + bool InputEofError() const { return m_InBitStream.ExtraBitsWereRead(); } + + // size of used real data from input stream + UInt64 GetStreamSize() const { return m_InBitStream.GetStreamSize(); } + + // size of virtual input stream processed + UInt64 GetInputProcessedSize() const { return m_InBitStream.GetProcessedSize(); } +}; + +class CCOMCoder : public CCoder { public: CCOMCoder(): CCoder(false) {} }; +class CCOMCoder64 : public CCoder { public: CCOMCoder64(): CCoder(true) {} }; + +}}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateEncoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateEncoder.cpp new file mode 100644 index 0000000..afc5f12 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateEncoder.cpp @@ -0,0 +1,1045 @@ +// DeflateEncoder.cpp + +#include "StdAfx.h" + +#include "../../../C/Alloc.h" +#include "../../../C/HuffEnc.h" + +#include "../../Common/ComTry.h" + +#include "../Common/CWrappers.h" + +#include "DeflateEncoder.h" + +#undef NO_INLINE + +#ifdef _MSC_VER +#define NO_INLINE Z7_NO_INLINE +#else +#define NO_INLINE +#endif + +#define MAX_HUF_LEN_12 12 + +namespace NCompress { +namespace NDeflate { +namespace NEncoder { + +static const unsigned k_CodeValue_Len_Is_Literal_Flag = 1u << 15; + +static const unsigned kNumDivPassesMax = 10; // [0, 16); ratio/speed/ram tradeoff; use big value for better compression ratio. +static const unsigned kNumTables = 1u << kNumDivPassesMax; + +static const UInt32 kFixedHuffmanCodeBlockSizeMax = (1 << 8); // [0, (1 << 32)); ratio/speed tradeoff; use big value for better compression ratio. +static const UInt32 kDivideCodeBlockSizeMin = (1 << 7); // [1, (1 << 32)); ratio/speed tradeoff; use small value for better compression ratio. +static const UInt32 kDivideBlockSizeMin = (1 << 6); // [1, (1 << 32)); ratio/speed tradeoff; use small value for better compression ratio. + +static const UInt32 kMaxUncompressedBlockSize = ((1 << 16) - 1) * 1; // [1, (1 << 32)) +static const UInt32 kMatchArraySize = kMaxUncompressedBlockSize * 10; // [kMatchMaxLen * 2, (1 << 32)) +static const UInt32 kMatchArrayLimit = kMatchArraySize - kMatchMaxLen * 4 * sizeof(UInt16); +static const UInt32 kBlockUncompressedSizeThreshold = kMaxUncompressedBlockSize - + kMatchMaxLen - kNumOpts; + +// static const unsigned kMaxCodeBitLength = 11; +static const unsigned kMaxLevelBitLength = 7; + +static const Byte kNoLiteralStatPrice = 11; +static const Byte kNoLenStatPrice = 11; +static const Byte kNoPosStatPrice = 6; + +MY_ALIGN(64) +static Byte g_LenSlots[kNumLenSymbolsMax]; + +#define kNumLogBits 9 // do not change it +MY_ALIGN(64) +static Byte g_FastPos[1 << kNumLogBits]; + +class CFastPosInit +{ +public: + CFastPosInit() + { + unsigned i; + for (i = 0; i < kNumLenSlots; i++) + { + unsigned c = kLenStart32[i]; + const unsigned j = 1u << kLenDirectBits32[i]; + for (unsigned k = 0; k < j; k++, c++) + g_LenSlots[c] = (Byte)i; + } + + const unsigned kFastSlots = kNumLogBits * 2; + unsigned c = 0; + for (Byte slotFast = 0; slotFast < kFastSlots; slotFast++) + { + const unsigned k = 1u << kDistDirectBits[slotFast]; + for (unsigned j = 0; j < k; j++, c++) + g_FastPos[c] = slotFast; + } + } +}; + +static CFastPosInit g_FastPosInit; + +inline unsigned GetPosSlot(UInt32 pos) +{ + /* + if (pos < 0x200) + return g_FastPos[pos]; + return g_FastPos[pos >> 8] + 16; + */ + // const unsigned zz = (pos < ((UInt32)1 << (kNumLogBits))) ? 0 : 8; + /* + const unsigned zz = (kNumLogBits - 1) & + ((UInt32)0 - (((((UInt32)1 << kNumLogBits) - 1) - pos) >> 31)); + */ + const unsigned zz = (kNumLogBits - 1) & + (((((UInt32)1 << kNumLogBits) - 1) - pos) >> (31 - 3)); + return g_FastPos[pos >> zz] + (zz * 2); +} + + +void CEncProps::Normalize() +{ + int level = Level; + if (level < 0) level = 5; + Level = level; + if (algo < 0) algo = (level < 5 ? 0 : 1); + if (fb < 0) fb = (level < 7 ? 32 : (level < 9 ? 64 : 128)); + if (btMode < 0) btMode = (algo == 0 ? 0 : 1); + if (mc == 0) mc = (16 + ((unsigned)fb >> 1)); + if (numPasses == (UInt32)(Int32)-1) numPasses = (level < 7 ? 1 : (level < 9 ? 3 : 10)); +} + +void CCoder::SetProps(const CEncProps *props2) +{ + CEncProps props = *props2; + props.Normalize(); + + m_MatchFinderCycles = props.mc; + { + unsigned fb = (unsigned)props.fb; + if (fb < kMatchMinLen) + fb = kMatchMinLen; + if (fb > m_MatchMaxLen) + fb = m_MatchMaxLen; + m_NumFastBytes = fb; + } + _fastMode = (props.algo == 0); + _btMode = (props.btMode != 0); + + m_NumDivPasses = props.numPasses; + if (m_NumDivPasses == 0) + m_NumDivPasses = 1; + if (m_NumDivPasses == 1) + m_NumPasses = 1; + else if (m_NumDivPasses <= kNumDivPassesMax) + m_NumPasses = 2; + else + { + m_NumPasses = 2 + (m_NumDivPasses - kNumDivPassesMax); + m_NumDivPasses = kNumDivPassesMax; + } +} + +CCoder::CCoder(bool deflate64Mode): + m_Values(NULL), + m_OnePosMatchesMemory(NULL), + m_DistanceMemory(NULL), + m_Created(false), + m_Deflate64Mode(deflate64Mode), + m_Tables(NULL) +{ + m_MatchMaxLen = deflate64Mode ? kMatchMaxLen64 : kMatchMaxLen32; + m_NumLenCombinations = deflate64Mode ? kNumLenSymbols64 : kNumLenSymbols32; + m_LenStart = deflate64Mode ? kLenStart64 : kLenStart32; + m_LenDirectBits = deflate64Mode ? kLenDirectBits64 : kLenDirectBits32; + { + CEncProps props; + SetProps(&props); + } + MatchFinder_Construct(&_lzInWindow); +} + +HRESULT CCoder::Create() +{ + // COM_TRY_BEGIN + if (!m_Values) + { + m_Values = (CCodeValue *)MyAlloc(kMaxUncompressedBlockSize * sizeof(CCodeValue)); + if (!m_Values) + return E_OUTOFMEMORY; + } + if (!m_Tables) + { + m_Tables = (CTables *)MyAlloc(kNumTables * sizeof(CTables)); + if (!m_Tables) + return E_OUTOFMEMORY; + } + + if (m_IsMultiPass) + { + if (!m_OnePosMatchesMemory) + { + m_OnePosMatchesMemory = (UInt16 *)::MidAlloc(kMatchArraySize * sizeof(UInt16)); + if (!m_OnePosMatchesMemory) + return E_OUTOFMEMORY; + } + } + else + { + if (!m_DistanceMemory) + { + m_DistanceMemory = (UInt16 *)MyAlloc((kMatchMaxLen + 2) * 2 * sizeof(UInt16)); + if (!m_DistanceMemory) + return E_OUTOFMEMORY; + m_MatchDistances = m_DistanceMemory; + } + } + + if (!m_Created) + { + _lzInWindow.btMode = (Byte)(_btMode ? 1 : 0); + _lzInWindow.numHashBytes = 3; + _lzInWindow.numHashBytes_Min = 3; + if (!MatchFinder_Create(&_lzInWindow, + m_Deflate64Mode ? kHistorySize64 : kHistorySize32, + kNumOpts + kMaxUncompressedBlockSize, + m_NumFastBytes, m_MatchMaxLen - m_NumFastBytes, &g_AlignedAlloc)) + return E_OUTOFMEMORY; + if (!m_OutStream.Create(1 << 20)) + return E_OUTOFMEMORY; + } + if (m_MatchFinderCycles != 0) + _lzInWindow.cutValue = m_MatchFinderCycles; + m_Created = true; + return S_OK; + // COM_TRY_END +} + +HRESULT CCoder::BaseSetEncoderProperties2(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) +{ + CEncProps props; + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + PROPID propID = propIDs[i]; + if (propID >= NCoderPropID::kReduceSize) + continue; + if (prop.vt != VT_UI4) + return E_INVALIDARG; + UInt32 v = (UInt32)prop.ulVal; + switch (propID) + { + case NCoderPropID::kNumPasses: props.numPasses = v; break; + case NCoderPropID::kNumFastBytes: props.fb = (int)v; break; + case NCoderPropID::kMatchFinderCycles: props.mc = v; break; + case NCoderPropID::kAlgorithm: props.algo = (int)v; break; + case NCoderPropID::kLevel: props.Level = (int)v; break; + case NCoderPropID::kNumThreads: break; + default: return E_INVALIDARG; + } + } + SetProps(&props); + return S_OK; +} + +void CCoder::Free() +{ + ::MidFree(m_OnePosMatchesMemory); m_OnePosMatchesMemory = NULL; + ::MyFree(m_DistanceMemory); m_DistanceMemory = NULL; + ::MyFree(m_Values); m_Values = NULL; + ::MyFree(m_Tables); m_Tables = NULL; +} + +CCoder::~CCoder() +{ + Free(); + MatchFinder_Free(&_lzInWindow, &g_AlignedAlloc); +} + +NO_INLINE void CCoder::GetMatches() +{ + if (m_IsMultiPass) + { + m_MatchDistances = m_OnePosMatchesMemory + m_Pos; + if (m_SecondPass) + { + m_Pos += *m_MatchDistances + 1; + return; + } + } + + UInt32 distanceTmp[kMatchMaxLen * 2 + 3]; + + const size_t numPairs = (size_t)((_btMode ? + Bt3Zip_MatchFinder_GetMatches(&_lzInWindow, distanceTmp): + Hc3Zip_MatchFinder_GetMatches(&_lzInWindow, distanceTmp)) - distanceTmp); + + UInt16 *matchDistances = m_MatchDistances; + *matchDistances++ = (UInt16)numPairs; + + if (numPairs != 0) + { + size_t i; + for (i = 0; i < numPairs; i += 2) + { + matchDistances[0] = (UInt16)distanceTmp[i]; + matchDistances[1] = (UInt16)distanceTmp[(size_t)i + 1]; + matchDistances += 2; + } + UInt32 len = distanceTmp[(size_t)numPairs - 2]; + if (len == m_NumFastBytes && m_NumFastBytes != m_MatchMaxLen) + { + UInt32 numAvail = Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) + 1; + const Byte *pby = Inline_MatchFinder_GetPointerToCurrentPos(&_lzInWindow) - 1; + const Byte *pby2 = pby - (distanceTmp[(size_t)numPairs - 1] + 1); + if (numAvail > m_MatchMaxLen) + numAvail = m_MatchMaxLen; + for (; len < numAvail && pby[len] == pby2[len]; len++); + matchDistances[-2] = (UInt16)len; + } + } + if (m_IsMultiPass) + m_Pos += (UInt32)numPairs + 1; + if (!m_SecondPass) + m_AdditionalOffset++; +} + +void CCoder::MovePos(UInt32 num) +{ + if (!m_SecondPass && num > 0) + { + if (_btMode) + Bt3Zip_MatchFinder_Skip(&_lzInWindow, num); + else + Hc3Zip_MatchFinder_Skip(&_lzInWindow, num); + m_AdditionalOffset += num; + } +} + +static const UInt32 kIfinityPrice = 0xFFFFFFF; + +NO_INLINE UInt32 CCoder::Backward(UInt32 &backRes, UInt32 cur) +{ + m_OptimumEndIndex = cur; + UInt32 posMem = m_Optimum[cur].PosPrev; + UInt16 backMem = m_Optimum[cur].BackPrev; + do + { + UInt32 posPrev = posMem; + UInt16 backCur = backMem; + backMem = m_Optimum[posPrev].BackPrev; + posMem = m_Optimum[posPrev].PosPrev; + m_Optimum[posPrev].BackPrev = backCur; + m_Optimum[posPrev].PosPrev = (UInt16)cur; + cur = posPrev; + } + while (cur > 0); + backRes = m_Optimum[0].BackPrev; + m_OptimumCurrentIndex = m_Optimum[0].PosPrev; + return m_OptimumCurrentIndex; +} + +NO_INLINE UInt32 CCoder::GetOptimal(UInt32 &backRes) +{ + if (m_OptimumEndIndex != m_OptimumCurrentIndex) + { + UInt32 len = m_Optimum[m_OptimumCurrentIndex].PosPrev - m_OptimumCurrentIndex; + backRes = m_Optimum[m_OptimumCurrentIndex].BackPrev; + m_OptimumCurrentIndex = m_Optimum[m_OptimumCurrentIndex].PosPrev; + return len; + } + m_OptimumCurrentIndex = m_OptimumEndIndex = 0; + + GetMatches(); + + UInt32 lenEnd; + { + const UInt32 numDistancePairs = m_MatchDistances[0]; + if (numDistancePairs == 0) + return 1; + const UInt16 *matchDistances = m_MatchDistances + 1; + lenEnd = matchDistances[(size_t)numDistancePairs - 2]; + + if (lenEnd > m_NumFastBytes) + { + backRes = matchDistances[(size_t)numDistancePairs - 1]; + MovePos(lenEnd - 1); + return lenEnd; + } + + m_Optimum[1].Price = m_LiteralPrices[*(Inline_MatchFinder_GetPointerToCurrentPos(&_lzInWindow) - m_AdditionalOffset)]; + m_Optimum[1].PosPrev = 0; + + m_Optimum[2].Price = kIfinityPrice; + m_Optimum[2].PosPrev = 1; + + UInt32 offs = 0; + + for (UInt32 i = kMatchMinLen; i <= lenEnd; i++) + { + UInt32 distance = matchDistances[(size_t)offs + 1]; + m_Optimum[i].PosPrev = 0; + m_Optimum[i].BackPrev = (UInt16)distance; + m_Optimum[i].Price = m_LenPrices[(size_t)i - kMatchMinLen] + m_PosPrices[GetPosSlot(distance)]; + if (i == matchDistances[offs]) + offs += 2; + } + } + + UInt32 cur = 0; + + for (;;) + { + ++cur; + if (cur == lenEnd || cur == kNumOptsBase || m_Pos >= kMatchArrayLimit) + return Backward(backRes, cur); + GetMatches(); + const UInt16 *matchDistances = m_MatchDistances + 1; + const UInt32 numDistancePairs = m_MatchDistances[0]; + UInt32 newLen = 0; + if (numDistancePairs != 0) + { + newLen = matchDistances[(size_t)numDistancePairs - 2]; + if (newLen > m_NumFastBytes) + { + UInt32 len = Backward(backRes, cur); + m_Optimum[cur].BackPrev = matchDistances[(size_t)numDistancePairs - 1]; + m_OptimumEndIndex = cur + newLen; + m_Optimum[cur].PosPrev = (UInt16)m_OptimumEndIndex; + MovePos(newLen - 1); + return len; + } + } + UInt32 curPrice = m_Optimum[cur].Price; + { + const UInt32 curAnd1Price = curPrice + m_LiteralPrices[*(Inline_MatchFinder_GetPointerToCurrentPos(&_lzInWindow) + cur - m_AdditionalOffset)]; + COptimal &optimum = m_Optimum[(size_t)cur + 1]; + if (curAnd1Price < optimum.Price) + { + optimum.Price = curAnd1Price; + optimum.PosPrev = (UInt16)cur; + } + } + if (numDistancePairs == 0) + continue; + while (lenEnd < cur + newLen) + m_Optimum[++lenEnd].Price = kIfinityPrice; + UInt32 offs = 0; + UInt32 distance = matchDistances[(size_t)offs + 1]; + curPrice += m_PosPrices[GetPosSlot(distance)]; + for (UInt32 lenTest = kMatchMinLen; ; lenTest++) + { + UInt32 curAndLenPrice = curPrice + m_LenPrices[(size_t)lenTest - kMatchMinLen]; + COptimal &optimum = m_Optimum[cur + lenTest]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = (UInt16)cur; + optimum.BackPrev = (UInt16)distance; + } + if (lenTest == matchDistances[offs]) + { + offs += 2; + if (offs == numDistancePairs) + break; + curPrice -= m_PosPrices[GetPosSlot(distance)]; + distance = matchDistances[(size_t)offs + 1]; + curPrice += m_PosPrices[GetPosSlot(distance)]; + } + } + } +} + +UInt32 CCoder::GetOptimalFast(UInt32 &backRes) +{ + GetMatches(); + UInt32 numDistancePairs = m_MatchDistances[0]; + if (numDistancePairs == 0) + return 1; + UInt32 lenMain = m_MatchDistances[(size_t)numDistancePairs - 1]; + backRes = m_MatchDistances[numDistancePairs]; + MovePos(lenMain - 1); + return lenMain; +} + +void CTables::InitStructures() +{ + UInt32 i; + for (i = 0; i < 256; i++) + litLenLevels[i] = 8; + litLenLevels[i++] = 13; + for (;i < kFixedMainTableSize; i++) + litLenLevels[i] = 5; + for (i = 0; i < kFixedDistTableSize; i++) + distLevels[i] = 5; +} + +NO_INLINE void CCoder::LevelTableDummy(const Byte *levels, unsigned numLevels, UInt32 *freqs) +{ + unsigned prevLen = 0xFF; + unsigned nextLen = levels[0]; + unsigned count = 0; + unsigned maxCount = 7; + unsigned minCount = 4; + + if (nextLen == 0) + { + maxCount = 138; + minCount = 3; + } + + for (unsigned n = 0; n < numLevels; n++) + { + unsigned curLen = nextLen; + nextLen = (n < numLevels - 1) ? levels[(size_t)n + 1] : 0xFF; + count++; + if (count < maxCount && curLen == nextLen) + continue; + + if (count < minCount) + freqs[curLen] += (UInt32)count; + else if (curLen != 0) + { + if (curLen != prevLen) + { + freqs[curLen]++; + count--; + } + freqs[kTableLevelRepNumber]++; + } + else if (count <= 10) + freqs[kTableLevel0Number]++; + else + freqs[kTableLevel0Number2]++; + + count = 0; + prevLen = curLen; + + if (nextLen == 0) + { + maxCount = 138; + minCount = 3; + } + else if (curLen == nextLen) + { + maxCount = 6; + minCount = 3; + } + else + { + maxCount = 7; + minCount = 4; + } + } +} + +NO_INLINE void CCoder::WriteBits(UInt32 value, unsigned numBits) +{ + m_OutStream.WriteBits(value, numBits); +} + +#define WRITE_HF2(codes, lens, i) m_OutStream.WriteBits(codes[i], lens[i]) +#define WRITE_HF2_NO_INLINE(codes, lens, i) WriteBits(codes[i], lens[i]) +#define WRITE_HF(i) WriteBits(codes[i], lens[i]) + +NO_INLINE void CCoder::LevelTableCode(const Byte *levels, unsigned numLevels, const Byte *lens, const UInt32 *codes) +{ + unsigned prevLen = 0xFF; + unsigned nextLen = levels[0]; + unsigned count = 0; + unsigned maxCount = 7; + unsigned minCount = 4; + + if (nextLen == 0) + { + maxCount = 138; + minCount = 3; + } + + for (unsigned n = 0; n < numLevels; n++) + { + unsigned curLen = nextLen; + nextLen = (n < numLevels - 1) ? levels[(size_t)n + 1] : 0xFF; + count++; + if (count < maxCount && curLen == nextLen) + continue; + + if (count < minCount) + for (unsigned i = 0; i < count; i++) + WRITE_HF(curLen); + else if (curLen != 0) + { + if (curLen != prevLen) + { + WRITE_HF(curLen); + count--; + } + WRITE_HF(kTableLevelRepNumber); + WriteBits(count - 3, 2); + } + else if (count <= 10) + { + WRITE_HF(kTableLevel0Number); + WriteBits(count - 3, 3); + } + else + { + WRITE_HF(kTableLevel0Number2); + WriteBits(count - 11, 7); + } + + count = 0; + prevLen = curLen; + + if (nextLen == 0) + { + maxCount = 138; + minCount = 3; + } + else if (curLen == nextLen) + { + maxCount = 6; + minCount = 3; + } + else + { + maxCount = 7; + minCount = 4; + } + } +} + +NO_INLINE void CCoder::MakeTables(unsigned maxHuffLen) +{ + Huffman_Generate(mainFreqs, mainCodes, m_NewLevels.litLenLevels, kFixedMainTableSize, maxHuffLen); + Huffman_Generate(distFreqs, distCodes, m_NewLevels.distLevels, kDistTableSize64, maxHuffLen); +} + +static NO_INLINE UInt32 Huffman_GetPrice(const UInt32 *freqs, const Byte *lens, UInt32 num) +{ + UInt32 price = 0; + UInt32 i; + for (i = 0; i < num; i++) + price += lens[i] * freqs[i]; + return price; +} + +static NO_INLINE UInt32 Huffman_GetPrice_Spec( + const UInt32 *freqs, const Byte *lens, UInt32 num, + const Byte *extraBits, UInt32 extraBase) +{ + return + Huffman_GetPrice(freqs, lens, num) + + Huffman_GetPrice(freqs + extraBase, extraBits, num - extraBase); +} + +NO_INLINE UInt32 CCoder::GetLzBlockPrice() const +{ + return + Huffman_GetPrice_Spec(mainFreqs, m_NewLevels.litLenLevels, + kFixedMainTableSize, m_LenDirectBits, kSymbolMatch) + + Huffman_GetPrice_Spec(distFreqs, m_NewLevels.distLevels, + kDistTableSize64, kDistDirectBits, 0); +} + +NO_INLINE void CCoder::TryBlock() +{ + memset(mainFreqs, 0, sizeof(mainFreqs)); + memset(distFreqs, 0, sizeof(distFreqs)); + + m_ValueIndex = 0; + UInt32 blockSize = BlockSizeRes; + BlockSizeRes = 0; + for (;;) + { + if (m_OptimumCurrentIndex == m_OptimumEndIndex) + { + if (m_Pos >= kMatchArrayLimit + || BlockSizeRes >= blockSize + || (!m_SecondPass && ((Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) == 0) || m_ValueIndex >= m_ValueBlockSize))) + break; + } + UInt32 pos; + UInt32 len; + if (_fastMode) + len = GetOptimalFast(pos); + else + len = GetOptimal(pos); + CCodeValue &codeValue = m_Values[m_ValueIndex++]; + if (len >= kMatchMinLen) + { + const UInt32 newLen = len - kMatchMinLen; + codeValue.Len = (UInt16)newLen; + mainFreqs[kSymbolMatch + (size_t)g_LenSlots[newLen]]++; + codeValue.Pos = (UInt16)pos; + distFreqs[GetPosSlot(pos)]++; + } + else + { + const unsigned b = *(Inline_MatchFinder_GetPointerToCurrentPos(&_lzInWindow) - m_AdditionalOffset); + mainFreqs[b]++; + codeValue.Len = k_CodeValue_Len_Is_Literal_Flag; + codeValue.Pos = (UInt16)b; + } + m_AdditionalOffset -= len; + BlockSizeRes += len; + } + mainFreqs[kSymbolEndOfBlock]++; + m_AdditionalOffset += BlockSizeRes; + m_SecondPass = true; +} + +NO_INLINE void CCoder::SetPrices(const CLevels &levels) +{ + if (_fastMode) + return; + UInt32 i; + for (i = 0; i < 256; i++) + { + Byte price = levels.litLenLevels[i]; + m_LiteralPrices[i] = ((price != 0) ? price : kNoLiteralStatPrice); + } + + for (i = 0; i < m_NumLenCombinations; i++) + { + UInt32 slot = g_LenSlots[i]; + Byte price = levels.litLenLevels[kSymbolMatch + (size_t)slot]; + m_LenPrices[i] = (Byte)(((price != 0) ? price : kNoLenStatPrice) + m_LenDirectBits[slot]); + } + + for (i = 0; i < kDistTableSize64; i++) + { + Byte price = levels.distLevels[i]; + m_PosPrices[i] = (Byte)(((price != 0) ? price: kNoPosStatPrice) + kDistDirectBits[i]); + } +} + +#if MAX_HUF_LEN_12 > 12 +// Huffman_ReverseBits() now supports 12-bits values only. +#error Stop_Compiling_Bad_MAX_HUF_LEN_12 +#endif +static NO_INLINE void Huffman_ReverseBits(UInt32 *codes, const Byte *lens, UInt32 num) +{ + const Byte * const lens_lim = lens + num; + do + { + // we should change constants, if lens[*] can be larger than 12. + UInt32 x = *codes; + x = ((x & (0x555 )) << 2) + (x & (0xAAA )); + x = ((x & (0x333 << 1)) << 4) | (x & (0xCCC << 1)); + x = ((x & (0xF0F << 3)) << 8) | (x & (0x0F0 << 3)); + // we can use (x) instead of (x & (0xFF << 7)), if we support garabage data after (*lens) bits. + *codes++ = (((x & (0xFF << 7)) << 16) | x) >> (*lens ^ 31); + } + while (++lens != lens_lim); +} + +NO_INLINE void CCoder::WriteBlock() +{ + Huffman_ReverseBits(mainCodes, m_NewLevels.litLenLevels, kFixedMainTableSize); + Huffman_ReverseBits(distCodes, m_NewLevels.distLevels, kDistTableSize64); + + CCodeValue *values = m_Values; + const CCodeValue * const values_lim = values + m_ValueIndex; + + if (values != values_lim) + do + { + const UInt32 len = values->Len; + const UInt32 dist = values->Pos; + if (len == k_CodeValue_Len_Is_Literal_Flag) + WRITE_HF2(mainCodes, m_NewLevels.litLenLevels, dist); + else + { + const unsigned lenSlot = g_LenSlots[len]; + WRITE_HF2(mainCodes, m_NewLevels.litLenLevels, kSymbolMatch + lenSlot); + m_OutStream.WriteBits(len - m_LenStart[lenSlot], m_LenDirectBits[lenSlot]); + const unsigned posSlot = GetPosSlot(dist); + WRITE_HF2(distCodes, m_NewLevels.distLevels, posSlot); + m_OutStream.WriteBits(dist - kDistStart[posSlot], kDistDirectBits[posSlot]); + } + } + while (++values != values_lim); + WRITE_HF2_NO_INLINE(mainCodes, m_NewLevels.litLenLevels, kSymbolEndOfBlock); +} + +static UInt32 GetStorePrice(UInt32 blockSize, unsigned bitPosition) +{ + UInt32 price = 0; + do + { + UInt32 nextBitPosition = (bitPosition + kFinalBlockFieldSize + kBlockTypeFieldSize) & 7; + unsigned numBitsForAlign = nextBitPosition > 0 ? (8 - nextBitPosition): 0; + UInt32 curBlockSize = (blockSize < (1 << 16)) ? blockSize : (1 << 16) - 1; + price += kFinalBlockFieldSize + kBlockTypeFieldSize + numBitsForAlign + (2 + 2) * 8 + curBlockSize * 8; + bitPosition = 0; + blockSize -= curBlockSize; + } + while (blockSize != 0); + return price; +} + +void CCoder::WriteStoreBlock(UInt32 blockSize, UInt32 additionalOffset, bool finalBlock) +{ + do + { + UInt32 curBlockSize = (blockSize < (1 << 16)) ? blockSize : (1 << 16) - 1; + blockSize -= curBlockSize; + WriteBits((finalBlock && (blockSize == 0) ? NFinalBlockField::kFinalBlock: NFinalBlockField::kNotFinalBlock), kFinalBlockFieldSize); + WriteBits(NBlockType::kStored, kBlockTypeFieldSize); + m_OutStream.FlushByte(); + WriteBits((UInt16)curBlockSize, kStoredBlockLengthFieldSize); + WriteBits((UInt16)~curBlockSize, kStoredBlockLengthFieldSize); + const Byte *data = Inline_MatchFinder_GetPointerToCurrentPos(&_lzInWindow)- additionalOffset; + for (UInt32 i = 0; i < curBlockSize; i++) + m_OutStream.WriteByte(data[i]); + additionalOffset -= curBlockSize; + } + while (blockSize != 0); +} + +NO_INLINE UInt32 CCoder::TryDynBlock(unsigned tableIndex, UInt32 numPasses) +{ + CTables &t = m_Tables[tableIndex]; + BlockSizeRes = t.BlockSizeRes; + UInt32 posTemp = t.m_Pos; + SetPrices(t); + + for (UInt32 p = 0; p < numPasses; p++) + { + m_Pos = posTemp; + TryBlock(); + const unsigned numHuffBits = + m_ValueIndex > 18000 ? MAX_HUF_LEN_12 : + m_ValueIndex > 7000 ? 11 : + m_ValueIndex > 2000 ? 10 : 9; + MakeTables(numHuffBits); + SetPrices(m_NewLevels); + } + + (CLevels &)t = m_NewLevels; + + m_NumLitLenLevels = kMainTableSize; + while (m_NumLitLenLevels > kNumLitLenCodesMin && m_NewLevels.litLenLevels[(size_t)m_NumLitLenLevels - 1] == 0) + m_NumLitLenLevels--; + + m_NumDistLevels = kDistTableSize64; + while (m_NumDistLevels > kNumDistCodesMin && m_NewLevels.distLevels[(size_t)m_NumDistLevels - 1] == 0) + m_NumDistLevels--; + + UInt32 levelFreqs[kLevelTableSize]; + memset(levelFreqs, 0, sizeof(levelFreqs)); + + LevelTableDummy(m_NewLevels.litLenLevels, m_NumLitLenLevels, levelFreqs); + LevelTableDummy(m_NewLevels.distLevels, m_NumDistLevels, levelFreqs); + + Huffman_Generate(levelFreqs, levelCodes, levelLens, kLevelTableSize, kMaxLevelBitLength); + + m_NumLevelCodes = kNumLevelCodesMin; + for (UInt32 i = 0; i < kLevelTableSize; i++) + { + Byte level = levelLens[kCodeLengthAlphabetOrder[i]]; + if (level > 0 && i >= m_NumLevelCodes) + m_NumLevelCodes = i + 1; + m_LevelLevels[i] = level; + } + + return GetLzBlockPrice() + + Huffman_GetPrice_Spec(levelFreqs, levelLens, kLevelTableSize, kLevelDirectBits, kTableDirectLevels) + + kNumLenCodesFieldSize + kNumDistCodesFieldSize + kNumLevelCodesFieldSize + + m_NumLevelCodes * kLevelFieldSize + kFinalBlockFieldSize + kBlockTypeFieldSize; +} + +NO_INLINE UInt32 CCoder::TryFixedBlock(unsigned tableIndex) +{ + CTables &t = m_Tables[tableIndex]; + BlockSizeRes = t.BlockSizeRes; + m_Pos = t.m_Pos; + m_NewLevels.SetFixedLevels(); + SetPrices(m_NewLevels); + TryBlock(); + return kFinalBlockFieldSize + kBlockTypeFieldSize + GetLzBlockPrice(); +} + +NO_INLINE UInt32 CCoder::GetBlockPrice(unsigned tableIndex, unsigned numDivPasses) +{ + CTables &t = m_Tables[tableIndex]; + t.StaticMode = false; + UInt32 price = TryDynBlock(tableIndex, m_NumPasses); + t.BlockSizeRes = BlockSizeRes; + UInt32 numValues = m_ValueIndex; + UInt32 posTemp = m_Pos; + UInt32 additionalOffsetEnd = m_AdditionalOffset; + + if (m_CheckStatic && m_ValueIndex <= kFixedHuffmanCodeBlockSizeMax) + { + const UInt32 fixedPrice = TryFixedBlock(tableIndex); + t.StaticMode = (fixedPrice < price); + if (t.StaticMode) + price = fixedPrice; + } + + const UInt32 storePrice = GetStorePrice(BlockSizeRes, 0); // bitPosition + t.StoreMode = (storePrice <= price); + if (t.StoreMode) + price = storePrice; + + t.UseSubBlocks = false; + + if (numDivPasses > 1 && numValues >= kDivideCodeBlockSizeMin) + { + CTables &t0 = m_Tables[(tableIndex << 1)]; + (CLevels &)t0 = t; + t0.BlockSizeRes = t.BlockSizeRes >> 1; + t0.m_Pos = t.m_Pos; + UInt32 subPrice = GetBlockPrice((tableIndex << 1), numDivPasses - 1); + + UInt32 blockSize2 = t.BlockSizeRes - t0.BlockSizeRes; + if (t0.BlockSizeRes >= kDivideBlockSizeMin && blockSize2 >= kDivideBlockSizeMin) + { + CTables &t1 = m_Tables[(tableIndex << 1) + 1]; + (CLevels &)t1 = t; + t1.BlockSizeRes = blockSize2; + t1.m_Pos = m_Pos; + m_AdditionalOffset -= t0.BlockSizeRes; + subPrice += GetBlockPrice((tableIndex << 1) + 1, numDivPasses - 1); + t.UseSubBlocks = (subPrice < price); + if (t.UseSubBlocks) + price = subPrice; + } + } + + m_AdditionalOffset = additionalOffsetEnd; + m_Pos = posTemp; + return price; +} + +void CCoder::CodeBlock(unsigned tableIndex, bool finalBlock) +{ + CTables &t = m_Tables[tableIndex]; + if (t.UseSubBlocks) + { + CodeBlock((tableIndex << 1), false); + CodeBlock((tableIndex << 1) + 1, finalBlock); + } + else + { + if (t.StoreMode) + WriteStoreBlock(t.BlockSizeRes, m_AdditionalOffset, finalBlock); + else + { + WriteBits((finalBlock ? NFinalBlockField::kFinalBlock: NFinalBlockField::kNotFinalBlock), kFinalBlockFieldSize); + if (t.StaticMode) + { + WriteBits(NBlockType::kFixedHuffman, kBlockTypeFieldSize); + TryFixedBlock(tableIndex); + unsigned i; + const unsigned kMaxStaticHuffLen = 9; + for (i = 0; i < kFixedMainTableSize; i++) + mainFreqs[i] = (UInt32)1 << (kMaxStaticHuffLen - m_NewLevels.litLenLevels[i]); + for (i = 0; i < kFixedDistTableSize; i++) + distFreqs[i] = (UInt32)1 << (kMaxStaticHuffLen - m_NewLevels.distLevels[i]); + MakeTables(kMaxStaticHuffLen); + } + else + { + if (m_NumDivPasses > 1 || m_CheckStatic) + TryDynBlock(tableIndex, 1); + WriteBits(NBlockType::kDynamicHuffman, kBlockTypeFieldSize); + WriteBits(m_NumLitLenLevels - kNumLitLenCodesMin, kNumLenCodesFieldSize); + WriteBits(m_NumDistLevels - kNumDistCodesMin, kNumDistCodesFieldSize); + WriteBits(m_NumLevelCodes - kNumLevelCodesMin, kNumLevelCodesFieldSize); + + for (UInt32 i = 0; i < m_NumLevelCodes; i++) + WriteBits(m_LevelLevels[i], kLevelFieldSize); + + Huffman_ReverseBits(levelCodes, levelLens, kLevelTableSize); + LevelTableCode(m_NewLevels.litLenLevels, m_NumLitLenLevels, levelLens, levelCodes); + LevelTableCode(m_NewLevels.distLevels, m_NumDistLevels, levelLens, levelCodes); + } + WriteBlock(); + } + m_AdditionalOffset -= t.BlockSizeRes; + } +} + + +HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */ , const UInt64 * /* outSize */ , ICompressProgressInfo *progress) +{ + m_CheckStatic = (m_NumPasses != 1 || m_NumDivPasses != 1); + m_IsMultiPass = (m_CheckStatic || (m_NumPasses != 1 || m_NumDivPasses != 1)); + + /* we can set stream mode before MatchFinder_Create + if default MatchFinder mode was not STREAM_MODE) */ + // MatchFinder_SET_STREAM_MODE(&_lzInWindow); + + CSeqInStreamWrap _seqInStream; + _seqInStream.Init(inStream); + MatchFinder_SET_STREAM(&_lzInWindow, &_seqInStream.vt) + + RINOK(Create()) + + m_ValueBlockSize = (7 << 10) + (1 << 12) * m_NumDivPasses; + + UInt64 nowPos = 0; + + MatchFinder_Init(&_lzInWindow); + m_OutStream.SetStream(outStream); + m_OutStream.Init(); + + m_OptimumEndIndex = m_OptimumCurrentIndex = 0; + + CTables &t = m_Tables[1]; + t.m_Pos = 0; + t.InitStructures(); + + m_AdditionalOffset = 0; + do + { + t.BlockSizeRes = kBlockUncompressedSizeThreshold; + m_SecondPass = false; + GetBlockPrice(1, m_NumDivPasses); + CodeBlock(1, Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) == 0); + nowPos += m_Tables[1].BlockSizeRes; + if (progress != NULL) + { + UInt64 packSize = m_OutStream.GetProcessedSize(); + RINOK(progress->SetRatioInfo(&nowPos, &packSize)) + } + } + while (Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) != 0); + + if (_seqInStream.Res != S_OK) + return _seqInStream.Res; + + if (_lzInWindow.result != SZ_OK) + return SResToHRESULT(_lzInWindow.result); + return m_OutStream.Flush(); +} + +HRESULT CCoder::BaseCode(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + try { return CodeReal(inStream, outStream, inSize, outSize, progress); } + catch(const COutBufferException &e) { return e.ErrorCode; } + catch(...) { return E_FAIL; } +} + +Z7_COM7F_IMF(CCOMCoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) + { return BaseCode(inStream, outStream, inSize, outSize, progress); } + +Z7_COM7F_IMF(CCOMCoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)) + { return BaseSetEncoderProperties2(propIDs, props, numProps); } + +Z7_COM7F_IMF(CCOMCoder64::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) + { return BaseCode(inStream, outStream, inSize, outSize, progress); } + +Z7_COM7F_IMF(CCOMCoder64::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)) + { return BaseSetEncoderProperties2(propIDs, props, numProps); } + +}}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateEncoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateEncoder.h new file mode 100644 index 0000000..be181e1 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateEncoder.h @@ -0,0 +1,203 @@ +// DeflateEncoder.h + +#ifndef ZIP7_INC_DEFLATE_ENCODER_H +#define ZIP7_INC_DEFLATE_ENCODER_H + +#include "../../../C/LzFind.h" + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "BitlEncoder.h" +#include "DeflateConst.h" + +namespace NCompress { +namespace NDeflate { +namespace NEncoder { + +struct CCodeValue +{ + UInt16 Len; + UInt16 Pos; + void SetAsLiteral() { Len = (1 << 15); } + bool IsLiteral() const { return (Len >= (1 << 15)); } +}; + +struct COptimal +{ + UInt32 Price; + UInt16 PosPrev; + UInt16 BackPrev; +}; + +const UInt32 kNumOptsBase = 1 << 12; +const UInt32 kNumOpts = kNumOptsBase + kMatchMaxLen; + +class CCoder; + +struct CTables: public CLevels +{ + bool UseSubBlocks; + bool StoreMode; + bool StaticMode; + UInt32 BlockSizeRes; + UInt32 m_Pos; + void InitStructures(); +}; + + +struct CEncProps +{ + int Level; + int algo; + int fb; + int btMode; + UInt32 mc; + UInt32 numPasses; + + CEncProps() + { + Level = -1; + mc = 0; + algo = fb = btMode = -1; + numPasses = (UInt32)(Int32)-1; + } + void Normalize(); +}; + +class CCoder +{ + CMatchFinder _lzInWindow; + CBitlEncoder m_OutStream; + +public: + CCodeValue *m_Values; + + UInt16 *m_MatchDistances; + UInt32 m_NumFastBytes; + bool _fastMode; + bool _btMode; + + UInt16 *m_OnePosMatchesMemory; + UInt16 *m_DistanceMemory; + + UInt32 m_Pos; + + unsigned m_NumPasses; + unsigned m_NumDivPasses; + bool m_CheckStatic; + bool m_IsMultiPass; + UInt32 m_ValueBlockSize; + + UInt32 m_NumLenCombinations; + UInt32 m_MatchMaxLen; + const Byte *m_LenStart; + const Byte *m_LenDirectBits; + + bool m_Created; + bool m_Deflate64Mode; + + Byte m_LevelLevels[kLevelTableSize]; + unsigned m_NumLitLenLevels; + unsigned m_NumDistLevels; + UInt32 m_NumLevelCodes; + UInt32 m_ValueIndex; + + bool m_SecondPass; + UInt32 m_AdditionalOffset; + + UInt32 m_OptimumEndIndex; + UInt32 m_OptimumCurrentIndex; + + Byte m_LiteralPrices[256]; + Byte m_LenPrices[kNumLenSymbolsMax]; + Byte m_PosPrices[kDistTableSize64]; + + CLevels m_NewLevels; + UInt32 mainFreqs[kFixedMainTableSize]; + UInt32 distFreqs[kDistTableSize64]; + UInt32 mainCodes[kFixedMainTableSize]; + UInt32 distCodes[kDistTableSize64]; + UInt32 levelCodes[kLevelTableSize]; + Byte levelLens[kLevelTableSize]; + + UInt32 BlockSizeRes; + + CTables *m_Tables; + COptimal m_Optimum[kNumOpts]; + + UInt32 m_MatchFinderCycles; + + void GetMatches(); + void MovePos(UInt32 num); + UInt32 Backward(UInt32 &backRes, UInt32 cur); + UInt32 GetOptimal(UInt32 &backRes); + UInt32 GetOptimalFast(UInt32 &backRes); + + void LevelTableDummy(const Byte *levels, unsigned numLevels, UInt32 *freqs); + + void WriteBits(UInt32 value, unsigned numBits); + void LevelTableCode(const Byte *levels, unsigned numLevels, const Byte *lens, const UInt32 *codes); + + void MakeTables(unsigned maxHuffLen); + UInt32 GetLzBlockPrice() const; + void TryBlock(); + UInt32 TryDynBlock(unsigned tableIndex, UInt32 numPasses); + + UInt32 TryFixedBlock(unsigned tableIndex); + + void SetPrices(const CLevels &levels); + void WriteBlock(); + + HRESULT Create(); + void Free(); + + void WriteStoreBlock(UInt32 blockSize, UInt32 additionalOffset, bool finalBlock); + void WriteTables(bool writeMode, bool finalBlock); + + void WriteBlockData(bool writeMode, bool finalBlock); + + UInt32 GetBlockPrice(unsigned tableIndex, unsigned numDivPasses); + void CodeBlock(unsigned tableIndex, bool finalBlock); + + void SetProps(const CEncProps *props2); +public: + CCoder(bool deflate64Mode = false); + ~CCoder(); + + HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + + HRESULT BaseCode(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + + HRESULT BaseSetEncoderProperties2(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); +}; + + +class CCOMCoder Z7_final: + public ICompressCoder, + public ICompressSetCoderProperties, + public CMyUnknownImp, + public CCoder +{ + Z7_IFACES_IMP_UNK_2(ICompressCoder, ICompressSetCoderProperties) +public: + CCOMCoder(): CCoder(false) {} +}; + +class CCOMCoder64 Z7_final: + public ICompressCoder, + public ICompressSetCoderProperties, + public CMyUnknownImp, + public CCoder +{ + Z7_IFACES_IMP_UNK_2(ICompressCoder, ICompressSetCoderProperties) +public: + CCOMCoder64(): CCoder(true) {} +}; + +}}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateRegister.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateRegister.cpp new file mode 100644 index 0000000..eaedb84 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/DeflateRegister.cpp @@ -0,0 +1,25 @@ +// DeflateRegister.cpp + +#include "StdAfx.h" + +#include "../Common/RegisterCodec.h" + +#include "DeflateDecoder.h" +#if !defined(Z7_EXTRACT_ONLY) && !defined(Z7_DEFLATE_EXTRACT_ONLY) +#include "DeflateEncoder.h" +#endif + +namespace NCompress { +namespace NDeflate { + +REGISTER_CODEC_CREATE(CreateDec, NDecoder::CCOMCoder) + +#if !defined(Z7_EXTRACT_ONLY) && !defined(Z7_DEFLATE_EXTRACT_ONLY) +REGISTER_CODEC_CREATE(CreateEnc, NEncoder::CCOMCoder) +#else +#define CreateEnc NULL +#endif + +REGISTER_CODEC_2(Deflate, CreateDec, CreateEnc, 0x40108, "Deflate") + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/HuffmanDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/HuffmanDecoder.h new file mode 100644 index 0000000..318190d --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/HuffmanDecoder.h @@ -0,0 +1,527 @@ +// Compress/HuffmanDecoder.h + +#ifndef ZIP7_INC_COMPRESS_HUFFMAN_DECODER_H +#define ZIP7_INC_COMPRESS_HUFFMAN_DECODER_H + +#include "../../../C/CpuArch.h" +#include "../../Common/MyTypes.h" + +namespace NCompress { +namespace NHuffman { + +// const unsigned kNumTableBits_Default = 9; + +#if 0 || 0 && defined(MY_CPU_64BIT) +// for debug or optimization: +// 64-BIT limit array can be faster for some compilers. +// for debug or optimization: +#define Z7_HUFF_USE_64BIT_LIMIT +#else +// sizet value variable allows to eliminate some move operation in some compilers. +// for debug or optimization: +// #define Z7_HUFF_USE_SIZET_VALUE +#endif + +// v0 must normalized to (32 bits) : (v0 < ((UInt64)1 << 32)) + +#ifdef Z7_HUFF_USE_64BIT_LIMIT + typedef UInt64 CLimitInt; + typedef UInt64 CValueInt; + // all _limits[*] are normalized and limited by ((UInt64)1 << 32). + // we don't use (v1) in this branch + #define Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) 32 + #define Z7_HUFF_TABLE_COMPARE(huf, kNumTableBits, v0, v1) \ + ((NCompress::NHuffman::CLimitInt)v0 >= (huf)->_limits[0]) + #define Z7_HUFF_GET_VAL_FOR_LIMITS(v0, v1, kNumBitsMax, kNumTableBits) (v0) + #define Z7_HUFF_GET_VAL_FOR_TABLE( v0, v1, kNumBitsMax, kNumTableBits) ((v0) >> (32 - kNumTableBits)) + #define Z7_HUFF_PRECALC_V1(kNumTableBits, v0, v1) +#else + typedef UInt32 CLimitInt; + typedef + #ifdef Z7_HUFF_USE_SIZET_VALUE + size_t + #else + UInt32 + #endif + CValueInt; + // v1 must be precalculated from v0 in this branch + // _limits[0] and (v1) are normalized and limited by (1 << kNumTableBits). + // _limits[non_0] are normalized and limited by (1 << kNumBitsMax). + #define Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) (kNumBitsMax) + #define Z7_HUFF_TABLE_COMPARE(huf, kNumTableBits, v0, v1) \ + ((NCompress::NHuffman::CLimitInt)v1 >= (huf)->_limits[0]) + #define Z7_HUFF_GET_VAL_FOR_LIMITS(v0, v1, kNumBitsMax, kNumTableBits) ((v0) >> (32 - kNumBitsMax)) + #define Z7_HUFF_GET_VAL_FOR_TABLE( v0, v1, kNumBitsMax, kNumTableBits) (v1) + #define Z7_HUFF_PRECALC_V1(kNumTableBits, v0, v1) const UInt32 v1 = ((v0) >> (32 - kNumTableBits)); +#endif + + +enum enum_BuildMode +{ + k_BuildMode_Partial = 0, + k_BuildMode_Full = 1, + k_BuildMode_Full_or_Empty = 2 +}; + + +template +struct CDecoderBase +{ + CLimitInt _limits[kNumBitsMax + 2 - kNumTableBits]; + UInt32 _poses[kNumBitsMax - kNumTableBits]; // unsigned +union +{ + // if defined(MY_CPU_64BIT), we need 64-bit alignment for _symbols. + // if !defined(MY_CPU_64BIT), we need 32-bit alignment for _symbols + // but we provide alignment for _lens. + // _symbols also will be aligned, if _lens are aligned + #if defined(MY_CPU_64BIT) + UInt64 + #else + UInt32 + #endif + _pad_align[m_NumSymbols < (1u << sizeof(symType) * 8) ? 1 : -1]; + /* if symType is Byte, we use 16-bytes padding to avoid cache + bank conflict between _lens and _symbols: */ + Byte _lens[(1 << kNumTableBits) + (sizeof(symType) == 1 ? 16 : 0)]; +} _u; + symType _symbols[(1 << kNumTableBits) + m_NumSymbols - (kNumTableBits + 1)]; + + /* + Z7_FORCE_INLINE + bool IsFull() const + { + return _limits[kNumBitsMax - kNumTableBits] == + (CLimitInt)1u << Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax); + } + Z7_FORCE_INLINE + bool IsEmpty() const + { + return _limits[kNumBitsMax - kNumTableBits] == 0; + } + Z7_FORCE_INLINE + bool Is_Full_or_Empty() const + { + return 0 == (_limits[kNumBitsMax - kNumTableBits] & + ~((CLimitInt)1 << Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax))); + } + */ + + Z7_FORCE_INLINE + bool Build(const Byte *lens, enum_BuildMode buidMode = k_BuildMode_Partial) throw() + { + unsigned counts[kNumBitsMax + 1]; + size_t i; + for (i = 0; i <= kNumBitsMax; i++) + counts[i] = 0; + for (i = 0; i < m_NumSymbols; i++) + counts[lens[i]]++; + + UInt32 sum = 0; + for (i = 1; i <= kNumTableBits; i++) + { + sum <<= 1; + sum += counts[i]; + } + + CLimitInt startPos = (CLimitInt)sum; + _limits[0] = + #ifdef Z7_HUFF_USE_64BIT_LIMIT + startPos << (Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) - kNumTableBits); + #else + startPos; + #endif + + for (i = kNumTableBits + 1; i <= kNumBitsMax; i++) + { + startPos <<= 1; + _poses[i - (kNumTableBits + 1)] = (UInt32)(startPos - sum); + const unsigned cnt = counts[i]; + counts[i] = sum; + sum += cnt; + startPos += cnt; + _limits[i - kNumTableBits] = startPos << (Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) - i); + } + + _limits[kNumBitsMax + 1 - kNumTableBits] = + (CLimitInt)1 << Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax); + + if (buidMode == k_BuildMode_Partial) + { + if (startPos > (1u << kNumBitsMax)) return false; + } + else + { + if (buidMode != k_BuildMode_Full && startPos == 0) return true; + if (startPos != (1u << kNumBitsMax)) return false; + } + size_t sum2 = 0; + for (i = 1; i <= kNumTableBits; i++) + { + const unsigned cnt = counts[i] << (kNumTableBits - i); + counts[i] = (unsigned)sum2 >> (kNumTableBits - i); + memset(_u._lens + sum2, (int)i, cnt); + sum2 += cnt; + } + +#ifdef MY_CPU_64BIT + symType4 + // UInt64 // for symType = UInt16 + // UInt32 // for symType = Byte +#else + UInt32 +#endif + v = 0; + for (i = 0; i < m_NumSymbols; i++, + v += + 1 + + ( (UInt32)1 << (sizeof(symType) * 8 * 1)) + // 0x00010001 // for symType = UInt16 + // 0x00000101 // for symType = Byte +#ifdef MY_CPU_64BIT + + ((symType4)1 << (sizeof(symType) * 8 * 2)) + + ((symType4)1 << (sizeof(symType) * 8 * 3)) + // 0x0001000100010001 // for symType = UInt16 + // 0x0000000001010101 // for symType = Byte +#endif + ) + { + const unsigned len = lens[i]; + if (len == 0) + continue; + const size_t offset = counts[len]; + counts[len] = (unsigned)offset + 1; + if (len >= kNumTableBits) + _symbols[offset] = (symType)v; + else + { + Byte *s2 = (Byte *)(void *)_symbols + (offset << + (kNumTableBits + sizeof(symType) / 2 - len)); + Byte *lim = s2 + ((size_t)1 << + (kNumTableBits + sizeof(symType) / 2 - len)); + if (len >= kNumTableBits - 2) + { + *(symType2 *)(void *)(s2 ) = (symType2)v; + *(symType2 *)(void *)(lim - sizeof(symType) * 2) = (symType2)v; + } + else + { +#ifdef MY_CPU_64BIT + symType4 *s = (symType4 *)(void *)s2; + do + { + s[0] = v; s[1] = v; s += 2; + } + while (s != (const symType4 *)(const void *)lim); +#else + symType2 *s = (symType2 *)(void *)s2; + do + { + s[0] = (symType2)v; s[1] = (symType2)v; s += 2; + s[0] = (symType2)v; s[1] = (symType2)v; s += 2; + } + while (s != (const symType2 *)(const void *)lim); +#endif + } + } + } + return true; + } + + +#define Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES(_numBits_, kNumBitsMax, error_op) if (_numBits_ > kNumBitsMax) { error_op } +#define Z7_HUFF_DECODE_ERROR_SYM_CHECK_NO( _numBits_, kNumBitsMax, error_op) + + +#define Z7_HUFF_DECODE_BASE_TREE_BRANCH(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + get_val_for_limits, \ + check_op, error_op, _numBits_) \ +{ \ + const NHuffman::CValueInt _val = get_val_for_limits(v0, v1, kNumBitsMax, kNumTableBits); \ + _numBits_ = kNumTableBits + 1; \ + if ((NCompress::NHuffman::CLimitInt)_val >= (huf)->_limits[1]) \ + do { _numBits_++; } \ + while ((NCompress::NHuffman::CLimitInt)_val >= (huf)->_limits[_numBits_ - kNumTableBits]); \ + check_op(_numBits_, kNumBitsMax, error_op) \ + sym = (huf)->_symbols[(/* (UInt32) */ (_val >> ((Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) - (unsigned)_numBits_)))) \ + - (huf)->_poses[_numBits_ - (kNumTableBits + 1)]]; \ +} + +/* + Z7_HUFF_DECODE_BASE_TREE_BRANCH(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + get_val_for_limits, \ + check_op, error_op, _numBits_) \ + +*/ + +#define Z7_HUFF_DECODE_BASE(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + get_val_for_table, get_val_for_limits, \ + check_op, error_op, move_pos_op, after_op, bs) \ +{ \ + if (Z7_HUFF_TABLE_COMPARE(huf, kNumTableBits, v0, v1)) \ + { \ + const NHuffman::CValueInt _val = get_val_for_limits(v0, v1, kNumBitsMax, kNumTableBits); \ + size_t _numBits_ = kNumTableBits + 1; \ + if ((NCompress::NHuffman::CLimitInt)_val >= (huf)->_limits[1]) \ + do { _numBits_++; } \ + while ((NCompress::NHuffman::CLimitInt)_val >= (huf)->_limits[_numBits_ - kNumTableBits]); \ + check_op(_numBits_, kNumBitsMax, error_op) \ + sym = (huf)->_symbols[(/* (UInt32) */ (_val >> ((Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) - (unsigned)_numBits_)))) \ + - (huf)->_poses[_numBits_ - (kNumTableBits + 1)]]; \ + move_pos_op(bs, _numBits_); \ + } \ + else \ + { \ + const size_t _val = get_val_for_table(v0, v1, kNumBitsMax, kNumTableBits); \ + const size_t _numBits_ = (huf)->_u._lens[_val]; \ + sym = (huf)->_symbols[_val]; \ + move_pos_op(bs, _numBits_); \ + } \ + after_op \ +} + +#define Z7_HUFF_DECODE_10(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + check_op, error_op, move_pos_op, after_op, bs) \ + Z7_HUFF_DECODE_BASE(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + Z7_HUFF_GET_VAL_FOR_TABLE, \ + Z7_HUFF_GET_VAL_FOR_LIMITS, \ + check_op, error_op, move_pos_op, after_op, bs) \ + + +#define Z7_HUFF_DECODE_VAL_IN_HIGH32(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, \ + check_op, error_op, move_pos_op, after_op, bs) \ +{ \ + Z7_HUFF_PRECALC_V1(kNumTableBits, v0, _v1_temp) \ + Z7_HUFF_DECODE_10(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, _v1_temp, \ + check_op, error_op, move_pos_op, after_op, bs) \ +} + +#if 0 || defined(Z7_HUFF_USE_64BIT_LIMIT) +// this branch uses bitStream->GetValue_InHigh32bits(). +#define Z7_HUFF_DECODE_0(sym, huf, kNumBitsMax, kNumTableBits, bitStream, \ + check_op, error_op, move_pos_op) \ +{ \ + const UInt32 v0 = (bitStream)->GetValue_InHigh32bits(); \ + Z7_HUFF_PRECALC_V1(kNumTableBits, v0, v1); \ + Z7_HUFF_DECODE_BASE(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + Z7_HUFF_GET_VAL_FOR_TABLE, \ + Z7_HUFF_GET_VAL_FOR_LIMITS, \ + check_op, error_op, move_pos_op, {}, bitStream) \ +} +#else +/* +this branch uses bitStream->GetValue(). +So we use SIMPLE versions for v0, v1 calculation: + v0 is normalized for kNumBitsMax + v1 is normalized for kNumTableBits +*/ +#define Z7_HUFF_GET_VAL_FOR_LIMITS_SIMPLE(v0, v1, kNumBitsMax, kNumTableBits) v0 +#define Z7_HUFF_GET_VAL_FOR_TABLE_SIMPLE( v0, v1, kNumBitsMax, kNumTableBits) v1 +#define Z7_HUFF_DECODE_0(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op, move_pos_op) \ +{ \ + const UInt32 v0 = (bitStream)->GetValue(kNumBitsMax); \ + const UInt32 v1 = v0 >> (kNumBitsMax - kNumTableBits); \ + Z7_HUFF_DECODE_BASE(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + Z7_HUFF_GET_VAL_FOR_TABLE_SIMPLE, \ + Z7_HUFF_GET_VAL_FOR_LIMITS_SIMPLE, \ + check_op, error_op, move_pos_op, {}, bitStream) \ +} +#endif + +#define Z7_HUFF_bitStream_MovePos(bitStream, numBits) (bitStream)->MovePos((unsigned)(numBits)) + +#define Z7_HUFF_DECODE_1(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op) \ + Z7_HUFF_DECODE_0(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op, \ + Z7_HUFF_bitStream_MovePos) + +// MovePosCheck + +#define Z7_HUFF_DECODE_2(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op) \ + Z7_HUFF_DECODE_0(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op, \ + Z7_HUFF_bitStream_MovePos) + +// MovePosCheck + +#define Z7_HUFF_DECODE_CHECK(sym, huf, kNumBitsMax, kNumTableBits, bitStream, error_op) \ + Z7_HUFF_DECODE_1( sym, huf, kNumBitsMax, kNumTableBits, bitStream, \ + Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES, error_op) + + template + Z7_FORCE_INLINE + bool Decode2(TBitDecoder *bitStream, unsigned &sym) const + { + Z7_HUFF_DECODE_CHECK(sym, this, kNumBitsMax, kNumTableBits, bitStream, + { return false; } + ) + return true; + } + + template + Z7_FORCE_INLINE + bool Decode_SymCheck_MovePosCheck(TBitDecoder *bitStream, unsigned &sym) const + { + Z7_HUFF_DECODE_0(sym, this, kNumBitsMax, kNumTableBits, bitStream, + Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES, + { return false; }, + { return (bitStream)->MovePosCheck; } + ) + } + + template + Z7_FORCE_INLINE + unsigned Decode(TBitDecoder *bitStream) const + { + unsigned sym; + Z7_HUFF_DECODE_CHECK(sym, this, kNumBitsMax, kNumTableBits, bitStream, + { return (unsigned)(int)(Int32)0xffffffff; } + ) + return sym; + } + + + template + Z7_FORCE_INLINE + unsigned DecodeFull(TBitDecoder *bitStream) const + { + /* + const UInt32 val = bitStream->GetValue(kNumBitsMax); + if (val < _limits[kNumTableBits]) + { + const unsigned pair = _u._lens[(size_t)(val >> (kNumBitsMax - kNumTableBits))]; + bitStream->MovePos(pair & kPairLenMask); + return pair >> kNumPairLenBits; + } + + unsigned numBits; + for (numBits = kNumTableBits + 1; val >= _limits[numBits]; numBits++); + + bitStream->MovePos(numBits); + return _symbols[_poses[numBits] + (unsigned) + ((val - _limits[(size_t)numBits - 1]) >> (kNumBitsMax - numBits))]; + */ + unsigned sym; + Z7_HUFF_DECODE_2(sym, this, kNumBitsMax, kNumTableBits, bitStream, + Z7_HUFF_DECODE_ERROR_SYM_CHECK_NO, {} + ) + return sym; + } +}; + + +template +struct CDecoder: public CDecoderBase + {}; + +template +struct CDecoder256: public CDecoderBase + {}; + + +template +class CDecoder7b +{ +public: + Byte _lens[1 << 7]; + + bool Build(const Byte *lens, bool full) throw() + { + const unsigned kNumBitsMax = 7; + + unsigned counts[kNumBitsMax + 1]; + unsigned _poses[kNumBitsMax + 1]; + unsigned _limits[kNumBitsMax + 1]; + unsigned i; + for (i = 0; i <= kNumBitsMax; i++) + counts[i] = 0; + for (i = 0; i < numSymbols; i++) + counts[lens[i]]++; + + _limits[0] = 0; + const unsigned kMaxValue = 1u << kNumBitsMax; + unsigned startPos = 0; + unsigned sum = 0; + + for (i = 1; i <= kNumBitsMax; i++) + { + const unsigned cnt = counts[i]; + startPos += cnt << (kNumBitsMax - i); + _limits[i] = startPos; + counts[i] = sum; + _poses[i] = sum; + sum += cnt; + } + + counts[0] = sum; + _poses[0] = sum; + + if (full) + { + if (startPos != kMaxValue) + return false; + } + else + { + if (startPos > kMaxValue) + return false; + } + + + for (i = 0; i < numSymbols; i++) + { + const unsigned len = lens[i]; + if (len == 0) + continue; + const unsigned offset = counts[len]++; + { + Byte *dest = _lens + _limits[(size_t)len - 1] + + ((offset - _poses[len]) << (kNumBitsMax - len)); + const unsigned num = (unsigned)1 << (kNumBitsMax - len); + const unsigned val = (i << 3) + len; + for (unsigned k = 0; k < num; k++) + dest[k] = (Byte)val; + } + } + + if (!full) + { + const unsigned limit = _limits[kNumBitsMax]; + const unsigned num = ((unsigned)1 << kNumBitsMax) - limit; + Byte *dest = _lens + limit; + for (unsigned k = 0; k < num; k++) + dest[k] = (Byte) + // (0x1f << 3); + ((0x1f << 3) + 0x7); + } + + return true; + } + +#define Z7_HUFF_DECODER_7B_DECODE(dest, huf, get_val, move_pos, bs) \ + { \ + const unsigned pair = huf->_lens[(size_t)get_val(7)]; \ + const unsigned numBits = pair & 0x7; \ + move_pos(bs, numBits); \ + dest = pair >> 3; \ + } + + template + unsigned Decode(TBitDecoder *bitStream) const + { + const unsigned pair = _lens[(size_t)bitStream->GetValue(7)]; + bitStream->MovePos(pair & 0x7); + return pair >> 3; + } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeDecoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeDecoder.cpp new file mode 100644 index 0000000..bcf5d4f --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeDecoder.cpp @@ -0,0 +1,250 @@ +// ImplodeDecoder.cpp + +#include "StdAfx.h" + +#include "../../Common/Defs.h" + +#include "ImplodeDecoder.h" + +namespace NCompress { +namespace NImplode { +namespace NDecoder { + +bool CHuffmanDecoder::Build(const Byte *lens, unsigned numSymbols) throw() +{ + unsigned counts[kNumHuffmanBits + 1]; + unsigned i; + for (i = 0; i <= kNumHuffmanBits; i++) + counts[i] = 0; + for (i = 0; i < numSymbols; i++) + counts[lens[i]]++; + + const UInt32 kMaxValue = (UInt32)1 << kNumHuffmanBits; + // _limits[0] = kMaxValue; + UInt32 startPos = kMaxValue; + unsigned sum = 0; + + for (i = 1; i <= kNumHuffmanBits; i++) + { + const unsigned cnt = counts[i]; + const UInt32 range = (UInt32)cnt << (kNumHuffmanBits - i); + if (startPos < range) + return false; + startPos -= range; + _limits[i] = startPos; + _poses[i] = sum; + sum += cnt; + counts[i] = sum; + } + // counts[0] += sum; + if (startPos != 0) + return false; + for (i = 0; i < numSymbols; i++) + { + const unsigned len = lens[i]; + if (len != 0) + _symbols[--counts[len]] = (Byte)i; + } + return true; +} + + +unsigned CHuffmanDecoder::Decode(CInBit *inStream) const throw() +{ + const UInt32 val = inStream->GetValue(kNumHuffmanBits); + size_t numBits; + for (numBits = 1; val < _limits[numBits]; numBits++); + const unsigned sym = _symbols[_poses[numBits] + + (unsigned)((val - _limits[numBits]) >> (kNumHuffmanBits - numBits))]; + inStream->MovePos(numBits); + return sym; +} + + + +static const unsigned kNumLenDirectBits = 8; + +static const unsigned kNumDistDirectBitsSmall = 6; +static const unsigned kNumDistDirectBitsBig = 7; + +static const unsigned kLitTableSize = (1 << 8); +static const unsigned kDistTableSize = 64; +static const unsigned kLenTableSize = 64; + +static const UInt32 kHistorySize = (1 << kNumDistDirectBitsBig) * kDistTableSize; // 8 KB + + +CCoder::CCoder(): + _flags(0), + _fullStreamMode(false) +{} + + +bool CCoder::BuildHuff(CHuffmanDecoder &decoder, unsigned numSymbols) +{ + Byte levels[kMaxHuffTableSize]; + unsigned numRecords = (unsigned)_inBitStream.ReadAlignedByte() + 1; + unsigned index = 0; + do + { + const unsigned b = (unsigned)_inBitStream.ReadAlignedByte(); + const unsigned level = (b & 0xF) + 1; + const unsigned rep = ((unsigned)b >> 4) + 1; + if (index + rep > numSymbols) + return false; + for (unsigned j = 0; j < rep; j++) + levels[index++] = (Byte)level; + } + while (--numRecords); + + if (index != numSymbols) + return false; + return decoder.Build(levels, numSymbols); +} + + +HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + if (!_inBitStream.Create(1 << 18)) + return E_OUTOFMEMORY; + if (!_outWindowStream.Create(kHistorySize << 1)) // 16 KB + return E_OUTOFMEMORY; + if (!outSize) + return E_INVALIDARG; + + _outWindowStream.SetStream(outStream); + _outWindowStream.Init(false); + _inBitStream.SetStream(inStream); + _inBitStream.Init(); + + const unsigned numDistDirectBits = (_flags & 2) ? + kNumDistDirectBitsBig: + kNumDistDirectBitsSmall; + const bool literalsOn = ((_flags & 4) != 0); + const UInt32 minMatchLen = (literalsOn ? 3 : 2); + + if (literalsOn) + if (!BuildHuff(_litDecoder, kLitTableSize)) + return S_FALSE; + if (!BuildHuff(_lenDecoder, kLenTableSize)) + return S_FALSE; + if (!BuildHuff(_distDecoder, kDistTableSize)) + return S_FALSE; + + UInt64 prevProgress = 0; + bool moreOut = false; + UInt64 pos = 0, unPackSize = *outSize; + + while (pos < unPackSize) + { + if (pos - prevProgress >= (1u << 18) && progress) + { + prevProgress = pos; + const UInt64 packSize = _inBitStream.GetProcessedSize(); + RINOK(progress->SetRatioInfo(&packSize, &pos)) + } + + if (_inBitStream.ReadBits(1) != 0) + { + Byte b; + if (literalsOn) + { + const unsigned sym = _litDecoder.Decode(&_inBitStream); + // if (sym >= kLitTableSize) break; + b = (Byte)sym; + } + else + b = (Byte)_inBitStream.ReadBits(8); + _outWindowStream.PutByte(b); + pos++; + } + else + { + const UInt32 lowDistBits = _inBitStream.ReadBits(numDistDirectBits); + UInt32 dist = (UInt32)_distDecoder.Decode(&_inBitStream); + // if (dist >= kDistTableSize) break; + dist = (dist << numDistDirectBits) + lowDistBits; + unsigned len = _lenDecoder.Decode(&_inBitStream); + // if (len >= kLenTableSize) break; + if (len == kLenTableSize - 1) + len += _inBitStream.ReadBits(kNumLenDirectBits); + len += minMatchLen; + { + const UInt64 limit = unPackSize - pos; + // limit != 0 + if (len > limit) + { + moreOut = true; + len = (UInt32)limit; + } + } + do + { + // len != 0 + if (dist < pos) + { + _outWindowStream.CopyBlock(dist, len); + pos += len; + break; + } + _outWindowStream.PutByte(0); + pos++; + } + while (--len); + } + } + + HRESULT res = _outWindowStream.Flush(); + + if (res == S_OK) + { + if (_fullStreamMode) + { + if (moreOut) + res = S_FALSE; + if (inSize && *inSize != _inBitStream.GetProcessedSize()) + res = S_FALSE; + } + if (pos != unPackSize) + res = S_FALSE; + } + + return res; +} + + +Z7_COM7F_IMF(CCoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + try { return CodeReal(inStream, outStream, inSize, outSize, progress); } + // catch(const CInBufferException &e) { return e.ErrorCode; } + // catch(const CLzOutWindowException &e) { return e.ErrorCode; } + catch(const CSystemException &e) { return e.ErrorCode; } + catch(...) { return S_FALSE; } +} + + +Z7_COM7F_IMF(CCoder::SetDecoderProperties2(const Byte *data, UInt32 size)) +{ + if (size == 0) + return E_NOTIMPL; + _flags = data[0]; + return S_OK; +} + + +Z7_COM7F_IMF(CCoder::SetFinishMode(UInt32 finishMode)) +{ + _fullStreamMode = (finishMode != 0); + return S_OK; +} + + +Z7_COM7F_IMF(CCoder::GetInStreamProcessedSize(UInt64 *value)) +{ + *value = _inBitStream.GetProcessedSize(); + return S_OK; +} + +}}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeDecoder.h new file mode 100644 index 0000000..6032515 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeDecoder.h @@ -0,0 +1,61 @@ +// ImplodeDecoder.h + +#ifndef ZIP7_INC_COMPRESS_IMPLODE_DECODER_H +#define ZIP7_INC_COMPRESS_IMPLODE_DECODER_H + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" + +#include "BitlDecoder.h" +#include "LzOutWindow.h" + +namespace NCompress { +namespace NImplode { +namespace NDecoder { + +typedef NBitl::CDecoder CInBit; + +const unsigned kNumHuffmanBits = 16; +const unsigned kMaxHuffTableSize = 1 << 8; + +class CHuffmanDecoder +{ + UInt32 _limits[kNumHuffmanBits + 1]; + UInt32 _poses[kNumHuffmanBits + 1]; + Byte _symbols[kMaxHuffTableSize]; +public: + bool Build(const Byte *lens, unsigned numSymbols) throw(); + unsigned Decode(CInBit *inStream) const throw(); +}; + + +Z7_CLASS_IMP_NOQIB_4( + CCoder + , ICompressCoder + , ICompressSetDecoderProperties2 + , ICompressSetFinishMode + , ICompressGetInStreamProcessedSize +) + Byte _flags; + bool _fullStreamMode; + + CLzOutWindow _outWindowStream; + CInBit _inBitStream; + + CHuffmanDecoder _litDecoder; + CHuffmanDecoder _lenDecoder; + CHuffmanDecoder _distDecoder; + + bool BuildHuff(CHuffmanDecoder &table, unsigned numSymbols); + HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); +public: + CCoder(); +}; + +}}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeHuffmanDecoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeHuffmanDecoder.cpp new file mode 100644 index 0000000..7d31bb9 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeHuffmanDecoder.cpp @@ -0,0 +1,3 @@ +// ImplodeHuffmanDecoder.cpp + +#include "StdAfx.h" diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeHuffmanDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeHuffmanDecoder.h new file mode 100644 index 0000000..dcf8ac6 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ImplodeHuffmanDecoder.h @@ -0,0 +1,6 @@ +// ImplodeHuffmanDecoder.h + +#ifndef ZIP7_INC_IMPLODE_HUFFMAN_DECODER_H +#define ZIP7_INC_IMPLODE_HUFFMAN_DECODER_H + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzOutWindow.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzOutWindow.cpp new file mode 100644 index 0000000..aae02eb --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzOutWindow.cpp @@ -0,0 +1,14 @@ +// LzOutWindow.cpp + +#include "StdAfx.h" + +#include "LzOutWindow.h" + +void CLzOutWindow::Init(bool solid) throw() +{ + if (!solid) + COutBuffer::Init(); + #ifdef Z7_NO_EXCEPTIONS + ErrorCode = S_OK; + #endif +} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzOutWindow.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzOutWindow.h new file mode 100644 index 0000000..599b124 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzOutWindow.h @@ -0,0 +1,102 @@ +// LzOutWindow.h + +#ifndef ZIP7_INC_LZ_OUT_WINDOW_H +#define ZIP7_INC_LZ_OUT_WINDOW_H + +#include "../Common/OutBuffer.h" + +#ifndef Z7_NO_EXCEPTIONS +typedef COutBufferException CLzOutWindowException; +#endif + +class CLzOutWindow: public COutBuffer +{ +public: + void Init(bool solid = false) throw(); + + // distance >= 0, len > 0, + bool CopyBlock(UInt32 distance, UInt32 len) + { + UInt32 pos = _pos - distance - 1; + if (distance >= _pos) + { + if (!_overDict || distance >= _bufSize) + return false; + pos += _bufSize; + } + if (_limitPos - _pos > len && _bufSize - pos > len) + { + const Byte *src = _buf + pos; + Byte *dest = _buf + _pos; + _pos += len; + do + *dest++ = *src++; + while (--len != 0); + } + else do + { + UInt32 pos2; + if (pos == _bufSize) + pos = 0; + pos2 = _pos; + _buf[pos2++] = _buf[pos++]; + _pos = pos2; + if (pos2 == _limitPos) + FlushWithCheck(); + } + while (--len != 0); + return true; + } + + void PutByte(Byte b) + { + UInt32 pos = _pos; + _buf[pos++] = b; + _pos = pos; + if (pos == _limitPos) + FlushWithCheck(); + } + + void PutBytes(const Byte *data, UInt32 size) + { + if (size == 0) + return; + UInt32 pos = _pos; + Byte *buf = _buf; + buf[pos++] = *data++; + size--; + for (;;) + { + UInt32 limitPos = _limitPos; + UInt32 rem = limitPos - pos; + if (rem == 0) + { + _pos = pos; + FlushWithCheck(); + pos = _pos; + continue; + } + + if (size == 0) + break; + + if (rem > size) + rem = size; + size -= rem; + do + buf[pos++] = *data++; + while (--rem); + } + _pos = pos; + } + + Byte GetByte(UInt32 distance) const + { + UInt32 pos = _pos - distance - 1; + if (distance >= _pos) + pos += _bufSize; + return _buf[pos]; + } +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzfseDecoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzfseDecoder.cpp new file mode 100644 index 0000000..b224a33 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzfseDecoder.cpp @@ -0,0 +1,950 @@ +// LzfseDecoder.cpp + +/* +This code implements LZFSE data decompressing. +The code from "LZFSE compression library" was used. + +2018 : Igor Pavlov : BSD 3-clause License : the code in this file +2015-2017 : Apple Inc : BSD 3-clause License : original "LZFSE compression library" code + +The code in the "LZFSE compression library" is licensed under the "BSD 3-clause License": +---- +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +---- +*/ + +#include "StdAfx.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#include "../../../C/CpuArch.h" + +#include "LzfseDecoder.h" + +namespace NCompress { +namespace NLzfse { + +static const Byte kSignature_LZFSE_V1 = 0x31; // '1' +static const Byte kSignature_LZFSE_V2 = 0x32; // '2' + + +HRESULT CDecoder::GetUInt32(UInt32 &val) +{ + Byte b[4]; + for (unsigned i = 0; i < 4; i++) + if (!m_InStream.ReadByte(b[i])) + return S_FALSE; + val = GetUi32(b); + return S_OK; +} + + + +HRESULT CDecoder::DecodeUncompressed(UInt32 unpackSize) +{ + PRF(printf("\nUncompressed %7u\n", unpackSize)); + + const unsigned kBufSize = 1 << 8; + Byte buf[kBufSize]; + for (;;) + { + if (unpackSize == 0) + return S_OK; + UInt32 cur = unpackSize; + if (cur > kBufSize) + cur = kBufSize; + const UInt32 cur2 = (UInt32)m_InStream.ReadBytes(buf, cur); + m_OutWindowStream.PutBytes(buf, cur2); + if (cur != cur2) + return S_FALSE; + } +} + + + +HRESULT CDecoder::DecodeLzvn(UInt32 unpackSize, UInt32 packSize) +{ + PRF(printf("\nLZVN 0x%07x 0x%07x\n", unpackSize, packSize)); + + UInt32 D = 0; + + for (;;) + { + if (packSize == 0) + return S_FALSE; + Byte b; + if (!m_InStream.ReadByte(b)) + return S_FALSE; + packSize--; + + UInt32 M; + UInt32 L; + + if (b >= 0xE0) + { + /* + large L - 11100000 LLLLLLLL + small L - 1110LLLL + + large Rep - 11110000 MMMMMMMM + small Rep - 1111MMMM + */ + + M = b & 0xF; + if (M == 0) + { + if (packSize == 0) + return S_FALSE; + Byte b1; + if (!m_InStream.ReadByte(b1)) + return S_FALSE; + packSize--; + M = (UInt32)b1 + 16; + } + L = 0; + if ((b & 0x10) == 0) + { + // Literals only + L = M; + M = 0; + } + } + + // ERROR codes + else if ((b & 0xF0) == 0x70) // 0111xxxx + return S_FALSE; + else if ((b & 0xF0) == 0xD0) // 1101xxxx + return S_FALSE; + + else + { + if ((b & 0xE0) == 0xA0) + { + // medium - 101LLMMM DDDDDDMM DDDDDDDD + if (packSize < 2) + return S_FALSE; + Byte b1; + if (!m_InStream.ReadByte(b1)) + return S_FALSE; + packSize--; + + Byte b2; + if (!m_InStream.ReadByte(b2)) + return S_FALSE; + packSize--; + L = (((UInt32)b >> 3) & 3); + M = (((UInt32)b & 7) << 2) + (b1 & 3); + D = ((UInt32)b1 >> 2) + ((UInt32)b2 << 6); + } + else + { + L = (UInt32)b >> 6; + M = ((UInt32)b >> 3) & 7; + if ((b & 0x7) == 6) + { + // REP - LLMMM110 + if (L == 0) + { + // spec + if (M == 0) + break; // EOS + if (M <= 2) + continue; // NOP + return S_FALSE; // UNDEFINED + } + } + else + { + if (packSize == 0) + return S_FALSE; + Byte b1; + if (!m_InStream.ReadByte(b1)) + return S_FALSE; + packSize--; + + // large - LLMMM111 DDDDDDDD DDDDDDDD + // small - LLMMMDDD DDDDDDDD + D = ((UInt32)b & 7); + if (D == 7) + { + if (packSize == 0) + return S_FALSE; + Byte b2; + if (!m_InStream.ReadByte(b2)) + return S_FALSE; + packSize--; + D = b2; + } + D = (D << 8) + b1; + } + } + + M += 3; + } + { + for (unsigned i = 0; i < L; i++) + { + if (packSize == 0 || unpackSize == 0) + return S_FALSE; + Byte b1; + if (!m_InStream.ReadByte(b1)) + return S_FALSE; + packSize--; + m_OutWindowStream.PutByte(b1); + unpackSize--; + } + } + + if (M != 0) + { + if (unpackSize == 0 || D == 0) + return S_FALSE; + unsigned cur = M; + if (cur > unpackSize) + cur = (unsigned)unpackSize; + if (!m_OutWindowStream.CopyBlock(D - 1, cur)) + return S_FALSE; + unpackSize -= cur; + if (cur != M) + return S_FALSE; + } + } + + if (unpackSize != 0) + return S_FALSE; + + // LZVN encoder writes 7 additional zero bytes + if (packSize < 7) + return S_FALSE; + for (unsigned i = 0; i < 7; i++) + { + Byte b; + if (!m_InStream.ReadByte(b)) + return S_FALSE; + if (b != 0) + return S_FALSE; + } + packSize -= 7; + if (packSize) + { + PRF(printf("packSize after unused = %u\n", packSize)); + // if (packSize <= 0x100) { Byte buf[0x100]; m_InStream.ReadBytes(buf, packSize); } + /* Lzvn block that is used in HFS can contain junk data + (at least 256 bytes) after payload data. Why? + We ignore that junk data, if it's HFS (LzvnMode) mode. */ + if (!LzvnMode) + return S_FALSE; + } + return S_OK; +} + + + +// ---------- LZFSE ---------- + +#define MATCHES_PER_BLOCK 10000 +#define LITERALS_PER_BLOCK (4 * MATCHES_PER_BLOCK) + +#define NUM_L_SYMBOLS 20 +#define NUM_M_SYMBOLS 20 +#define NUM_D_SYMBOLS 64 +#define NUM_LIT_SYMBOLS 256 + +#define NUM_SYMBOLS ( \ + NUM_L_SYMBOLS + \ + NUM_M_SYMBOLS + \ + NUM_D_SYMBOLS + \ + NUM_LIT_SYMBOLS) + +#define NUM_L_STATES (1 << 6) +#define NUM_M_STATES (1 << 6) +#define NUM_D_STATES (1 << 8) +#define NUM_LIT_STATES (1 << 10) + + +typedef UInt32 CFseState; + + +static UInt32 SumFreqs(const UInt16 *freqs, unsigned num) +{ + UInt32 sum = 0; + for (unsigned i = 0; i < num; i++) + sum += (UInt32)freqs[i]; + return sum; +} + + +static Z7_FORCE_INLINE unsigned CountZeroBits(UInt32 val, UInt32 mask) +{ + for (unsigned i = 0;;) + { + if (val & mask) + return i; + i++; + mask >>= 1; + } +} + + +static Z7_FORCE_INLINE void InitLitTable(const UInt16 *freqs, UInt32 *table) +{ + for (unsigned i = 0; i < NUM_LIT_SYMBOLS; i++) + { + unsigned f = freqs[i]; + if (f == 0) + continue; + + // 0 < f <= numStates + // 0 <= k <= numStatesLog + // numStates <= (f<> k) - f; + + /* + CEntry + { + Byte k; + Byte symbol; + UInt16 delta; + }; + */ + + UInt32 e = ((UInt32)i << 8) + k; + k += 16; + UInt32 d = e + ((UInt32)f << k) - ((UInt32)NUM_LIT_STATES << 16); + UInt32 step = (UInt32)1 << k; + + unsigned j = 0; + do + { + *table++ = d; + d += step; + } + while (++j < j0); + + e--; + step >>= 1; + + for (j = j0; j < f; j++) + { + *table++ = e; + e += step; + } + } +} + + +typedef struct +{ + Byte totalBits; + Byte extraBits; + UInt16 delta; + UInt32 vbase; +} CExtraEntry; + + +static void InitExtraDecoderTable(unsigned numStates, + unsigned numSymbols, + const UInt16 *freqs, + const Byte *vbits, + CExtraEntry *table) +{ + UInt32 vbase = 0; + + for (unsigned i = 0; i < numSymbols; i++) + { + unsigned f = freqs[i]; + unsigned extraBits = vbits[i]; + + if (f != 0) + { + unsigned k = CountZeroBits(f, numStates); + unsigned j0 = ((2 * numStates) >> k) - f; + + unsigned j = 0; + do + { + CExtraEntry *e = table++; + e->totalBits = (Byte)(k + extraBits); + e->extraBits = (Byte)extraBits; + e->delta = (UInt16)(((f + j) << k) - numStates); + e->vbase = vbase; + } + while (++j < j0); + + f -= j0; + k--; + + for (j = 0; j < f; j++) + { + CExtraEntry *e = table++; + e->totalBits = (Byte)(k + extraBits); + e->extraBits = (Byte)extraBits; + e->delta = (UInt16)(j << k); + e->vbase = vbase; + } + } + + vbase += ((UInt32)1 << extraBits); + } +} + + +static const Byte k_L_extra[NUM_L_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 5, 8 +}; + +static const Byte k_M_extra[NUM_M_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 8, 11 +}; + +static const Byte k_D_extra[NUM_D_SYMBOLS] = +{ + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, + 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15 +}; + + + +// ---------- CBitStream ---------- + +typedef struct +{ + UInt32 accum; + unsigned numBits; // [0, 31] - Number of valid bits in (accum), other bits are 0 +} CBitStream; + + +static Z7_FORCE_INLINE int FseInStream_Init(CBitStream *s, + int n, // [-7, 0], (-n == number_of_unused_bits) in last byte + const Byte **pbuf) +{ + *pbuf -= 4; + s->accum = GetUi32(*pbuf); + if (n) + { + s->numBits = (unsigned)(n + 32); + if ((s->accum >> s->numBits) != 0) + return -1; // ERROR, encoder should have zeroed the upper bits + } + else + { + *pbuf += 1; + s->accum >>= 8; + s->numBits = 24; + } + return 0; // OK +} + + +// 0 <= numBits < 32 +#define mask31(x, numBits) ((x) & (((UInt32)1 << (numBits)) - 1)) + +#define FseInStream_FLUSH \ + { const unsigned nbits = (31 - in.numBits) & (unsigned)-8; \ + if (nbits) { \ + buf -= (nbits >> 3); \ + if (buf < buf_check) return S_FALSE; \ + UInt32 v = GetUi32(buf); \ + in.accum = (in.accum << nbits) | mask31(v, nbits); \ + in.numBits += nbits; }} + + + +static Z7_FORCE_INLINE UInt32 BitStream_Pull(CBitStream *s, unsigned numBits) +{ + s->numBits -= numBits; + const UInt32 v = s->accum >> s->numBits; + s->accum = mask31(s->accum, s->numBits); + return v; +} + + +#define DECODE_LIT(dest, pstate) { \ + UInt32 e = lit_decoder[pstate]; \ + pstate = (CFseState)((e >> 16) + BitStream_Pull(&in, e & 0xff)); \ + dest = (Byte)(e >> 8); } + + +static Z7_FORCE_INLINE UInt32 FseDecodeExtra(CFseState *pstate, + const CExtraEntry *table, + CBitStream *s) +{ + const CExtraEntry *e = &table[*pstate]; + UInt32 v = BitStream_Pull(s, e->totalBits); + unsigned extraBits = e->extraBits; + *pstate = (CFseState)(e->delta + (v >> extraBits)); + return e->vbase + mask31(v, extraBits); +} + + +#define freqs_L (freqs) +#define freqs_M (freqs_L + NUM_L_SYMBOLS) +#define freqs_D (freqs_M + NUM_M_SYMBOLS) +#define freqs_LIT (freqs_D + NUM_D_SYMBOLS) + +#define GET_BITS_64(v, offset, num, dest) dest = (UInt32) ((v >> (offset)) & ((1 << (num)) - 1)); +#define GET_BITS_64_Int32(v, offset, num, dest) dest = (Int32)((v >> (offset)) & ((1 << (num)) - 1)); +#define GET_BITS_32(v, offset, num, dest) dest = (CFseState)((v >> (offset)) & ((1 << (num)) - 1)); + + +HRESULT CDecoder::DecodeLzfse(UInt32 unpackSize, Byte version) +{ + PRF(printf("\nLZFSE-%d %7u", version - '0', unpackSize)); + + UInt32 numLiterals; + UInt32 litPayloadSize; + Int32 literal_bits; + + UInt32 lit_state_0; + UInt32 lit_state_1; + UInt32 lit_state_2; + UInt32 lit_state_3; + + UInt32 numMatches; + UInt32 lmdPayloadSize; + Int32 lmd_bits; + + CFseState l_state; + CFseState m_state; + CFseState d_state; + + UInt16 freqs[NUM_SYMBOLS]; + + if (version == kSignature_LZFSE_V1) + { + return E_NOTIMPL; + // we need examples to test LZFSE-V1 code + /* + const unsigned k_v1_SubHeaderSize = 7 * 4 + 7 * 2; + const unsigned k_v1_HeaderSize = k_v1_SubHeaderSize + NUM_SYMBOLS * 2; + _buffer.AllocAtLeast(k_v1_HeaderSize); + if (m_InStream.ReadBytes(_buffer, k_v1_HeaderSize) != k_v1_HeaderSize) + return S_FALSE; + + const Byte *buf = _buffer; + #define GET_32(offs, dest) dest = GetUi32(buf + offs) + #define GET_16(offs, dest) dest = GetUi16(buf + offs) + + UInt32 payload_bytes; + GET_32(0, payload_bytes); + GET_32(4, numLiterals); + GET_32(8, numMatches); + GET_32(12, litPayloadSize); + GET_32(16, lmdPayloadSize); + if (litPayloadSize > (1 << 20) || lmdPayloadSize > (1 << 20)) + return S_FALSE; + GET_32(20, literal_bits); + if (literal_bits < -7 || literal_bits > 0) + return S_FALSE; + + GET_16(24, lit_state_0); + GET_16(26, lit_state_1); + GET_16(28, lit_state_2); + GET_16(30, lit_state_3); + + GET_32(32, lmd_bits); + if (lmd_bits < -7 || lmd_bits > 0) + return S_FALSE; + + GET_16(36, l_state); + GET_16(38, m_state); + GET_16(40, d_state); + + for (unsigned i = 0; i < NUM_SYMBOLS; i++) + freqs[i] = GetUi16(buf + k_v1_SubHeaderSize + i * 2); + */ + } + else + { + UInt32 headerSize; + { + const unsigned kPreHeaderSize = 4 * 2; // signature and upackSize + const unsigned kHeaderSize = 8 * 3; + Byte temp[kHeaderSize]; + if (m_InStream.ReadBytes(temp, kHeaderSize) != kHeaderSize) + return S_FALSE; + + UInt64 v; + + v = GetUi64(temp); + GET_BITS_64(v, 0, 20, numLiterals) + GET_BITS_64(v, 20, 20, litPayloadSize) + GET_BITS_64(v, 40, 20, numMatches) + GET_BITS_64_Int32(v, 60, 3 + 1, literal_bits) // (NumberOfUsedBits - 1) + literal_bits -= 7; // (-NumberOfUnusedBits) + if (literal_bits > 0) + return S_FALSE; + // GET_BITS_64(v, 63, 1, unused); + + v = GetUi64(temp + 8); + GET_BITS_64(v, 0, 10, lit_state_0) + GET_BITS_64(v, 10, 10, lit_state_1) + GET_BITS_64(v, 20, 10, lit_state_2) + GET_BITS_64(v, 30, 10, lit_state_3) + GET_BITS_64(v, 40, 20, lmdPayloadSize) + GET_BITS_64_Int32(v, 60, 3 + 1, lmd_bits) + lmd_bits -= 7; + if (lmd_bits > 0) + return S_FALSE; + // GET_BITS_64(v, 63, 1, unused) + + UInt32 v32 = GetUi32(temp + 20); + // (total header size in bytes; this does not + // correspond to a field in the uncompressed header version, + // but is required; we wouldn't know the size of the + // compresssed header otherwise. + GET_BITS_32(v32, 0, 10, l_state) + GET_BITS_32(v32, 10, 10, m_state) + GET_BITS_32(v32, 20, 10 + 2, d_state) + // GET_BITS_64(v, 62, 2, unused) + + headerSize = GetUi32(temp + 16); + if (headerSize <= kPreHeaderSize + kHeaderSize) + return S_FALSE; + headerSize -= kPreHeaderSize + kHeaderSize; + } + + // no freqs case is not allowed ? + // memset(freqs, 0, sizeof(freqs)); + // if (headerSize != 0) + { + static const Byte numBitsTable[32] = + { + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14, + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14 + }; + + static const Byte valueTable[32] = + { + 0, 2, 1, 4, 0, 3, 1, 8, 0, 2, 1, 5, 0, 3, 1, 24, + 0, 2, 1, 6, 0, 3, 1, 8, 0, 2, 1, 7, 0, 3, 1, 24 + }; + + UInt32 accum = 0; + unsigned numBits = 0; + + for (unsigned i = 0; i < NUM_SYMBOLS; i++) + { + while (numBits <= 14 && headerSize != 0) + { + Byte b; + if (!m_InStream.ReadByte(b)) + return S_FALSE; + accum |= (UInt32)b << numBits; + numBits += 8; + headerSize--; + } + + unsigned b = (unsigned)accum & 31; + unsigned n = numBitsTable[b]; + if (numBits < n) + return S_FALSE; + numBits -= n; + UInt32 f = valueTable[b]; + if (n >= 8) + f += ((accum >> 4) & (0x3ff >> (14 - n))); + accum >>= n; + freqs[i] = (UInt16)f; + } + + if (numBits >= 8 || headerSize != 0) + return S_FALSE; + } + } + + PRF(printf(" Literals=%6u Matches=%6u", numLiterals, numMatches)); + + if (numLiterals > LITERALS_PER_BLOCK + || (numLiterals & 3) != 0 + || numMatches > MATCHES_PER_BLOCK + || lit_state_0 >= NUM_LIT_STATES + || lit_state_1 >= NUM_LIT_STATES + || lit_state_2 >= NUM_LIT_STATES + || lit_state_3 >= NUM_LIT_STATES + || l_state >= NUM_L_STATES + || m_state >= NUM_M_STATES + || d_state >= NUM_D_STATES) + return S_FALSE; + + // only full table is allowed ? + if ( SumFreqs(freqs_L, NUM_L_SYMBOLS) != NUM_L_STATES + || SumFreqs(freqs_M, NUM_M_SYMBOLS) != NUM_M_STATES + || SumFreqs(freqs_D, NUM_D_SYMBOLS) != NUM_D_STATES + || SumFreqs(freqs_LIT, NUM_LIT_SYMBOLS) != NUM_LIT_STATES) + return S_FALSE; + + + const unsigned kPad = 16; + + // ---------- Decode literals ---------- + + { + _literals.AllocAtLeast(LITERALS_PER_BLOCK + 16); + _buffer.AllocAtLeast(kPad + litPayloadSize); + memset(_buffer, 0, kPad); + + if (m_InStream.ReadBytes(_buffer + kPad, litPayloadSize) != litPayloadSize) + return S_FALSE; + + UInt32 lit_decoder[NUM_LIT_STATES]; + InitLitTable(freqs_LIT, lit_decoder); + + const Byte *buf_start = _buffer + kPad; + const Byte *buf_check = buf_start - 4; + const Byte *buf = buf_start + litPayloadSize; + CBitStream in; + if (FseInStream_Init(&in, literal_bits, &buf) != 0) + return S_FALSE; + + Byte *lit = _literals; + const Byte *lit_limit = lit + numLiterals; + for (; lit < lit_limit; lit += 4) + { + FseInStream_FLUSH + DECODE_LIT (lit[0], lit_state_0) + DECODE_LIT (lit[1], lit_state_1) + FseInStream_FLUSH + DECODE_LIT (lit[2], lit_state_2) + DECODE_LIT (lit[3], lit_state_3) + } + + if ((buf_start - buf) * 8 != (int)in.numBits) + return S_FALSE; + } + + + // ---------- Decode LMD ---------- + + _buffer.AllocAtLeast(kPad + lmdPayloadSize); + memset(_buffer, 0, kPad); + if (m_InStream.ReadBytes(_buffer + kPad, lmdPayloadSize) != lmdPayloadSize) + return S_FALSE; + + CExtraEntry l_decoder[NUM_L_STATES]; + CExtraEntry m_decoder[NUM_M_STATES]; + CExtraEntry d_decoder[NUM_D_STATES]; + + InitExtraDecoderTable(NUM_L_STATES, NUM_L_SYMBOLS, freqs_L, k_L_extra, l_decoder); + InitExtraDecoderTable(NUM_M_STATES, NUM_M_SYMBOLS, freqs_M, k_M_extra, m_decoder); + InitExtraDecoderTable(NUM_D_STATES, NUM_D_SYMBOLS, freqs_D, k_D_extra, d_decoder); + + const Byte *buf_start = _buffer + kPad; + const Byte *buf_check = buf_start - 4; + const Byte *buf = buf_start + lmdPayloadSize; + CBitStream in; + if (FseInStream_Init(&in, lmd_bits, &buf)) + return S_FALSE; + + const Byte *lit = _literals; + const Byte *lit_limit = lit + numLiterals; + + UInt32 D = 0; + + for (;;) + { + if (numMatches == 0) + break; + numMatches--; + + FseInStream_FLUSH + + unsigned L = (unsigned)FseDecodeExtra(&l_state, l_decoder, &in); + + FseInStream_FLUSH + + unsigned M = (unsigned)FseDecodeExtra(&m_state, m_decoder, &in); + + FseInStream_FLUSH + + { + UInt32 new_D = FseDecodeExtra(&d_state, d_decoder, &in); + if (new_D) + D = new_D; + } + + if (L != 0) + { + if (L > (size_t)(lit_limit - lit)) + return S_FALSE; + unsigned cur = L; + if (cur > unpackSize) + cur = (unsigned)unpackSize; + m_OutWindowStream.PutBytes(lit, cur); + unpackSize -= cur; + lit += cur; + if (cur != L) + return S_FALSE; + } + + if (M != 0) + { + if (unpackSize == 0 || D == 0) + return S_FALSE; + unsigned cur = M; + if (cur > unpackSize) + cur = (unsigned)unpackSize; + if (!m_OutWindowStream.CopyBlock(D - 1, cur)) + return S_FALSE; + unpackSize -= cur; + if (cur != M) + return S_FALSE; + } + } + + if (unpackSize != 0) + return S_FALSE; + + // LZFSE encoder writes 8 additional zero bytes before LMD payload + // We test it: + if ((size_t)(buf - buf_start) * 8 + in.numBits != 64) + return S_FALSE; + if (GetUi64(buf_start) != 0) + return S_FALSE; + + return S_OK; +} + + +HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + PRF(printf("\n\nLzfseDecoder %7u %7u\n", (unsigned)*outSize, (unsigned)*inSize)); + + const UInt32 kLzfseDictSize = 1 << 18; + if (!m_OutWindowStream.Create(kLzfseDictSize)) + return E_OUTOFMEMORY; + if (!m_InStream.Create(1 << 18)) + return E_OUTOFMEMORY; + + m_OutWindowStream.SetStream(outStream); + m_OutWindowStream.Init(false); + m_InStream.SetStream(inStream); + m_InStream.Init(); + + CCoderReleaser coderReleaser(this); + + UInt64 prevOut = 0; + UInt64 prevIn = 0; + + if (LzvnMode) + { + if (!outSize || !inSize) + return E_NOTIMPL; + const UInt64 unpackSize = *outSize; + const UInt64 packSize = *inSize; + if (unpackSize > (UInt32)(Int32)-1 + || packSize > (UInt32)(Int32)-1) + return S_FALSE; + RINOK(DecodeLzvn((UInt32)unpackSize, (UInt32)packSize)) + } + else + for (;;) + { + const UInt64 pos = m_OutWindowStream.GetProcessedSize(); + const UInt64 packPos = m_InStream.GetProcessedSize(); + + if (progress && ((pos - prevOut) >= (1 << 22) || (packPos - prevIn) >= (1 << 22))) + { + RINOK(progress->SetRatioInfo(&packPos, &pos)) + prevIn = packPos; + prevOut = pos; + } + + UInt32 v; + RINOK(GetUInt32(v)) + if ((v & 0xFFFFFF) != 0x787662) // bvx + return S_FALSE; + v >>= 24; + + if (v == 0x24) // '$', end of stream + break; + + UInt32 unpackSize; + RINOK(GetUInt32(unpackSize)) + + UInt32 cur = unpackSize; + if (outSize) + { + const UInt64 rem = *outSize - pos; + if (cur > rem) + cur = (UInt32)rem; + } + unpackSize -= cur; + + HRESULT res; + if (v == kSignature_LZFSE_V1 || v == kSignature_LZFSE_V2) + res = DecodeLzfse(cur, (Byte)v); + else if (v == 0x6E) // 'n' + { + UInt32 packSize; + res = GetUInt32(packSize); + if (res == S_OK) + res = DecodeLzvn(cur, packSize); + } + else if (v == 0x2D) // '-' + res = DecodeUncompressed(cur); + else + return E_NOTIMPL; + + if (res != S_OK) + return res; + + if (unpackSize != 0) + return S_FALSE; + } + + coderReleaser.NeedFlush = false; + HRESULT res = m_OutWindowStream.Flush(); + if (res == S_OK) + if ((!LzvnMode && inSize && *inSize != m_InStream.GetProcessedSize()) + || (outSize && *outSize != m_OutWindowStream.GetProcessedSize())) + res = S_FALSE; + return res; +} + + +Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + try { return CodeReal(inStream, outStream, inSize, outSize, progress); } + catch(const CInBufferException &e) { return e.ErrorCode; } + catch(const CLzOutWindowException &e) { return e.ErrorCode; } + catch(...) { return E_OUTOFMEMORY; } + // catch(...) { return S_FALSE; } +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzfseDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzfseDecoder.h new file mode 100644 index 0000000..b7227dc --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzfseDecoder.h @@ -0,0 +1,63 @@ +// LzfseDecoder.h + +#ifndef ZIP7_INC_LZFSE_DECODER_H +#define ZIP7_INC_LZFSE_DECODER_H + +#include "../../Common/MyBuffer.h" +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" + +#include "LzOutWindow.h" + +namespace NCompress { +namespace NLzfse { + +Z7_CLASS_IMP_NOQIB_1( + CDecoder + , ICompressCoder +) + CLzOutWindow m_OutWindowStream; + CInBuffer m_InStream; + CByteBuffer _literals; + CByteBuffer _buffer; + + class CCoderReleaser + { + CDecoder *m_Coder; + public: + bool NeedFlush; + CCoderReleaser(CDecoder *coder): m_Coder(coder), NeedFlush(true) {} + ~CCoderReleaser() + { + if (NeedFlush) + m_Coder->m_OutWindowStream.Flush(); + } + }; + friend class CCoderReleaser; + + HRESULT GetUInt32(UInt32 &val); + + HRESULT DecodeUncompressed(UInt32 unpackSize); + HRESULT DecodeLzvn(UInt32 unpackSize, UInt32 packSize); + HRESULT DecodeLzfse(UInt32 unpackSize, Byte version); + + HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); +public: + bool LzvnMode; + + CDecoder(): + LzvnMode(false) + {} + + // sizes are checked in Code() + // UInt64 GetInputProcessedSize() const { return m_InStream.GetProcessedSize(); } + // UInt64 GetOutputProcessedSize() const { return m_OutWindowStream.GetProcessedSize(); } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzhDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzhDecoder.h new file mode 100644 index 0000000..204c52c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzhDecoder.h @@ -0,0 +1,68 @@ +// LzhDecoder.h + +#ifndef ZIP7_INC_COMPRESS_LZH_DECODER_H +#define ZIP7_INC_COMPRESS_LZH_DECODER_H + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" + +#include "BitmDecoder.h" +#include "HuffmanDecoder.h" +#include "LzOutWindow.h" + +namespace NCompress { +namespace NLzh { +namespace NDecoder { + +const unsigned kMatchMinLen = 3; +const unsigned kMatchMaxLen = 256; +const unsigned NC = 256 + kMatchMaxLen - kMatchMinLen + 1; +const unsigned NUM_CODE_BITS = 16; +const unsigned NUM_DIC_BITS_MAX = 25; +const unsigned NT = NUM_CODE_BITS + 3; +const unsigned NP = NUM_DIC_BITS_MAX + 1; +const unsigned NPT = NP; // Max(NT, NP) + +class CCoder +{ + CLzOutWindow _outWindow; + NBitm::CDecoder _inBitStream; + + int _symbolT; + int _symbolC; + UInt32 DictSize; + // bool FinishMode; + + NHuffman::CDecoder256 _decoderT; + NHuffman::CDecoder _decoderC; + + class CCoderReleaser + { + CCoder *_coder; + public: + CCoderReleaser(CCoder *coder): _coder(coder) {} + void Disable() { _coder = NULL; } + ~CCoderReleaser() { if (_coder) _coder->_outWindow.Flush(); } + }; + friend class CCoderReleaser; + + bool ReadTP(unsigned num, unsigned numBits, int spec); + bool ReadC(); + + HRESULT CodeReal(UInt32 outSize, ICompressProgressInfo *progress); +public: + CCoder(): DictSize(1 << 16) + // , FinishMode(true) + {} + void SetDictSize(UInt32 dictSize) { DictSize = dictSize; } + UInt64 GetInputProcessedSize() const { return _inBitStream.GetProcessedSize(); } + HRESULT Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + UInt32 outSize, ICompressProgressInfo *progress); +}; + +}}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzmsDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzmsDecoder.h new file mode 100644 index 0000000..bb21262 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzmsDecoder.h @@ -0,0 +1,209 @@ +// LzmsDecoder.h +// The code is based on LZMS description from wimlib code + +#ifndef ZIP7_INC_LZMS_DECODER_H +#define ZIP7_INC_LZMS_DECODER_H + +#include "../../../C/CpuArch.h" +#include "../../../C/HuffEnc.h" + +#include "HuffmanDecoder.h" + +namespace NCompress { +namespace NLzms { + +const unsigned k_NumLitSyms = 256; +const unsigned k_NumLenSyms = 54; +const unsigned k_NumPosSyms = 799; +const unsigned k_NumPowerSyms = 8; + +const unsigned k_NumProbBits = 6; +const unsigned k_ProbLimit = 1 << k_NumProbBits; +const unsigned k_InitialProb = 48; +const UInt32 k_InitialHist = 0x55555555; + +const unsigned k_NumReps = 3; + +const unsigned k_NumMainProbs = 16; +const unsigned k_NumMatchProbs = 32; +const unsigned k_NumRepProbs = 64; + +const unsigned k_NumHuffmanBits = 15; + +template +class CHuffDecoder: public NCompress::NHuffman::CDecoder +{ +public: + UInt32 RebuildRem; + UInt32 NumSyms; + UInt32 Freqs[m_NumSyms]; + + void Generate() throw() + { + UInt32 vals[m_NumSyms]; + Byte levels[m_NumSyms]; + + // We need to check that our algorithm is OK, when optimal Huffman tree uses more than 15 levels !!! + Huffman_Generate(Freqs, vals, levels, NumSyms, k_NumHuffmanBits); + + for (UInt32 i = NumSyms; i < m_NumSyms; i++) + levels[i] = 0; + + this->Build(levels, /* NumSyms, */ NHuffman::k_BuildMode_Full); + } + + void Rebuild() throw() + { + Generate(); + RebuildRem = m_RebuildFreq; + const UInt32 num = NumSyms; + for (UInt32 i = 0; i < num; i++) + Freqs[i] = (Freqs[i] >> 1) + 1; + } + +public: + void Init(UInt32 numSyms = m_NumSyms) throw() + { + RebuildRem = m_RebuildFreq; + NumSyms = numSyms; + for (UInt32 i = 0; i < numSyms; i++) + Freqs[i] = 1; + // for (; i < m_NumSyms; i++) Freqs[i] = 0; + Generate(); + } +}; + + +struct CProbEntry +{ + UInt32 Prob; + UInt64 Hist; + + void Init() + { + Prob = k_InitialProb; + Hist = k_InitialHist; + } + + UInt32 GetProb() const throw() + { + UInt32 prob = Prob; + if (prob == 0) + prob = 1; + else if (prob == k_ProbLimit) + prob = k_ProbLimit - 1; + return prob; + } + + void Update(unsigned bit) throw() + { + Prob += (UInt32)((Int32)(Hist >> (k_ProbLimit - 1)) - (Int32)bit); + Hist = (Hist << 1) | bit; + } +}; + + +struct CRangeDecoder +{ + UInt32 range; + UInt32 code; + const Byte *cur; + // const Byte *end; + + void Init(const Byte *data, size_t /* size */) throw() + { + range = 0xFFFFFFFF; + code = (((UInt32)GetUi16(data)) << 16) | GetUi16(data + 2); + cur = data + 4; + // end = data + size; + } + + void Normalize() + { + if (range <= 0xFFFF) + { + range <<= 16; + code <<= 16; + // if (cur >= end) throw 1; + code |= GetUi16(cur); + cur += 2; + } + } + + unsigned Decode(UInt32 *state, UInt32 numStates, struct CProbEntry *probs) + { + UInt32 st = *state; + CProbEntry *entry = &probs[st]; + st = (st << 1) & (numStates - 1); + + const UInt32 prob = entry->GetProb(); + + if (range <= 0xFFFF) + { + range <<= 16; + code <<= 16; + // if (cur >= end) throw 1; + code |= GetUi16(cur); + cur += 2; + } + + const UInt32 bound = (range >> k_NumProbBits) * prob; + + if (code < bound) + { + range = bound; + *state = st; + entry->Update(0); + return 0; + } + else + { + range -= bound; + code -= bound; + *state = st | 1; + entry->Update(1); + return 1; + } + } +}; + + +class CDecoder +{ + // CRangeDecoder _rc; + size_t _pos; + + UInt32 _reps[k_NumReps + 1]; + UInt64 _deltaReps[k_NumReps + 1]; + + UInt32 mainState; + UInt32 matchState; + UInt32 lzRepStates[k_NumReps]; + UInt32 deltaRepStates[k_NumReps]; + + struct CProbEntry mainProbs[k_NumMainProbs]; + struct CProbEntry matchProbs[k_NumMatchProbs]; + + struct CProbEntry lzRepProbs[k_NumReps][k_NumRepProbs]; + struct CProbEntry deltaRepProbs[k_NumReps][k_NumRepProbs]; + + CHuffDecoder m_LitDecoder; + CHuffDecoder m_PosDecoder; + CHuffDecoder m_LenDecoder; + CHuffDecoder m_PowerDecoder; + CHuffDecoder m_DeltaDecoder; + + Int32 *_x86_history; + + HRESULT CodeReal(const Byte *in, size_t inSize, Byte *out, size_t outSize); +public: + CDecoder(); + ~CDecoder(); + + HRESULT Code(const Byte *in, size_t inSize, Byte *out, size_t outSize); + size_t GetUnpackSize() const { return _pos; } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Lzx.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Lzx.h new file mode 100644 index 0000000..39fca0c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Lzx.h @@ -0,0 +1,61 @@ +// Lzx.h + +#ifndef ZIP7_INC_COMPRESS_LZX_H +#define ZIP7_INC_COMPRESS_LZX_H + +#include "../../Common/MyTypes.h" + +namespace NCompress { +namespace NLzx { + +const unsigned kBlockType_NumBits = 3; +const unsigned kBlockType_Verbatim = 1; +const unsigned kBlockType_Aligned = 2; +const unsigned kBlockType_Uncompressed = 3; + +const unsigned kNumHuffmanBits = 16; +const unsigned kNumReps = 3; + +const unsigned kNumLenSlots = 8; +const unsigned kMatchMinLen = 2; +const unsigned kNumLenSymbols = 249; +const unsigned kMatchMaxLen = kMatchMinLen + (kNumLenSlots - 1) + kNumLenSymbols - 1; + +const unsigned kNumAlignLevelBits = 3; +const unsigned kNumAlignBits = 3; +const unsigned kAlignTableSize = 1 << kNumAlignBits; + +const unsigned kNumPosSlots = 50; +const unsigned kNumPosLenSlots = kNumPosSlots * kNumLenSlots; + +const unsigned kMainTableSize = 256 + kNumPosLenSlots; +const unsigned kLevelTableSize = 20; +const unsigned kMaxTableSize = kMainTableSize; + +const unsigned kNumLevelBits = 4; + +const unsigned kLevelSym_Zero1 = 17; +const unsigned kLevelSym_Zero2 = 18; +const unsigned kLevelSym_Same = 19; + +const unsigned kLevelSym_Zero1_Start = 4; +const unsigned kLevelSym_Zero1_NumBits = 4; + +const unsigned kLevelSym_Zero2_Start = kLevelSym_Zero1_Start + (1 << kLevelSym_Zero1_NumBits); +const unsigned kLevelSym_Zero2_NumBits = 5; + +const unsigned kLevelSym_Same_NumBits = 1; +const unsigned kLevelSym_Same_Start = 4; + +const unsigned kNumDictBits_Min = 15; +const unsigned kNumDictBits_Max = 21; +const UInt32 kDictSize_Max = (UInt32)1 << kNumDictBits_Max; + +const unsigned kNumLinearPosSlotBits = 17; +// const unsigned kNumPowerPosSlots = 38; +// const unsigned kNumPowerPosSlots = (kNumLinearPosSlotBits + 1) * 2; // non-including two first linear slots. +const unsigned kNumPowerPosSlots = (kNumLinearPosSlotBits + 2) * 2; // including two first linear slots. + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzxDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzxDecoder.h new file mode 100644 index 0000000..b4556a6 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/LzxDecoder.h @@ -0,0 +1,106 @@ +// LzxDecoder.h + +#ifndef ZIP7_INC_LZX_DECODER_H +#define ZIP7_INC_LZX_DECODER_H + +#include "HuffmanDecoder.h" +#include "Lzx.h" + +namespace NCompress { +namespace NLzx { + +const unsigned kAdditionalOutputBufSize = 32 * 2; + +const unsigned kNumTableBits_Main = 11; +const unsigned kNumTableBits_Len = 8; + +// if (kNumLenSymols_Big <= 256) we can use NHuffman::CDecoder256 +// if (kNumLenSymols_Big > 256) we must use NHuffman::CDecoder +// const unsigned kNumLenSymols_Big_Start = kNumLenSlots - 1 + kMatchMinLen; // 8 - 1 + 2 +const unsigned kNumLenSymols_Big_Start = 0; +// const unsigned kNumLenSymols_Big_Start = 0; +const unsigned kNumLenSymols_Big = kNumLenSymols_Big_Start + kNumLenSymbols; + +#if 1 + // for smallest structure size: + const unsigned kPosSlotOffset = 0; +#else + // use virtual entries for mispredicted branches: + const unsigned kPosSlotOffset = 256 / kNumLenSlots; +#endif + +class CBitByteDecoder; + +class CDecoder +{ +public: + UInt32 _pos; + UInt32 _winSize; + Byte *_win; + + bool _overDict; + bool _isUncompressedBlock; + bool _skipByte; + bool _keepHistory; + bool _keepHistoryForNext; + bool _needAlloc; + bool _wimMode; + Byte _numDictBits; + + // unsigned _numAlignBits_PosSlots; + // unsigned _numAlignBits; + UInt32 _numAlignBits_Dist; +private: + unsigned _numPosLenSlots; + UInt32 _unpackBlockSize; + + UInt32 _writePos; + + UInt32 _x86_translationSize; + UInt32 _x86_processedSize; + Byte *_x86_buf; + + Byte *_unpackedData; +public: + Byte _extra[kPosSlotOffset + kNumPosSlots]; + UInt32 _reps[kPosSlotOffset + kNumPosSlots]; + + NHuffman::CDecoder _mainDecoder; + NHuffman::CDecoder256 _lenDecoder; + NHuffman::CDecoder7b _alignDecoder; +private: + Byte _mainLevels[kMainTableSize]; + Byte _lenLevels[kNumLenSymols_Big]; + + HRESULT Flush() throw(); + bool ReadTables(CBitByteDecoder &_bitStream) throw(); + + HRESULT CodeSpec(const Byte *inData, size_t inSize, UInt32 outSize) throw(); + HRESULT SetParams2(unsigned numDictBits) throw(); +public: + CDecoder() throw(); + ~CDecoder() throw(); + + void Set_WimMode(bool wimMode) { _wimMode = wimMode; } + void Set_KeepHistory(bool keepHistory) { _keepHistory = keepHistory; } + void Set_KeepHistoryForNext(bool keepHistoryForNext) { _keepHistoryForNext = keepHistoryForNext; } + + HRESULT Set_ExternalWindow_DictBits(Byte *win, unsigned numDictBits) + { + _needAlloc = false; + _win = win; + _winSize = (UInt32)1 << numDictBits; + return SetParams2(numDictBits); + } + HRESULT Set_DictBits_and_Alloc(unsigned numDictBits) throw(); + + HRESULT Code_WithExceedReadWrite(const Byte *inData, size_t inSize, UInt32 outSize) throw(); + + bool WasBlockFinished() const { return _unpackBlockSize == 0; } + const Byte *GetUnpackData() const { return _unpackedData; } + UInt32 GetUnpackSize() const { return _pos - _writePos; } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Mtf8.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Mtf8.h new file mode 100644 index 0000000..5fce30e --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Mtf8.h @@ -0,0 +1,225 @@ +// Mtf8.h + +#ifndef ZIP7_INC_COMPRESS_MTF8_H +#define ZIP7_INC_COMPRESS_MTF8_H + +#include "../../../C/CpuArch.h" + +namespace NCompress { + +struct CMtf8Encoder +{ + Byte Buf[256]; + + unsigned FindAndMove(Byte v) throw() + { +#if 1 + Byte b = Buf[0]; + if (v == b) + return 0; + Buf[0] = v; + for (unsigned pos = 0;;) + { + Byte a; + a = Buf[++pos]; Buf[pos] = b; if (v == a) return pos; + b = Buf[++pos]; Buf[pos] = a; if (v == b) return pos; + } +#else + size_t pos; + for (pos = 0; Buf[pos] != v; pos++); + const unsigned resPos = (unsigned)pos; + for (; pos >= 8; pos -= 8) + { + Buf[pos] = Buf[pos - 1]; + Buf[pos - 1] = Buf[pos - 2]; + Buf[pos - 2] = Buf[pos - 3]; + Buf[pos - 3] = Buf[pos - 4]; + Buf[pos - 4] = Buf[pos - 5]; + Buf[pos - 5] = Buf[pos - 6]; + Buf[pos - 6] = Buf[pos - 7]; + Buf[pos - 7] = Buf[pos - 8]; + } + for (; pos != 0; pos--) + Buf[pos] = Buf[pos - 1]; + Buf[0] = v; + return resPos; +#endif + } +}; + +/* +struct CMtf8Decoder +{ + Byte Buf[256]; + + void StartInit() { memset(Buf, 0, sizeof(Buf)); } + void Add(unsigned pos, Byte val) { Buf[pos] = val; } + Byte GetHead() const { return Buf[0]; } + Byte GetAndMove(unsigned pos) + { + Byte res = Buf[pos]; + for (; pos >= 8; pos -= 8) + { + Buf[pos] = Buf[pos - 1]; + Buf[pos - 1] = Buf[pos - 2]; + Buf[pos - 2] = Buf[pos - 3]; + Buf[pos - 3] = Buf[pos - 4]; + Buf[pos - 4] = Buf[pos - 5]; + Buf[pos - 5] = Buf[pos - 6]; + Buf[pos - 6] = Buf[pos - 7]; + Buf[pos - 7] = Buf[pos - 8]; + } + for (; pos > 0; pos--) + Buf[pos] = Buf[pos - 1]; + Buf[0] = res; + return res; + } +}; +*/ + +#ifdef MY_CPU_64BIT + typedef UInt64 CMtfVar; + #define Z7_MTF_MOVS 3 +#else + typedef UInt32 CMtfVar; + #define Z7_MTF_MOVS 2 +#endif + +#define Z7_MTF_MASK ((1 << Z7_MTF_MOVS) - 1) + + +struct CMtf8Decoder +{ + CMtfVar Buf[256 >> Z7_MTF_MOVS]; + + void StartInit() { memset(Buf, 0, sizeof(Buf)); } + void Add(unsigned pos, Byte val) { Buf[pos >> Z7_MTF_MOVS] |= ((CMtfVar)val << ((pos & Z7_MTF_MASK) << 3)); } + Byte GetHead() const { return (Byte)Buf[0]; } + + Z7_FORCE_INLINE + Byte GetAndMove(unsigned pos) throw() + { + const UInt32 lim = ((UInt32)pos >> Z7_MTF_MOVS); + pos = (pos & Z7_MTF_MASK) << 3; + CMtfVar prev = (Buf[lim] >> pos) & 0xFF; + + UInt32 i = 0; + + + /* + if ((lim & 1) != 0) + { + CMtfVar next = Buf[0]; + Buf[0] = (next << 8) | prev; + prev = (next >> (Z7_MTF_MASK << 3)); + i = 1; + lim -= 1; + } + for (; i < lim; i += 2) + { + CMtfVar n0 = Buf[i]; + CMtfVar n1 = Buf[i + 1]; + Buf[i ] = (n0 << 8) | prev; + Buf[i + 1] = (n1 << 8) | (n0 >> (Z7_MTF_MASK << 3)); + prev = (n1 >> (Z7_MTF_MASK << 3)); + } + */ + + for (; i < lim; i++) + { + const CMtfVar n0 = Buf[i]; + Buf[i ] = (n0 << 8) | prev; + prev = (n0 >> (Z7_MTF_MASK << 3)); + } + + + const CMtfVar next = Buf[i]; + const CMtfVar mask = (((CMtfVar)0x100 << pos) - 1); + Buf[i] = (next & ~mask) | (((next << 8) | prev) & mask); + return (Byte)Buf[0]; + } +}; + +/* +const int kSmallSize = 64; +class CMtf8Decoder +{ + Byte SmallBuffer[kSmallSize]; + int SmallSize; + int Counts[16]; + int Size; +public: + Byte Buf[256]; + + Byte GetHead() const + { + if (SmallSize > 0) + return SmallBuffer[kSmallSize - SmallSize]; + return Buf[0]; + } + + void Init(int size) + { + Size = size; + SmallSize = 0; + for (int i = 0; i < 16; i++) + { + Counts[i] = ((size >= 16) ? 16 : size); + size -= Counts[i]; + } + } + + void Add(unsigned pos, Byte val) + { + Buf[pos] = val; + } + + Byte GetAndMove(int pos) + { + if (pos < SmallSize) + { + Byte *p = SmallBuffer + kSmallSize - SmallSize; + Byte res = p[pos]; + for (; pos > 0; pos--) + p[pos] = p[pos - 1]; + SmallBuffer[kSmallSize - SmallSize] = res; + return res; + } + if (SmallSize == kSmallSize) + { + int i = Size - 1; + int g = 16; + do + { + g--; + int offset = (g << 4); + for (int t = Counts[g] - 1; t >= 0; t--, i--) + Buf[i] = Buf[offset + t]; + } + while (g != 0); + + for (i = kSmallSize - 1; i >= 0; i--) + Buf[i] = SmallBuffer[i]; + Init(Size); + } + pos -= SmallSize; + int g; + for (g = 0; pos >= Counts[g]; g++) + pos -= Counts[g]; + int offset = (g << 4); + Byte res = Buf[offset + pos]; + for (pos; pos < 16 - 1; pos++) + Buf[offset + pos] = Buf[offset + pos + 1]; + + SmallSize++; + SmallBuffer[kSmallSize - SmallSize] = res; + + Counts[g]--; + return res; + } +}; +*/ + +} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/PpmdZip.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/PpmdZip.cpp new file mode 100644 index 0000000..e59e52f --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/PpmdZip.cpp @@ -0,0 +1,309 @@ +// PpmdZip.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" + +#include "../Common/RegisterCodec.h" +#include "../Common/StreamUtils.h" + +#include "PpmdZip.h" + +namespace NCompress { +namespace NPpmdZip { + +static const UInt32 kBufSize = 1 << 20; + +bool CBuf::Alloc() +{ + if (!Buf) + Buf = (Byte *)::MidAlloc(kBufSize); + return (Buf != NULL); +} + +CDecoder::CDecoder(bool fullFileMode): + _fullFileMode(fullFileMode) +{ + Ppmd8_Construct(&_ppmd); + _ppmd.Stream.In = &_inStream.vt; +} + +CDecoder::~CDecoder() +{ + Ppmd8_Free(&_ppmd, &g_BigAlloc); +} + +Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + // try { + + if (!_outStream.Alloc()) + return E_OUTOFMEMORY; + if (!_inStream.Alloc(1 << 20)) + return E_OUTOFMEMORY; + + _inStream.Stream = inStream; + _inStream.Init(); + + { + Byte buf[2]; + for (int i = 0; i < 2; i++) + buf[i] = _inStream.ReadByte(); + if (_inStream.Extra) + return S_FALSE; + + const UInt32 val = GetUi16(buf); + const unsigned order = (val & 0xF) + 1; + const UInt32 mem = ((val >> 4) & 0xFF) + 1; + const unsigned restor = (val >> 12); + if (order < 2 || restor > 2) + return S_FALSE; + + #ifndef PPMD8_FREEZE_SUPPORT + if (restor == 2) + return E_NOTIMPL; + #endif + + if (!Ppmd8_Alloc(&_ppmd, mem << 20, &g_BigAlloc)) + return E_OUTOFMEMORY; + + if (!Ppmd8_Init_RangeDec(&_ppmd)) + return S_FALSE; + Ppmd8_Init(&_ppmd, order, restor); + } + + bool wasFinished = false; + UInt64 processedSize = 0; + + for (;;) + { + size_t size = kBufSize; + if (outSize) + { + const UInt64 rem = *outSize - processedSize; + if (size > rem) + { + size = (size_t)rem; + if (size == 0) + break; + } + } + + int sym; + Byte *buf = _outStream.Buf; + const Byte *lim = buf + size; + + do + { + sym = Ppmd8_DecodeSymbol(&_ppmd); + if (_inStream.Extra || sym < 0) + break; + *buf++ = (Byte)sym; + } + while (buf != lim); + + const size_t cur = (size_t)(buf - _outStream.Buf); + processedSize += cur; + + RINOK(WriteStream(outStream, _outStream.Buf, cur)) + + RINOK(_inStream.Res) + if (_inStream.Extra) + return S_FALSE; + + if (sym < 0) + { + if (sym != -1) + return S_FALSE; + wasFinished = true; + break; + } + + if (progress) + { + const UInt64 inProccessed = _inStream.GetProcessed(); + RINOK(progress->SetRatioInfo(&inProccessed, &processedSize)) + } + } + + RINOK(_inStream.Res) + + if (_fullFileMode) + { + if (!wasFinished) + { + const int res = Ppmd8_DecodeSymbol(&_ppmd); + RINOK(_inStream.Res) + if (_inStream.Extra || res != -1) + return S_FALSE; + } + if (!Ppmd8_RangeDec_IsFinishedOK(&_ppmd)) + return S_FALSE; + + if (inSize && *inSize != _inStream.GetProcessed()) + return S_FALSE; + } + + return S_OK; + + // } catch (...) { return E_FAIL; } +} + + +Z7_COM7F_IMF(CDecoder::SetFinishMode(UInt32 finishMode)) +{ + _fullFileMode = (finishMode != 0); + return S_OK; +} + +Z7_COM7F_IMF(CDecoder::GetInStreamProcessedSize(UInt64 *value)) +{ + *value = _inStream.GetProcessed(); + return S_OK; +} + + + +// ---------- Encoder ---------- + +void CEncProps::Normalize(int level) +{ + if (level < 0) level = 5; + if (level == 0) level = 1; + if (level > 9) level = 9; + if (MemSizeMB == (UInt32)(Int32)-1) + MemSizeMB = 1 << (level - 1); + const unsigned kMult = 16; + for (UInt32 m = 1; m < MemSizeMB; m <<= 1) + if (ReduceSize <= (m << 20) / kMult) + { + MemSizeMB = m; + break; + } + if (Order == -1) Order = 3 + level; + if (Restor == -1) + Restor = level < 7 ? + PPMD8_RESTORE_METHOD_RESTART : + PPMD8_RESTORE_METHOD_CUT_OFF; +} + +CEncoder::~CEncoder() +{ + Ppmd8_Free(&_ppmd, &g_BigAlloc); +} + +Z7_COM7F_IMF(CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps)) +{ + int level = -1; + CEncProps props; + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + const PROPID propID = propIDs[i]; + if (propID > NCoderPropID::kReduceSize) + continue; + if (propID == NCoderPropID::kReduceSize) + { + props.ReduceSize = (UInt32)(Int32)-1; + if (prop.vt == VT_UI8 && prop.uhVal.QuadPart < (UInt32)(Int32)-1) + props.ReduceSize = (UInt32)prop.uhVal.QuadPart; + continue; + } + if (prop.vt != VT_UI4) + return E_INVALIDARG; + const UInt32 v = (UInt32)prop.ulVal; + switch (propID) + { + case NCoderPropID::kUsedMemorySize: + if (v < (1 << 20) || v > (1 << 28)) + return E_INVALIDARG; + props.MemSizeMB = v >> 20; + break; + case NCoderPropID::kOrder: + if (v < PPMD8_MIN_ORDER || v > PPMD8_MAX_ORDER) + return E_INVALIDARG; + props.Order = (Byte)v; + break; + case NCoderPropID::kNumThreads: break; + case NCoderPropID::kLevel: level = (int)v; break; + case NCoderPropID::kAlgorithm: + if (v >= PPMD8_RESTORE_METHOD_UNSUPPPORTED) + return E_INVALIDARG; + props.Restor = (int)v; + break; + default: return E_INVALIDARG; + } + } + props.Normalize(level); + _props = props; + return S_OK; +} + +CEncoder::CEncoder() +{ + _props.Normalize(-1); + _ppmd.Stream.Out = &_outStream.vt; + Ppmd8_Construct(&_ppmd); +} + +Z7_COM7F_IMF(CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)) +{ + if (!_inStream.Alloc()) + return E_OUTOFMEMORY; + if (!_outStream.Alloc(1 << 20)) + return E_OUTOFMEMORY; + if (!Ppmd8_Alloc(&_ppmd, _props.MemSizeMB << 20, &g_BigAlloc)) + return E_OUTOFMEMORY; + + _outStream.Stream = outStream; + _outStream.Init(); + + Ppmd8_Init_RangeEnc(&_ppmd) + Ppmd8_Init(&_ppmd, (unsigned)_props.Order, (unsigned)_props.Restor); + + { + const unsigned val = + ((unsigned)_props.Order - 1) + + (((unsigned)_props.MemSizeMB - 1) << 4) + + ((unsigned)_props.Restor << 12); + _outStream.WriteByte((Byte)(val & 0xFF)); + _outStream.WriteByte((Byte)(val >> 8)); + } + RINOK(_outStream.Res) + + UInt64 processed = 0; + for (;;) + { + UInt32 size; + RINOK(inStream->Read(_inStream.Buf, kBufSize, &size)) + if (size == 0) + { + Ppmd8_EncodeSymbol(&_ppmd, -1); + Ppmd8_Flush_RangeEnc(&_ppmd); + return _outStream.Flush(); + } + + processed += size; + const Byte *buf = _inStream.Buf; + const Byte *lim = buf + size; + do + { + Ppmd8_EncodeSymbol(&_ppmd, *buf); + if (_outStream.Res != S_OK) + break; + } + while (++buf != lim); + + RINOK(_outStream.Res) + + if (progress) + { + const UInt64 outProccessed = _outStream.GetProcessed(); + RINOK(progress->SetRatioInfo(&processed, &outProccessed)) + } + } +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/PpmdZip.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/PpmdZip.h new file mode 100644 index 0000000..6cc6af1 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/PpmdZip.h @@ -0,0 +1,78 @@ +// PpmdZip.h + +#ifndef ZIP7_INC_COMPRESS_PPMD_ZIP_H +#define ZIP7_INC_COMPRESS_PPMD_ZIP_H + +#include "../../../C/Alloc.h" +#include "../../../C/Ppmd8.h" + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/CWrappers.h" + +namespace NCompress { +namespace NPpmdZip { + +struct CBuf +{ + Byte *Buf; + + CBuf(): Buf(NULL) {} + ~CBuf() { ::MidFree(Buf); } + bool Alloc(); +}; + + +Z7_CLASS_IMP_NOQIB_3( + CDecoder + , ICompressCoder + , ICompressSetFinishMode + , ICompressGetInStreamProcessedSize +) + bool _fullFileMode; + CByteInBufWrap _inStream; + CBuf _outStream; + CPpmd8 _ppmd; +public: + CDecoder(bool fullFileMode = true); + ~CDecoder(); +}; + + +struct CEncProps +{ + UInt32 MemSizeMB; + UInt32 ReduceSize; + int Order; + int Restor; + + CEncProps() + { + MemSizeMB = (UInt32)(Int32)-1; + ReduceSize = (UInt32)(Int32)-1; + Order = -1; + Restor = -1; + } + void Normalize(int level); +}; + + +Z7_CLASS_IMP_NOQIB_2( + CEncoder + , ICompressCoder + , ICompressSetCoderProperties +) + CByteOutBufWrap _outStream; + CBuf _inStream; + CPpmd8 _ppmd; + CEncProps _props; +public: + CEncoder(); + ~CEncoder(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/QuantumDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/QuantumDecoder.h new file mode 100644 index 0000000..fd5baea --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/QuantumDecoder.h @@ -0,0 +1,60 @@ +// QuantumDecoder.h + +#ifndef ZIP7_INC_COMPRESS_QUANTUM_DECODER_H +#define ZIP7_INC_COMPRESS_QUANTUM_DECODER_H + +#include "../../Common/MyTypes.h" + +namespace NCompress { +namespace NQuantum { + +const unsigned kNumLitSelectorBits = 2; +const unsigned kNumLitSelectors = 1 << kNumLitSelectorBits; +const unsigned kNumLitSymbols = 1 << (8 - kNumLitSelectorBits); +const unsigned kNumMatchSelectors = 3; +const unsigned kNumSelectors = kNumLitSelectors + kNumMatchSelectors; +const unsigned kNumSymbolsMax = kNumLitSymbols; // 64 + +class CRangeDecoder; + +class CModelDecoder +{ + unsigned NumItems; + unsigned ReorderCount; + Byte Vals[kNumSymbolsMax]; + UInt16 Freqs[kNumSymbolsMax + 1]; +public: + Byte _pad[64 - 10]; // for structure size alignment + + void Init(unsigned numItems, unsigned startVal); + unsigned Decode(CRangeDecoder *rc); +}; + + +class CDecoder +{ + UInt32 _winSize; + UInt32 _winPos; + UInt32 _winSize_allocated; + bool _overWin; + Byte *_win; + unsigned _numDictBits; + + CModelDecoder m_Selector; + CModelDecoder m_Literals[kNumLitSelectors]; + CModelDecoder m_PosSlot[kNumMatchSelectors]; + CModelDecoder m_LenSlot; + + void Init(); + HRESULT CodeSpec(const Byte *inData, size_t inSize, UInt32 outSize); +public: + HRESULT Code(const Byte *inData, size_t inSize, UInt32 outSize, bool keepHistory); + HRESULT SetParams(unsigned numDictBits); + + CDecoder(): _win(NULL), _numDictBits(0) {} + const Byte * GetDataPtr() const { return _win + _winPos; } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar1Decoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar1Decoder.h new file mode 100644 index 0000000..e1e22e2 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar1Decoder.h @@ -0,0 +1,70 @@ +// Rar1Decoder.h +// According to unRAR license, this code may not be used to develop +// a program that creates RAR archives + +#ifndef ZIP7_INC_COMPRESS_RAR1_DECODER_H +#define ZIP7_INC_COMPRESS_RAR1_DECODER_H + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" + +#include "BitmDecoder.h" +#include "LzOutWindow.h" + +namespace NCompress { +namespace NRar1 { + +const unsigned kNumRepDists = 4; + +Z7_CLASS_IMP_COM_2( + CDecoder + , ICompressCoder + , ICompressSetDecoderProperties2 +) + bool _isSolid; + bool _solidAllowed; + bool StMode; + + CLzOutWindow m_OutWindowStream; + NBitm::CDecoder m_InBitStream; + + UInt64 m_UnpackSize; + + UInt32 LastDist; + UInt32 LastLength; + + UInt32 m_RepDistPtr; + UInt32 m_RepDists[kNumRepDists]; + + int FlagsCnt; + UInt32 FlagBuf, AvrPlc, AvrPlcB, AvrLn1, AvrLn2, AvrLn3; + unsigned Buf60, NumHuf, LCount; + UInt32 Nhfb, Nlzb, MaxDist3; + + UInt32 ChSet[256], ChSetA[256], ChSetB[256], ChSetC[256]; + UInt32 Place[256], PlaceA[256], PlaceB[256], PlaceC[256]; + UInt32 NToPl[256], NToPlB[256], NToPlC[256]; + + UInt32 ReadBits(unsigned numBits); + HRESULT CopyBlock(UInt32 distance, UInt32 len); + UInt32 DecodeNum(const Byte *numTab); + HRESULT ShortLZ(); + HRESULT LongLZ(); + HRESULT HuffDecode(); + void GetFlagsBuf(); + void CorrHuff(UInt32 *CharSet, UInt32 *NumToPlace); + void OldUnpWriteBuf(); + + HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + +public: + CDecoder(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar2Decoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar2Decoder.h new file mode 100644 index 0000000..c9ddedb --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar2Decoder.h @@ -0,0 +1,111 @@ +// Rar2Decoder.h +// According to unRAR license, this code may not be used to develop +// a program that creates RAR archives + +#ifndef ZIP7_INC_COMPRESS_RAR2_DECODER_H +#define ZIP7_INC_COMPRESS_RAR2_DECODER_H + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" + +#include "BitmDecoder.h" +#include "HuffmanDecoder.h" +#include "LzOutWindow.h" + +namespace NCompress { +namespace NRar2 { + +const unsigned kNumReps = 4; +const unsigned kDistTableSize = 48; +const unsigned kNumLen2Symbols = 8; +const unsigned kLenTableSize = 28; +const unsigned kMainTableSize = 256 + 2 + kNumReps + kNumLen2Symbols + kLenTableSize; +const unsigned kHeapTablesSizesSum = kMainTableSize + kDistTableSize + kLenTableSize; +const unsigned k_MM_TableSize = 256 + 1; +const unsigned k_MM_NumChanelsMax = 4; +const unsigned k_MM_TablesSizesSum = k_MM_TableSize * k_MM_NumChanelsMax; +const unsigned kMaxTableSize = k_MM_TablesSizesSum; + +namespace NMultimedia { + +struct CFilter +{ + int K1,K2,K3,K4,K5; + int D1,D2,D3,D4; + int LastDelta; + UInt32 Dif[11]; + UInt32 ByteCount; + int LastChar; + + void Init() { memset(this, 0, sizeof(*this)); } + Byte Decode(int &channelDelta, Byte delta); +}; + +struct CFilter2 +{ + CFilter m_Filters[k_MM_NumChanelsMax]; + int m_ChannelDelta; + unsigned CurrentChannel; + + void Init() { memset(this, 0, sizeof(*this)); } + Byte Decode(Byte delta) + { + return m_Filters[CurrentChannel].Decode(m_ChannelDelta, delta); + } +}; + +} + +typedef NBitm::CDecoder CBitDecoder; + +const unsigned kNumHufBits = 15; + +Z7_CLASS_IMP_NOQIB_2( + CDecoder + , ICompressCoder + , ICompressSetDecoderProperties2 +) + bool _isSolid; + bool _solidAllowed; + bool m_TablesOK; + bool m_AudioMode; + + CLzOutWindow m_OutWindowStream; + CBitDecoder m_InBitStream; + + UInt32 m_RepDistPtr; + UInt32 m_RepDists[kNumReps]; + UInt32 m_LastLength; + unsigned m_NumChannels; + + NHuffman::CDecoder m_MainDecoder; + NHuffman::CDecoder256 m_DistDecoder; + NHuffman::CDecoder256 m_LenDecoder; + NHuffman::CDecoder m_MMDecoders[k_MM_NumChanelsMax]; + + UInt64 m_PackSize; + + NMultimedia::CFilter2 m_MmFilter; + Byte m_LastLevels[kMaxTableSize]; + + void InitStructures(); + UInt32 ReadBits(unsigned numBits); + bool ReadTables(); + bool ReadLastTables(); + + bool DecodeMm(UInt32 pos); + bool DecodeLz(Int32 pos); + + HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + +public: + CDecoder(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar3Decoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar3Decoder.h new file mode 100644 index 0000000..48fc212 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar3Decoder.h @@ -0,0 +1,283 @@ +// Rar3Decoder.h +// According to unRAR license, this code may not be used to develop +// a program that creates RAR archives + +/* This code uses Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ + +#ifndef ZIP7_INC_COMPRESS_RAR3_DECODER_H +#define ZIP7_INC_COMPRESS_RAR3_DECODER_H + +#include "../../../C/Ppmd7.h" + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" + +#include "BitmDecoder.h" +#include "HuffmanDecoder.h" +#include "Rar3Vm.h" + +namespace NCompress { +namespace NRar3 { + +const unsigned kNumHuffmanBits = 15; + +const UInt32 kWindowSize = 1 << 22; +const UInt32 kWindowMask = kWindowSize - 1; + +const unsigned kNumReps = 4; +const unsigned kNumLen2Symbols = 8; +const unsigned kLenTableSize = 28; +const unsigned kMainTableSize = 256 + 3 + kNumReps + kNumLen2Symbols + kLenTableSize; +const unsigned kDistTableSize = 60; + +const unsigned kNumAlignBits = 4; +const unsigned kAlignTableSize = (1 << kNumAlignBits) + 1; + +const unsigned kTablesSizesSum = kMainTableSize + kDistTableSize + kAlignTableSize + kLenTableSize; + +class CBitDecoder +{ + UInt32 _value; + unsigned _bitPos; +public: + CInBuffer Stream; + + bool Create(UInt32 bufSize) { return Stream.Create(bufSize); } + void SetStream(ISequentialInStream *inStream) { Stream.SetStream(inStream);} + + void Init() + { + Stream.Init(); + _bitPos = 0; + _value = 0; + } + + bool ExtraBitsWereRead() const + { + return (Stream.NumExtraBytes > 4 || _bitPos < (Stream.NumExtraBytes << 3)); + } + + UInt64 GetProcessedSize() const { return Stream.GetProcessedSize() - (_bitPos >> 3); } + + void AlignToByte() + { + _bitPos &= ~(unsigned)7; + _value = _value & ((1 << _bitPos) - 1); + } + + Z7_FORCE_INLINE + UInt32 GetValue(unsigned numBits) + { + if (_bitPos < numBits) + { + _bitPos += 8; + _value = (_value << 8) | Stream.ReadByte(); + if (_bitPos < numBits) + { + _bitPos += 8; + _value = (_value << 8) | Stream.ReadByte(); + } + } + return _value >> (_bitPos - numBits); + } + + Z7_FORCE_INLINE + UInt32 GetValue_InHigh32bits() + { + return GetValue(kNumHuffmanBits) << (32 - kNumHuffmanBits); + } + + + Z7_FORCE_INLINE + void MovePos(unsigned numBits) + { + _bitPos -= numBits; + _value = _value & ((1 << _bitPos) - 1); + } + + UInt32 ReadBits(unsigned numBits) + { + const UInt32 res = GetValue(numBits); + MovePos(numBits); + return res; + } + + UInt32 ReadBits_upto8(unsigned numBits) + { + if (_bitPos < numBits) + { + _bitPos += 8; + _value = (_value << 8) | Stream.ReadByte(); + } + _bitPos -= numBits; + const UInt32 res = _value >> _bitPos; + _value = _value & ((1 << _bitPos) - 1); + return res; + } + + Byte ReadByteFromAligned() + { + if (_bitPos == 0) + return Stream.ReadByte(); + const unsigned bitsPos = _bitPos - 8; + const Byte b = (Byte)(_value >> bitsPos); + _value = _value & ((1 << bitsPos) - 1); + _bitPos = bitsPos; + return b; + } +}; + + +struct CByteIn +{ + IByteIn IByteIn_obj; + CBitDecoder BitDecoder; +}; + + +struct CFilter: public NVm::CProgram +{ + CRecordVector GlobalData; + UInt32 BlockStart; + UInt32 BlockSize; + UInt32 ExecCount; + + CFilter(): BlockStart(0), BlockSize(0), ExecCount(0) {} +}; + +struct CTempFilter: public NVm::CProgramInitState +{ + UInt32 BlockStart; + UInt32 BlockSize; + bool NextWindow; + + UInt32 FilterIndex; + + CTempFilter() + { + // all filters must contain at least FixedGlobal block + AllocateEmptyFixedGlobal(); + } +}; + + +Z7_CLASS_IMP_NOQIB_2( + CDecoder + , ICompressCoder + , ICompressSetDecoderProperties2 +) + bool _isSolid; + bool _solidAllowed; + // bool _errorMode; + + bool _lzMode; + bool _unsupportedFilter; + + CByteIn m_InBitStream; + Byte *_window; + UInt32 _winPos; + UInt32 _wrPtr; + UInt64 _lzSize; + UInt64 _unpackSize; + UInt64 _writtenFileSize; // if it's > _unpackSize, then _unpackSize only written + ISequentialOutStream *_outStream; + + NHuffman::CDecoder m_MainDecoder; + UInt32 kDistStart[kDistTableSize]; + NHuffman::CDecoder256 m_DistDecoder; + NHuffman::CDecoder256 m_AlignDecoder; + NHuffman::CDecoder256 m_LenDecoder; + + UInt32 _reps[kNumReps]; + UInt32 _lastLength; + + Byte m_LastLevels[kTablesSizesSum]; + + Byte *_vmData; + Byte *_vmCode; + NVm::CVm _vm; + CRecordVector _filters; + CRecordVector _tempFilters; + unsigned _numEmptyTempFilters; + UInt32 _lastFilter; + + UInt32 PrevAlignBits; + UInt32 PrevAlignCount; + + bool TablesRead; + bool TablesOK; + bool PpmError; + + int PpmEscChar; + CPpmd7 _ppmd; + + HRESULT WriteDataToStream(const Byte *data, UInt32 size); + HRESULT WriteData(const Byte *data, UInt32 size); + HRESULT WriteArea(UInt32 startPtr, UInt32 endPtr); + void ExecuteFilter(unsigned tempFilterIndex, NVm::CBlockRef &outBlockRef); + HRESULT WriteBuf(); + + void InitFilters(); + bool AddVmCode(UInt32 firstByte, UInt32 codeSize); + bool ReadVmCodeLZ(); + bool ReadVmCodePPM(); + + UInt32 ReadBits(unsigned numBits); + + HRESULT InitPPM(); + // int DecodePpmSymbol(); + HRESULT DecodePPM(Int32 num, bool &keepDecompressing); + + HRESULT ReadTables(bool &keepDecompressing); + HRESULT ReadEndOfBlock(bool &keepDecompressing); + HRESULT DecodeLZ(bool &keepDecompressing); + HRESULT CodeReal(ICompressProgressInfo *progress); + + bool InputEofError() const { return m_InBitStream.BitDecoder.ExtraBitsWereRead(); } + bool InputEofError_Fast() const { return (m_InBitStream.BitDecoder.Stream.NumExtraBytes > 2); } + + void CopyBlock(UInt32 dist, UInt32 len) + { + _lzSize += len; + UInt32 pos = (_winPos - dist - 1) & kWindowMask; + Byte *window = _window; + UInt32 winPos = _winPos; + if (kWindowSize - winPos > len && kWindowSize - pos > len) + { + const Byte *src = window + pos; + Byte *dest = window + winPos; + _winPos += len; + do + *dest++ = *src++; + while (--len != 0); + return; + } + do + { + window[winPos] = window[pos]; + winPos = (winPos + 1) & kWindowMask; + pos = (pos + 1) & kWindowMask; + } + while (--len != 0); + _winPos = winPos; + } + + void PutByte(Byte b) + { + const UInt32 wp = _winPos; + _window[wp] = b; + _winPos = (wp + 1) & kWindowMask; + _lzSize++; + } + +public: + CDecoder(); + ~CDecoder(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar3Vm.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar3Vm.h new file mode 100644 index 0000000..fde7e95 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar3Vm.h @@ -0,0 +1,195 @@ +// Rar3Vm.h +// According to unRAR license, this code may not be used to develop +// a program that creates RAR archives + +#ifndef ZIP7_INC_COMPRESS_RAR3_VM_H +#define ZIP7_INC_COMPRESS_RAR3_VM_H + +#include "../../../C/CpuArch.h" + +#include "../../Common/MyVector.h" + +#define Z7_RARVM_STANDARD_FILTERS +// #define Z7_RARVM_VM_ENABLE + +namespace NCompress { +namespace NRar3 { + +class CMemBitDecoder +{ + const Byte *_data; + UInt32 _bitSize; + UInt32 _bitPos; +public: + void Init(const Byte *data, UInt32 byteSize) + { + _data = data; + _bitSize = (byteSize << 3); + _bitPos = 0; + } + UInt32 ReadBits(unsigned numBits); + UInt32 ReadBit(); + bool Avail() const { return (_bitPos < _bitSize); } + + UInt32 ReadEncodedUInt32(); +}; + +namespace NVm { + +inline UInt32 GetValue32(const void *addr) { return GetUi32(addr); } +inline void SetValue32(void *addr, UInt32 value) { SetUi32(addr, value) } + +const unsigned kNumRegBits = 3; +const UInt32 kNumRegs = 1 << kNumRegBits; +const UInt32 kNumGpRegs = kNumRegs - 1; + +const UInt32 kSpaceSize = 0x40000; +const UInt32 kSpaceMask = kSpaceSize - 1; +const UInt32 kGlobalOffset = 0x3C000; +const UInt32 kGlobalSize = 0x2000; +const UInt32 kFixedGlobalSize = 64; + +namespace NGlobalOffset +{ + const UInt32 kBlockSize = 0x1C; + const UInt32 kBlockPos = 0x20; + const UInt32 kExecCount = 0x2C; + const UInt32 kGlobalMemOutSize = 0x30; +} + + +#ifdef Z7_RARVM_VM_ENABLE + +enum ECommand +{ + CMD_MOV, CMD_CMP, CMD_ADD, CMD_SUB, CMD_JZ, CMD_JNZ, CMD_INC, CMD_DEC, + CMD_JMP, CMD_XOR, CMD_AND, CMD_OR, CMD_TEST, CMD_JS, CMD_JNS, CMD_JB, + CMD_JBE, CMD_JA, CMD_JAE, CMD_PUSH, CMD_POP, CMD_CALL, CMD_RET, CMD_NOT, + CMD_SHL, CMD_SHR, CMD_SAR, CMD_NEG, CMD_PUSHA,CMD_POPA, CMD_PUSHF,CMD_POPF, + CMD_MOVZX,CMD_MOVSX,CMD_XCHG, CMD_MUL, CMD_DIV, CMD_ADC, CMD_SBB, CMD_PRINT, + + CMD_MOVB, CMD_CMPB, CMD_ADDB, CMD_SUBB, CMD_INCB, CMD_DECB, + CMD_XORB, CMD_ANDB, CMD_ORB, CMD_TESTB,CMD_NEGB, + CMD_SHLB, CMD_SHRB, CMD_SARB, CMD_MULB +}; + +enum EOpType {OP_TYPE_REG, OP_TYPE_INT, OP_TYPE_REGMEM, OP_TYPE_NONE}; + +// Addr in COperand object can link (point) to CVm object!!! + +struct COperand +{ + EOpType Type; + UInt32 Data; + UInt32 Base; + COperand(): Type(OP_TYPE_NONE), Data(0), Base(0) {} +}; + +struct CCommand +{ + ECommand OpCode; + bool ByteMode; + COperand Op1, Op2; +}; + +#endif + + +struct CBlockRef +{ + UInt32 Offset; + UInt32 Size; +}; + + +class CProgram +{ + #ifdef Z7_RARVM_VM_ENABLE + void ReadProgram(const Byte *code, UInt32 codeSize); +public: + CRecordVector Commands; + #endif + +public: + #ifdef Z7_RARVM_STANDARD_FILTERS + int StandardFilterIndex; + #endif + + bool IsSupported; + CRecordVector StaticData; + + bool PrepareProgram(const Byte *code, UInt32 codeSize); +}; + + +struct CProgramInitState +{ + UInt32 InitR[kNumGpRegs]; + CRecordVector GlobalData; + + void AllocateEmptyFixedGlobal() + { + GlobalData.ClearAndSetSize(NVm::kFixedGlobalSize); + memset(&GlobalData[0], 0, NVm::kFixedGlobalSize); + } +}; + + +class CVm +{ + static UInt32 GetValue(bool byteMode, const void *addr) + { + if (byteMode) + return(*(const Byte *)addr); + else + return GetUi32(addr); + } + + static void SetValue(bool byteMode, void *addr, UInt32 value) + { + if (byteMode) + *(Byte *)addr = (Byte)value; + else + SetUi32(addr, value) + } + + UInt32 GetFixedGlobalValue32(UInt32 globalOffset) { return GetValue(false, &Mem[kGlobalOffset + globalOffset]); } + + void SetBlockSize(UInt32 v) { SetValue(&Mem[kGlobalOffset + NGlobalOffset::kBlockSize], v); } + void SetBlockPos(UInt32 v) { SetValue(&Mem[kGlobalOffset + NGlobalOffset::kBlockPos], v); } +public: + static void SetValue(void *addr, UInt32 value) { SetValue(false, addr, value); } + +private: + + #ifdef Z7_RARVM_VM_ENABLE + UInt32 GetOperand32(const COperand *op) const; + void SetOperand32(const COperand *op, UInt32 val); + Byte GetOperand8(const COperand *op) const; + void SetOperand8(const COperand *op, Byte val); + UInt32 GetOperand(bool byteMode, const COperand *op) const; + void SetOperand(bool byteMode, const COperand *op, UInt32 val); + bool ExecuteCode(const CProgram *prg); + #endif + + #ifdef Z7_RARVM_STANDARD_FILTERS + bool ExecuteStandardFilter(unsigned filterIndex); + #endif + + Byte *Mem; + UInt32 R[kNumRegs + 1]; // R[kNumRegs] = 0 always (speed optimization) + UInt32 Flags; + +public: + CVm(); + ~CVm(); + bool Create(); + void SetMemory(UInt32 pos, const Byte *data, UInt32 dataSize); + bool Execute(CProgram *prg, const CProgramInitState *initState, + CBlockRef &outBlockRef, CRecordVector &outGlobalData); + const Byte *GetDataPointer(UInt32 offset) const { return Mem + offset; } +}; + +#endif + +}}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar5Decoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar5Decoder.h new file mode 100644 index 0000000..66c1c3c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/Rar5Decoder.h @@ -0,0 +1,132 @@ +// Rar5Decoder.h +// According to unRAR license, this code may not be used to develop +// a program that creates RAR archives + +#ifndef ZIP7_INC_COMPRESS_RAR5_DECODER_H +#define ZIP7_INC_COMPRESS_RAR5_DECODER_H + +#include "../../Common/MyBuffer2.h" +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "HuffmanDecoder.h" + +namespace NCompress { +namespace NRar5 { + +class CBitDecoder; + +struct CFilter +{ + Byte Type; + Byte Channels; + UInt32 Size; + UInt64 Start; +}; + + +const unsigned kNumReps = 4; +const unsigned kLenTableSize = 11 * 4; +const unsigned kMainTableSize = 256 + 1 + 1 + kNumReps + kLenTableSize; +const unsigned kExtraDistSymbols_v7 = 16; +const unsigned kDistTableSize_v6 = 64; +const unsigned kDistTableSize_MAX = 64 + kExtraDistSymbols_v7; +const unsigned kNumAlignBits = 4; +const unsigned kAlignTableSize = 1 << kNumAlignBits; + +const unsigned kNumHufBits = 15; + +const unsigned k_NumHufTableBits_Main = 10; +const unsigned k_NumHufTableBits_Dist = 7; +const unsigned k_NumHufTableBits_Len = 7; +const unsigned k_NumHufTableBits_Align = 6; + +const unsigned DICT_SIZE_BITS_MAX = 40; + +Z7_CLASS_IMP_NOQIB_2( + CDecoder + , ICompressCoder + , ICompressSetDecoderProperties2 +) + bool _useAlignBits; + bool _isLastBlock; + bool _unpackSize_Defined; + // bool _packSize_Defined; + + bool _unsupportedFilter; + Byte _lzError; + bool _writeError; + + bool _isSolid; + // bool _solidAllowed; + bool _is_v7; + bool _tableWasFilled; + bool _wasInit; + + Byte _exitType; + + // Byte _dictSizeLog; + size_t _dictSize; + Byte *_window; + size_t _winPos; + size_t _winSize; + size_t _dictSize_forCheck; + size_t _limit; + const Byte *_buf_Res; + UInt64 _lzSize; + size_t _reps[kNumReps]; + unsigned _bitPos_Res; + UInt32 _lastLen; + + // unsigned _numCorrectDistSymbols; + unsigned _numUnusedFilters; + unsigned _numFilters; + + UInt64 _lzWritten; + UInt64 _lzFileStart; + UInt64 _unpackSize; + // UInt64 _packSize; + UInt64 _lzEnd; + UInt64 _writtenFileSize; + UInt64 _filterEnd; + UInt64 _progress_Pack; + UInt64 _progress_Unpack; + CAlignedBuffer _filterSrc; + CAlignedBuffer _filterDst; + + CFilter *_filters; + size_t _winSize_Allocated; + ISequentialInStream *_inStream; + ISequentialOutStream *_outStream; + ICompressProgressInfo *_progress; + Byte *_inputBuf; + + NHuffman::CDecoder m_MainDecoder; + NHuffman::CDecoder256 m_DistDecoder; + NHuffman::CDecoder256 m_AlignDecoder; + NHuffman::CDecoder256 m_LenDecoder; + Byte m_LenPlusTable[DICT_SIZE_BITS_MAX]; + + void InitFilters() + { + _numUnusedFilters = 0; + _numFilters = 0; + } + void DeleteUnusedFilters(); + HRESULT WriteData(const Byte *data, size_t size); + HRESULT ExecuteFilter(const CFilter &f); + HRESULT WriteBuf(); + HRESULT AddFilter(CBitDecoder &_bitStream); + HRESULT ReadTables(CBitDecoder &_bitStream); + HRESULT DecodeLZ2(const CBitDecoder &_bitStream) throw(); + HRESULT DecodeLZ(); + HRESULT CodeReal(); +public: + CDecoder(); + ~CDecoder(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ShrinkDecoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ShrinkDecoder.cpp new file mode 100644 index 0000000..c8e2083 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ShrinkDecoder.cpp @@ -0,0 +1,244 @@ +// ShrinkDecoder.cpp + +#include "StdAfx.h" + +#include "../../../C/Alloc.h" + +#include "../Common/InBuffer.h" +#include "../Common/OutBuffer.h" + +#include "BitlDecoder.h" +#include "ShrinkDecoder.h" + +namespace NCompress { +namespace NShrink { + +static const UInt32 kEmpty = 256; // kNumItems; +static const UInt32 kBufferSize = (1 << 18); +static const unsigned kNumMinBits = 9; + +HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + NBitl::CBaseDecoder inBuffer; + COutBuffer outBuffer; + + if (!inBuffer.Create(kBufferSize)) + return E_OUTOFMEMORY; + if (!outBuffer.Create(kBufferSize)) + return E_OUTOFMEMORY; + + inBuffer.SetStream(inStream); + inBuffer.Init(); + + outBuffer.SetStream(outStream); + outBuffer.Init(); + + { + for (unsigned i = 0; i < kNumItems; i++) + _parents[i] = kEmpty; + } + + UInt64 outPrev = 0, inPrev = 0; + unsigned numBits = kNumMinBits; + unsigned head = 257; + int lastSym = -1; + Byte lastChar = 0; + bool moreOut = false; + + HRESULT res = S_FALSE; + + for (;;) + { + _inProcessed = inBuffer.GetProcessedSize(); + const UInt64 nowPos = outBuffer.GetProcessedSize(); + + bool eofCheck = false; + + if (outSize && nowPos >= *outSize) + { + if (!_fullStreamMode || moreOut) + { + res = S_OK; + break; + } + eofCheck = true; + // Is specSym(=256) allowed after end of stream ? + // Do we need to read it here ? + } + + if (progress) + { + if (nowPos - outPrev >= (1 << 20) || _inProcessed - inPrev >= (1 << 20)) + { + outPrev = nowPos; + inPrev = _inProcessed; + res = progress->SetRatioInfo(&_inProcessed, &nowPos); + if (res != SZ_OK) + { + // break; + return res; + } + } + } + + UInt32 sym = inBuffer.ReadBits(numBits); + + if (inBuffer.ExtraBitsWereRead()) + { + res = S_OK; + break; + } + + if (sym == 256) + { + sym = inBuffer.ReadBits(numBits); + + if (inBuffer.ExtraBitsWereRead()) + break; + + if (sym == 1) + { + if (numBits >= kNumMaxBits) + break; + numBits++; + continue; + } + if (sym != 2) + { + break; + // continue; // info-zip just ignores such code + } + { + /* + ---------- Free leaf nodes ---------- + Note : that code can mark _parents[lastSym] as free, and next + inserted node will be Orphan in that case. + */ + + unsigned i; + for (i = 256; i < kNumItems; i++) + _stack[i] = 0; + for (i = 257; i < kNumItems; i++) + { + unsigned par = _parents[i]; + if (par != kEmpty) + _stack[par] = 1; + } + for (i = 257; i < kNumItems; i++) + if (_stack[i] == 0) + _parents[i] = kEmpty; + head = 257; + continue; + } + } + + if (eofCheck) + { + // It's can be error case. + // That error can be detected later in (*inSize != _inProcessed) check. + res = S_OK; + break; + } + + bool needPrev = false; + if (head < kNumItems && lastSym >= 0) + { + while (head < kNumItems && _parents[head] != kEmpty) + head++; + if (head < kNumItems) + { + /* + if (head == lastSym), it updates Orphan to self-linked Orphan and creates two problems: + 1) we must check _stack[i++] overflow in code that walks tree nodes. + 2) self-linked node can not be removed. So such self-linked nodes can occupy all _parents items. + */ + needPrev = true; + _parents[head] = (UInt16)lastSym; + _suffixes[head] = (Byte)lastChar; + head++; + } + } + + lastSym = (int)sym; + unsigned cur = sym; + unsigned i = 0; + + while (cur >= 256) + { + _stack[i++] = _suffixes[cur]; + cur = _parents[cur]; + // don't change that code: + // Orphan Check and self-linked Orphan check (_stack overflow check); + if (cur == kEmpty || i >= kNumItems) + break; + } + + if (cur == kEmpty || i >= kNumItems) + break; + + _stack[i++] = (Byte)cur; + lastChar = (Byte)cur; + + if (needPrev) + _suffixes[(size_t)head - 1] = (Byte)cur; + + if (outSize) + { + const UInt64 limit = *outSize - nowPos; + if (i > limit) + { + moreOut = true; + i = (unsigned)limit; + } + } + + do + outBuffer.WriteByte(_stack[--i]); + while (i); + } + + RINOK(outBuffer.Flush()) + + if (res == S_OK) + if (_fullStreamMode) + { + if (moreOut) + res = S_FALSE; + const UInt64 nowPos = outBuffer.GetProcessedSize(); + if (outSize && *outSize != nowPos) + res = S_FALSE; + if (inSize && *inSize != _inProcessed) + res = S_FALSE; + } + + return res; +} + + +Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + try { return CodeReal(inStream, outStream, inSize, outSize, progress); } + // catch(const CInBufferException &e) { return e.ErrorCode; } + // catch(const COutBufferException &e) { return e.ErrorCode; } + catch(const CSystemException &e) { return e.ErrorCode; } + catch(...) { return S_FALSE; } +} + + +Z7_COM7F_IMF(CDecoder::SetFinishMode(UInt32 finishMode)) +{ + _fullStreamMode = (finishMode != 0); + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::GetInStreamProcessedSize(UInt64 *value)) +{ + *value = _inProcessed; + return S_OK; +} + + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ShrinkDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ShrinkDecoder.h new file mode 100644 index 0000000..5e7f78f --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ShrinkDecoder.h @@ -0,0 +1,35 @@ +// ShrinkDecoder.h + +#ifndef ZIP7_INC_COMPRESS_SHRINK_DECODER_H +#define ZIP7_INC_COMPRESS_SHRINK_DECODER_H + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +namespace NCompress { +namespace NShrink { + +const unsigned kNumMaxBits = 13; +const unsigned kNumItems = 1 << kNumMaxBits; + +Z7_CLASS_IMP_NOQIB_3( + CDecoder + , ICompressCoder + , ICompressSetFinishMode + , ICompressGetInStreamProcessedSize +) + bool _fullStreamMode; + UInt64 _inProcessed; + + UInt16 _parents[kNumItems]; + Byte _suffixes[kNumItems]; + Byte _stack[kNumItems]; + + HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/XpressDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/XpressDecoder.h new file mode 100644 index 0000000..7d17602 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/XpressDecoder.h @@ -0,0 +1,18 @@ +// XpressDecoder.h + +#ifndef ZIP7_INC_XPRESS_DECODER_H +#define ZIP7_INC_XPRESS_DECODER_H + +#include "../../Common/MyTypes.h" + +namespace NCompress { +namespace NXpress { + +// (out) buffer size must be larger than (outSize) for the following value: +const unsigned kAdditionalOutputBufSize = 32; + +HRESULT Decode_WithExceedWrite(const Byte *in, size_t inSize, Byte *out, size_t outSize); + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZDecoder.h new file mode 100644 index 0000000..fd67f8f --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZDecoder.h @@ -0,0 +1,47 @@ +// ZDecoder.h + +#ifndef ZIP7_INC_COMPRESS_Z_DECODER_H +#define ZIP7_INC_COMPRESS_Z_DECODER_H + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +namespace NCompress { +namespace NZ { + +// Z decoder decodes Z data stream, including 3 bytes of header. + +class CDecoder +{ + UInt16 *_parents; + Byte *_suffixes; + Byte *_stack; + unsigned _numMaxBits; + +public: + CDecoder(): _parents(NULL), _suffixes(NULL), _stack(NULL), /* _prop(0), */ _numMaxBits(0) {} + ~CDecoder(); + void Free(); + // UInt64 PackSize; + + HRESULT Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + ICompressProgressInfo *progress); +}; + +/* + There is no end_of_payload_marker in Z stream. + Z decoder stops decoding, if it reaches end of input stream. + + CheckStream function: + (size) must be at least 3 bytes (size of Z header). + if (size) is larger than size of real Z stream in (data), CheckStream can return false. +*/ + +const unsigned kRecommendedCheckSize = 64; + +bool CheckStream(const Byte *data, size_t size); + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibDecoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibDecoder.cpp new file mode 100644 index 0000000..dc01894 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibDecoder.cpp @@ -0,0 +1,136 @@ +// ZlibDecoder.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" + +#include "../Common/StreamUtils.h" + +#include "ZlibDecoder.h" + +namespace NCompress { +namespace NZlib { + +#define DEFLATE_TRY_BEGIN try { +#define DEFLATE_TRY_END } catch(...) { return S_FALSE; } + +#define ADLER_MOD 65521 +#define ADLER_LOOP_MAX 5550 + +UInt32 Adler32_Update(UInt32 adler, const Byte *data, size_t size); +UInt32 Adler32_Update(UInt32 adler, const Byte *data, size_t size) +{ + if (size == 0) + return adler; + UInt32 a = adler & 0xffff; + UInt32 b = adler >> 16; + do + { + size_t cur = size; + if (cur > ADLER_LOOP_MAX) + cur = ADLER_LOOP_MAX; + size -= cur; + const Byte *lim = data + cur; + if (cur >= 4) + { + lim -= 4 - 1; + do + { + a += data[0]; b += a; + a += data[1]; b += a; + a += data[2]; b += a; + a += data[3]; b += a; + data += 4; + } + while (data < lim); + lim += 4 - 1; + } + if (data != lim) { a += *data++; b += a; + if (data != lim) { a += *data++; b += a; + if (data != lim) { a += *data++; b += a; }}} + a %= ADLER_MOD; + b %= ADLER_MOD; + } + while (size); + return (b << 16) + a; +} + +Z7_COM7F_IMF(COutStreamWithAdler::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + HRESULT result = S_OK; + if (_stream) + result = _stream->Write(data, size, &size); + _adler = Adler32_Update(_adler, (const Byte *)data, size); + _size += size; + if (processedSize) + *processedSize = size; + return result; +} + +Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + DEFLATE_TRY_BEGIN + _inputProcessedSize_Additional = 0; + AdlerStream.Create_if_Empty(); + DeflateDecoder.Create_if_Empty(); + DeflateDecoder->Set_NeedFinishInput(true); + + if (inSize && *inSize < 2) + return S_FALSE; + { + Byte buf[2]; + RINOK(ReadStream_FALSE(inStream, buf, 2)) + if (!IsZlib(buf)) + return S_FALSE; + } + _inputProcessedSize_Additional = 2; + AdlerStream->SetStream(outStream); + AdlerStream->Init(); + // NDeflate::NDecoder::Code() ignores inSize + /* + UInt64 inSize2 = 0; + if (inSize) + inSize2 = *inSize - 2; + */ + const HRESULT res = DeflateDecoder.Interface()->Code(inStream, AdlerStream, + /* inSize ? &inSize2 : */ NULL, outSize, progress); + AdlerStream->ReleaseStream(); + + if (res == S_OK) + { + UInt32 footer32[1]; + UInt32 processedSize; + RINOK(DeflateDecoder->ReadUnusedFromInBuf(footer32, 4, &processedSize)) + if (processedSize != 4) + { + size_t processedSize2 = 4 - processedSize; + RINOK(ReadStream(inStream, (Byte *)(void *)footer32 + processedSize, &processedSize2)) + _inputProcessedSize_Additional += (Int32)processedSize2; + processedSize += (UInt32)processedSize2; + } + + if (processedSize == 4) + { + const UInt32 adler = GetBe32a(footer32); + if (adler != AdlerStream->GetAdler()) + return S_FALSE; // adler error + } + else if (!IsAdlerOptional) + return S_FALSE; // unexpeced end of stream (can't read adler) + else + { + // IsAdlerOptional == true + if (processedSize != 0) + { + // we exclude adler bytes from processed size: + _inputProcessedSize_Additional -= (Int32)processedSize; + return S_FALSE; + } + } + } + return res; + DEFLATE_TRY_END +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibDecoder.h new file mode 100644 index 0000000..5488653 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibDecoder.h @@ -0,0 +1,78 @@ +// ZlibDecoder.h + +#ifndef ZIP7_INC_ZLIB_DECODER_H +#define ZIP7_INC_ZLIB_DECODER_H + +#include "DeflateDecoder.h" + +namespace NCompress { +namespace NZlib { + +const UInt32 ADLER_INIT_VAL = 1; + +Z7_CLASS_IMP_NOQIB_1( + COutStreamWithAdler + , ISequentialOutStream +) + UInt32 _adler; + CMyComPtr _stream; + UInt64 _size; +public: + void SetStream(ISequentialOutStream *stream) { _stream = stream; } + void ReleaseStream() { _stream.Release(); } + void Init() { _adler = ADLER_INIT_VAL; _size = 0; } + UInt32 GetAdler() const { return _adler; } + UInt64 GetSize() const { return _size; } +}; + +Z7_CLASS_IMP_NOQIB_1( + CDecoder + , ICompressCoder +) + CMyComPtr2 AdlerStream; + CMyComPtr2 DeflateDecoder; + Int32 _inputProcessedSize_Additional; +public: + bool IsAdlerOptional; + + CDecoder(): IsAdlerOptional(false) {} + UInt64 GetInputProcessedSize() const + { + return (UInt64)( + (Int64)DeflateDecoder->GetInputProcessedSize() + + (Int64)_inputProcessedSize_Additional); + } + UInt64 GetOutputProcessedSize() const { return AdlerStream->GetSize(); } +}; + +static bool inline IsZlib(const Byte *p) +{ + if ((p[0] & 0xF) != 8) // method + return false; + if (((unsigned)p[0] >> 4) > 7) // logar_window_size minus 8. + return false; + if ((p[1] & 0x20) != 0) // dictPresent + return false; + if ((((UInt32)p[0] << 8) + p[1]) % 31 != 0) + return false; + return true; +} + +// IsZlib_3bytes checks 2 bytes of zlib header and starting byte of Deflate stream + +static bool inline IsZlib_3bytes(const Byte *p) +{ + if (!IsZlib(p)) + return false; + const unsigned val = p[2]; + const unsigned blockType = (val >> 1) & 0x3; + if (blockType == 3) // unsupported block type for deflate + return false; + if (blockType == NCompress::NDeflate::NBlockType::kStored && (val >> 3) != 0) + return false; + return true; +} + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibEncoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibEncoder.cpp new file mode 100644 index 0000000..3670d30 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibEncoder.cpp @@ -0,0 +1,61 @@ +// ZlibEncoder.cpp + +#include "StdAfx.h" + +#include "../Common/StreamUtils.h" + +#include "ZlibEncoder.h" + +namespace NCompress { +namespace NZlib { + +#define DEFLATE_TRY_BEGIN try { +#define DEFLATE_TRY_END } catch(...) { return S_FALSE; } + +UInt32 Adler32_Update(UInt32 adler, const Byte *buf, size_t size); + +Z7_COM7F_IMF(CInStreamWithAdler::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + const HRESULT result = _stream->Read(data, size, &size); + _adler = Adler32_Update(_adler, (const Byte *)data, size); + _size += size; + if (processedSize) + *processedSize = size; + return result; +} + +void CEncoder::Create() +{ + if (!DeflateEncoder) + DeflateEncoder = DeflateEncoderSpec = new NDeflate::NEncoder::CCOMCoder; +} + +Z7_COM7F_IMF(CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 * /* outSize */, ICompressProgressInfo *progress)) +{ + DEFLATE_TRY_BEGIN + if (!AdlerStream) + AdlerStream = AdlerSpec = new CInStreamWithAdler; + Create(); + + { + Byte buf[2] = { 0x78, 0xDA }; + RINOK(WriteStream(outStream, buf, 2)) + } + + AdlerSpec->SetStream(inStream); + AdlerSpec->Init(); + const HRESULT res = DeflateEncoder->Code(AdlerStream, outStream, inSize, NULL, progress); + AdlerSpec->ReleaseStream(); + + RINOK(res) + + { + const UInt32 a = AdlerSpec->GetAdler(); + const Byte buf[4] = { (Byte)(a >> 24), (Byte)(a >> 16), (Byte)(a >> 8), (Byte)(a) }; + return WriteStream(outStream, buf, 4); + } + DEFLATE_TRY_END +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibEncoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibEncoder.h new file mode 100644 index 0000000..484a307 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZlibEncoder.h @@ -0,0 +1,42 @@ +// ZlibEncoder.h + +#ifndef ZIP7_INC_ZLIB_ENCODER_H +#define ZIP7_INC_ZLIB_ENCODER_H + +#include "DeflateEncoder.h" + +namespace NCompress { +namespace NZlib { + +Z7_CLASS_IMP_NOQIB_1( + CInStreamWithAdler + , ISequentialInStream +) + CMyComPtr _stream; + UInt32 _adler; + UInt64 _size; +public: + void SetStream(ISequentialInStream *stream) { _stream = stream; } + void ReleaseStream() { _stream.Release(); } + void Init() { _adler = 1; _size = 0; } // ADLER_INIT_VAL + UInt32 GetAdler() const { return _adler; } + UInt64 GetSize() const { return _size; } +}; + +Z7_CLASS_IMP_NOQIB_1( + CEncoder + , ICompressCoder +) + CInStreamWithAdler *AdlerSpec; + CMyComPtr AdlerStream; + CMyComPtr DeflateEncoder; +public: + NCompress::NDeflate::NEncoder::CCOMCoder *DeflateEncoderSpec; + + void Create(); + UInt64 GetInputProcessedSize() const { return AdlerSpec->GetSize(); } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZstdDecoder.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZstdDecoder.cpp new file mode 100644 index 0000000..c958475 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZstdDecoder.cpp @@ -0,0 +1,437 @@ +// ZstdDecoder.cpp + +#include "StdAfx.h" + +// #include + +#include "../../../C/Alloc.h" + +#include "../Common/CWrappers.h" +#include "../Common/StreamUtils.h" + +#include "ZstdDecoder.h" + +namespace NCompress { +namespace NZstd { + +static const size_t k_Zstd_BlockSizeMax = 1 << 17; +/* + we set _outStepMask as (k_Zstd_BlockSizeMax - 1), because: + - cycSize in zstd decoder for isCyclicMode is aligned for (1 << 17) only. + So some write sizes will be multiple of ((1 << 17) * n). + - Also it can be optimal to flush data after each block decoding. +*/ + +CDecoder::CDecoder(): + _outStepMask(k_Zstd_BlockSizeMax - 1) // must be = (1 << x) - 1 + , _dec(NULL) + , _inProcessed(0) + , _inBufSize(1u << 19) // larger value will reduce the number of memcpy() calls in CZstdDec code + , _inBuf(NULL) + , FinishMode(false) + , DisableHash(False) + // , DisableHash(True) // for debug : fast decoding without hash calculation +{ + // ZstdDecInfo_Clear(&ResInfo); +} + +CDecoder::~CDecoder() +{ + if (_dec) + ZstdDec_Destroy(_dec); + MidFree(_inBuf); +} + + +Z7_COM7F_IMF(CDecoder::SetInBufSize(UInt32 , UInt32 size)) + { _inBufSize = size; return S_OK; } +Z7_COM7F_IMF(CDecoder::SetOutBufSize(UInt32 , UInt32 size)) +{ + // we round it down: + size >>= 1; + size |= size >> (1 << 0); + size |= size >> (1 << 1); + size |= size >> (1 << 2); + size |= size >> (1 << 3); + size |= size >> (1 << 4); + _outStepMask = size; // it's (1 << x) - 1 now + return S_OK; +} + +Z7_COM7F_IMF(CDecoder::SetDecoderProperties2(const Byte * /* prop */, UInt32 /* size */)) +{ + // if (size != 3 && size != 5) return E_NOTIMPL; + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::SetFinishMode(UInt32 finishMode)) +{ + FinishMode = (finishMode != 0); + // FinishMode = false; // for debug + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::ReadUnusedFromInBuf(void *data, UInt32 size, UInt32 *processedSize)) +{ + size_t cur = ZstdDec_ReadUnusedFromInBuf(_dec, _afterDecoding_tempPos, data, size); + _afterDecoding_tempPos += cur; + size -= (UInt32)cur; + if (size) + { + const size_t rem = _state.inLim - _state.inPos; + if (size > rem) + size = (UInt32)rem; + if (size) + { + memcpy((Byte *)data + cur, _state.inBuf + _state.inPos, size); + _state.inPos += size; + cur += size; + } + } + *processedSize = (UInt32)cur; + return S_OK; +} + + + +HRESULT CDecoder::Prepare(const UInt64 *outSize) +{ + _inProcessed = 0; + _afterDecoding_tempPos = 0; + ZstdDecState_Clear(&_state); + ZstdDecInfo_CLEAR(&ResInfo) + // _state.outStep = _outStepMask + 1; // must be = (1 << x) + _state.disableHash = DisableHash; + if (outSize) + { + _state.outSize_Defined = True; + _state.outSize = *outSize; + // _state.outSize = 0; // for debug + } + if (!_dec) + { + _dec = ZstdDec_Create(&g_AlignedAlloc, &g_BigAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } + if (!_inBuf || _inBufSize != _inBufSize_Allocated) + { + MidFree(_inBuf); + _inBuf = NULL; + _inBufSize_Allocated = 0; + _inBuf = (Byte *)MidAlloc(_inBufSize); + if (!_inBuf) + return E_OUTOFMEMORY; + _inBufSize_Allocated = _inBufSize; + } + _state.inBuf = _inBuf; + ZstdDec_Init(_dec); + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + RINOK(Prepare(outSize)) + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + UInt64 writtenSize = 0; + bool readWasFinished = false; + SRes sres = SZ_OK; + HRESULT hres = S_OK; + HRESULT hres_Read = S_OK; + + for (;;) + { + if (_state.inPos == _state.inLim && !readWasFinished) + { + _state.inPos = 0; + _state.inLim = _inBufSize; + hres_Read = ReadStream(inStream, _inBuf, &_state.inLim); + // _state.inLim -= 5; readWasFinished = True; // for debug + if (_state.inLim != _inBufSize || hres_Read != S_OK) + { + // hres_Read = 99; // for debug + readWasFinished = True; + } + } + { + const size_t inPos_Start = _state.inPos; + sres = ZstdDec_Decode(_dec, &_state); + _inProcessed += _state.inPos - inPos_Start; + } + /* + if (_state.status == ZSTD_STATUS_FINISHED_FRAME) + printf("\nfinished frame pos=%8x, checksum=%08x\n", (unsigned)_state.outProcessed, (unsigned)_state.info.checksum); + */ + const bool needStop = (sres != SZ_OK) + || _state.status == ZSTD_STATUS_OUT_REACHED + || (outSize && *outSize < _state.outProcessed) + || (readWasFinished && _state.inPos == _state.inLim + && ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(&_state)); + + size_t size = _state.winPos - _state.wrPos; // full write size + if (size) + { + if (!needStop) + { + // we try to flush on aligned positions, if possible + size = _state.needWrite_Size; // minimal required write size + const size_t alignedPos = _state.winPos & ~(size_t)_outStepMask; + if (alignedPos > _state.wrPos) + { + const size_t size2 = alignedPos - _state.wrPos; // optimized aligned size + if (size < size2) + size = size2; + } + } + if (size) + { + { + size_t curSize = size; + if (outSize) + { + const UInt64 rem = *outSize - writtenSize; + if (curSize > rem) + curSize = (size_t)rem; + } + if (curSize) + { + // printf("Write wrPos=%8x, size=%8x\n", (unsigned)_state.wrPos, (unsigned)size); + hres = WriteStream(outStream, _state.win + _state.wrPos, curSize); + if (hres != S_OK) + break; + writtenSize += curSize; // it's real size of data that was written to stream + } + } + _state.wrPos += size; // virtual written size, that will be reported to CZstdDec + // _state.needWrite_Size = 0; // optional + } + } + + if (needStop) + break; + if (progress) + if (_inProcessed - inPrev >= (1 << 27) + || _state.outProcessed - outPrev >= (1 << 28)) + { + inPrev = _inProcessed; + outPrev = _state.outProcessed; + RINOK(progress->SetRatioInfo(&inPrev, &outPrev)) + } + } + + if (hres == S_OK) + { + ZstdDec_GetResInfo(_dec, &_state, sres, &ResInfo); + sres = ResInfo.decode_SRes; + /* now (ResInfo.decode_SRes) can contain 2 extra error codes: + - SZ_ERROR_NO_ARCHIVE : if no frames + - SZ_ERROR_INPUT_EOF : if ZSTD_STATUS_NEEDS_MORE_INPUT + */ + _inProcessed -= ResInfo.extraSize; + if (hres_Read != S_OK && _state.inLim == _state.inPos && readWasFinished) + { + /* if (there is stream reading error, + and decoding was stopped because of end of input stream), + then we use reading error as main error code */ + if (sres == SZ_OK || + sres == SZ_ERROR_INPUT_EOF || + sres == SZ_ERROR_NO_ARCHIVE) + hres = hres_Read; + } + if (sres == SZ_ERROR_INPUT_EOF && !FinishMode) + { + /* SZ_ERROR_INPUT_EOF case is allowed case for (!FinishMode) mode. + So we restore SZ_OK result for that case: */ + ResInfo.decode_SRes = sres = SZ_OK; + } + if (hres == S_OK) + { + hres = SResToHRESULT(sres); + if (hres == S_OK && FinishMode) + { + if ((inSize && *inSize != _inProcessed) + || ResInfo.is_NonFinishedFrame + || (outSize && (*outSize != writtenSize || writtenSize != _state.outProcessed))) + hres = S_FALSE; + } + } + } + return hres; +} + + +Z7_COM7F_IMF(CDecoder::GetInStreamProcessedSize(UInt64 *value)) +{ + *value = _inProcessed; + return S_OK; +} + + +#ifndef Z7_NO_READ_FROM_CODER_ZSTD + +Z7_COM7F_IMF(CDecoder::SetOutStreamSize(const UInt64 *outSize)) +{ + _inProcessed = 0; + _hres_Read = S_OK; + _hres_Decode = S_OK; + _writtenSize = 0; + _readWasFinished = false; + _wasFinished = false; + return Prepare(outSize); +} + + +Z7_COM7F_IMF(CDecoder::SetInStream(ISequentialInStream *inStream)) + { _inStream = inStream; return S_OK; } +Z7_COM7F_IMF(CDecoder::ReleaseInStream()) + { _inStream.Release(); return S_OK; } + + +// if SetInStream() mode: the caller must call GetFinishResult() after full decoding +// to check that there decoding was finished correctly + +HRESULT CDecoder::GetFinishResult() +{ + if (_state.winPos != _state.wrPos || !_wasFinished) + return FinishMode ? S_FALSE : S_OK; + // _state.winPos == _state.wrPos + // _wasFinished == true + if (FinishMode && _hres_Decode == S_OK && _state.outSize_Defined && _state.outSize != _writtenSize) + _hres_Decode = S_FALSE; + return _hres_Decode; +} + + +Z7_COM7F_IMF(CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + if (processedSize) + *processedSize = 0; + + for (;;) + { + if (_state.outSize_Defined) + { + // _writtenSize <= _state.outSize + const UInt64 rem = _state.outSize - _writtenSize; + if (size > rem) + size = (UInt32)rem; + } + { + size_t cur = _state.winPos - _state.wrPos; + if (cur) + { + // _state.winPos != _state.wrPos; + // so there is some decoded data that was not written still + if (size == 0) + { + // if (FinishMode) and we are not allowed to write more, then it's data error + if (FinishMode && _state.outSize_Defined && _state.outSize == _writtenSize) + return S_FALSE; + return S_OK; + } + if (cur > size) + cur = (size_t)size; + // cur != 0 + memcpy(data, _state.win + _state.wrPos, cur); + _state.wrPos += cur; + _writtenSize += cur; + data = (void *)((Byte *)data + cur); + if (processedSize) + *processedSize += (UInt32)cur; + size -= (UInt32)cur; + continue; + } + } + + // _state.winPos == _state.wrPos + if (_wasFinished) + { + if (_hres_Decode == S_OK && FinishMode + && _state.outSize_Defined && _state.outSize != _writtenSize) + _hres_Decode = S_FALSE; + return _hres_Decode; + } + + // _wasFinished == false + if (size == 0 && _state.outSize_Defined && _state.outSize != _state.outProcessed) + { + /* size == 0 : so the caller don't need more data now. + _state.outSize > _state.outProcessed : so more data will be requested + later by caller for full processing. + So we exit without ZstdDec_Decode() call, because we don't want + ZstdDec_Decode() to start new block decoding + */ + return S_OK; + } + // size != 0 || !_state.outSize_Defined || _state.outSize == _state.outProcessed) + + if (_state.inPos == _state.inLim && !_readWasFinished) + { + _state.inPos = 0; + _state.inLim = _inBufSize; + _hres_Read = ReadStream(_inStream, _inBuf, &_state.inLim); + if (_state.inLim != _inBufSize || _hres_Read != S_OK) + { + // _hres_Read = 99; // for debug + _readWasFinished = True; + } + } + + SRes sres; + { + const SizeT inPos_Start = _state.inPos; + sres = ZstdDec_Decode(_dec, &_state); + _inProcessed += _state.inPos - inPos_Start; + } + + const bool inFinished = (_state.inPos == _state.inLim) && _readWasFinished; + + _wasFinished = (sres != SZ_OK) + || _state.status == ZSTD_STATUS_OUT_REACHED + || (_state.outSize_Defined && _state.outSize < _state.outProcessed) + || (inFinished + && ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(&_state)); + + if (!_wasFinished) + continue; + + // _wasFinished == true + /* (_state.winPos != _state.wrPos) is possible here. + So we still can have some data to flush, + but we must all result variables . + */ + HRESULT hres = S_OK; + ZstdDec_GetResInfo(_dec, &_state, sres, &ResInfo); + sres = ResInfo.decode_SRes; + _inProcessed -= ResInfo.extraSize; + if (_hres_Read != S_OK && inFinished) + { + if (sres == SZ_OK || + sres == SZ_ERROR_INPUT_EOF || + sres == SZ_ERROR_NO_ARCHIVE) + hres = _hres_Read; + } + if (sres == SZ_ERROR_INPUT_EOF && !FinishMode) + ResInfo.decode_SRes = sres = SZ_OK; + if (hres == S_OK) + { + hres = SResToHRESULT(sres); + if (hres == S_OK && FinishMode) + if (!inFinished + || ResInfo.is_NonFinishedFrame + || (_state.outSize_Defined && _state.outSize != _state.outProcessed)) + hres = S_FALSE; + } + _hres_Decode = hres; + } +} + +#endif + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZstdDecoder.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZstdDecoder.h new file mode 100644 index 0000000..6e15a96 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Compress/ZstdDecoder.h @@ -0,0 +1,98 @@ +// ZstdDecoder.h + +#ifndef ZIP7_INC_ZSTD_DECODER_H +#define ZIP7_INC_ZSTD_DECODER_H + +#include "../../../C/ZstdDec.h" + +#include "../../Common/MyCom.h" +#include "../ICoder.h" + +namespace NCompress { +namespace NZstd { + +#ifdef Z7_NO_READ_FROM_CODER +#define Z7_NO_READ_FROM_CODER_ZSTD +#endif + +#ifndef Z7_NO_READ_FROM_CODER_ZSTD +// #define Z7_NO_READ_FROM_CODER_ZSTD +#endif + +class CDecoder Z7_final: + public ICompressCoder, + public ICompressSetDecoderProperties2, + public ICompressSetFinishMode, + public ICompressGetInStreamProcessedSize, + public ICompressReadUnusedFromInBuf, + public ICompressSetBufSize, + #ifndef Z7_NO_READ_FROM_CODER_ZSTD + public ICompressSetInStream, + public ICompressSetOutStreamSize, + public ISequentialInStream, + #endif + public CMyUnknownImp +{ + Z7_COM_QI_BEGIN2(ICompressCoder) + Z7_COM_QI_ENTRY(ICompressSetDecoderProperties2) + Z7_COM_QI_ENTRY(ICompressSetFinishMode) + Z7_COM_QI_ENTRY(ICompressGetInStreamProcessedSize) + Z7_COM_QI_ENTRY(ICompressReadUnusedFromInBuf) + Z7_COM_QI_ENTRY(ICompressSetBufSize) + #ifndef Z7_NO_READ_FROM_CODER_ZSTD + Z7_COM_QI_ENTRY(ICompressSetInStream) + Z7_COM_QI_ENTRY(ICompressSetOutStreamSize) + Z7_COM_QI_ENTRY(ISequentialInStream) + #endif + Z7_COM_QI_END + Z7_COM_ADDREF_RELEASE + + Z7_IFACE_COM7_IMP(ICompressCoder) + Z7_IFACE_COM7_IMP(ICompressSetDecoderProperties2) + Z7_IFACE_COM7_IMP(ICompressSetFinishMode) + Z7_IFACE_COM7_IMP(ICompressGetInStreamProcessedSize) + Z7_IFACE_COM7_IMP(ICompressReadUnusedFromInBuf) + Z7_IFACE_COM7_IMP(ICompressSetBufSize) + #ifndef Z7_NO_READ_FROM_CODER_ZSTD + Z7_IFACE_COM7_IMP(ICompressSetOutStreamSize) + Z7_IFACE_COM7_IMP(ICompressSetInStream) + Z7_IFACE_COM7_IMP(ISequentialInStream) + #endif + + HRESULT Prepare(const UInt64 *outSize); + + UInt32 _outStepMask; + CZstdDecHandle _dec; +public: + UInt64 _inProcessed; + CZstdDecState _state; + +private: + UInt32 _inBufSize; + UInt32 _inBufSize_Allocated; + Byte *_inBuf; + size_t _afterDecoding_tempPos; + + #ifndef Z7_NO_READ_FROM_CODER_ZSTD + CMyComPtr _inStream; + HRESULT _hres_Read; + HRESULT _hres_Decode; + UInt64 _writtenSize; + bool _readWasFinished; + bool _wasFinished; + #endif + +public: + bool FinishMode; + Byte DisableHash; + CZstdDecResInfo ResInfo; + + HRESULT GetFinishResult(); + + CDecoder(); + ~CDecoder(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha1.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha1.cpp new file mode 100644 index 0000000..30b1a8f --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha1.cpp @@ -0,0 +1,94 @@ +// HmacSha1.cpp + +#include "StdAfx.h" + +#include + +#include "../../../C/CpuArch.h" + +#include "HmacSha1.h" + +namespace NCrypto { +namespace NSha1 { + +void CHmac::SetKey(const Byte *key, size_t keySize) +{ + MY_ALIGN (16) + UInt32 temp[SHA1_NUM_BLOCK_WORDS]; + size_t i; + + for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) + temp[i] = 0; + + if (keySize > kBlockSize) + { + _sha.Init(); + _sha.Update(key, keySize); + _sha.Final((Byte *)temp); + } + else + memcpy(temp, key, keySize); + + for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) + temp[i] ^= 0x36363636; + + _sha.Init(); + _sha.Update((const Byte *)temp, kBlockSize); + + for (i = 0; i < SHA1_NUM_BLOCK_WORDS; i++) + temp[i] ^= 0x36363636 ^ 0x5C5C5C5C; + + _sha2.Init(); + _sha2.Update((const Byte *)temp, kBlockSize); +} + + +void CHmac::Final(Byte *mac) +{ + _sha.Final(mac); + _sha2.Update(mac, kDigestSize); + _sha2.Final(mac); +} + + +void CHmac::GetLoopXorDigest1(void *mac, UInt32 numIteration) +{ + MY_ALIGN (16) UInt32 block [SHA1_NUM_BLOCK_WORDS]; + MY_ALIGN (16) UInt32 block2[SHA1_NUM_BLOCK_WORDS]; + MY_ALIGN (16) UInt32 mac2 [SHA1_NUM_BLOCK_WORDS]; + + _sha. PrepareBlock((Byte *)block, SHA1_DIGEST_SIZE); + _sha2.PrepareBlock((Byte *)block2, SHA1_DIGEST_SIZE); + + block[0] = ((const UInt32 *)mac)[0]; + block[1] = ((const UInt32 *)mac)[1]; + block[2] = ((const UInt32 *)mac)[2]; + block[3] = ((const UInt32 *)mac)[3]; + block[4] = ((const UInt32 *)mac)[4]; + + mac2[0] = block[0]; + mac2[1] = block[1]; + mac2[2] = block[2]; + mac2[3] = block[3]; + mac2[4] = block[4]; + + for (UInt32 i = 0; i < numIteration; i++) + { + _sha. GetBlockDigest((const Byte *)block, (Byte *)block2); + _sha2.GetBlockDigest((const Byte *)block2, (Byte *)block); + + mac2[0] ^= block[0]; + mac2[1] ^= block[1]; + mac2[2] ^= block[2]; + mac2[3] ^= block[3]; + mac2[4] ^= block[4]; + } + + ((UInt32 *)mac)[0] = mac2[0]; + ((UInt32 *)mac)[1] = mac2[1]; + ((UInt32 *)mac)[2] = mac2[2]; + ((UInt32 *)mac)[3] = mac2[3]; + ((UInt32 *)mac)[4] = mac2[4]; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha1.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha1.h new file mode 100644 index 0000000..a36c78e --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha1.h @@ -0,0 +1,31 @@ +// HmacSha1.h +// Implements HMAC-SHA-1 (RFC2104, FIPS-198) + +#ifndef ZIP7_INC_CRYPTO_HMAC_SHA1_H +#define ZIP7_INC_CRYPTO_HMAC_SHA1_H + +#include "Sha1Cls.h" + +namespace NCrypto { +namespace NSha1 { + +// Use: SetKey(key, keySize); for () Update(data, size); FinalFull(mac); + +class CHmac +{ + CContext _sha; + CContext _sha2; +public: + void SetKey(const Byte *key, size_t keySize); + void Update(const Byte *data, size_t dataSize) { _sha.Update(data, dataSize); } + + // Final() : mac is recommended to be aligned for 4 bytes + // GetLoopXorDigest1() : mac is required to be aligned for 4 bytes + // The caller can use: UInt32 mac[NSha1::kNumDigestWords] and typecast to (Byte *) and (void *); + void Final(Byte *mac); + void GetLoopXorDigest1(void *mac, UInt32 numIteration); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha256.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha256.cpp new file mode 100644 index 0000000..56f8ede --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha256.cpp @@ -0,0 +1,53 @@ +// HmacSha256.cpp + +#include "StdAfx.h" + +#include + +#include "../../../C/CpuArch.h" + +#include "HmacSha256.h" + +namespace NCrypto { +namespace NSha256 { + +void CHmac::SetKey(const Byte *key, size_t keySize) +{ + MY_ALIGN (16) + UInt32 temp[SHA256_NUM_BLOCK_WORDS]; + size_t i; + + for (i = 0; i < SHA256_NUM_BLOCK_WORDS; i++) + temp[i] = 0; + + if (keySize > kBlockSize) + { + Sha256_Init(&_sha); + Sha256_Update(&_sha, key, keySize); + Sha256_Final(&_sha, (Byte *)temp); + } + else + memcpy(temp, key, keySize); + + for (i = 0; i < SHA256_NUM_BLOCK_WORDS; i++) + temp[i] ^= 0x36363636; + + Sha256_Init(&_sha); + Sha256_Update(&_sha, (const Byte *)temp, kBlockSize); + + for (i = 0; i < SHA256_NUM_BLOCK_WORDS; i++) + temp[i] ^= 0x36363636 ^ 0x5C5C5C5C; + + Sha256_Init(&_sha2); + Sha256_Update(&_sha2, (const Byte *)temp, kBlockSize); +} + + +void CHmac::Final(Byte *mac) +{ + Sha256_Final(&_sha, mac); + Sha256_Update(&_sha2, mac, SHA256_DIGEST_SIZE); + Sha256_Final(&_sha2, mac); +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha256.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha256.h new file mode 100644 index 0000000..4039c0c --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/HmacSha256.h @@ -0,0 +1,27 @@ +// HmacSha256.h +// Implements HMAC-SHA-256 (RFC2104, FIPS-198) + +#ifndef ZIP7_INC_CRYPTO_HMAC_SHA256_H +#define ZIP7_INC_CRYPTO_HMAC_SHA256_H + +#include "../../../C/Sha256.h" + +namespace NCrypto { +namespace NSha256 { + +const unsigned kBlockSize = SHA256_BLOCK_SIZE; +const unsigned kDigestSize = SHA256_DIGEST_SIZE; + +class CHmac +{ + CSha256 _sha; + CSha256 _sha2; +public: + void SetKey(const Byte *key, size_t keySize); + void Update(const Byte *data, size_t dataSize) { Sha256_Update(&_sha, data, dataSize); } + void Final(Byte *mac); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Pbkdf2HmacSha1.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Pbkdf2HmacSha1.cpp new file mode 100644 index 0000000..3d9e4c1 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Pbkdf2HmacSha1.cpp @@ -0,0 +1,48 @@ +// Pbkdf2HmacSha1.cpp + +#include "StdAfx.h" + +#include + +#include "../../../C/CpuArch.h" + +#include "HmacSha1.h" +#include "Pbkdf2HmacSha1.h" + +namespace NCrypto { +namespace NSha1 { + +void Pbkdf2Hmac(const Byte *pwd, size_t pwdSize, + const Byte *salt, size_t saltSize, + UInt32 numIterations, + Byte *key, size_t keySize) +{ + MY_ALIGN (16) + CHmac baseCtx; + baseCtx.SetKey(pwd, pwdSize); + + for (UInt32 i = 1; keySize != 0; i++) + { + MY_ALIGN (16) + CHmac ctx; + ctx = baseCtx; + ctx.Update(salt, saltSize); + + MY_ALIGN (16) + UInt32 u[kNumDigestWords]; + SetBe32(u, i) + + ctx.Update((const Byte *)u, 4); + ctx.Final((Byte *)u); + + ctx = baseCtx; + ctx.GetLoopXorDigest1((void *)u, numIterations - 1); + + const unsigned curSize = (keySize < kDigestSize) ? (unsigned)keySize : kDigestSize; + memcpy(key, (const Byte *)u, curSize); + key += curSize; + keySize -= curSize; + } +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Pbkdf2HmacSha1.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Pbkdf2HmacSha1.h new file mode 100644 index 0000000..89cbe41 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Pbkdf2HmacSha1.h @@ -0,0 +1,19 @@ +// Pbkdf2HmacSha1.h +// Password-Based Key Derivation Function (RFC 2898, PKCS #5) based on HMAC-SHA-1 + +#ifndef ZIP7_INC_CRYPTO_PBKDF2_HMAC_SHA1_H +#define ZIP7_INC_CRYPTO_PBKDF2_HMAC_SHA1_H + +#include + +#include "../../Common/MyTypes.h" + +namespace NCrypto { +namespace NSha1 { + +void Pbkdf2Hmac(const Byte *pwd, size_t pwdSize, const Byte *salt, size_t saltSize, + UInt32 numIterations, Byte *key, size_t keySize); + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Sha1Cls.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Sha1Cls.h new file mode 100644 index 0000000..56ae040 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/Sha1Cls.h @@ -0,0 +1,37 @@ +// Crypto/Sha1Cls.h + +#ifndef ZIP7_INC_CRYPTO_SHA1_CLS_H +#define ZIP7_INC_CRYPTO_SHA1_CLS_H + +#include "../../../C/Sha1.h" + +namespace NCrypto { +namespace NSha1 { + +const unsigned kNumBlockWords = SHA1_NUM_BLOCK_WORDS; +const unsigned kNumDigestWords = SHA1_NUM_DIGEST_WORDS; + +const unsigned kBlockSize = SHA1_BLOCK_SIZE; +const unsigned kDigestSize = SHA1_DIGEST_SIZE; + +class CContext +{ + CSha1 _s; + +public: + void Init() throw() { Sha1_Init(&_s); } + void Update(const Byte *data, size_t size) throw() { Sha1_Update(&_s, data, size); } + void Final(Byte *digest) throw() { Sha1_Final(&_s, digest); } + void PrepareBlock(Byte *block, unsigned size) const throw() + { + Sha1_PrepareBlock(&_s, block, size); + } + void GetBlockDigest(const Byte *blockData, Byte *destDigest) const throw() + { + Sha1_GetBlockDigest(&_s, blockData, destDigest); + } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/WzAes.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/WzAes.cpp new file mode 100644 index 0000000..3f2adab --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/WzAes.cpp @@ -0,0 +1,231 @@ +// Crypto/WzAes.cpp +/* +This code implements Brian Gladman's scheme +specified in "A Password Based File Encryption Utility". + +Note: you must include MyAes.cpp to project to initialize AES tables +*/ + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" + +#include "../Common/StreamUtils.h" + +#include "Pbkdf2HmacSha1.h" +#include "RandGen.h" +#include "WzAes.h" + +namespace NCrypto { +namespace NWzAes { + +const unsigned kAesKeySizeMax = 32; + +static const UInt32 kNumKeyGenIterations = 1000; + +Z7_COM7F_IMF(CBaseCoder::CryptoSetPassword(const Byte *data, UInt32 size)) +{ + if (size > kPasswordSizeMax) + return E_INVALIDARG; + _key.Password.Wipe(); + _key.Password.CopyFrom(data, (size_t)size); + return S_OK; +} + +void CBaseCoder::Init2() +{ + _hmacOverCalc = 0; + const unsigned dkSizeMax32 = (2 * kAesKeySizeMax + kPwdVerifSize + 3) / 4; + Byte dk[dkSizeMax32 * 4]; + + const unsigned keySize = _key.GetKeySize(); + const unsigned dkSize = 2 * keySize + ((kPwdVerifSize + 3) & ~(unsigned)3); + + // for (unsigned ii = 0; ii < 1000; ii++) + { + NSha1::Pbkdf2Hmac( + _key.Password, _key.Password.Size(), + _key.Salt, _key.GetSaltSize(), + kNumKeyGenIterations, + dk, dkSize); + } + + Hmac()->SetKey(dk + keySize, keySize); + memcpy(_key.PwdVerifComputed, dk + 2 * keySize, kPwdVerifSize); + + // Aes_SetKey_Enc(_aes.Aes() + 8, dk, keySize); + // AesCtr2_Init(&_aes); + _aesCoderSpec->SetKeySize(keySize); + if (_aesCoderSpec->SetKey(dk, keySize) != S_OK) throw 2; + if (_aesCoderSpec->Init() != S_OK) throw 3; +} + +Z7_COM7F_IMF(CBaseCoder::Init()) +{ + return S_OK; +} + +HRESULT CEncoder::WriteHeader(ISequentialOutStream *outStream) +{ + unsigned saltSize = _key.GetSaltSize(); + MY_RAND_GEN(_key.Salt, saltSize); + Init2(); + RINOK(WriteStream(outStream, _key.Salt, saltSize)) + return WriteStream(outStream, _key.PwdVerifComputed, kPwdVerifSize); +} + +HRESULT CEncoder::WriteFooter(ISequentialOutStream *outStream) +{ + MY_ALIGN (16) + UInt32 mac[NSha1::kNumDigestWords]; + Hmac()->Final((Byte *)mac); + return WriteStream(outStream, mac, kMacSize); +} + +/* +Z7_COM7F_IMF(CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size)) +{ + if (size != 1) + return E_INVALIDARG; + _key.Init(); + return SetKeyMode(data[0]) ? S_OK : E_INVALIDARG; +} +*/ + +HRESULT CDecoder::ReadHeader(ISequentialInStream *inStream) +{ + const unsigned saltSize = _key.GetSaltSize(); + const unsigned extraSize = saltSize + kPwdVerifSize; + Byte temp[kSaltSizeMax + kPwdVerifSize]; + RINOK(ReadStream_FAIL(inStream, temp, extraSize)) + unsigned i; + for (i = 0; i < saltSize; i++) + _key.Salt[i] = temp[i]; + for (i = 0; i < kPwdVerifSize; i++) + _pwdVerifFromArchive[i] = temp[saltSize + i]; + return S_OK; +} + +static inline bool CompareArrays(const Byte *p1, const Byte *p2, unsigned size) +{ + for (unsigned i = 0; i < size; i++) + if (p1[i] != p2[i]) + return false; + return true; +} + +bool CDecoder::Init_and_CheckPassword() +{ + Init2(); + return CompareArrays(_key.PwdVerifComputed, _pwdVerifFromArchive, kPwdVerifSize); +} + +HRESULT CDecoder::CheckMac(ISequentialInStream *inStream, bool &isOK) +{ + isOK = false; + MY_ALIGN (16) + Byte mac1[kMacSize]; + RINOK(ReadStream_FAIL(inStream, mac1, kMacSize)) + MY_ALIGN (16) + UInt32 mac2[NSha1::kNumDigestWords]; + Hmac()->Final((Byte *)mac2); + isOK = CompareArrays(mac1, (const Byte *)mac2, kMacSize); + if (_hmacOverCalc) + isOK = false; + return S_OK; +} + +/* + +CAesCtr2::CAesCtr2(): + aes((4 + AES_NUM_IVMRK_WORDS) * 4) +{ + // offset = ((0 - (unsigned)(ptrdiff_t)aes) & 0xF) / sizeof(UInt32); + // first 16 bytes are buffer for last block data. + // so the ivAES is aligned for (Align + 16). +} + +void AesCtr2_Init(CAesCtr2 *p) +{ + UInt32 *ctr = p->Aes() + 4; + unsigned i; + for (i = 0; i < 4; i++) + ctr[i] = 0; + p->pos = AES_BLOCK_SIZE; +} + +// (size != 16 * N) is allowed only for last call + +void AesCtr2_Code(CAesCtr2 *p, Byte *data, SizeT size) +{ + unsigned pos = p->pos; + UInt32 *buf32 = p->Aes(); + if (size == 0) + return; + + if (pos != AES_BLOCK_SIZE) + { + const Byte *buf = (const Byte *)buf32; + do + *data++ ^= buf[pos++]; + while (--size != 0 && pos != AES_BLOCK_SIZE); + } + + // (size == 0 || pos == AES_BLOCK_SIZE) + + if (size >= 16) + { + SizeT size2 = size >> 4; + g_AesCtr_Code(buf32 + 4, data, size2); + size2 <<= 4; + data += size2; + size -= size2; + // (pos == AES_BLOCK_SIZE) + } + + // (size < 16) + + if (size != 0) + { + unsigned j; + const Byte *buf; + for (j = 0; j < 4; j++) + buf32[j] = 0; + g_AesCtr_Code(buf32 + 4, (Byte *)buf32, 1); + buf = (const Byte *)buf32; + pos = 0; + do + *data++ ^= buf[pos++]; + while (--size != 0); + } + + p->pos = pos; +} +*/ + +/* (size != 16 * N) is allowed only for last Filter() call */ + +Z7_COM7F_IMF2(UInt32, CEncoder::Filter(Byte *data, UInt32 size)) +{ + // AesCtr2_Code(&_aes, data, size); + size = _aesCoder->Filter(data, size); + Hmac()->Update(data, size); + return size; +} + +Z7_COM7F_IMF2(UInt32, CDecoder::Filter(Byte *data, UInt32 size)) +{ + if (size >= 16) + size &= ~(UInt32)15; + if (_hmacOverCalc < size) + { + Hmac()->Update(data + _hmacOverCalc, size - _hmacOverCalc); + _hmacOverCalc = size; + } + // AesCtr2_Code(&_aes, data, size); + size = _aesCoder->Filter(data, size); + _hmacOverCalc -= size; + return size; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/WzAes.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/WzAes.h new file mode 100644 index 0000000..113ec81 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/WzAes.h @@ -0,0 +1,161 @@ +// Crypto/WzAes.h +/* +This code implements Brian Gladman's scheme +specified in "A Password Based File Encryption Utility": + - AES encryption (128,192,256-bit) in Counter (CTR) mode. + - HMAC-SHA1 authentication for encrypted data (10 bytes) + - Keys are derived by PPKDF2(RFC2898)-HMAC-SHA1 from ASCII password and + Salt (saltSize = aesKeySize / 2). + - 2 bytes contain Password Verifier's Code +*/ + +#ifndef ZIP7_INC_CRYPTO_WZ_AES_H +#define ZIP7_INC_CRYPTO_WZ_AES_H + +#include "../../Common/MyBuffer.h" + +#include "../IPassword.h" + +#include "HmacSha1.h" +#include "MyAes.h" + +namespace NCrypto { +namespace NWzAes { + +/* ICompressFilter::Init() does nothing for this filter. + + Call to init: + Encoder: + CryptoSetPassword(); + WriteHeader(); + Decoder: + [CryptoSetPassword();] + ReadHeader(); + [CryptoSetPassword();] Init_and_CheckPassword(); + [CryptoSetPassword();] Init_and_CheckPassword(); +*/ + +const UInt32 kPasswordSizeMax = 99; // 128; + +const unsigned kSaltSizeMax = 16; +const unsigned kPwdVerifSize = 2; +const unsigned kMacSize = 10; + +enum EKeySizeMode +{ + kKeySizeMode_AES128 = 1, + kKeySizeMode_AES192 = 2, + kKeySizeMode_AES256 = 3 +}; + +struct CKeyInfo +{ + EKeySizeMode KeySizeMode; + Byte Salt[kSaltSizeMax]; + Byte PwdVerifComputed[kPwdVerifSize]; + + CByteBuffer Password; + + unsigned GetKeySize() const { return (8 * KeySizeMode + 8); } + unsigned GetSaltSize() const { return (4 * KeySizeMode + 4); } + unsigned GetNumSaltWords() const { return (KeySizeMode + 1); } + + CKeyInfo(): KeySizeMode(kKeySizeMode_AES256) {} + + void Wipe() + { + Password.Wipe(); + Z7_memset_0_ARRAY(Salt); + Z7_memset_0_ARRAY(PwdVerifComputed); + } + + ~CKeyInfo() { Wipe(); } +}; + +/* +struct CAesCtr2 +{ + unsigned pos; + CAlignedBuffer aes; + UInt32 *Aes() { return (UInt32 *)(Byte *)aes; } + + // unsigned offset; + // UInt32 aes[4 + AES_NUM_IVMRK_WORDS + 3]; + // UInt32 *Aes() { return aes + offset; } + CAesCtr2(); +}; + +void AesCtr2_Init(CAesCtr2 *p); +void AesCtr2_Code(CAesCtr2 *p, Byte *data, SizeT size); +*/ + +class CBaseCoder: + public ICompressFilter, + public ICryptoSetPassword, + public CMyUnknownImp +{ + Z7_COM_UNKNOWN_IMP_1(ICryptoSetPassword) + Z7_COM7F_IMP(Init()) +public: + Z7_IFACE_COM7_IMP(ICryptoSetPassword) +protected: + CKeyInfo _key; + + // NSha1::CHmac _hmac; + // NSha1::CHmac *Hmac() { return &_hmac; } + CAlignedBuffer1 _hmacBuf; + UInt32 _hmacOverCalc; + + NSha1::CHmac *Hmac() { return (NSha1::CHmac *)(void *)(Byte *)_hmacBuf; } + + // CAesCtr2 _aes; + CAesCoder *_aesCoderSpec; + CMyComPtr _aesCoder; + CBaseCoder(): + _hmacBuf(sizeof(NSha1::CHmac)) + { + _aesCoderSpec = new CAesCtrCoder(32); + _aesCoder = _aesCoderSpec; + } + + void Init2(); +public: + unsigned GetHeaderSize() const { return _key.GetSaltSize() + kPwdVerifSize; } + unsigned GetAddPackSize() const { return GetHeaderSize() + kMacSize; } + + bool SetKeyMode(unsigned mode) + { + if (mode < kKeySizeMode_AES128 || mode > kKeySizeMode_AES256) + return false; + _key.KeySizeMode = (EKeySizeMode)mode; + return true; + } + + virtual ~CBaseCoder() {} +}; + +class CEncoder Z7_final: + public CBaseCoder +{ + Z7_COM7F_IMP2(UInt32, Filter(Byte *data, UInt32 size)) +public: + HRESULT WriteHeader(ISequentialOutStream *outStream); + HRESULT WriteFooter(ISequentialOutStream *outStream); +}; + +class CDecoder Z7_final: + public CBaseCoder + // public ICompressSetDecoderProperties2 +{ + Byte _pwdVerifFromArchive[kPwdVerifSize]; + Z7_COM7F_IMP2(UInt32, Filter(Byte *data, UInt32 size)) +public: + // Z7_IFACE_COM7_IMP(ICompressSetDecoderProperties2) + HRESULT ReadHeader(ISequentialInStream *inStream); + bool Init_and_CheckPassword(); + HRESULT CheckMac(ISequentialInStream *inStream, bool &isOK); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipCrypto.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipCrypto.cpp new file mode 100644 index 0000000..047e2bd --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipCrypto.cpp @@ -0,0 +1,114 @@ +// Crypto/ZipCrypto.cpp + +#include "StdAfx.h" + +#include "../../../C/7zCrc.h" + +#include "../Common/StreamUtils.h" + +#include "RandGen.h" +#include "ZipCrypto.h" + +namespace NCrypto { +namespace NZip { + +#define UPDATE_KEYS(b) { \ + key0 = CRC_UPDATE_BYTE(key0, b); \ + key1 = (key1 + (key0 & 0xFF)) * 0x8088405 + 1; \ + key2 = CRC_UPDATE_BYTE(key2, (Byte)(key1 >> 24)); } \ + +#define DECRYPT_BYTE_1 UInt32 temp = key2 | 2; +#define DECRYPT_BYTE_2 ((Byte)((temp * (temp ^ 1)) >> 8)) + +Z7_COM7F_IMF(CCipher::CryptoSetPassword(const Byte *data, UInt32 size)) +{ + UInt32 key0 = 0x12345678; + UInt32 key1 = 0x23456789; + UInt32 key2 = 0x34567890; + + for (UInt32 i = 0; i < size; i++) + UPDATE_KEYS(data[i]) + + KeyMem0 = key0; + KeyMem1 = key1; + KeyMem2 = key2; + + return S_OK; +} + +Z7_COM7F_IMF(CCipher::Init()) +{ + return S_OK; +} + +HRESULT CEncoder::WriteHeader_Check16(ISequentialOutStream *outStream, UInt16 crc) +{ + Byte h[kHeaderSize]; + + /* PKZIP before 2.0 used 2 byte CRC check. + PKZIP 2.0+ used 1 byte CRC check. It's more secure. + We also use 1 byte CRC. */ + + MY_RAND_GEN(h, kHeaderSize - 1); + // h[kHeaderSize - 2] = (Byte)(crc); + h[kHeaderSize - 1] = (Byte)(crc >> 8); + + RestoreKeys(); + Filter(h, kHeaderSize); + return WriteStream(outStream, h, kHeaderSize); +} + +Z7_COM7F_IMF2(UInt32, CEncoder::Filter(Byte *data, UInt32 size)) +{ + UInt32 key0 = this->Key0; + UInt32 key1 = this->Key1; + UInt32 key2 = this->Key2; + + for (UInt32 i = 0; i < size; i++) + { + Byte b = data[i]; + DECRYPT_BYTE_1 + data[i] = (Byte)(b ^ DECRYPT_BYTE_2); + UPDATE_KEYS(b) + } + + this->Key0 = key0; + this->Key1 = key1; + this->Key2 = key2; + + return size; +} + +HRESULT CDecoder::ReadHeader(ISequentialInStream *inStream) +{ + return ReadStream_FAIL(inStream, _header, kHeaderSize); +} + +void CDecoder::Init_BeforeDecode() +{ + RestoreKeys(); + Filter(_header, kHeaderSize); +} + +Z7_COM7F_IMF2(UInt32, CDecoder::Filter(Byte *data, UInt32 size)) +{ + UInt32 key0 = this->Key0; + UInt32 key1 = this->Key1; + UInt32 key2 = this->Key2; + + for (UInt32 i = 0; i < size; i++) + { + DECRYPT_BYTE_1 + Byte b = (Byte)(data[i] ^ DECRYPT_BYTE_2); + UPDATE_KEYS(b) + data[i] = b; + } + + this->Key0 = key0; + this->Key1 = key1; + this->Key2 = key2; + + return size; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipCrypto.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipCrypto.h new file mode 100644 index 0000000..485470e --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipCrypto.h @@ -0,0 +1,80 @@ +// Crypto/ZipCrypto.h + +#ifndef ZIP7_INC_CRYPTO_ZIP_CRYPTO_H +#define ZIP7_INC_CRYPTO_ZIP_CRYPTO_H + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" +#include "../IPassword.h" + +namespace NCrypto { +namespace NZip { + +const unsigned kHeaderSize = 12; + +/* ICompressFilter::Init() does nothing for this filter. + Call to init: + Encoder: + CryptoSetPassword(); + WriteHeader(); + Decoder: + [CryptoSetPassword();] + ReadHeader(); + [CryptoSetPassword();] Init_and_GetCrcByte(); + [CryptoSetPassword();] Init_and_GetCrcByte(); +*/ + +class CCipher: + public ICompressFilter, + public ICryptoSetPassword, + public CMyUnknownImp +{ + Z7_COM_UNKNOWN_IMP_1(ICryptoSetPassword) + Z7_COM7F_IMP(Init()) +public: + Z7_IFACE_COM7_IMP(ICryptoSetPassword) +protected: + UInt32 Key0; + UInt32 Key1; + UInt32 Key2; + + UInt32 KeyMem0; + UInt32 KeyMem1; + UInt32 KeyMem2; + + void RestoreKeys() + { + Key0 = KeyMem0; + Key1 = KeyMem1; + Key2 = KeyMem2; + } + +public: + virtual ~CCipher() + { + Key0 = KeyMem0 = + Key1 = KeyMem1 = + Key2 = KeyMem2 = 0; + } +}; + +class CEncoder Z7_final: public CCipher +{ + Z7_COM7F_IMP2(UInt32, Filter(Byte *data, UInt32 size)) +public: + HRESULT WriteHeader_Check16(ISequentialOutStream *outStream, UInt16 crc); +}; + +class CDecoder Z7_final: public CCipher +{ + Z7_COM7F_IMP2(UInt32, Filter(Byte *data, UInt32 size)) +public: + Byte _header[kHeaderSize]; + HRESULT ReadHeader(ISequentialInStream *inStream); + void Init_BeforeDecode(); +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipStrong.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipStrong.cpp new file mode 100644 index 0000000..c4e8311 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipStrong.cpp @@ -0,0 +1,265 @@ +// Crypto/ZipStrong.cpp + +#include "StdAfx.h" + +#include "../../../C/7zCrc.h" +#include "../../../C/CpuArch.h" + +#include "../Common/StreamUtils.h" + +#include "Sha1Cls.h" +#include "ZipStrong.h" + +namespace NCrypto { +namespace NZipStrong { + +static const UInt16 kAES128 = 0x660E; + +/* + DeriveKey() function is similar to CryptDeriveKey() from Windows. + New version of MSDN contains the following condition in CryptDeriveKey() description: + "If the hash is not a member of the SHA-2 family and the required key is for either 3DES or AES". + Now we support ZipStrong for AES only. And it uses SHA1. + Our DeriveKey() code is equal to CryptDeriveKey() in Windows for such conditions: (SHA1 + AES). + if (method != AES && method != 3DES), probably we need another code. +*/ + +static void DeriveKey2(const UInt32 *digest32, Byte c, UInt32 *dest32) +{ + const unsigned kBufSize = 64; + MY_ALIGN (16) + UInt32 buf32[kBufSize / 4]; + memset(buf32, c, kBufSize); + for (unsigned i = 0; i < NSha1::kNumDigestWords; i++) + buf32[i] ^= digest32[i]; + MY_ALIGN (16) + NSha1::CContext sha; + sha.Init(); + sha.Update((const Byte *)buf32, kBufSize); + sha.Final((Byte *)dest32); +} + +static void DeriveKey(NSha1::CContext &sha, Byte *key) +{ + MY_ALIGN (16) + UInt32 digest32[NSha1::kNumDigestWords]; + sha.Final((Byte *)digest32); + MY_ALIGN (16) + UInt32 temp32[NSha1::kNumDigestWords * 2]; + DeriveKey2(digest32, 0x36, temp32); + DeriveKey2(digest32, 0x5C, temp32 + NSha1::kNumDigestWords); + memcpy(key, temp32, 32); +} + +void CKeyInfo::SetPassword(const Byte *data, UInt32 size) +{ + MY_ALIGN (16) + NSha1::CContext sha; + sha.Init(); + sha.Update(data, size); + DeriveKey(sha, MasterKey); +} + + + +CDecoder::CDecoder() +{ + CAesCbcDecoder *d = new CAesCbcDecoder(); + _cbcDecoder = d; + _aesFilter = d; +} + +Z7_COM7F_IMF(CDecoder::CryptoSetPassword(const Byte *data, UInt32 size)) +{ + _key.SetPassword(data, size); + return S_OK; +} + +Z7_COM7F_IMF(CDecoder::Init()) +{ + return S_OK; +} + +Z7_COM7F_IMF2(UInt32, CDecoder::Filter(Byte *data, UInt32 size)) +{ + return _aesFilter->Filter(data, size); +} + + +HRESULT CDecoder::ReadHeader(ISequentialInStream *inStream, UInt32 crc, UInt64 unpackSize) +{ + Byte temp[4]; + RINOK(ReadStream_FALSE(inStream, temp, 2)) + _ivSize = GetUi16(temp); + if (_ivSize == 0) + { + memset(_iv, 0, 16); + SetUi32(_iv + 0, crc) + SetUi64(_iv + 4, unpackSize) + _ivSize = 12; + } + else if (_ivSize == 16) + { + RINOK(ReadStream_FALSE(inStream, _iv, _ivSize)) + } + else + return E_NOTIMPL; + RINOK(ReadStream_FALSE(inStream, temp, 4)) + _remSize = GetUi32(temp); + // const UInt32 kAlign = 16; + if (_remSize < 16 || _remSize > (1 << 18)) + return E_NOTIMPL; + if (_remSize > _bufAligned.Size()) + { + _bufAligned.AllocAtLeast(_remSize); + if (!(Byte *)_bufAligned) + return E_OUTOFMEMORY; + } + return ReadStream_FALSE(inStream, _bufAligned, _remSize); +} + +HRESULT CDecoder::Init_and_CheckPassword(bool &passwOK) +{ + passwOK = false; + if (_remSize < 16) + return E_NOTIMPL; + Byte * const p = _bufAligned; + const unsigned format = GetUi16a(p); + if (format != 3) + return E_NOTIMPL; + unsigned algId = GetUi16a(p + 2); + if (algId < kAES128) + return E_NOTIMPL; + algId -= kAES128; + if (algId > 2) + return E_NOTIMPL; + const unsigned bitLen = GetUi16a(p + 4); + const unsigned flags = GetUi16a(p + 6); + if (algId * 64 + 128 != bitLen) + return E_NOTIMPL; + _key.KeySize = 16 + algId * 8; + const bool cert = ((flags & 2) != 0); + + if (flags & 0x4000) + { + // Use 3DES for rd data + return E_NOTIMPL; + } + + if (cert) + { + return E_NOTIMPL; + } + else + { + if ((flags & 1) == 0) + return E_NOTIMPL; + } + + UInt32 rdSize = GetUi16a(p + 8); + + if (rdSize + 16 > _remSize) + return E_NOTIMPL; + + const unsigned kPadSize = kAesPadAllign; // is equal to blockSize of cipher for rd + + /* + if (cert) + { + if ((rdSize & 0x7) != 0) + return E_NOTIMPL; + } + else + */ + { + // PKCS7 padding + if (rdSize < kPadSize) + return E_NOTIMPL; + if (rdSize & (kPadSize - 1)) + return E_NOTIMPL; + } + + memmove(p, p + 10, rdSize); + const Byte *p2 = p + rdSize + 10; + UInt32 reserved = GetUi32(p2); + p2 += 4; + + /* + if (cert) + { + UInt32 numRecipients = reserved; + + if (numRecipients == 0) + return E_NOTIMPL; + + { + UInt32 hashAlg = GetUi16(p2); + hashAlg = hashAlg; + UInt32 hashSize = GetUi16(p2 + 2); + hashSize = hashSize; + p2 += 4; + + reserved = reserved; + // return E_NOTIMPL; + + for (unsigned r = 0; r < numRecipients; r++) + { + UInt32 specSize = GetUi16(p2); + p2 += 2; + p2 += specSize; + } + } + } + else + */ + { + if (reserved != 0) + return E_NOTIMPL; + } + + UInt32 validSize = GetUi16(p2); + p2 += 2; + const size_t validOffset = (size_t)(p2 - p); + if ((validSize & 0xF) != 0 || validOffset + validSize != _remSize) + return E_NOTIMPL; + + { + RINOK(_cbcDecoder->SetKey(_key.MasterKey, _key.KeySize)) + RINOK(_cbcDecoder->SetInitVector(_iv, 16)) + // SetInitVector() calls also Init() + RINOK(_cbcDecoder->Init()) // it's optional + Filter(p, rdSize); + + rdSize -= kPadSize; + for (unsigned i = 0; i < kPadSize; i++) + if (p[(size_t)rdSize + i] != kPadSize) + return S_OK; // passwOK = false; + } + + MY_ALIGN (16) + Byte fileKey[32]; + MY_ALIGN (16) + NSha1::CContext sha; + sha.Init(); + sha.Update(_iv, _ivSize); + sha.Update(p, rdSize); + DeriveKey(sha, fileKey); + + RINOK(_cbcDecoder->SetKey(fileKey, _key.KeySize)) + RINOK(_cbcDecoder->SetInitVector(_iv, 16)) + // SetInitVector() calls also Init() + RINOK(_cbcDecoder->Init()) // it's optional + + memmove(p, p + validOffset, validSize); + Filter(p, validSize); + + if (validSize < 4) + return E_NOTIMPL; + validSize -= 4; + if (GetUi32(p + validSize) != CrcCalc(p, validSize)) + return S_OK; + passwOK = true; + return S_OK; +} + +}} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipStrong.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipStrong.h new file mode 100644 index 0000000..a2b5cdd --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/7zip/Crypto/ZipStrong.h @@ -0,0 +1,73 @@ +// Crypto/ZipStrong.h + +#ifndef ZIP7_INC_CRYPTO_ZIP_STRONG_H +#define ZIP7_INC_CRYPTO_ZIP_STRONG_H + +#include "../../Common/MyBuffer2.h" + +#include "../IPassword.h" + +#include "MyAes.h" + +namespace NCrypto { +namespace NZipStrong { + +/* ICompressFilter::Init() does nothing for this filter. + Call to init: + Decoder: + [CryptoSetPassword();] + ReadHeader(); + [CryptoSetPassword();] Init_and_CheckPassword(); + [CryptoSetPassword();] Init_and_CheckPassword(); +*/ + +struct CKeyInfo +{ + Byte MasterKey[32]; + UInt32 KeySize; + + void SetPassword(const Byte *data, UInt32 size); + + void Wipe() + { + Z7_memset_0_ARRAY(MasterKey); + } +}; + + +const unsigned kAesPadAllign = AES_BLOCK_SIZE; + +Z7_CLASS_IMP_COM_2( + CDecoder + , ICompressFilter + , ICryptoSetPassword +) + CAesCbcDecoder *_cbcDecoder; + CMyComPtr _aesFilter; + CKeyInfo _key; + CAlignedBuffer _bufAligned; + + UInt32 _ivSize; + Byte _iv[16]; + UInt32 _remSize; +public: + HRESULT ReadHeader(ISequentialInStream *inStream, UInt32 crc, UInt64 unpackSize); + HRESULT Init_and_CheckPassword(bool &passwOK); + UInt32 GetPadSize(UInt32 packSize32) const + { + // Padding is to align to blockSize of cipher. + // Change it, if is not AES + return kAesPadAllign - (packSize32 & (kAesPadAllign - 1)); + } + CDecoder(); + ~CDecoder() { Wipe(); } + void Wipe() + { + Z7_memset_0_ARRAY(_iv); + _key.Wipe(); + } +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/MyMap.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/MyMap.h new file mode 100644 index 0000000..9ca5566 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/MyMap.h @@ -0,0 +1,28 @@ +// MyMap.h + +#ifndef ZIP7_INC_COMMON_MY_MAP_H +#define ZIP7_INC_COMMON_MY_MAP_H + +#include "MyTypes.h" +#include "MyVector.h" + +class CMap32 +{ + struct CNode + { + UInt32 Key; + UInt32 Keys[2]; + UInt32 Values[2]; + UInt16 Len; + Byte IsLeaf[2]; + }; + CRecordVector Nodes; + +public: + + void Clear() { Nodes.Clear(); } + bool Find(UInt32 key, UInt32 &valueRes) const throw(); + bool Set(UInt32 key, UInt32 value); // returns true, if there is such key already +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/MyXml.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/MyXml.h new file mode 100644 index 0000000..b22d7e4 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/MyXml.h @@ -0,0 +1,45 @@ +// MyXml.h + +#ifndef ZIP7_INC_MY_XML_H +#define ZIP7_INC_MY_XML_H + +#include "MyString.h" + +struct CXmlProp +{ + AString Name; + AString Value; +}; + +class CXmlItem +{ +public: + AString Name; + bool IsTag; + CObjectVector Props; + CObjectVector SubItems; + + const char * ParseItem(const char *s, int numAllowedLevels); + + bool IsTagged(const char *tag) const throw(); + int FindProp(const char *propName) const throw(); + AString GetPropVal(const char *propName) const; + AString GetSubString() const; + const AString * GetSubStringPtr() const throw(); + int FindSubTag(const char *tag) const throw(); + const CXmlItem *FindSubTag_GetPtr(const char *tag) const throw(); + AString GetSubStringForTag(const char *tag) const; + void AppendTo(AString &s) const; +}; + +struct CXml +{ + CXmlItem Root; + + bool Parse(const char *s); + // void AppendTo(AString &s) const; +}; + +void z7_xml_DecodeString(AString &s); + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/Random.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/Random.h new file mode 100644 index 0000000..3fbb416 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Common/Random.h @@ -0,0 +1,14 @@ +// Common/Random.h + +#ifndef ZIP7_INC_COMMON_RANDOM_H +#define ZIP7_INC_COMMON_RANDOM_H + +class CRandom +{ +public: + void Init(); + void Init(unsigned seed); + int Generate() const; +}; + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Console.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Console.h new file mode 100644 index 0000000..818b8d4 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Console.h @@ -0,0 +1,52 @@ +// Windows/Console.h + +#ifndef ZIP7_INC_WINDOWS_CONSOLE_H +#define ZIP7_INC_WINDOWS_CONSOLE_H + +#include "Defs.h" + +namespace NWindows { +namespace NConsole { + +class CBase +{ +protected: + HANDLE m_Object; +public: + void Attach(HANDLE handle) { m_Object = handle; } + bool GetMode(DWORD &mode) + { return BOOLToBool(::GetConsoleMode(m_Object, &mode)); } + bool SetMode(DWORD mode) + { return BOOLToBool(::SetConsoleMode(m_Object, mode)); } +}; + +class CIn: public CBase +{ +public: + bool PeekEvents(PINPUT_RECORD events, DWORD numEvents, DWORD &numEventsRead) + { return BOOLToBool(::PeekConsoleInput(m_Object, events, numEvents, &numEventsRead)); } + bool PeekEvent(INPUT_RECORD &event, DWORD &numEventsRead) + { return PeekEvents(&event, 1, numEventsRead); } + bool ReadEvents(PINPUT_RECORD events, DWORD numEvents, DWORD &numEventsRead) + { return BOOLToBool(::ReadConsoleInput(m_Object, events, numEvents, &numEventsRead)); } + bool ReadEvent(INPUT_RECORD &event, DWORD &numEventsRead) + { return ReadEvents(&event, 1, numEventsRead); } + bool GetNumberOfEvents(DWORD &numEvents) + { return BOOLToBool(::GetNumberOfConsoleInputEvents(m_Object, &numEvents)); } + + bool WriteEvents(const INPUT_RECORD *events, DWORD numEvents, DWORD &numEventsWritten) + { return BOOLToBool(::WriteConsoleInput(m_Object, events, numEvents, &numEventsWritten)); } + bool WriteEvent(const INPUT_RECORD &event, DWORD &numEventsWritten) + { return WriteEvents(&event, 1, numEventsWritten); } + + bool Read(LPVOID buffer, DWORD numChars, DWORD &numCharsRead) + { return BOOLToBool(::ReadConsole(m_Object, buffer, numChars, &numCharsRead, NULL)); } + + bool Flush() + { return BOOLToBool(::FlushConsoleInputBuffer(m_Object)); } + +}; + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Menu.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Menu.h new file mode 100644 index 0000000..e1de4c4 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Menu.h @@ -0,0 +1,170 @@ +// Windows/Menu.h + +#ifndef ZIP7_INC_WINDOWS_MENU_H +#define ZIP7_INC_WINDOWS_MENU_H + +#include "../Common/MyWindows.h" +#include "../Common/MyString.h" + +#include "Defs.h" + +namespace NWindows { + +#ifndef MIIM_STRING +#define MIIM_STRING 0x00000040 +#endif +/* +#ifndef MIIM_BITMAP +#define MIIM_BITMAP 0x00000080 +#endif +*/ +#ifndef MIIM_FTYPE +#define MIIM_FTYPE 0x00000100 +#endif + +struct CMenuItem +{ + UString StringValue; + UINT fMask; + UINT fType; + UINT fState; + UINT wID; + HMENU hSubMenu; + HBITMAP hbmpChecked; + HBITMAP hbmpUnchecked; + ULONG_PTR dwItemData; + // LPTSTR dwTypeData; + // UINT cch; + // HBITMAP hbmpItem; + bool IsString() const { return (fMask & (MIIM_TYPE | MIIM_STRING)) != 0; } + bool IsSeparator() const { return (fType == MFT_SEPARATOR); } + CMenuItem(): fMask(0), fType(0), fState(0), wID(0), + hSubMenu(NULL), hbmpChecked(NULL), hbmpUnchecked(NULL), dwItemData(0) {} +}; + +class CMenu +{ + HMENU _menu; +public: + CMenu(): _menu(NULL) {} + operator HMENU() const { return _menu; } + void Attach(HMENU menu) { _menu = menu; } + + HMENU Detach() + { + const HMENU menu = _menu; + _menu = NULL; + return menu; + } + + bool Create() + { + _menu = ::CreateMenu(); + return (_menu != NULL); + } + + bool CreatePopup() + { + _menu = ::CreatePopupMenu(); + return (_menu != NULL); + } + + bool Destroy() + { + if (!_menu) + return false; + return BOOLToBool(::DestroyMenu(Detach())); + } + + int GetItemCount() const + { + #ifdef UNDER_CE + for (unsigned i = 0;; i++) + { + CMenuItem item; + item.fMask = MIIM_STATE; + if (!GetItem(i, true, item)) + return (int)i; + } + #else + return GetMenuItemCount(_menu); + #endif + } + + HMENU GetSubMenu(int pos) const { return ::GetSubMenu(_menu, pos); } + #ifndef UNDER_CE + /* + bool GetItemString(UINT idItem, UINT flag, CSysString &result) + { + result.Empty(); + int len = ::GetMenuString(_menu, idItem, 0, 0, flag); + int len2 = ::GetMenuString(_menu, idItem, result.GetBuf(len + 2), len + 1, flag); + if (len > len2) + len = len2; + result.ReleaseBuf_CalcLen(len + 2); + return (len != 0); + } + */ + UINT GetItemID(int pos) const { return ::GetMenuItemID(_menu, pos); } + UINT GetItemState(UINT id, UINT flags) const { return ::GetMenuState(_menu, id, flags); } + #endif + + bool GetItemInfo(UINT itemIndex, bool byPosition, LPMENUITEMINFO itemInfo) const + { return BOOLToBool(::GetMenuItemInfo(_menu, itemIndex, BoolToBOOL(byPosition), itemInfo)); } + bool SetItemInfo(UINT itemIndex, bool byPosition, LPMENUITEMINFO itemInfo) + { return BOOLToBool(::SetMenuItemInfo(_menu, itemIndex, BoolToBOOL(byPosition), itemInfo)); } + + bool AppendItem(UINT flags, UINT_PTR newItemID, LPCTSTR newItem) + { return BOOLToBool(::AppendMenu(_menu, flags, newItemID, newItem)); } + + bool Insert(UINT position, UINT flags, UINT_PTR idNewItem, LPCTSTR newItem) + { return BOOLToBool(::InsertMenu(_menu, position, flags, idNewItem, newItem)); } + + #ifndef UNDER_CE + bool InsertItem(UINT itemIndex, bool byPosition, LPCMENUITEMINFO itemInfo) + { return BOOLToBool(::InsertMenuItem(_menu, itemIndex, BoolToBOOL(byPosition), itemInfo)); } + #endif + + bool RemoveItem(UINT item, UINT flags) { return BOOLToBool(::RemoveMenu(_menu, item, flags)); } + void RemoveAllItemsFrom(UINT index) { while (RemoveItem(index, MF_BYPOSITION)); } + void RemoveAllItems() { RemoveAllItemsFrom(0); } + + #ifndef _UNICODE + bool GetItemInfo(UINT itemIndex, bool byPosition, LPMENUITEMINFOW itemInfo) const + { return BOOLToBool(::GetMenuItemInfoW(_menu, itemIndex, BoolToBOOL(byPosition), itemInfo)); } + bool InsertItem(UINT itemIndex, bool byPosition, LPMENUITEMINFOW itemInfo) + { return BOOLToBool(::InsertMenuItemW(_menu, itemIndex, BoolToBOOL(byPosition), itemInfo)); } + bool SetItemInfo(UINT itemIndex, bool byPosition, LPMENUITEMINFOW itemInfo) + { return BOOLToBool(::SetMenuItemInfoW(_menu, itemIndex, BoolToBOOL(byPosition), itemInfo)); } + bool AppendItem(UINT flags, UINT_PTR newItemID, LPCWSTR newItem); + #endif + + bool GetItem(UINT itemIndex, bool byPosition, CMenuItem &item) const; + bool SetItem(UINT itemIndex, bool byPosition, const CMenuItem &item); + bool InsertItem(UINT itemIndex, bool byPosition, const CMenuItem &item); + + int Track(UINT flags, int x, int y, HWND hWnd) { return ::TrackPopupMenuEx(_menu, flags, x, y, hWnd, NULL); } + + bool CheckRadioItem(UINT idFirst, UINT idLast, UINT idCheck, UINT flags) + { return BOOLToBool(::CheckMenuRadioItem(_menu, idFirst, idLast, idCheck, flags)); } + + DWORD CheckItem(UINT id, UINT uCheck) { return ::CheckMenuItem(_menu, id, uCheck); } + DWORD CheckItemByID(UINT id, bool check) { return CheckItem(id, MF_BYCOMMAND | (check ? MF_CHECKED : MF_UNCHECKED)); } + + BOOL EnableItem(UINT uIDEnableItem, UINT uEnable) { return EnableMenuItem(_menu, uIDEnableItem, uEnable); } +}; + +class CMenuDestroyer +{ + CMenu *_menu; +public: + CMenuDestroyer(CMenu &menu): _menu(&menu) {} + CMenuDestroyer(): _menu(NULL) {} + ~CMenuDestroyer() { if (_menu) _menu->Destroy(); } + void Attach(CMenu &menu) { _menu = &menu; } + void Disable() { _menu = NULL; } +}; + +} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/NationalTime.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/NationalTime.h new file mode 100644 index 0000000..32ba479 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/NationalTime.h @@ -0,0 +1,20 @@ +// Windows/NationalTime.h + +#ifndef ZIP7_INC_WINDOWS_NATIONAL_TIME_H +#define ZIP7_INC_WINDOWS_NATIONAL_TIME_H + +#include "../Common/MyString.h" + +namespace NWindows { +namespace NNational { +namespace NTime { + +bool MyGetTimeFormat(LCID locale, DWORD flags, CONST SYSTEMTIME *time, + LPCTSTR format, CSysString &resultString); + +bool MyGetDateFormat(LCID locale, DWORD flags, CONST SYSTEMTIME *time, + LPCTSTR format, CSysString &resultString); + +}}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Net.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Net.h new file mode 100644 index 0000000..a954bdb --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/Net.h @@ -0,0 +1,87 @@ +// Windows/Net.h + +#ifndef ZIP7_INC_WINDOWS_NET_H +#define ZIP7_INC_WINDOWS_NET_H + +#include "../Common/MyString.h" +#include "../Common/MyWindows.h" + +namespace NWindows { +namespace NNet { + +struct CResourceBase +{ + DWORD Scope; + DWORD Type; + DWORD DisplayType; + DWORD Usage; + bool LocalNameIsDefined; + bool RemoteNameIsDefined; + bool CommentIsDefined; + bool ProviderIsDefined; +}; + +struct CResource: public CResourceBase +{ + CSysString LocalName; + CSysString RemoteName; + CSysString Comment; + CSysString Provider; +}; + +#ifdef _UNICODE +typedef CResource CResourceW; +#else +struct CResourceW: public CResourceBase +{ + UString LocalName; + UString RemoteName; + UString Comment; + UString Provider; +}; +#endif + +class CEnum +{ + HANDLE _handle; + bool _handleAllocated; + DWORD Open(DWORD scope, DWORD type, DWORD usage, LPNETRESOURCE netResource); + DWORD Next(LPDWORD lpcCount, LPVOID lpBuffer, LPDWORD lpBufferSize); + #ifndef _UNICODE + DWORD Open(DWORD scope, DWORD type, DWORD usage, LPNETRESOURCEW netResource); + DWORD NextW(LPDWORD lpcCount, LPVOID lpBuffer, LPDWORD lpBufferSize); + #endif +protected: + bool IsHandleAllocated() const { return _handleAllocated; } +public: + CEnum(): _handleAllocated(false) {} + ~CEnum() { Close(); } + DWORD Close(); + DWORD Open(DWORD scope, DWORD type, DWORD usage, const CResource *resource); + DWORD Next(CResource &resource); + #ifndef _UNICODE + DWORD Open(DWORD scope, DWORD type, DWORD usage, const CResourceW *resource); + DWORD Next(CResourceW &resource); + #endif +}; + +DWORD GetResourceParent(const CResource &resource, CResource &parentResource); +#ifndef _UNICODE +DWORD GetResourceParent(const CResourceW &resource, CResourceW &parentResource); +#endif + +DWORD GetResourceInformation(const CResource &resource, + CResource &destResource, CSysString &systemPathPart); +#ifndef _UNICODE +DWORD GetResourceInformation(const CResourceW &resource, + CResourceW &destResource, UString &systemPathPart); +#endif + +DWORD AddConnection2(const CResource &resource, LPCTSTR password, LPCTSTR userName, DWORD flags); +#ifndef _UNICODE +DWORD AddConnection2(const CResourceW &resource, LPCWSTR password, LPCWSTR userName, DWORD flags); +#endif + +}} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/ProcessMessages.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/ProcessMessages.h new file mode 100644 index 0000000..aa98bbb --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/ProcessMessages.h @@ -0,0 +1,12 @@ +// Windows/ProcessMessages.h + +#ifndef ZIP7_INC_WINDOWS_PROCESS_MESSAGES_H +#define ZIP7_INC_WINDOWS_PROCESS_MESSAGES_H + +namespace NWindows { + +void ProcessMessages(HWND window); + +} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/ProcessUtils.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/ProcessUtils.h new file mode 100644 index 0000000..b1fce3a --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/ProcessUtils.h @@ -0,0 +1,138 @@ +// Windows/ProcessUtils.h + +#ifndef ZIP7_INC_WINDOWS_PROCESS_UTILS_H +#define ZIP7_INC_WINDOWS_PROCESS_UTILS_H + +#include "../Common/MyWindows.h" + +#ifndef Z7_OLD_WIN_SDK + +#if defined(__MINGW32__) || defined(__MINGW64__) +#include +#else +#include +#endif + +#else // Z7_OLD_WIN_SDK + +typedef struct _MODULEINFO { + LPVOID lpBaseOfDll; + DWORD SizeOfImage; + LPVOID EntryPoint; +} MODULEINFO, *LPMODULEINFO; + +typedef struct _PROCESS_MEMORY_COUNTERS { + DWORD cb; + DWORD PageFaultCount; + SIZE_T PeakWorkingSetSize; + SIZE_T WorkingSetSize; + SIZE_T QuotaPeakPagedPoolUsage; + SIZE_T QuotaPagedPoolUsage; + SIZE_T QuotaPeakNonPagedPoolUsage; + SIZE_T QuotaNonPagedPoolUsage; + SIZE_T PagefileUsage; + SIZE_T PeakPagefileUsage; +} PROCESS_MEMORY_COUNTERS; +typedef PROCESS_MEMORY_COUNTERS *PPROCESS_MEMORY_COUNTERS; + +#endif // Z7_OLD_WIN_SDK + +#include "../Common/MyString.h" + +#include "Defs.h" +#include "Handle.h" + +namespace NWindows { + +class CProcess: public CHandle +{ +public: + bool Open(DWORD desiredAccess, bool inheritHandle, DWORD processId) + { + _handle = ::OpenProcess(desiredAccess, inheritHandle, processId); + return (_handle != NULL); + } + + #ifndef UNDER_CE + + bool GetExitCodeProcess(LPDWORD lpExitCode) { return BOOLToBool(::GetExitCodeProcess(_handle, lpExitCode)); } + bool Terminate(UINT exitCode) { return BOOLToBool(::TerminateProcess(_handle, exitCode)); } + #if (WINVER >= 0x0500) + DWORD GetGuiResources (DWORD uiFlags) { return ::GetGuiResources(_handle, uiFlags); } + #endif + bool SetPriorityClass(DWORD dwPriorityClass) { return BOOLToBool(::SetPriorityClass(_handle, dwPriorityClass)); } + DWORD GetPriorityClass() { return ::GetPriorityClass(_handle); } + // bool GetIoCounters(PIO_COUNTERS lpIoCounters ) { return BOOLToBool(::GetProcessIoCounters(_handle, lpIoCounters )); } + + bool GetTimes(LPFILETIME creationTime, LPFILETIME exitTime, LPFILETIME kernelTime, LPFILETIME userTime) + { return BOOLToBool(::GetProcessTimes(_handle, creationTime, exitTime, kernelTime, userTime)); } + + DWORD WaitForInputIdle(DWORD milliseconds) { return ::WaitForInputIdle(_handle, milliseconds); } + + // Debug + + bool ReadMemory(LPCVOID baseAddress, LPVOID buffer, SIZE_T size, SIZE_T* numberOfBytesRead) + { return BOOLToBool(::ReadProcessMemory(_handle, baseAddress, buffer, size, numberOfBytesRead)); } + + bool WriteMemory(LPVOID baseAddress, LPCVOID buffer, SIZE_T size, SIZE_T* numberOfBytesWritten) + { return BOOLToBool(::WriteProcessMemory(_handle, baseAddress, + #ifdef Z7_OLD_WIN_SDK + (LPVOID) + #endif + buffer, + size, numberOfBytesWritten)); } + + bool FlushInstructionCache(LPCVOID baseAddress = NULL, SIZE_T size = 0) + { return BOOLToBool(::FlushInstructionCache(_handle, baseAddress, size)); } + + LPVOID VirtualAlloc(LPVOID address, SIZE_T size, DWORD allocationType, DWORD protect) + { return VirtualAllocEx(_handle, address, size, allocationType, protect); } + + bool VirtualFree(LPVOID address, SIZE_T size, DWORD freeType) + { return BOOLToBool(::VirtualFreeEx(_handle, address, size, freeType)); } + + // Process Status API (PSAPI) + + /* + bool EmptyWorkingSet() + { return BOOLToBool(::EmptyWorkingSet(_handle)); } + bool EnumModules(HMODULE *hModules, DWORD arraySizeInBytes, LPDWORD receivedBytes) + { return BOOLToBool(::EnumProcessModules(_handle, hModules, arraySizeInBytes, receivedBytes)); } + DWORD MyGetModuleBaseName(HMODULE hModule, LPTSTR baseName, DWORD size) + { return ::GetModuleBaseName(_handle, hModule, baseName, size); } + bool MyGetModuleBaseName(HMODULE hModule, CSysString &name) + { + const unsigned len = MAX_PATH + 100; + const DWORD resultLen = MyGetModuleBaseName(hModule, name.GetBuf(len), len); + name.ReleaseBuf_CalcLen(len); + return (resultLen != 0); + } + + DWORD MyGetModuleFileNameEx(HMODULE hModule, LPTSTR baseName, DWORD size) + { return ::GetModuleFileNameEx(_handle, hModule, baseName, size); } + bool MyGetModuleFileNameEx(HMODULE hModule, CSysString &name) + { + const unsigned len = MAX_PATH + 100; + const DWORD resultLen = MyGetModuleFileNameEx(hModule, name.GetBuf(len), len); + name.ReleaseBuf_CalcLen(len); + return (resultLen != 0); + } + + bool GetModuleInformation(HMODULE hModule, LPMODULEINFO moduleInfo) + { return BOOLToBool(::GetModuleInformation(_handle, hModule, moduleInfo, sizeof(MODULEINFO))); } + bool GetMemoryInfo(PPROCESS_MEMORY_COUNTERS memCounters) + { return BOOLToBool(::GetProcessMemoryInfo(_handle, memCounters, sizeof(PROCESS_MEMORY_COUNTERS))); } + */ + + #endif + + WRes Create(LPCWSTR imageName, const UString ¶ms, LPCWSTR curDir); + + DWORD Wait() { return ::WaitForSingleObject(_handle, INFINITE); } +}; + +WRes MyCreateProcess(LPCWSTR imageName, const UString ¶ms); + +} + +#endif diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/PropVariantUtils.cpp b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/PropVariantUtils.cpp new file mode 100644 index 0000000..6daee83 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/PropVariantUtils.cpp @@ -0,0 +1,161 @@ +// PropVariantUtils.cpp + +#include "StdAfx.h" + +#include "../Common/IntToString.h" + +#include "PropVariantUtils.h" + +using namespace NWindows; + +static void AddHex(AString &s, UInt32 v) +{ + char sz[16]; + sz[0] = '0'; + sz[1] = 'x'; + ConvertUInt32ToHex(v, sz + 2); + s += sz; +} + + +AString TypePairToString(const CUInt32PCharPair *pairs, unsigned num, UInt32 value) +{ + char sz[16]; + const char *p = NULL; + for (unsigned i = 0; i < num; i++) + { + const CUInt32PCharPair &pair = pairs[i]; + if (pair.Value == value) + p = pair.Name; + } + if (!p) + { + ConvertUInt32ToString(value, sz); + p = sz; + } + return (AString)p; +} + +void PairToProp(const CUInt32PCharPair *pairs, unsigned num, UInt32 value, NCOM::CPropVariant &prop) +{ + prop = TypePairToString(pairs, num, value); +} + + +AString TypeToString(const char * const table[], unsigned num, UInt32 value) +{ + char sz[16]; + const char *p = NULL; + if (value < num) + p = table[value]; + if (!p) + { + ConvertUInt32ToString(value, sz); + p = sz; + } + return (AString)p; +} + +void TypeToProp(const char * const table[], unsigned num, UInt32 value, NWindows::NCOM::CPropVariant &prop) +{ + char sz[16]; + const char *p = NULL; + if (value < num) + p = table[value]; + if (!p) + { + ConvertUInt32ToString(value, sz); + p = sz; + } + prop = p; +} + + +AString FlagsToString(const char * const *names, unsigned num, UInt32 flags) +{ + AString s; + for (unsigned i = 0; i < num; i++) + { + UInt32 flag = (UInt32)1 << i; + if ((flags & flag) != 0) + { + const char *name = names[i]; + if (name && name[0] != 0) + { + s.Add_OptSpaced(name); + flags &= ~flag; + } + } + } + if (flags != 0) + { + s.Add_Space_if_NotEmpty(); + AddHex(s, flags); + } + return s; +} + +AString FlagsToString(const CUInt32PCharPair *pairs, unsigned num, UInt32 flags) +{ + AString s; + for (unsigned i = 0; i < num; i++) + { + const CUInt32PCharPair &p = pairs[i]; + UInt32 flag = (UInt32)1 << (unsigned)p.Value; + if ((flags & flag) != 0) + { + if (p.Name[0] != 0) + s.Add_OptSpaced(p.Name); + } + flags &= ~flag; + } + if (flags != 0) + { + s.Add_Space_if_NotEmpty(); + AddHex(s, flags); + } + return s; +} + +void FlagsToProp(const char * const *names, unsigned num, UInt32 flags, NCOM::CPropVariant &prop) +{ + prop = FlagsToString(names, num, flags); +} + +void FlagsToProp(const CUInt32PCharPair *pairs, unsigned num, UInt32 flags, NCOM::CPropVariant &prop) +{ + prop = FlagsToString(pairs, num, flags); +} + + +static AString Flags64ToString(const CUInt32PCharPair *pairs, unsigned num, UInt64 flags) +{ + AString s; + for (unsigned i = 0; i < num; i++) + { + const CUInt32PCharPair &p = pairs[i]; + UInt64 flag = (UInt64)1 << (unsigned)p.Value; + if ((flags & flag) != 0) + { + if (p.Name[0] != 0) + s.Add_OptSpaced(p.Name); + } + flags &= ~flag; + } + if (flags != 0) + { + { + char sz[32]; + sz[0] = '0'; + sz[1] = 'x'; + ConvertUInt64ToHex(flags, sz + 2); + s.Add_OptSpaced(sz); + } + } + return s; +} + +void Flags64ToProp(const CUInt32PCharPair *pairs, unsigned num, UInt64 flags, NCOM::CPropVariant &prop) +{ + prop = Flags64ToString(pairs, num, flags); +} diff --git a/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/PropVariantUtils.h b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/PropVariantUtils.h new file mode 100644 index 0000000..78adb50 --- /dev/null +++ b/ext/lzma_sdk_wrapper/lzma_sdk/CPP/Windows/PropVariantUtils.h @@ -0,0 +1,34 @@ +// Windows/PropVariantUtils.h + +#ifndef ZIP7_INC_PROP_VARIANT_UTILS_H +#define ZIP7_INC_PROP_VARIANT_UTILS_H + +#include "../Common/MyString.h" + +#include "PropVariant.h" + +struct CUInt32PCharPair +{ + UInt32 Value; + const char *Name; +}; + +AString TypePairToString(const CUInt32PCharPair *pairs, unsigned num, UInt32 value); +void PairToProp(const CUInt32PCharPair *pairs, unsigned num, UInt32 value, NWindows::NCOM::CPropVariant &prop); + +AString FlagsToString(const char * const *names, unsigned num, UInt32 flags); +AString FlagsToString(const CUInt32PCharPair *pairs, unsigned num, UInt32 flags); +void FlagsToProp(const char * const *names, unsigned num, UInt32 flags, NWindows::NCOM::CPropVariant &prop); +void FlagsToProp(const CUInt32PCharPair *pairs, unsigned num, UInt32 flags, NWindows::NCOM::CPropVariant &prop); + +AString TypeToString(const char * const table[], unsigned num, UInt32 value); +void TypeToProp(const char * const table[], unsigned num, UInt32 value, NWindows::NCOM::CPropVariant &prop); + +#define PAIR_TO_PROP(pairs, value, prop) PairToProp(pairs, Z7_ARRAY_SIZE(pairs), value, prop) +#define FLAGS_TO_PROP(pairs, value, prop) FlagsToProp(pairs, Z7_ARRAY_SIZE(pairs), value, prop) +#define TYPE_TO_PROP(table, value, prop) TypeToProp(table, Z7_ARRAY_SIZE(table), value, prop) + +void Flags64ToProp(const CUInt32PCharPair *pairs, unsigned num, UInt64 flags, NWindows::NCOM::CPropVariant &prop); +#define FLAGS64_TO_PROP(pairs, value, prop) Flags64ToProp(pairs, Z7_ARRAY_SIZE(pairs), value, prop) + +#endif diff --git a/lib/ruby_lzma/archive/reader.rb b/lib/ruby_lzma/archive/reader.rb index 828f808..0ecb442 100644 --- a/lib/ruby_lzma/archive/reader.rb +++ b/lib/ruby_lzma/archive/reader.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "fileutils" +require "stringio" module RubyLzma module Archive @@ -16,8 +17,10 @@ class Reader # Default maximum total extraction size (10 GB) DEFAULT_MAX_TOTAL_SIZE = 10 * 1024 * 1024 * 1024 - # Default maximum compression ratio (1000:1 - typical archive bombs exceed this) - DEFAULT_MAX_COMPRESSION_RATIO = 1000 + # Default maximum compression ratio (10000:1) + # LZMA2 legitimately achieves high ratios on repetitive data (1000-5000:1), + # while actual archive bombs typically exceed 1,000,000:1. + DEFAULT_MAX_COMPRESSION_RATIO = 10_000 # Default maximum size for entries with unknown compression ratio (10 MB) # In solid archives, individual compressed_size may be 0 (unknown) diff --git a/lib/ruby_lzma/format.rb b/lib/ruby_lzma/format.rb index f5de476..dab6d84 100644 --- a/lib/ruby_lzma/format.rb +++ b/lib/ruby_lzma/format.rb @@ -24,12 +24,14 @@ def self.detect(path) end end - # Check if format is supported by current C API + # Check if format is supported # # @param format [Integer] Format constant # @return [Boolean] true if fully supported def self.supported?(format) - format == FFI::Constants::SZ_FORMAT_7Z + [FFI::Constants::SZ_FORMAT_7Z, + FFI::Constants::SZ_FORMAT_ZIP, + FFI::Constants::SZ_FORMAT_TAR].include?(format) end # Get format name diff --git a/spec/archive/multi_format_spec.rb b/spec/archive/multi_format_spec.rb new file mode 100644 index 0000000..1215b6f --- /dev/null +++ b/spec/archive/multi_format_spec.rb @@ -0,0 +1,183 @@ +# frozen_string_literal: true + +RSpec.describe "Multi-format Archive Support" do + let(:test_7z) { fixture_path("test.7z") } + let(:test_zip) { fixture_path("test.zip") } + let(:test_tar) { fixture_path("test.tar") } + let(:encrypted_zip) { fixture_path("test_encrypted.zip") } + let(:output_dir) { File.join(__dir__, "..", "tmp", "multi_format") } + + before do + FileUtils.mkdir_p(output_dir) + end + + after do + FileUtils.rm_rf(output_dir) + end + + describe "ZIP format" do + it "opens and reads a ZIP archive" do + RubyLzma::Archive::Reader.open(test_zip, format: RubyLzma::FFI::Constants::SZ_FORMAT_ZIP) do |reader| + expect(reader.size).to eq(2) + + names = reader.entries.map(&:name) + expect(names).to include("test.txt") + expect(names).to include("test2.txt") + end + end + + it "extracts files from a ZIP archive" do + RubyLzma::Archive::Reader.open(test_zip, format: RubyLzma::FFI::Constants::SZ_FORMAT_ZIP) do |reader| + entry = reader.entries.find { |e| e.name == "test.txt" } + expect(entry).not_to be_nil + + data = reader.extract_data(entry) + expect(data).to eq("Hello, World!") + end + end + + it "extracts all files from a ZIP archive" do + RubyLzma::Archive::Reader.open(test_zip, format: RubyLzma::FFI::Constants::SZ_FORMAT_ZIP) do |reader| + reader.extract_all(output_dir) + + files = Dir.glob("#{output_dir}/**/*").select { |f| File.file?(f) } + expect(files.size).to eq(2) + end + end + + it "opens a ZIP archive from memory buffer" do + data = File.binread(test_zip) + RubyLzma::Archive::Reader.open_buffer(data, format: RubyLzma::FFI::Constants::SZ_FORMAT_ZIP) do |reader| + expect(reader.size).to eq(2) + extracted = reader.extract_data(0) + expect(extracted).not_to be_empty + end + end + + it "opens a ZIP archive from StringIO" do + require "stringio" + data = File.binread(test_zip) + buffer = StringIO.new(data) + + RubyLzma::Archive::Reader.open_buffer(buffer, format: RubyLzma::FFI::Constants::SZ_FORMAT_ZIP) do |reader| + expect(reader.size).to eq(2) + end + end + + it "opens encrypted ZIP with correct password" do + RubyLzma::Archive::Reader.open( + encrypted_zip, + format: RubyLzma::FFI::Constants::SZ_FORMAT_ZIP, + password: "TestPass123" + ) do |reader| + expect(reader.size).to eq(1) + expect(reader.entries.first.name).to eq("secret.txt") + + data = reader.extract_data(0) + expect(data).to eq("Secret ZIP content") + end + end + end + + describe "TAR format" do + it "opens and reads a TAR archive" do + RubyLzma::Archive::Reader.open(test_tar, format: RubyLzma::FFI::Constants::SZ_FORMAT_TAR) do |reader| + # TAR may include additional entries (e.g., directory entries, metadata) + expect(reader.size).to be >= 2 + + names = reader.entries.map(&:name) + expect(names).to include("test.txt") + expect(names).to include("test2.txt") + end + end + + it "extracts files from a TAR archive" do + RubyLzma::Archive::Reader.open(test_tar, format: RubyLzma::FFI::Constants::SZ_FORMAT_TAR) do |reader| + entry = reader.entries.find { |e| e.name == "test.txt" } + expect(entry).not_to be_nil + + data = reader.extract_data(entry) + expect(data).to eq("Hello, World!") + end + end + + it "extracts all files from a TAR archive" do + RubyLzma::Archive::Reader.open(test_tar, format: RubyLzma::FFI::Constants::SZ_FORMAT_TAR) do |reader| + reader.extract_all(output_dir) + + files = Dir.glob("#{output_dir}/**/*").select { |f| File.file?(f) } + expect(files.size).to eq(2) + end + end + + it "opens a TAR archive from memory buffer" do + data = File.binread(test_tar) + RubyLzma::Archive::Reader.open_buffer(data, format: RubyLzma::FFI::Constants::SZ_FORMAT_TAR) do |reader| + expect(reader.size).to be >= 2 + + file_entry = reader.entries.find { |e| e.file? } + expect(file_entry).not_to be_nil + extracted = reader.extract_data(file_entry) + expect(extracted).not_to be_empty + end + end + end + + describe "7z format (regression)" do + it "continues to work with default format" do + RubyLzma::Archive::Reader.open(test_7z) do |reader| + expect(reader.size).to eq(3) + end + end + + it "continues to work with explicit 7z format" do + RubyLzma::Archive::Reader.open(test_7z, format: RubyLzma::FFI::Constants::SZ_FORMAT_7Z) do |reader| + expect(reader.size).to eq(3) + end + end + end + + describe "StringIO handling" do + it "handles StringIO without requiring it externally" do + # This tests the fix for the StringIO NameError bug + # The reader.rb now requires 'stringio' itself + data = File.binread(test_7z) + buffer = StringIO.new(data) + + RubyLzma::Archive::Reader.open_buffer(buffer) do |reader| + expect(reader.size).to eq(3) + end + end + + it "handles plain String buffers" do + data = File.binread(test_7z) + + RubyLzma::Archive::Reader.open_buffer(data) do |reader| + expect(reader.size).to eq(3) + end + end + + it "handles StringIO with ZIP format" do + data = File.binread(test_zip) + buffer = StringIO.new(data) + + RubyLzma::Archive::Reader.open_buffer(buffer, format: RubyLzma::FFI::Constants::SZ_FORMAT_ZIP) do |reader| + expect(reader.size).to eq(2) + end + end + end + + describe "format error handling" do + it "raises error when opening a ZIP with 7z format" do + expect { + RubyLzma::Archive::Reader.new(test_zip, format: RubyLzma::FFI::Constants::SZ_FORMAT_7Z) + }.to raise_error(RubyLzma::Error) + end + + it "raises error when opening a TAR with 7z format" do + expect { + RubyLzma::Archive::Reader.new(test_tar, format: RubyLzma::FFI::Constants::SZ_FORMAT_7Z) + }.to raise_error(RubyLzma::Error) + end + end +end diff --git a/spec/archive/reader_spec.rb b/spec/archive/reader_spec.rb index e814172..f858f7b 100644 --- a/spec/archive/reader_spec.rb +++ b/spec/archive/reader_spec.rb @@ -256,7 +256,7 @@ it "has default extraction limits" do expect(described_class.max_entry_size).to eq(1024 * 1024 * 1024) # 1 GB expect(described_class.max_total_size).to eq(10 * 1024 * 1024 * 1024) # 10 GB - expect(described_class.max_compression_ratio).to eq(1000) + expect(described_class.max_compression_ratio).to eq(10_000) expect(described_class.max_size_for_unknown_ratio).to eq(10 * 1024 * 1024) # 10 MB end diff --git a/spec/fixtures/test.tar b/spec/fixtures/test.tar new file mode 100644 index 0000000..1341fd6 Binary files /dev/null and b/spec/fixtures/test.tar differ diff --git a/spec/fixtures/test.zip b/spec/fixtures/test.zip new file mode 100644 index 0000000..9456e4f Binary files /dev/null and b/spec/fixtures/test.zip differ diff --git a/spec/fixtures/test_encrypted.zip b/spec/fixtures/test_encrypted.zip new file mode 100644 index 0000000..0eed4c9 Binary files /dev/null and b/spec/fixtures/test_encrypted.zip differ diff --git a/spec/format_spec.rb b/spec/format_spec.rb index 391b4cd..a59fa25 100644 --- a/spec/format_spec.rb +++ b/spec/format_spec.rb @@ -33,12 +33,12 @@ expect(described_class.supported?(RubyLzma::FFI::Constants::SZ_FORMAT_7Z)).to be true end - it "returns false for ZIP format (requires C++ API)" do - expect(described_class.supported?(RubyLzma::FFI::Constants::SZ_FORMAT_ZIP)).to be false + it "returns true for ZIP format" do + expect(described_class.supported?(RubyLzma::FFI::Constants::SZ_FORMAT_ZIP)).to be true end - it "returns false for TAR format (requires C++ API)" do - expect(described_class.supported?(RubyLzma::FFI::Constants::SZ_FORMAT_TAR)).to be false + it "returns true for TAR format" do + expect(described_class.supported?(RubyLzma::FFI::Constants::SZ_FORMAT_TAR)).to be true end end