Add XMem functions (in progress) and LZO, fix ZLib.

libs/compression/compression.cpp

@@ -1,5 +1,5 @@
 #include "compression.h"
-//#include "lzokay.h"
+#include "minilzo.h"
 
 #define XBOXAPI __declspec(dllimport)
 #include "xcompress.h"
@@ -13,8 +13,8 @@ QByteArray Compression::CompressXMem(const QByteArray &data)
 {
     XMEMCODEC_PARAMETERS_LZX lzxParams = {};
     lzxParams.Flags = 0;
-    lzxParams.WindowSize = XCOMPRESS_LZX_BLOCK_SIZE;
+    lzxParams.WindowSize = 0x20000;
-    lzxParams.CompressionPartitionSize = XCOMPRESS_LZX_BLOCK_SIZE;
+    lzxParams.CompressionPartitionSize = 0x80000;
 
     XMEMCOMPRESSION_CONTEXT ctx = nullptr;
     if (FAILED(XMemCreateCompressionContext(XMEMCODEC_LZX, &lzxParams, 0, &ctx)) || !ctx)
@@ -34,25 +34,35 @@ QByteArray Compression::CompressXMem(const QByteArray &data)
     return output;
 }
 
-QByteArray Compression::DecompressXMem(const QByteArray &data)
+QByteArray Compression::DecompressXMem(const QByteArray &data, int flags, int windowSize, int partSize)
 {
+    if (data.isEmpty())
+        return {};
+
     XMEMCODEC_PARAMETERS_LZX lzxParams = {};
-    lzxParams.Flags = 0;
+    lzxParams.Flags = flags;
-    lzxParams.WindowSize = XCOMPRESS_LZX_BLOCK_SIZE;
+    lzxParams.WindowSize = windowSize;
-    lzxParams.CompressionPartitionSize = XCOMPRESS_LZX_BLOCK_SIZE;
+    lzxParams.CompressionPartitionSize = partSize;
 
     XMEMDECOMPRESSION_CONTEXT ctx = nullptr;
     if (FAILED(XMemCreateDecompressionContext(XMEMCODEC_LZX, &lzxParams, 0, &ctx)) || !ctx)
-        return QByteArray();
+        return {};
 
-    QByteArray output(data.size(), 0);
+    // Allocate large enough buffer for decompression (16 MB is a safe upper bound)
-    SIZE_T actualSize = data.size();
+    const SIZE_T kMaxOutSize = 16 * 1024 * 1024;
+    QByteArray output(static_cast<int>(kMaxOutSize), Qt::Uninitialized);
+    SIZE_T actualSize = kMaxOutSize;
+
+    HRESULT hr = XMemDecompress(ctx,
+                                output.data(), &actualSize,
+                                data.constData(), data.size() + 16);
 
-    HRESULT hr = XMemDecompress(ctx, output.data(), &actualSize, data.constData(), data.size());
     XMemDestroyDecompressionContext(ctx);
 
-    if (FAILED(hr))
+    if (FAILED(hr)) {
-        return QByteArray();
+        qWarning() << "XMemDecompress failed with HRESULT:" << hr;
+        return {};
+    }
 
     output.resize(static_cast<int>(actualSize));
     return output;
@@ -68,6 +78,46 @@ quint32 Compression::CalculateAdler32Checksum(const QByteArray &data) {
     return adler;
 }
 
+qint64 Compression::FindZlibOffset(const QByteArray &bytes)
+{
+    static const QByteArray iwffs("IWffs");
+    auto idx = bytes.indexOf(iwffs);
+    if (idx != -1)
+        return idx + 0x4000;
+
+    const char header = 0x78;           // z-lib: 0x78 [FLG]
+    int pos = -1;
+    while ((pos = bytes.indexOf(header, pos + 1)) != -1)
+    {
+        QByteArray window = bytes.mid(pos, 0x20);
+        if (!window.contains(QByteArray::fromHex("000000")) &&
+            !window.contains(QByteArray::fromHex("FFFFFF")))
+            return pos;
+    }
+    return -1;
+}
+
+QByteArray Compression::StripHashBlocks(const QByteArray &raw,
+                                        int dataChunkSize,
+                                        int hashChunkSize)
+{
+    QByteArray cleaned;
+    cleaned.reserve(raw.size());        // upper bound
+
+    int p = 0;
+    while (p < raw.size())
+    {
+        const int chunk = qMin(dataChunkSize, raw.size() - p);
+        cleaned.append(raw.constData() + p, chunk);
+        p += chunk;
+
+        // skip hash bytes if they are still inside the buffer
+        if (p < raw.size())
+            p += qMin(hashChunkSize, raw.size() - p);
+    }
+    return cleaned;
+}
+
 QByteArray Compression::DecompressZLIB(const QByteArray &aCompressedData) {
     if (aCompressedData.isEmpty()) {
         return {};
@@ -95,22 +145,23 @@ QByteArray Compression::DecompressZLIB(const QByteArray &aCompressedData) {
 
         ret = inflate(&strm, Z_NO_FLUSH);
 
-        if (strm.avail_out < buffer.size()) {  // Data has been written to the buffer
+        if (strm.avail_out < buffer.size()) {
             decompressed.append(buffer.constData(), buffer.size() - strm.avail_out);
         }
 
         if (ret == Z_STREAM_END) {
-            break;  // Proper end of the data stream
+            break;
         }
 
         if (ret == Z_BUF_ERROR && strm.avail_out == 0) {
-            // Buffer was completely used, resize it
+            buffer.resize(buffer.size() * 2);
-            int newSize = buffer.size() * 2;  // Double the buffer size
-            buffer.resize(newSize);
         } else if (ret != Z_OK) {
-            qWarning() << "Zlib error:" << zError(ret);
+            size_t errorOffset = strm.total_in;
+            qWarning() << "Zlib error:" << zError(ret)
+                       << "at offset" << errorOffset
+                       << "of" << aCompressedData.size() << "bytes";
             inflateEnd(&strm);
-            return {};  // Return on other errors
+            return decompressed;
         }
     } while (ret != Z_STREAM_END);
 
@@ -118,6 +169,7 @@ QByteArray Compression::DecompressZLIB(const QByteArray &aCompressedData) {
     return decompressed;
 }
 
+
 QByteArray Compression::CompressZLIB(const QByteArray &aData) {
     return CompressZLIBWithSettings(aData);
 }
@@ -242,6 +294,27 @@ QByteArray Compression::CompressDeflateWithSettings(const QByteArray &aData, int
     return compressed;
 }
 
+QByteArray Compression::DecompressLZO(const QByteArray &aCompressedData, quint32 aDestSize) {
+    static bool ok = (lzo_init() == LZO_E_OK);
+    if (!ok)
+        throw std::runtime_error("lzo_init failed");
+
+    QByteArray dst(aDestSize, Qt::Uninitialized);
+    lzo_uint out = aDestSize;
+
+    int rc = lzo1x_decompress_safe(
+        reinterpret_cast<const lzo_bytep>(aCompressedData.constData()),
+        static_cast<lzo_uint>(aCompressedData.size()),
+        reinterpret_cast<lzo_bytep>(dst.data()),
+        &out,
+        nullptr);
+
+    if (rc != LZO_E_OK || out != aDestSize)
+        throw std::runtime_error("LZO decompression error");
+
+    return dst;
+}
+
 QByteArray Compression::DecompressOodle(const QByteArray &aCompressedData, quint32 aDecompressedSize) {
     return pDecompressOodle(aCompressedData, aCompressedData.length(), aDecompressedSize);
 }

libs/compression/compression.h

@@ -47,6 +47,10 @@ class Compression {
 public:
     static quint32 CalculateAdler32Checksum(const QByteArray &data);
     static QByteArray DecompressZLIB(const QByteArray &aCompressedData);
+    static qint64 FindZlibOffset(const QByteArray &bytes);
+    static QByteArray StripHashBlocks(const QByteArray &raw,
+                                      int dataChunkSize = 0x200000,
+                                      int hashChunkSize = 0x2000);
     static QByteArray CompressZLIB(const QByteArray &aData);
     static QByteArray CompressZLIBWithSettings(const QByteArray &aData,
                                                int aCompressionLevel = Z_BEST_COMPRESSION,
@@ -64,13 +68,14 @@ public:
                                                   int aStrategy = Z_DEFAULT_STRATEGY,
                                                   const QByteArray &aDictionary = {});
 
+    static QByteArray DecompressLZO(const QByteArray &aCompressedData, quint32 aDestSize);
+
     static QByteArray DecompressOodle(const QByteArray &aCompressedData, quint32 aDecompressedSize);
     static QByteArray CompressOodle(const QByteArray &aData);
 
 
     static QByteArray CompressXMem(const QByteArray &data);
-    static QByteArray DecompressXMem(const QByteArray &data);
+    static QByteArray DecompressXMem(const QByteArray &data, int flags = 0, int windowSize = 0x20000, int partSize = 0x80000);
-
 private:
     static quint32 pGetOodleCompressedBounds(quint32 aBufferSize);
     static QByteArray pCompressOodle(QByteArray aBuffer, quint32 aBufferSize, quint32 aOutputBufferSize,