Add simple Ogg and IFF parsers

CMakeLists.txt

@@ -273,11 +273,15 @@ file(GLOB kaitai_sources "${SRC_DIR}/kaitai/*.cc")
 add_library(parser
     ${INCLUDE_DIR}/parser/parser.h
     ${INCLUDE_DIR}/parser/stream.h
+    ${INCLUDE_DIR}/parser/uniff.h
+    ${INCLUDE_DIR}/parser/unogg.h
     ${INCLUDE_DIR}/parser/unpyc.h
     ${INCLUDE_DIR}/parser/unpng.h
     ${INCLUDE_DIR}/parser/utils.h
     ${kaitai_headers}
     ${SRC_DIR}/parser/parser.cc
+    ${SRC_DIR}/parser/uniff.cc
+    ${SRC_DIR}/parser/unogg.cc
     ${SRC_DIR}/parser/unpyc.cc
     ${SRC_DIR}/parser/unpng.cc
     ${SRC_DIR}/parser/utils.cc

include/parser/stream.h

@@ -18,6 +18,8 @@
 
 #include <cassert>
 
+#include <string>
+
 #include "data/repack.h"
 #include "dbif/info.h"
 #include "dbif/types.h"
@@ -94,11 +96,12 @@ class StreamParser {
     return res;
   }
 
-  data::BinData getDataUntil(const QString& name, const data::Repacker& repack,
-                             data::BinData termination,
-                             const data::FieldHighType& high_type,
-                             bool include_termination = true) {
-    assert(termination.size() == 1);
+  template <class Func>
+  data::BinData getDataUntilCond(const QString& name,
+                                 const data::Repacker& repack,
+                                 Func&& termination_condition,
+                                 const data::FieldHighType& high_type,
+                                 bool include_termination = true) {
     data::BinData res(repack.to_width, 0);
     size_t num_elements = 1;
     size_t src_size = repack.repackSize(num_elements);
@@ -116,7 +119,7 @@ class StreamParser {
       data = repack.repack(data, 0, num_elements);
 
       for (size_t dataIndex = 0; dataIndex < data.size(); ++dataIndex) {
-        if (data[dataIndex] == termination) {
+        if (termination_condition(data[dataIndex])) {
           if (include_termination) {
             dataIndex += 1;
           }
@@ -240,18 +243,39 @@ class StreamParser {
     return res;
   }
 
-  std::vector<uint8_t> getBytesUntil(const QString& name, uint8_t termination,
-                                     bool include_termination = true) {
-    auto data = getDataUntil(name, data::Repacker(),
-                             data::BinData::fromRawData(8, {termination}),
-                             data::FieldHighType(), include_termination);
+  std::string getString(const QString& name, uint64_t len) {
+    auto data =
+        getData(name, data::Repacker(), len,
+                data::FieldHighType::string(data::FieldHighType::STRING_RAW));
+    std::string res(len, '\0');
+    for (size_t i = 0; i < data.size(); i++) {
+      res[i] = static_cast<char>(data.element64(i));
+    }
+    return res;
+  }
+
+  template <class Func>
+  std::vector<uint8_t> getBytesUntilCond(const QString& name,
+                                         Func&& termination_condition,
+                                         bool include_termination = true) {
+    auto data = getDataUntilCond(name, data::Repacker(),
+                                 std::forward<Func>(termination_condition),
+                                 data::FieldHighType(), include_termination);
     std::vector<uint8_t> res;
     for (size_t i = 0; i < data.size(); i++) {
       res.push_back(static_cast<uint8_t>(data.element64(i)));
     }
     return res;
   }
 
+  std::vector<uint8_t> getBytesUntil(const QString& name, uint8_t termination,
+                                     bool include_termination = true) {
+    auto bindata_termination = data::BinData::fromRawData(8, {termination});
+    return getBytesUntilCond(
+        name, [=](data::BinData c) { return c == bindata_termination; },
+        include_termination);
+  }
+
   uint8_t getByte(const QString& name) {
     auto data = getData(name, data::Repacker(), 1, data::FieldHighType());
     if (data.size() == 0) {

include/parser/uniff.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2017 Marek Rusinowski
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include "data/bindata.h"
+#include "dbif/types.h"
+#include "parser/parser.h"
+
+namespace veles {
+namespace parser {
+
+void unIffFileBlob(
+    const dbif::ObjectHandle& blob, uint64_t start = 0,
+    const dbif::ObjectHandle& parent_chunk = dbif::ObjectHandle());
+
+class IffParser : public Parser {
+ public:
+  IffParser() : Parser("iff (LE)", data::BinData(8, {})) {}
+  void parse(const dbif::ObjectHandle& blob, uint64_t start,
+             const dbif::ObjectHandle& parent_chunk) override {
+    unIffFileBlob(blob, start, parent_chunk);
+  }
+};
+
+}  // namespace parser
+}  // namespace veles

include/parser/unogg.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2017 Marek Rusinowski
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include "data/bindata.h"
+#include "dbif/types.h"
+#include "parser/parser.h"
+
+namespace veles {
+namespace parser {
+
+void unOggFileBlob(
+    const dbif::ObjectHandle& blob, uint64_t start = 0,
+    const dbif::ObjectHandle& parent_chunk = dbif::ObjectHandle());
+
+class OggParser : public Parser {
+ public:
+  OggParser() : Parser("ogg", data::BinData(8, {'O', 'g', 'g', 'S'})) {}
+  void parse(const dbif::ObjectHandle& blob, uint64_t start,
+             const dbif::ObjectHandle& parent_chunk) override {
+    unOggFileBlob(blob, start, parent_chunk);
+  }
+};
+
+}  // namespace parser
+}  // namespace veles

src/parser/uniff.cc

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2017 Marek Rusinowski
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <limits>
+#include <string>
+
+#include "parser/unogg.h"
+
+#include "parser/stream.h"
+
+namespace veles {
+namespace parser {
+
+enum {
+  OGG_HEADER_TYPE_CONTINUED_PACKET = 0x1,
+  OGG_HEADER_TYPE_FIRST_PAGE = 0x2,
+  OGG_HEADER_TYPE_LAST_PAGE = 0x4
+};
+
+static std::string formGroup = "FORM", listGroup = "LIST", catGroup = "CAT ";
+
+static void parseChunkGroup(StreamParser* parser, size_t size) {
+  size_t pos = 0;
+  while (pos < size && !parser->eof()) {
+    parser->startChunk("chunk", "chunk");
+    auto bytes = parser->getString("id", 4);
+    uint32_t chunk_size = parser->getLe32("size");
+    if (bytes == formGroup || bytes == listGroup || bytes == catGroup) {
+      parseChunkGroup(parser, chunk_size);
+    } else {
+      parser->getBytes("data", chunk_size);
+    }
+    parser->endChunk();
+  }
+}
+
+void unIffFileBlob(const dbif::ObjectHandle& blob, uint64_t start,
+                   const dbif::ObjectHandle& parent_chunk) {
+  StreamParser parser(blob, start, parent_chunk);
+  parseChunkGroup(&parser, std::numeric_limits<size_t>::max());
+}
+
+}  // namespace parser
+}  // namespace veles

src/parser/unogg.cc

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2017 Marek Rusinowski
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "parser/unogg.h"
+
+#include "parser/stream.h"
+
+namespace veles {
+namespace parser {
+
+enum {
+  OGG_HEADER_TYPE_CONTINUED_PACKET = 0x1,
+  OGG_HEADER_TYPE_FIRST_PAGE = 0x2,
+  OGG_HEADER_TYPE_LAST_PAGE = 0x4
+};
+
+void unOggFileBlob(const dbif::ObjectHandle& blob, uint64_t start,
+                   const dbif::ObjectHandle& parent_chunk) {
+  StreamParser parser(blob, start, parent_chunk);
+  std::vector<unsigned> packets_size;
+  for (unsigned ogg_page_num = 0; !parser.eof(); ++ogg_page_num) {
+    parser.startChunk("ogg_page", QString("page[%1]").arg(ogg_page_num));
+    parser.getBytes("sig", 4);
+    parser.getByte("version");
+    uint8_t header_type_flags = parser.getByte("header_type_flags");
+    QStringList page_comment;
+    if ((header_type_flags & OGG_HEADER_TYPE_CONTINUED_PACKET) != 0) {
+      page_comment << "continued packet";
+    }
+    if ((header_type_flags & OGG_HEADER_TYPE_FIRST_PAGE) != 0) {
+      page_comment << "first page";
+    }
+    if ((header_type_flags & OGG_HEADER_TYPE_LAST_PAGE) != 0) {
+      page_comment << "last page";
+    }
+    parser.setComment(page_comment.join(", "));
+    parser.getLe64("granule_position");
+    parser.getLe32("stream_id");
+    parser.getLe32("page_num");
+    parser.getLe32("crc32");
+    uint8_t page_segments = parser.getByte("page_segments");
+
+    parser.startChunk("ogg_segment_table", "segment_table");
+    packets_size.clear();
+    for (unsigned segment = 0; segment < page_segments;) {
+      unsigned packet_size = 0;
+      parser.getBytesUntilCond(
+          QString("packet_size[%1]").arg(packets_size.size()),
+          [&, page_segments](data::BinData c) {
+            uint8_t v = c.element64();
+            packet_size += v;
+            ++segment;
+            return v < 255 || segment >= page_segments;
+          });
+      packets_size.push_back(packet_size);
+    }
+    parser.endChunk();
+
+    parser.startChunk("ogg_packet_data", "packet_data");
+    for (size_t i = 0; i < packets_size.size(); ++i) {
+      parser.getBytes(QString("packet[%1]").arg(i), packets_size[i]);
+    }
+    parser.endChunk();
+
+    parser.endChunk();
+  }
+}
+
+}  // namespace parser
+}  // namespace veles

src/parser/utils.cc

@@ -25,6 +25,8 @@
 #include "kaitai/png_parser.h"
 #include "kaitai/quicktime_mov_parser.h"
 #include "kaitai/zip_parser.h"
+#include "parser/uniff.h"
+#include "parser/unogg.h"
 #include "parser/unpng.h"
 #include "parser/unpyc.h"
 
@@ -64,6 +66,8 @@ dbif::ObjectHandle makeSubBlob(const dbif::ObjectHandle& parent,
 
 QList<Parser*> createAllParsers() {
   QList<Parser*> res;
+  res.append(new IffParser());
+  res.append(new OggParser());
   res.append(new PycParser());
   res.append(new PngParser());
   res.append(new kaitai::ZipParser());