crc and hdt cookie

ate47 · Mar 17, 2024 · ae0aab9 · ae0aab9
1 parent d13f1ff
commit ae0aab9
Show file tree

Hide file tree

Showing 10 changed files with 2,205 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -23,6 +23,8 @@ This tool is using and providing
 - [asmjit](https://github.com/asmjit/asmjit) - [ZLib license](https://github.com/asmjit/asmjit/blob/master/LICENSE.md).
 - [ladislav-zezula/CascLib](https://github.com/ladislav-zezula/CascLib) - [MIT license](https://github.com/ladislav-zezula/CascLib/blob/master/LICENSE).
 - [lz4/lz4](https://github.com/lz4/lz4) - [Custom license](https://github.com/lz4/lz4/blob/dev/lib/LICENSE).
+- [vimpunk/mio](https://github.com/vimpunk/mio) - [MIT license](https://github.com/vimpunk/mio/blob/master/LICENSE).
+- [JustasMasiulis/xorstr](https://github.com/JustasMasiulis/xorstr) - [Apache License 2.0](https://github.com/JustasMasiulis/xorstr/blob/master/LICENSE)
 
 ## Downloads
 

diff --git a/docs/_config.yml b/docs/_config.yml
@@ -3,5 +3,5 @@ plugins:
 - jekyll-remote-theme
 
 title: Atian Tools
-description: Set of tools
+description: ate47's set of tools
 
diff --git a/src/cli/tools/lib/actslibtest.cpp b/src/cli/tools/lib/actslibtest.cpp
@@ -1,6 +1,9 @@
 #include <actslib/actslib.hpp>
 #include <actslib/profiler.hpp>
 #include <includes.hpp>
+#include <actslib/crc.hpp>
+#include <actslib/hdt.hpp>
+#include <mio.hpp>
 
 namespace {
 	int actslibtest(Process& proc, int argc, const char* argv[]) {
@@ -56,9 +59,45 @@ namespace {
 		return tool::OK;
 	}
 
+	int actslibhdtcookie(Process& proc, int argc, const char* argv[]) {
+		if (argc < 3) {
+			return tool::BAD_USAGE;
+		}
+
+		std::ifstream is{ argv[2], std::ios::binary };
+
+		if (!is) {
+			LOG_ERROR("Can't open {}", argv[2]);
+			return tool::BASIC_ERROR;
+		}
+
+		try {
+			actslib::hdt::HDTCookie cookie{ is };
+			is.close();
+			LOG_INFO("Type:   {}", actslib::hdt::FormatName(cookie.GetType()));
+			LOG_INFO("Format: {}", cookie.GetFormat());
+
+			LOG_INFO("Props:");
+			for (const auto& [key, val] : cookie) {
+				LOG_INFO("'{}' = {}", key, val);
+			}
+		}
+		catch (std::runtime_error& err) {
+			LOG_ERROR("Can't read profiler {}", err.what());
+			is.close();
+			return tool::BASIC_ERROR;
+		}
+		is.close();
+
+		return tool::OK;
+	}
+
 }
 
 #ifndef CI_BUILD
 ADD_TOOL("actslibtest", "", "Acts lib test", nullptr, actslibtest);
-ADD_TOOL("actslibprofiler", " [profile file]", "Read profiler", nullptr, actslibprofiler);
-#endif
+
+ADD_TOOL("actslibhdt", " [hdt]", "Read hdt cookie", nullptr, actslibhdtcookie);
+#endif
+
+ADD_TOOL("actslibprofiler", " [profile file]", "Read profiler", nullptr, actslibprofiler);
diff --git a/src/lib/actslib/crc.cpp b/src/lib/actslib/crc.cpp
@@ -0,0 +1,86 @@
+#include "actslib.hpp"
+#include "crc.hpp"
+
+namespace {
+	const uint32_t crc32Table[] { 
+		0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 
+		0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, 
+		0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57,
+		0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a,
+		0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e,
+		0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 
+		0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696,
+		0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,
+		0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3,
+		0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f,
+		0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, 0x61c69362, 0x93ad1061, 0x80fde395,
+		0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, 
+		0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 
+		0x44694011, 0x5739b3e5, 0xa55230e6, 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 
+		0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 
+		0x563c5f93, 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, 
+		0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8,
+		0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, 
+		0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 
+		0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 
+		0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 
+		0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 
+		0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 
+		0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, 
+		0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a,
+		0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, 
+		0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x0105ec76, 0x12551f82, 
+		0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 
+		0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351
+	};
+	const uint16_t crc16Table[] {
+		0x0000, 0xc0c1, 0xc181, 0x0140, 0xc301, 0x03c0, 0x0280, 0xc241, 0xc601, 0x06c0, 0x0780, 0xc741, 0x0500, 0xc5c1, 
+		0xc481, 0x0440, 0xcc01, 0x0cc0, 0x0d80, 0xcd41, 0x0f00, 0xcfc1, 0xce81, 0x0e40, 0x0a00, 0xcac1, 0xcb81, 0x0b40, 
+		0xc901, 0x09c0, 0x0880, 0xc841, 0xd801, 0x18c0, 0x1980, 0xd941, 0x1b00, 0xdbc1, 0xda81, 0x1a40, 0x1e00, 0xdec1, 
+		0xdf81, 0x1f40, 0xdd01, 0x1dc0, 0x1c80, 0xdc41, 0x1400, 0xd4c1, 0xd581, 0x1540, 0xd701, 0x17c0, 0x1680, 0xd641, 
+		0xd201, 0x12c0, 0x1380, 0xd341, 0x1100, 0xd1c1, 0xd081, 0x1040, 0xf001, 0x30c0, 0x3180, 0xf141, 0x3300, 0xf3c1, 
+		0xf281, 0x3240, 0x3600, 0xf6c1, 0xf781, 0x3740, 0xf501, 0x35c0, 0x3480, 0xf441, 0x3c00, 0xfcc1, 0xfd81, 0x3d40, 
+		0xff01, 0x3fc0, 0x3e80, 0xfe41, 0xfa01, 0x3ac0, 0x3b80, 0xfb41, 0x3900, 0xf9c1, 0xf881, 0x3840, 0x2800, 0xe8c1,
+		0xe981, 0x2940, 0xeb01, 0x2bc0, 0x2a80, 0xea41, 0xee01, 0x2ec0, 0x2f80, 0xef41, 0x2d00, 0xedc1, 0xec81, 0x2c40, 
+		0xe401, 0x24c0, 0x2580, 0xe541, 0x2700, 0xe7c1, 0xe681, 0x2640, 0x2200, 0xe2c1, 0xe381, 0x2340, 0xe101, 0x21c0, 
+		0x2080, 0xe041, 0xa001, 0x60c0, 0x6180, 0xa141, 0x6300, 0xa3c1, 0xa281, 0x6240, 0x6600, 0xa6c1, 0xa781, 0x6740, 
+		0xa501, 0x65c0, 0x6480, 0xa441, 0x6c00, 0xacc1, 0xad81, 0x6d40, 0xaf01, 0x6fc0, 0x6e80, 0xae41, 0xaa01, 0x6ac0, 
+		0x6b80, 0xab41, 0x6900, 0xa9c1, 0xa881, 0x6840, 0x7800, 0xb8c1, 0xb981, 0x7940, 0xbb01, 0x7bc0, 0x7a80, 0xba41,
+		0xbe01, 0x7ec0, 0x7f80, 0xbf41, 0x7d00, 0xbdc1, 0xbc81, 0x7c40, 0xb401, 0x74c0, 0x7580, 0xb541, 0x7700, 0xb7c1,
+		0xb681, 0x7640, 0x7200, 0xb2c1, 0xb381, 0x7340, 0xb101, 0x71c0, 0x7080, 0xb041, 0x5000, 0x90c1, 0x9181, 0x5140, 
+		0x9301, 0x53c0, 0x5280, 0x9241, 0x9601, 0x56c0, 0x5780, 0x9741, 0x5500, 0x95c1, 0x9481, 0x5440, 0x9c01, 0x5cc0, 
+		0x5d80, 0x9d41, 0x5f00, 0x9fc1, 0x9e81, 0x5e40, 0x5a00, 0x9ac1, 0x9b81, 0x5b40, 0x9901, 0x59c0, 0x5880, 0x9841, 
+		0x8801, 0x48c0, 0x4980, 0x8941, 0x4b00, 0x8bc1, 0x8a81, 0x4a40, 0x4e00, 0x8ec1, 0x8f81, 0x4f40, 0x8d01, 0x4dc0,
+		0x4c80, 0x8c41, 0x4400, 0x84c1, 0x8581, 0x4540, 0x8701, 0x47c0, 0x4680, 0x8641, 0x8201, 0x42c0, 0x4380, 0x8341,
+		0x4100, 0x81c1, 0x8081, 0x4040
+	};
+
+	const uint8_t crc8Table[] {
+		0x00, 0x07, 0x0e, 0x09, 0x1c, 0x1b, 0x12, 0x15, 0x38, 0x3f, 0x36, 0x31, 0x24, 0x23, 0x2a, 0x2d, 0x70, 0x77, 0x7e,
+		0x79, 0x6c, 0x6b, 0x62, 0x65, 0x48, 0x4f, 0x46, 0x41, 0x54, 0x53, 0x5a, 0x5d, 0xe0, 0xe7, 0xee, 0xe9, 0xfc, 0xfb, 
+		0xf2, 0xf5, 0xd8, 0xdf, 0xd6, 0xd1, 0xc4, 0xc3, 0xca, 0xcd, 0x90, 0x97, 0x9e, 0x99, 0x8c, 0x8b, 0x82, 0x85, 0xa8, 
+		0xaf, 0xa6, 0xa1, 0xb4, 0xb3, 0xba, 0xbd, 0xc7, 0xc0, 0xc9, 0xce, 0xdb, 0xdc, 0xd5, 0xd2, 0xff, 0xf8, 0xf1, 0xf6,
+		0xe3, 0xe4, 0xed, 0xea, 0xb7, 0xb0, 0xb9, 0xbe, 0xab, 0xac, 0xa5, 0xa2, 0x8f, 0x88, 0x81, 0x86, 0x93, 0x94, 0x9d,
+		0x9a, 0x27, 0x20, 0x29, 0x2e, 0x3b, 0x3c, 0x35, 0x32, 0x1f, 0x18, 0x11, 0x16, 0x03, 0x04, 0x0d, 0x0a, 0x57, 0x50, 
+		0x59, 0x5e, 0x4b, 0x4c, 0x45, 0x42, 0x6f, 0x68, 0x61, 0x66, 0x73, 0x74, 0x7d, 0x7a, 0x89, 0x8e, 0x87, 0x80, 0x95, 
+		0x92, 0x9b, 0x9c, 0xb1, 0xb6, 0xbf, 0xb8, 0xad, 0xaa, 0xa3, 0xa4, 0xf9, 0xfe, 0xf7, 0xf0, 0xe5, 0xe2, 0xeb, 0xec, 
+		0xc1, 0xc6, 0xcf, 0xc8, 0xdd, 0xda, 0xd3, 0xd4, 0x69, 0x6e, 0x67, 0x60, 0x75, 0x72, 0x7b, 0x7c, 0x51, 0x56, 0x5f,
+		0x58, 0x4d, 0x4a, 0x43, 0x44, 0x19, 0x1e, 0x17, 0x10, 0x05, 0x02, 0x0b, 0x0c, 0x21, 0x26, 0x2f, 0x28, 0x3d, 0x3a, 
+		0x33, 0x34, 0x4e, 0x49, 0x40, 0x47, 0x52, 0x55, 0x5c, 0x5b, 0x76, 0x71, 0x78, 0x7f, 0x6a, 0x6d, 0x64, 0x63, 0x3e, 
+		0x39, 0x30, 0x37, 0x22, 0x25, 0x2c, 0x2b, 0x06, 0x01, 0x08, 0x0f, 0x1a, 0x1d, 0x14, 0x13, 0xae, 0xa9, 0xa0, 0xa7, 
+		0xb2, 0xb5, 0xbc, 0xbb, 0x96, 0x91, 0x98, 0x9f, 0x8a, 0x8d, 0x84, 0x83, 0xde, 0xd9, 0xd0, 0xd7, 0xc2, 0xc5, 0xcc, 
+		0xcb, 0xe6, 0xe1, 0xe8, 0xef, 0xfa, 0xfd, 0xf4, 0xf3
+	};
+}
+
+namespace actslib::crc {
+	void CRC8::Update(const char val) {
+		value = crc8Table[(value ^ val) & 0xFF];
+	}
+	void CRC16::Update(const char val) {
+		value = (crc16Table[(value ^ val) & 0xFF] ^ (value >> 8u)) & 0xFFFF;
+	}
+	void CRC32::Update(const char val) {
+		value = crc32Table[(value ^ val) & 0xFF] ^ (value >> 8u);
+	}
+}
diff --git a/src/lib/actslib/crc.hpp b/src/lib/actslib/crc.hpp
@@ -0,0 +1,49 @@
+#pragma once
+#include "io.hpp"
+
+namespace actslib::crc {
+	template<typename CRCType>
+	class CRC {
+	public:
+		CRCType value{};
+		CRC() {}
+
+		virtual void Update(const char val) = 0;
+		void Update(const char* buffer, size_t offset, size_t length) {
+			for (size_t i = 0; i < length; i++) {
+				Update(buffer[offset + i]);
+			}
+		}
+
+		constexpr void Reset() {
+			value = 0;
+		}
+
+		constexpr void WriteCRC(std::ostream& os) const {
+			actslib::io::WriteUInt<CRCType>(os, value);
+		}
+
+		constexpr bool CheckCRC(std::istream& is) const {
+			return actslib::io::ReadUInt<CRCType>(is) == value;
+		}
+	};
+
+	class CRC8 : public CRC<uint8_t> {
+	public:
+		void Update(const char val) override;
+	};
+	class CRC16 : public CRC<uint16_t> {
+	public:
+		using CRC::Update;
+		using CRC::Reset;
+		using CRC::WriteCRC;
+		void Update(const char val) override;
+	};
+	class CRC32 : public CRC<uint32_t> {
+	public:
+		using CRC::Update;
+		using CRC::Reset;
+		using CRC::WriteCRC;
+		void Update(const char val) override;
+	};
+}
diff --git a/src/lib/actslib/hdt.cpp b/src/lib/actslib/hdt.cpp
@@ -0,0 +1,19 @@
+#include "hdt.hpp"
+
+namespace actslib::hdt {
+	namespace {
+		const char* HDTCookieTypeNames[] {
+			"unknown",
+			"global",
+			"header",
+			"dictionary",
+			"triples",
+			"index",
+			"qepcore_merge",
+		};
+	}
+
+	const char* FormatName(HDTCookieType type) {
+		return type >= 0 && type < HDTCookieType::HCT_COUNT ? HDTCookieTypeNames[type] : "bad_type";
+	}
+}
diff --git a/src/lib/actslib/hdt.hpp b/src/lib/actslib/hdt.hpp
@@ -0,0 +1,102 @@
+#pragma once
+#include "actslib.hpp"
+#include "crc.hpp"
+
+namespace actslib::hdt {
+	constexpr const char HDT_COOKIE_MAGIC[] = "$HDT";
+
+	enum HDTCookieType : uint8_t {
+		HCT_UNKNOWN = 0,
+		HCT_GLOBAL,
+		HCT_HEADER,
+		HCT_DICTIONARY,
+		HCT_TRIPLES,
+		HCT_INDEX,
+		HCT_QEPCORE_MERGE,
+		HCT_COUNT
+	};
+
+	const char* FormatName(HDTCookieType type);
+
+	class HDTCookie {
+		HDTCookieType type;
+		std::string format;
+		std::unordered_map<std::string, std::string> props{};
+	public:
+		using iterator = decltype(props)::iterator;
+		using const_iterator = decltype(props)::const_iterator;
+
+		HDTCookie(HDTCookieType type, std::string format) : type(type), format(format) {}
+		HDTCookie(std::istream& is) {
+			actslib::crc::CRC16 crc{};
+			char buffer[sizeof(HDT_COOKIE_MAGIC) - 1];
+			is.read(buffer, sizeof(HDT_COOKIE_MAGIC) - 1);
+
+			if (memcmp(buffer, HDT_COOKIE_MAGIC, sizeof(HDT_COOKIE_MAGIC) - 1)) {
+				throw std::runtime_error("Invalid HDT cookie magic");
+			}
+			crc.Update(buffer, 0, sizeof(HDT_COOKIE_MAGIC) - 1);
+
+			type = (HDTCookieType)is.get();
+			crc.Update((const char)type);
+
+			std::getline(is, format, '\0');
+			crc.Update(format.data(), 0, format.size() + 1);
+
+			std::string propsVal;
+			std::getline(is, propsVal, '\0');
+
+			crc.Update(propsVal.data(), 0, propsVal.size() + 1);
+
+			size_t off{};
+			while (off < propsVal.size()) {
+				size_t idx = propsVal.find(';', off);
+
+				if (idx == std::string::npos) {
+					idx = propsVal.size();
+				}
+
+				size_t idxeq = propsVal.find('=', off);
+
+				if (idxeq != std::string::npos && idxeq < idx) {
+					// split
+					props[propsVal.substr(off, idxeq - off)] = propsVal.substr(idxeq + 1, idx - idxeq - 1);
+				}
+
+				off = idx + 1;
+			}
+
+			if (!crc.CheckCRC(is)) {
+				throw std::runtime_error("Invalid HDT cookie CRC");
+			}
+		}
+
+		constexpr HDTCookieType GetType() const {
+			return type;
+		}
+
+		constexpr const std::string& GetFormat() const {
+			return format;
+		}
+
+		constexpr const std::unordered_map<std::string, std::string>& GetProperties() const {
+			return props;
+		}
+
+		iterator begin() {
+			return props.begin();
+		}
+
+		iterator end() {
+			return props.end();
+		}
+
+		const_iterator cbegin() const {
+			return props.begin();
+		}
+
+		const_iterator cend() const {
+			return props.cend();
+		}
+	};
+}