#5349. Implemented SHA256 checksum verification for downloaded missions.

Of course, it is done only if XML file with list of mission contains "sha256" attribute on "downloadLocation" or "localisationPack" element. 

idlib/hashing/sha256.*
  Since SHA256 is computed on-the-fly during download, and download itself is much slower anyway, I took simple standalone but unoptimized implementation of SHA256:
    https://github.com/B-Con/crypto-algorithms
LICENSE.txt:
  Since author wants acknowledgement, added it =)

game/Http/HttpRequest.*
  Added EnableSha256 and GetSha256 methods.
  Basically, if you call the first method, then checksum will be computed.

game/Missions/Download.*
  Added method VerifySha256Checksum, which receives expected sha256 and enables its validation when download is over.
  Added MALFORMED status for the case when checksum verification failed (want special error message).
  Also removed excessive constructor.

game/Missions/MissionManager.*
  LoadModListFromXml now reads and checks "sha256" field.
  Validation is perhaps not necessary, but better let it be.
  For that reason, LoadModListFromXml now returns bool (false if malformed XML).
  Also added MALFORMED status for requests by external users.

game/DownloadMenu.cpp:
  Enabled SHA256 verification for mission/localization packs in CDownloadMenu::StartDownload.
  Print custom error message if mission has wrong checksum.
  Print custom error message if missionlist XML is malformed.
  (the other requests just use error message for FAILED, since they don't return MALFORMED)


git-svn-id: http://svn.thedarkmod.com/svn/darkmod_src/trunk@9022 49c82d7f-2e2a-0410-a16f-ae8f201b507f

Author: stgatilov

LICENSE.txt

@@ -1138,3 +1138,19 @@ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY
 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
+
+
+sha256 implementation
+---------------------------------------------------------------------------
+idlib/hashing/sha256.*
+
+These are basic implementations of standard cryptography algorithms,
+written by Brad Conte ([email protected]) from scratch and without any cross-licensing.
+They exist to provide publically accessible, restriction-free implementations
+of popular cryptographic algorithms, like AES and SHA-1.
+
+This code is released into the public domain free of any restrictions.
+The author requests acknowledgement if the code is used, but does not require it.
+This code is provided free of any liability and without any quality claims by the author.
+
+Source: https://github.com/B-Con/crypto-algorithms

game/DownloadMenu.cpp

@@ -68,6 +68,23 @@ void CDownloadMenu::HandleCommands(const idStr& cmd, idUserInterface* gui)
 				}
 				break;
 
+				case CMissionManager::MALFORMED:
+				{
+					gui->HandleNamedEvent("onAvailableMissionsRefreshed"); // hide progress dialog
+
+					// Issue a failure message
+					gameLocal.Printf("Server response is incorrect\n");
+
+					GuiMessage msg;
+					msg.title = common->Translate( "#str_02147" );	// Malformed response
+					msg.message = common->Translate( "#str_02138" );	// The server returned wrong information.\nPlease report to maintainers.
+					msg.type = GuiMessage::MSG_OK;
+					msg.okCmd = "close_msg_box";
+
+					gameLocal.AddMainMenuMessage(msg);
+				}
+				break;
+
 				case CMissionManager::SUCCESSFUL:
 				{
 					gui->HandleNamedEvent("onAvailableMissionsRefreshed"); // hide progress dialog
@@ -90,6 +107,7 @@ void CDownloadMenu::HandleCommands(const idStr& cmd, idUserInterface* gui)
 
 			switch (status)
 			{
+				case CMissionManager::MALFORMED:
 				case CMissionManager::FAILED:
 				{
 					gui->HandleNamedEvent("onDownloadableMissionDetailsDownloadFailed"); // hide progress dialog
@@ -128,6 +146,7 @@ void CDownloadMenu::HandleCommands(const idStr& cmd, idUserInterface* gui)
 
 			switch (status)
 			{
+				case CMissionManager::MALFORMED:
 				case CMissionManager::FAILED:
 				{
 					gui->HandleNamedEvent("onFailedToDownloadScreenshot");
@@ -421,6 +440,7 @@ void CDownloadMenu::StartDownload(idUserInterface* gui)
 		}
 
 		CDownloadPtr download(new CDownload(mod.missionUrls, missionPath, true));
+		download->VerifySha256Checksum(mod.missionSha256);
         // gnartsch: In case only the language pack needs to be downloaded, do not add the mission itself to the download list.
 		//           In that case we did not add any urls for the mission itself anyway.
         int id = -1;
@@ -446,6 +466,7 @@ void CDownloadMenu::StartDownload(idUserInterface* gui)
 			DM_LOG(LC_MAINMENU, LT_INFO)LOGSTRING("Will download the l10n pack to %s.", l10nPackPath.c_str());
 
 			CDownloadPtr l10nDownload(new CDownload(mod.l10nPackUrls, l10nPackPath, true));
+			download->VerifySha256Checksum(mod.l10nPackSha256);
 
 			l10nId = gameLocal.m_DownloadManager->AddDownload(l10nDownload);
 
@@ -627,6 +648,8 @@ idStr CDownloadMenu::GetMissionDownloadProgressString(int modIndex)
 		return common->Translate( "#str_02180" );	// "queued "
 	case CDownload::FAILED:
 		return common->Translate( "#str_02181" );	// "failed "
+	case CDownload::MALFORMED:
+		return common->Translate( "#str_02518" );	// "malformed "
 	case CDownload::IN_PROGRESS:
 	{
 		double totalFraction = download->GetProgressFraction(); 
@@ -657,7 +680,7 @@ void CDownloadMenu::ShowDownloadResult(idUserInterface* gui)
 
 	int successfulDownloads = 0;
 	int failedDownloads = 0;
-
+	int malformedDownloads = 0;
 	int canceledDownloads = 0; //Agent Jones
 
 	const DownloadableModList& mods = gameLocal.m_MissionManager->GetDownloadableMods();
@@ -680,6 +703,9 @@ void CDownloadMenu::ShowDownloadResult(idUserInterface* gui)
 		case CDownload::FAILED:
 			failedDownloads++;
 			break;
+		case CDownload::MALFORMED:
+			malformedDownloads++;
+			break;
 		case CDownload::CANCELED:
 			canceledDownloads++;//Agent Jones Test
 			break;
@@ -748,14 +774,19 @@ void CDownloadMenu::ShowDownloadResult(idUserInterface* gui)
 			GetPlural(successfulDownloads, common->Translate("#str_02144"), common->Translate("#str_02145")),
 			successfulDownloads);
 	}
-
 	if (failedDownloads > 0)
 	{
 		// "\n%d mission(s) couldn't be downloaded. Please check your disk space (or maybe some file is write protected) and try again."
 		msg.message += va(common->Translate("#str_02146"),
 			failedDownloads);
 	}
-	if (failedDownloads > 0 || successfulDownloads > 0)//Agent Jones Test
+	if (malformedDownloads > 0)
+	{
+		//"\n%d mission(s) are wrong on mirrors.\nPlease report to maintainers."
+		msg.message += va(common->Translate("#str_02693"), malformedDownloads);
+	}
+
+	if (failedDownloads > 0 || successfulDownloads > 0 || malformedDownloads > 0)//Agent Jones Test
 		gameLocal.AddMainMenuMessage(msg);
 	else{
 		//AJ test

game/Http/HttpRequest.cpp

@@ -31,6 +31,8 @@
 
 #include "curl/curl.h"
 #define ExtLibs
+#include "hashing/sha256.h"
+
 
 CHttpRequest::CHttpRequest(CHttpConnection& conn, const std::string& url) :
 	_conn(&conn),
@@ -51,6 +53,28 @@ CHttpRequest::CHttpRequest(CHttpConnection& conn, const std::string& url, const
 	_progress(0)
 {}
 
+CHttpRequest::~CHttpRequest() {}
+
+void CHttpRequest::EnableSha256()
+{
+	_computeSha256 = true;
+}
+
+idStr CHttpRequest::GetSha256() const 
+{
+	if (!_sha256state)
+		return "";
+	uint8_t digest[32];
+	sha256_final(_sha256state.get(), digest);
+	idStr res;
+	for (int i = 0; i < 32; i++) {
+		char tmp[8];
+		idStr::snPrintf(tmp, sizeof(tmp), "%02x", digest[i]);
+		res += tmp;
+	}
+	return res;
+}
+
 // Agent Jones #3766
 int CHttpRequest::TDMHttpProgressFunc(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
 {
@@ -154,6 +178,12 @@ void CHttpRequest::Perform()
 		_destStream.open(_destFilename.c_str(), std::ofstream::out|std::ofstream::binary);
 	}
 
+	if (_computeSha256)
+	{
+		_sha256state.reset(new SHA256_CTX());
+		sha256_init(_sha256state.get());
+	}
+
 	CURLcode result = ExtLibs::curl_easy_perform( _handle );
 
 	if (!_destFilename.empty())
@@ -271,6 +301,11 @@ size_t CHttpRequest::WriteMemoryCallback(void* ptr, size_t size, size_t nmemb, C
 
 	self->UpdateProgress();
 
+	if (self->_computeSha256)
+	{
+		sha256_update(self->_sha256state.get(), (uint8_t*)ptr, bytesToCopy);
+	}
+
 	return static_cast<size_t>(bytesToCopy);
 }
 
@@ -288,6 +323,11 @@ size_t CHttpRequest::WriteFileCallback(void* ptr, size_t size, size_t nmemb, CHt
 
 	self->UpdateProgress();
 
+	if (self->_computeSha256)
+	{
+		sha256_update(self->_sha256state.get(), (uint8_t*)ptr, bytesToCopy);
+	}
+
 	return static_cast<size_t>(bytesToCopy);
 }
 

game/Http/HttpRequest.h

@@ -26,6 +26,8 @@ typedef void CURL;
 // Shared_ptr typedef
 typedef std::shared_ptr<pugi::xml_document> XmlDocumentPtr;
 
+typedef struct SHA256_CTX SHA256_CTX;
+
 /**
  * greebo: An object representing a single HttpRequest, holding 
  * the result (string) and status information.
@@ -69,11 +71,18 @@ class CHttpRequest
 
 	double _progress;
 
+	bool _computeSha256 = false;
+	std::unique_ptr<SHA256_CTX> _sha256state;
+
 public:
 	CHttpRequest(CHttpConnection& conn, const std::string& url);
 
 	CHttpRequest(CHttpConnection& conn, const std::string& url, const std::string& destFilename);
 
+	~CHttpRequest();
+
+	void EnableSha256();
+
 	// Callbacks for CURL
 	static size_t WriteMemoryCallback(void* ptr, size_t size, size_t nmemb, CHttpRequest* self);
 	static size_t WriteFileCallback(void* ptr, size_t size, size_t nmemb, CHttpRequest* self);
@@ -96,6 +105,7 @@ class CHttpRequest
 	// Returns the result as XML document
 	XmlDocumentPtr GetResultXml();
 
+	idStr GetSha256() const;
 private:
 	void InitRequest();
 

game/Missions/Download.cpp

@@ -23,15 +23,6 @@
 #include "../ZipLoader/ZipLoader.h"
 #include "MissionManager.h"
 
-CDownload::CDownload(const idStr& url, const idStr& destFilename, bool enablePK4check) :
-	_curUrl(0),
-	_destFilename(destFilename),
-	_status(NOT_STARTED_YET),
-	_pk4CheckEnabled(enablePK4check),
-	_relatedDownload(-1)
-{
-	_urls.Append(url);
-}
 
 CDownload::CDownload(const idStringList& urls, const idStr& destFilename, bool enablePK4check) :
 	_urls(urls),
@@ -47,6 +38,11 @@ CDownload::~CDownload()
 	Stop(false);
 }
 
+void CDownload::VerifySha256Checksum(const idStr &sha256)
+{
+	_expectedSha256 = sha256;
+}
+
 CDownload::DownloadStatus CDownload::GetStatus()
 {
 	return _status;
@@ -134,13 +130,29 @@ void CDownload::Perform()
 
 		// Create a new request
 		_request = gameLocal.m_HttpConnection->CreateRequest(url.c_str(), _tempFilename.c_str());
+		// stgatilov: if we have checksum to verify, enable its computation on-the-fly
+		if (_expectedSha256.Length())
+			_request->EnableSha256();
 
 		// Start the download, blocks until finished or aborted
 		_request->Perform();
 
 		if (_request->GetStatus() == CHttpRequest::OK)
 		{
-			// Check the downloaded file
+			// stgatilov: Verify checksum if specified
+			if (_expectedSha256.Length())
+			{
+				idStr computedSha256 = _request->GetSha256();
+				if (computedSha256 != _expectedSha256)
+				{
+					DM_LOG(LC_MAINMENU, LT_DEBUG)LOGSTRING("Wrong checksum at '%s': %s instead of %s", url.c_str(), computedSha256.c_str(), _expectedSha256.c_str());
+					CMissionManager::DoRemoveFile(_tempFilename.c_str());
+					_status = MALFORMED;
+					break;
+				}
+			}
+
+			// Check that downloaded file is a zip indeed
 			if (_pk4CheckEnabled)
 			{
 				bool valid = CheckValidPK4(_tempFilename);

game/Missions/Download.h

@@ -61,6 +61,7 @@ class CDownload
 		FAILED,
 		SUCCESS,
 		CANCELED,//Agent Jones
+		MALFORMED,	//stgatilov: checksum invalid
 	};
 
 	// Some additional data which can be set by clients
@@ -87,19 +88,14 @@ class CDownload
 	ThreadPtr _thread;
 
 	bool _pk4CheckEnabled;
+	idStr _expectedSha256;	//empty if check is ignored
 
 	UserData _userData;
 
 	// The ID of the related download (mission + l10n)
 	int	_relatedDownload;
 
 public:
-	/** 
-	 * greebo: Construct a new Download using the given URL.
-	 * The download data will be saved to the given destFilename;
-	 */
-	CDownload(const idStr& url, const idStr& destFilename, bool enablePK4check = false);
-
 	/**
 	 * greebo: Construct a new Download using the given list of 
 	 * alternative URLs. If downloading from the first URL fails
@@ -110,6 +106,11 @@ class CDownload
 
 	~CDownload();
 
+	// stgatilov: sha256 must be valid 64-digit hex number or empty string.
+	// If empty, then sha256 check is disabled (default).
+	// If not empty, then sha256 checksum of downloaded file will be computed and verified after download.
+	void VerifySha256Checksum(const idStr &sha256);
+
 	// Start the download. This will spawn a new thread and execution
 	// will return to the calling code.
 	void Start();