refactor(string): Add functions for handling UTF8 encoded strings

Core/GameEngine/Include/Common/AsciiString.h

@@ -364,6 +364,16 @@ class AsciiString
 	Bool isNotEmpty() const { return !isEmpty(); }
 	Bool isNotNone() const { return !isNone(); }
 
+	/**
+		return true iff the string is valid UTF-8.
+	*/
+	Bool isValidUtf8() const;
+
+	/**
+		convert the given UnicodeString to UTF-8 and store it in self.
+	*/
+	void convertToUtf8(const UnicodeString& unicodeStr);
+
 //
 // You might think it would be a good idea to overload the * operator
 // to allow for an implicit conversion to an char*. This is

Core/GameEngine/Include/Common/UnicodeString.h

@@ -340,6 +340,11 @@ class UnicodeString
 	*/
 	Bool nextToken(UnicodeString* token, UnicodeString delimiters = UnicodeString::TheEmptyString);
 
+	/**
+		convert the given UTF-8 encoded AsciiString to WideChar and store it in self.
+	*/
+	void convertFromUtf8(const AsciiString& asciiStr);
+
 //
 // You might think it would be a good idea to overload the * operator
 // to allow for an implicit conversion to an WideChar*. This is

Core/GameEngine/Source/Common/System/AsciiString.cpp

@@ -45,6 +45,7 @@
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
 
 #include "Common/CriticalSection.h"
+#include "utf8.h"
 
 
 // -----------------------------------------------------
@@ -534,3 +535,24 @@ Bool AsciiString::nextToken(AsciiString* tok, const char* seps)
 		return false;
 	}
 }
+
+//-----------------------------------------------------------------------------
+Bool AsciiString::isValidUtf8() const
+{
+	return utf8_validate_string(str());
+}
+
+//-----------------------------------------------------------------------------
+void AsciiString::convertToUtf8(const UnicodeString& unicodeStr)
+{
+	const size_t requiredBytes = get_size_as_utf8(unicodeStr.str());
+	if (requiredBytes == 0)
+	{
+		DEBUG_CRASH(("AsciiString::convertToUtf8: failed to get size as UTF-8"));
+		return;
+	}
+
+	ensureUniqueBufferOfSize(static_cast<int>(requiredBytes), false, NULL, NULL);
+	convert_widechar_to_utf8(unicodeStr.str(), peek(), requiredBytes);
+	validate();
+}

Core/GameEngine/Source/Common/System/UnicodeString.cpp

@@ -45,6 +45,7 @@
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
 
 #include "Common/CriticalSection.h"
+#include "utf8.h"
 
 
 // -----------------------------------------------------
@@ -465,3 +466,18 @@ Bool UnicodeString::nextToken(UnicodeString* tok, UnicodeString delimiters)
 		return false;
 	}
 }
+
+//-----------------------------------------------------------------------------
+void UnicodeString::convertFromUtf8(const AsciiString& asciiStr)
+{
+	const size_t requiredBytes = get_size_as_widechar(asciiStr.str());
+	if (requiredBytes == 0)
+	{
+		DEBUG_CRASH(("UnicodeString::convertFromUtf8: failed to get size as WideChar"));
+		return;
+	}
+
+	ensureUniqueBufferOfSize(static_cast<int>(requiredBytes / sizeof(WideChar)), false, NULL, NULL);
+	convert_utf8_to_widechar(asciiStr.str(), peek(), requiredBytes);
+	validate();
+}

Core/GameEngine/Source/GameNetwork/GameInfo.cpp

@@ -44,6 +44,7 @@
 #include "GameNetwork/LANAPI.h"						// for testing packet size
 #include "GameNetwork/LANAPICallbacks.h"	// for testing packet size
 #include "strtok_r.h"
+#include "utf8.h"
 
 
 
@@ -1032,6 +1033,11 @@ Bool ParseAsciiStringToGameInfo(GameInfo *game, AsciiString options)
 	//DEBUG_LOG(("Saw options of %s", options.str()));
 	DEBUG_LOG(("ParseAsciiStringToGameInfo - parsing [%s]", options.str()));
 
+	if (!options.isValidUtf8())
+	{
+		DEBUG_LOG(("ParseAsciiStringToGameInfo - options string is not valid UTF-8"));
+		return false;
+	}
 
 	while ( (keyValPair = strtok_r(bufPtr, ";", &strPos)) != NULL )
 	{
@@ -1167,8 +1173,9 @@ Bool ParseAsciiStringToGameInfo(GameInfo *game, AsciiString options)
 								DEBUG_LOG(("ParseAsciiStringToGameInfo - slotValue name is empty, quitting"));
 								break;
 							}
+
 							UnicodeString name;
-              				name.set(MultiByteToWideCharSingleLine(slotValue.str() +1).c_str());
+							name.convertFromUtf8(slotValue.str() + 1);
 
 							//DEBUG_LOG(("ParseAsciiStringToGameInfo - name is %s", slotValue.str()+1));
 

Core/GameEngine/Source/GameNetwork/GameSpy/Thread/ThreadUtils.cpp

@@ -27,15 +27,16 @@
 // Author: Matthew D. Campbell, July 2002
 
 #include "PreRTS.h"	// This must go first in EVERY cpp file in the GameEngine
+#include "utf8.h"
 
 //-------------------------------------------------------------------------
 
 std::wstring MultiByteToWideCharSingleLine( const char *orig )
 {
-	Int len = strlen(orig);
-	WideChar *dest = NEW WideChar[len+1];
+	const size_t size = get_size_as_widechar( orig );
+	WideChar *dest = NEW WideChar[size / sizeof(WideChar)];
 
-	MultiByteToWideChar(CP_UTF8, 0, orig, -1, dest, len);
+	convert_utf8_to_widechar( orig, dest, size );
 	WideChar *c = NULL;
 	do
 	{
@@ -56,7 +57,6 @@ std::wstring MultiByteToWideCharSingleLine( const char *orig )
 	}
 	while ( c != NULL );
 
-	dest[len] = 0;
 	std::wstring ret = dest;
 	delete[] dest;
 	return ret;
@@ -65,12 +65,11 @@ std::wstring MultiByteToWideCharSingleLine( const char *orig )
 std::string WideCharStringToMultiByte( const WideChar *orig )
 {
 	std::string ret;
-	Int len = WideCharToMultiByte( CP_UTF8, 0, orig, wcslen(orig), NULL, 0, NULL, NULL ) + 1;
-	if (len > 0)
+	const size_t size = get_size_as_utf8( orig );
+	if (size > 0)
 	{
-		char *dest = NEW char[len];
-		WideCharToMultiByte( CP_UTF8, 0, orig, -1, dest, len, NULL, NULL );
-		dest[len-1] = 0;
+		char *dest = NEW char[size];
+		convert_widechar_to_utf8( orig, dest, size );
 		ret = dest;
 		delete[] dest;
 	}

Core/Libraries/Source/WWVegas/WWLib/CMakeLists.txt

@@ -133,6 +133,8 @@ set(WWLIB_SRC
     trim.cpp
     trim.h
     uarray.h
+    utf8.h
+    utf8.cpp
     vector.cpp
     Vector.h
     visualc.h