CodeProject: Convert between std::string and std::wstring, UTF-8 and UTF-16. Free source code and programming help

Report Article

Discuss

Send to a friend

11 votes for this Article.

Popularity: 3.69 Rating: 3.55 out of 5

Note: This is an unedited contribution. If this article is inappropriate, needs attention or copies someone else's work without reference then please Report This Article

Download source code - 0.9 Kb

Introduction

I needed to convert between UTF-8 coded std::string and UTF-16 coded std::wstring. I found here and there converting functions for native C strings, but these leave the memory handling to the caller. Not nice in modern times.

The best converter is probably the one from unicode.org. Here is a wrapper around this one which converts the STL strings.

Unlike other articles, this one has no other dependencies, does not introduce yet another string class, it only converts the STL strings, and that's it. And it's better than the widely found

std::wstring widestring(sourcestring.begin(), sourcestring.end());

which only works for Ascii text.

Source

The header goes like this:

#ifndef UTFCONVERTER__H__
#define UTFCONVERTER__H__

namespace UtfConverter
{
    std::wstring FromUtf8(const std::string& utf8string);
    std::string ToUtf8(const std::wstring& widestring);
}

#endif

I guess this is simple and easy enough to use.

Here is the source code:

#include "stdafx.h"

#include "UtfConverter.h"

#include "ConvertUTF.h"


namespace UtfConverter
{

    std::wstring FromUtf8(const std::string& utf8string)
    {
        size_t widesize = utf8string.length();
        if (sizeof(wchar_t) == 2)
        {
            wchar_t* widestringnative = new wchar_t[widesize+1];
            const UTF8* sourcestart = reinterpret_cast<const UTF8*>(utf8string.c_str());
            const UTF8* sourceend = sourcestart + widesize;
            UTF16* targetstart = reinterpret_cast<UTF16*>(widestringnative);
            UTF16* targetend = targetstart + widesize+1;
            ConversionResult res = ConvertUTF8toUTF16(&sourcestart, sourceend, &targetstart, targetend, strictConversion);
            if (res != conversionOK)
            {
                delete [] widestringnative;
                throw std::exception("La falla!");
            }
            *targetstart = 0;
            std::wstring resultstring(widestringnative);
            delete [] widestringnative;
            return resultstring;
        }
        else if (sizeof(wchar_t) == 4)
        {
            wchar_t* widestringnative = new wchar_t[widesize];
            const UTF8* sourcestart = reinterpret_cast<const UTF8*>(utf8string.c_str());
            const UTF8* sourceend = sourcestart + widesize;
            UTF32* targetstart = reinterpret_cast<UTF32*>(widestringnative);
            UTF32* targetend = targetstart + widesize;
            ConversionResult res = ConvertUTF8toUTF32(&sourcestart, sourceend, &targetstart, targetend, strictConversion);
            if (res != conversionOK)
            {
                delete [] widestringnative;
                throw std::exception("La falla!");
            }
            *targetstart = 0;
            std::wstring resultstring(widestringnative);
            delete [] widestringnative;
            return resultstring;
        }
        else
        {
            throw std::exception("La falla!");
        }
        return L"";
    }

    std::string ToUtf8(const std::wstring& widestring)
    {
        size_t widesize = widestring.length();

        if (sizeof(wchar_t) == 2)
        {
            size_t utf8size = 3 * widesize + 1;
            char* utf8stringnative = new char[utf8size];
            const UTF16* sourcestart = reinterpret_cast<const UTF16*>(widestring.c_str());
            const UTF16* sourceend = sourcestart + widesize;
            UTF8* targetstart = reinterpret_cast<UTF8*>(utf8stringnative);
            UTF8* targetend = targetstart + utf8size;
            ConversionResult res = ConvertUTF16toUTF8(&sourcestart, sourceend, &targetstart, targetend, strictConversion);
            if (res != conversionOK)
            {
                delete [] utf8stringnative;
                throw std::exception("La falla!");
            }
            *targetstart = 0;
            std::string resultstring(utf8stringnative);
            delete [] utf8stringnative;
            return resultstring;
        }
        else if (sizeof(wchar_t) == 4)
        {
            size_t utf8size = 4 * widesize + 1;
            char* utf8stringnative = new char[utf8size];
            const UTF32* sourcestart = reinterpret_cast<const UTF32*>(widestring.c_str());
            const UTF32* sourceend = sourcestart + widesize;
            UTF8* targetstart = reinterpret_cast<UTF8*>(utf8stringnative);
            UTF8* targetend = targetstart + utf8size;
            ConversionResult res = ConvertUTF32toUTF8(&sourcestart, sourceend, &targetstart, targetend, strictConversion);
            if (res != conversionOK)
            {
                delete [] utf8stringnative;
                throw std::exception("La falla!");
            }
            *targetstart = 0;
            std::string resultstring(utf8stringnative);
            delete [] utf8stringnative;
            return resultstring;
        }
        else
        {
            throw std::exception("La falla!");
        }
        return "";
    }
}

How to do it better

Here's another version that avoids using new and delete, by writing directly into the string buffer. Does anyone know whether this is okay?

#include "stdafx.h"

#include "UtfConverter.h"

#include "ConvertUTF.h"


namespace UtfConverter
{

    std::wstring FromUtf8(const std::string& utf8string)
    {
        size_t widesize = utf8string.length();
        if (sizeof(wchar_t) == 2)
        {
            std::wstring resultstring;
            resultstring.resize(widesize+1, L'\0');
            const UTF8* sourcestart = reinterpret_cast<const UTF8*>(utf8string.c_str());
            const UTF8* sourceend = sourcestart + widesize;
            UTF16* targetstart = reinterpret_cast<UTF16*>(&resultstring[0]);
            UTF16* targetend = targetstart + widesize;
            ConversionResult res = ConvertUTF8toUTF16(&sourcestart, sourceend, &targetstart, targetend, strictConversion);
            if (res != conversionOK)
            {
                throw std::exception("La falla!");
            }
            *targetstart = 0;
            return resultstring;
        }
        else if (sizeof(wchar_t) == 4)
        {
            std::wstring resultstring;
            resultstring.resize(widesize+1, L'\0');
            const UTF8* sourcestart = reinterpret_cast<const UTF8*>(utf8string.c_str());
            const UTF8* sourceend = sourcestart + widesize;
            UTF32* targetstart = reinterpret_cast<UTF32*>(&resultstring[0]);
            UTF32* targetend = targetstart + widesize;
            ConversionResult res = ConvertUTF8toUTF32(&sourcestart, sourceend, &targetstart, targetend, strictConversion);
            if (res != conversionOK)
            {
                throw std::exception("La falla!");
            }
            *targetstart = 0;
            return resultstring;
        }
        else
        {
            throw std::exception("La falla!");
        }
        return L"";
    }

    std::string ToUtf8(const std::wstring& widestring)
    {
        size_t widesize = widestring.length();

        if (sizeof(wchar_t) == 2)
        {
            size_t utf8size = 3 * widesize + 1;
            std::string resultstring;
            resultstring.resize(utf8size, '\0');
            const UTF16* sourcestart = reinterpret_cast<const UTF16*>(widestring.c_str());
            const UTF16* sourceend = sourcestart + widesize;
            UTF8* targetstart = reinterpret_cast<UTF8*>(&resultstring[0]);
            UTF8* targetend = targetstart + utf8size;
            ConversionResult res = ConvertUTF16toUTF8(&sourcestart, sourceend, &targetstart, targetend, strictConversion);
            if (res != conversionOK)
            {
                throw std::exception("La falla!");
            }
            *targetstart = 0;
            return resultstring;
        }
        else if (sizeof(wchar_t) == 4)
        {
            size_t utf8size = 4 * widesize + 1;
            std::string resultstring;
            resultstring.resize(utf8size, '\0');
            const UTF32* sourcestart = reinterpret_cast<const UTF32*>(widestring.c_str());
            const UTF32* sourceend = sourcestart + widesize;
            UTF8* targetstart = reinterpret_cast<UTF8*>(&resultstring[0]);
            UTF8* targetend = targetstart + utf8size;
            ConversionResult res = ConvertUTF32toUTF8(&sourcestart, sourceend, &targetstart, targetend, strictConversion);
            if (res != conversionOK)
            {
                throw std::exception("La falla!");
            }
            *targetstart = 0;
            return resultstring;
        }
        else
        {
            throw std::exception("La falla!");
        }
        return "";
    }
}

<> >

How to use it

Just add them to your project. Download the unicode converter from http://www.unicode.org/Public/PROGRAMS/CVTUTF/ and add that to the project, too. It should just work.

Of course you can throw whatever exception you like upon failure.

I must admit I tried it only for 2-byte wchar_t.

Comments are welcome.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

About the Author

rh_

Location:

Germany

Other popular String handling articles:

The Complete Guide to C++ Strings, Part I - Win32 Character Encodings
A guide to the multitude of string types used in Windows.
The Complete Guide to C++ Strings, Part II - String Wrapper Classes
A guide to the string wrapper classes provided by Visual C++ and class libraries
CString Management
Learn how to effectively use CStrings.
CString-clone Using Standard C++
A Drop-In replacement for CString that builds on the Standard C++ Library's basic_string template
Wildcard string compare (globbing)
Matches a string against a wildcard string such as "*.*" or "bl?h.*" etc. This is good for file globbing or to match hostmasks.

Article Top

You must Sign In to use this message board.

Msgs 1 to 25 of 25 (Total in Forum: 25) (Refresh)

FirstPrevNext

The easiest way to do the same conversion

steveb

10:21 23 Oct '08

To std::string:


str.assign(ws.begin(), ws.end());

To std::wstring


ws.assign(str.begin(), str.end());

1.00/5 (1 vote)

trouble appending string to conversion

Member 4574348

3:53 18 Oct '08

Having read your article I replaced the conversion I was using in some existing code compiled with VS2005 running on XP:

std::string szName = typeid(*this).name();
std::wstring wszName(szName.length(), L' ');
std::copy(szName.begin(),szName.end(),wszName.begin());
wszName += L" some message\n";
ATLTRACE(wszName.c_str());

with:

std::wstring wszName = FromUtf8(szName);

but using the new conversion don't see the second string appended as it was with the original (limited) conversion.

Jonathan

Incorrect size set in ToUTF8

DEmberton

3:55 19 Jun '08

I've no idea what this calculation is meant to acheive: size_t utf8size = 3 * widesize + 1; I was using the returned string in a comparison and it failed despite the other string being identical because the size set in the string is wrong (for a 3 character string the above sets it to 10). Simple change to size_t utf8size = widesize; And now it works.
Sign In·View Thread·PermaLink

Re: Incorrect size set in ToUTF8

peterchen

5:47 21 Aug '08

One "wide" character may require more than one UTF-8 characters.
If you are using
size_t utf8size = widesize;
this works very well with strings only containing ASCII characters, but as soon as you go beyond that, you can expect buffer overruns.

We are a big screwed up dysfunctional psychotic happy family - some more screwed up, others more happy, but everybody's psychotic joint venture definition of CP
blog: TDD - the Aha! | Linkify!| FoldWithUs! | sighist

thank you

wipehindy

13:59 15 Feb '08

just thank you for this code
Sign In·View Thread·PermaLink

what is “L” in：resultstring.resize(widesize+1, L'\0');

Eva ranee

21:34 6 Jan '08

The "L" means long type, is right? Follow the C++ technical
Sign In·View Thread·PermaLink

Re: what is “L” in：resultstring.resize(widesize+1, L'\0');

Mircea Puiu

0:41 7 Jan '08

unicode SkyWalker
Sign In·View Thread·PermaLink

UNICODE is not the same as UTF16

christophe.hermier@quickfds.com

22:55 30 Sep '07

> Unicode is an encoding standard in which all characters are two bytes long

NO ! NO ! NO ! Please stop spreading the confusion

Unicode is a repertoire of all characters.

What Microsoft calls "UNICODE" is UTF-16, that is just one way to convert a unicode number in the unicode repertoire into bytes.
(As you may guess it introduces "endian" troubles)

UTF-8 is another way to convert unicode numbers to bytes...

4.00/5 (3 votes)

CA2T, CA2W

kpnut

5:35 30 Aug '07

If this is Windows, wouldn't CA2W, CA2T etc do what you need? Ken Davis
Sign In·View Thread·PermaLink

Is this OK?

Stephen Hewitt

22:12 20 May '07

From your code:

UTF16* targetstart = reinterpret_cast(&resultstring[0]);
UTF16* targetend = targetstart + widesize;

In general this is not OK. The string interface is crafted to allow optimizations. One such optimization is to allocate the string’s buffer in chunks to make insertion in the middle of the string more efficient and to avoid the need for copying data when is string’s size increases. If this is the case then c_str copies the data into one big buffer which hangs around until the string is destroyed or a non-const method is called; but there is no requirement for the data to be copied back. This hack may work on some specific STL implementations (perhaps even most) but is not portable.

Steve

Bug in the code?

_ema_

14:51 10 May '07

Unless I am using a different version of the library from unicode.org, otherwise there could be a bug in the sample.

In the FromUtf8 method, I think it should allocate an extra char for widestringnative (e.g. 'new wchar_t[widesize+1]' instead of 'new wchar_t[widesize]'). Otherwise later on the line '*targetstart = 0' will be writing '0' to the memory *after* widestringnative.

It would be nice if rh_ (or others) can confirm this. Thanks!

Re: Bug in the code?

rh_

21:32 20 May '07

I think you are right. This demonstrates nicely the reason why the other version which avoids using new and delete is better.
Sign In·View Thread·PermaLink

Simplest way

tracker2002

5:06 13 Feb '07

std::wstring string_to_wstring(const std::string &strSingleByte) { xiwstring strUnicode; strUnicode.resize(strSingleByte.size()); std::transform(strSingleByte.begin(), strSingleByte.end(), strUnicode.begin(), btowc); return strUnicode; }
Sign In·View Thread·PermaLink

Re: Simplest way

rh_

21:04 13 Feb '07

This appears interesting, but it surely is locale-dependent. Also, as I understand from the manpages of btowc and wctob, they cannot handle multi-byte sequences. So this is a good way to convert, say, ISO-8859-15 or Windows-1252 to UTF-16.

For converting multi-byte sequences, we would need a function with an internal finite automaton, using a static variable or, probably better, a transformation class defining an operator() and remembering the previous bytes of the current milti-byte sequence privately.

Here is a page which sketches a possible solution: http://www.codeguru.com/forum/archive/index.php/t-340943.html

UTF-8 and Multibyte are not the same

Ted Dunlop

6:59 12 Feb '07

UTF-8 encoding is not the same as multi-byte character encoding. The major problem that you will encounter using either the C++ conversions or the Windows conversions is the need for locales supporting the conversion. You cannot guareentee that your client will have the required locales and you may get an unexpected result.

The unicode implementation is great for converting between UTF-8 and UTF-16. A generic solution for multi-byte to UTF-16 and back is the ICU (http://icu.sourceforge.net) which also supports complex text layout and localization. Since it lets you add localization tables it can handle almost any conversion scenario imaginable.

If you want to convert between UTF-8 and UTF-16 use the unicode consortium's implementation. If you want to convert between multi-byte and unicode, make sure that you have the correct locales or use the International Componets for Unicode.

1.00/5 (1 vote)

Re: UTF-8 and Multibyte are not the same

rh_

1:21 13 Feb '07

The online help says that the functions WideCharToMultiByte and MultiByteToWideChar support the parameter CP_UTF8 under "Windows 98/Me, Windows NT 4.0 and later". Why should they depend on the locale when called with CP_UTF8? I thought UTF-8 is UTF-8.

Anyway, they need "#include ", and if we have some general doubt about how they are implemented, the unicode consortium routines may be the wiser choice.

Thank you for the link to ICU! For my current task this would be overkill, but it's good to know.

MultiByteToWideChar and WideCharToMultiByte...

Johann Gerell

20:27 11 Feb '07

...are all you need! Of course, they are not cross-platform, so away from Windows, your solution has its place. -- Time you enjoy wasting is not wasted time - Bertrand Russel
Sign In·View Thread·PermaLink

Re: MultiByteToWideChar and WideCharToMultiByte...

rh_

23:37 11 Feb '07

Well, yes and no. MultiByteToWideChar and WideCharToMultiByte also need a buffer with a fixed size which must be provided before. Certainly you can replace ConvertUTF16toUTF8 by WideCharToMultiByte etc. in my code, such that you don't need to download the stuff from unicode.org.
Sign In·View Thread·PermaLink

Re: MultiByteToWideChar and WideCharToMultiByte...

Johann Gerell

23:54 11 Feb '07

rh_ wrote: need a buffer with a fixed size which must be provided before Nope. They can first be queried about the needed space. -- Time you enjoy wasting is not wasted time - Bertrand Russel
Sign In·View Thread·PermaLink

Re: MultiByteToWideChar and WideCharToMultiByte...

rh_

2:32 12 Feb '07

Ah, that's nice. This would call for an implementation like this:


#include "stdafx.h"
#include "UtfConverter.h"

namespace UtfConverter
{

	std::wstring FromUtf8(const std::string& utf8string)
	{
		int widesize = ::MultiByteToWideChar(CP_UTF8, 0, utf8string.c_str(), -1, NULL, 0);
		if (widesize == ERROR_NO_UNICODE_TRANSLATION)
		{
			throw std::exception("Invalid UTF-8 sequence.");
		}
		if (widesize == 0)
		{
			throw std::exception("Error in conversion.");
		}

		std::wstring resultstring;
		resultstring.resize(widesize, L'\0');

		int convresult = ::MultiByteToWideChar(CP_UTF8, 0, utf8string.c_str(), -1, reinterpret_cast(&resultstring[0]), widesize);

		if (convresult != widesize)
		{
			throw std::exception("La falla!");
		}

		return resultstring;
	}

	std::string ToUtf8(const std::wstring& widestring)
	{
		int utf8size = ::WideCharToMultiByte(CP_UTF8, 0, widestring.c_str(), -1, NULL, 0, NULL, NULL);
		if (utf8size == 0)
		{
			throw std::exception("Error in conversion.");
		}

		std::string resultstring;
		resultstring.resize(utf8size, '\0');

		int convresult = ::WideCharToMultiByte(CP_UTF8, 0, widestring.c_str(), -1, reinterpret_cast(&resultstring[0]), utf8size, NULL, NULL);

		if (convresult != utf8size)
		{
			throw std::exception("La falla!");
		}

		return resultstring;
	}
}

Re: MultiByteToWideChar and WideCharToMultiByte...

Johann Gerell

2:49 12 Feb '07

Something like that, yes...

rh_ wrote:
std::string resultstring;
resultstring.resize(utf8size, L'\0');

int convresult = ::WideCharToMultiByte(CP_UTF8, 0, widestring.c_str(), -1, reinterpret_cast(&resultstring[0]), utf8size, NULL, NULL);

...but not like that.

Use a vector<char> instead of the string. Never play with a string as a consecutive sequence of characters like that. That's what vector is for. The same goes for wstring higher up. And set the size directly in the ctor instead. No need for the second argument, since 0 is the default value.

--
Time you enjoy wasting is not wasted time - Bertrand Russel

Re: MultiByteToWideChar and WideCharToMultiByte...

rh_

3:34 12 Feb '07

Codeproject is really a great place to learn good coding!

So here is the improved, safer version which uses vector. Also nice that I could throw out the ugly reinterpret_cast.

#include "stdafx.h"
#include "UtfConverter.h"
#include <vector>

namespace UtfConverter
{

	std::wstring FromUtf8(const std::string& utf8string)
	{
		int widesize = ::MultiByteToWideChar(CP_UTF8, 0, utf8string.c_str(), -1, NULL, 0);
		if (widesize == ERROR_NO_UNICODE_TRANSLATION)
		{
			throw std::exception("Invalid UTF-8 sequence.");
		}
		if (widesize == 0)
		{
			throw std::exception("Error in conversion.");
		}

		std::vector<wchar_t> resultstring(widesize);

		int convresult = ::MultiByteToWideChar(CP_UTF8, 0, utf8string.c_str(), -1, &resultstring[0], widesize);

		if (convresult != widesize)
		{
			throw std::exception("La falla!");
		}

		return std::wstring(&resultstring[0]);
	}

	std::string ToUtf8(const std::wstring& widestring)
	{
		int utf8size = ::WideCharToMultiByte(CP_UTF8, 0, widestring.c_str(), -1, NULL, 0, NULL, NULL);
		if (utf8size == 0)
		{
			throw std::exception("Error in conversion.");
		}

		std::vector<char> resultstring(utf8size);

		int convresult = ::WideCharToMultiByte(CP_UTF8, 0, widestring.c_str(), -1, &resultstring[0], utf8size, NULL, NULL);

		if (convresult != utf8size)
		{
			throw std::exception("La falla!");
		}

		return std::string(&resultstring[0]);
	}
}

Re: MultiByteToWideChar and WideCharToMultiByte...

Bjornar

4:44 19 Feb '07

Good. The need for reinterpret_cast is usually the result of a architectual mishap
Sign In·View Thread·PermaLink	5.00/5 (1 vote)

Re: MultiByteToWideChar and WideCharToMultiByte...

konmel

7:53 7 Nov '07

How about this? std::string str = "Hello"; std::wstring str2(str.length(), L' '); std::copy(str.begin(), str.end(), str2.begin());
Sign In·View Thread·PermaLink

Re: MultiByteToWideChar and WideCharToMultiByte...

chipmunk

12:25 7 Aug '08

thank you very much !
Sign In·View Thread·PermaLink

Last Visit: Monday, December 1, 2008 7:56 PM Last Update:Monday, December 1, 2008 7:56 PM

General News Question Answer Joke Rant Admin