A UTF-16 Class for Reading and Writing Unicode Files

// UTF16File.h: interface for the CUTF16File class.
//
// Version 5.0, 2 February 2004.
//
// Jeffrey Walton
//
//	Modified by Jordan Walters 27/04/2005 to work with non-Unicode
//	builds as well.
//
//////////////////////////////////////////////////////////////////////

#if !defined(AFX_UTF16File_H__32BEF8AC_25E0_482F_8B00_C40775BCDB81__INCLUDED_)
#define AFX_UTF16File_H__32BEF8AC_25E0_482F_8B00_C40775BCDB81__INCLUDED_
 
#if _MSC_VER > 1000
#pragma once
#endif // _MSC_VER > 1000

#pragma warning(push, 3)
#include <list>
#pragma warning(pop)
 

//
// Under a hex editor, file[0] = 0xFF
//                     file[1] = 0xFE
//
// for a proper UTF-16 BOM
//
// This is different than the in-memory
//   representation of: mem[0] = 0xFE
//                      mem[1] = 0xFF
//
// on an Intel CPU
//
const unsigned char UNICODE_BOM[2]				= {unsigned char(0xFF), unsigned char(0xFE)};
const unsigned char UNICODE_RBOM[2]				= {unsigned char(0xFE), unsigned char(0xFF)};
 
const INT ACCUMULATOR_CHAR_COUNT				= 2048;
 
class CUTF16File: public CStdioFile
{
public:
	
	CUTF16File();
	CUTF16File(LPCTSTR lpszFileName, UINT nOpenFlags);
 
	virtual BOOL	Open(LPCTSTR lpszFileName, UINT nOpenFlags, CFileException* pError = NULL);
	virtual BOOL	ReadString(CString& rString);
    virtual LPTSTR  ReadString(LPTSTR lpsz, UINT nMax);
	virtual VOID	WriteString(LPCTSTR lpsz, BOOL bAsUnicode = FALSE);
 
    virtual LONG    Seek(LONG lOff, UINT nFrom);
 
    BOOL            IsUnicodeFile() { return m_bIsUnicode; }
 
protected:
 
	BOOL            ReadUnicodeString(CString& szString);
    LPTSTR          ReadUnicodeString(LPTSTR lpsz, UINT nMax);
 
    virtual VOID    WriteANSIString(LPCSTR lpsz);
    virtual VOID    WriteUnicodeString(LPCWSTR lpsz);
 
	BOOL m_bIsUnicode;
    BOOL m_bByteSwapped;
 
private:
 
	BOOL LoadAccumulator();
 
    std::list<WCHAR> m_Accumulator;
	DWORD	m_dwCurrentFilePointer;
};
 
#endif // !defined(AFX_UTF16File_H__32BEF8AC_25E0_482F_8B00_C40775BCDB81__INCLUDED_)

// UTF16File.cpp: implementation of the CUTF16File class.
//
// Version 5.0, 2 February 2004.
//
// Jeffrey Walton
//
//	Modified by Jordan Walters 27/04/2005 to work with non-Unicode
//	builds as well.
//
//////////////////////////////////////////////////////////////////////

#include "stdafx.h"
#include "UTF16File.h"
#include <atlconv.h>
 
#ifdef _DEBUG
#undef THIS_FILE
static char THIS_FILE[]=__FILE__;
#define new DEBUG_NEW
#endif
 
//////////////////////////////////////////////////////////////////////
// Construction/Destruction
//////////////////////////////////////////////////////////////////////

CUTF16File::CUTF16File(): CStdioFile(),
	m_bIsUnicode(FALSE),
	m_bByteSwapped(FALSE),
	m_dwCurrentFilePointer(0)
{
}
 
CUTF16File::CUTF16File(LPCTSTR lpszFileName, UINT nOpenFlags) :
	CStdioFile(lpszFileName, nOpenFlags), 
	m_bIsUnicode(FALSE),
	m_bByteSwapped(FALSE),
	m_dwCurrentFilePointer(0)
{
	char uchBOM[2] = {0};
 
	// We only need the BOM check if reading.
	if(CFile::modeWrite == (nOpenFlags & CFile::modeWrite)) { return; }
 
	// BOM is two bytes
	if(CFile::GetLength() < 2) { return; }
 
	m_dwCurrentFilePointer += CStdioFile::Read(reinterpret_cast<LPVOID>(uchBOM), 2);
 
	if(uchBOM[0] == UNICODE_BOM[0] &&  uchBOM[1] == UNICODE_BOM[1])
	{
		m_bIsUnicode   = TRUE;
		m_bByteSwapped = FALSE;
	} 
 
	if(uchBOM[0] == UNICODE_RBOM[0] &&  uchBOM[1] == UNICODE_RBOM[1])
	{
		m_bIsUnicode   = TRUE;
		m_bByteSwapped = TRUE;
	}
 
	// Not a BOM mark - its an ANSI file
	//   so punt to CStdioFile...
	if(FALSE == m_bIsUnicode)
	{
			m_dwCurrentFilePointer = 0;
		CStdioFile::Seek(0, CFile::begin);
	}
 
	m_Accumulator.clear();
}
 
BOOL CUTF16File::Open(LPCTSTR lpszFileName, UINT nOpenFlags, CFileException* pError /*=NULL*/)
{
	BOOL bResult = FALSE;
 
	unsigned char uchBOM[3] = {0};
 
	bResult = CStdioFile::Open(lpszFileName, nOpenFlags, pError);
 
	// We only need the BOM check if reading.
	if(CFile::modeWrite == (nOpenFlags & CFile::modeWrite)) { return bResult; }
 
	// BOM is two bytes
	if(CFile::GetLength() < 2) { return bResult; }
 
	if(TRUE == bResult)
	{
		m_dwCurrentFilePointer += CStdioFile::Read(reinterpret_cast<LPVOID>(uchBOM), 2);
 
		if(uchBOM[0] == UNICODE_BOM[0] &&  uchBOM[1] == UNICODE_BOM[1])
		{
			m_bIsUnicode   = TRUE;
			m_bByteSwapped = FALSE;
		} 
 
		if(uchBOM[0] == UNICODE_RBOM[0] &&  uchBOM[1] == UNICODE_RBOM[1])
		{
			m_bIsUnicode   = TRUE;
			m_bByteSwapped = TRUE;
		}
 
		// Not a BOM mark - its an ANSI file
		//   so punt to CStdioFile...
		if(FALSE == m_bIsUnicode)
		{
			m_dwCurrentFilePointer = 0;
			CStdioFile::Seek( 0, CFile::begin );
		}
	}
 
	m_Accumulator.clear();
 
	return bResult;
}
 
BOOL CUTF16File::ReadString( CString& rString )
{
	if(TRUE == m_bIsUnicode)
	{
		return ReadUnicodeString(rString);
	}
 
	return CStdioFile::ReadString(rString);
}
 
LPTSTR CUTF16File::ReadString(LPTSTR lpsz, UINT nMax)
{
	if(TRUE == m_bIsUnicode)
	{
		return ReadUnicodeString(lpsz, nMax);
	}
 
	return CStdioFile::ReadString(lpsz, nMax);
}
 
BOOL CUTF16File::ReadUnicodeString(CString& rString)
{
	USES_CONVERSION;
 
	BOOL bRead = FALSE;
 
	WCHAR c[2] = {0};
 
	rString.Empty();
 
	LoadAccumulator();
 
	while(FALSE == m_Accumulator.empty())
	{
		bRead = TRUE;
 
		c[0] = m_Accumulator.front();
 
		m_Accumulator.pop_front();
 
		if(L'\r' == c[0] || L'\n' == c[0])
		{
			// Set the file pointer to the current position of this carriage return - 
			// or one after as it has been read.
			m_dwCurrentFilePointer += 2;
 
			c[0] = m_Accumulator.front();
 
			m_Accumulator.pop_front();
 
			if(L'\r' == c[0] || L'\n' == c[0])
			{
				m_dwCurrentFilePointer += 2;
				Seek(m_dwCurrentFilePointer, CFile::begin);
			}
			break;
		}
 
		m_dwCurrentFilePointer += 2;
		rString += W2T(c);
 
		if(TRUE == m_Accumulator.empty())
		{
			LoadAccumulator();
		}
	}
 
	return bRead;;
}
 
/***
*char *fgets(string, count, stream) - input string from a stream
*
*Purpose:
*       get a string, up to count-1 chars or '\n', whichever comes first,
*       append '\0' and put the whole thing into string. the '\n' IS included
*       in the string. if count<=1 no input is requested. if EOF is found
*       immediately, return NULL. if EOF found after chars read, let EOF
*       finish the string as '\n' would.
*
***/
 
LPTSTR CUTF16File::ReadUnicodeString( LPTSTR lpsz, UINT nMax )
{
	USES_CONVERSION;
 
	BOOL bRead = FALSE;
 
	LPTSTR p = lpsz;
	WCHAR c[2] = {0};
 
	ASSERT(lpsz != NULL);
	ASSERT(AfxIsValidAddress(lpsz, nMax));
	ASSERT(m_pStream != NULL);
 
	if(nMax <= 1) { return lpsz; }
 
	LoadAccumulator();
 
	while(FALSE == m_Accumulator.empty() && --nMax)
	{
		bRead = TRUE;        
 
		c[0] = m_Accumulator.front();
		m_dwCurrentFilePointer += 2;
		*p++ = *(W2T(c));
 
		m_Accumulator.pop_front();
 
		if(L'\r' == c[0] || L'\n' == c[0])
		{
			// Set the file pointer to the current position of this carriage return - 
			// or one after as it has been read.
			m_dwCurrentFilePointer += 2;
 
			c[0] = m_Accumulator.front();
 
			m_Accumulator.pop_front();
 
			if(L'\r' == c[0] || L'\n' == c[0])
			{
				m_dwCurrentFilePointer += 2;
				Seek(m_dwCurrentFilePointer, CFile::begin);
			}
			break;
		}
 
		if(TRUE == m_Accumulator.empty())
		{
			LoadAccumulator();
		}
	}
 
	*p = L'\0';
 
	return TRUE == bRead ? lpsz : NULL;
}
 
VOID CUTF16File::WriteString( LPCTSTR lpsz, BOOL bAsUnicode /*= FALSE */ )
{
	USES_CONVERSION;
	
	if(TRUE == bAsUnicode)
	{
		WriteUnicodeString(T2W(lpsz));
	}
	else
	{
		WriteANSIString(lpsz);
	}
}
 
BOOL CUTF16File::LoadAccumulator()
{
	BYTE cbBuffer[ACCUMULATOR_CHAR_COUNT * sizeof(WCHAR)];
 
	UINT uCount = CStdioFile::Read(cbBuffer, ACCUMULATOR_CHAR_COUNT * sizeof(WCHAR));
 
	WCHAR* pwszBuffer = reinterpret_cast<WCHAR*>(cbBuffer);
 
	for(UINT i = 0; i < uCount / 2; i++)
	{
		WCHAR c = *pwszBuffer++;
 
		if(TRUE == m_bByteSwapped)
		{
			BYTE b1 = BYTE(c >> 8);   // high order
			BYTE b2 = BYTE(c & 0xFF); // low order

			c = WCHAR(b1 | (b2 << 8));
		}
 
		m_Accumulator.push_back(c);
	}
 
	return 0 == uCount;
}
 
LONG CUTF16File::Seek(LONG lOff, UINT nFrom)
{
	LONG lResult = CStdioFile::Seek(lOff, nFrom);
	m_dwCurrentFilePointer = CStdioFile::Seek(0, CFile::current);
 
	m_Accumulator.clear();
 
//	LoadAccumulator();

	// Should there be a test here to set fp = 2 if Unicode,
	//  and the user asks for CFile::begin???

	return lResult;
}
 
VOID CUTF16File::WriteANSIString( LPCSTR lpsz )
{
	CStdioFile::WriteString(lpsz);
}
 
VOID CUTF16File::WriteUnicodeString(LPCWSTR lpsz)
{
	if(0 == CFile::GetPosition())
	{
		CFile::Write(static_cast<LPVOID>(LPVOID(UNICODE_BOM)), sizeof(UNICODE_BOM));
	}
 
	CFile::Write(lpsz, wcslen(lpsz) * sizeof(WCHAR));
}