third-party libs

third-party/BugTrap-1.3-57bd515/source/Encoding.cpp

@@ -0,0 +1,896 @@
+/*
+ * This is a part of the BugTrap package.
+ * Copyright (c) 2005-2009 IntelleSoft.
+ * All rights reserved.
+ *
+ * Description: Text encoding/decoding.
+ * Author: Maksim Pyatkovskiy.
+ *
+ * This source code is only intended as a supplement to the
+ * BugTrap package reference and related electronic documentation
+ * provided with the product. See these sources for detailed
+ * information regarding the BugTrap package.
+ */
+
+#include "StdAfx.h"
+#include "Encoding.h"
+#include "MemStream.h"
+#include "BugTrapUtils.h"
+#include "TextFormat.h"
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#endif
+
+/// Maximal 7-bit ASCII value.
+#define ASCII_MAX              0x007F
+/// Non-ASCII character value.
+#define NON_ASCII_CHAR         0x80
+/// Maximal UTF-8 2-byte sequence (32 * 64 = 2048).
+#define UTF8_2_MAX             0x07FF
+/// Maximal UTF-8 3-byte sequence (16 * 64 * 64 = 2048).
+#define UTF8_3_MAX             0xFFFF
+/// Prefix of UTF-8 2-byte sequence.
+#define UTF8_1ST_OF_2          0xC0    // 110xxxxx
+/// Prefix of UTF-8 3-byte sequence.
+#define UTF8_1ST_OF_3          0xE0    // 1110xxxx
+/// Prefix of UTF-8 4-byte sequence.
+#define UTF8_1ST_OF_4          0xF0    // 11110xxx
+/// Prefix of UTF-8 bytes from sequence.
+#define UTF8_TRAIL             0x80    // 10xxxxxx
+
+/// Single object instance.
+CUTF8Decoder CUTF8Decoder::m_instance;
+/// Single object instance.
+CUTF16LeDecoder CUTF16LeDecoder::m_instance;
+/// Single object instance.
+CUTF16BeDecoder CUTF16BeDecoder::m_instance;
+/// Single object instance.
+CAnsiDecoder CAnsiDecoder::m_instance;
+
+/**
+ * @param pchValue - pointer to the character normally stored in the string.
+ * @param nCharSize - upon return keeps character size.
+ * @return Unicode character value.
+ */
+static DWORD GetUnicodeValue(const TCHAR* pchValue, size_t& nCharSize)
+{
+#ifndef _UNICODE
+	WCHAR arrUnicodeChar[2];
+	arrUnicodeChar[0] = arrUnicodeChar[1] = 0;
+	nCharSize = IsDBCSLeadByte(*pchValue) ? 2 : 1;
+	MultiByteToWideChar(CP_ACP, 0, pchValue, (int)nCharSize, arrUnicodeChar, countof(arrUnicodeChar));
+#else
+	const WCHAR* arrUnicodeChar = pchValue;
+#endif
+	DWORD dwUnicodeChar;
+	if ((HIGH_SURROGATE_START <= arrUnicodeChar[0] && arrUnicodeChar[0] <= HIGH_SURROGATE_END) &&
+		(LOW_SURROGATE_START <= arrUnicodeChar[1] && arrUnicodeChar[1] <= LOW_SURROGATE_END))
+	{
+		dwUnicodeChar = ((arrUnicodeChar[0] - HIGH_SURROGATE_START) << 10) + (arrUnicodeChar[1] - LOW_SURROGATE_START) + 0x10000;
+#ifdef _UNICODE
+		nCharSize = 2;
+#endif
+	}
+	else
+	{
+#ifndef _UNICODE
+		_ASSERTE(arrUnicodeChar[1] == 0);
+#endif
+		dwUnicodeChar = arrUnicodeChar[0];
+#ifdef _UNICODE
+		nCharSize = 1;
+#endif
+	}
+	return dwUnicodeChar;
+}
+
+/**
+ * @param dwUnicodeChar - Unicode character value.
+ * @return true if data has been successfully written.
+ */
+bool CUTF8EncStream::WriteUTF8Bin(DWORD dwUnicodeChar)
+{
+	_ASSERTE(m_pOutputStream != NULL);
+	if (dwUnicodeChar <= ASCII_MAX)
+	{
+		// 7-bit ASCII data is stored as-is
+		if (! m_pOutputStream->WriteByte((BYTE)dwUnicodeChar))
+			return false;
+	}
+	else if (dwUnicodeChar <= UTF8_2_MAX)
+	{
+		// Use upper 5 bits in first byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_1ST_OF_2 | ((dwUnicodeChar >> 6) & 0x1F))))
+			return false;
+		// Use lower 6 bits in second byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_TRAIL | (dwUnicodeChar & 0x3F))))
+			return false;
+	}
+	else if (dwUnicodeChar <= UTF8_3_MAX)
+	{
+		// Use upper 4 bits in first byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_1ST_OF_3 | ((dwUnicodeChar >> 12) & 0x0F))))
+			return false;
+		// Use middle 6 bits in second byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_TRAIL | ((dwUnicodeChar >> 6) & 0x3F))))
+			return false;
+		// Use lower 6 bits in third byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_TRAIL | (dwUnicodeChar & 0x3F))))
+			return false;
+	}
+	else
+	{
+		// Use upper 3 bits in first byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_1ST_OF_4 | ((dwUnicodeChar >> 18) & 0x07))))
+			return false;
+		// Use middle 6 bits in second byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_TRAIL | ((dwUnicodeChar >> 12) & 0x3F))))
+			return false;
+		// Use middle 6 bits in third byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_TRAIL | ((dwUnicodeChar >> 6) & 0x3F))))
+			return false;
+		// Use lower 6 bits in fourth byte
+		if (! m_pOutputStream->WriteByte((BYTE)(UTF8_TRAIL | (dwUnicodeChar & 0x3F))))
+			return false;
+	}
+	return true;
+}
+
+/**
+ * @param dwUnicodeChar - Unicode character value.
+ * @return true if data has been successfully written.
+ */
+bool CUTF8EncStream::WriteUTF8Hex(DWORD dwUnicodeChar)
+{
+	_ASSERTE(m_pOutputStream != NULL);
+	CHAR szHexValue[16];
+	size_t nLength;
+	if (dwUnicodeChar <= ASCII_MAX)
+	{
+		if (! m_pOutputStream->WriteByte((BYTE)dwUnicodeChar))
+			return false;
+	}
+	else if (dwUnicodeChar <= UTF8_3_MAX)
+	{
+		nLength = sprintf_s(szHexValue, countof(szHexValue), "&#x%04lX;", dwUnicodeChar);
+		if (! m_pOutputStream->WriteBytes((const BYTE*)szHexValue, nLength))
+			return false;
+	}
+	else
+	{
+		nLength = sprintf_s(szHexValue, countof(szHexValue), "&#x%06lX;", dwUnicodeChar);
+		if (! m_pOutputStream->WriteBytes((const BYTE*)szHexValue, nLength))
+			return false;
+	}
+	return true;
+}
+
+/**
+ * @param pszString - string to be written.
+ * @return true if data has been written.
+ */
+bool CUTF8EncStream::WriteUTF8Bin(PCTSTR pszString)
+{
+	size_t nPosition = 0;
+	while (pszString[nPosition] != _T('\0'))
+	{
+		size_t nCharSize;
+		if (! WriteUTF8Bin(pszString + nPosition, nCharSize))
+			return false;
+		nPosition += nCharSize;
+	}
+	return true;
+}
+
+/**
+ * @param pszString - string to be written.
+ * @return true if data has been written.
+ */
+bool CUTF8EncStream::WriteUTF8Hex(PCTSTR pszString)
+{
+	size_t nPosition = 0;
+	while (pszString[nPosition] != _T('\0'))
+	{
+		size_t nCharSize;
+		if (! WriteUTF8Hex(pszString + nPosition, nCharSize))
+			return false;
+		nPosition += nCharSize;
+	}
+	return true;
+}
+
+/**
+ * @param pchValue - pointer to the character normally stored in the string.
+ * @param nCharSize - upon return keeps character size.
+ * @return true if data has been written.
+ */
+bool CUTF8EncStream::WriteUTF8Bin(const TCHAR* pchValue, size_t& nCharSize)
+{
+	DWORD dwUnicodeChar = GetUnicodeValue(pchValue, nCharSize);
+	return WriteUTF8Bin(dwUnicodeChar);
+}
+
+/**
+ * @param pchValue - pointer to the character normally stored in the string.
+ * @param nCharSize - upon return keeps number of bytes in character.
+ * @return true if data has been written.
+ */
+bool CUTF8EncStream::WriteUTF8Hex(const TCHAR* pchValue, size_t& nCharSize)
+{
+	DWORD dwUnicodeChar = GetUnicodeValue(pchValue, nCharSize);
+	return WriteUTF8Hex(dwUnicodeChar);
+}
+
+/**
+ * @brief Write a 32-bit integer in a compressed format to the buffer.
+ * @param nValue - integer value.
+ * @param pBuffer - pointer to the buffer.
+ * @param nPosition - position in the buffer.
+ * @param nBufferSize - size of output buffer.
+ * @return true if operation has been completed successfully.
+ */
+bool Write7BitEncodedInt(size_t nValue, PBYTE pBuffer, size_t& nPosition, size_t nBufferSize)
+{
+	size_t nOldPosition = nPosition;
+	while (nValue >= 0x80)
+	{
+		if (nPosition >= nBufferSize)
+			goto error;
+		pBuffer[nPosition++] = (BYTE)(nValue | 0x80);
+		nValue >>= 7;
+	}
+	if (nPosition < nBufferSize)
+	{
+		pBuffer[nPosition++] = (BYTE)nValue;
+		return true;
+	}
+
+error:
+	nPosition = nOldPosition;
+	return false;
+}
+
+/**
+ * @brief Read a 32-bit integer in a compressed format to the buffer.
+ * @param pBuffer - pointer to the buffer.
+ * @param nPosition - position in the buffer.
+ * @param nBufferSize - size of output buffer.
+ * @return decoded integer value.
+ */
+size_t Read7BitEncodedInt(PBYTE pBuffer, size_t& nPosition, size_t nBufferSize)
+{
+	size_t nValue = 0, nShift = 0;
+	for (;;)
+	{
+		if (nPosition >= nBufferSize)
+			break;
+		BYTE bTempValue = pBuffer[nPosition++];
+		nValue |= (bTempValue & 0x7F) << (nShift & 0x1F);
+		nShift += 7;
+		if ((bTempValue & 0x80) == 0)
+			break;
+	}
+	return nValue;
+}
+
+/**
+ * @brief Write length prefixed Unicode string to the buffer.
+ * @param rEncStream - shared encoder.
+ * @param pszString - string value.
+ * @param pBuffer - pointer to the buffer.
+ * @param nPosition - position in the buffer.
+ * @param nBufferSize - size of output buffer.
+ * @return true if operation has been completed successfully.
+ */
+bool WriteBinaryString(CUTF8EncStream& rEncStream, PCTSTR pszString, PBYTE pBuffer, size_t& nPosition, size_t nBufferSize)
+{
+	CStream* pStream = rEncStream.GetStream();
+	_ASSERTE(pStream != NULL);
+	if (pStream == NULL)
+		return false;
+	unsigned uStreamFeatures = pStream->GetFeatures();
+	_ASSERTE((uStreamFeatures & (CStream::SF_GETLENGTH | CStream::SF_SETPOSITION)) == (CStream::SF_GETLENGTH | CStream::SF_SETPOSITION));
+	if ((uStreamFeatures & (CStream::SF_GETLENGTH | CStream::SF_SETPOSITION)) != (CStream::SF_GETLENGTH | CStream::SF_SETPOSITION))
+		return false;
+	size_t nOldPosition = nPosition;
+	rEncStream.Reset();
+	rEncStream.WriteUTF8Bin(pszString);
+	pStream->SetPosition(0, FILE_BEGIN);
+	size_t nLength = pStream->GetLength();
+	if (! Write7BitEncodedInt(nLength, pBuffer, nPosition, nBufferSize))
+		return false;
+	if (nBufferSize - nPosition >= nLength)
+	{
+		size_t nNumRead = pStream->ReadBytes(pBuffer + nPosition, nLength);
+		_ASSERTE(nNumRead == nLength);
+		if (nNumRead == nLength)
+		{
+			nPosition += nLength;
+			return true;
+		}
+	}
+	nPosition = nOldPosition;
+	return false;
+}
+
+/**
+ * @brief Write length prefixed Unicode string to the buffer.
+ * @param pszString - string value.
+ * @param pBuffer - pointer to the buffer.
+ * @param nPosition - position in the buffer.
+ * @param nBufferSize - size of output buffer.
+ * @return true if operation has been completed successfully.
+ */
+bool WriteBinaryString(PCTSTR pszString, PBYTE pBuffer, size_t& nPosition, size_t nBufferSize)
+{
+	CMemStream MemStream;
+	CUTF8EncStream EncStream(&MemStream);
+	return WriteBinaryString(EncStream, pszString, pBuffer, nPosition, nBufferSize);
+}
+
+/**
+ * @brief Decode UTF-8 character to normal character.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - number of bytes in the array.
+ * @param arrChar - output buffer for the character.
+ * @param nCharSize - number of symbols in one character.
+ * @return number of bytes in one character.
+ */
+size_t UTF8DecodeChar(const BYTE* pBytes, size_t nNumBytes, TCHAR arrChar[2], size_t& nCharSize)
+{
+	nCharSize = MAXSIZE_T;
+	if (nNumBytes == 0)
+		return 0;
+	BYTE bValue = pBytes[0];
+	DWORD dwUnicodeChar;
+	size_t nNumBytesInChar;
+	if ((bValue & UTF8_1ST_OF_4) == UTF8_1ST_OF_4)
+	{
+		if (4 > nNumBytes)
+			return 0;
+		nNumBytesInChar = 4;
+		dwUnicodeChar = ((DWORD)pBytes[0] & ~UTF8_1ST_OF_4 & 0x07) << 18;
+		dwUnicodeChar |= ((DWORD)pBytes[1] & ~UTF8_TRAIL & 0x3F) << 12;
+		dwUnicodeChar |= ((DWORD)pBytes[2] & ~UTF8_TRAIL & 0x3F) << 6;
+		dwUnicodeChar |= (DWORD)pBytes[3] & ~UTF8_TRAIL;
+	}
+	else if ((bValue & UTF8_1ST_OF_3) == UTF8_1ST_OF_3)
+	{
+		if (3 > nNumBytes)
+			return 0;
+		nNumBytesInChar = 3;
+		dwUnicodeChar = ((DWORD)pBytes[0] & ~UTF8_1ST_OF_3 & 0x0F) << 12;
+		dwUnicodeChar |= ((DWORD)pBytes[1] & ~UTF8_TRAIL & 0x3F) << 6;
+		dwUnicodeChar |= (DWORD)pBytes[2] & ~UTF8_TRAIL;
+	}
+	else if ((bValue & UTF8_1ST_OF_2) == UTF8_1ST_OF_2)
+	{
+		if (2 > nNumBytes)
+			return 0;
+		nNumBytesInChar = 2;
+		dwUnicodeChar = ((DWORD)pBytes[0] & ~UTF8_1ST_OF_2 & 0x1F) << 6;
+		dwUnicodeChar |= (DWORD)pBytes[1] & ~UTF8_TRAIL;
+	}
+	else if ((bValue & NON_ASCII_CHAR) == 0)
+	{
+		if (1 > nNumBytes)
+			return 0;
+		nNumBytesInChar = 1;
+		dwUnicodeChar = (DWORD)pBytes[0];
+	}
+	else
+		return 0;
+	const WCHAR* arrUnicodeChar = (const WCHAR*)&dwUnicodeChar;
+#ifdef _UNICODE
+	if ((HIGH_SURROGATE_START <= arrUnicodeChar[0] && arrUnicodeChar[0] <= HIGH_SURROGATE_END) &&
+		(LOW_SURROGATE_START <= arrUnicodeChar[1] && arrUnicodeChar[1] <= LOW_SURROGATE_END))
+	{
+		nCharSize = 2;
+		arrChar[0] = arrUnicodeChar[0];
+		arrChar[1] = arrUnicodeChar[1];
+	}
+	else
+	{
+		nCharSize = 1;
+		arrChar[0] = arrUnicodeChar[0];
+		_ASSERTE(arrUnicodeChar[1] == 0);
+	}
+#else
+	size_t nWordsInChar;
+	if ((HIGH_SURROGATE_START <= arrUnicodeChar[0] && arrUnicodeChar[0] <= HIGH_SURROGATE_END) &&
+		(LOW_SURROGATE_START <= arrUnicodeChar[1] && arrUnicodeChar[1] <= LOW_SURROGATE_END))
+	{
+		nWordsInChar = 2;
+	}
+	else
+	{
+		nWordsInChar = 1;
+	}
+	nCharSize = WideCharToMultiByte(CP_ACP, 0, arrUnicodeChar, (int)nWordsInChar, arrChar, countof(arrChar), NULL, NULL);
+#endif
+	return nNumBytesInChar;
+}
+
+/**
+ * @brief Decode UTF-16 character to normal character.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - number of bytes in the array.
+ * @param arrChar - output buffer for the character.
+ * @param nCharSize - number of symbols in one character.
+ * @return number of bytes in one character.
+ */
+size_t UTF16BeDecodeChar(BYTE* pBytes, size_t nNumBytes, TCHAR arrChar[2], size_t& nCharSize)
+{
+	if (UTF16BeToLeChar(pBytes, nNumBytes) < 0)
+	{
+		nCharSize = MAXSIZE_T;
+		return 0;
+	}
+	return UTF16LeDecodeChar(pBytes, nNumBytes, arrChar, nCharSize);
+}
+
+/**
+ * @brief Decode UTF-16 character to normal character.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - number of bytes in the array.
+ * @param arrChar - output buffer for the character.
+ * @param nCharSize - number of symbols in one character.
+ * @return number of bytes in one character.
+ */
+size_t UTF16LeDecodeChar(const BYTE* pBytes, size_t nNumBytes, TCHAR arrChar[2], size_t& nCharSize)
+{
+	size_t nNumBytesInChar = GetUnicodeCharSize(pBytes);
+	if (nNumBytes >= nNumBytesInChar)
+	{
+#ifdef _UNICODE
+		_ASSERTE(nNumBytesInChar == sizeof(WCHAR) || nNumBytesInChar == 2 * sizeof(WCHAR));
+		arrChar[0] = *(const WCHAR*)pBytes;
+		if (nNumBytesInChar == 2 * sizeof(WCHAR))
+			arrChar[1] = *((const WCHAR*)pBytes + 1);
+		nCharSize = nNumBytesInChar / sizeof(WCHAR);
+#else
+		nCharSize = WideCharToMultiByte(CP_ACP, 0, (const WCHAR*)pBytes, (int)nNumBytesInChar, arrChar, countof(arrChar), NULL, NULL) / sizeof(WCHAR);
+		if (nCharSize == 0)
+			nNumBytesInChar = 0;
+#endif
+	}
+	else
+		nCharSize = MAXSIZE_T;
+	return nNumBytesInChar;
+}
+
+/**
+ * @brief Decode Ansi character to normal character.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - number of bytes in the array.
+ * @param arrChar - output buffer for the character.
+ * @param nCharSize - number of symbols in one character.
+ * @return number of bytes in one character.
+ */
+size_t AnsiDecodeChar(const BYTE* pBytes, size_t nNumBytes, TCHAR arrChar[2], size_t& nCharSize)
+{
+	int nNumBytesInChar = IsDBCSLeadByte(*pBytes) ? 2 : 1;
+	if (nNumBytes >= (size_t)nNumBytesInChar)
+	{
+#ifdef _UNICODE
+		nCharSize = MultiByteToWideChar(CP_ACP, 0, (const CHAR*)pBytes, nNumBytesInChar, arrChar, countof(arrChar));
+		if (nCharSize == 0)
+			nNumBytesInChar = 0;
+#else
+		arrChar[0] = *(const CHAR*)pBytes;
+		if (nNumBytesInChar == 2)
+			arrChar[1] = *((const CHAR*)pBytes + 1);
+		nCharSize = nNumBytesInChar;
+#endif
+	}
+	else
+		nCharSize = MAXSIZE_T;
+	return nNumBytesInChar;
+}
+
+/**
+ * @brief Decode UTF-8 string to normal text.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - number of bytes in the array.
+ * @param pszString - decoded string.
+ * @param nBufferSize - size of output buffer (in characters).
+ * @return number of characters in resulting string including null-terminator.
+ */
+size_t UTF8DecodeString(const BYTE* pBytes, size_t nNumBytes, PTSTR pszString, size_t nBufferSize)
+{
+	if (! nBufferSize)
+		return 0;
+	bool bAddNullTerminator;
+	if (nNumBytes == MAXSIZE_T)
+	{
+		nNumBytes = strlen((PCSTR)pBytes);
+		--nBufferSize;
+		bAddNullTerminator = true;
+	}
+	else
+		bAddNullTerminator = false;
+	size_t nBytePos = 0, nCharPos = 0;
+	while (nBytePos < nNumBytes)
+	{
+		TCHAR arrChar[2];
+		size_t nCharSize;
+		size_t nNumBytesInChar = UTF8DecodeChar(pBytes + nBytePos, nNumBytes - nBytePos, arrChar, nCharSize);
+		if (nNumBytesInChar == 0 || nCharPos + nCharSize > nBufferSize)
+			break;
+		nBytePos += nNumBytesInChar;
+		pszString[nCharPos++] = arrChar[0];
+		if (nCharSize > 1)
+			pszString[nCharPos++] = arrChar[1];
+	}
+	if (bAddNullTerminator)
+		pszString[nCharPos++] = _T('\0');
+	return nCharPos;
+}
+
+/**
+ * @brief Decode UTF-16 string to normal text.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - number of bytes in the array.
+ * @param pszString - decoded string.
+ * @param nBufferSize - size of output buffer (in characters).
+ * @return number of characters in resulting string including null-terminator.
+ */
+size_t UTF16BeDecodeString(BYTE* pBytes, size_t nNumBytes, PTSTR pszString, size_t nBufferSize)
+{
+	if (UTF16BeToLeString(pBytes, nNumBytes) == 0)
+		return 0;
+	return UTF8DecodeString(pBytes, nNumBytes, pszString, nBufferSize);
+}
+
+/**
+ * @brief Decode UTF-16 string to normal text.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - number of bytes in the array.
+ * @param pszString - decoded string.
+ * @param nBufferSize - size of output buffer (in characters).
+ * @return number of characters in resulting string including null-terminator.
+ */
+size_t UTF16LeDecodeString(const BYTE* pBytes, size_t nNumBytes, PTSTR pszString, size_t nBufferSize)
+{
+#ifdef _UNICODE
+	if (nNumBytes == MAXSIZE_T)
+		nNumBytes = (wcslen((const WCHAR*)pBytes) + 1) * sizeof(WCHAR);
+	if (nNumBytes > nBufferSize)
+		nNumBytes = nBufferSize;
+	CopyMemory(pszString, pBytes, nNumBytes);
+	return nNumBytes;
+#else
+	return WideCharToMultiByte(CP_ACP, 0, (const WCHAR*)pBytes, (int)(nNumBytes / sizeof(WCHAR)), pszString, (int)nBufferSize, NULL, NULL);
+#endif
+}
+
+/**
+ * @brief Decode Ansi string to normal text.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - number of bytes in the array.
+ * @param pszString - decoded string.
+ * @param nBufferSize - size of output buffer (in characters).
+ * @return number of characters in resulting string including null-terminator.
+ */
+size_t AnsiDecodeString(const BYTE* pBytes, size_t nNumBytes, PTSTR pszString, size_t nBufferSize)
+{
+#ifdef _UNICODE
+	return MultiByteToWideChar(CP_ACP, 0, (const CHAR*)pBytes, (int)nNumBytes, pszString, (int)nBufferSize);
+#else
+	if (nNumBytes == MAXSIZE_T)
+		nNumBytes = strlen((const CHAR*)pBytes) + 1;
+	if (nNumBytes > nBufferSize)
+		nNumBytes = nBufferSize;
+	CopyMemory(pszString, pBytes, nNumBytes);
+	return nNumBytes;
+#endif
+}
+
+/**
+ * @brief Get number of bytes representing one character.
+ * @param pBytes - array of bytes.
+ * @return number of bytes in one character.
+ */
+size_t GetUTF8CharSize(const BYTE* pBytes)
+{
+	BYTE bValue = *pBytes;
+	if ((bValue & UTF8_1ST_OF_4) == UTF8_1ST_OF_4)
+		return 4;
+	else if ((bValue & UTF8_1ST_OF_3) == UTF8_1ST_OF_3)
+		return 3;
+	else if ((bValue & UTF8_1ST_OF_2) == UTF8_1ST_OF_2)
+		return 2;
+	else
+		return 1;
+}
+
+/**
+ * @brief Used to check if next character starts surrogate pair.
+ * @param pBytes - array of bytes.
+ * @return true if next character is high surrogate character.
+ */
+bool IsUnicodeLeadChar(const BYTE* pBytes)
+{
+	WCHAR chValue = *((const WCHAR*)pBytes);
+	return (HIGH_SURROGATE_START <= chValue && chValue <= HIGH_SURROGATE_END);
+}
+
+/**
+ * @brief Get number of bytes representing one character.
+ * @param pBytes - array of bytes.
+ * @return number of bytes in one character.
+ */
+size_t GetUnicodeCharSize(const BYTE* pBytes)
+{
+	WCHAR chValue1 = ((const WCHAR*)pBytes)[0];
+	if (HIGH_SURROGATE_START <= chValue1 && chValue1 <= HIGH_SURROGATE_END)
+	{
+		WCHAR chValue2 = ((const WCHAR*)pBytes)[1];
+		if (LOW_SURROGATE_START <= chValue2 && chValue2 <= LOW_SURROGATE_END)
+			return (2 * sizeof(WCHAR));
+	}
+	return (sizeof(WCHAR));
+}
+
+/**
+ * @brief Convert UTF-16 value from big-endian to low-endian format.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - buffer size in bytes.
+ * @return number of bytes in one character.
+ */
+size_t UTF16BeToLeChar(BYTE* pBytes, size_t nNumBytes)
+{
+	if (nNumBytes < 1)
+		return MAXSIZE_T;
+	WCHAR chValue1 = SWAP16(((WCHAR*)pBytes)[0]);
+	if (LOW_SURROGATE_START <= chValue1 && chValue1 <= LOW_SURROGATE_END)
+	{
+		if (nNumBytes < 2)
+			return MAXSIZE_T;
+		WCHAR chValue2 = SWAP16(((WCHAR*)pBytes)[1]);
+		if (HIGH_SURROGATE_START <= chValue2 && chValue2 <= HIGH_SURROGATE_END)
+		{
+			((WCHAR*)pBytes)[0] = chValue2;
+			((WCHAR*)pBytes)[1] = chValue1;
+			return (2 * sizeof(WCHAR));
+		}
+	}
+	*(WCHAR*)pBytes = chValue1;
+	return (sizeof(WCHAR));
+}
+
+/**
+ * @brief Convert UTF-16 value from big-endian to low-endian format.
+ * @param pBytes - array of bytes.
+ * @param nNumBytes - buffer size in bytes.
+ * @return number of processed bytes.
+ */
+size_t UTF16BeToLeString(BYTE* pBytes, size_t nNumBytes)
+{
+	size_t nBytePos = 0;
+	while (nBytePos < nNumBytes)
+	{
+		size_t nResult = UTF16BeToLeChar(pBytes + nBytePos, nNumBytes - nBytePos);
+		if (nResult == MAXSIZE_T)
+			break;
+		nBytePos += nResult;
+	}
+	return nBytePos;
+}
+
+/**
+ * @param pchValue - pointer to the character normally stored in the string.
+ * @param nCharSize - upon return keeps character size.
+ * @return size of the character after UTF-8 encoding.
+ */
+size_t GetCharSizeInUTF8(const TCHAR* pchValue, size_t& nCharSize)
+{
+	DWORD dwUnicodeChar = GetUnicodeValue(pchValue, nCharSize);
+	if (dwUnicodeChar <= ASCII_MAX)
+		return 1;
+	else if (dwUnicodeChar <= UTF8_2_MAX)
+		return 2;
+	else if (dwUnicodeChar <= UTF8_3_MAX)
+		return 3;
+	else
+		return 4;
+}
+
+/**
+ * @param pszString - input string that needs to be encoded in UTF-8 format.
+ * @return size of the character encoded in UTF-8 format.
+ */
+size_t GetStringSizeInUTF8(PCTSTR pszString)
+{
+	_ASSERTE(pszString != NULL);
+	size_t nUTF8StringSize = 0;
+	while (*pszString)
+	{
+		size_t nCharSize;
+		nUTF8StringSize += GetCharSizeInUTF8(pszString, nCharSize);
+		pszString += nCharSize;
+	}
+	return nUTF8StringSize;
+}
+
+/**
+ * @param eEncoding - text encoding.
+ * @return pointer to corresponding decoder object.
+ */
+CBaseDecoder* CBaseDecoder::GetDecoder(TEXT_ENCODING eEncoding)
+{
+	switch (eEncoding)
+	{
+	case TXTENC_UTF8:
+		return &CUTF8Decoder::GetInstance();
+	case TXTENC_UTF16LE:
+		return &CUTF16LeDecoder::GetInstance();
+	case TXTENC_UTF16BE:
+		return &CUTF16BeDecoder::GetInstance();
+	case TXTENC_ANSI:
+		return &CAnsiDecoder::GetInstance();
+	default:
+		return NULL;
+	}
+}
+
+/**
+ * @param pszEncoding - text encoding.
+ * @return pointer to corresponding decoder object.
+ */
+CBaseDecoder* CBaseDecoder::GetDecoder(PCTSTR pszEncoding)
+{
+	if (_tcsicmp(pszEncoding, _T("UTF-8")) == 0)
+		return &CUTF8Decoder::GetInstance();
+	else if (_tcsicmp(pszEncoding, _T("UTF-16")) == 0)
+		return &CUTF16LeDecoder::GetInstance();
+	else
+		return &CAnsiDecoder::GetInstance();
+}
+
+/**
+ * @param pInputStream - input stream.
+ */
+void CDecInputStream::SetInputStream(CInputStream* pInputStream)
+{
+	m_pInputStream = pInputStream;
+	m_nInputBufferPos = 0;
+	m_nInputBufferLength = 0;
+	m_bEndOfFile = false;
+}
+
+/**
+ * @param nNumBytes - number of bytes to read.
+ * @return number of bytes available in a buffer.
+ */
+size_t CDecInputStream::FillBuffer(size_t nNumBytes)
+{
+	_ASSERTE(m_pInputStream != NULL);
+	size_t nBytesLeft = m_nInputBufferLength - m_nInputBufferPos;
+	if (nBytesLeft < nNumBytes && ! m_bEndOfFile)
+	{
+		MoveMemory(m_arrInputBuffer, m_arrInputBuffer + m_nInputBufferPos, nBytesLeft);
+		m_nInputBufferPos = 0;
+		size_t nFreeSize = sizeof(m_arrInputBuffer) - nBytesLeft;
+		size_t nBytesRead = m_pInputStream->ReadBytes(m_arrInputBuffer + nBytesLeft, nFreeSize);
+		if (nBytesRead == MAXSIZE_T)
+			return MAXSIZE_T;
+		m_nInputBufferLength = nBytesLeft + nBytesRead;
+		m_bEndOfFile = nBytesRead < nFreeSize;
+		nBytesLeft = m_nInputBufferLength;
+	}
+	return nBytesLeft;
+}
+
+/**
+ * @param arrChar - character data.
+ * @return number of characters in one symbol.
+ */
+size_t CDecInputStream::ReadChar(TCHAR arrChar[2])
+{
+	// one character in UTF-16 encoding may require up to 4 bytes
+	// one character in UTF-8 encoding may require up to 4 bytes
+	// one character in ANSI encoding may require up to 2 bytes
+	// make sure at least one character can be read from the buffer
+	// (i.e. at least 4 bytes should be available)
+	size_t nBytesLeft = FillBuffer(4);
+	if (nBytesLeft == 0 || nBytesLeft == MAXSIZE_T)
+		return nBytesLeft;
+	size_t nNumBytesInChar, nCharSize;
+	_ASSERTE(m_pDecoder != NULL);
+	nNumBytesInChar = m_pDecoder->DecodeChar(m_arrInputBuffer + m_nInputBufferPos, nBytesLeft, arrChar, nCharSize);
+	if (nCharSize > 0)
+		m_nInputBufferPos += nNumBytesInChar;
+	else
+		++m_nInputBufferPos;
+	return nCharSize;
+}
+
+/**
+ * @param eEncoding - stream encoding.
+ * @return true if stream encoding has been read from the stream.
+ */
+bool CDecInputStream::ReadPreamble(TEXT_ENCODING& eEncoding)
+{
+	// make sure at least 4 bytes are available in a buffer;
+	// this should be enough for any preamble.
+	size_t nBytesLeft = FillBuffer(4);
+	if (nBytesLeft == 0 || nBytesLeft == MAXSIZE_T)
+		return false;
+	PBYTE pBytes = m_arrInputBuffer + m_nInputBufferPos;
+	if (nBytesLeft >= sizeof(g_arrUTF16LEPreamble) &&
+		memcmp(pBytes, g_arrUTF16LEPreamble, sizeof(g_arrUTF16LEPreamble)) == 0)
+	{
+		eEncoding = TXTENC_UTF16LE;
+		m_nInputBufferPos += sizeof(g_arrUTF16LEPreamble);
+		return true;
+	}
+	else if (nBytesLeft >= sizeof(g_arrUTF16BEPreamble) &&
+		memcmp(pBytes, g_arrUTF16BEPreamble, sizeof(g_arrUTF16BEPreamble)) == 0)
+	{
+		eEncoding = TXTENC_UTF16BE;
+		m_nInputBufferPos += sizeof(g_arrUTF16BEPreamble);
+		return true;
+	}
+	else if (nBytesLeft >= sizeof(g_arrUTF8Preamble) &&
+		memcmp(pBytes, g_arrUTF8Preamble, sizeof(g_arrUTF8Preamble)) == 0)
+	{
+		eEncoding = TXTENC_UTF8;
+		m_nInputBufferPos += sizeof(g_arrUTF8Preamble);
+		return true;
+	}
+	return false;
+}
+
+/**
+ * @param arrChar - character data.
+ * @return number of characters in one symbol.
+ */
+size_t CStrInputStream::ReadChar(TCHAR arrChar[2])
+{
+	_ASSERTE(m_pStrStream != NULL);
+	size_t nLength = m_pStrStream->GetLength();
+	if (m_nPosition >= nLength)
+		return 0;
+	arrChar[0] = ((PCTSTR)m_pStrStream)[m_nPosition++];
+#ifdef _UNICODE
+	if (IsUnicodeLeadChar((const BYTE*)arrChar))
+#else
+	if (IsDBCSLeadByte(*arrChar))
+#endif
+	{
+		if (m_nPosition >= nLength)
+		{
+			_ASSERT(FALSE);
+			return MAXSIZE_T;
+		}
+		arrChar[1] = ((PCTSTR)m_pStrStream)[m_nPosition++];
+		return 2;
+	}
+	return 1;
+}
+
+/**
+ * @return true if byte order mark has been read from the stream.
+ */
+bool CCharInputStream::CheckEncoding(void)
+{
+	TEXT_ENCODING eEncoding;
+	if (! ReadPreamble(eEncoding))
+		return false;
+	return SetDecoder(CBaseDecoder::GetDecoder(eEncoding));
+}
+
+/**
+ * @param eDefaultEncoding - default encoding.
+ * @return true if byte order mark has been read from the stream.
+ */
+bool CCharInputStream::CheckEncoding(TEXT_ENCODING eDefaultEncoding)
+{
+	TEXT_ENCODING eEncoding;
+	bool bResult = ReadPreamble(eEncoding);
+	if (! bResult)
+		eEncoding = eDefaultEncoding;
+	return SetDecoder(CBaseDecoder::GetDecoder(eEncoding));
+}