diff --git a/ConvertUTF.c b/ConvertUTF.c deleted file mode 100644 index de6bf13..0000000 --- a/ConvertUTF.c +++ /dev/null @@ -1,578 +0,0 @@ -/* - * https://web.archive.org/web/20090529064329/http://www.unicode.org:80/Public/PROGRAMS/CVTUTF/ - * - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - - /* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Source code file. - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Sept 2001: fixed const & error conditions per - mods suggested by S. Parent & A. Lillich. - June 2002: Tim Dodd added detection and handling of incomplete - source sequences, enhanced error detection, added casts - to eliminate compiler warnings. - July 2003: slight mods to back out aggressive FFFE detection. - Jan 2004: updated switches in from-UTF8 conversions. - Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. - - See the header file "ConvertUTF.h" for complete documentation. - - ------------------------------------------------------------------------ */ - - -#include "ConvertUTF.h" -#ifdef CVTUTF_DEBUG -#include -#endif - -static const int halfShift = 10; /* used for shifting by 10 bits */ - -static const UTF32 halfBase = 0x0010000UL; -static const UTF32 halfMask = 0x3FFUL; - -#define UNI_SUR_HIGH_START (UTF32)0xD800 -#define UNI_SUR_HIGH_END (UTF32)0xDBFF -#define UNI_SUR_LOW_START (UTF32)0xDC00 -#define UNI_SUR_LOW_END (UTF32)0xDFFF -#define false 0 -#define true 1 - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF16( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - if (target >= targetEnd) { - result = targetExhausted; break; - } - ch = *source++; - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } - else { - *target++ = (UTF16)ch; /* normal case */ - } - } - else if (ch > UNI_MAX_LEGAL_UTF32) { - if (flags == strictConversion) { - result = sourceIllegal; - } - else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } - else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - --source; /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF32( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF32* target = *targetStart; - UTF32 ch, ch2; - while (source < sourceEnd) { - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } - else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } - else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - if (target >= targetEnd) { - source = oldSource; /* Back up source pointer! */ - result = targetExhausted; break; - } - *target++ = ch; - } - *sourceStart = source; - *targetStart = target; -#ifdef CVTUTF_DEBUG - if (result == sourceIllegal) { - fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); - fflush(stderr); - } -#endif - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Index into the table below with the first byte of a UTF-8 sequence to - * get the number of trailing bytes that are supposed to follow it. - * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is - * left as-is for anyone who may want to do such conversion, which was - * allowed in earlier algorithms. - */ -static const char trailingBytesForUTF8[256] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 -}; - -/* - * Magic values subtracted from a buffer value during UTF8 conversion. - * This table contains as many values as there might be trailing bytes - * in a UTF-8 sequence. - */ -static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, - 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; - -/* - * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed - * into the first byte, depending on how many bytes follow. There are - * as many entries in this table as there are UTF-8 sequence types. - * (I.e., one byte sequence, two byte... etc.). Remember that sequencs - * for *legal* UTF-8 will be 4 or fewer bytes total. - */ -static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -/* --------------------------------------------------------------------- */ - -/* The interface converts a whole buffer to avoid function-call overhead. - * Constants have been gathered. Loops & conditionals have been removed as - * much as possible for efficiency, in favor of drop-through switches. - * (See "Note A" at the bottom of the file for equivalent code.) - * If your compiler supports it, the "isLegalUTF8" call can be turned - * into an inline function. - */ - - /* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF8( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - UTF32 ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } - else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } - else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* Figure out how many bytes the result will require */ - if (ch < (UTF32)0x80) { - bytesToWrite = 1; - } - else if (ch < (UTF32)0x800) { - bytesToWrite = 2; - } - else if (ch < (UTF32)0x10000) { - bytesToWrite = 3; - } - else if (ch < (UTF32)0x110000) { - bytesToWrite = 4; - } - else { - bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - } - - target += bytesToWrite; - if (target > targetEnd) { - source = oldSource; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Utility routine to tell whether a sequence of bytes is legal UTF-8. - * This must be called with the length pre-determined by the first byte. - * If not calling this from ConvertUTF8to*, then the length can be set by: - * length = trailingBytesForUTF8[*source]+1; - * and the sequence is illegal right away if there aren't that many bytes - * available. - * If presented with a length > 4, this returns false. The Unicode - * definition of UTF-8 goes up to 4-byte sequences. - */ - -static Boolean isLegalUTF8(const UTF8* source, int length) { - UTF8 a; - const UTF8* srcptr = source + length; - switch (length) { - default: return false; - /* Everything else falls through when "true"... */ - case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 2: if ((a = (*--srcptr)) > 0xBF) return false; - - switch (*source) { - /* no fall-through in this inner switch */ - case 0xE0: if (a < 0xA0) return false; break; - case 0xED: if (a > 0x9F) return false; break; - case 0xF0: if (a < 0x90) return false; break; - case 0xF4: if (a > 0x8F) return false; break; - default: if (a < 0x80) return false; - } - - case 1: if (*source >= 0x80 && *source < 0xC2) return false; - } - if (*source > 0xF4) return false; - return true; -} - -/* --------------------------------------------------------------------- */ - -/* - * Exported function to return whether a UTF-8 sequence is legal or not. - * This is not used here; it's just exported. - */ -Boolean isLegalUTF8Sequence(const UTF8* source, const UTF8* sourceEnd) { - int length = trailingBytesForUTF8[*source] + 1; - if (source + length > sourceEnd) { - return false; - } - return isLegalUTF8(source, length); -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF16( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (!isLegalUTF8(source, extraBytesToRead + 1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead + 1); /* Back up source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead + 1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } - else { - *target++ = (UTF16)ch; /* normal case */ - } - } - else if (ch > UNI_MAX_UTF16) { - if (flags == strictConversion) { - result = sourceIllegal; - source -= (extraBytesToRead + 1); /* return to the start */ - break; /* Bail out; shouldn't continue */ - } - else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } - else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - source -= (extraBytesToRead + 1); /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF8( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - ch = *source++; - if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* - * Figure out how many bytes the result will require. Turn any - * illegally large UTF32 things (> Plane 17) into replacement chars. - */ - if (ch < (UTF32)0x80) { - bytesToWrite = 1; - } - else if (ch < (UTF32)0x800) { - bytesToWrite = 2; - } - else if (ch < (UTF32)0x10000) { - bytesToWrite = 3; - } - else if (ch <= UNI_MAX_LEGAL_UTF32) { - bytesToWrite = 4; - } - else { - bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - result = sourceIllegal; - } - - target += bytesToWrite; - if (target > targetEnd) { - --source; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF32( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF32* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (!isLegalUTF8(source, extraBytesToRead + 1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; - case 4: ch += *source++; ch <<= 6; - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead + 1); /* Back up the source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_LEGAL_UTF32) { - /* - * UTF-16 surrogate values are illegal in UTF-32, and anything - * over Plane 17 (> 0x10FFFF) is illegal. - */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead + 1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } - else { - *target++ = ch; - } - } - else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ - result = sourceIllegal; - *target++ = UNI_REPLACEMENT_CHAR; - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- - - Note A. - The fall-through switches in UTF-8 reading code save a - temp variable, some decrements & conditionals. The switches - are equivalent to the following loop: - { - int tmpBytesToRead = extraBytesToRead+1; - do { - ch += *source++; - --tmpBytesToRead; - if (tmpBytesToRead) ch <<= 6; - } while (tmpBytesToRead > 0); - } - In UTF-8 writing code, the switches on "bytesToWrite" are - similarly unrolled loops. - - --------------------------------------------------------------------- */ \ No newline at end of file diff --git a/ConvertUTF.h b/ConvertUTF.h deleted file mode 100644 index 528c963..0000000 --- a/ConvertUTF.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - * https://web.archive.org/web/20090529064329/http://www.unicode.org:80/Public/PROGRAMS/CVTUTF/ - * - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - - /* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Header file. - - Several funtions are included here, forming a complete set of - conversions between the three formats. UTF-7 is not included - here, but is handled in a separate source file. - - Each of these routines takes pointers to input buffers and output - buffers. The input buffers are const. - - Each routine converts the text between *sourceStart and sourceEnd, - putting the result into the buffer between *targetStart and - targetEnd. Note: the end pointers are *after* the last item: e.g. - *(sourceEnd - 1) is the last item. - - The return result indicates whether the conversion was successful, - and if not, whether the problem was in the source or target buffers. - (Only the first encountered problem is indicated.) - - After the conversion, *sourceStart and *targetStart are both - updated to point to the end of last text successfully converted in - the respective buffers. - - Input parameters: - sourceStart - pointer to a pointer to the source buffer. - The contents of this are modified on return so that - it points at the next thing to be converted. - targetStart - similarly, pointer to pointer to the target buffer. - sourceEnd, targetEnd - respectively pointers to the ends of the - two buffers, for overflow checking only. - - These conversion functions take a ConversionFlags argument. When this - flag is set to strict, both irregular sequences and isolated surrogates - will cause an error. When the flag is set to lenient, both irregular - sequences and isolated surrogates are converted. - - Whether the flag is strict or lenient, all illegal sequences will cause - an error return. This includes sequences such as: , , - or in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code - must check for illegal sequences. - - When the flag is set to lenient, characters over 0x10FFFF are converted - to the replacement character; otherwise (when the flag is set to strict) - they constitute an error. - - Output parameters: - The value "sourceIllegal" is returned from some routines if the input - sequence is malformed. When "sourceIllegal" is returned, the source - value will point to the illegal value that caused the problem. E.g., - in UTF-8 when a sequence is malformed, it points to the start of the - malformed sequence. - - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Fixes & updates, Sept 2001. - - ------------------------------------------------------------------------ */ - - /* --------------------------------------------------------------------- - The following 4 definitions are compiler-specific. - The C standard does not guarantee that wchar_t has at least - 16 bits, so wchar_t is no less portable than unsigned short! - All should be unsigned values to avoid sign extension during - bit mask & shift operations. - ------------------------------------------------------------------------ */ - -typedef unsigned long UTF32; /* at least 32 bits */ -typedef unsigned short UTF16; /* at least 16 bits */ -typedef unsigned char UTF8; /* typically 8 bits */ -typedef unsigned char Boolean; /* 0 or 1 */ - -/* Some fundamental constants */ -#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD -#define UNI_MAX_BMP (UTF32)0x0000FFFF -#define UNI_MAX_UTF16 (UTF32)0x0010FFFF -#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF -#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF - -typedef enum { - conversionOK, /* conversion successful */ - sourceExhausted, /* partial character in source, but hit end */ - targetExhausted, /* insuff. room in target for conversion */ - sourceIllegal /* source sequence is illegal/malformed */ -} ConversionResult; - -typedef enum { - strictConversion = 0, - lenientConversion -} ConversionFlags; - -/* This is for C++ and does no harm in C */ -#ifdef __cplusplus -extern "C" { -#endif - - ConversionResult ConvertUTF8toUTF16( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - - ConversionResult ConvertUTF16toUTF8( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - - ConversionResult ConvertUTF8toUTF32( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - - ConversionResult ConvertUTF32toUTF8( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - - ConversionResult ConvertUTF16toUTF32( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - - ConversionResult ConvertUTF32toUTF16( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - - Boolean isLegalUTF8Sequence(const UTF8* source, const UTF8* sourceEnd); - -#ifdef __cplusplus -} -#endif - -/* --------------------------------------------------------------------- */ \ No newline at end of file diff --git a/ConvertUTF_readme.txt b/ConvertUTF_readme.txt deleted file mode 100644 index 402eaa4..0000000 --- a/ConvertUTF_readme.txt +++ /dev/null @@ -1,43 +0,0 @@ -https://web.archive.org/web/20090529064329/http://www.unicode.org:80/Public/PROGRAMS/CVTUTF/ - -The accompanying C source code file "ConvertUTF.c" and the associated header -file "ConvertUTF.h" provide for conversion between various transformation -formats of Unicode characters. The following conversions are supported: - - UTF-32 to UTF-16 - UTF-32 to UTF-8 - UTF-16 to UTF-32 - UTF-16 to UTF-8 - UTF-8 to UTF-16 - UTF-8 to UTF-32 - -In addition, there is a test harness which runs various tests. - -The files "CVTUTF7.C" and "CVTUTF7.H" are for archival and historical purposes -only. They have not been updated to Unicode 3.0 or later and should be -considered obsolescent. "CVTUTF7.C" contains two functions that can convert -between UCS2 (i.e., the BMP characters only) and UTF-7. Surrogates are -not supported, the code has not been tested, and should be considered -unsuitable for general purpose use. - -Please submit any bug reports about these programs here: - - http://www.unicode.org/unicode/reporting.html - -Version 1.0: initial version. - -Version 1.1: corrected some minor problems; added stricter checks. - -Version 1.2: corrected switch statements associated with "extraBytesToRead" - in 4 & 5 byte cases, in functions for conversion from UTF8. - Note: formally, the 4 & 5 byte cases are illegal in the latest - UTF8, but the table and this code has always catered for those, - cases since at one time they were legal. - -Version 1.3: Updated UTF-8 legality check; - updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions - Updated UTF-8 legality tests in harness.c - - -Last update: October 19, 2004 - diff --git a/SimpleIni.h b/SimpleIni.h index 34d179e..67e22fc 100644 --- a/SimpleIni.h +++ b/SimpleIni.h @@ -3038,8 +3038,8 @@ class SI_ConvertA { #define SI_Case SI_GenericCase #define SI_NoCase SI_GenericNoCase -#include -#include "ConvertUTF.h" +#include + /** * Converts UTF-8 to a wchar_t (or equivalent) using the Unicode reference @@ -3121,28 +3121,38 @@ class SI_ConvertW { size_t a_uOutputDataSize) { if (m_bStoreIsUtf8) { - // This uses the Unicode reference implementation to do the - // conversion from UTF-8 to wchar_t. The required files are - // ConvertUTF.h and ConvertUTF.c which should be included in - // the distribution but are publicly available from unicode.org - // at http://www.unicode.org/Public/PROGRAMS/CVTUTF/ - ConversionResult retval; - const UTF8 * pUtf8 = (const UTF8 *) a_pInputData; - if (sizeof(wchar_t) == sizeof(UTF32)) { - UTF32 * pUtf32 = (UTF32 *) a_pOutputData; - retval = ConvertUTF8toUTF32( - &pUtf8, pUtf8 + a_uInputDataLen, - &pUtf32, pUtf32 + a_uOutputDataSize, - lenientConversion); - } - else if (sizeof(wchar_t) == sizeof(UTF16)) { - UTF16 * pUtf16 = (UTF16 *) a_pOutputData; - retval = ConvertUTF8toUTF16( - &pUtf8, pUtf8 + a_uInputDataLen, - &pUtf16, pUtf16 + a_uOutputDataSize, - lenientConversion); + /* CAVEAT: mbrtoc32() uses the current locale encoding. + * Non-ASCII characters are only decoded correctly when the + * caller has set a UTF-8 locale, e.g.: + * setlocale(LC_CTYPE, "en_US.UTF-8") or setlocale(LC_ALL, "") */ + const char * src = a_pInputData; + const char * srcEnd = a_pInputData + a_uInputDataLen; + SI_CHAR * dst = a_pOutputData; + SI_CHAR * dstEnd = a_pOutputData + a_uOutputDataSize; + mbstate_t state; + memset(&state, 0, sizeof(state)); + while (src < srcEnd) { + if (dst >= dstEnd) return false; + char32_t cp; + size_t n = mbrtoc32(&cp, src, (size_t)(srcEnd - src), &state); + if (n == (size_t)-1 || n == (size_t)-2) return false; + if (n == (size_t)-3) { /* cp already set, no bytes consumed */ } + else if (n == 0) { cp = 0; src++; } + else { src += n; } + if (sizeof(SI_CHAR) >= sizeof(char32_t)) { + *dst++ = (SI_CHAR)cp; + } else { + if (cp < 0x10000) { + *dst++ = (SI_CHAR)cp; + } else { + if (dst + 1 >= dstEnd) return false; + cp -= 0x10000; + *dst++ = (SI_CHAR)(0xD800 + (cp >> 10)); + *dst++ = (SI_CHAR)(0xDC00 + (cp & 0x3FF)); + } + } } - return retval == conversionOK; + return true; } // convert to wchar_t @@ -3210,35 +3220,35 @@ class SI_ConvertW { ) { if (m_bStoreIsUtf8) { - // calc input string length (SI_CHAR type and size independent) - size_t uInputLen = 0; - while (a_pInputData[uInputLen]) { - ++uInputLen; - } - ++uInputLen; // include the NULL char - - // This uses the Unicode reference implementation to do the - // conversion from wchar_t to UTF-8. The required files are - // ConvertUTF.h and ConvertUTF.c which should be included in - // the distribution but are publicly available from unicode.org - // at http://www.unicode.org/Public/PROGRAMS/CVTUTF/ - ConversionResult retval; - UTF8 * pUtf8 = (UTF8 *) a_pOutputData; - if (sizeof(wchar_t) == sizeof(UTF32)) { - const UTF32 * pUtf32 = (const UTF32 *) a_pInputData; - retval = ConvertUTF32toUTF8( - &pUtf32, pUtf32 + uInputLen, - &pUtf8, pUtf8 + a_uOutputDataSize, - lenientConversion); - } - else if (sizeof(wchar_t) == sizeof(UTF16)) { - const UTF16 * pUtf16 = (const UTF16 *) a_pInputData; - retval = ConvertUTF16toUTF8( - &pUtf16, pUtf16 + uInputLen, - &pUtf8, pUtf8 + a_uOutputDataSize, - lenientConversion); + /* CAVEAT: c32rtomb() uses the current locale encoding. + * Non-ASCII characters are only encoded correctly when the + * caller has set a UTF-8 locale, e.g.: + * setlocale(LC_CTYPE, "en_US.UTF-8") or setlocale(LC_ALL, "") */ + const SI_CHAR * src = a_pInputData; + char * dst = a_pOutputData; + char * dstEnd = a_pOutputData + a_uOutputDataSize; + mbstate_t state; + memset(&state, 0, sizeof(state)); + while (*src) { + char32_t cp; + if (sizeof(SI_CHAR) >= sizeof(char32_t)) { + cp = (char32_t)(*src++); + } else { + char16_t w = (char16_t)(*src++); + if (w >= 0xD800 && w <= 0xDBFF && *src >= 0xDC00 && *src <= 0xDFFF) + cp = 0x10000 + ((char32_t)(w - 0xD800) << 10) + ((char16_t)(*src++) - 0xDC00); + else + cp = (char32_t)w; + } + char buf[4]; /* 4 bytes max for any UTF-8 code point */ + size_t n = c32rtomb(buf, cp, &state); + if (n == (size_t)-1 || (size_t)(dstEnd - dst) < n) return false; + memcpy(dst, buf, n); + dst += n; } - return retval == conversionOK; + if (dst >= dstEnd) return false; + *dst = '\0'; + return true; } else { size_t retval = wcstombs(a_pOutputData, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index afdfb69..ea29e92 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,7 +38,6 @@ set(TEST_SOURCES ts-multiline.cpp ts-casesensitivity.cpp ts-generic.cpp - ${CMAKE_SOURCE_DIR}/ConvertUTF.c ) # ts-wchar.cpp uses wchar_t which is primarily for Windows