/* ********************************************************************** * Copyright (C) 1999-2009, International Business Machines * Corporation and others. All Rights Reserved. ********************************************************************** * * ucnv_cnv.h: * Definitions for converter implementations. * * Modification History: * * Date Name Description * 05/09/00 helena Added implementation to handle fallback mappings. * 06/29/2000 helena Major rewrite of the callback APIs. */ #ifndef UCNV_CNV_H #define UCNV_CNV_H #include "unicode/utypes.h" #if !UCONFIG_NO_CONVERSION #include "unicode/ucnv.h" #include "unicode/ucnv_err.h" #include "unicode/uset.h" #include "uset_imp.h" U_CDECL_BEGIN /* this is used in fromUnicode DBCS tables as an "unassigned" marker */ #define missingCharMarker 0xFFFF /* * #define missingUCharMarker 0xfffe * * commented out because there are actually two values used in toUnicode tables: * U+fffe "unassigned" * U+ffff "illegal" */ /** Forward declaration, see ucnv_bld.h */ struct UConverterSharedData; typedef struct UConverterSharedData UConverterSharedData; /* function types for UConverterImpl ---------------------------------------- */ /* struct with arguments for UConverterLoad and ucnv_load() */ typedef struct { int32_t size; /* sizeof(UConverterLoadArgs) */ int32_t nestedLoads; /* count nested ucnv_load() calls */ UBool onlyTestIsLoadable; /* input: don't actually load */ UBool reserved0; /* reserved - for good alignment of the pointers */ int16_t reserved; /* reserved - for good alignment of the pointers */ uint32_t options; const char *pkg, *name, *locale; } UConverterLoadArgs; typedef void (*UConverterLoad) (UConverterSharedData *sharedData, UConverterLoadArgs *pArgs, const uint8_t *raw, UErrorCode *pErrorCode); typedef void (*UConverterUnload) (UConverterSharedData *sharedData); typedef void (*UConverterOpen) (UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *pErrorCode); typedef void (*UConverterClose) (UConverter *cnv); typedef enum UConverterResetChoice { UCNV_RESET_BOTH, UCNV_RESET_TO_UNICODE, UCNV_RESET_FROM_UNICODE } UConverterResetChoice; typedef void (*UConverterReset) (UConverter *cnv, UConverterResetChoice choice); /* * Converter implementation function(s) for ucnv_toUnicode(). * If the toUnicodeWithOffsets function pointer is NULL, * then the toUnicode function will be used and the offsets will be set to -1. * * Must maintain state across buffers. Use toUBytes[toULength] for partial input * sequences; it will be checked in ucnv.c at the end of the input stream * to detect truncated input. * Some converters may need additional detection and may then set U_TRUNCATED_CHAR_FOUND. * * The toUnicodeWithOffsets must write exactly as many offset values as target * units. Write offset values of -1 for when the source index corresponding to * the output unit is not known (e.g., the character started in an earlier buffer). * The pArgs->offsets pointer need not be moved forward. * * At function return, either one of the following conditions must be true: * - U_BUFFER_OVERFLOW_ERROR and the target is full: target==targetLimit * - another error code with toUBytes[toULength] set to the offending input * - no error, and the source is consumed: source==sourceLimit * * The ucnv.c code will handle the end of the input (reset) * (reset, and truncation detection) and callbacks. */ typedef void (*UConverterToUnicode) (UConverterToUnicodeArgs *, UErrorCode *); /* * Same rules as for UConverterToUnicode. * A lead surrogate is kept in fromUChar32 across buffers, and if an error * occurs, then the offending input code point must be put into fromUChar32 * as well. */ typedef void (*UConverterFromUnicode) (UConverterFromUnicodeArgs *, UErrorCode *); /* * Converter implementation function for ucnv_convertEx(), for direct conversion * between two charsets without pivoting through UTF-16. * The rules are the same as for UConverterToUnicode and UConverterFromUnicode. * In addition, * - The toUnicode side must behave and keep state exactly like the * UConverterToUnicode implementation for the same source charset. * - A U_USING_DEFAULT_WARNING can be set to request to temporarily fall back * to pivoting. When this function is called, the conversion framework makes * sure that this warning is not set on input. * - Continuing a partial match and flushing the toUnicode replay buffer * are handled by pivoting, using the toUnicode and fromUnicode functions. */ typedef void (*UConverterConvert) (UConverterFromUnicodeArgs *pFromUArgs, UConverterToUnicodeArgs *pToUArgs, UErrorCode *pErrorCode); /* * Converter implementation function for ucnv_getNextUChar(). * If the function pointer is NULL, then the toUnicode function will be used. * * Will be called at a character boundary (toULength==0). * May return with * - U_INDEX_OUTOFBOUNDS_ERROR if there was no output for the input * (the return value will be ignored) * - U_TRUNCATED_CHAR_FOUND or another error code (never U_BUFFER_OVERFLOW_ERROR!) * with toUBytes[toULength] set to the offending input * (the return value will be ignored) * - return UCNV_GET_NEXT_UCHAR_USE_TO_U, without moving the source pointer, * to indicate that the ucnv.c code shall call the toUnicode function instead * - return a real code point result * * Unless UCNV_GET_NEXT_UCHAR_USE_TO_U is returned, the source bytes must be consumed. * * The ucnv.c code will handle the end of the input (reset) * (except for truncation detection!) and callbacks. */ typedef UChar32 (*UConverterGetNextUChar) (UConverterToUnicodeArgs *, UErrorCode *); typedef void (*UConverterGetStarters)(const UConverter* converter, UBool starters[256], UErrorCode *pErrorCode); /* If this function pointer is null or if the function returns null * the name field in static data struct should be returned by * ucnv_getName() API function */ typedef const char * (*UConverterGetName) (const UConverter *cnv); /** * Write the codepage substitution character. * If this function is not set, then ucnv_cbFromUWriteSub() writes * the substitution character from UConverter. * For stateful converters, it is typically necessary to handle this * specificially for the converter in order to properly maintain the state. */ typedef void (*UConverterWriteSub) (UConverterFromUnicodeArgs *pArgs, int32_t offsetIndex, UErrorCode *pErrorCode); /** * For converter-specific safeClone processing * If this function is not set, then ucnv_safeClone assumes that the converter has no private data that changes * after the converter is done opening. * If this function is set, then it is called just after a memcpy() of * converter data to the new, empty converter, and is expected to set up * the initial state of the converter. It is not expected to increment the * reference counts of the standard data types such as the shared data. */ typedef UConverter * (*UConverterSafeClone) (const UConverter *cnv, void *stackBuffer, int32_t *pBufferSize, UErrorCode *status); /** * Filters for some ucnv_getUnicodeSet() implementation code. */ typedef enum UConverterSetFilter { UCNV_SET_FILTER_NONE, UCNV_SET_FILTER_DBCS_ONLY, UCNV_SET_FILTER_2022_CN, UCNV_SET_FILTER_SJIS, UCNV_SET_FILTER_GR94DBCS, UCNV_SET_FILTER_HZ, UCNV_SET_FILTER_COUNT } UConverterSetFilter; /** * Fills the set of Unicode code points that can be converted by an ICU converter. * The API function ucnv_getUnicodeSet() clears the USet before calling * the converter's getUnicodeSet() implementation; the converter should only * add the appropriate code points to allow recursive use. * For example, the ISO-2022-JP converter will call each subconverter's * getUnicodeSet() implementation to consecutively add code points to * the same USet, which will result in a union of the sets of all subconverters. * * For more documentation, see ucnv_getUnicodeSet() in ucnv.h. */ typedef void (*UConverterGetUnicodeSet) (const UConverter *cnv, const USetAdder *sa, UConverterUnicodeSet which, UErrorCode *pErrorCode); UBool CONVERSION_U_SUCCESS (UErrorCode err); /** * UConverterImpl contains all the data and functions for a converter type. * Its function pointers work much like a C++ vtable. * Many converter types need to define only a subset of the functions; * when a function pointer is NULL, then a default action will be performed. * * Every converter type must implement toUnicode, fromUnicode, and getNextUChar, * otherwise the converter may crash. * Every converter type that has variable-length codepage sequences should * also implement toUnicodeWithOffsets and fromUnicodeWithOffsets for * correct offset handling. * All other functions may or may not be implemented - it depends only on * whether the converter type needs them. * * When open() fails, then close() will be called, if present. */ struct UConverterImpl { UConverterType type; UConverterLoad load; UConverterUnload unload; UConverterOpen open; UConverterClose close; UConverterReset reset; UConverterToUnicode toUnicode; UConverterToUnicode toUnicodeWithOffsets; UConverterFromUnicode fromUnicode; UConverterFromUnicode fromUnicodeWithOffsets; UConverterGetNextUChar getNextUChar; UConverterGetStarters getStarters; UConverterGetName getName; UConverterWriteSub writeSub; UConverterSafeClone safeClone; UConverterGetUnicodeSet getUnicodeSet; UConverterConvert toUTF8; UConverterConvert fromUTF8; }; extern const UConverterSharedData _MBCSData, _Latin1Data, _UTF8Data, _UTF16BEData, _UTF16LEData, _UTF32BEData, _UTF32LEData, _ISO2022Data, _LMBCSData1,_LMBCSData2, _LMBCSData3, _LMBCSData4, _LMBCSData5, _LMBCSData6, _LMBCSData8,_LMBCSData11,_LMBCSData16,_LMBCSData17,_LMBCSData18,_LMBCSData19, _HZData,_ISCIIData, _SCSUData, _ASCIIData, _UTF7Data, _Bocu1Data, _UTF16Data, _UTF32Data, _CESU8Data, _IMAPData; U_CDECL_END /** Always use fallbacks from codepage to Unicode */ #define TO_U_USE_FALLBACK(useFallback) TRUE #define UCNV_TO_U_USE_FALLBACK(cnv) TRUE /** Use fallbacks from Unicode to codepage when cnv->useFallback or for private-use code points */ #define IS_PRIVATE_USE(c) ((uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000) #define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || IS_PRIVATE_USE(c)) #define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c) /** * Magic number for ucnv_getNextUChar(), returned by a * getNextUChar() implementation to indicate to use the converter's toUnicode() * instead of the native function. * @internal */ #define UCNV_GET_NEXT_UCHAR_USE_TO_U -9 U_CFUNC void ucnv_getCompleteUnicodeSet(const UConverter *cnv, const USetAdder *sa, UConverterUnicodeSet which, UErrorCode *pErrorCode); U_CFUNC void ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv, const USetAdder *sa, UConverterUnicodeSet which, UErrorCode *pErrorCode); U_CFUNC void ucnv_fromUWriteBytes(UConverter *cnv, const char *bytes, int32_t length, char **target, const char *targetLimit, int32_t **offsets, int32_t sourceIndex, UErrorCode *pErrorCode); U_CFUNC void ucnv_toUWriteUChars(UConverter *cnv, const UChar *uchars, int32_t length, UChar **target, const UChar *targetLimit, int32_t **offsets, int32_t sourceIndex, UErrorCode *pErrorCode); U_CFUNC void ucnv_toUWriteCodePoint(UConverter *cnv, UChar32 c, UChar **target, const UChar *targetLimit, int32_t **offsets, int32_t sourceIndex, UErrorCode *pErrorCode); #endif #endif /* UCNV_CNV */