1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
4 **********************************************************************
5 * Copyright (C) 2005-2013, International Business Machines
6 * Corporation and others. All Rights Reserved.
7 **********************************************************************
12 * created on: 2005Aug04
13 * created by: Andy Heninger
15 * ICU Character Set Detection, API for C
17 * Draft version 18 Oct 2005
24 #include "unicode/utypes.h"
26 #if !UCONFIG_NO_CONVERSION
28 #include "unicode/localpointer.h"
29 #include "unicode/uenum.h"
33 * \brief C API: Charset Detection API
35 * This API provides a facility for detecting the
36 * charset or encoding of character data in an unknown text format.
37 * The input data can be from an array of bytes.
39 * Character set detection is at best an imprecise operation. The detection
40 * process will attempt to identify the charset that best matches the characteristics
41 * of the byte data, but the process is partly statistical in nature, and
42 * the results can not be guaranteed to always be correct.
44 * For best accuracy in charset detection, the input data should be primarily
45 * in a single language, and a minimum of a few hundred bytes worth of plain text
46 * in the language are needed. The detection process will attempt to
47 * ignore html or xml style markup that could otherwise obscure the content.
51 struct UCharsetDetector;
53 * Structure representing a charset detector
56 typedef struct UCharsetDetector UCharsetDetector;
60 * Opaque structure representing a match that was identified
61 * from a charset detection operation.
64 typedef struct UCharsetMatch UCharsetMatch;
67 * Open a charset detector.
69 * @param status Any error conditions occurring during the open
70 * operation are reported back in this variable.
71 * @return the newly opened charset detector.
74 U_STABLE UCharsetDetector * U_EXPORT2
75 ucsdet_open(UErrorCode *status);
78 * Close a charset detector. All storage and any other resources
79 * owned by this charset detector will be released. Failure to
80 * close a charset detector when finished with it can result in
81 * memory leaks in the application.
83 * @param ucsd The charset detector to be closed.
86 U_STABLE void U_EXPORT2
87 ucsdet_close(UCharsetDetector *ucsd);
89 #if U_SHOW_CPLUSPLUS_API
94 * \class LocalUCharsetDetectorPointer
95 * "Smart pointer" class, closes a UCharsetDetector via ucsdet_close().
96 * For most methods see the LocalPointerBase base class.
98 * @see LocalPointerBase
102 U_DEFINE_LOCAL_OPEN_POINTER(LocalUCharsetDetectorPointer, UCharsetDetector, ucsdet_close);
109 * Set the input byte data whose charset is to detected.
111 * Ownership of the input text byte array remains with the caller.
112 * The input string must not be altered or deleted until the charset
113 * detector is either closed or reset to refer to different input text.
115 * @param ucsd the charset detector to be used.
116 * @param textIn the input text of unknown encoding. .
117 * @param len the length of the input text, or -1 if the text
119 * @param status any error conditions are reported back in this variable.
123 U_STABLE void U_EXPORT2
124 ucsdet_setText(UCharsetDetector *ucsd, const char *textIn, int32_t len, UErrorCode *status);
127 /** Set the declared encoding for charset detection.
128 * The declared encoding of an input text is an encoding obtained
129 * by the user from an http header or xml declaration or similar source that
130 * can be provided as an additional hint to the charset detector.
132 * How and whether the declared encoding will be used during the
133 * detection process is TBD.
135 * @param ucsd the charset detector to be used.
136 * @param encoding an encoding for the current data obtained from
137 * a header or declaration or other source outside
138 * of the byte data itself.
139 * @param length the length of the encoding name, or -1 if the name string
141 * @param status any error conditions are reported back in this variable.
145 U_STABLE void U_EXPORT2
146 ucsdet_setDeclaredEncoding(UCharsetDetector *ucsd, const char *encoding, int32_t length, UErrorCode *status);
150 * Return the charset that best matches the supplied input data.
152 * Note though, that because the detection
153 * only looks at the start of the input data,
154 * there is a possibility that the returned charset will fail to handle
155 * the full set of input data.
157 * The returned UCharsetMatch object is owned by the UCharsetDetector.
158 * It will remain valid until the detector input is reset, or until
159 * the detector is closed.
161 * The function will fail if
163 * <li>no charset appears to match the data.</li>
164 * <li>no input text has been provided</li>
167 * @param ucsd the charset detector to be used.
168 * @param status any error conditions are reported back in this variable.
169 * @return a UCharsetMatch representing the best matching charset,
170 * or NULL if no charset matches the byte data.
174 U_STABLE const UCharsetMatch * U_EXPORT2
175 ucsdet_detect(UCharsetDetector *ucsd, UErrorCode *status);
179 * Find all charset matches that appear to be consistent with the input,
180 * returning an array of results. The results are ordered with the
181 * best quality match first.
183 * Because the detection only looks at a limited amount of the
184 * input byte data, some of the returned charsets may fail to handle
185 * the all of input data.
187 * The returned UCharsetMatch objects are owned by the UCharsetDetector.
188 * They will remain valid until the detector is closed or modified
193 * <li>no charsets appear to match the input data.</li>
194 * <li>no input text has been provided</li>
197 * @param ucsd the charset detector to be used.
198 * @param matchesFound pointer to a variable that will be set to the
199 * number of charsets identified that are consistent with
200 * the input data. Output only.
201 * @param status any error conditions are reported back in this variable.
202 * @return A pointer to an array of pointers to UCharSetMatch objects.
203 * This array, and the UCharSetMatch instances to which it refers,
204 * are owned by the UCharsetDetector, and will remain valid until
205 * the detector is closed or modified.
208 U_STABLE const UCharsetMatch ** U_EXPORT2
209 ucsdet_detectAll(UCharsetDetector *ucsd, int32_t *matchesFound, UErrorCode *status);
214 * Get the name of the charset represented by a UCharsetMatch.
216 * The storage for the returned name string is owned by the
217 * UCharsetMatch, and will remain valid while the UCharsetMatch
220 * The name returned is suitable for use with the ICU conversion APIs.
222 * @param ucsm The charset match object.
223 * @param status Any error conditions are reported back in this variable.
224 * @return The name of the matching charset.
228 U_STABLE const char * U_EXPORT2
229 ucsdet_getName(const UCharsetMatch *ucsm, UErrorCode *status);
232 * Get a confidence number for the quality of the match of the byte
233 * data with the charset. Confidence numbers range from zero to 100,
234 * with 100 representing complete confidence and zero representing
237 * The confidence values are somewhat arbitrary. They define an
238 * an ordering within the results for any single detection operation
239 * but are not generally comparable between the results for different input.
241 * A confidence value of ten does have a general meaning - it is used
242 * for charsets that can represent the input data, but for which there
243 * is no other indication that suggests that the charset is the correct one.
244 * Pure 7 bit ASCII data, for example, is compatible with a
245 * great many charsets, most of which will appear as possible matches
246 * with a confidence of 10.
248 * @param ucsm The charset match object.
249 * @param status Any error conditions are reported back in this variable.
250 * @return A confidence number for the charset match.
254 U_STABLE int32_t U_EXPORT2
255 ucsdet_getConfidence(const UCharsetMatch *ucsm, UErrorCode *status);
258 * Get the RFC 3066 code for the language of the input data.
260 * The Charset Detection service is intended primarily for detecting
261 * charsets, not language. For some, but not all, charsets, a language is
262 * identified as a byproduct of the detection process, and that is what
263 * is returned by this function.
266 * 1. Language information is not available for input data encoded in
267 * all charsets. In particular, no language is identified
268 * for UTF-8 input data.
270 * 2. Closely related languages may sometimes be confused.
272 * If more accurate language detection is required, a linguistic
273 * analysis package should be used.
275 * The storage for the returned name string is owned by the
276 * UCharsetMatch, and will remain valid while the UCharsetMatch
279 * @param ucsm The charset match object.
280 * @param status Any error conditions are reported back in this variable.
281 * @return The RFC 3066 code for the language of the input data, or
282 * an empty string if the language could not be determined.
286 U_STABLE const char * U_EXPORT2
287 ucsdet_getLanguage(const UCharsetMatch *ucsm, UErrorCode *status);
291 * Get the entire input text as a UChar string, placing it into
292 * a caller-supplied buffer. A terminating
293 * NUL character will be appended to the buffer if space is available.
295 * The number of UChars in the output string, not including the terminating
298 * If the supplied buffer is smaller than required to hold the output,
299 * the contents of the buffer are undefined. The full output string length
300 * (in UChars) is returned as always, and can be used to allocate a buffer
301 * of the correct size.
304 * @param ucsm The charset match object.
305 * @param buf A UChar buffer to be filled with the converted text data.
306 * @param cap The capacity of the buffer in UChars.
307 * @param status Any error conditions are reported back in this variable.
308 * @return The number of UChars in the output string.
312 U_STABLE int32_t U_EXPORT2
313 ucsdet_getUChars(const UCharsetMatch *ucsm,
314 UChar *buf, int32_t cap, UErrorCode *status);
319 * Get an iterator over the set of all detectable charsets -
320 * over the charsets that are known to the charset detection
323 * The returned UEnumeration provides access to the names of
327 * The state of the Charset detector that is passed in does not
328 * affect the result of this function, but requiring a valid, open
329 * charset detector as a parameter insures that the charset detection
330 * service has been safely initialized and that the required detection
334 * <b>Note:</b> Multiple different charset encodings in a same family may use
335 * a single shared name in this implementation. For example, this method returns
336 * an array including "ISO-8859-1" (ISO Latin 1), but not including "windows-1252"
337 * (Windows Latin 1). However, actual detection result could be "windows-1252"
338 * when the input data matches Latin 1 code points with any points only available
341 * @param ucsd a Charset detector.
342 * @param status Any error conditions are reported back in this variable.
343 * @return an iterator providing access to the detectable charset names.
346 U_STABLE UEnumeration * U_EXPORT2
347 ucsdet_getAllDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status);
350 * Test whether input filtering is enabled for this charset detector.
351 * Input filtering removes text that appears to be HTML or xml
352 * markup from the input before applying the code page detection
355 * @param ucsd The charset detector to check.
356 * @return TRUE if filtering is enabled.
360 U_STABLE UBool U_EXPORT2
361 ucsdet_isInputFilterEnabled(const UCharsetDetector *ucsd);
365 * Enable filtering of input text. If filtering is enabled,
366 * text within angle brackets ("<" and ">") will be removed
367 * before detection, which will remove most HTML or xml markup.
369 * @param ucsd the charset detector to be modified.
370 * @param filter <code>true</code> to enable input text filtering.
371 * @return The previous setting.
375 U_STABLE UBool U_EXPORT2
376 ucsdet_enableInputFilter(UCharsetDetector *ucsd, UBool filter);
378 #ifndef U_HIDE_INTERNAL_API
380 * Get an iterator over the set of detectable charsets -
381 * over the charsets that are enabled by the specified charset detector.
383 * The returned UEnumeration provides access to the names of
386 * @param ucsd a Charset detector.
387 * @param status Any error conditions are reported back in this variable.
388 * @return an iterator providing access to the detectable charset names by
389 * the specified charset detector.
392 U_INTERNAL UEnumeration * U_EXPORT2
393 ucsdet_getDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status);
396 * Enable or disable individual charset encoding.
397 * A name of charset encoding must be included in the names returned by
398 * {@link #getAllDetectableCharsets()}.
400 * @param ucsd a Charset detector.
401 * @param encoding encoding the name of charset encoding.
402 * @param enabled <code>TRUE</code> to enable, or <code>FALSE</code> to disable the
404 * @param status receives the return status. When the name of charset encoding
405 * is not supported, U_ILLEGAL_ARGUMENT_ERROR is set.
408 U_INTERNAL void U_EXPORT2
409 ucsdet_setDetectableCharset(UCharsetDetector *ucsd, const char *encoding, UBool enabled, UErrorCode *status);
410 #endif /* U_HIDE_INTERNAL_API */
413 #endif /* __UCSDET_H */