2 *******************************************************************************
4 * Copyright (C) 2003-2009, International Business Machines
5 * Corporation and others. All Rights Reserved.
7 *******************************************************************************
8 * file name: gensprep.c
10 * tab size: 8 (not used)
13 * created on: 2003-02-06
14 * created by: Ram Viswanadha
16 * This program reads the Profile.txt files,
17 * parses them, and extracts the data for StringPrep profile.
18 * It then preprocesses it and writes a binary file for efficient use
19 * in various StringPrep conversion processes.
22 #define USPREP_TYPE_NAMES_ARRAY 1
34 #include "unicode/udata.h"
35 #include "unicode/utypes.h"
36 #include "unicode/putil.h"
43 UBool beVerbose=FALSE, haveCopyright=TRUE;
45 #define NORM_CORRECTIONS_FILE_NAME "NormalizationCorrections.txt"
47 #define NORMALIZE_DIRECTIVE "normalize"
48 #define NORMALIZE_DIRECTIVE_LEN 9
49 #define CHECK_BIDI_DIRECTIVE "check-bidi"
50 #define CHECK_BIDI_DIRECTIVE_LEN 10
52 /* prototypes --------------------------------------------------------------- */
55 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode);
58 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode);
61 /* -------------------------------------------------------------------------- */
63 static UOption options[]={
65 UOPTION_HELP_QUESTION_MARK,
72 { "normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0 },
73 { "norm-correction", NULL, NULL, NULL, 'm', UOPT_REQUIRES_ARG, 0 },
74 { "check-bidi", NULL, NULL, NULL, 'k', UOPT_NO_ARG, 0},
75 { "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 },
93 static int printHelp(int argc, char* argv[]){
95 * Broken into chucks because the C89 standard says the minimum
96 * required supported string length is 509 bytes.
99 "Usage: %s [-options] [file_name]\n"
101 "Read the files specified and\n"
102 "create a binary file [package-name]_[bundle-name]." DATA_TYPE " with the StringPrep profile data\n"
107 "\t-h or -? or --help print this usage text\n"
108 "\t-v or --verbose verbose output\n"
109 "\t-c or --copyright include a copyright notice\n");
111 "\t-d or --destdir destination directory, followed by the path\n"
112 "\t-s or --sourcedir source directory of ICU data, followed by the path\n"
113 "\t-b or --bundle-name generate the ouput data file with the name specified\n"
114 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
115 "\t followed by path, defaults to %s\n",
116 u_getDataDirectory());
118 "\t-n or --normalize turn on the option for normalization and include mappings\n"
119 "\t from NormalizationCorrections.txt from the given path,\n"
120 "\t e.g: /test/icu/source/data/unidata\n");
122 "\t-m or --norm-correction use NormalizationCorrections.txt from the given path\n"
123 "\t when the input file contains a normalization directive.\n"
124 "\t unlike -n/--normalize, this option does not force the\n"
125 "\t normalization.\n");
127 "\t-k or --check-bidi turn on the option for checking for BiDi in the profile\n"
128 "\t-u or --unicode version of Unicode to be used with this profile followed by the version\n"
130 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
135 main(int argc, char* argv[]) {
137 char* filename = NULL;
139 const char *srcDir=NULL, *destDir=NULL, *icuUniDataDir=NULL;
140 const char *bundleName=NULL, *inputFileName = NULL;
142 int32_t sprepOptions = 0;
144 UErrorCode errorCode=U_ZERO_ERROR;
146 U_MAIN_INIT_ARGS(argc, argv);
148 /* preset then read command line options */
149 options[DESTDIR].value=u_getDataDirectory();
150 options[SOURCEDIR].value="";
151 options[UNICODE_VERSION].value="0"; /* don't assume the unicode version */
152 options[BUNDLE_NAME].value = DATA_NAME;
153 options[NORMALIZE].value = "";
155 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
157 /* error handling, printing usage message */
160 "error in command line argument \"%s\"\n",
163 if(argc<0 || options[HELP].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
164 return printHelp(argc, argv);
168 /* get the options values */
169 beVerbose=options[VERBOSE].doesOccur;
170 haveCopyright=options[COPYRIGHT].doesOccur;
171 srcDir=options[SOURCEDIR].value;
172 destDir=options[DESTDIR].value;
173 bundleName = options[BUNDLE_NAME].value;
174 if(options[NORMALIZE].doesOccur) {
175 icuUniDataDir = options[NORMALIZE].value;
177 icuUniDataDir = options[NORM_CORRECTION_DIR].value;
181 /* print the help message */
182 return printHelp(argc, argv);
184 inputFileName = argv[1];
186 if(!options[UNICODE_VERSION].doesOccur){
187 return printHelp(argc, argv);
189 if(options[ICUDATADIR].doesOccur) {
190 u_setDataDirectory(options[ICUDATADIR].value);
195 "gensprep writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE
196 " because UCONFIG_NO_IDNA is set, \n"
197 "see icu/source/common/unicode/uconfig.h\n");
198 generateData(destDir, bundleName);
202 setUnicodeVersion(options[UNICODE_VERSION].value);
203 filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + 300); /* hopefully this should be enough */
205 /* prepare the filename beginning with the source dir */
206 if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL && uprv_strchr(srcDir,U_FILE_ALT_SEP_CHAR) == NULL){
208 filename[1] = U_FILE_SEP_CHAR;
209 uprv_strcpy(filename+2,srcDir);
211 uprv_strcpy(filename, srcDir);
214 basename=filename+uprv_strlen(filename);
215 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
216 *basename++=U_FILE_SEP_CHAR;
222 /* process the file */
223 uprv_strcpy(basename,inputFileName);
224 parseMappings(filename,FALSE, &errorCode);
225 if(U_FAILURE(errorCode)) {
226 fprintf(stderr, "Could not open file %s for reading. Error: %s \n", filename, u_errorName(errorCode));
230 if(options[NORMALIZE].doesOccur){ /* this option might be set by @normalize;; in the source file */
231 /* set up directory for NormalizationCorrections.txt */
232 uprv_strcpy(filename,icuUniDataDir);
233 basename=filename+uprv_strlen(filename);
234 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
235 *basename++=U_FILE_SEP_CHAR;
238 *basename++=U_FILE_SEP_CHAR;
239 uprv_strcpy(basename,NORM_CORRECTIONS_FILE_NAME);
241 parseNormalizationCorrections(filename,&errorCode);
242 if(U_FAILURE(errorCode)){
243 fprintf(stderr,"Could not open file %s for reading \n", filename);
246 sprepOptions |= _SPREP_NORMALIZATION_ON;
249 if(options[CHECK_BIDI].doesOccur){ /* this option might be set by @check-bidi;; in the source file */
250 sprepOptions |= _SPREP_CHECK_BIDI_ON;
253 setOptions(sprepOptions);
255 /* process parsed data */
256 if(U_SUCCESS(errorCode)) {
257 /* write the data file */
258 generateData(destDir, bundleName);
272 static void U_CALLCONV
273 normalizationCorrectionsLineFn(void *context,
274 char *fields[][2], int32_t fieldCount,
275 UErrorCode *pErrorCode) {
276 uint32_t mapping[40];
280 UVersionInfo version;
281 UVersionInfo thisVersion;
283 /* get the character code, field 0 */
284 code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
285 if(U_FAILURE(*pErrorCode)) {
286 fprintf(stderr, "gensprep: error parsing NormalizationCorrections.txt mapping at %s\n", fields[0][0]);
289 /* Original (erroneous) decomposition */
292 /* parse the mapping string */
293 length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);
295 /* ignore corrected decomposition */
297 u_versionFromString(version,fields[3][0] );
298 u_versionFromString(thisVersion, "3.2.0");
302 if(U_FAILURE(*pErrorCode)) {
303 fprintf(stderr, "gensprep error parsing NormalizationCorrections.txt of U+%04lx - %s\n",
304 (long)code, u_errorName(*pErrorCode));
308 /* store the mapping */
309 if( version[0] > thisVersion[0] ||
310 ((version[0]==thisVersion[0]) && (version[1] > thisVersion[1]))
312 storeMapping(code,mapping, length, USPREP_MAP, pErrorCode);
314 setUnicodeVersionNC(version);
318 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) {
321 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
325 u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode);
327 /* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */
329 if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) {
330 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
335 static void U_CALLCONV
336 strprepProfileLineFn(void *context,
337 char *fields[][2], int32_t fieldCount,
338 UErrorCode *pErrorCode) {
339 uint32_t mapping[40];
343 /*UBool* mapWithNorm = (UBool*) context;*/
344 const char* typeName;
345 uint32_t rangeStart=0,rangeEnd =0;
346 const char* filename = (const char*) context;
349 s = u_skipWhitespace(fields[0][0]);
351 /* special directive */
353 length = fields[0][1] - s;
354 if (length >= NORMALIZE_DIRECTIVE_LEN
355 && uprv_strncmp(s, NORMALIZE_DIRECTIVE, NORMALIZE_DIRECTIVE_LEN) == 0) {
356 options[NORMALIZE].doesOccur = TRUE;
359 else if (length >= CHECK_BIDI_DIRECTIVE_LEN
360 && uprv_strncmp(s, CHECK_BIDI_DIRECTIVE, CHECK_BIDI_DIRECTIVE_LEN) == 0) {
361 options[CHECK_BIDI].doesOccur = TRUE;
365 fprintf(stderr, "gensprep error parsing a directive %s.", fields[0][0]);
369 typeName = fields[2][0];
372 if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){
374 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
375 if(U_FAILURE(*pErrorCode)){
376 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
380 /* store the range */
381 storeRange(rangeStart,rangeEnd,USPREP_UNASSIGNED, pErrorCode);
383 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){
385 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
386 if(U_FAILURE(*pErrorCode)){
387 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
391 /* store the range */
392 storeRange(rangeStart,rangeEnd,USPREP_PROHIBITED, pErrorCode);
394 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){
396 /* get the character code, field 0 */
397 code=(uint32_t)uprv_strtoul(s, &end, 16);
398 if(end<=s || end!=fields[0][1]) {
399 fprintf(stderr, "gensprep: syntax error in field 0 at %s\n", fields[0][0]);
400 *pErrorCode=U_PARSE_ERROR;
404 /* parse the mapping string */
405 length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode);
407 /* store the mapping */
408 storeMapping(code,mapping, length,USPREP_MAP, pErrorCode);
411 *pErrorCode = U_INVALID_FORMAT_ERROR;
414 if(U_FAILURE(*pErrorCode)) {
415 fprintf(stderr, "gensprep error parsing %s line %s at %s. Error: %s\n",filename,
416 fields[0][0],fields[2][0],u_errorName(*pErrorCode));
423 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode) {
426 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
430 u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode);
432 /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/
434 if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
435 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
441 #endif /* #if !UCONFIG_NO_IDNA */
444 * Hey, Emacs, please set the following:
447 * indent-tabs-mode: nil