2 * Copyright 2006-2007 Adrian Thurston <thurston@cs.queensu.ca>
5 /* This file is part of Ragel.
7 * Ragel is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * Ragel is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Ragel; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
52 * The Scanner for Importing
56 machine inline_token_scan;
60 # Import scanner tokens.
65 IMP_Define IMP_Word IMP_UInt => {
66 int base = tok_ts - token_data;
70 directToParser( inclToParser, fileName, line, column, TK_Word,
71 token_strings[base+nameOff], token_lens[base+nameOff] );
72 directToParser( inclToParser, fileName, line, column, '=', 0, 0 );
73 directToParser( inclToParser, fileName, line, column, TK_UInt,
74 token_strings[base+numOff], token_lens[base+numOff] );
75 directToParser( inclToParser, fileName, line, column, ';', 0, 0 );
78 # Assignment of number.
79 IMP_Word '=' IMP_UInt => {
80 int base = tok_ts - token_data;
84 directToParser( inclToParser, fileName, line, column, TK_Word,
85 token_strings[base+nameOff], token_lens[base+nameOff] );
86 directToParser( inclToParser, fileName, line, column, '=', 0, 0 );
87 directToParser( inclToParser, fileName, line, column, TK_UInt,
88 token_strings[base+numOff], token_lens[base+numOff] );
89 directToParser( inclToParser, fileName, line, column, ';', 0, 0 );
93 IMP_Define IMP_Word IMP_Literal => {
94 int base = tok_ts - token_data;
98 directToParser( inclToParser, fileName, line, column, TK_Word,
99 token_strings[base+nameOff], token_lens[base+nameOff] );
100 directToParser( inclToParser, fileName, line, column, '=', 0, 0 );
101 directToParser( inclToParser, fileName, line, column, TK_Literal,
102 token_strings[base+litOff], token_lens[base+litOff] );
103 directToParser( inclToParser, fileName, line, column, ';', 0, 0 );
106 # Assignment of literal.
107 IMP_Word '=' IMP_Literal => {
108 int base = tok_ts - token_data;
112 directToParser( inclToParser, fileName, line, column, TK_Word,
113 token_strings[base+nameOff], token_lens[base+nameOff] );
114 directToParser( inclToParser, fileName, line, column, '=', 0, 0 );
115 directToParser( inclToParser, fileName, line, column, TK_Literal,
116 token_strings[base+litOff], token_lens[base+litOff] );
117 directToParser( inclToParser, fileName, line, column, ';', 0, 0 );
120 # Catch everything else.
127 void Scanner::flushImport()
130 int *pe = token_data + cur_token;
134 machine inline_token_scan;
142 cur_token = pe - tok_ts;
143 int ts_offset = tok_ts - token_data;
144 memmove( token_data, token_data+ts_offset, cur_token*sizeof(token_data[0]) );
145 memmove( token_strings, token_strings+ts_offset, cur_token*sizeof(token_strings[0]) );
146 memmove( token_lens, token_lens+ts_offset, cur_token*sizeof(token_lens[0]) );
150 void Scanner::directToParser( Parser *toParser, char *tokFileName, int tokLine,
151 int tokColumn, int type, char *tokdata, int toklen )
156 cerr << "scanner:" << tokLine << ":" << tokColumn <<
157 ": sending token to the parser " << Parser_lelNames[type];
158 cerr << " " << toklen;
160 cerr << " " << tokdata;
164 loc.fileName = tokFileName;
168 toParser->token( loc, type, tokdata, toklen );
171 void Scanner::importToken( int token, char *start, char *end )
173 if ( cur_token == max_tokens )
176 token_data[cur_token] = token;
178 token_strings[cur_token] = 0;
179 token_lens[cur_token] = 0;
182 int toklen = end-start;
183 token_lens[cur_token] = toklen;
184 token_strings[cur_token] = new char[toklen+1];
185 memcpy( token_strings[cur_token], start, toklen );
186 token_strings[cur_token][toklen] = 0;
191 void Scanner::pass( int token, char *start, char *end )
193 if ( importMachines )
194 importToken( token, start, end );
202 /* If no errors and we are at the bottom of the include stack (the
203 * source file listed on the command line) then write out the data. */
204 if ( includeDepth == 0 && machineSpec == 0 && machineName == 0 )
205 xmlEscapeHost( output, ts, te-ts );
209 * The scanner for processing sections, includes, imports, etc.
213 machine section_parse;
219 void Scanner::init( )
224 bool Scanner::active()
229 if ( parser == 0 && ! parserExistsError ) {
230 scan_error() << "this specification has no name, nor does any previous"
231 " specification" << endl;
232 parserExistsError = true;
241 ostream &Scanner::scan_error()
243 /* Maintain the error count. */
245 cerr << fileName << ":" << line << ":" << column << ": ";
249 bool Scanner::recursiveInclude( char *inclFileName, char *inclSectionName )
251 for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
252 if ( strcmp( si->fileName, inclFileName ) == 0 &&
253 strcmp( si->sectionName, inclSectionName ) == 0 )
261 void Scanner::updateCol()
266 //cerr << "adding " << te - from << " to column" << endl;
271 void Scanner::handleMachine()
273 /* Assign a name to the machine. */
274 char *machine = word;
276 if ( !importMachines && inclSectionTarg == 0 ) {
277 ignoreSection = false;
279 ParserDictEl *pdEl = parserDict.find( machine );
281 pdEl = new ParserDictEl( machine );
282 pdEl->value = new Parser( fileName, machine, sectionLoc );
284 parserDict.insert( pdEl );
287 parser = pdEl->value;
289 else if ( !importMachines && strcmp( inclSectionTarg, machine ) == 0 ) {
290 /* found include target */
291 ignoreSection = false;
292 parser = inclToParser;
295 /* ignoring section */
296 ignoreSection = true;
301 void Scanner::handleInclude()
304 char *inclSectionName = word;
305 char **includeChecks = 0;
307 /* Implement defaults for the input file and section name. */
308 if ( inclSectionName == 0 )
309 inclSectionName = parser->sectionName;
312 includeChecks = makeIncludePathChecks( fileName, lit, lit_len );
314 char *test = new char[strlen(fileName)+1];
315 strcpy( test, fileName );
317 includeChecks = new char*[2];
319 includeChecks[0] = test;
320 includeChecks[1] = 0;
324 ifstream *inFile = tryOpenInclude( includeChecks, found );
326 scan_error() << "include: failed to locate file" << endl;
327 char **tried = includeChecks;
328 while ( *tried != 0 )
329 scan_error() << "include: attempted: \"" << *tried++ << '\"' << endl;
332 /* Check for a recursive include structure. Add the current file/section
333 * name then check if what we are including is already in the stack. */
334 includeStack.append( IncludeStackItem( fileName, parser->sectionName ) );
336 if ( recursiveInclude( includeChecks[found], inclSectionName ) )
337 scan_error() << "include: this is a recursive include operation" << endl;
339 Scanner scanner( includeChecks[found], *inFile, output, parser,
340 inclSectionName, includeDepth+1, false );
345 /* Remove the last element (len-1) */
346 includeStack.remove( -1 );
351 void Scanner::handleImport()
354 char **importChecks = makeIncludePathChecks( fileName, lit, lit_len );
356 /* Open the input file for reading. */
358 ifstream *inFile = tryOpenInclude( importChecks, found );
360 scan_error() << "import: could not open import file " <<
361 "for reading" << endl;
362 char **tried = importChecks;
363 while ( *tried != 0 )
364 scan_error() << "import: attempted: \"" << *tried++ << '\"' << endl;
367 Scanner scanner( importChecks[found], *inFile, output, parser,
368 0, includeDepth+1, true );
370 scanner.importToken( 0, 0, 0 );
371 scanner.flushImport();
377 machine section_parse;
379 # Need the defines representing tokens.
382 action clear_words { word = lit = 0; word_len = lit_len = 0; }
383 action store_word { word = tokdata; word_len = toklen; }
384 action store_lit { lit = tokdata; lit_len = toklen; }
386 action mach_err { scan_error() << "bad machine statement" << endl; }
387 action incl_err { scan_error() << "bad include statement" << endl; }
388 action import_err { scan_error() << "bad import statement" << endl; }
389 action write_err { scan_error() << "bad write statement" << endl; }
391 action handle_machine { handleMachine(); }
392 action handle_include { handleInclude(); }
393 action handle_import { handleImport(); }
396 ( KW_Machine TK_Word @store_word ';' ) @handle_machine
397 <>err mach_err <>eof mach_err;
400 TK_Word @store_word ( TK_Literal @store_lit )? |
401 TK_Literal @store_lit
405 ( KW_Include include_names ';' ) @handle_include
406 <>err incl_err <>eof incl_err;
409 ( KW_Import TK_Literal @store_lit ';' ) @handle_import
410 <>err import_err <>eof import_err;
414 if ( active() && machineSpec == 0 && machineName == 0 ) {
416 " def_name=\"" << parser->sectionName << "\""
417 " line=\"" << line << "\""
418 " col=\"" << column << "\""
425 if ( active() && machineSpec == 0 && machineName == 0 )
426 output << "<arg>" << tokdata << "</arg>";
431 if ( active() && machineSpec == 0 && machineName == 0 )
432 output << "</write>\n";
436 ( KW_Write @write_command
437 ( TK_Word @write_arg )+ ';' @write_close )
438 <>err write_err <>eof write_err;
442 /* Send the token off to the parser. */
444 directToParser( parser, fileName, line, column, type, tokdata, toklen );
447 # Catch everything else.
449 ^( KW_Machine | KW_Include | KW_Import | KW_Write ) @handle_token;
460 void Scanner::token( int type, char c )
462 token( type, &c, &c + 1 );
465 void Scanner::token( int type )
470 void Scanner::token( int type, char *start, char *end )
476 tokdata = new char[toklen+1];
477 memcpy( tokdata, start, toklen );
481 processToken( type, tokdata, toklen );
484 void Scanner::processToken( int type, char *tokdata, int toklen )
498 machine section_parse;
504 /* Record the last token for use in controlling the scan of subsequent
509 void Scanner::startSection( )
511 parserExistsError = false;
513 if ( includeDepth == 0 ) {
514 if ( machineSpec == 0 && machineName == 0 )
515 output << "</host>\n";
518 sectionLoc.fileName = fileName;
519 sectionLoc.line = line;
523 void Scanner::endSection( )
525 /* Execute the eof actions for the section parser. */
526 processToken( -1, 0, 0 );
528 /* Close off the section with the parser. */
531 loc.fileName = fileName;
535 parser->token( loc, TK_EndSection, 0, 0 );
538 if ( includeDepth == 0 ) {
539 if ( machineSpec == 0 && machineName == 0 ) {
540 /* The end section may include a newline on the end, so
541 * we use the last line, which will count the newline. */
542 output << "<host line=\"" << line << "\">";
547 bool isAbsolutePath( const char *path )
550 return isalpha( path[0] ) && path[1] == ':' && path[2] == '\\';
552 return path[0] == '/';
556 char **Scanner::makeIncludePathChecks( char *thisFileName, char *fileName, int fnlen )
558 char **checks = new char*[2];
561 bool caseInsensitive = false;
563 char *data = prepareLitString( InputLoc(), fileName, fnlen,
564 length, caseInsensitive );
567 if ( isAbsolutePath( data ) )
568 checks[nextCheck++] = data;
570 /* Search from the the location of the current file. */
571 const char *lastSlash = strrchr( thisFileName, PATH_SEP );
572 if ( lastSlash == 0 )
573 checks[nextCheck++] = data;
575 long givenPathLen = (lastSlash - thisFileName) + 1;
576 long checklen = givenPathLen + length;
577 char *check = new char[checklen+1];
578 memcpy( check, thisFileName, givenPathLen );
579 memcpy( check+givenPathLen, data, length );
581 checks[nextCheck++] = check;
584 /* Search from the include paths given on the command line. */
585 for ( ArgsVector::Iter incp = includePaths; incp.lte(); incp++ ) {
586 long pathLen = strlen( *incp );
587 long checkLen = pathLen + 1 + length;
588 char *check = new char[checkLen+1];
589 memcpy( check, *incp, pathLen );
590 check[pathLen] = PATH_SEP;
591 memcpy( check+pathLen+1, data, length );
593 checks[nextCheck++] = check;
597 checks[nextCheck] = 0;
601 ifstream *Scanner::tryOpenInclude( char **pathChecks, long &found )
603 char **check = pathChecks;
604 ifstream *inFile = new ifstream;
606 while ( *check != 0 ) {
607 inFile->open( *check );
608 if ( inFile->is_open() ) {
609 found = check - pathChecks;
623 # This is sent by the driver code.
633 # Identifiers, numbers, commetns, and other common things.
634 ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
636 hex_number = '0x' [0-9a-fA-F]+;
639 '/*' ( any | NL )* :>> '*/';
644 c_cpp_comment = c_comment | cpp_comment;
646 ruby_comment = '#' [^\n]* NL;
648 # These literal forms are common to host code and ragel.
649 s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
650 d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';
651 host_re_literal = '/' ([^/\\] | NL | '\\' (any | NL))* '/';
653 whitespace = [ \t] | NL;
654 pound_comment = '#' [^\n]* NL;
656 # An inline block of code for Ruby.
657 inline_code_ruby := |*
658 # Inline expression keywords.
659 "fpc" => { token( KW_PChar ); };
660 "fc" => { token( KW_Char ); };
661 "fcurs" => { token( KW_CurState ); };
662 "ftargs" => { token( KW_TargState ); };
664 whitespaceOn = false;
668 # Inline statement keywords.
670 whitespaceOn = false;
673 "fexec" => { token( KW_Exec, 0, 0 ); };
675 whitespaceOn = false;
679 whitespaceOn = false;
683 whitespaceOn = false;
687 whitespaceOn = false;
691 whitespaceOn = false;
695 ident => { token( TK_Word, ts, te ); };
697 number => { token( TK_UInt, ts, te ); };
698 hex_number => { token( TK_Hex, ts, te ); };
700 ( s_literal | d_literal | host_re_literal )
701 => { token( IL_Literal, ts, te ); };
705 token( IL_WhiteSpace, ts, te );
708 ruby_comment => { token( IL_Comment, ts, te ); };
710 "::" => { token( TK_NameSep, ts, te ); };
712 # Some symbols need to go to the parser as with their cardinal value as
713 # the token type (as opposed to being sent as anonymous symbols)
714 # because they are part of the sequences which we interpret. The * ) ;
715 # symbols cause whitespace parsing to come back on. This gets turned
716 # off by some keywords.
720 token( *ts, ts, te );
721 if ( inlineBlockType == SemiTerminated )
727 token( *ts, ts, te );
730 [,(] => { token( *ts, ts, te ); };
733 token( IL_Symbol, ts, te );
738 if ( --curly_count == 0 && inlineBlockType == CurlyDelimited ) {
739 /* Inline code block ends. */
744 /* Either a semi terminated inline block or only the closing
745 * brace of some inner scope, not the block's closing brace. */
746 token( IL_Symbol, ts, te );
751 scan_error() << "unterminated code block" << endl;
754 # Send every other character as a symbol.
755 any => { token( IL_Symbol, ts, te ); };
759 # An inline block of code for languages other than Ruby.
761 # Inline expression keywords.
762 "fpc" => { token( KW_PChar ); };
763 "fc" => { token( KW_Char ); };
764 "fcurs" => { token( KW_CurState ); };
765 "ftargs" => { token( KW_TargState ); };
767 whitespaceOn = false;
771 # Inline statement keywords.
773 whitespaceOn = false;
776 "fexec" => { token( KW_Exec, 0, 0 ); };
778 whitespaceOn = false;
782 whitespaceOn = false;
786 whitespaceOn = false;
790 whitespaceOn = false;
794 whitespaceOn = false;
798 ident => { token( TK_Word, ts, te ); };
800 number => { token( TK_UInt, ts, te ); };
801 hex_number => { token( TK_Hex, ts, te ); };
803 ( s_literal | d_literal )
804 => { token( IL_Literal, ts, te ); };
808 token( IL_WhiteSpace, ts, te );
811 c_cpp_comment => { token( IL_Comment, ts, te ); };
813 "::" => { token( TK_NameSep, ts, te ); };
815 # Some symbols need to go to the parser as with their cardinal value as
816 # the token type (as opposed to being sent as anonymous symbols)
817 # because they are part of the sequences which we interpret. The * ) ;
818 # symbols cause whitespace parsing to come back on. This gets turned
819 # off by some keywords.
823 token( *ts, ts, te );
824 if ( inlineBlockType == SemiTerminated )
830 token( *ts, ts, te );
833 [,(] => { token( *ts, ts, te ); };
836 token( IL_Symbol, ts, te );
841 if ( --curly_count == 0 && inlineBlockType == CurlyDelimited ) {
842 /* Inline code block ends. */
847 /* Either a semi terminated inline block or only the closing
848 * brace of some inner scope, not the block's closing brace. */
849 token( IL_Symbol, ts, te );
854 scan_error() << "unterminated code block" << endl;
857 # Send every other character as a symbol.
858 any => { token( IL_Symbol, ts, te ); };
862 # Escape sequences in OR expressions.
863 '\\0' => { token( RE_Char, '\0' ); };
864 '\\a' => { token( RE_Char, '\a' ); };
865 '\\b' => { token( RE_Char, '\b' ); };
866 '\\t' => { token( RE_Char, '\t' ); };
867 '\\n' => { token( RE_Char, '\n' ); };
868 '\\v' => { token( RE_Char, '\v' ); };
869 '\\f' => { token( RE_Char, '\f' ); };
870 '\\r' => { token( RE_Char, '\r' ); };
871 '\\\n' => { updateCol(); };
872 '\\' any => { token( RE_Char, ts+1, te ); };
874 # Range dash in an OR expression.
875 '-' => { token( RE_Dash, 0, 0 ); };
877 # Terminate an OR expression.
878 ']' => { token( RE_SqClose ); fret; };
881 scan_error() << "unterminated OR literal" << endl;
884 # Characters in an OR expression.
885 [^\]] => { token( RE_Char, ts, te ); };
889 ragel_re_literal := |*
890 # Escape sequences in regular expressions.
891 '\\0' => { token( RE_Char, '\0' ); };
892 '\\a' => { token( RE_Char, '\a' ); };
893 '\\b' => { token( RE_Char, '\b' ); };
894 '\\t' => { token( RE_Char, '\t' ); };
895 '\\n' => { token( RE_Char, '\n' ); };
896 '\\v' => { token( RE_Char, '\v' ); };
897 '\\f' => { token( RE_Char, '\f' ); };
898 '\\r' => { token( RE_Char, '\r' ); };
899 '\\\n' => { updateCol(); };
900 '\\' any => { token( RE_Char, ts+1, te ); };
902 # Terminate an OR expression.
904 token( RE_Slash, ts, te );
908 # Special characters.
909 '.' => { token( RE_Dot ); };
910 '*' => { token( RE_Star ); };
912 '[' => { token( RE_SqOpen ); fcall or_literal; };
913 '[^' => { token( RE_SqOpenNeg ); fcall or_literal; };
916 scan_error() << "unterminated regular expression" << endl;
919 # Characters in an OR expression.
920 [^\/] => { token( RE_Char, ts, te ); };
923 # We need a separate token space here to avoid the ragel keywords.
924 write_statement := |*
925 ident => { token( TK_Word, ts, te ); } ;
926 [ \t\n]+ => { updateCol(); };
927 ';' => { token( ';' ); fgoto parser_def; };
930 scan_error() << "unterminated write statement" << endl;
934 # Parser definitions.
936 'machine' => { token( KW_Machine ); };
937 'include' => { token( KW_Include ); };
938 'import' => { token( KW_Import ); };
941 fgoto write_statement;
943 'action' => { token( KW_Action ); };
944 'alphtype' => { token( KW_AlphType ); };
945 'prepush' => { token( KW_PrePush ); };
946 'postpop' => { token( KW_PostPop ); };
948 # FIXME: Enable this post 5.17.
949 # 'range' => { token( KW_Range ); };
953 inlineBlockType = SemiTerminated;
954 if ( hostLang->lang == HostLang::Ruby )
955 fcall inline_code_ruby;
961 inlineBlockType = SemiTerminated;
962 if ( hostLang->lang == HostLang::Ruby )
963 fcall inline_code_ruby;
968 token( KW_Variable );
969 inlineBlockType = SemiTerminated;
970 if ( hostLang->lang == HostLang::Ruby )
971 fcall inline_code_ruby;
975 'when' => { token( KW_When ); };
976 'inwhen' => { token( KW_InWhen ); };
977 'outwhen' => { token( KW_OutWhen ); };
978 'eof' => { token( KW_Eof ); };
979 'err' => { token( KW_Err ); };
980 'lerr' => { token( KW_Lerr ); };
981 'to' => { token( KW_To ); };
982 'from' => { token( KW_From ); };
983 'export' => { token( KW_Export ); };
986 ident => { token( TK_Word, ts, te ); } ;
989 number => { token( TK_UInt, ts, te ); };
990 hex_number => { token( TK_Hex, ts, te ); };
992 # Literals, with optionals.
993 ( s_literal | d_literal ) [i]?
994 => { token( TK_Literal, ts, te ); };
996 '[' => { token( RE_SqOpen ); fcall or_literal; };
997 '[^' => { token( RE_SqOpenNeg ); fcall or_literal; };
999 '/' => { token( RE_Slash ); fgoto ragel_re_literal; };
1002 pound_comment => { updateCol(); };
1004 ':=' => { token( TK_ColonEquals ); };
1007 ">~" => { token( TK_StartToState ); };
1008 "$~" => { token( TK_AllToState ); };
1009 "%~" => { token( TK_FinalToState ); };
1010 "<~" => { token( TK_NotStartToState ); };
1011 "@~" => { token( TK_NotFinalToState ); };
1012 "<>~" => { token( TK_MiddleToState ); };
1014 # From State actions
1015 ">*" => { token( TK_StartFromState ); };
1016 "$*" => { token( TK_AllFromState ); };
1017 "%*" => { token( TK_FinalFromState ); };
1018 "<*" => { token( TK_NotStartFromState ); };
1019 "@*" => { token( TK_NotFinalFromState ); };
1020 "<>*" => { token( TK_MiddleFromState ); };
1023 ">/" => { token( TK_StartEOF ); };
1024 "$/" => { token( TK_AllEOF ); };
1025 "%/" => { token( TK_FinalEOF ); };
1026 "</" => { token( TK_NotStartEOF ); };
1027 "@/" => { token( TK_NotFinalEOF ); };
1028 "<>/" => { token( TK_MiddleEOF ); };
1030 # Global Error actions.
1031 ">!" => { token( TK_StartGblError ); };
1032 "$!" => { token( TK_AllGblError ); };
1033 "%!" => { token( TK_FinalGblError ); };
1034 "<!" => { token( TK_NotStartGblError ); };
1035 "@!" => { token( TK_NotFinalGblError ); };
1036 "<>!" => { token( TK_MiddleGblError ); };
1038 # Local error actions.
1039 ">^" => { token( TK_StartLocalError ); };
1040 "$^" => { token( TK_AllLocalError ); };
1041 "%^" => { token( TK_FinalLocalError ); };
1042 "<^" => { token( TK_NotStartLocalError ); };
1043 "@^" => { token( TK_NotFinalLocalError ); };
1044 "<>^" => { token( TK_MiddleLocalError ); };
1047 "<>" => { token( TK_Middle ); };
1050 '>?' => { token( TK_StartCond ); };
1051 '$?' => { token( TK_AllCond ); };
1052 '%?' => { token( TK_LeavingCond ); };
1054 '..' => { token( TK_DotDot ); };
1055 '**' => { token( TK_StarStar ); };
1056 '--' => { token( TK_DashDash ); };
1057 '->' => { token( TK_Arrow ); };
1058 '=>' => { token( TK_DoubleArrow ); };
1060 ":>" => { token( TK_ColonGt ); };
1061 ":>>" => { token( TK_ColonGtGt ); };
1062 "<:" => { token( TK_LtColon ); };
1064 # Opening of longest match.
1065 "|*" => { token( TK_BarStar ); };
1067 # Separater for name references.
1068 "::" => { token( TK_NameSep, ts, te ); };
1076 [ \t\r]+ => { updateCol(); };
1078 # If we are in a single line machine then newline may end the spec.
1081 if ( singleLineSpec ) {
1088 if ( lastToken == KW_Export || lastToken == KW_Entry )
1093 inlineBlockType = CurlyDelimited;
1094 if ( hostLang->lang == HostLang::Ruby )
1095 fcall inline_code_ruby;
1102 scan_error() << "unterminated ragel section" << endl;
1105 any => { token( *ts ); } ;
1108 # Outside code scanner. These tokens get passed through.
1110 ident => { pass( IMP_Word, ts, te ); };
1111 number => { pass( IMP_UInt, ts, te ); };
1112 ruby_comment => { pass(); };
1113 ( s_literal | d_literal | host_re_literal )
1114 => { pass( IMP_Literal, ts, te ); };
1118 singleLineSpec = false;
1124 singleLineSpec = true;
1128 whitespace+ => { pass(); };
1130 any => { pass( *ts, 0, 0 ); };
1133 # Outside code scanner. These tokens get passed through.
1135 'define' => { pass( IMP_Define, 0, 0 ); };
1136 ident => { pass( IMP_Word, ts, te ); };
1137 number => { pass( IMP_UInt, ts, te ); };
1138 c_cpp_comment => { pass(); };
1139 ( s_literal | d_literal ) => { pass( IMP_Literal, ts, te ); };
1143 singleLineSpec = false;
1149 singleLineSpec = true;
1153 whitespace+ => { pass(); };
1155 any => { pass( *ts, 0, 0 ); };
1161 void Scanner::do_scan()
1164 char *buf = new char[bufsize];
1165 int cs, act, have = 0;
1168 /* The stack is two deep, one level for going into ragel defs from the main
1169 * machines which process outside code, and another for going into or literals
1170 * from either a ragel spec, or a regular expression. */
1172 int curly_count = 0;
1173 bool execute = true;
1174 bool singleLineSpec = false;
1175 InlineBlockType inlineBlockType = CurlyDelimited;
1177 /* Init the section parser and the character scanner. */
1181 /* Set up the start state. FIXME: After 5.20 is released the nocs write
1182 * init option should be used, the main machine eliminated and this statement moved
1183 * above the write init. */
1184 if ( hostLang->lang == HostLang::Ruby )
1185 cs = rlscan_en_main_ruby;
1187 cs = rlscan_en_main;
1190 char *p = buf + have;
1191 int space = bufsize - have;
1194 /* We filled up the buffer trying to scan a token. Grow it. */
1195 bufsize = bufsize * 2;
1196 char *newbuf = new char[bufsize];
1198 /* Recompute p and space. */
1200 space = bufsize - have;
1202 /* Patch up pointers possibly in use. */
1204 ts = newbuf + ( ts - buf );
1205 te = newbuf + ( te - buf );
1207 /* Copy the new buffer in. */
1208 memcpy( newbuf, buf, have );
1213 input.read( p, space );
1214 int len = input.gcount();
1217 /* If we see eof then append the eof var. */
1226 /* Check if we failed. */
1227 if ( cs == rlscan_error ) {
1228 /* Machine failed before finding a token. I'm not yet sure if this
1230 scan_error() << "scanner error" << endl;
1234 /* Decide if we need to preserve anything. */
1235 char *preserve = ts;
1237 /* Now set up the prefix. */
1238 if ( preserve == 0 )
1241 /* There is data that needs to be shifted over. */
1242 have = pe - preserve;
1243 memmove( buf, preserve, have );
1244 unsigned int shiftback = preserve - buf;