2 * Copyright 2006 Adrian Thurston <thurston@cs.queensu.ca>
5 /* This file is part of Ragel.
7 * Ragel is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * Ragel is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Ragel; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 #include "parsedata.h"
40 /* This is used for tracking the current stack of include file/machine pairs. It is
41 * is used to detect and recursive include structure. */
42 struct IncludeStackItem
44 IncludeStackItem( char *fileName, char *sectionName )
45 : fileName(fileName), sectionName(sectionName) {}
51 typedef Vector<IncludeStackItem> IncludeStack;
52 IncludeStack includeStack;
62 Scanner( char *fileName, istream &input,
63 Parser *inclToParser, char *inclSectionTarg,
66 fileName(fileName), input(input),
67 inclToParser(inclToParser),
68 inclSectionTarg(inclSectionTarg),
69 include_depth(include_depth),
70 line(1), column(1), lastnl(0),
71 parser(0), active(false),
72 parserExistsError(false), ragelDefOpen(false),
76 bool recursiveInclude( IncludeStack &includeStack,
77 char *inclFileName, char *inclSectionName );
79 char *prepareFileName( char *fileName, int len )
82 Token tokenFnStr, tokenRes;
83 tokenFnStr.data = fileName;
84 tokenFnStr.length = len;
85 tokenFnStr.prepareLitString( tokenRes, caseInsensitive );
90 void token( int type, char *start, char *end );
91 void token( int type, char *string );
92 void token( int type );
103 Parser *inclToParser;
104 char *inclSectionTarg;
110 int word_len, lit_len;
112 char *tokstart, *tokend;
116 /* Set by machine statements, these persist from section to section
117 * allowing for unnamed sections. */
121 /* This is set if ragel has already emitted an error stating that
122 * no section name has been seen and thus no parser exists. */
123 bool parserExistsError;
126 /* This is for inline code. By default it is on. It goes off for
127 * statements and values in inline blocks which are parsed. */
132 machine section_parse;
137 void Scanner::init( )
142 bool Scanner::parserExists()
147 if ( ! parserExistsError ) {
148 error() << "include: there is no previous specification name" << endl;
149 parserExistsError = true;
154 ostream &Scanner::error()
156 /* Maintain the error count. */
159 cerr << fileName << ":" << line << ":" << column << ": ";
163 bool Scanner::recursiveInclude( IncludeStack &includeStack,
164 char *inclFileName, char *inclSectionName )
166 for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
167 if ( strcmp( si->fileName, inclFileName ) == 0 &&
168 strcmp( si->sectionName, inclSectionName ) == 0 )
176 void Scanner::updateCol()
181 //cerr << "adding " << tokend - from << " to column" << endl;
182 column += tokend - from;
186 void Scanner::token( int type, char *string )
188 token( type, string, string + strlen(string) );
191 void Scanner::token( int type )
197 machine section_parse;
199 # This relies on the the kelbt implementation and the order
200 # that tokens are declared.
207 action clear_words { word = lit = 0; word_len = lit_len = 0; }
208 action store_word { word = tokdata; word_len = toklen; }
209 action store_lit { lit = tokdata; lit_len = toklen; }
211 action mach_err { error() << "bad machine statement" << endl; }
212 action incl_err { error() << "bad include statement" << endl; }
213 action write_err { error() << "bad write statement" << endl; }
215 action handle_machine
217 /* Assign a name to the machine. */
218 char *machine = word;
219 //cerr << "scanner: machine statement: " << machine << endl;
221 if ( inclSectionTarg == 0 ) {
224 ParserDictEl *pdEl = parserDict.find( machine );
226 //cerr << "scanner: using existing parser" << endl;
229 //cerr << "scanner: creating a new parser" << endl;
230 pdEl = new ParserDictEl( machine );
231 pdEl->value = new Parser( fileName, machine, sectionLoc );
233 parserDict.insert( pdEl );
236 parser = pdEl->value;
238 else if ( strcmp( inclSectionTarg, machine ) == 0 ) {
239 //cerr << "scanner: found include target" << endl;
241 parser = inclToParser;
244 //cerr << "scanner: ignoring section" << endl;
251 ( KW_Machine TK_Word @store_word ';' ) @handle_machine
252 <>err mach_err <>eof mach_err;
254 action handle_include
256 if ( active && parserExists() ) {
257 char *inclSectionName = word;
258 char *inclFileName = 0;
260 /* Implement defaults for the input file and section name. */
261 if ( inclSectionName == 0 )
262 inclSectionName = parser->sectionName;
265 inclFileName = prepareFileName( lit, lit_len );
267 inclFileName = fileName;
269 /* Open the file and process it. */
270 //cerr << "scanner: include: " << inclSectionName << " " << inclFileName << endl;
272 /* Check for a recursive include structure. Add the current file/section
273 * name then check if what we are including is already in the stack. */
274 includeStack.append( IncludeStackItem( fileName, parser->sectionName ) );
276 if ( recursiveInclude( includeStack, inclFileName, inclSectionName ) )
277 error() << "include: this is a recursive include operation" << endl;
279 /* Open the input file for reading. */
280 ifstream *inFile = new ifstream( inclFileName );
281 if ( ! inFile->is_open() ) {
282 error() << "include: could not open " <<
283 inclFileName << " for reading" << endl;
286 Scanner scanner( inclFileName, *inFile, parser,
287 inclSectionName, include_depth+1 );
293 /* Remove the last element (len-1) */
294 includeStack.remove( -1 );
299 TK_Word @store_word ( TK_Literal @store_lit )? |
300 TK_Literal @store_lit
304 ( KW_Include include_names ';' ) @handle_include
305 <>err incl_err <>eof incl_err;
311 if ( strcmp( tokdata, "data" ) != 0 &&
312 strcmp( tokdata, "init" ) != 0 &&
313 strcmp( tokdata, "exec" ) != 0 &&
314 strcmp( tokdata, "eof" ) != 0 )
316 error() << "unknown write command" << endl;
318 *outStream << " <write what=\"" << tokdata << "\">";
325 *outStream << "<option>" << tokdata << "</option>";
330 *outStream << "</write>\n";
334 ( KW_Write TK_Word @write_command
335 ( TK_Word @write_option )* ';' @write_close )
336 <>err write_err <>eof write_err;
340 /* Send the token off to the parser. */
341 if ( active && parserExists() ) {
344 //cerr << "scanner:" << line << ":" << column <<
345 // ": sending token to the parser " << lelNames[*p];
346 //if ( tokdata != 0 )
347 // cerr << " " << tokdata;
350 loc.fileName = fileName;
354 parser->token( loc, type, tokdata, toklen );
358 # Catch everything else.
359 everything_else = ^( KW_Machine | KW_Include | KW_Write ) @handle_token;
369 void Scanner::token( int type, char *start, char *end )
378 tokdata = new char[toklen+1];
379 memcpy( tokdata, start, toklen );
384 machine section_parse;
391 void Scanner::startSection( )
393 parserExistsError = false;
395 if ( include_depth == 0 ) {
396 if ( machineSpec == 0 && machineName == 0 )
397 *outStream << "</host>\n";
398 ragelDefOpen = false;
401 sectionLoc.fileName = fileName;
402 sectionLoc.line = line;
406 void Scanner::openRagelDef()
408 if ( ! ragelDefOpen ) {
410 *outStream << "<ragel_def name=\"" << parser->sectionName << "\">\n";
414 void Scanner::endSection( )
416 /* Execute the eof actions for the section parser. */
418 machine section_parse;
422 /* Close off the section with the parser. */
423 if ( active && parserExists() ) {
425 loc.fileName = fileName;
429 parser->token( loc, TK_EndSection, 0, 0 );
432 if ( include_depth == 0 ) {
433 if ( ragelDefOpen ) {
434 *outStream << "</ragel_def>\n";
435 ragelDefOpen = false;
438 if ( machineSpec == 0 && machineName == 0 ) {
439 /* The end section may include a newline on the end, so
440 * we use the last line, which will count the newline. */
441 *outStream << "<host line=\"" << line << "\">";
449 # This is sent by the driver code.
459 # Identifiers, numbers, commetns, and other common things.
460 ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
462 hex_number = '0x' [0-9a-fA-F]+;
465 '/*' ( any | NL )* :>> '*/';
470 c_cpp_comment = c_comment | cpp_comment;
472 # These literal forms are common to C-like host code and ragel.
473 s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
474 d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';
476 whitespace = [ \t] | NL;
477 pound_comment = '#' [^\n]* NL;
479 # An inline block of code. This is specified as a scanned, but is sent to
480 # the parser as one long block. The inline_block pointer is used to handle
481 # the preservation of the data.
483 # Inline expression keywords.
484 "fpc" => { token( KW_PChar ); };
485 "fc" => { token( KW_Char ); };
486 "fcurs" => { token( KW_CurState ); };
487 "ftargs" => { token( KW_TargState ); };
489 whitespaceOn = false;
493 # Inline statement keywords.
495 whitespaceOn = false;
498 "fexec" => { token( KW_Exec, 0, 0 ); };
500 whitespaceOn = false;
504 whitespaceOn = false;
508 whitespaceOn = false;
512 whitespaceOn = false;
516 whitespaceOn = false;
520 ident => { token( TK_Word, tokstart, tokend ); };
522 number => { token( TK_UInt, tokstart, tokend ); };
523 hex_number => { token( TK_Hex, tokstart, tokend ); };
525 ( s_literal | d_literal )
526 => { token( IL_Literal, tokstart, tokend ); };
530 token( IL_WhiteSpace, tokstart, tokend );
532 c_cpp_comment => { token( IL_Comment, tokstart, tokend ); };
534 "::" => { token( TK_NameSep, tokstart, tokend ); };
536 # Some symbols need to go to the parser as with their cardinal value as
537 # the token type (as opposed to being sent as anonymous symbols)
538 # because they are part of the sequences which we interpret. The * ) ;
539 # symbols cause whitespace parsing to come back on. This gets turned
540 # off by some keywords.
544 token( *tokstart, tokstart, tokend );
545 if ( inlineBlockType == SemiTerminated )
551 token( *tokstart, tokstart, tokend );
554 [,(] => { token( *tokstart, tokstart, tokend ); };
557 token( IL_Symbol, tokstart, tokend );
562 if ( --curly_count == 0 && inlineBlockType == CurlyDelimited ) {
563 /* Inline code block ends. */
568 /* Either a semi terminated inline block or only the closing
569 * brace of some inner scope, not the block's closing brace. */
570 token( IL_Symbol, tokstart, tokend );
574 # Send every other character as a symbol.
575 any => { token( IL_Symbol, tokstart, tokend ); };
579 # Escape sequences in OR expressions.
580 '\\0' => { token( RE_Char, "\0" ); };
581 '\\a' => { token( RE_Char, "\a" ); };
582 '\\b' => { token( RE_Char, "\b" ); };
583 '\\t' => { token( RE_Char, "\t" ); };
584 '\\n' => { token( RE_Char, "\n" ); };
585 '\\v' => { token( RE_Char, "\v" ); };
586 '\\f' => { token( RE_Char, "\f" ); };
587 '\\r' => { token( RE_Char, "\r" ); };
588 '\\\n' => { updateCol(); };
589 '\\' any => { token( RE_Char, tokstart+1, tokend ); };
591 # Range dash in an OR expression.
592 '-' => { token( RE_Dash, 0, 0 ); };
594 # Terminate an OR expression.
595 ']' => { token( RE_SqClose ); fret; };
597 # Characters in an OR expression.
598 [^\]] => { token( RE_Char, tokstart, tokend ); };
602 # Escape sequences in regular expressions.
603 '\\0' => { token( RE_Char, "\0" ); };
604 '\\a' => { token( RE_Char, "\a" ); };
605 '\\b' => { token( RE_Char, "\b" ); };
606 '\\t' => { token( RE_Char, "\t" ); };
607 '\\n' => { token( RE_Char, "\n" ); };
608 '\\v' => { token( RE_Char, "\v" ); };
609 '\\f' => { token( RE_Char, "\f" ); };
610 '\\r' => { token( RE_Char, "\r" ); };
611 '\\\n' => { updateCol(); };
612 '\\' any => { token( RE_Char, tokstart+1, tokend ); };
614 # Terminate an OR expression.
616 token( RE_Slash, tokstart, tokend );
620 # Special characters.
621 '.' => { token( RE_Dot ); };
622 '*' => { token( RE_Star ); };
624 '[' => { token( RE_SqOpen ); fcall or_literal; };
625 '[^' => { token( RE_SqOpenNeg ); fcall or_literal; };
627 # Characters in an OR expression.
628 [^\/] => { token( RE_Char, tokstart, tokend ); };
631 write_statement := |*
632 ident => { token( TK_Word, tokstart, tokend ); } ;
633 [ \t\n]+ => { updateCol(); };
634 ';' => { token( ';' ); fgoto parser_def; };
637 # Parser definitions.
639 'machine' => { token( KW_Machine ); };
640 'include' => { token( KW_Include ); };
643 fgoto write_statement;
645 'action' => { token( KW_Action ); };
646 'alphtype' => { token( KW_AlphType ); };
648 # FIXME: Enable this post 5.17.
649 # 'range' => { token( KW_Range ); };
653 inlineBlockType = SemiTerminated;
658 inlineBlockType = SemiTerminated;
662 token( KW_Variable );
663 inlineBlockType = SemiTerminated;
666 'when' => { token( KW_When ); };
667 'eof' => { token( KW_Eof ); };
668 'err' => { token( KW_Err ); };
669 'lerr' => { token( KW_Lerr ); };
670 'to' => { token( KW_To ); };
671 'from' => { token( KW_From ); };
674 ident => { token( TK_Word, tokstart, tokend ); } ;
677 number => { token( TK_UInt, tokstart, tokend ); };
678 hex_number => { token( TK_Hex, tokstart, tokend ); };
680 # Literals, with optionals.
681 ( s_literal | d_literal ) [i]?
682 => { token( TK_Literal, tokstart, tokend ); };
684 '[' => { token( RE_SqOpen ); fcall or_literal; };
685 '[^' => { token( RE_SqOpenNeg ); fcall or_literal; };
687 '/' => { token( RE_Slash ); fgoto re_literal; };
690 pound_comment => { updateCol(); };
692 ':=' => { token( TK_ColonEquals ); };
695 ">~" => { token( TK_StartToState ); };
696 "$~" => { token( TK_AllToState ); };
697 "%~" => { token( TK_FinalToState ); };
698 "<~" => { token( TK_NotStartToState ); };
699 "@~" => { token( TK_NotFinalToState ); };
700 "<>~" => { token( TK_MiddleToState ); };
703 ">*" => { token( TK_StartFromState ); };
704 "$*" => { token( TK_AllFromState ); };
705 "%*" => { token( TK_FinalFromState ); };
706 "<*" => { token( TK_NotStartFromState ); };
707 "@*" => { token( TK_NotFinalFromState ); };
708 "<>*" => { token( TK_MiddleFromState ); };
711 ">/" => { token( TK_StartEOF ); };
712 "$/" => { token( TK_AllEOF ); };
713 "%/" => { token( TK_FinalEOF ); };
714 "</" => { token( TK_NotStartEOF ); };
715 "@/" => { token( TK_NotFinalEOF ); };
716 "<>/" => { token( TK_MiddleEOF ); };
718 # Global Error actions.
719 ">!" => { token( TK_StartGblError ); };
720 "$!" => { token( TK_AllGblError ); };
721 "%!" => { token( TK_FinalGblError ); };
722 "<!" => { token( TK_NotStartGblError ); };
723 "@!" => { token( TK_NotFinalGblError ); };
724 "<>!" => { token( TK_MiddleGblError ); };
726 # Local error actions.
727 ">^" => { token( TK_StartLocalError ); };
728 "$^" => { token( TK_AllLocalError ); };
729 "%^" => { token( TK_FinalLocalError ); };
730 "<^" => { token( TK_NotStartLocalError ); };
731 "@^" => { token( TK_NotFinalLocalError ); };
732 "<>^" => { token( TK_MiddleLocalError ); };
735 "<>" => { token( TK_Middle ); };
738 '>?' => { token( TK_StartCond ); };
739 '$?' => { token( TK_AllCond ); };
740 '%?' => { token( TK_LeavingCond ); };
742 '..' => { token( TK_DotDot ); };
743 '**' => { token( TK_StarStar ); };
744 '--' => { token( TK_DashDash ); };
745 '->' => { token( TK_Arrow ); };
746 '=>' => { token( TK_DoubleArrow ); };
748 ":>" => { token( TK_ColonGt ); };
749 ":>>" => { token( TK_ColonGtGt ); };
750 "<:" => { token( TK_LtColon ); };
752 # Opening of longest match.
753 "|*" => { token( TK_BarStar ); };
756 /* In order to generate anything we must be in the top level file
757 * and the current spec must be active and there must not have been
758 * any parse errors. */
764 [ \t]+ => { updateCol(); };
766 # If we are in a single line machine then newline may end the spec.
769 if ( singleLineSpec ) {
770 /* In order to generate anything we must be in the top level file
771 * and the current spec must be active and there must not have been
772 * any parse errors. */
781 inlineBlockType = CurlyDelimited;
785 any => { token( *tokstart ); } ;
791 /* If no errors and we are at the bottom of the include stack (the
792 * source file listed on the command line) then write out the data. */
793 if ( include_depth == 0 && machineSpec == 0 && machineName == 0 )
794 xmlEscapeHost( *outStream, tokstart, tokend-tokstart );
797 # Outside code scanner. These tokens get passed through.
801 c_cpp_comment => pass;
802 s_literal | d_literal => pass;
805 singleLineSpec = false;
811 singleLineSpec = true;
824 void Scanner::do_scan()
827 char *buf = new char[bufsize];
828 const char last_char = 0;
829 int cs, act, have = 0;
833 bool singleLineSpec = false;
834 InlineBlockType inlineBlockType;
839 char *p = buf + have;
840 int space = bufsize - have;
843 /* We filled up the buffer trying to scan a token. Grow it. */
844 bufsize = bufsize * 2;
845 char *newbuf = new char[bufsize];
846 //cerr << "FULL BUFFER, NEW SIZE: " << bufsize << endl;
848 /* Recompute p and space. */
850 space = bufsize - have;
852 /* Patch up pointers possibly in use. */
854 tokstart = newbuf + ( tokstart - buf );
855 tokend = newbuf + ( tokend - buf );
857 /* Copy the new buffer in. */
858 memcpy( newbuf, buf, have );
863 input.read( p, space );
864 int len = input.gcount();
866 /* If we see eof then append the EOF char. */
868 p[0] = last_char, len = 1;
875 /* Check if we failed. */
876 if ( cs == rlscan_error ) {
877 /* Machine failed before finding a token. */
878 //cerr << "PARSE ERROR" << endl;
882 /* Decide if we need to preserve anything. */
883 char *preserve = tokstart;
885 /* Now set up the prefix. */
889 /* There is data that needs to be shifted over. */
890 have = pe - preserve;
891 memmove( buf, preserve, have );
892 unsigned int shiftback = preserve - buf;
894 tokstart -= shiftback;
904 void scan( char *fileName, istream &input )
906 Scanner scanner( fileName, input, 0, 0, 0 );