2 * Copyright 2006-2007 Adrian Thurston <thurston@cs.queensu.ca>
5 /* This file is part of Ragel.
7 * Ragel is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * Ragel is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Ragel; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 #include "parsedata.h"
39 extern char *Parser_lelNames[];
41 /* This is used for tracking the current stack of include file/machine pairs. It is
42 * is used to detect and recursive include structure. */
43 struct IncludeStackItem
45 IncludeStackItem( char *fileName, char *sectionName )
46 : fileName(fileName), sectionName(sectionName) {}
52 typedef Vector<IncludeStackItem> IncludeStack;
62 Scanner( char *fileName, istream &input, ostream &output,
63 Parser *inclToParser, char *inclSectionTarg,
66 fileName(fileName), input(input), output(output),
67 inclToParser(inclToParser),
68 inclSectionTarg(inclSectionTarg),
69 includeDepth(includeDepth),
70 line(1), column(1), lastnl(0),
71 parser(0), ignoreSection(false),
72 parserExistsError(false),
77 bool recursiveInclude( char *inclFileName, char *inclSectionName );
79 char *prepareFileName( char *fileName, int len )
82 Token tokenFnStr, tokenRes;
83 tokenFnStr.data = fileName;
84 tokenFnStr.length = len;
85 tokenFnStr.prepareLitString( tokenRes, caseInsensitive );
90 void token( int type, char *start, char *end );
91 void token( int type, char c );
92 void token( int type );
98 ostream &scan_error();
103 Parser *inclToParser;
104 char *inclSectionTarg;
110 int word_len, lit_len;
112 char *tokstart, *tokend;
116 /* Set by machine statements, these persist from section to section
117 * allowing for unnamed sections. */
120 IncludeStack includeStack;
122 /* This is set if ragel has already emitted an error stating that
123 * no section name has been seen and thus no parser exists. */
124 bool parserExistsError;
126 /* This is for inline code. By default it is on. It goes off for
127 * statements and values in inline blocks which are parsed. */
130 /* Keeps a record of the previous token sent to the section parser. */
135 machine section_parse;
140 void Scanner::init( )
145 bool Scanner::active()
150 if ( parser == 0 && ! parserExistsError ) {
151 scan_error() << "there is no previous specification name" << endl;
152 parserExistsError = true;
161 ostream &Scanner::scan_error()
163 /* Maintain the error count. */
165 cerr << fileName << ":" << line << ":" << column << ": ";
169 bool Scanner::recursiveInclude( char *inclFileName, char *inclSectionName )
171 for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
172 if ( strcmp( si->fileName, inclFileName ) == 0 &&
173 strcmp( si->sectionName, inclSectionName ) == 0 )
181 void Scanner::updateCol()
186 //cerr << "adding " << tokend - from << " to column" << endl;
187 column += tokend - from;
191 void Scanner::token( int type, char c )
193 token( type, &c, &c + 1 );
196 void Scanner::token( int type )
202 machine section_parse;
204 # This relies on the the kelbt implementation and the order
205 # that tokens are declared.
212 action clear_words { word = lit = 0; word_len = lit_len = 0; }
213 action store_word { word = tokdata; word_len = toklen; }
214 action store_lit { lit = tokdata; lit_len = toklen; }
216 action mach_err { scan_error() << "bad machine statement" << endl; }
217 action incl_err { scan_error() << "bad include statement" << endl; }
218 action write_err { scan_error() << "bad write statement" << endl; }
220 action handle_machine
222 /* Assign a name to the machine. */
223 char *machine = word;
225 if ( inclSectionTarg == 0 ) {
226 ignoreSection = false;
228 ParserDictEl *pdEl = parserDict.find( machine );
230 pdEl = new ParserDictEl( machine );
231 pdEl->value = new Parser( fileName, machine, sectionLoc );
233 parserDict.insert( pdEl );
236 parser = pdEl->value;
238 else if ( strcmp( inclSectionTarg, machine ) == 0 ) {
239 /* found include target */
240 ignoreSection = false;
241 parser = inclToParser;
244 /* ignoring section */
245 ignoreSection = true;
251 ( KW_Machine TK_Word @store_word ';' ) @handle_machine
252 <>err mach_err <>eof mach_err;
254 action handle_include
257 char *inclSectionName = word;
258 char *inclFileName = 0;
260 /* Implement defaults for the input file and section name. */
261 if ( inclSectionName == 0 )
262 inclSectionName = parser->sectionName;
265 inclFileName = prepareFileName( lit, lit_len );
267 inclFileName = fileName;
269 /* Check for a recursive include structure. Add the current file/section
270 * name then check if what we are including is already in the stack. */
271 includeStack.append( IncludeStackItem( fileName, parser->sectionName ) );
273 if ( recursiveInclude( inclFileName, inclSectionName ) )
274 scan_error() << "include: this is a recursive include operation" << endl;
276 /* Open the input file for reading. */
277 ifstream *inFile = new ifstream( inclFileName );
278 if ( ! inFile->is_open() ) {
279 scan_error() << "include: could not open " <<
280 inclFileName << " for reading" << endl;
283 Scanner scanner( inclFileName, *inFile, output, parser,
284 inclSectionName, includeDepth+1 );
290 /* Remove the last element (len-1) */
291 includeStack.remove( -1 );
296 TK_Word @store_word ( TK_Literal @store_lit )? |
297 TK_Literal @store_lit
301 ( KW_Include include_names ';' ) @handle_include
302 <>err incl_err <>eof incl_err;
306 if ( active() && machineSpec == 0 && machineName == 0 ) {
308 " def_name=\"" << parser->sectionName << "\""
309 " line=\"" << line << "\""
310 " col=\"" << column << "\""
317 if ( active() && machineSpec == 0 && machineName == 0 )
318 output << "<arg>" << tokdata << "</arg>";
323 if ( active() && machineSpec == 0 && machineName == 0 )
324 output << "</write>\n";
328 ( KW_Write @write_command
329 ( TK_Word @write_arg )+ ';' @write_close )
330 <>err write_err <>eof write_err;
334 /* Send the token off to the parser. */
339 cerr << "scanner:" << line << ":" << column <<
340 ": sending token to the parser " << Parser_lelNames[*p];
341 cerr << " " << toklen;
343 cerr << " " << tokdata;
347 loc.fileName = fileName;
351 parser->token( loc, type, tokdata, toklen );
355 # Catch everything else.
356 everything_else = ^( KW_Machine | KW_Include | KW_Write ) @handle_token;
366 void Scanner::token( int type, char *start, char *end )
375 tokdata = new char[toklen+1];
376 memcpy( tokdata, start, toklen );
381 machine section_parse;
387 /* Record the last token for use in controlling the scan of subsequent
392 void Scanner::startSection( )
394 parserExistsError = false;
396 if ( includeDepth == 0 ) {
397 if ( machineSpec == 0 && machineName == 0 )
398 output << "</host>\n";
401 sectionLoc.fileName = fileName;
402 sectionLoc.line = line;
406 void Scanner::endSection( )
408 /* Execute the eof actions for the section parser. */
410 machine section_parse;
414 /* Close off the section with the parser. */
417 loc.fileName = fileName;
421 parser->token( loc, TK_EndSection, 0, 0 );
424 if ( includeDepth == 0 ) {
425 if ( machineSpec == 0 && machineName == 0 ) {
426 /* The end section may include a newline on the end, so
427 * we use the last line, which will count the newline. */
428 output << "<host line=\"" << line << "\">";
436 # This is sent by the driver code.
446 # Identifiers, numbers, commetns, and other common things.
447 ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
449 hex_number = '0x' [0-9a-fA-F]+;
452 '/*' ( any | NL )* :>> '*/';
457 c_cpp_comment = c_comment | cpp_comment;
459 # These literal forms are common to C-like host code and ragel.
460 s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
461 d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';
463 whitespace = [ \t] | NL;
464 pound_comment = '#' [^\n]* NL;
466 # An inline block of code. This is specified as a scanned, but is sent to
467 # the parser as one long block. The inline_block pointer is used to handle
468 # the preservation of the data.
470 # Inline expression keywords.
471 "fpc" => { token( KW_PChar ); };
472 "fc" => { token( KW_Char ); };
473 "fcurs" => { token( KW_CurState ); };
474 "ftargs" => { token( KW_TargState ); };
476 whitespaceOn = false;
480 # Inline statement keywords.
482 whitespaceOn = false;
485 "fexec" => { token( KW_Exec, 0, 0 ); };
487 whitespaceOn = false;
491 whitespaceOn = false;
495 whitespaceOn = false;
499 whitespaceOn = false;
503 whitespaceOn = false;
507 ident => { token( TK_Word, tokstart, tokend ); };
509 number => { token( TK_UInt, tokstart, tokend ); };
510 hex_number => { token( TK_Hex, tokstart, tokend ); };
512 ( s_literal | d_literal )
513 => { token( IL_Literal, tokstart, tokend ); };
517 token( IL_WhiteSpace, tokstart, tokend );
519 c_cpp_comment => { token( IL_Comment, tokstart, tokend ); };
521 "::" => { token( TK_NameSep, tokstart, tokend ); };
523 # Some symbols need to go to the parser as with their cardinal value as
524 # the token type (as opposed to being sent as anonymous symbols)
525 # because they are part of the sequences which we interpret. The * ) ;
526 # symbols cause whitespace parsing to come back on. This gets turned
527 # off by some keywords.
531 token( *tokstart, tokstart, tokend );
532 if ( inlineBlockType == SemiTerminated )
538 token( *tokstart, tokstart, tokend );
541 [,(] => { token( *tokstart, tokstart, tokend ); };
544 token( IL_Symbol, tokstart, tokend );
549 if ( --curly_count == 0 && inlineBlockType == CurlyDelimited ) {
550 /* Inline code block ends. */
555 /* Either a semi terminated inline block or only the closing
556 * brace of some inner scope, not the block's closing brace. */
557 token( IL_Symbol, tokstart, tokend );
562 scan_error() << "unterminated code block" << endl;
565 # Send every other character as a symbol.
566 any => { token( IL_Symbol, tokstart, tokend ); };
570 # Escape sequences in OR expressions.
571 '\\0' => { token( RE_Char, '\0' ); };
572 '\\a' => { token( RE_Char, '\a' ); };
573 '\\b' => { token( RE_Char, '\b' ); };
574 '\\t' => { token( RE_Char, '\t' ); };
575 '\\n' => { token( RE_Char, '\n' ); };
576 '\\v' => { token( RE_Char, '\v' ); };
577 '\\f' => { token( RE_Char, '\f' ); };
578 '\\r' => { token( RE_Char, '\r' ); };
579 '\\\n' => { updateCol(); };
580 '\\' any => { token( RE_Char, tokstart+1, tokend ); };
582 # Range dash in an OR expression.
583 '-' => { token( RE_Dash, 0, 0 ); };
585 # Terminate an OR expression.
586 ']' => { token( RE_SqClose ); fret; };
589 scan_error() << "unterminated OR literal" << endl;
592 # Characters in an OR expression.
593 [^\]] => { token( RE_Char, tokstart, tokend ); };
598 # Escape sequences in regular expressions.
599 '\\0' => { token( RE_Char, '\0' ); };
600 '\\a' => { token( RE_Char, '\a' ); };
601 '\\b' => { token( RE_Char, '\b' ); };
602 '\\t' => { token( RE_Char, '\t' ); };
603 '\\n' => { token( RE_Char, '\n' ); };
604 '\\v' => { token( RE_Char, '\v' ); };
605 '\\f' => { token( RE_Char, '\f' ); };
606 '\\r' => { token( RE_Char, '\r' ); };
607 '\\\n' => { updateCol(); };
608 '\\' any => { token( RE_Char, tokstart+1, tokend ); };
610 # Terminate an OR expression.
612 token( RE_Slash, tokstart, tokend );
616 # Special characters.
617 '.' => { token( RE_Dot ); };
618 '*' => { token( RE_Star ); };
620 '[' => { token( RE_SqOpen ); fcall or_literal; };
621 '[^' => { token( RE_SqOpenNeg ); fcall or_literal; };
624 scan_error() << "unterminated regular expression" << endl;
627 # Characters in an OR expression.
628 [^\/] => { token( RE_Char, tokstart, tokend ); };
631 # We need a separate token space here to avoid the ragel keywords.
632 write_statement := |*
633 ident => { token( TK_Word, tokstart, tokend ); } ;
634 [ \t\n]+ => { updateCol(); };
635 ';' => { token( ';' ); fgoto parser_def; };
638 scan_error() << "unterminated write statement" << endl;
642 # Parser definitions.
644 'machine' => { token( KW_Machine ); };
645 'include' => { token( KW_Include ); };
648 fgoto write_statement;
650 'action' => { token( KW_Action ); };
651 'alphtype' => { token( KW_AlphType ); };
653 # FIXME: Enable this post 5.17.
654 # 'range' => { token( KW_Range ); };
658 inlineBlockType = SemiTerminated;
663 inlineBlockType = SemiTerminated;
667 token( KW_Variable );
668 inlineBlockType = SemiTerminated;
671 'when' => { token( KW_When ); };
672 'eof' => { token( KW_Eof ); };
673 'err' => { token( KW_Err ); };
674 'lerr' => { token( KW_Lerr ); };
675 'to' => { token( KW_To ); };
676 'from' => { token( KW_From ); };
677 'export' => { token( KW_Export ); };
678 'entry' => { token( KW_Entry ); };
681 ident => { token( TK_Word, tokstart, tokend ); } ;
684 number => { token( TK_UInt, tokstart, tokend ); };
685 hex_number => { token( TK_Hex, tokstart, tokend ); };
687 # Literals, with optionals.
688 ( s_literal | d_literal ) [i]?
689 => { token( TK_Literal, tokstart, tokend ); };
691 '[' => { token( RE_SqOpen ); fcall or_literal; };
692 '[^' => { token( RE_SqOpenNeg ); fcall or_literal; };
694 '/' => { token( RE_Slash ); fgoto re_literal; };
697 pound_comment => { updateCol(); };
699 ':=' => { token( TK_ColonEquals ); };
702 ">~" => { token( TK_StartToState ); };
703 "$~" => { token( TK_AllToState ); };
704 "%~" => { token( TK_FinalToState ); };
705 "<~" => { token( TK_NotStartToState ); };
706 "@~" => { token( TK_NotFinalToState ); };
707 "<>~" => { token( TK_MiddleToState ); };
710 ">*" => { token( TK_StartFromState ); };
711 "$*" => { token( TK_AllFromState ); };
712 "%*" => { token( TK_FinalFromState ); };
713 "<*" => { token( TK_NotStartFromState ); };
714 "@*" => { token( TK_NotFinalFromState ); };
715 "<>*" => { token( TK_MiddleFromState ); };
718 ">/" => { token( TK_StartEOF ); };
719 "$/" => { token( TK_AllEOF ); };
720 "%/" => { token( TK_FinalEOF ); };
721 "</" => { token( TK_NotStartEOF ); };
722 "@/" => { token( TK_NotFinalEOF ); };
723 "<>/" => { token( TK_MiddleEOF ); };
725 # Global Error actions.
726 ">!" => { token( TK_StartGblError ); };
727 "$!" => { token( TK_AllGblError ); };
728 "%!" => { token( TK_FinalGblError ); };
729 "<!" => { token( TK_NotStartGblError ); };
730 "@!" => { token( TK_NotFinalGblError ); };
731 "<>!" => { token( TK_MiddleGblError ); };
733 # Local error actions.
734 ">^" => { token( TK_StartLocalError ); };
735 "$^" => { token( TK_AllLocalError ); };
736 "%^" => { token( TK_FinalLocalError ); };
737 "<^" => { token( TK_NotStartLocalError ); };
738 "@^" => { token( TK_NotFinalLocalError ); };
739 "<>^" => { token( TK_MiddleLocalError ); };
742 "<>" => { token( TK_Middle ); };
745 '>?' => { token( TK_StartCond ); };
746 '$?' => { token( TK_AllCond ); };
747 '%?' => { token( TK_LeavingCond ); };
749 '..' => { token( TK_DotDot ); };
750 '**' => { token( TK_StarStar ); };
751 '--' => { token( TK_DashDash ); };
752 '->' => { token( TK_Arrow ); };
753 '=>' => { token( TK_DoubleArrow ); };
755 ":>" => { token( TK_ColonGt ); };
756 ":>>" => { token( TK_ColonGtGt ); };
757 "<:" => { token( TK_LtColon ); };
759 # Opening of longest match.
760 "|*" => { token( TK_BarStar ); };
768 [ \t\r]+ => { updateCol(); };
770 # If we are in a single line machine then newline may end the spec.
773 if ( singleLineSpec ) {
780 if ( lastToken == KW_Export || lastToken == KW_Entry )
785 inlineBlockType = CurlyDelimited;
791 scan_error() << "unterminated ragel section" << endl;
794 any => { token( *tokstart ); } ;
800 /* If no errors and we are at the bottom of the include stack (the
801 * source file listed on the command line) then write out the data. */
802 if ( includeDepth == 0 && machineSpec == 0 && machineName == 0 )
803 xmlEscapeHost( output, tokstart, tokend-tokstart );
806 # Outside code scanner. These tokens get passed through.
810 c_cpp_comment => pass;
811 s_literal | d_literal => pass;
814 singleLineSpec = false;
820 singleLineSpec = true;
833 void Scanner::do_scan()
836 char *buf = new char[bufsize];
837 const char last_char = 0;
838 int cs, act, have = 0;
842 bool singleLineSpec = false;
843 InlineBlockType inlineBlockType;
848 char *p = buf + have;
849 int space = bufsize - have;
852 /* We filled up the buffer trying to scan a token. Grow it. */
853 bufsize = bufsize * 2;
854 char *newbuf = new char[bufsize];
856 /* Recompute p and space. */
858 space = bufsize - have;
860 /* Patch up pointers possibly in use. */
862 tokstart = newbuf + ( tokstart - buf );
863 tokend = newbuf + ( tokend - buf );
865 /* Copy the new buffer in. */
866 memcpy( newbuf, buf, have );
871 input.read( p, space );
872 int len = input.gcount();
874 /* If we see eof then append the EOF char. */
876 p[0] = last_char, len = 1;
883 /* Check if we failed. */
884 if ( cs == rlscan_error ) {
885 /* Machine failed before finding a token. I'm not yet sure if this
887 scan_error() << "scanner error" << endl;
891 /* Decide if we need to preserve anything. */
892 char *preserve = tokstart;
894 /* Now set up the prefix. */
898 /* There is data that needs to be shifted over. */
899 have = pe - preserve;
900 memmove( buf, preserve, have );
901 unsigned int shiftback = preserve - buf;
903 tokstart -= shiftback;
913 void scan( char *fileName, istream &input, ostream &output )
915 Scanner scanner( fileName, input, output, 0, 0, 0 );
920 eofLoc.fileName = fileName;
922 eofLoc.line = scanner.line;