src/doctokenizer.l

   1 /******************************************************************************
   2  *
   3  * $Id: $
   4  *
   5  *
   6  * Copyright (C) 1997-2012 by Dimitri van Heesch.
   7  *
   8  * Permission to use, copy, modify, and distribute this software and its
   9  * documentation under the terms of the GNU General Public License is hereby
  10  * granted. No representations are made about the suitability of this software
  11  * for any purpose. It is provided "as is" without express or implied warranty.
  12  * See the GNU General Public License for more details.
  13  *
  14  * Documents produced by Doxygen are derivative works derived from the
  15  * input used in their production; they are not affected by this license.
  16  *
  17  */
  18
  19
  20 %{
  21
  22 #include <qfile.h>
  23 #include <qstring.h>
  24 #include <qstack.h>
  25 #include <qdict.h>
  26 #include <qregexp.h>
  27
  28 #include "doctokenizer.h"
  29 #include "cmdmapper.h"
  30 #include "config.h"
  31 #include "message.h"
  32 #include "section.h"
  33 #include "membergroup.h"
  34 #include "definition.h"
  35 #include "doxygen.h"
  36 #include "portable.h"
  37
  38 #define YY_NEVER_INTERACTIVE 1
  39 #define YY_NO_INPUT 1
  40
  41 //--------------------------------------------------------------------------
  42
  43 // context for tokenizer phase
  44 static int g_commentState;
  45 TokenInfo *g_token = 0;
  46 static int g_inputPos = 0;
  47 static const char *g_inputString;
  48 static QCString g_fileName;
  49 static bool g_insidePre;
  50
  51 // context for section finding phase
  52 static Definition  *g_definition;
  53 static MemberGroup *g_memberGroup;
  54 static QCString     g_secLabel;
  55 static QCString     g_secTitle;
  56 static SectionInfo::SectionType g_secType;
  57 static QCString     g_endMarker;
  58
  59 struct DocLexerContext
  60 {
  61   TokenInfo *token;
  62   int rule;
  63   int inputPos;
  64   const char *inputString;
  65   YY_BUFFER_STATE state;
  66 };
  67
  68 static QStack<DocLexerContext> g_lexerStack;
  69
  70 //--------------------------------------------------------------------------
  71
  72 void doctokenizerYYpushContext()
  73 {
  74   DocLexerContext *ctx = new DocLexerContext;
  75   ctx->rule = YY_START;
  76   ctx->token = g_token;
  77   ctx->inputPos = g_inputPos;
  78   ctx->inputString = g_inputString;
  79   ctx->state = YY_CURRENT_BUFFER;
  80   g_lexerStack.push(ctx);
  81   yy_switch_to_buffer(yy_create_buffer(doctokenizerYYin, YY_BUF_SIZE));
  82 }
  83
  84 bool doctokenizerYYpopContext()
  85 {
  86   if (g_lexerStack.isEmpty()) return FALSE;
  87   DocLexerContext *ctx = g_lexerStack.pop();
  88   g_inputPos = ctx->inputPos;
  89   g_inputString = ctx->inputString;
  90   yy_delete_buffer(YY_CURRENT_BUFFER);
  91   yy_switch_to_buffer(ctx->state);
  92   BEGIN(ctx->rule);
  93   delete ctx;
  94   return TRUE;
  95 }
  96
  97
  98 //--------------------------------------------------------------------------
  99
 100 const char *tokToString(int token)
 101 {
 102   switch (token)
 103   {
 104     case 0:              return "TK_EOF";
 105     case TK_WORD:        return "TK_WORD";
 106     case TK_LNKWORD:     return "TK_LNKWORD";
 107     case TK_WHITESPACE:  return "TK_WHITESPACE";
 108     case TK_LISTITEM:    return "TK_LISTITEM";
 109     case TK_ENDLIST:     return "TK_ENDLIST";
 110     case TK_COMMAND:     return "TK_COMMAND";
 111     case TK_HTMLTAG:     return "TK_HTMLTAG";
 112     case TK_SYMBOL:      return "TK_SYMBOL";
 113     case TK_NEWPARA:     return "TK_NEWPARA";
 114     case TK_RCSTAG:      return "TK_RCSTAG";
 115     case TK_URL:         return "TK_URL";
 116   }
 117   return "ERROR";
 118 }
 119
 120 static int computeIndent(const char *str,int length)
 121 {
 122   int i;
 123   int indent=0;
 124   static int tabSize=Config_getInt("TAB_SIZE");
 125   for (i=0;i<length;i++)
 126   {
 127     if (str[i]=='\t')
 128     {
 129       indent+=tabSize - (indent%tabSize);
 130     }
 131     else if (str[i]=='\n')
 132     {
 133       indent=0;
 134     }
 135     else
 136     {
 137       indent++;
 138     }
 139   }
 140   return indent;
 141 }
 142
 143 //--------------------------------------------------------------------------
 144
 145 static void processSection()
 146 {
 147   //printf("%s: found section/anchor with name `%s'\n",g_fileName.data(),g_secLabel.data());
 148   QCString file;
 149   if (g_memberGroup)
 150   {
 151     file = g_memberGroup->parent()->getOutputFileBase();
 152   }
 153   else if (g_definition)
 154   {
 155     file = g_definition->getOutputFileBase();
 156   }
 157   else
 158   {
 159     warn(g_fileName,yylineno,"Found section/anchor %s without context\n",g_secLabel.data());
 160   }
 161   SectionInfo *si=0;
 162   if ((si=Doxygen::sectionDict.find(g_secLabel)))
 163   {
 164     si->fileName = file;
 165     //si = new SectionInfo(file,g_secLabel,g_secTitle,g_secType);
 166     //Doxygen::sectionDict.insert(g_secLabel,si);
 167   }
 168 }
 169
 170 static void handleHtmlTag()
 171 {
 172   QCString tagText=yytext;
 173   g_token->attribs.clear();
 174   g_token->endTag = FALSE;
 175   g_token->emptyTag = FALSE;
 176
 177   // Check for end tag
 178   int startNamePos=1;
 179   if (tagText.at(1)=='/')
 180   {
 181     g_token->endTag = TRUE;
 182     startNamePos++;
 183   }
 184
 185   // Parse the name portion
 186   int i = startNamePos;
 187   for (i=startNamePos; i < (int)yyleng; i++)
 188   {
 189     // Check for valid HTML/XML name chars (including namespaces)
 190     char c = tagText.at(i);
 191     if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
 192   }
 193   g_token->name = tagText.mid(startNamePos,i-startNamePos);
 194
 195   // Parse the attributes. Each attribute is a name, value pair
 196   // The result is stored in g_token->attribs.
 197   int startName,endName,startAttrib,endAttrib;
 198   while (i<(int)yyleng)
 199   {
 200     char c=tagText.at(i);
 201     // skip spaces
 202     while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 203     // check for end of the tag
 204     if (c == '>') break;
 205     // Check for XML style "empty" tag.
 206     if (c == '/')
 207     {
 208       g_token->emptyTag = TRUE;
 209       break;
 210     }
 211     startName=i;
 212     // search for end of name
 213     while (i<(int)yyleng && !isspace(c) && c!='=') { c=tagText.at(++i); }
 214     endName=i;
 215     HtmlAttrib opt;
 216     opt.name  = tagText.mid(startName,endName-startName).lower();
 217     // skip spaces
 218     while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 219     if (tagText.at(i)=='=') // option has value
 220     {
 221       c=tagText.at(++i);
 222       // skip spaces
 223       while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 224       if (tagText.at(i)=='\'') // option '...'
 225       {
 226         c=tagText.at(++i);
 227         startAttrib=i;
 228
 229         // search for matching quote
 230         while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
 231         endAttrib=i;
 232         if (i<(int)yyleng) c=tagText.at(++i);
 233       }
 234       else if (tagText.at(i)=='"') // option "..."
 235       {
 236         c=tagText.at(++i);
 237         startAttrib=i;
 238         // search for matching quote
 239         while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
 240         endAttrib=i;
 241         if (i<(int)yyleng) c=tagText.at(++i);
 242       }
 243       else // value without any quotes
 244       {
 245         startAttrib=i;
 246         // search for separator or end symbol
 247         while (i<(int)yyleng && !isspace(c) && c!='>') { c=tagText.at(++i); }
 248         endAttrib=i;
 249         if (i<(int)yyleng) c=tagText.at(++i);
 250       }
 251       opt.value  = tagText.mid(startAttrib,endAttrib-startAttrib);
 252     }
 253     else // start next option
 254     {
 255     }
 256     //printf("=====> Adding option name=<%s> value=<%s>\n",
 257     //    opt.name.data(),opt.value.data());
 258     g_token->attribs.append(&opt);
 259   }
 260 }
 261
 262 static QCString stripEmptyLines(const QCString &s)
 263 {
 264   if (s.isEmpty()) return QCString();
 265   int end=s.length();
 266   int start=0,p=0;
 267   // skip leading empty lines
 268   for (;;)
 269   {
 270     int c;
 271     while ((c=s[p]) && (c==' ' || c=='\t')) p++;
 272     if (s[p]=='\n')
 273     {
 274       start=++p;
 275     }
 276     else
 277     {
 278       break;
 279     }
 280   }
 281   // skip trailing empty lines
 282   p=end-1;
 283   if (p>=start && s.at(p)=='\n') p--;
 284   while (p>=start)
 285   {
 286     int c;
 287     while ((c=s[p]) && (c==' ' || c=='\t')) p--;
 288     if (s[p]=='\n')
 289     {
 290       end=p;
 291     }
 292     else
 293     {
 294       break;
 295     }
 296     p--;
 297   }
 298   //printf("stripEmptyLines(%d-%d)\n",start,end);
 299   return s.mid(start,end-start);
 300 }
 301
 302 //--------------------------------------------------------------------------
 303
 304 #undef  YY_INPUT
 305 #define YY_INPUT(buf,result,max_size) result=yyread(buf,max_size);
 306
 307 static int yyread(char *buf,int max_size)
 308 {
 309   int c=0;
 310   const char *src=g_inputString+g_inputPos;
 311   while ( c < max_size && *src ) *buf++ = *src++, c++;
 312   g_inputPos+=c;
 313   return c;
 314 }
 315
 316 //--------------------------------------------------------------------------
 317
 318 %}
 319
 320 CMD       ("\\"|"@")
 321 WS        [ \t\r\n]
 322 NONWS     [^ \t\r\n]
 323 BLANK     [ \t\r]
 324 ID        "$"?[a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]*
 325 LABELID   [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-]*
 326 PHPTYPE   [\\:a-z_A-Z0-9\x80-\xFF\-]+
 327 CITEID    [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-:/]*
 328 MAILADR   ("mailto:")?[a-z_A-Z0-9.+-]+"@"[a-z_A-Z0-9-]+("."[a-z_A-Z0-9\-]+)+[a-z_A-Z0-9\-]+
 329 OPTSTARS  ("//"{BLANK}*)?"*"*{BLANK}*
 330 LISTITEM  {BLANK}*[-]("#")?{WS}
 331 MLISTITEM {BLANK}*[+*]{WS}
 332 OLISTITEM {BLANK}*[1-9][0-9]*"."{BLANK}
 333 ENDLIST   {BLANK}*"."{BLANK}*\n
 334 ATTRIB    {ID}{WS}*("="{WS}*(("\""[^\"]*"\"")|("'"[^\']*"'")|[^ \t\r\n'"><]+))?
 335 URLCHAR   [a-z_A-Z0-9\!\~\,\:\;\'\$\?\@\&\%\#\.\-\+\/\=]
 336 URLMASK   ({URLCHAR}+([({]{URLCHAR}*[)}])?)+
 337 FILESCHAR [a-z_A-Z0-9\\:\\\/\-\+]
 338 FILEECHAR [a-z_A-Z0-9\-\+]
 339 HFILEMASK ("."{FILESCHAR}*{FILEECHAR}+)*
 340 FILEMASK  ({FILESCHAR}*{FILEECHAR}+("."{FILESCHAR}*{FILEECHAR}+)*)|{HFILEMASK}
 341 LINKMASK  [^ \t\n\r\\@<&${}]+("("[^\n)]*")")?({BLANK}*("const"|"volatile"){BLANK}+)?
 342 VERBATIM  "verbatim"{BLANK}*
 343 SPCMD1    {CMD}([a-z_A-Z][a-z_A-Z0-9]*|{VERBATIM})
 344 SPCMD2    {CMD}[\\@<>&$#%~".|]
 345 SPCMD3    {CMD}form#[0-9]+
 346 SPCMD4    {CMD}"::"
 347 INOUT     "in"|"out"|("in"{BLANK}*","{BLANK}*"out")|("out"{BLANK}*","{BLANK}*"in")
 348 PARAMIO   {CMD}param{BLANK}*"["{BLANK}*{INOUT}{BLANK}*"]"
 349 TEMPCHAR  [a-z_A-Z0-9,: \t\*\&]
 350 FUNCCHAR  [a-z_A-Z0-9,:\<\> \t\*\&\[\]]
 351 SCOPESEP  "::"|"#"|"."
 352 TEMPLPART "<"{TEMPCHAR}*">"
 353 SCOPEPRE  {ID}{TEMPLPART}?{SCOPESEP}
 354 SCOPEKEYS ":"({ID}":")*
 355 SCOPECPP  {SCOPEPRE}*(~)?{ID}("<"{TEMPCHAR}*">")?
 356 SCOPEOBJC {SCOPEPRE}?{ID}{SCOPEKEYS}?
 357 SCOPEMASK {SCOPECPP}|{SCOPEOBJC}
 358 FUNCARG   "("{FUNCCHAR}*")"({BLANK}*("volatile"|"const"){BLANK})?
 359 OPNEW     {BLANK}+"new"({BLANK}*"[]")?
 360 OPDEL     {BLANK}+"delete"({BLANK}*"[]")?
 361 OPNORM    {OPNEW}|{OPDEL}|"+"|"-"|"*"|"/"|"%"|"^"|"&"|"|"|"~"|"!"|"="|"<"|">"|"+="|"-="|"*="|"/="|"%="|"^="|"&="|"|="|"<<"|">>"|"<<="|">>="|"=="|"!="|"<="|">="|"&&"|"||"|"++"|"--"|","|"->*"|"->"|"[]"|"()"
 362 OPCAST    {BLANK}+[^<(\r\n.,][^(\r\n.,]*
 363 OPMASK    ({BLANK}*{OPNORM}{FUNCARG})
 364 OPMASKOPT ({BLANK}*{OPNORM}{FUNCARG}?)|({OPCAST}{FUNCARG})
 365 LNKWORD1  ("::"|"#")?{SCOPEMASK}
 366 CVSPEC    {BLANK}*("const"|"volatile")
 367 LNKWORD2  (({SCOPEPRE}*"operator"{OPMASK})|({SCOPEPRE}"operator"{OPMASKOPT})|(("::"|"#"){SCOPEPRE}*"operator"{OPMASKOPT})){CVSPEC}?
 368 LNKWORD3  ([0-9a-z_A-Z\-]+("/"|"\\"))*[0-9a-z_A-Z\-]+("."[0-9a-z_A-Z]+)+
 369 CHARWORDQ [^ \t\n\r\\@<>()\[\]:;\?{}&%$#,."=']
 370 ESCWORD   ("%"{ID}(("::"|"."){ID})*)|("%'")
 371 WORD1     {ESCWORD}|{CHARWORDQ}+|"{"|"}"|"'\"'"|("\""[^"\n]*\n?[^"\n]*"\"")
 372 WORD2     "."|","|"("|")"|"["|"]"|":"|";"|"\?"|"="|"'"
 373 WORD1NQ   {ESCWORD}|{CHARWORDQ}+|"{"|"}"
 374 WORD2NQ   "."|","|"("|")"|"["|"]"|":"|";"|"\?"|"="|"'"
 375 HTMLTAG   "<"(("/")?){ID}({WS}+{ATTRIB})*{WS}*(("/")?)">"
 376 HTMLKEYL  "strong"|"center"|"table"|"caption"|"small"|"code"|"dfn"|"var"|"img"|"pre"|"sub"|"sup"|"tr"|"td"|"th"|"ol"|"ul"|"li"|"tt"|"kbd"|"em"|"hr"|"dl"|"dt"|"dd"|"br"|"i"|"a"|"b"|"p"
 377 HTMLKEYU  "STRONG"|"CENTER"|"TABLE"|"CAPTION"|"SMALL"|"CODE"|"DFN"|"VAR"|"IMG"|"PRE"|"SUB"|"SUP"|"TR"|"TD"|"TH"|"OL"|"UL"|"LI"|"TT"|"KBD"|"EM"|"HR"|"DL"|"DT"|"DD"|"BR"|"I"|"A"|"B"|"P"
 378 HTMLKEYW  {HTMLKEYL}|{HTMLKEYU}
 379 REFWORD2  ("#"|"::")?({ID}{TEMPLPART}?("."|"#"|"::"|"-"|"/"))*({ID}(":")?){FUNCARG}?
 380 REFWORD3  ({ID}":")*{ID}":"?
 381 REFWORD   {LABELID}|{REFWORD2}|{REFWORD3}
 382
 383 %option noyywrap
 384 %option yylineno
 385
 386 %x St_Para
 387 %x St_Comment
 388 %x St_Title
 389 %x St_TitleN
 390 %x St_TitleQ
 391 %x St_TitleA
 392 %x St_TitleV
 393 %x St_Code
 394 %x St_CodeOpt
 395 %x St_XmlCode
 396 %x St_HtmlOnly
 397 %x St_ManOnly
 398 %x St_LatexOnly
 399 %x St_RtfOnly
 400 %x St_XmlOnly
 401 %x St_Verbatim
 402 %x St_Dot
 403 %x St_Msc
 404 %x St_Param
 405 %x St_XRefItem
 406 %x St_XRefItem2
 407 %x St_File
 408 %x St_Pattern
 409 %x St_Link
 410 %x St_Cite
 411 %x St_Ref
 412 %x St_Ref2
 413 %x St_IntRef
 414 %x St_Text
 415 %x St_SkipTitle
 416 %x St_Anchor
 417 %x St_Snippet
 418
 419 %x St_Sections
 420 %s St_SecLabel1
 421 %s St_SecLabel2
 422 %s St_SecTitle
 423 %x St_SecSkip
 424
 425 %%
 426 <St_Para>\r            /* skip carriage return */
 427 <St_Para>^{LISTITEM}   { /* list item */
 428                          QCString text=yytext;
 429                          int dashPos = text.findRev('-');
 430                          g_token->isEnumList = text.at(dashPos+1)=='#';
 431                          g_token->id         = -1;
 432                          g_token->indent     = computeIndent(yytext,dashPos);
 433                          return TK_LISTITEM;
 434                        }
 435 <St_Para>^{MLISTITEM}  { /* list item */
 436                          if (!Doxygen::markdownSupport || g_insidePre)
 437                          {
 438                            REJECT;
 439                          }
 440                          else
 441                          {
 442                            QCString text=yytext;
 443                            static QRegExp re("[*+]");
 444                            int listPos = text.findRev(re);
 445                            g_token->isEnumList = FALSE;
 446                            g_token->id         = -1;
 447                            g_token->indent     = computeIndent(yytext,listPos);
 448                            return TK_LISTITEM;
 449                          }
 450                        }
 451 <St_Para>^{OLISTITEM}  { /* numbered list item */
 452                          if (!Doxygen::markdownSupport || g_insidePre)
 453                          {
 454                            REJECT;
 455                          }
 456                          else
 457                          {
 458                            QCString text=yytext;
 459                            static QRegExp re("[1-9]");
 460                            int digitPos = text.find(re);
 461                            int dotPos = text.find('.',digitPos);
 462                            g_token->isEnumList = TRUE;
 463                            g_token->id         = atoi(QCString(yytext).mid(digitPos,dotPos-digitPos));
 464                            g_token->indent     = computeIndent(yytext,digitPos);
 465                            return TK_LISTITEM;
 466                          }
 467                        }
 468 <St_Para>{BLANK}*\n{LISTITEM}     { /* list item on next line */
 469                          QCString text=yytext;
 470                          text=text.right(text.length()-text.find('\n')-1);
 471                          int dashPos = text.findRev('-');
 472                          g_token->isEnumList = text.at(dashPos+1)=='#';
 473                          g_token->id         = -1;
 474                          g_token->indent     = computeIndent(text,dashPos);
 475                          return TK_LISTITEM;
 476                        }
 477 <St_Para>{BLANK}*\n{MLISTITEM}     { /* list item on next line */
 478                          if (!Doxygen::markdownSupport || g_insidePre)
 479                          {
 480                            REJECT;
 481                          }
 482                          else
 483                          {
 484                            QCString text=yytext;
 485                            static QRegExp re("[*+]");
 486                            text=text.right(text.length()-text.find('\n')-1);
 487                            int markPos = text.findRev(re);
 488                            g_token->isEnumList = FALSE;
 489                            g_token->id         = -1;
 490                            g_token->indent     = computeIndent(text,markPos);
 491                            return TK_LISTITEM;
 492                          }
 493                        }
 494 <St_Para>{BLANK}*\n{OLISTITEM}     { /* list item on next line */
 495                          if (!Doxygen::markdownSupport || g_insidePre)
 496                          {
 497                            REJECT;
 498                          }
 499                          else
 500                          {
 501                            QCString text=yytext;
 502                            int nl=text.findRev('\n');
 503                            int len=text.length();
 504                            text=text.right(len-nl-1);
 505                            static QRegExp re("[1-9]");
 506                            int digitPos = text.find(re);
 507                            int dotPos = text.find('.',digitPos);
 508                            g_token->isEnumList = TRUE;
 509                            g_token->id         = atoi(QCString(text).mid(digitPos,dotPos-digitPos));
 510                            g_token->indent     = computeIndent(text,digitPos);
 511                            return TK_LISTITEM;
 512                          }
 513                        }
 514 <St_Para>^{ENDLIST}       { /* end list */
 515                          int dotPos = QCString(yytext).findRev('.');
 516                          g_token->indent     = computeIndent(yytext,dotPos);
 517                          return TK_ENDLIST;
 518                        }
 519 <St_Para>{BLANK}*\n{ENDLIST}      { /* end list on next line */
 520                          QCString text=yytext;
 521                          text=text.right(text.length()-text.find('\n')-1);
 522                          int dotPos = text.findRev('.');
 523                          g_token->indent     = computeIndent(text,dotPos);
 524                          return TK_ENDLIST;
 525                        }
 526 <St_Para>"{"{BLANK}*"@link" {
 527                          g_token->name = "javalink";
 528                          return TK_COMMAND;
 529                        }
 530 <St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
 531                          g_token->name = "inheritdoc";
 532                          return TK_COMMAND;
 533                        }
 534 <St_Para>"@_fakenl"     { // artificial new line
 535                              yylineno++;
 536                           }
 537 <St_Para>{SPCMD3}      {
 538                          g_token->name = "form";
 539                          bool ok;
 540                          g_token->id = QCString(yytext).right((int)yyleng-6).toInt(&ok);
 541                          ASSERT(ok);
 542                          return TK_COMMAND;
 543                        }
 544 <St_Para>{SPCMD1}      |
 545 <St_Para>{SPCMD2}      |
 546 <St_Para>{SPCMD4}      { /* special command */
 547                          g_token->name = yytext+1;
 548                          g_token->name = g_token->name.stripWhiteSpace();
 549                          g_token->paramDir=TokenInfo::Unspecified;
 550                          return TK_COMMAND;
 551                        }
 552 <St_Para>{PARAMIO}     { /* param [in,out] command */
 553                          g_token->name = "param";
 554                          QCString s(yytext);
 555                          bool isIn  = s.find("in")!=-1;
 556                          bool isOut = s.find("out")!=-1;
 557                          if (isIn)
 558                          {
 559                            if (isOut)
 560                            {
 561                              g_token->paramDir=TokenInfo::InOut;
 562                            }
 563                            else
 564                            {
 565                              g_token->paramDir=TokenInfo::In;
 566                            }
 567                          }
 568                          else if (isOut)
 569                          {
 570                            g_token->paramDir=TokenInfo::Out;
 571                          }
 572                          else
 573                          {
 574                            g_token->paramDir=TokenInfo::Unspecified;
 575                          }
 576                          return TK_COMMAND;
 577                        }
 578 <St_Para>("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK}/\. { // URL.
 579                          g_token->name=yytext;
 580                          g_token->isEMailAddr=FALSE;
 581                          return TK_URL;
 582                        }
 583 <St_Para>("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK} { // URL
 584                          g_token->name=yytext;
 585                          g_token->isEMailAddr=FALSE;
 586                          return TK_URL;
 587                        }
 588 <St_Para>"<"("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK}">" { // URL
 589                          g_token->name=yytext;
 590                          g_token->name = g_token->name.mid(1,g_token->name.length()-2);
 591                          g_token->isEMailAddr=FALSE;
 592                          return TK_URL;
 593                        }
 594 <St_Para>{MAILADR}     { // Mail address
 595                          g_token->name=yytext;
 596                          g_token->name.stripPrefix("mailto:");
 597                          g_token->isEMailAddr=TRUE;
 598                          return TK_URL;
 599                        }
 600 <St_Para>"<"{MAILADR}">" { // Mail address
 601                          g_token->name=yytext;
 602                          g_token->name = g_token->name.mid(1,g_token->name.length()-2);
 603                          g_token->name.stripPrefix("mailto:");
 604                          g_token->isEMailAddr=TRUE;
 605                          return TK_URL;
 606                        }
 607 <St_Para>"$"{ID}":"[^\n$]+"$" { /* RCS tag */
 608                          QCString tagName(yytext+1);
 609                          int index=tagName.find(':');
 610                          g_token->name = tagName.left(index+1);
 611                          g_token->text = tagName.mid(index+2,tagName.length()-index-3);
 612                          return TK_RCSTAG;
 613                        }
 614 <St_Para,St_HtmlOnly>"$("{ID}")"   { /* environment variable */
 615                          QCString name = &yytext[2];
 616                          name = name.left(name.length()-1);
 617                          QCString value = portable_getenv(name);
 618                          for (int i=value.length()-1;i>=0;i--) unput(value.at(i));
 619                        }
 620 <St_Para>{HTMLTAG}     { /* html tag */
 621                          handleHtmlTag();
 622                          return TK_HTMLTAG;
 623                        }
 624 <St_Para,St_Text>"&"{ID}";" { /* special symbol */
 625                          g_token->name = yytext;
 626                          return TK_SYMBOL;
 627                        }
 628
 629   /********* patterns for linkable words ******************/
 630
 631 <St_Para>{ID}/"<"{HTMLKEYW}">" { /* this rule is to prevent opening html
 632                                   * tag to be recognized as a templated classes
 633                                   */
 634                          g_token->name = yytext;
 635                          return TK_LNKWORD;
 636                         }
 637 <St_Para>{LNKWORD1}/"<br>"           | // prevent <br> html tag to be parsed as template arguments
 638 <St_Para>{LNKWORD1}                  |
 639 <St_Para>{LNKWORD1}{FUNCARG}         |
 640 <St_Para>{LNKWORD2}                  |
 641 <St_Para>{LNKWORD3}    {
 642                          g_token->name = yytext;
 643                          return TK_LNKWORD;
 644                        }
 645 <St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
 646                          g_token->name = yytext;
 647                          g_token->name = g_token->name.left(g_token->name.length()-1);
 648                          unput(yytext[(int)yyleng-1]);
 649                          return TK_LNKWORD;
 650                        }
 651   /********* patterns for normal words ******************/
 652
 653 <St_Para,St_Text>{WORD1} |
 654 <St_Para,St_Text>{WORD2} { /* function call */
 655                          if (yytext[0]=='%') // strip % if present
 656                            g_token->name = &yytext[1];
 657                          else
 658                            g_token->name = yytext;
 659                          return TK_WORD;
 660
 661                          /* the following is dummy code to please the
 662                           * compiler, removing this results in a warning
 663                           * on my machine
 664                           */
 665                          goto find_rule;
 666                        }
 667 <St_Text>({ID}".")+{ID} {
 668                           g_token->name = yytext;
 669                           return TK_WORD;
 670                         }
 671 <St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
 672                                                           // avoid interpretation as "operator <"
 673                            g_token->name = yytext;
 674                            return TK_WORD;
 675                          }
 676
 677   /*******************************************************/
 678
 679 <St_Para,St_Text>{BLANK}+      |
 680 <St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
 681                          g_token->chars=yytext;
 682                          return TK_WHITESPACE;
 683                        }
 684 <St_Text>[\\@<>&$#%~]  {
 685                          g_token->name = yytext;
 686                          return TK_COMMAND;
 687                        }
 688 <St_Para>({BLANK}*\n)+{BLANK}*\n{BLANK}* {
 689                          // g_insidePre was always FALSE, so the next section
 690                          // was never executed, now g_insidePre is set properly
 691                          // again, so the section is commented out to keep the
 692                          // old behavior.
 693                          //if (g_insidePre)
 694                          //{
 695                          //  /* Inside a <pre>..</pre> blank lines are treated
 696                          //   * as whitespace.
 697                          //   */
 698                          //  g_token->chars=yytext;
 699                          //  return TK_WHITESPACE;
 700                          //}
 701                          //else // found end of a paragraph
 702                          {
 703                            g_token->indent=computeIndent(yytext,(int)yyleng);
 704                            int i;
 705                            // put back the indentation (needed for list items)
 706                            for (i=0;i<g_token->indent;i++)
 707                            {
 708                              unput(' ');
 709                            }
 710                            // tell flex that after putting the last indent
 711                            // back we are at the beginning of the line
 712                            YY_CURRENT_BUFFER->yy_at_bol=1;
 713                            // start of a new paragraph
 714                            return TK_NEWPARA;
 715                          }
 716                        }
 717 <St_CodeOpt>{BLANK}*"{"(".")?{LABELID}"}" {
 718                          g_token->name = yytext;
 719                          int i=g_token->name.find('{');
 720                          g_token->name = g_token->name.mid(i+1,g_token->name.length()-i-2);
 721                          BEGIN(St_Code);
 722                        }
 723 <St_CodeOpt>\n         |
 724 <St_CodeOpt>.          {
 725                          unput(*yytext);
 726                          BEGIN(St_Code);
 727                        }
 728 <St_Code>{WS}*{CMD}"endcode" {
 729                          return RetVal_OK;
 730                        }
 731 <St_XmlCode>{WS}*"</code>" {
 732                          return RetVal_OK;
 733                        }
 734 <St_Code,St_XmlCode>[^\\@\n<]+  |
 735 <St_Code,St_XmlCode>\n          |
 736 <St_Code,St_XmlCode>.           {
 737                          g_token->verb+=yytext;
 738                        }
 739 <St_HtmlOnly>{CMD}"endhtmlonly" {
 740                          return RetVal_OK;
 741                        }
 742 <St_HtmlOnly>[^\\@\n$]+    |
 743 <St_HtmlOnly>\n            |
 744 <St_HtmlOnly>.             {
 745                          g_token->verb+=yytext;
 746                        }
 747 <St_ManOnly>{CMD}"endmanonly" {
 748                          return RetVal_OK;
 749                        }
 750 <St_ManOnly>[^\\@\n$]+    |
 751 <St_ManOnly>\n            |
 752 <St_ManOnly>.             {
 753                          g_token->verb+=yytext;
 754                        }
 755 <St_RtfOnly>{CMD}"endrtfonly" {
 756                          return RetVal_OK;
 757                        }
 758 <St_RtfOnly>[^\\@\n$]+    |
 759 <St_RtfOnly>\n            |
 760 <St_RtfOnly>.             {
 761                          g_token->verb+=yytext;
 762                        }
 763 <St_LatexOnly>{CMD}"endlatexonly" {
 764                          return RetVal_OK;
 765                        }
 766 <St_LatexOnly>[^\\@\n]+     |
 767 <St_LatexOnly>\n            |
 768 <St_LatexOnly>.             {
 769                          g_token->verb+=yytext;
 770                        }
 771 <St_XmlOnly>{CMD}"endxmlonly" {
 772                          return RetVal_OK;
 773                        }
 774 <St_XmlOnly>[^\\@\n]+  |
 775 <St_XmlOnly>\n         |
 776 <St_XmlOnly>.          {
 777                          g_token->verb+=yytext;
 778                        }
 779 <St_Verbatim>{CMD}"endverbatim" {
 780                          g_token->verb=stripEmptyLines(g_token->verb);
 781                          return RetVal_OK;
 782                        }
 783 <St_Verbatim>[^\\@\n]+ |
 784 <St_Verbatim>\n        |
 785 <St_Verbatim>.         { /* Verbatim text */
 786                          g_token->verb+=yytext;
 787                        }
 788 <St_Dot>{CMD}"enddot"  {
 789                          return RetVal_OK;
 790                        }
 791 <St_Dot>[^\\@\n]+      |
 792 <St_Dot>\n             |
 793 <St_Dot>.              { /* dot text */
 794                          g_token->verb+=yytext;
 795                        }
 796 <St_Msc>{CMD}"endmsc"  {
 797                          return RetVal_OK;
 798                        }
 799 <St_Msc>[^\\@\n]+      |
 800 <St_Msc>\n             |
 801 <St_Msc>.              { /* msc text */
 802                          g_token->verb+=yytext;
 803                        }
 804 <St_Title>"\""         { // quoted title
 805                          BEGIN(St_TitleQ);
 806                        }
 807 <St_Title>[ \t]+       {
 808                          g_token->chars=yytext;
 809                          return TK_WHITESPACE;
 810                        }
 811 <St_Title>.            { // non-quoted title
 812                          unput(*yytext);
 813                          BEGIN(St_TitleN);
 814                        }
 815 <St_Title>\n           {
 816                          unput(*yytext);
 817                          return 0;
 818                        }
 819 <St_TitleN>"&"{ID}";"  { /* symbol */
 820                          g_token->name = yytext;
 821                          return TK_SYMBOL;
 822                        }
 823 <St_TitleN>{HTMLTAG}   {
 824                        }
 825 <St_TitleN>{SPCMD1}    |
 826 <St_TitleN>{SPCMD2}    { /* special command */
 827                          g_token->name = yytext+1;
 828                          g_token->paramDir=TokenInfo::Unspecified;
 829                          return TK_COMMAND;
 830                        }
 831 <St_TitleN>{WORD1}     |
 832 <St_TitleN>{WORD2}     { /* word */
 833                          if (yytext[0]=='%') // strip % if present
 834                            g_token->name = &yytext[1];
 835                          else
 836                            g_token->name = yytext;
 837                          return TK_WORD;
 838                        }
 839 <St_TitleN>[ \t]+      {
 840                          g_token->chars=yytext;
 841                          return TK_WHITESPACE;
 842                        }
 843 <St_TitleN>\n          { /* new line => end of title */
 844                          unput(*yytext);
 845                          return 0;
 846                        }
 847 <St_TitleQ>"&"{ID}";"  { /* symbol */
 848                          g_token->name = yytext;
 849                          return TK_SYMBOL;
 850                        }
 851 <St_TitleQ>{SPCMD1}    |
 852 <St_TitleQ>{SPCMD2}    { /* special command */
 853                          g_token->name = yytext+1;
 854                          g_token->paramDir=TokenInfo::Unspecified;
 855                          return TK_COMMAND;
 856                        }
 857 <St_TitleQ>{WORD1NQ}   |
 858 <St_TitleQ>{WORD2NQ}   { /* word */
 859                          g_token->name = yytext;
 860                          return TK_WORD;
 861                        }
 862 <St_TitleQ>[ \t]+      {
 863                          g_token->chars=yytext;
 864                          return TK_WHITESPACE;
 865                        }
 866 <St_TitleQ>"\""        { /* closing quote => end of title */
 867                          BEGIN(St_TitleA);
 868                          return 0;
 869                        }
 870 <St_TitleQ>\n          { /* new line => end of title */
 871                          unput(*yytext);
 872                          return 0;
 873                        }
 874 <St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
 875                          g_token->name = yytext;
 876                          g_token->name = g_token->name.left(
 877                                g_token->name.find('=')).stripWhiteSpace();
 878                          BEGIN(St_TitleV);
 879                        }
 880 <St_TitleV>[^ \t\r\n]+ { // attribute value
 881                          g_token->chars = yytext;
 882                          BEGIN(St_TitleN);
 883                          return TK_WORD;
 884                        }
 885 <St_TitleV,St_TitleA>. {
 886                          unput(*yytext);
 887                          return 0;
 888                        }
 889 <St_TitleV,St_TitleA>\n  {
 890                          return 0;
 891                        }
 892
 893 <St_Anchor>{LABELID}{WS}? { // anchor
 894                          g_token->name = QCString(yytext).stripWhiteSpace();
 895                          return TK_WORD;
 896                        }
 897 <St_Anchor>.           {
 898                          unput(*yytext);
 899                          return 0;
 900                        }
 901 <St_Cite>{CITEID}      { // label to cite
 902                          g_token->name=yytext;
 903                          return TK_WORD;
 904                        }
 905 <St_Cite>{BLANK}       { // white space
 906                          unput(' ');
 907                          return 0;
 908                        }
 909 <St_Cite>\n            { // new line
 910                          unput(*yytext);
 911                          return 0;
 912                        }
 913 <St_Cite>.             { // any other character
 914                          unput(*yytext);
 915                          return 0;
 916                        }
 917 <St_Ref>{REFWORD}      { // label to refer to
 918                          g_token->name=yytext;
 919                          return TK_WORD;
 920                        }
 921 <St_Ref>{BLANK}        { // white space
 922                          unput(' ');
 923                          return 0;
 924                        }
 925 <St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
 926                          BEGIN(St_Ref2);
 927                        }
 928 <St_Ref>\n             { // new line
 929                          unput(*yytext);
 930                          return 0;
 931                        }
 932 <St_Ref>.              { // any other character
 933                          unput(*yytext);
 934                          return 0;
 935                        }
 936 <St_IntRef>[A-Z_a-z0-9.:/#\-\+\(\)]+ {
 937                          g_token->name = yytext;
 938                          return TK_WORD;
 939                        }
 940 <St_IntRef>{BLANK}+"\"" {
 941                          BEGIN(St_Ref2);
 942                        }
 943 <St_Ref2>"&"{ID}";"    { /* symbol */
 944                          g_token->name = yytext;
 945                          return TK_SYMBOL;
 946                        }
 947 <St_Ref2>{SPCMD1}      |
 948 <St_Ref2>{SPCMD2}      { /* special command */
 949                          g_token->name = yytext+1;
 950                          g_token->paramDir=TokenInfo::Unspecified;
 951                          return TK_COMMAND;
 952                        }
 953 <St_Ref2>{WORD1NQ}     |
 954 <St_Ref2>{WORD2NQ}     {
 955                          /* word */
 956                          g_token->name = yytext;
 957                          return TK_WORD;
 958                        }
 959 <St_Ref2>[ \t]+        {
 960                          g_token->chars=yytext;
 961                          return TK_WHITESPACE;
 962                        }
 963 <St_Ref2>"\""|\n       { /* " or \n => end of title */
 964                          return 0;
 965                        }
 966 <St_XRefItem>{LABELID} {
 967                          g_token->name=yytext;
 968                        }
 969 <St_XRefItem>" "       {
 970                          BEGIN(St_XRefItem2);
 971                        }
 972 <St_XRefItem2>[0-9]+"." {
 973                          QCString numStr=yytext;
 974                          numStr=numStr.left((int)yyleng-1);
 975                          g_token->id=numStr.toInt();
 976                          return RetVal_OK;
 977                        }
 978 <St_Para,St_Title,St_Ref2>"<!--"     { /* html style comment block */
 979                          g_commentState = YY_START;
 980                          BEGIN(St_Comment);
 981                        }
 982 <St_Param>"\""[^\n\"]+"\"" {
 983                          g_token->name = yytext+1;
 984                          g_token->name = g_token->name.left((int)yyleng-2);
 985                          return TK_WORD;
 986                        }
 987 <St_Param>({PHPTYPE}{BLANK}*"|"{BLANK}*)*{PHPTYPE}{WS}+("&")?"$"{LABELID} {
 988                          QCString params = yytext;
 989                          int j = params.find('&');
 990                          int i = params.find('$');
 991                          if (j<i && j!=-1) i=j;
 992                          QCString types = params.left(i).stripWhiteSpace();
 993                          g_token->name = types+"#"+params.mid(i);
 994                          return TK_WORD;
 995                        }
 996 <St_Param>[^ \t\n,]+   {
 997                          g_token->name = yytext;
 998                          return TK_WORD;
 999                        }
1000 <St_Param>{WS}*","{WS}*  /* param separator */
1001 <St_Param>{WS}         {
1002                          g_token->chars=yytext;
1003                          return TK_WHITESPACE;
1004                        }
1005 <St_File>{FILEMASK}    {
1006                          g_token->name = yytext;
1007                          return TK_WORD;
1008                        }
1009 <St_File>"\""[^\n\"]+"\"" {
1010                          QCString text=yytext;
1011                          g_token->name = text.mid(1,text.length()-2);
1012                          return TK_WORD;
1013                        }
1014 <St_Pattern>[^\r\n]+   {
1015                          g_token->name = yytext;
1016                          g_token->name = g_token->name.stripWhiteSpace();
1017                          return TK_WORD;
1018                        }
1019 <St_Link>{LINKMASK}|{REFWORD}    {
1020                          g_token->name = yytext;
1021                          return TK_WORD;
1022                        }
1023 <St_Comment>"-->"      { /* end of html comment */
1024                          BEGIN(g_commentState);
1025                        }
1026 <St_Comment>[^-\n]+       /* inside html comment */
1027 <St_Comment>.             /* inside html comment */
1028
1029      /* State for skipping title (all chars until the end of the line) */
1030
1031 <St_SkipTitle>.
1032 <St_SkipTitle>\n       { return 0; }
1033
1034      /* State for the pass used to find the anchors and sections */
1035
1036 <St_Sections>[^\n@\\]+
1037 <St_Sections>"@@"|"\\\\"
1038 <St_Sections>{CMD}"anchor"{BLANK}+  {
1039                                       g_secType = SectionInfo::Anchor;
1040                                       BEGIN(St_SecLabel1);
1041                                     }
1042 <St_Sections>{CMD}"section"{BLANK}+ {
1043                                       g_secType = SectionInfo::Section;
1044                                       BEGIN(St_SecLabel2);
1045                                     }
1046 <St_Sections>{CMD}"subsection"{BLANK}+ {
1047                                       g_secType = SectionInfo::Subsection;
1048                                       BEGIN(St_SecLabel2);
1049                                     }
1050 <St_Sections>{CMD}"subsubsection"{BLANK}+ {
1051                                       g_secType = SectionInfo::Subsubsection;
1052                                       BEGIN(St_SecLabel2);
1053                                     }
1054 <St_Sections>{CMD}"paragraph"{BLANK}+ {
1055                                       g_secType = SectionInfo::Paragraph;
1056                                       BEGIN(St_SecLabel2);
1057                                     }
1058 <St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9]  {
1059                                       g_endMarker="endverbatim";
1060                                       BEGIN(St_SecSkip);
1061                                     }
1062 <St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1063                                       g_endMarker="enddot";
1064                                       BEGIN(St_SecSkip);
1065                                     }
1066 <St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1067                                       g_endMarker="endmsc";
1068                                       BEGIN(St_SecSkip);
1069                                     }
1070 <St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1071                                       g_endMarker="endhtmlonly";
1072                                       BEGIN(St_SecSkip);
1073                                     }
1074 <St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1075                                       g_endMarker="endlatexonly";
1076                                       BEGIN(St_SecSkip);
1077                                     }
1078 <St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1079                                       g_endMarker="endxmlonly";
1080                                       BEGIN(St_SecSkip);
1081                                     }
1082 <St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1083                                       g_endMarker="endcode";
1084                                       BEGIN(St_SecSkip);
1085                                     }
1086 <St_Sections>"<!--"                 {
1087                                       g_endMarker="-->";
1088                                       BEGIN(St_SecSkip);
1089                                     }
1090 <St_SecSkip>{CMD}{ID}               {
1091                                       if (strcmp(yytext+1,g_endMarker)==0)
1092                                       {
1093                                         BEGIN(St_Sections);
1094                                       }
1095                                     }
1096 <St_SecSkip>"-->"                   {
1097                                       if (strcmp(yytext,g_endMarker)==0)
1098                                       {
1099                                         BEGIN(St_Sections);
1100                                       }
1101                                     }
1102 <St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1103 <St_SecSkip>.
1104 <St_SecSkip>\n
1105 <St_Sections>.
1106 <St_Sections>\n
1107 <St_SecLabel1>{LABELID} {
1108                          g_secLabel = yytext;
1109                          processSection();
1110                          BEGIN(St_Sections);
1111                        }
1112 <St_SecLabel2>{LABELID}{BLANK}+ |
1113 <St_SecLabel2>{LABELID}         {
1114                          g_secLabel = yytext;
1115                          g_secLabel = g_secLabel.stripWhiteSpace();
1116                          BEGIN(St_SecTitle);
1117                        }
1118 <St_SecTitle>[^\n]+    |
1119 <St_SecTitle>[^\n]*\n  {
1120                          g_secTitle = yytext;
1121                          g_secTitle = g_secTitle.stripWhiteSpace();
1122                          processSection();
1123                          BEGIN(St_Sections);
1124                        }
1125 <St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1126                          warn(g_fileName,yylineno,"warning: Unexpected character `%s' while looking for section label or title",yytext);
1127                        }
1128
1129 <St_Snippet>[^\n]+     |
1130 <St_Snippet>[^\n]*\n   {
1131                          g_token->name = yytext;
1132                          g_token->name = g_token->name.stripWhiteSpace();
1133                          return TK_WORD;
1134                        }
1135
1136      /* Generic rules that work for all states */
1137 <*>\n                  {
1138                          warn(g_fileName,yylineno,"warning: Unexpected new line character");
1139                        }
1140 <*>[\\@<>&$#%~"=]      { /* unescaped special character */
1141                          //warn(g_fileName,yylineno,"warning: Unexpected character `%s', assuming command \\%s was meant.",yytext,yytext);
1142                          g_token->name = yytext;
1143                          return TK_COMMAND;
1144                        }
1145 <*>.                   {
1146                          warn(g_fileName,yylineno,"warning: Unexpected character `%s'",yytext);
1147                        }
1148 %%
1149
1150 //--------------------------------------------------------------------------
1151
1152 void doctokenizerYYFindSections(const char *input,Definition *d,
1153                                 MemberGroup *mg,const char *fileName)
1154 {
1155   if (input==0) return;
1156   g_inputString = input;
1157   //printf("parsing --->`%s'<---\n",input);
1158   g_inputPos    = 0;
1159   g_definition  = d;
1160   g_memberGroup = mg;
1161   g_fileName    = fileName;
1162   BEGIN(St_Sections);
1163   doctokenizerYYlineno = 1;
1164   doctokenizerYYlex();
1165 }
1166
1167 void doctokenizerYYinit(const char *input,const char *fileName)
1168 {
1169   g_inputString = input;
1170   g_inputPos    = 0;
1171   g_fileName    = fileName;
1172   g_insidePre   = FALSE;
1173   BEGIN(St_Para);
1174 }
1175
1176 void doctokenizerYYsetStatePara()
1177 {
1178   BEGIN(St_Para);
1179 }
1180
1181 void doctokenizerYYsetStateTitle()
1182 {
1183   BEGIN(St_Title);
1184 }
1185
1186 void doctokenizerYYsetStateTitleAttrValue()
1187 {
1188   BEGIN(St_TitleV);
1189 }
1190
1191 void doctokenizerYYsetStateCode()
1192 {
1193   g_token->verb="";
1194   g_token->name="";
1195   BEGIN(St_CodeOpt);
1196 }
1197
1198 void doctokenizerYYsetStateXmlCode()
1199 {
1200   g_token->verb="";
1201   g_token->name="";
1202   BEGIN(St_XmlCode);
1203 }
1204
1205 void doctokenizerYYsetStateHtmlOnly()
1206 {
1207   g_token->verb="";
1208   BEGIN(St_HtmlOnly);
1209 }
1210
1211 void doctokenizerYYsetStateManOnly()
1212 {
1213   g_token->verb="";
1214   BEGIN(St_ManOnly);
1215 }
1216
1217 void doctokenizerYYsetStateRtfOnly()
1218 {
1219   g_token->verb="";
1220   BEGIN(St_RtfOnly);
1221 }
1222
1223 void doctokenizerYYsetStateXmlOnly()
1224 {
1225   g_token->verb="";
1226   BEGIN(St_XmlOnly);
1227 }
1228
1229 void doctokenizerYYsetStateLatexOnly()
1230 {
1231   g_token->verb="";
1232   BEGIN(St_LatexOnly);
1233 }
1234
1235 void doctokenizerYYsetStateVerbatim()
1236 {
1237   g_token->verb="";
1238   BEGIN(St_Verbatim);
1239 }
1240
1241 void doctokenizerYYsetStateDot()
1242 {
1243   g_token->verb="";
1244   BEGIN(St_Dot);
1245 }
1246
1247 void doctokenizerYYsetStateMsc()
1248 {
1249   g_token->verb="";
1250   BEGIN(St_Msc);
1251 }
1252
1253 void doctokenizerYYsetStateParam()
1254 {
1255   BEGIN(St_Param);
1256 }
1257
1258 void doctokenizerYYsetStateXRefItem()
1259 {
1260   BEGIN(St_XRefItem);
1261 }
1262
1263 void doctokenizerYYsetStateFile()
1264 {
1265   BEGIN(St_File);
1266 }
1267
1268 void doctokenizerYYsetStatePattern()
1269 {
1270   BEGIN(St_Pattern);
1271 }
1272
1273 void doctokenizerYYsetStateLink()
1274 {
1275   BEGIN(St_Link);
1276 }
1277
1278 void doctokenizerYYsetStateCite()
1279 {
1280   BEGIN(St_Cite);
1281 }
1282
1283 void doctokenizerYYsetStateRef()
1284 {
1285   BEGIN(St_Ref);
1286 }
1287
1288 void doctokenizerYYsetStateInternalRef()
1289 {
1290   BEGIN(St_IntRef);
1291 }
1292
1293 void doctokenizerYYsetStateText()
1294 {
1295   BEGIN(St_Text);
1296 }
1297
1298 void doctokenizerYYsetStateSkipTitle()
1299 {
1300   BEGIN(St_SkipTitle);
1301 }
1302
1303 void doctokenizerYYsetStateAnchor()
1304 {
1305   BEGIN(St_Anchor);
1306 }
1307
1308 void doctokenizerYYsetStateSnippet()
1309 {
1310   BEGIN(St_Snippet);
1311 }
1312
1313 void doctokenizerYYcleanup()
1314 {
1315   yy_delete_buffer( YY_CURRENT_BUFFER );
1316 }
1317
1318 void doctokenizerYYsetInsidePre(bool b)
1319 {
1320   g_insidePre = b;
1321 }
1322
1323 void doctokenizerYYpushBackHtmlTag(const char *tag)
1324 {
1325   QCString tagName = tag;
1326   int i,l = tagName.length();
1327   unput('>');
1328   for (i=l-1;i>=0;i--)
1329   {
1330     unput(tag[i]);
1331   }
1332   unput('<');
1333 }
1334
1335 #if !defined(YY_FLEX_SUBMINOR_VERSION)
1336 extern "C" { // some bogus code to keep the compiler happy
1337     void doctokenizerYYdummy() { yy_flex_realloc(0,0); }
1338 }
1339 #endif
1340