src/doctokenizer.l

   1 /******************************************************************************
   2  *
   3  * $Id: $
   4  *
   5  *
   6  * Copyright (C) 1997-2014 by Dimitri van Heesch.
   7  *
   8  * Permission to use, copy, modify, and distribute this software and its
   9  * documentation under the terms of the GNU General Public License is hereby
  10  * granted. No representations are made about the suitability of this software
  11  * for any purpose. It is provided "as is" without express or implied warranty.
  12  * See the GNU General Public License for more details.
  13  *
  14  * Documents produced by Doxygen are derivative works derived from the
  15  * input used in their production; they are not affected by this license.
  16  *
  17  */
  18
  19
  20 %{
  21
  22 #include <ctype.h>
  23
  24 #include <qfile.h>
  25 #include <qstring.h>
  26 #include <qstack.h>
  27 #include <qdict.h>
  28 #include <qregexp.h>
  29
  30 #include "doctokenizer.h"
  31 #include "cmdmapper.h"
  32 #include "config.h"
  33 #include "message.h"
  34 #include "section.h"
  35 #include "membergroup.h"
  36 #include "definition.h"
  37 #include "doxygen.h"
  38 #include "portable.h"
  39
  40 #define YY_NEVER_INTERACTIVE 1
  41 #define YY_NO_INPUT 1
  42
  43 //--------------------------------------------------------------------------
  44
  45 // context for tokenizer phase
  46 static int g_commentState;
  47 TokenInfo *g_token = 0;
  48 static int g_inputPos = 0;
  49 static const char *g_inputString;
  50 static QCString g_fileName;
  51 static bool g_insidePre;
  52
  53 // context for section finding phase
  54 static Definition  *g_definition;
  55 static MemberGroup *g_memberGroup;
  56 static QCString     g_secLabel;
  57 static QCString     g_secTitle;
  58 static SectionInfo::SectionType g_secType;
  59 static QCString     g_endMarker;
  60 static int          g_autoListLevel;
  61
  62 struct DocLexerContext
  63 {
  64   TokenInfo *token;
  65   int rule;
  66   int autoListLevel;
  67   int inputPos;
  68   const char *inputString;
  69   YY_BUFFER_STATE state;
  70 };
  71
  72 static QStack<DocLexerContext> g_lexerStack;
  73
  74 //--------------------------------------------------------------------------
  75
  76 void doctokenizerYYpushContext()
  77 {
  78   DocLexerContext *ctx = new DocLexerContext;
  79   ctx->rule = YY_START;
  80   ctx->autoListLevel = g_autoListLevel;
  81   ctx->token = g_token;
  82   ctx->inputPos = g_inputPos;
  83   ctx->inputString = g_inputString;
  84   ctx->state = YY_CURRENT_BUFFER;
  85   g_lexerStack.push(ctx);
  86   yy_switch_to_buffer(yy_create_buffer(doctokenizerYYin, YY_BUF_SIZE));
  87 }
  88
  89 bool doctokenizerYYpopContext()
  90 {
  91   if (g_lexerStack.isEmpty()) return FALSE;
  92   DocLexerContext *ctx = g_lexerStack.pop();
  93   g_autoListLevel = ctx->autoListLevel;
  94   g_inputPos = ctx->inputPos;
  95   g_inputString = ctx->inputString;
  96   yy_delete_buffer(YY_CURRENT_BUFFER);
  97   yy_switch_to_buffer(ctx->state);
  98   BEGIN(ctx->rule);
  99   delete ctx;
 100   return TRUE;
 101 }
 102
 103
 104 //--------------------------------------------------------------------------
 105
 106 const char *tokToString(int token)
 107 {
 108   switch (token)
 109   {
 110     case 0:              return "TK_EOF";
 111     case TK_WORD:        return "TK_WORD";
 112     case TK_LNKWORD:     return "TK_LNKWORD";
 113     case TK_WHITESPACE:  return "TK_WHITESPACE";
 114     case TK_LISTITEM:    return "TK_LISTITEM";
 115     case TK_ENDLIST:     return "TK_ENDLIST";
 116     case TK_COMMAND:     return "TK_COMMAND";
 117     case TK_HTMLTAG:     return "TK_HTMLTAG";
 118     case TK_SYMBOL:      return "TK_SYMBOL";
 119     case TK_NEWPARA:     return "TK_NEWPARA";
 120     case TK_RCSTAG:      return "TK_RCSTAG";
 121     case TK_URL:         return "TK_URL";
 122   }
 123   return "ERROR";
 124 }
 125
 126 static int computeIndent(const char *str,int length)
 127 {
 128   int i;
 129   int indent=0;
 130   static int tabSize=Config_getInt("TAB_SIZE");
 131   for (i=0;i<length;i++)
 132   {
 133     if (str[i]=='\t')
 134     {
 135       indent+=tabSize - (indent%tabSize);
 136     }
 137     else if (str[i]=='\n')
 138     {
 139       indent=0;
 140     }
 141     else
 142     {
 143       indent++;
 144     }
 145   }
 146   return indent;
 147 }
 148
 149 //--------------------------------------------------------------------------
 150
 151 static void processSection()
 152 {
 153   //printf("%s: found section/anchor with name `%s'\n",g_fileName.data(),g_secLabel.data());
 154   QCString file;
 155   if (g_memberGroup)
 156   {
 157     file = g_memberGroup->parent()->getOutputFileBase();
 158   }
 159   else if (g_definition)
 160   {
 161     file = g_definition->getOutputFileBase();
 162   }
 163   else
 164   {
 165     warn(g_fileName,yylineno,"Found section/anchor %s without context\n",g_secLabel.data());
 166   }
 167   SectionInfo *si=0;
 168   if ((si=Doxygen::sectionDict->find(g_secLabel)))
 169   {
 170     si->fileName = file;
 171     //si = new SectionInfo(file,g_secLabel,g_secTitle,g_secType);
 172     //Doxygen::sectionDict.insert(g_secLabel,si);
 173   }
 174 }
 175
 176 static void handleHtmlTag()
 177 {
 178   QCString tagText=yytext;
 179   g_token->attribs.clear();
 180   g_token->endTag = FALSE;
 181   g_token->emptyTag = FALSE;
 182
 183   // Check for end tag
 184   int startNamePos=1;
 185   if (tagText.at(1)=='/')
 186   {
 187     g_token->endTag = TRUE;
 188     startNamePos++;
 189   }
 190
 191   // Parse the name portion
 192   int i = startNamePos;
 193   for (i=startNamePos; i < (int)yyleng; i++)
 194   {
 195     // Check for valid HTML/XML name chars (including namespaces)
 196     char c = tagText.at(i);
 197     if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
 198   }
 199   g_token->name = tagText.mid(startNamePos,i-startNamePos);
 200
 201   // Parse the attributes. Each attribute is a name, value pair
 202   // The result is stored in g_token->attribs.
 203   int startName,endName,startAttrib,endAttrib;
 204   while (i<(int)yyleng)
 205   {
 206     char c=tagText.at(i);
 207     // skip spaces
 208     while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 209     // check for end of the tag
 210     if (c == '>') break;
 211     // Check for XML style "empty" tag.
 212     if (c == '/')
 213     {
 214       g_token->emptyTag = TRUE;
 215       break;
 216     }
 217     startName=i;
 218     // search for end of name
 219     while (i<(int)yyleng && !isspace(c) && c!='=') { c=tagText.at(++i); }
 220     endName=i;
 221     HtmlAttrib opt;
 222     opt.name  = tagText.mid(startName,endName-startName).lower();
 223     // skip spaces
 224     while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 225     if (tagText.at(i)=='=') // option has value
 226     {
 227       c=tagText.at(++i);
 228       // skip spaces
 229       while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 230       if (tagText.at(i)=='\'') // option '...'
 231       {
 232         c=tagText.at(++i);
 233         startAttrib=i;
 234
 235         // search for matching quote
 236         while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
 237         endAttrib=i;
 238         if (i<(int)yyleng) c=tagText.at(++i);
 239       }
 240       else if (tagText.at(i)=='"') // option "..."
 241       {
 242         c=tagText.at(++i);
 243         startAttrib=i;
 244         // search for matching quote
 245         while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
 246         endAttrib=i;
 247         if (i<(int)yyleng) c=tagText.at(++i);
 248       }
 249       else // value without any quotes
 250       {
 251         startAttrib=i;
 252         // search for separator or end symbol
 253         while (i<(int)yyleng && !isspace(c) && c!='>') { c=tagText.at(++i); }
 254         endAttrib=i;
 255         if (i<(int)yyleng) c=tagText.at(++i);
 256       }
 257       opt.value  = tagText.mid(startAttrib,endAttrib-startAttrib);
 258     }
 259     else // start next option
 260     {
 261     }
 262     //printf("=====> Adding option name=<%s> value=<%s>\n",
 263     //    opt.name.data(),opt.value.data());
 264     g_token->attribs.append(&opt);
 265   }
 266 }
 267
 268 static QCString stripEmptyLines(const QCString &s)
 269 {
 270   if (s.isEmpty()) return QCString();
 271   int end=s.length();
 272   int start=0,p=0;
 273   // skip leading empty lines
 274   for (;;)
 275   {
 276     int c;
 277     while ((c=s[p]) && (c==' ' || c=='\t')) p++;
 278     if (s[p]=='\n')
 279     {
 280       start=++p;
 281     }
 282     else
 283     {
 284       break;
 285     }
 286   }
 287   // skip trailing empty lines
 288   p=end-1;
 289   if (p>=start && s.at(p)=='\n') p--;
 290   while (p>=start)
 291   {
 292     int c;
 293     while ((c=s[p]) && (c==' ' || c=='\t')) p--;
 294     if (s[p]=='\n')
 295     {
 296       end=p;
 297     }
 298     else
 299     {
 300       break;
 301     }
 302     p--;
 303   }
 304   //printf("stripEmptyLines(%d-%d)\n",start,end);
 305   return s.mid(start,end-start);
 306 }
 307
 308 //--------------------------------------------------------------------------
 309
 310 #undef  YY_INPUT
 311 #define YY_INPUT(buf,result,max_size) result=yyread(buf,max_size);
 312
 313 static int yyread(char *buf,int max_size)
 314 {
 315   int c=0;
 316   const char *src=g_inputString+g_inputPos;
 317   while ( c < max_size && *src ) *buf++ = *src++, c++;
 318   g_inputPos+=c;
 319   return c;
 320 }
 321
 322 //--------------------------------------------------------------------------
 323 #define REAL_YY_DECL int doctokenizerYYlex (void)
 324 #define YY_DECL static int local_doctokinizer(void)
 325 #define LOCAL_YY_DECL local_doctokinizer()
 326
 327 %}
 328
 329 CMD       ("\\"|"@")
 330 WS        [ \t\r\n]
 331 NONWS     [^ \t\r\n]
 332 BLANK     [ \t\r]
 333 ID        "$"?[a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]*
 334 LABELID   [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-]*
 335 PHPTYPE   [\\:a-z_A-Z0-9\x80-\xFF\-]+
 336 CITEID    [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-:/]*
 337 MAILADR   ("mailto:")?[a-z_A-Z0-9.+-]+"@"[a-z_A-Z0-9-]+("."[a-z_A-Z0-9\-]+)+[a-z_A-Z0-9\-]+
 338 OPTSTARS  ("//"{BLANK}*)?"*"*{BLANK}*
 339 LISTITEM  {BLANK}*[-]("#")?{WS}
 340 MLISTITEM {BLANK}*[+*]{WS}
 341 OLISTITEM {BLANK}*[1-9][0-9]*"."{BLANK}
 342 ENDLIST   {BLANK}*"."{BLANK}*\n
 343 ATTRNAME  [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-]*
 344 ATTRIB    {ATTRNAME}{WS}*("="{WS}*(("\""[^\"]*"\"")|("'"[^\']*"'")|[^ \t\r\n'"><]+))?
 345 URLCHAR   [a-z_A-Z0-9\!\~\,\:\;\'\$\?\@\&\%\#\.\-\+\/\=]
 346 URLMASK   ({URLCHAR}+([({]{URLCHAR}*[)}])?)+
 347 FILESCHAR [a-z_A-Z0-9\\:\\\/\-\+@&#]
 348 FILEECHAR [a-z_A-Z0-9\-\+@&#]
 349 HFILEMASK ("."{FILESCHAR}*{FILEECHAR}+)*
 350 FILEMASK  ({FILESCHAR}*{FILEECHAR}+("."{FILESCHAR}*{FILEECHAR}+)*)|{HFILEMASK}
 351 LINKMASK  [^ \t\n\r\\@<&${}]+("("[^\n)]*")")?({BLANK}*("const"|"volatile"){BLANK}+)?
 352 VERBATIM  "verbatim"{BLANK}*
 353 SPCMD1    {CMD}([a-z_A-Z][a-z_A-Z0-9]*|{VERBATIM}|"--"|"---")
 354 SPCMD2    {CMD}[\\@<>&$#%~".|]
 355 SPCMD3    {CMD}form#[0-9]+
 356 SPCMD4    {CMD}"::"
 357 INOUT     "inout"|"in"|"out"|("in"{BLANK}*","{BLANK}*"out")|("out"{BLANK}*","{BLANK}*"in")
 358 PARAMIO   {CMD}param{BLANK}*"["{BLANK}*{INOUT}{BLANK}*"]"
 359 TEMPCHAR  [a-z_A-Z0-9.,: \t\*\&\(\)\[\]]
 360 FUNCCHAR  [a-z_A-Z0-9,:\<\> \t\^\*\&\[\]]
 361 FUNCPART  {FUNCCHAR}*("("{FUNCCHAR}*")"{FUNCCHAR}*)?
 362 SCOPESEP  "::"|"#"|"."
 363 TEMPLPART "<"{TEMPCHAR}*">"
 364 ANONNS    "anonymous_namespace{"[^}]*"}"
 365 SCOPEPRE  (({ID}{TEMPLPART}?)|{ANONNS}){SCOPESEP}
 366 SCOPEKEYS ":"({ID}":")*
 367 SCOPECPP  {SCOPEPRE}*(~)?{ID}{TEMPLPART}?
 368 SCOPEOBJC {SCOPEPRE}?{ID}{SCOPEKEYS}?
 369 SCOPEMASK {SCOPECPP}|{SCOPEOBJC}
 370 FUNCARG   "("{FUNCPART}")"({BLANK}*("volatile"|"const"){BLANK})?
 371 OPNEW     {BLANK}+"new"({BLANK}*"[]")?
 372 OPDEL     {BLANK}+"delete"({BLANK}*"[]")?
 373 OPNORM    {OPNEW}|{OPDEL}|"+"|"-"|"*"|"/"|"%"|"^"|"&"|"|"|"~"|"!"|"="|"<"|">"|"+="|"-="|"*="|"/="|"%="|"^="|"&="|"|="|"<<"|">>"|"<<="|">>="|"=="|"!="|"<="|">="|"&&"|"||"|"++"|"--"|","|"->*"|"->"|"[]"|"()"
 374 OPCAST    {BLANK}+[^<(\r\n.,][^(\r\n.,]*
 375 OPMASK    ({BLANK}*{OPNORM}{FUNCARG})
 376 OPMASKOPT ({BLANK}*{OPNORM}{FUNCARG}?)|({OPCAST}{FUNCARG})
 377 LNKWORD1  ("::"|"#")?{SCOPEMASK}
 378 CVSPEC    {BLANK}*("const"|"volatile")
 379 LNKWORD2  (({SCOPEPRE}*"operator"{OPMASK})|({SCOPEPRE}"operator"{OPMASKOPT})|(("::"|"#"){SCOPEPRE}*"operator"{OPMASKOPT})){CVSPEC}?
 380 LNKWORD3  ([0-9a-z_A-Z\-]+("/"|"\\"))*[0-9a-z_A-Z\-]+("."[0-9a-z_A-Z]+)+
 381 CHARWORDQ [^ \t\n\r\\@<>()\[\]:;\?{}&%$#,."=']
 382 ESCWORD   ("%"{ID}(("::"|"."){ID})*)|("%'")
 383 WORD1     {ESCWORD}|{CHARWORDQ}+|"{"|"}"|"'\"'"|("\""[^"\n]*\n?[^"\n]*"\"")
 384 WORD2     "."|","|"("|")"|"["|"]"|":"|";"|"\?"|"="|"'"
 385 WORD1NQ   {ESCWORD}|{CHARWORDQ}+|"{"|"}"
 386 WORD2NQ   "."|","|"("|")"|"["|"]"|":"|";"|"\?"|"="|"'"
 387 HTMLTAG   "<"(("/")?){ID}({WS}+{ATTRIB})*{WS}*(("/")?)">"
 388 HTMLKEYL  "strong"|"center"|"table"|"caption"|"small"|"code"|"dfn"|"var"|"img"|"pre"|"sub"|"sup"|"tr"|"td"|"th"|"ol"|"ul"|"li"|"tt"|"kbd"|"em"|"hr"|"dl"|"dt"|"dd"|"br"|"i"|"a"|"b"|"p"
 389 HTMLKEYU  "STRONG"|"CENTER"|"TABLE"|"CAPTION"|"SMALL"|"CODE"|"DFN"|"VAR"|"IMG"|"PRE"|"SUB"|"SUP"|"TR"|"TD"|"TH"|"OL"|"UL"|"LI"|"TT"|"KBD"|"EM"|"HR"|"DL"|"DT"|"DD"|"BR"|"I"|"A"|"B"|"P"
 390 HTMLKEYW  {HTMLKEYL}|{HTMLKEYU}
 391 REFWORD2  ("#"|"::")?((({ID}{TEMPLPART}?)|{ANONNS})("."|"#"|"::"|"-"|"/"))*({ID}{TEMPLPART}?(":")?){FUNCARG}?
 392 REFWORD3  ({ID}":")*{ID}":"?
 393 REFWORD   {LABELID}|{REFWORD2}|{REFWORD3}|{LNKWORD2}
 394
 395 %option noyywrap
 396 %option yylineno
 397
 398 %x St_Para
 399 %x St_Comment
 400 %x St_Title
 401 %x St_TitleN
 402 %x St_TitleQ
 403 %x St_TitleA
 404 %x St_TitleV
 405 %x St_Code
 406 %x St_CodeOpt
 407 %x St_XmlCode
 408 %x St_HtmlOnly
 409 %x St_HtmlOnlyOption
 410 %x St_ManOnly
 411 %x St_LatexOnly
 412 %x St_RtfOnly
 413 %x St_XmlOnly
 414 %x St_DbOnly
 415 %x St_Verbatim
 416 %x St_Dot
 417 %x St_Msc
 418 %x St_Param
 419 %x St_XRefItem
 420 %x St_XRefItem2
 421 %x St_File
 422 %x St_Pattern
 423 %x St_Link
 424 %x St_Cite
 425 %x St_Ref
 426 %x St_Ref2
 427 %x St_IntRef
 428 %x St_Text
 429 %x St_SkipTitle
 430 %x St_Anchor
 431 %x St_Snippet
 432
 433 %x St_Sections
 434 %s St_SecLabel1
 435 %s St_SecLabel2
 436 %s St_SecTitle
 437 %x St_SecSkip
 438
 439 %%
 440 <St_Para>\r            /* skip carriage return */
 441 <St_Para>^{LISTITEM}   { /* list item */
 442                          QCString text=yytext;
 443                          int dashPos = text.findRev('-');
 444                          g_token->isEnumList = text.at(dashPos+1)=='#';
 445                          g_token->id         = -1;
 446                          g_token->indent     = computeIndent(yytext,dashPos);
 447                          return TK_LISTITEM;
 448                        }
 449 <St_Para>^{MLISTITEM}  { /* list item */
 450                          if (!Doxygen::markdownSupport || g_insidePre)
 451                          {
 452                            REJECT;
 453                          }
 454                          else
 455                          {
 456                            QCString text=yytext;
 457                            static QRegExp re("[*+]");
 458                            int listPos = text.findRev(re);
 459                            g_token->isEnumList = FALSE;
 460                            g_token->id         = -1;
 461                            g_token->indent     = computeIndent(yytext,listPos);
 462                            return TK_LISTITEM;
 463                          }
 464                        }
 465 <St_Para>^{OLISTITEM}  { /* numbered list item */
 466                          if (!Doxygen::markdownSupport || g_insidePre)
 467                          {
 468                            REJECT;
 469                          }
 470                          else
 471                          {
 472                            QCString text=yytext;
 473                            static QRegExp re("[1-9]");
 474                            int digitPos = text.find(re);
 475                            int dotPos = text.find('.',digitPos);
 476                            g_token->isEnumList = TRUE;
 477                            g_token->id         = atoi(QCString(yytext).mid(digitPos,dotPos-digitPos));
 478                            g_token->indent     = computeIndent(yytext,digitPos);
 479                            return TK_LISTITEM;
 480                          }
 481                        }
 482 <St_Para>{BLANK}*\n{LISTITEM}     { /* list item on next line */
 483                          QCString text=yytext;
 484                          text=text.right(text.length()-text.find('\n')-1);
 485                          int dashPos = text.findRev('-');
 486                          g_token->isEnumList = text.at(dashPos+1)=='#';
 487                          g_token->id         = -1;
 488                          g_token->indent     = computeIndent(text,dashPos);
 489                          return TK_LISTITEM;
 490                        }
 491 <St_Para>{BLANK}*\n{MLISTITEM}     { /* list item on next line */
 492                          if (!Doxygen::markdownSupport || g_insidePre)
 493                          {
 494                            REJECT;
 495                          }
 496                          else
 497                          {
 498                            QCString text=yytext;
 499                            static QRegExp re("[*+]");
 500                            text=text.right(text.length()-text.find('\n')-1);
 501                            int markPos = text.findRev(re);
 502                            g_token->isEnumList = FALSE;
 503                            g_token->id         = -1;
 504                            g_token->indent     = computeIndent(text,markPos);
 505                            return TK_LISTITEM;
 506                          }
 507                        }
 508 <St_Para>{BLANK}*\n{OLISTITEM}     { /* list item on next line */
 509                          if (!Doxygen::markdownSupport || g_insidePre)
 510                          {
 511                            REJECT;
 512                          }
 513                          else
 514                          {
 515                            QCString text=yytext;
 516                            int nl=text.findRev('\n');
 517                            int len=text.length();
 518                            text=text.right(len-nl-1);
 519                            static QRegExp re("[1-9]");
 520                            int digitPos = text.find(re);
 521                            int dotPos = text.find('.',digitPos);
 522                            g_token->isEnumList = TRUE;
 523                            g_token->id         = atoi(QCString(text).mid(digitPos,dotPos-digitPos));
 524                            g_token->indent     = computeIndent(text,digitPos);
 525                            return TK_LISTITEM;
 526                          }
 527                        }
 528 <St_Para>^{ENDLIST}       { /* end list */
 529                          int dotPos = QCString(yytext).findRev('.');
 530                          g_token->indent     = computeIndent(yytext,dotPos);
 531                          return TK_ENDLIST;
 532                        }
 533 <St_Para>{BLANK}*\n{ENDLIST}      { /* end list on next line */
 534                          QCString text=yytext;
 535                          text=text.right(text.length()-text.find('\n')-1);
 536                          int dotPos = text.findRev('.');
 537                          g_token->indent     = computeIndent(text,dotPos);
 538                          return TK_ENDLIST;
 539                        }
 540 <St_Para>"{"{BLANK}*"@link" {
 541                          g_token->name = "javalink";
 542                          return TK_COMMAND;
 543                        }
 544 <St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
 545                          g_token->name = "inheritdoc";
 546                          return TK_COMMAND;
 547                        }
 548 <St_Para>"@_fakenl"     { // artificial new line
 549                              yylineno++;
 550                           }
 551 <St_Para>{SPCMD3}      {
 552                          g_token->name = "form";
 553                          bool ok;
 554                          g_token->id = QCString(yytext).right((int)yyleng-6).toInt(&ok);
 555                          ASSERT(ok);
 556                          return TK_COMMAND;
 557                        }
 558 <St_Para>{CMD}"n"\n    { /* \n followed by real newline */
 559                          yylineno++;
 560                          g_token->name = yytext+1;
 561                          g_token->name = g_token->name.stripWhiteSpace();
 562                          g_token->paramDir=TokenInfo::Unspecified;
 563                          return TK_COMMAND;
 564                        }
 565 <St_Para>{SPCMD1}      |
 566 <St_Para>{SPCMD2}      |
 567 <St_Para>{SPCMD4}      { /* special command */
 568                          g_token->name = yytext+1;
 569                          g_token->name = g_token->name.stripWhiteSpace();
 570                          g_token->paramDir=TokenInfo::Unspecified;
 571                          return TK_COMMAND;
 572                        }
 573 <St_Para>{PARAMIO}     { /* param [in,out] command */
 574                          g_token->name = "param";
 575                          QCString s(yytext);
 576                          bool isIn  = s.find("in")!=-1;
 577                          bool isOut = s.find("out")!=-1;
 578                          if (isIn)
 579                          {
 580                            if (isOut)
 581                            {
 582                              g_token->paramDir=TokenInfo::InOut;
 583                            }
 584                            else
 585                            {
 586                              g_token->paramDir=TokenInfo::In;
 587                            }
 588                          }
 589                          else if (isOut)
 590                          {
 591                            g_token->paramDir=TokenInfo::Out;
 592                          }
 593                          else
 594                          {
 595                            g_token->paramDir=TokenInfo::Unspecified;
 596                          }
 597                          return TK_COMMAND;
 598                        }
 599 <St_Para>("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK}/\. { // URL.
 600                          g_token->name=yytext;
 601                          g_token->isEMailAddr=FALSE;
 602                          return TK_URL;
 603                        }
 604 <St_Para>("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK} { // URL
 605                          g_token->name=yytext;
 606                          g_token->isEMailAddr=FALSE;
 607                          return TK_URL;
 608                        }
 609 <St_Para>"<"("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK}">" { // URL
 610                          g_token->name=yytext;
 611                          g_token->name = g_token->name.mid(1,g_token->name.length()-2);
 612                          g_token->isEMailAddr=FALSE;
 613                          return TK_URL;
 614                        }
 615 <St_Para>{MAILADR}     { // Mail address
 616                          g_token->name=yytext;
 617                          g_token->name.stripPrefix("mailto:");
 618                          g_token->isEMailAddr=TRUE;
 619                          return TK_URL;
 620                        }
 621 <St_Para>"<"{MAILADR}">" { // Mail address
 622                          g_token->name=yytext;
 623                          g_token->name = g_token->name.mid(1,g_token->name.length()-2);
 624                          g_token->name.stripPrefix("mailto:");
 625                          g_token->isEMailAddr=TRUE;
 626                          return TK_URL;
 627                        }
 628 <St_Para>"$"{ID}":"[^\n$]+"$" { /* RCS tag */
 629                          QCString tagName(yytext+1);
 630                          int index=tagName.find(':');
 631                          g_token->name = tagName.left(index);
 632                          int text_begin = index+2;
 633                          int text_end = tagName.length()-1;
 634                          if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
 635                          {
 636                                  ++text_begin;
 637                                  if (tagName[text_end-1]=='#')
 638                                          --text_end;
 639                          }
 640                          g_token->text = tagName.mid(text_begin,text_end-text_begin);
 641                          return TK_RCSTAG;
 642                        }
 643 <St_Para,St_HtmlOnly>"$("{ID}")"   { /* environment variable */
 644                          QCString name = &yytext[2];
 645                          name = name.left(name.length()-1);
 646                          QCString value = portable_getenv(name);
 647                          for (int i=value.length()-1;i>=0;i--) unput(value.at(i));
 648                        }
 649 <St_Para>{HTMLTAG}     { /* html tag */
 650                          handleHtmlTag();
 651                          return TK_HTMLTAG;
 652                        }
 653 <St_Para,St_Text>"&"{ID}";" { /* special symbol */
 654                          g_token->name = yytext;
 655                          return TK_SYMBOL;
 656                        }
 657
 658   /********* patterns for linkable words ******************/
 659
 660 <St_Para>{ID}/"<"{HTMLKEYW}">" { /* this rule is to prevent opening html
 661                                   * tag to be recognized as a templated classes
 662                                   */
 663                          g_token->name = yytext;
 664                          return TK_LNKWORD;
 665                         }
 666 <St_Para>{LNKWORD1}/"<br>"           | // prevent <br> html tag to be parsed as template arguments
 667 <St_Para>{LNKWORD1}                  |
 668 <St_Para>{LNKWORD1}{FUNCARG}         |
 669 <St_Para>{LNKWORD2}                  |
 670 <St_Para>{LNKWORD3}    {
 671                          g_token->name = yytext;
 672                          return TK_LNKWORD;
 673                        }
 674 <St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
 675                          g_token->name = yytext;
 676                          g_token->name = g_token->name.left(g_token->name.length()-1);
 677                          unput(yytext[(int)yyleng-1]);
 678                          return TK_LNKWORD;
 679                        }
 680   /********* patterns for normal words ******************/
 681
 682 <St_Para,St_Text>{WORD1} |
 683 <St_Para,St_Text>{WORD2} { /* function call */
 684                          if (yytext[0]=='%') // strip % if present
 685                            g_token->name = &yytext[1];
 686                          else
 687                            g_token->name = yytext;
 688                          return TK_WORD;
 689
 690                          /* the following is dummy code to please the
 691                           * compiler, removing this results in a warning
 692                           * on my machine
 693                           */
 694                          goto find_rule;
 695                        }
 696 <St_Text>({ID}".")+{ID} {
 697                           g_token->name = yytext;
 698                           return TK_WORD;
 699                         }
 700 <St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
 701                                                           // avoid interpretation as "operator <"
 702                            g_token->name = yytext;
 703                            return TK_WORD;
 704                          }
 705
 706   /*******************************************************/
 707
 708 <St_Para,St_Text>{BLANK}+      |
 709 <St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
 710                          g_token->chars=yytext;
 711                          return TK_WHITESPACE;
 712                        }
 713 <St_Text>[\\@<>&$#%~]  {
 714                          g_token->name = yytext;
 715                          return TK_COMMAND;
 716                        }
 717 <St_Para>({BLANK}*\n)+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
 718                          if (g_insidePre || g_autoListLevel==0)
 719                          {
 720                            REJECT;
 721                          }
 722                        }
 723 <St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
 724                          if (!Doxygen::markdownSupport || g_insidePre || g_autoListLevel==0)
 725                          {
 726                            REJECT;
 727                          }
 728                        }
 729 <St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
 730                          if (!Doxygen::markdownSupport || g_insidePre || g_autoListLevel==0)
 731                          {
 732                            REJECT;
 733                          }
 734                        }
 735 <St_Para>({BLANK}*\n)+{BLANK}*\n{BLANK}* {
 736                          g_token->indent=computeIndent(yytext,(int)yyleng);
 737                          int i;
 738                          // put back the indentation (needed for list items)
 739                          for (i=0;i<g_token->indent;i++)
 740                          {
 741                            unput(' ');
 742                          }
 743                          // tell flex that after putting the last indent
 744                          // back we are at the beginning of the line
 745                          YY_CURRENT_BUFFER->yy_at_bol=1;
 746                          // start of a new paragraph
 747                          return TK_NEWPARA;
 748                        }
 749 <St_CodeOpt>{BLANK}*"{"(".")?{LABELID}"}" {
 750                          g_token->name = yytext;
 751                          int i=g_token->name.find('{'); /* } to keep vi happy */
 752                          g_token->name = g_token->name.mid(i+1,g_token->name.length()-i-2);
 753                          BEGIN(St_Code);
 754                        }
 755 <St_CodeOpt>\n         |
 756 <St_CodeOpt>.          {
 757                          unput(*yytext);
 758                          BEGIN(St_Code);
 759                        }
 760 <St_Code>{WS}*{CMD}"endcode" {
 761                          return RetVal_OK;
 762                        }
 763 <St_XmlCode>{WS}*"</code>" {
 764                          return RetVal_OK;
 765                        }
 766 <St_Code,St_XmlCode>[^\\@\n<]+  |
 767 <St_Code,St_XmlCode>\n          |
 768 <St_Code,St_XmlCode>.           {
 769                          g_token->verb+=yytext;
 770                        }
 771 <St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
 772                          g_token->name="block";
 773                          BEGIN(St_HtmlOnly);
 774                         }
 775 <St_HtmlOnlyOption>.|\n {
 776                          unput(*yytext);
 777                          BEGIN(St_HtmlOnly);
 778                         }
 779 <St_HtmlOnly>{CMD}"endhtmlonly" {
 780                          return RetVal_OK;
 781                        }
 782 <St_HtmlOnly>[^\\@\n$]+    |
 783 <St_HtmlOnly>\n            |
 784 <St_HtmlOnly>.             {
 785                          g_token->verb+=yytext;
 786                        }
 787 <St_ManOnly>{CMD}"endmanonly" {
 788                          return RetVal_OK;
 789                        }
 790 <St_ManOnly>[^\\@\n$]+    |
 791 <St_ManOnly>\n            |
 792 <St_ManOnly>.             {
 793                          g_token->verb+=yytext;
 794                        }
 795 <St_RtfOnly>{CMD}"endrtfonly" {
 796                          return RetVal_OK;
 797                        }
 798 <St_RtfOnly>[^\\@\n$]+    |
 799 <St_RtfOnly>\n            |
 800 <St_RtfOnly>.             {
 801                          g_token->verb+=yytext;
 802                        }
 803 <St_LatexOnly>{CMD}"endlatexonly" {
 804                          return RetVal_OK;
 805                        }
 806 <St_LatexOnly>[^\\@\n]+     |
 807 <St_LatexOnly>\n            |
 808 <St_LatexOnly>.             {
 809                          g_token->verb+=yytext;
 810                        }
 811 <St_XmlOnly>{CMD}"endxmlonly" {
 812                          return RetVal_OK;
 813                        }
 814 <St_XmlOnly>[^\\@\n]+  |
 815 <St_XmlOnly>\n         |
 816 <St_XmlOnly>.          {
 817                          g_token->verb+=yytext;
 818                        }
 819 <St_DbOnly>{CMD}"enddocbookonly" {
 820                          return RetVal_OK;
 821                        }
 822 <St_DbOnly>[^\\@\n]+  |
 823 <St_DbOnly>\n         |
 824 <St_DbOnly>.          {
 825                          g_token->verb+=yytext;
 826                        }
 827 <St_Verbatim>{CMD}"endverbatim" {
 828                          g_token->verb=stripEmptyLines(g_token->verb);
 829                          return RetVal_OK;
 830                        }
 831 <St_Verbatim>[^\\@\n]+ |
 832 <St_Verbatim>\n        |
 833 <St_Verbatim>.         { /* Verbatim text */
 834                          g_token->verb+=yytext;
 835                        }
 836 <St_Dot>{CMD}"enddot"  {
 837                          return RetVal_OK;
 838                        }
 839 <St_Dot>[^\\@\n]+      |
 840 <St_Dot>\n             |
 841 <St_Dot>.              { /* dot text */
 842                          g_token->verb+=yytext;
 843                        }
 844 <St_Msc>{CMD}("endmsc"|"endvhdlflow")  {
 845                          return RetVal_OK;
 846                        }
 847 <St_Msc>[^\\@\n]+      |
 848 <St_Msc>\n             |
 849 <St_Msc>.              { /* msc text */
 850                          g_token->verb+=yytext;
 851                        }
 852 <St_Title>"\""         { // quoted title
 853                          BEGIN(St_TitleQ);
 854                        }
 855 <St_Title>[ \t]+       {
 856                          g_token->chars=yytext;
 857                          return TK_WHITESPACE;
 858                        }
 859 <St_Title>.            { // non-quoted title
 860                          unput(*yytext);
 861                          BEGIN(St_TitleN);
 862                        }
 863 <St_Title>\n           {
 864                          unput(*yytext);
 865                          return 0;
 866                        }
 867 <St_TitleN>"&"{ID}";"  { /* symbol */
 868                          g_token->name = yytext;
 869                          return TK_SYMBOL;
 870                        }
 871 <St_TitleN>{HTMLTAG}   {
 872                        }
 873 <St_TitleN>{SPCMD1}    |
 874 <St_TitleN>{SPCMD2}    { /* special command */
 875                          g_token->name = yytext+1;
 876                          g_token->paramDir=TokenInfo::Unspecified;
 877                          return TK_COMMAND;
 878                        }
 879 <St_TitleN>{WORD1}     |
 880 <St_TitleN>{WORD2}     { /* word */
 881                          if (yytext[0]=='%') // strip % if present
 882                            g_token->name = &yytext[1];
 883                          else
 884                            g_token->name = yytext;
 885                          return TK_WORD;
 886                        }
 887 <St_TitleN>[ \t]+      {
 888                          g_token->chars=yytext;
 889                          return TK_WHITESPACE;
 890                        }
 891 <St_TitleN>\n          { /* new line => end of title */
 892                          unput(*yytext);
 893                          return 0;
 894                        }
 895 <St_TitleQ>"&"{ID}";"  { /* symbol */
 896                          g_token->name = yytext;
 897                          return TK_SYMBOL;
 898                        }
 899 <St_TitleQ>{SPCMD1}    |
 900 <St_TitleQ>{SPCMD2}    { /* special command */
 901                          g_token->name = yytext+1;
 902                          g_token->paramDir=TokenInfo::Unspecified;
 903                          return TK_COMMAND;
 904                        }
 905 <St_TitleQ>{WORD1NQ}   |
 906 <St_TitleQ>{WORD2NQ}   { /* word */
 907                          g_token->name = yytext;
 908                          return TK_WORD;
 909                        }
 910 <St_TitleQ>[ \t]+      {
 911                          g_token->chars=yytext;
 912                          return TK_WHITESPACE;
 913                        }
 914 <St_TitleQ>"\""        { /* closing quote => end of title */
 915                          BEGIN(St_TitleA);
 916                          return 0;
 917                        }
 918 <St_TitleQ>\n          { /* new line => end of title */
 919                          unput(*yytext);
 920                          return 0;
 921                        }
 922 <St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
 923                          g_token->name = yytext;
 924                          g_token->name = g_token->name.left(
 925                                g_token->name.find('=')).stripWhiteSpace();
 926                          BEGIN(St_TitleV);
 927                        }
 928 <St_TitleV>[^ \t\r\n]+ { // attribute value
 929                          g_token->chars = yytext;
 930                          BEGIN(St_TitleN);
 931                          return TK_WORD;
 932                        }
 933 <St_TitleV,St_TitleA>. {
 934                          unput(*yytext);
 935                          return 0;
 936                        }
 937 <St_TitleV,St_TitleA>\n  {
 938                          return 0;
 939                        }
 940
 941 <St_Anchor>{LABELID}{WS}? { // anchor
 942                          g_token->name = QCString(yytext).stripWhiteSpace();
 943                          return TK_WORD;
 944                        }
 945 <St_Anchor>.           {
 946                          unput(*yytext);
 947                          return 0;
 948                        }
 949 <St_Cite>{CITEID}      { // label to cite
 950                          g_token->name=yytext;
 951                          return TK_WORD;
 952                        }
 953 <St_Cite>{BLANK}       { // white space
 954                          unput(' ');
 955                          return 0;
 956                        }
 957 <St_Cite>\n            { // new line
 958                          unput(*yytext);
 959                          return 0;
 960                        }
 961 <St_Cite>.             { // any other character
 962                          unput(*yytext);
 963                          return 0;
 964                        }
 965 <St_Ref>{REFWORD}      { // label to refer to
 966                          g_token->name=yytext;
 967                          return TK_WORD;
 968                        }
 969 <St_Ref>{BLANK}        { // white space
 970                          unput(' ');
 971                          return 0;
 972                        }
 973 <St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
 974                          BEGIN(St_Ref2);
 975                        }
 976 <St_Ref>\n             { // new line
 977                          unput(*yytext);
 978                          return 0;
 979                        }
 980 <St_Ref>.              { // any other character
 981                          unput(*yytext);
 982                          return 0;
 983                        }
 984 <St_IntRef>[A-Z_a-z0-9.:/#\-\+\(\)]+ {
 985                          g_token->name = yytext;
 986                          return TK_WORD;
 987                        }
 988 <St_IntRef>{BLANK}+"\"" {
 989                          BEGIN(St_Ref2);
 990                        }
 991 <St_Ref2>"&"{ID}";"    { /* symbol */
 992                          g_token->name = yytext;
 993                          return TK_SYMBOL;
 994                        }
 995 <St_Ref2>{SPCMD1}      |
 996 <St_Ref2>{SPCMD2}      { /* special command */
 997                          g_token->name = yytext+1;
 998                          g_token->paramDir=TokenInfo::Unspecified;
 999                          return TK_COMMAND;
1000                        }
1001 <St_Ref2>{WORD1NQ}     |
1002 <St_Ref2>{WORD2NQ}     {
1003                          /* word */
1004                          g_token->name = yytext;
1005                          return TK_WORD;
1006                        }
1007 <St_Ref2>[ \t]+        {
1008                          g_token->chars=yytext;
1009                          return TK_WHITESPACE;
1010                        }
1011 <St_Ref2>"\""|\n       { /* " or \n => end of title */
1012                          return 0;
1013                        }
1014 <St_XRefItem>{LABELID} {
1015                          g_token->name=yytext;
1016                        }
1017 <St_XRefItem>" "       {
1018                          BEGIN(St_XRefItem2);
1019                        }
1020 <St_XRefItem2>[0-9]+"." {
1021                          QCString numStr=yytext;
1022                          numStr=numStr.left((int)yyleng-1);
1023                          g_token->id=numStr.toInt();
1024                          return RetVal_OK;
1025                        }
1026 <St_Para,St_Title,St_Ref2>"<!--"     { /* html style comment block */
1027                          g_commentState = YY_START;
1028                          BEGIN(St_Comment);
1029                        }
1030 <St_Param>"\""[^\n\"]+"\"" {
1031                          g_token->name = yytext+1;
1032                          g_token->name = g_token->name.left((int)yyleng-2);
1033                          return TK_WORD;
1034                        }
1035 <St_Param>({PHPTYPE}{BLANK}*"|"{BLANK}*)*{PHPTYPE}{WS}+("&")?"$"{LABELID} {
1036                          QCString params = yytext;
1037                          int j = params.find('&');
1038                          int i = params.find('$');
1039                          if (j<i && j!=-1) i=j;
1040                          QCString types = params.left(i).stripWhiteSpace();
1041                          g_token->name = types+"#"+params.mid(i);
1042                          return TK_WORD;
1043                        }
1044 <St_Param>[^ \t\n,@\\]+  {
1045                          g_token->name = yytext;
1046                          if (g_token->name.at(yyleng-1)==':')
1047                          {
1048                            g_token->name=g_token->name.left(yyleng-1);
1049                          }
1050                          return TK_WORD;
1051                        }
1052 <St_Param>{WS}*","{WS}*  /* param separator */
1053 <St_Param>{WS}         {
1054                          g_token->chars=yytext;
1055                          return TK_WHITESPACE;
1056                        }
1057 <St_File>{FILEMASK}    {
1058                          g_token->name = yytext;
1059                          return TK_WORD;
1060                        }
1061 <St_File>"\""[^\n\"]+"\"" {
1062                          QCString text=yytext;
1063                          g_token->name = text.mid(1,text.length()-2);
1064                          return TK_WORD;
1065                        }
1066 <St_Pattern>[^\r\n]+   {
1067                          g_token->name = yytext;
1068                          g_token->name = g_token->name.stripWhiteSpace();
1069                          return TK_WORD;
1070                        }
1071 <St_Link>{LINKMASK}|{REFWORD}    {
1072                          g_token->name = yytext;
1073                          return TK_WORD;
1074                        }
1075 <St_Comment>"-->"      { /* end of html comment */
1076                          BEGIN(g_commentState);
1077                        }
1078 <St_Comment>[^-\n]+       /* inside html comment */
1079 <St_Comment>.             /* inside html comment */
1080
1081      /* State for skipping title (all chars until the end of the line) */
1082
1083 <St_SkipTitle>.
1084 <St_SkipTitle>\n       { return 0; }
1085
1086      /* State for the pass used to find the anchors and sections */
1087
1088 <St_Sections>[^\n@\\]+
1089 <St_Sections>"@@"|"\\\\"
1090 <St_Sections>{CMD}"anchor"{BLANK}+  {
1091                                       g_secType = SectionInfo::Anchor;
1092                                       BEGIN(St_SecLabel1);
1093                                     }
1094 <St_Sections>{CMD}"section"{BLANK}+ {
1095                                       g_secType = SectionInfo::Section;
1096                                       BEGIN(St_SecLabel2);
1097                                     }
1098 <St_Sections>{CMD}"subsection"{BLANK}+ {
1099                                       g_secType = SectionInfo::Subsection;
1100                                       BEGIN(St_SecLabel2);
1101                                     }
1102 <St_Sections>{CMD}"subsubsection"{BLANK}+ {
1103                                       g_secType = SectionInfo::Subsubsection;
1104                                       BEGIN(St_SecLabel2);
1105                                     }
1106 <St_Sections>{CMD}"paragraph"{BLANK}+ {
1107                                       g_secType = SectionInfo::Paragraph;
1108                                       BEGIN(St_SecLabel2);
1109                                     }
1110 <St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9]  {
1111                                       g_endMarker="endverbatim";
1112                                       BEGIN(St_SecSkip);
1113                                     }
1114 <St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1115                                       g_endMarker="enddot";
1116                                       BEGIN(St_SecSkip);
1117                                     }
1118 <St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1119                                       g_endMarker="endmsc";
1120                                       BEGIN(St_SecSkip);
1121                                     }
1122 <St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1123                                       g_endMarker="endhtmlonly";
1124                                       BEGIN(St_SecSkip);
1125                                     }
1126 <St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1127                                       g_endMarker="endlatexonly";
1128                                       BEGIN(St_SecSkip);
1129                                     }
1130 <St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1131                                       g_endMarker="endxmlonly";
1132                                       BEGIN(St_SecSkip);
1133                                     }
1134 <St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
1135                                       g_endMarker="enddocbookonly";
1136                                       BEGIN(St_SecSkip);
1137                                     }
1138 <St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1139                                       g_endMarker="endcode";
1140                                       BEGIN(St_SecSkip);
1141                                     }
1142 <St_Sections>"<!--"                 {
1143                                       g_endMarker="-->";
1144                                       BEGIN(St_SecSkip);
1145                                     }
1146 <St_SecSkip>{CMD}{ID}               {
1147                                       if (qstrcmp(yytext+1,g_endMarker)==0)
1148                                       {
1149                                         BEGIN(St_Sections);
1150                                       }
1151                                     }
1152 <St_SecSkip>"-->"                   {
1153                                       if (qstrcmp(yytext,g_endMarker)==0)
1154                                       {
1155                                         BEGIN(St_Sections);
1156                                       }
1157                                     }
1158 <St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1159 <St_SecSkip>.
1160 <St_SecSkip>\n
1161 <St_Sections>.
1162 <St_Sections>\n
1163 <St_SecLabel1>{LABELID} {
1164                          g_secLabel = yytext;
1165                          processSection();
1166                          BEGIN(St_Sections);
1167                        }
1168 <St_SecLabel2>{LABELID}{BLANK}+ |
1169 <St_SecLabel2>{LABELID}         {
1170                          g_secLabel = yytext;
1171                          g_secLabel = g_secLabel.stripWhiteSpace();
1172                          BEGIN(St_SecTitle);
1173                        }
1174 <St_SecTitle>[^\n]+    |
1175 <St_SecTitle>[^\n]*\n  {
1176                          g_secTitle = yytext;
1177                          g_secTitle = g_secTitle.stripWhiteSpace();
1178                          processSection();
1179                          BEGIN(St_Sections);
1180                        }
1181 <St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1182                          warn(g_fileName,yylineno,"Unexpected character `%s' while looking for section label or title",yytext);
1183                        }
1184
1185 <St_Snippet>[^\n]+     |
1186 <St_Snippet>[^\n]*\n   {
1187                          g_token->name = yytext;
1188                          g_token->name = g_token->name.stripWhiteSpace();
1189                          return TK_WORD;
1190                        }
1191
1192      /* Generic rules that work for all states */
1193 <*>\n                  {
1194                          warn(g_fileName,yylineno,"Unexpected new line character");
1195                        }
1196 <*>[\\@<>&$#%~"=]      { /* unescaped special character */
1197                          //warn(g_fileName,yylineno,"Unexpected character `%s', assuming command \\%s was meant.",yytext,yytext);
1198                          g_token->name = yytext;
1199                          return TK_COMMAND;
1200                        }
1201 <*>.                   {
1202                          warn(g_fileName,yylineno,"Unexpected character `%s'",yytext);
1203                        }
1204 %%
1205
1206 //--------------------------------------------------------------------------
1207
1208 void doctokenizerYYFindSections(const char *input,Definition *d,
1209                                 MemberGroup *mg,const char *fileName)
1210 {
1211   if (input==0) return;
1212   printlex(yy_flex_debug, TRUE, __FILE__, fileName);
1213   g_inputString = input;
1214   //printf("parsing --->`%s'<---\n",input);
1215   g_inputPos    = 0;
1216   g_definition  = d;
1217   g_memberGroup = mg;
1218   g_fileName    = fileName;
1219   BEGIN(St_Sections);
1220   doctokenizerYYlineno = 1;
1221   doctokenizerYYlex();
1222   printlex(yy_flex_debug, FALSE, __FILE__, fileName);
1223 }
1224
1225 void doctokenizerYYinit(const char *input,const char *fileName)
1226 {
1227   g_autoListLevel = 0;
1228   g_inputString = input;
1229   g_inputPos    = 0;
1230   g_fileName    = fileName;
1231   g_insidePre   = FALSE;
1232   BEGIN(St_Para);
1233 }
1234
1235 void doctokenizerYYsetStatePara()
1236 {
1237   BEGIN(St_Para);
1238 }
1239
1240 void doctokenizerYYsetStateTitle()
1241 {
1242   BEGIN(St_Title);
1243 }
1244
1245 void doctokenizerYYsetStateTitleAttrValue()
1246 {
1247   BEGIN(St_TitleV);
1248 }
1249
1250 void doctokenizerYYsetStateCode()
1251 {
1252   g_token->verb="";
1253   g_token->name="";
1254   BEGIN(St_CodeOpt);
1255 }
1256
1257 void doctokenizerYYsetStateXmlCode()
1258 {
1259   g_token->verb="";
1260   g_token->name="";
1261   BEGIN(St_XmlCode);
1262 }
1263
1264 void doctokenizerYYsetStateHtmlOnly()
1265 {
1266   g_token->verb="";
1267   g_token->name="";
1268   BEGIN(St_HtmlOnlyOption);
1269 }
1270
1271 void doctokenizerYYsetStateManOnly()
1272 {
1273   g_token->verb="";
1274   BEGIN(St_ManOnly);
1275 }
1276
1277 void doctokenizerYYsetStateRtfOnly()
1278 {
1279   g_token->verb="";
1280   BEGIN(St_RtfOnly);
1281 }
1282
1283 void doctokenizerYYsetStateXmlOnly()
1284 {
1285   g_token->verb="";
1286   BEGIN(St_XmlOnly);
1287 }
1288
1289 void doctokenizerYYsetStateDbOnly()
1290 {
1291   g_token->verb="";
1292   BEGIN(St_DbOnly);
1293 }
1294
1295 void doctokenizerYYsetStateLatexOnly()
1296 {
1297   g_token->verb="";
1298   BEGIN(St_LatexOnly);
1299 }
1300
1301 void doctokenizerYYsetStateVerbatim()
1302 {
1303   g_token->verb="";
1304   BEGIN(St_Verbatim);
1305 }
1306
1307 void doctokenizerYYsetStateDot()
1308 {
1309   g_token->verb="";
1310   BEGIN(St_Dot);
1311 }
1312
1313 void doctokenizerYYsetStateMsc()
1314 {
1315   g_token->verb="";
1316   BEGIN(St_Msc);
1317 }
1318
1319 void doctokenizerYYsetStateParam()
1320 {
1321   BEGIN(St_Param);
1322 }
1323
1324 void doctokenizerYYsetStateXRefItem()
1325 {
1326   BEGIN(St_XRefItem);
1327 }
1328
1329 void doctokenizerYYsetStateFile()
1330 {
1331   BEGIN(St_File);
1332 }
1333
1334 void doctokenizerYYsetStatePattern()
1335 {
1336   BEGIN(St_Pattern);
1337 }
1338
1339 void doctokenizerYYsetStateLink()
1340 {
1341   BEGIN(St_Link);
1342 }
1343
1344 void doctokenizerYYsetStateCite()
1345 {
1346   BEGIN(St_Cite);
1347 }
1348
1349 void doctokenizerYYsetStateRef()
1350 {
1351   BEGIN(St_Ref);
1352 }
1353
1354 void doctokenizerYYsetStateInternalRef()
1355 {
1356   BEGIN(St_IntRef);
1357 }
1358
1359 void doctokenizerYYsetStateText()
1360 {
1361   BEGIN(St_Text);
1362 }
1363
1364 void doctokenizerYYsetStateSkipTitle()
1365 {
1366   BEGIN(St_SkipTitle);
1367 }
1368
1369 void doctokenizerYYsetStateAnchor()
1370 {
1371   BEGIN(St_Anchor);
1372 }
1373
1374 void doctokenizerYYsetStateSnippet()
1375 {
1376   BEGIN(St_Snippet);
1377 }
1378
1379 void doctokenizerYYcleanup()
1380 {
1381   yy_delete_buffer( YY_CURRENT_BUFFER );
1382 }
1383
1384 void doctokenizerYYsetInsidePre(bool b)
1385 {
1386   g_insidePre = b;
1387 }
1388
1389 void doctokenizerYYpushBackHtmlTag(const char *tag)
1390 {
1391   QCString tagName = tag;
1392   int i,l = tagName.length();
1393   unput('>');
1394   for (i=l-1;i>=0;i--)
1395   {
1396     unput(tag[i]);
1397   }
1398   unput('<');
1399 }
1400
1401 void doctokenizerYYstartAutoList()
1402 {
1403   g_autoListLevel++;
1404 }
1405
1406 void doctokenizerYYendAutoList()
1407 {
1408   g_autoListLevel--;
1409 }
1410
1411 REAL_YY_DECL
1412 {
1413   printlex(yy_flex_debug, TRUE, __FILE__, g_fileName);
1414   int retval = LOCAL_YY_DECL;
1415   printlex(yy_flex_debug, FALSE, __FILE__, g_fileName);
1416   return retval;
1417 }
1418 #if !defined(YY_FLEX_SUBMINOR_VERSION)
1419 extern "C" { // some bogus code to keep the compiler happy
1420     void doctokenizerYYdummy() { yy_flex_realloc(0,0); }
1421 }
1422 #endif
1423