src/doctokenizer.l

   1 /******************************************************************************
   2  *
   3  * $Id: $
   4  *
   5  *
   6  * Copyright (C) 1997-2015 by Dimitri van Heesch.
   7  *
   8  * Permission to use, copy, modify, and distribute this software and its
   9  * documentation under the terms of the GNU General Public License is hereby
  10  * granted. No representations are made about the suitability of this software
  11  * for any purpose. It is provided "as is" without express or implied warranty.
  12  * See the GNU General Public License for more details.
  13  *
  14  * Documents produced by Doxygen are derivative works derived from the
  15  * input used in their production; they are not affected by this license.
  16  *
  17  */
  18
  19 %option never-interactive
  20 %option prefix="doctokenizerYY"
  21
  22 %{
  23
  24 #include <ctype.h>
  25
  26 #include <qfile.h>
  27 #include <qstring.h>
  28 #include <qstack.h>
  29 #include <qdict.h>
  30 #include <qregexp.h>
  31
  32 #include "doctokenizer.h"
  33 #include "cmdmapper.h"
  34 #include "config.h"
  35 #include "message.h"
  36 #include "section.h"
  37 #include "membergroup.h"
  38 #include "definition.h"
  39 #include "doxygen.h"
  40 #include "portable.h"
  41
  42 #define YY_NO_INPUT 1
  43 #define YY_NO_UNISTD_H 1
  44
  45 //--------------------------------------------------------------------------
  46
  47 // context for tokenizer phase
  48 static int g_commentState;
  49 TokenInfo *g_token = 0;
  50 static int g_inputPos = 0;
  51 static const char *g_inputString;
  52 static QCString g_fileName;
  53 static bool g_insidePre;
  54 static int g_sharpCount=0;
  55
  56 // context for section finding phase
  57 static Definition  *g_definition;
  58 static MemberGroup *g_memberGroup;
  59 static QCString     g_secLabel;
  60 static QCString     g_secTitle;
  61 static SectionInfo::SectionType g_secType;
  62 static QCString     g_endMarker;
  63 static int          g_autoListLevel;
  64
  65 struct DocLexerContext
  66 {
  67   TokenInfo *token;
  68   int rule;
  69   int autoListLevel;
  70   int inputPos;
  71   const char *inputString;
  72   YY_BUFFER_STATE state;
  73 };
  74
  75 static QStack<DocLexerContext> g_lexerStack;
  76
  77 //--------------------------------------------------------------------------
  78
  79 void doctokenizerYYpushContext()
  80 {
  81   DocLexerContext *ctx = new DocLexerContext;
  82   ctx->rule = YY_START;
  83   ctx->autoListLevel = g_autoListLevel;
  84   ctx->token = g_token;
  85   ctx->inputPos = g_inputPos;
  86   ctx->inputString = g_inputString;
  87   ctx->state = YY_CURRENT_BUFFER;
  88   g_lexerStack.push(ctx);
  89   yy_switch_to_buffer(yy_create_buffer(doctokenizerYYin, YY_BUF_SIZE));
  90 }
  91
  92 bool doctokenizerYYpopContext()
  93 {
  94   if (g_lexerStack.isEmpty()) return FALSE;
  95   DocLexerContext *ctx = g_lexerStack.pop();
  96   g_autoListLevel = ctx->autoListLevel;
  97   g_inputPos = ctx->inputPos;
  98   g_inputString = ctx->inputString;
  99   yy_delete_buffer(YY_CURRENT_BUFFER);
 100   yy_switch_to_buffer(ctx->state);
 101   BEGIN(ctx->rule);
 102   delete ctx;
 103   return TRUE;
 104 }
 105
 106
 107 //--------------------------------------------------------------------------
 108
 109 const char *tokToString(int token)
 110 {
 111   switch (token)
 112   {
 113     case 0:              return "TK_EOF";
 114     case TK_WORD:        return "TK_WORD";
 115     case TK_LNKWORD:     return "TK_LNKWORD";
 116     case TK_WHITESPACE:  return "TK_WHITESPACE";
 117     case TK_LISTITEM:    return "TK_LISTITEM";
 118     case TK_ENDLIST:     return "TK_ENDLIST";
 119     case TK_COMMAND:     return "TK_COMMAND";
 120     case TK_HTMLTAG:     return "TK_HTMLTAG";
 121     case TK_SYMBOL:      return "TK_SYMBOL";
 122     case TK_NEWPARA:     return "TK_NEWPARA";
 123     case TK_RCSTAG:      return "TK_RCSTAG";
 124     case TK_URL:         return "TK_URL";
 125   }
 126   return "ERROR";
 127 }
 128
 129 static int computeIndent(const char *str,int length)
 130 {
 131   int i;
 132   int indent=0;
 133   static int tabSize=Config_getInt(TAB_SIZE);
 134   for (i=0;i<length;i++)
 135   {
 136     if (str[i]=='\t')
 137     {
 138       indent+=tabSize - (indent%tabSize);
 139     }
 140     else if (str[i]=='\n')
 141     {
 142       indent=0;
 143     }
 144     else
 145     {
 146       indent++;
 147     }
 148   }
 149   return indent;
 150 }
 151
 152 //--------------------------------------------------------------------------
 153
 154 static void processSection()
 155 {
 156   //printf("%s: found section/anchor with name `%s'\n",g_fileName.data(),g_secLabel.data());
 157   QCString file;
 158   if (g_memberGroup)
 159   {
 160     file = g_memberGroup->parent()->getOutputFileBase();
 161   }
 162   else if (g_definition)
 163   {
 164     file = g_definition->getOutputFileBase();
 165   }
 166   else
 167   {
 168     warn(g_fileName,yylineno,"Found section/anchor %s without context\n",g_secLabel.data());
 169   }
 170   SectionInfo *si=0;
 171   if ((si=Doxygen::sectionDict->find(g_secLabel)))
 172   {
 173     si->fileName = file;
 174     si->type     = g_secType;
 175   }
 176 }
 177
 178 static void handleHtmlTag()
 179 {
 180   QCString tagText=yytext;
 181   g_token->attribs.clear();
 182   g_token->endTag = FALSE;
 183   g_token->emptyTag = FALSE;
 184
 185   // Check for end tag
 186   int startNamePos=1;
 187   if (tagText.at(1)=='/')
 188   {
 189     g_token->endTag = TRUE;
 190     startNamePos++;
 191   }
 192
 193   // Parse the name portion
 194   int i = startNamePos;
 195   for (i=startNamePos; i < (int)yyleng; i++)
 196   {
 197     // Check for valid HTML/XML name chars (including namespaces)
 198     char c = tagText.at(i);
 199     if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
 200   }
 201   g_token->name = tagText.mid(startNamePos,i-startNamePos);
 202
 203   // Parse the attributes. Each attribute is a name, value pair
 204   // The result is stored in g_token->attribs.
 205   int startName,endName,startAttrib,endAttrib;
 206   while (i<(int)yyleng)
 207   {
 208     char c=tagText.at(i);
 209     // skip spaces
 210     while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 211     // check for end of the tag
 212     if (c == '>') break;
 213     // Check for XML style "empty" tag.
 214     if (c == '/')
 215     {
 216       g_token->emptyTag = TRUE;
 217       break;
 218     }
 219     startName=i;
 220     // search for end of name
 221     while (i<(int)yyleng && !isspace(c) && c!='=') { c=tagText.at(++i); }
 222     endName=i;
 223     HtmlAttrib opt;
 224     opt.name  = tagText.mid(startName,endName-startName).lower();
 225     // skip spaces
 226     while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 227     if (tagText.at(i)=='=') // option has value
 228     {
 229       c=tagText.at(++i);
 230       // skip spaces
 231       while (i<(int)yyleng && isspace(c)) { c=tagText.at(++i); }
 232       if (tagText.at(i)=='\'') // option '...'
 233       {
 234         c=tagText.at(++i);
 235         startAttrib=i;
 236
 237         // search for matching quote
 238         while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
 239         endAttrib=i;
 240         if (i<(int)yyleng) c=tagText.at(++i);
 241       }
 242       else if (tagText.at(i)=='"') // option "..."
 243       {
 244         c=tagText.at(++i);
 245         startAttrib=i;
 246         // search for matching quote
 247         while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
 248         endAttrib=i;
 249         if (i<(int)yyleng) c=tagText.at(++i);
 250       }
 251       else // value without any quotes
 252       {
 253         startAttrib=i;
 254         // search for separator or end symbol
 255         while (i<(int)yyleng && !isspace(c) && c!='>') { c=tagText.at(++i); }
 256         endAttrib=i;
 257         if (i<(int)yyleng) c=tagText.at(++i);
 258       }
 259       opt.value  = tagText.mid(startAttrib,endAttrib-startAttrib);
 260     }
 261     else // start next option
 262     {
 263     }
 264     //printf("=====> Adding option name=<%s> value=<%s>\n",
 265     //    opt.name.data(),opt.value.data());
 266     g_token->attribs.append(&opt);
 267   }
 268 }
 269
 270 static QCString stripEmptyLines(const QCString &s)
 271 {
 272   if (s.isEmpty()) return QCString();
 273   int end=s.length();
 274   int start=0,p=0;
 275   // skip leading empty lines
 276   for (;;)
 277   {
 278     int c;
 279     while ((c=s[p]) && (c==' ' || c=='\t')) p++;
 280     if (s[p]=='\n')
 281     {
 282       start=++p;
 283     }
 284     else
 285     {
 286       break;
 287     }
 288   }
 289   // skip trailing empty lines
 290   p=end-1;
 291   if (p>=start && s.at(p)=='\n') p--;
 292   while (p>=start)
 293   {
 294     int c;
 295     while ((c=s[p]) && (c==' ' || c=='\t')) p--;
 296     if (s[p]=='\n')
 297     {
 298       end=p;
 299     }
 300     else
 301     {
 302       break;
 303     }
 304     p--;
 305   }
 306   //printf("stripEmptyLines(%d-%d)\n",start,end);
 307   return s.mid(start,end-start);
 308 }
 309
 310 //--------------------------------------------------------------------------
 311
 312 #undef  YY_INPUT
 313 #define YY_INPUT(buf,result,max_size) result=yyread(buf,max_size);
 314
 315 static int yyread(char *buf,int max_size)
 316 {
 317   int c=0;
 318   const char *src=g_inputString+g_inputPos;
 319   while ( c < max_size && *src ) *buf++ = *src++, c++;
 320   g_inputPos+=c;
 321   return c;
 322 }
 323
 324 //--------------------------------------------------------------------------
 325 //#define REAL_YY_DECL int doctokenizerYYlex (void)
 326 //#define YY_DECL static int local_doctokinizer(void)
 327 //#define LOCAL_YY_DECL local_doctokinizer()
 328
 329 %}
 330
 331 CMD       ("\\"|"@")
 332 WS        [ \t\r\n]
 333 NONWS     [^ \t\r\n]
 334 BLANK     [ \t\r]
 335 ID        "$"?[a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]*
 336 LABELID   [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-]*
 337 PHPTYPE   [\\:a-z_A-Z0-9\x80-\xFF\-]+
 338 CITESCHAR [a-z_A-Z0-9\x80-\xFF]
 339 CITEECHAR [a-z_A-Z0-9\x80-\xFF\-\+:\/]*
 340 CITEID    {CITESCHAR}{CITEECHAR}*("."{CITESCHAR}{CITEECHAR}*)*
 341 MAILADR   ("mailto:")?[a-z_A-Z0-9.+-]+"@"[a-z_A-Z0-9-]+("."[a-z_A-Z0-9\-]+)+[a-z_A-Z0-9\-]+
 342 OPTSTARS  ("//"{BLANK}*)?"*"*{BLANK}*
 343 LISTITEM  {BLANK}*[-]("#")?{WS}
 344 MLISTITEM {BLANK}*[+*]{WS}
 345 OLISTITEM {BLANK}*[1-9][0-9]*"."{BLANK}
 346 ENDLIST   {BLANK}*"."{BLANK}*\n
 347 ATTRNAME  [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-]*
 348 ATTRIB    {ATTRNAME}{WS}*("="{WS}*(("\""[^\"]*"\"")|("'"[^\']*"'")|[^ \t\r\n'"><]+))?
 349 URLCHAR   [a-z_A-Z0-9\!\~\,\:\;\'\$\?\@\&\%\#\.\-\+\/\=]
 350 URLMASK   ({URLCHAR}+([({]{URLCHAR}*[)}])?)+
 351 FILESCHAR [a-z_A-Z0-9\\:\\\/\-\+@&#]
 352 FILEECHAR [a-z_A-Z0-9\-\+@&#]
 353 HFILEMASK ("."{FILESCHAR}*{FILEECHAR}+)*
 354 FILEMASK  ({FILESCHAR}*{FILEECHAR}+("."{FILESCHAR}*{FILEECHAR}+)*)|{HFILEMASK}
 355 LINKMASK  [^ \t\n\r\\@<&${}]+("("[^\n)]*")")?({BLANK}*("const"|"volatile"){BLANK}+)?
 356 VERBATIM  "verbatim"{BLANK}*
 357 SPCMD1    {CMD}([a-z_A-Z][a-z_A-Z0-9]*|{VERBATIM}|"--"|"---")
 358 SPCMD2    {CMD}[\\@<>&$#%~".+|-]
 359 SPCMD3    {CMD}form#[0-9]+
 360 SPCMD4    {CMD}"::"
 361 INOUT     "inout"|"in"|"out"|("in"{BLANK}*","{BLANK}*"out")|("out"{BLANK}*","{BLANK}*"in")
 362 PARAMIO   {CMD}param{BLANK}*"["{BLANK}*{INOUT}{BLANK}*"]"
 363 TEMPCHAR  [a-z_A-Z0-9.,: \t\*\&\(\)\[\]]
 364 FUNCCHAR  [a-z_A-Z0-9,:\<\> \t\^\*\&\[\]]
 365 FUNCPART  {FUNCCHAR}*("("{FUNCCHAR}*")"{FUNCCHAR}*)?
 366 SCOPESEP  "::"|"#"|"."
 367 TEMPLPART "<"{TEMPCHAR}*">"
 368 ANONNS    "anonymous_namespace{"[^}]*"}"
 369 SCOPEPRE  (({ID}{TEMPLPART}?)|{ANONNS}){SCOPESEP}
 370 SCOPEKEYS ":"({ID}":")*
 371 SCOPECPP  {SCOPEPRE}*(~)?{ID}{TEMPLPART}?
 372 SCOPEOBJC {SCOPEPRE}?{ID}{SCOPEKEYS}?
 373 SCOPEMASK {SCOPECPP}|{SCOPEOBJC}
 374 FUNCARG   "("{FUNCPART}")"({BLANK}*("volatile"|"const"){BLANK})?
 375 FUNCARG2  "("{FUNCPART}")"({BLANK}*("volatile"|"const"))?
 376 OPNEW     {BLANK}+"new"({BLANK}*"[]")?
 377 OPDEL     {BLANK}+"delete"({BLANK}*"[]")?
 378 OPNORM    {OPNEW}|{OPDEL}|"+"|"-"|"*"|"/"|"%"|"^"|"&"|"|"|"~"|"!"|"="|"<"|">"|"+="|"-="|"*="|"/="|"%="|"^="|"&="|"|="|"<<"|">>"|"<<="|">>="|"=="|"!="|"<="|">="|"&&"|"||"|"++"|"--"|","|"->*"|"->"|"[]"|"()"
 379 OPCAST    {BLANK}+[^<(\r\n.,][^(\r\n.,]*
 380 OPMASK    ({BLANK}*{OPNORM}{FUNCARG})
 381 OPMASKOPT ({BLANK}*{OPNORM}{FUNCARG}?)|({OPCAST}{FUNCARG})
 382 OPMASKOP2 ({BLANK}*{OPNORM}{FUNCARG2}?)|({OPCAST}{FUNCARG2})
 383 LNKWORD1  ("::"|"#")?{SCOPEMASK}
 384 CVSPEC    {BLANK}*("const"|"volatile")
 385 LNKWORD2  (({SCOPEPRE}*"operator"{OPMASK})|({SCOPEPRE}"operator"{OPMASKOPT})|(("::"|"#"){SCOPEPRE}*"operator"{OPMASKOPT})){CVSPEC}?
 386 LNKWORD3  ([0-9a-z_A-Z\-]+("/"|"\\"))*[0-9a-z_A-Z\-]+("."[0-9a-z_A-Z]+)+
 387 CHARWORDQ [^ \t\n\r\\@<>()\[\]:;\?{}&%$#,."=']
 388 ESCWORD   ("%"{ID}(("::"|"."){ID})*)|("%'")
 389 WORD1     {ESCWORD}|{CHARWORDQ}+|"{"|"}"|"'\"'"|("\""[^"\n]*\n?[^"\n]*"\"")
 390 WORD2     "."|","|"("|")"|"["|"]"|":"|";"|"\?"|"="|"'"
 391 WORD1NQ   {ESCWORD}|{CHARWORDQ}+|"{"|"}"
 392 WORD2NQ   "."|","|"("|")"|"["|"]"|":"|";"|"\?"|"="|"'"
 393 CAPTION   [cC][aA][pP][tT][iI][oO][nN]
 394 HTMLTAG   "<"(("/")?){ID}({WS}+{ATTRIB})*{WS}*(("/")?)">"
 395 HTMLKEYL  "strong"|"center"|"table"|"caption"|"small"|"code"|"dfn"|"var"|"img"|"pre"|"sub"|"sup"|"tr"|"td"|"th"|"ol"|"ul"|"li"|"tt"|"kbd"|"em"|"hr"|"dl"|"dt"|"dd"|"br"|"i"|"a"|"b"|"p"
 396 HTMLKEYU  "STRONG"|"CENTER"|"TABLE"|"CAPTION"|"SMALL"|"CODE"|"DFN"|"VAR"|"IMG"|"PRE"|"SUB"|"SUP"|"TR"|"TD"|"TH"|"OL"|"UL"|"LI"|"TT"|"KBD"|"EM"|"HR"|"DL"|"DT"|"DD"|"BR"|"I"|"A"|"B"|"P"
 397 HTMLKEYW  {HTMLKEYL}|{HTMLKEYU}
 398 REFWORD2_PRE   ("#"|"::")?((({ID}{TEMPLPART}?)|{ANONNS})("."|"#"|"::"|"-"|"/"))*({ID}{TEMPLPART}?(":")?)
 399 REFWORD2       {REFWORD2_PRE}{FUNCARG2}?
 400 REFWORD2_NOCV  {REFWORD2_PRE}("("{FUNCPART}")")?
 401 REFWORD3       ({ID}":")*{ID}":"?
 402 REFWORD4_NOCV  (({SCOPEPRE}*"operator"{OPMASKOP2})|(("::"|"#"){SCOPEPRE}*"operator"{OPMASKOP2}))
 403 REFWORD4       {REFWORD4_NOCV}{CVSPEC}?
 404 REFWORD        {LABELID}|{REFWORD2}|{REFWORD3}|{REFWORD4}
 405 REFWORD_NOCV   {LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 406
 407 %option noyywrap
 408 %option yylineno
 409
 410 %x St_Para
 411 %x St_Comment
 412 %x St_Title
 413 %x St_TitleN
 414 %x St_TitleQ
 415 %x St_TitleA
 416 %x St_TitleV
 417 %x St_Code
 418 %x St_CodeOpt
 419 %x St_XmlCode
 420 %x St_HtmlOnly
 421 %x St_HtmlOnlyOption
 422 %x St_ManOnly
 423 %x St_LatexOnly
 424 %x St_RtfOnly
 425 %x St_XmlOnly
 426 %x St_DbOnly
 427 %x St_Verbatim
 428 %x St_Dot
 429 %x St_Msc
 430 %x St_PlantUMLOpt
 431 %x St_PlantUML
 432 %x St_Param
 433 %x St_XRefItem
 434 %x St_XRefItem2
 435 %x St_File
 436 %x St_Pattern
 437 %x St_Link
 438 %x St_Cite
 439 %x St_Ref
 440 %x St_Ref2
 441 %x St_IntRef
 442 %x St_Text
 443 %x St_SkipTitle
 444 %x St_Anchor
 445 %x St_Snippet
 446 %x St_SetScope
 447 %x St_SetScopeEnd
 448
 449 %x St_Sections
 450 %s St_SecLabel1
 451 %s St_SecLabel2
 452 %s St_SecTitle
 453 %x St_SecSkip
 454
 455 %%
 456 <St_Para>\r            /* skip carriage return */
 457 <St_Para>^{LISTITEM}   { /* list item */
 458                          QCString text=yytext;
 459                          int dashPos = text.findRev('-');
 460                          g_token->isEnumList = text.at(dashPos+1)=='#';
 461                          g_token->id         = -1;
 462                          g_token->indent     = computeIndent(yytext,dashPos);
 463                          return TK_LISTITEM;
 464                        }
 465 <St_Para>^{MLISTITEM}  { /* list item */
 466                          if (!Doxygen::markdownSupport || g_insidePre)
 467                          {
 468                            REJECT;
 469                          }
 470                          else
 471                          {
 472                            QCString text=yytext;
 473                            static QRegExp re("[*+]");
 474                            int listPos = text.findRev(re);
 475                            g_token->isEnumList = FALSE;
 476                            g_token->id         = -1;
 477                            g_token->indent     = computeIndent(yytext,listPos);
 478                            return TK_LISTITEM;
 479                          }
 480                        }
 481 <St_Para>^{OLISTITEM}  { /* numbered list item */
 482                          if (!Doxygen::markdownSupport || g_insidePre)
 483                          {
 484                            REJECT;
 485                          }
 486                          else
 487                          {
 488                            QCString text=yytext;
 489                            static QRegExp re("[1-9]");
 490                            int digitPos = text.find(re);
 491                            int dotPos = text.find('.',digitPos);
 492                            g_token->isEnumList = TRUE;
 493                            g_token->id         = atoi(QCString(yytext).mid(digitPos,dotPos-digitPos));
 494                            g_token->indent     = computeIndent(yytext,digitPos);
 495                            return TK_LISTITEM;
 496                          }
 497                        }
 498 <St_Para>{BLANK}*\n{LISTITEM}     { /* list item on next line */
 499                          QCString text=yytext;
 500                          text=text.right(text.length()-text.find('\n')-1);
 501                          int dashPos = text.findRev('-');
 502                          g_token->isEnumList = text.at(dashPos+1)=='#';
 503                          g_token->id         = -1;
 504                          g_token->indent     = computeIndent(text,dashPos);
 505                          return TK_LISTITEM;
 506                        }
 507 <St_Para>{BLANK}*\n{MLISTITEM}     { /* list item on next line */
 508                          if (!Doxygen::markdownSupport || g_insidePre)
 509                          {
 510                            REJECT;
 511                          }
 512                          else
 513                          {
 514                            QCString text=yytext;
 515                            static QRegExp re("[*+]");
 516                            text=text.right(text.length()-text.find('\n')-1);
 517                            int markPos = text.findRev(re);
 518                            g_token->isEnumList = FALSE;
 519                            g_token->id         = -1;
 520                            g_token->indent     = computeIndent(text,markPos);
 521                            return TK_LISTITEM;
 522                          }
 523                        }
 524 <St_Para>{BLANK}*\n{OLISTITEM}     { /* list item on next line */
 525                          if (!Doxygen::markdownSupport || g_insidePre)
 526                          {
 527                            REJECT;
 528                          }
 529                          else
 530                          {
 531                            QCString text=yytext;
 532                            int nl=text.findRev('\n');
 533                            int len=text.length();
 534                            text=text.right(len-nl-1);
 535                            static QRegExp re("[1-9]");
 536                            int digitPos = text.find(re);
 537                            int dotPos = text.find('.',digitPos);
 538                            g_token->isEnumList = TRUE;
 539                            g_token->id         = atoi(QCString(text).mid(digitPos,dotPos-digitPos));
 540                            g_token->indent     = computeIndent(text,digitPos);
 541                            return TK_LISTITEM;
 542                          }
 543                        }
 544 <St_Para>^{ENDLIST}       { /* end list */
 545                          int dotPos = QCString(yytext).findRev('.');
 546                          g_token->indent     = computeIndent(yytext,dotPos);
 547                          return TK_ENDLIST;
 548                        }
 549 <St_Para>{BLANK}*\n{ENDLIST}      { /* end list on next line */
 550                          QCString text=yytext;
 551                          text=text.right(text.length()-text.find('\n')-1);
 552                          int dotPos = text.findRev('.');
 553                          g_token->indent     = computeIndent(text,dotPos);
 554                          return TK_ENDLIST;
 555                        }
 556 <St_Para>"{"{BLANK}*"@link" {
 557                          g_token->name = "javalink";
 558                          return TK_COMMAND;
 559                        }
 560 <St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
 561                          g_token->name = "inheritdoc";
 562                          return TK_COMMAND;
 563                        }
 564 <St_Para>"@_fakenl"     { // artificial new line
 565                              yylineno++;
 566                           }
 567 <St_Para>{SPCMD3}      {
 568                          g_token->name = "form";
 569                          bool ok;
 570                          g_token->id = QCString(yytext).right((int)yyleng-6).toInt(&ok);
 571                          ASSERT(ok);
 572                          return TK_COMMAND;
 573                        }
 574 <St_Para>{CMD}"n"\n    { /* \n followed by real newline */
 575                          yylineno++;
 576                          g_token->name = yytext+1;
 577                          g_token->name = g_token->name.stripWhiteSpace();
 578                          g_token->paramDir=TokenInfo::Unspecified;
 579                          return TK_COMMAND;
 580                        }
 581 <St_Para>{SPCMD1}      |
 582 <St_Para>{SPCMD2}      |
 583 <St_Para>{SPCMD4}      { /* special command */
 584                          g_token->name = yytext+1;
 585                          g_token->name = g_token->name.stripWhiteSpace();
 586                          g_token->paramDir=TokenInfo::Unspecified;
 587                          return TK_COMMAND;
 588                        }
 589 <St_Para>{PARAMIO}     { /* param [in,out] command */
 590                          g_token->name = "param";
 591                          QCString s(yytext);
 592                          bool isIn  = s.find("in")!=-1;
 593                          bool isOut = s.find("out")!=-1;
 594                          if (isIn)
 595                          {
 596                            if (isOut)
 597                            {
 598                              g_token->paramDir=TokenInfo::InOut;
 599                            }
 600                            else
 601                            {
 602                              g_token->paramDir=TokenInfo::In;
 603                            }
 604                          }
 605                          else if (isOut)
 606                          {
 607                            g_token->paramDir=TokenInfo::Out;
 608                          }
 609                          else
 610                          {
 611                            g_token->paramDir=TokenInfo::Unspecified;
 612                          }
 613                          return TK_COMMAND;
 614                        }
 615 <St_Para>("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK}/\. { // URL.
 616                          g_token->name=yytext;
 617                          g_token->isEMailAddr=FALSE;
 618                          return TK_URL;
 619                        }
 620 <St_Para>("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK} { // URL
 621                          g_token->name=yytext;
 622                          g_token->isEMailAddr=FALSE;
 623                          return TK_URL;
 624                        }
 625 <St_Para>"<"("http:"|"https:"|"ftp:"|"file:"|"news:"){URLMASK}">" { // URL
 626                          g_token->name=yytext;
 627                          g_token->name = g_token->name.mid(1,g_token->name.length()-2);
 628                          g_token->isEMailAddr=FALSE;
 629                          return TK_URL;
 630                        }
 631 <St_Para>{MAILADR}     { // Mail address
 632                          g_token->name=yytext;
 633                          g_token->name.stripPrefix("mailto:");
 634                          g_token->isEMailAddr=TRUE;
 635                          return TK_URL;
 636                        }
 637 <St_Para>"<"{MAILADR}">" { // Mail address
 638                          g_token->name=yytext;
 639                          g_token->name = g_token->name.mid(1,g_token->name.length()-2);
 640                          g_token->name.stripPrefix("mailto:");
 641                          g_token->isEMailAddr=TRUE;
 642                          return TK_URL;
 643                        }
 644 <St_Para>"$"{ID}":"[^\n$]+"$" { /* RCS tag */
 645                          QCString tagName(yytext+1);
 646                          int index=tagName.find(':');
 647                          g_token->name = tagName.left(index);
 648                          int text_begin = index+2;
 649                          int text_end = tagName.length()-1;
 650                          if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
 651                          {
 652                                  ++text_begin;
 653                                  if (tagName[text_end-1]=='#')
 654                                          --text_end;
 655                          }
 656                          g_token->text = tagName.mid(text_begin,text_end-text_begin);
 657                          return TK_RCSTAG;
 658                        }
 659 <St_Para,St_HtmlOnly>"$("{ID}")"   { /* environment variable */
 660                          QCString name = &yytext[2];
 661                          name = name.left(name.length()-1);
 662                          QCString value = portable_getenv(name);
 663                          for (int i=value.length()-1;i>=0;i--) unput(value.at(i));
 664                        }
 665 <St_Para>{HTMLTAG}     { /* html tag */
 666                          handleHtmlTag();
 667                          return TK_HTMLTAG;
 668                        }
 669 <St_Para,St_Text>"&"{ID}";" { /* special symbol */
 670                          g_token->name = yytext;
 671                          return TK_SYMBOL;
 672                        }
 673
 674   /********* patterns for linkable words ******************/
 675
 676 <St_Para>{ID}/"<"{HTMLKEYW}">" { /* this rule is to prevent opening html
 677                                   * tag to be recognized as a templated classes
 678                                   */
 679                          g_token->name = yytext;
 680                          return TK_LNKWORD;
 681                         }
 682 <St_Para>{LNKWORD1}/"<br>"           | // prevent <br> html tag to be parsed as template arguments
 683 <St_Para>{LNKWORD1}                  |
 684 <St_Para>{LNKWORD1}{FUNCARG}         |
 685 <St_Para>{LNKWORD2}                  |
 686 <St_Para>{LNKWORD3}    {
 687                          g_token->name = yytext;
 688                          return TK_LNKWORD;
 689                        }
 690 <St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
 691                          g_token->name = yytext;
 692                          g_token->name = g_token->name.left(g_token->name.length()-1);
 693                          unput(yytext[(int)yyleng-1]);
 694                          return TK_LNKWORD;
 695                        }
 696   /********* patterns for normal words ******************/
 697
 698 <St_Para,St_Text>{WORD1} |
 699 <St_Para,St_Text>{WORD2} { /* function call */
 700                          if (yytext[0]=='%') // strip % if present
 701                            g_token->name = &yytext[1];
 702                          else
 703                            g_token->name = yytext;
 704                          return TK_WORD;
 705
 706                          /* the following is dummy code to please the
 707                           * compiler, removing this results in a warning
 708                           * on my machine
 709                           */
 710                          goto find_rule;
 711                        }
 712 <St_Text>({ID}".")+{ID} {
 713                           g_token->name = yytext;
 714                           return TK_WORD;
 715                         }
 716 <St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
 717                                                           // avoid interpretation as "operator <"
 718                            g_token->name = yytext;
 719                            return TK_WORD;
 720                          }
 721
 722   /*******************************************************/
 723
 724 <St_Para,St_Text>{BLANK}+      |
 725 <St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
 726                          g_token->chars=yytext;
 727                          return TK_WHITESPACE;
 728                        }
 729 <St_Text>[\\@<>&$#%~]  {
 730                          g_token->name = yytext;
 731                          return TK_COMMAND;
 732                        }
 733 <St_Para>({BLANK}*\n)+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
 734                          if (g_insidePre || g_autoListLevel==0)
 735                          {
 736                            REJECT;
 737                          }
 738                        }
 739 <St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
 740                          if (!Doxygen::markdownSupport || g_insidePre || g_autoListLevel==0)
 741                          {
 742                            REJECT;
 743                          }
 744                        }
 745 <St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
 746                          if (!Doxygen::markdownSupport || g_insidePre || g_autoListLevel==0)
 747                          {
 748                            REJECT;
 749                          }
 750                        }
 751 <St_Para>({BLANK}*\n)+{BLANK}*\n{BLANK}* {
 752                          g_token->indent=computeIndent(yytext,(int)yyleng);
 753                          int i;
 754                          // put back the indentation (needed for list items)
 755                          for (i=0;i<g_token->indent;i++)
 756                          {
 757                            unput(' ');
 758                          }
 759                          // tell flex that after putting the last indent
 760                          // back we are at the beginning of the line
 761                          YY_CURRENT_BUFFER->yy_at_bol=1;
 762                          // start of a new paragraph
 763                          return TK_NEWPARA;
 764                        }
 765 <St_CodeOpt>{BLANK}*"{"(".")?{LABELID}"}" {
 766                          g_token->name = yytext;
 767                          int i=g_token->name.find('{'); /* } to keep vi happy */
 768                          g_token->name = g_token->name.mid(i+1,g_token->name.length()-i-2);
 769                          BEGIN(St_Code);
 770                        }
 771 <St_CodeOpt>\n         |
 772 <St_CodeOpt>.          {
 773                          unput(*yytext);
 774                          BEGIN(St_Code);
 775                        }
 776 <St_Code>{WS}*{CMD}"endcode" {
 777                          return RetVal_OK;
 778                        }
 779 <St_XmlCode>{WS}*"</code>" {
 780                          return RetVal_OK;
 781                        }
 782 <St_Code,St_XmlCode>[^\\@\n<]+  |
 783 <St_Code,St_XmlCode>\n          |
 784 <St_Code,St_XmlCode>.           {
 785                          g_token->verb+=yytext;
 786                        }
 787 <St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
 788                          g_token->name="block";
 789                          BEGIN(St_HtmlOnly);
 790                         }
 791 <St_HtmlOnlyOption>.|\n {
 792                          unput(*yytext);
 793                          BEGIN(St_HtmlOnly);
 794                         }
 795 <St_HtmlOnly>{CMD}"endhtmlonly" {
 796                          return RetVal_OK;
 797                        }
 798 <St_HtmlOnly>[^\\@\n$]+    |
 799 <St_HtmlOnly>\n            |
 800 <St_HtmlOnly>.             {
 801                          g_token->verb+=yytext;
 802                        }
 803 <St_ManOnly>{CMD}"endmanonly" {
 804                          return RetVal_OK;
 805                        }
 806 <St_ManOnly>[^\\@\n$]+    |
 807 <St_ManOnly>\n            |
 808 <St_ManOnly>.             {
 809                          g_token->verb+=yytext;
 810                        }
 811 <St_RtfOnly>{CMD}"endrtfonly" {
 812                          return RetVal_OK;
 813                        }
 814 <St_RtfOnly>[^\\@\n$]+    |
 815 <St_RtfOnly>\n            |
 816 <St_RtfOnly>.             {
 817                          g_token->verb+=yytext;
 818                        }
 819 <St_LatexOnly>{CMD}"endlatexonly" {
 820                          return RetVal_OK;
 821                        }
 822 <St_LatexOnly>[^\\@\n]+     |
 823 <St_LatexOnly>\n            |
 824 <St_LatexOnly>.             {
 825                          g_token->verb+=yytext;
 826                        }
 827 <St_XmlOnly>{CMD}"endxmlonly" {
 828                          return RetVal_OK;
 829                        }
 830 <St_XmlOnly>[^\\@\n]+  |
 831 <St_XmlOnly>\n         |
 832 <St_XmlOnly>.          {
 833                          g_token->verb+=yytext;
 834                        }
 835 <St_DbOnly>{CMD}"enddocbookonly" {
 836                          return RetVal_OK;
 837                        }
 838 <St_DbOnly>[^\\@\n]+  |
 839 <St_DbOnly>\n         |
 840 <St_DbOnly>.          {
 841                          g_token->verb+=yytext;
 842                        }
 843 <St_Verbatim>{CMD}"endverbatim" {
 844                          g_token->verb=stripEmptyLines(g_token->verb);
 845                          return RetVal_OK;
 846                        }
 847 <St_Verbatim>[^\\@\n]+ |
 848 <St_Verbatim>\n        |
 849 <St_Verbatim>.         { /* Verbatim text */
 850                          g_token->verb+=yytext;
 851                        }
 852 <St_Dot>{CMD}"enddot"  {
 853                          return RetVal_OK;
 854                        }
 855 <St_Dot>[^\\@\n]+      |
 856 <St_Dot>\n             |
 857 <St_Dot>.              { /* dot text */
 858                          g_token->verb+=yytext;
 859                        }
 860 <St_Msc>{CMD}("endmsc")  {
 861                          return RetVal_OK;
 862                        }
 863 <St_Msc>[^\\@\n]+      |
 864 <St_Msc>\n             |
 865 <St_Msc>.              { /* msc text */
 866                          g_token->verb+=yytext;
 867                        }
 868 <St_PlantUMLOpt>{BLANK}*"{"[^}]*"}" { // case 1: file name is specified as {filename}
 869                          g_token->sectionId = QCString(yytext).stripWhiteSpace();
 870                          // skip curly brackets around the optional image name
 871                          g_token->sectionId = g_token->sectionId.mid(1,g_token->sectionId.length()-2).stripWhiteSpace();
 872                          return RetVal_OK;
 873                        }
 874 <St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
 875                          g_token->sectionId = QCString(yytext).stripWhiteSpace();
 876                          return RetVal_OK;
 877                        }
 878 <St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
 879                          g_token->sectionId = QCString(yytext).stripWhiteSpace();
 880                          return RetVal_OK;
 881                        }
 882 <St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}*/\n { // case 4: plain file name specified without title or attributes
 883                          g_token->sectionId = QCString(yytext).stripWhiteSpace();
 884                          return RetVal_OK;
 885                        }
 886 <St_PlantUMLOpt>.      {
 887                          g_token->sectionId = "";
 888                          unput(*yytext);
 889                          return RetVal_OK;
 890                        }
 891 <St_PlantUML>{CMD}"enduml"  {
 892                          return RetVal_OK;
 893                        }
 894 <St_PlantUML>[^\\@\n]+ |
 895 <St_PlantUML>\n        |
 896 <St_PlantUML>.         { /* plantuml text */
 897                          g_token->verb+=yytext;
 898                        }
 899 <St_Title>"\""         { // quoted title
 900                          BEGIN(St_TitleQ);
 901                        }
 902 <St_Title>[ \t]+       {
 903                          g_token->chars=yytext;
 904                          return TK_WHITESPACE;
 905                        }
 906 <St_Title>.            { // non-quoted title
 907                          unput(*yytext);
 908                          BEGIN(St_TitleN);
 909                        }
 910 <St_Title>\n           {
 911                          unput(*yytext);
 912                          return 0;
 913                        }
 914 <St_TitleN>"&"{ID}";"  { /* symbol */
 915                          g_token->name = yytext;
 916                          return TK_SYMBOL;
 917                        }
 918 <St_TitleN>{HTMLTAG}   {
 919                        }
 920 <St_TitleN>{SPCMD1}    |
 921 <St_TitleN>{SPCMD2}    { /* special command */
 922                          g_token->name = yytext+1;
 923                          g_token->paramDir=TokenInfo::Unspecified;
 924                          return TK_COMMAND;
 925                        }
 926 <St_TitleN>{ID}"="     { /* attribute */
 927                          if (yytext[0]=='%') // strip % if present
 928                            g_token->name = &yytext[1];
 929                          else
 930                            g_token->name = yytext;
 931                          return TK_WORD;
 932                        }
 933 <St_TitleN>{WORD1}     |
 934 <St_TitleN>{WORD2}     { /* word */
 935                          if (yytext[0]=='%') // strip % if present
 936                            g_token->name = &yytext[1];
 937                          else
 938                            g_token->name = yytext;
 939                          return TK_WORD;
 940                        }
 941 <St_TitleN>[ \t]+      {
 942                          g_token->chars=yytext;
 943                          return TK_WHITESPACE;
 944                        }
 945 <St_TitleN>\n          { /* new line => end of title */
 946                          unput(*yytext);
 947                          return 0;
 948                        }
 949 <St_TitleQ>"&"{ID}";"  { /* symbol */
 950                          g_token->name = yytext;
 951                          return TK_SYMBOL;
 952                        }
 953 <St_TitleQ>{SPCMD1}    |
 954 <St_TitleQ>{SPCMD2}    { /* special command */
 955                          g_token->name = yytext+1;
 956                          g_token->paramDir=TokenInfo::Unspecified;
 957                          return TK_COMMAND;
 958                        }
 959 <St_TitleQ>{WORD1NQ}   |
 960 <St_TitleQ>{WORD2NQ}   { /* word */
 961                          g_token->name = yytext;
 962                          return TK_WORD;
 963                        }
 964 <St_TitleQ>[ \t]+      {
 965                          g_token->chars=yytext;
 966                          return TK_WHITESPACE;
 967                        }
 968 <St_TitleQ>"\""        { /* closing quote => end of title */
 969                          BEGIN(St_TitleA);
 970                          return 0;
 971                        }
 972 <St_TitleQ>\n          { /* new line => end of title */
 973                          unput(*yytext);
 974                          return 0;
 975                        }
 976 <St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
 977                          g_token->name = yytext;
 978                          g_token->name = g_token->name.left(g_token->name.find('=')).stripWhiteSpace();
 979                          BEGIN(St_TitleV);
 980                        }
 981 <St_TitleV>[^ \t\r\n]+ { // attribute value
 982                          g_token->chars = yytext;
 983                          BEGIN(St_TitleN);
 984                          return TK_WORD;
 985                        }
 986 <St_TitleV,St_TitleA>. {
 987                          unput(*yytext);
 988                          return 0;
 989                        }
 990 <St_TitleV,St_TitleA>\n  {
 991                          return 0;
 992                        }
 993
 994 <St_Anchor>{LABELID}{WS}? { // anchor
 995                          g_token->name = QCString(yytext).stripWhiteSpace();
 996                          return TK_WORD;
 997                        }
 998 <St_Anchor>.           {
 999                          unput(*yytext);
1000                          return 0;
1001                        }
1002 <St_Cite>{CITEID}      { // label to cite
1003                          g_token->name=yytext;
1004                          return TK_WORD;
1005                        }
1006 <St_Cite>{BLANK}       { // white space
1007                          unput(' ');
1008                          return 0;
1009                        }
1010 <St_Cite>\n            { // new line
1011                          unput(*yytext);
1012                          return 0;
1013                        }
1014 <St_Cite>.             { // any other character
1015                          unput(*yytext);
1016                          return 0;
1017                        }
1018 <St_Ref>{REFWORD_NOCV}/{BLANK}("const")[a-z_A-Z0-9] { // see bug776988
1019                          g_token->name=yytext;
1020                          return TK_WORD;
1021                        }
1022 <St_Ref>{REFWORD_NOCV}/{BLANK}("volatile")[a-z_A-Z0-9] { // see bug776988
1023                          g_token->name=yytext;
1024                          return TK_WORD;
1025                        }
1026 <St_Ref>{REFWORD}      { // label to refer to
1027                          g_token->name=yytext;
1028                          return TK_WORD;
1029                        }
1030 <St_Ref>{BLANK}        { // white space
1031                          unput(' ');
1032                          return 0;
1033                        }
1034 <St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
1035                          BEGIN(St_Ref2);
1036                        }
1037 <St_Ref>\n             { // new line
1038                          unput(*yytext);
1039                          return 0;
1040                        }
1041 <St_Ref>.              { // any other character
1042                          unput(*yytext);
1043                          return 0;
1044                        }
1045 <St_IntRef>[A-Z_a-z0-9.:/#\-\+\(\)]+ {
1046                          g_token->name = yytext;
1047                          return TK_WORD;
1048                        }
1049 <St_IntRef>{BLANK}+"\"" {
1050                          BEGIN(St_Ref2);
1051                        }
1052 <St_SetScope>({SCOPEMASK}|{ANONNS}){BLANK} {
1053                          g_token->name = yytext;
1054                          g_token->name = g_token->name.stripWhiteSpace();
1055                          return TK_WORD;
1056                        }
1057 <St_SetScope>{SCOPEMASK}"<" {
1058                          g_token->name = yytext;
1059                          g_token->name = g_token->name.stripWhiteSpace();
1060                          g_sharpCount=1;
1061                          BEGIN(St_SetScopeEnd);
1062                        }
1063 <St_SetScope>{BLANK}   {
1064                        }
1065 <St_SetScopeEnd>"<"    {
1066                          g_token->name += yytext;
1067                          g_sharpCount++;
1068                        }
1069 <St_SetScopeEnd>">"    {
1070                          g_token->name += yytext;
1071                          g_sharpCount--;
1072                          if (g_sharpCount<=0)
1073                          {
1074                            return TK_WORD;
1075                          }
1076                        }
1077 <St_SetScopeEnd>.      {
1078                          g_token->name += yytext;
1079                        }
1080 <St_Ref2>"&"{ID}";"    { /* symbol */
1081                          g_token->name = yytext;
1082                          return TK_SYMBOL;
1083                        }
1084 <St_Ref2>{SPCMD1}      |
1085 <St_Ref2>{SPCMD2}      { /* special command */
1086                          g_token->name = yytext+1;
1087                          g_token->paramDir=TokenInfo::Unspecified;
1088                          return TK_COMMAND;
1089                        }
1090 <St_Ref2>{WORD1NQ}     |
1091 <St_Ref2>{WORD2NQ}     {
1092                          /* word */
1093                          g_token->name = yytext;
1094                          return TK_WORD;
1095                        }
1096 <St_Ref2>[ \t]+        {
1097                          g_token->chars=yytext;
1098                          return TK_WHITESPACE;
1099                        }
1100 <St_Ref2>"\""|\n       { /* " or \n => end of title */
1101                          return 0;
1102                        }
1103 <St_XRefItem>{LABELID} {
1104                          g_token->name=yytext;
1105                        }
1106 <St_XRefItem>" "       {
1107                          BEGIN(St_XRefItem2);
1108                        }
1109 <St_XRefItem2>[0-9]+"." {
1110                          QCString numStr=yytext;
1111                          numStr=numStr.left((int)yyleng-1);
1112                          g_token->id=numStr.toInt();
1113                          return RetVal_OK;
1114                        }
1115 <St_Para,St_Title,St_Ref2>"<!--"     { /* html style comment block */
1116                          g_commentState = YY_START;
1117                          BEGIN(St_Comment);
1118                        }
1119 <St_Param>"\""[^\n\"]+"\"" {
1120                          g_token->name = yytext+1;
1121                          g_token->name = g_token->name.left((int)yyleng-2);
1122                          return TK_WORD;
1123                        }
1124 <St_Param>({PHPTYPE}{BLANK}*"|"{BLANK}*)*{PHPTYPE}{WS}+("&")?"$"{LABELID} {
1125                          QCString params = yytext;
1126                          int j = params.find('&');
1127                          int i = params.find('$');
1128                          if (j<i && j!=-1) i=j;
1129                          QCString types = params.left(i).stripWhiteSpace();
1130                          g_token->name = types+"#"+params.mid(i);
1131                          return TK_WORD;
1132                        }
1133 <St_Param>[^ \t\n,@\\]+  {
1134                          g_token->name = yytext;
1135                          if (g_token->name.at(yyleng-1)==':')
1136                          {
1137                            g_token->name=g_token->name.left(yyleng-1);
1138                          }
1139                          return TK_WORD;
1140                        }
1141 <St_Param>{WS}*","{WS}*  /* param separator */
1142 <St_Param>{WS}         {
1143                          g_token->chars=yytext;
1144                          return TK_WHITESPACE;
1145                        }
1146 <St_File>{FILEMASK}    {
1147                          g_token->name = yytext;
1148                          return TK_WORD;
1149                        }
1150 <St_File>"\""[^\n\"]+"\"" {
1151                          QCString text=yytext;
1152                          g_token->name = text.mid(1,text.length()-2);
1153                          return TK_WORD;
1154                        }
1155 <St_Pattern>[^\r\n]+   {
1156                          g_token->name = yytext;
1157                          g_token->name = g_token->name.stripWhiteSpace();
1158                          return TK_WORD;
1159                        }
1160 <St_Link>{LINKMASK}|{REFWORD}    {
1161                          g_token->name = yytext;
1162                          return TK_WORD;
1163                        }
1164 <St_Comment>"-->"      { /* end of html comment */
1165                          BEGIN(g_commentState);
1166                        }
1167 <St_Comment>[^-]+      /* inside html comment */
1168 <St_Comment>.          /* inside html comment */
1169
1170      /* State for skipping title (all chars until the end of the line) */
1171
1172 <St_SkipTitle>.
1173 <St_SkipTitle>\n       { return 0; }
1174
1175      /* State for the pass used to find the anchors and sections */
1176
1177 <St_Sections>[^\n@\\<]+
1178 <St_Sections>"@@"|"\\\\"|"@<"|"\\<"
1179 <St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
1180                                       QCString tag=yytext;
1181                                       int s=tag.find("id=");
1182                                       if (s!=-1) // command has id attribute
1183                                       {
1184                                         char c=tag[s+3];
1185                                         if (c=='\'' || c=='"') // valid start
1186                                         {
1187                                           int e=tag.find(c,s+4);
1188                                           if (e!=-1) // found matching end
1189                                           {
1190                                             g_secType = SectionInfo::Table;
1191                                             g_secLabel=tag.mid(s+4,e-s-4); // extract id
1192                                             processSection();
1193                                           }
1194                                         }
1195                                       }
1196                                     }
1197 <St_Sections>{CMD}"anchor"{BLANK}+  {
1198                                       g_secType = SectionInfo::Anchor;
1199                                       BEGIN(St_SecLabel1);
1200                                     }
1201 <St_Sections>{CMD}"section"{BLANK}+ {
1202                                       g_secType = SectionInfo::Section;
1203                                       BEGIN(St_SecLabel2);
1204                                     }
1205 <St_Sections>{CMD}"subsection"{BLANK}+ {
1206                                       g_secType = SectionInfo::Subsection;
1207                                       BEGIN(St_SecLabel2);
1208                                     }
1209 <St_Sections>{CMD}"subsubsection"{BLANK}+ {
1210                                       g_secType = SectionInfo::Subsubsection;
1211                                       BEGIN(St_SecLabel2);
1212                                     }
1213 <St_Sections>{CMD}"paragraph"{BLANK}+ {
1214                                       g_secType = SectionInfo::Paragraph;
1215                                       BEGIN(St_SecLabel2);
1216                                     }
1217 <St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9]  {
1218                                       g_endMarker="endverbatim";
1219                                       BEGIN(St_SecSkip);
1220                                     }
1221 <St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1222                                       g_endMarker="enddot";
1223                                       BEGIN(St_SecSkip);
1224                                     }
1225 <St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1226                                       g_endMarker="endmsc";
1227                                       BEGIN(St_SecSkip);
1228                                     }
1229 <St_Sections>{CMD}"startuml"/[^a-z_A-Z0-9] {
1230                                       g_endMarker="enduml";
1231                                       BEGIN(St_SecSkip);
1232                                     }
1233 <St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1234                                       g_endMarker="endhtmlonly";
1235                                       BEGIN(St_SecSkip);
1236                                     }
1237 <St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1238                                       g_endMarker="endlatexonly";
1239                                       BEGIN(St_SecSkip);
1240                                     }
1241 <St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1242                                       g_endMarker="endxmlonly";
1243                                       BEGIN(St_SecSkip);
1244                                     }
1245 <St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
1246                                       g_endMarker="enddocbookonly";
1247                                       BEGIN(St_SecSkip);
1248                                     }
1249 <St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1250                                       g_endMarker="endcode";
1251                                       BEGIN(St_SecSkip);
1252                                     }
1253 <St_Sections>"<!--"                 {
1254                                       g_endMarker="-->";
1255                                       BEGIN(St_SecSkip);
1256                                     }
1257 <St_SecSkip>{CMD}{ID}               {
1258                                       if (qstrcmp(yytext+1,g_endMarker)==0)
1259                                       {
1260                                         BEGIN(St_Sections);
1261                                       }
1262                                     }
1263 <St_SecSkip>"-->"                   {
1264                                       if (qstrcmp(yytext,g_endMarker)==0)
1265                                       {
1266                                         BEGIN(St_Sections);
1267                                       }
1268                                     }
1269 <St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1270 <St_SecSkip>.
1271 <St_SecSkip>\n
1272 <St_Sections>.
1273 <St_Sections>\n
1274 <St_SecLabel1>{LABELID} {
1275                          g_secLabel = yytext;
1276                          processSection();
1277                          BEGIN(St_Sections);
1278                        }
1279 <St_SecLabel2>{LABELID}{BLANK}+ |
1280 <St_SecLabel2>{LABELID}         {
1281                          g_secLabel = yytext;
1282                          g_secLabel = g_secLabel.stripWhiteSpace();
1283                          BEGIN(St_SecTitle);
1284                        }
1285 <St_SecTitle>[^\n]+    |
1286 <St_SecTitle>[^\n]*\n  {
1287                          g_secTitle = yytext;
1288                          g_secTitle = g_secTitle.stripWhiteSpace();
1289                          processSection();
1290                          BEGIN(St_Sections);
1291                        }
1292 <St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1293                          warn(g_fileName,yylineno,"Unexpected character `%s' while looking for section label or title",yytext);
1294                        }
1295
1296 <St_Snippet>[^\n]+     |
1297 <St_Snippet>[^\n]*\n   {
1298                          g_token->name = yytext;
1299                          g_token->name = g_token->name.stripWhiteSpace();
1300                          return TK_WORD;
1301                        }
1302
1303      /* Generic rules that work for all states */
1304 <*>\n                  {
1305                          warn(g_fileName,yylineno,"Unexpected new line character");
1306                        }
1307 <*>[\\@<>&$#%~"=]      { /* unescaped special character */
1308                          //warn(g_fileName,yylineno,"Unexpected character `%s', assuming command \\%s was meant.",yytext,yytext);
1309                          g_token->name = yytext;
1310                          return TK_COMMAND;
1311                        }
1312 <*>.                   {
1313                          warn(g_fileName,yylineno,"Unexpected character `%s'",yytext);
1314                        }
1315 %%
1316
1317 //--------------------------------------------------------------------------
1318
1319 void doctokenizerYYFindSections(const char *input,Definition *d,
1320                                 MemberGroup *mg,const char *fileName)
1321 {
1322   if (input==0) return;
1323   printlex(yy_flex_debug, TRUE, __FILE__, fileName);
1324   g_inputString = input;
1325   //printf("parsing --->`%s'<---\n",input);
1326   g_inputPos    = 0;
1327   g_definition  = d;
1328   g_memberGroup = mg;
1329   g_fileName    = fileName;
1330   BEGIN(St_Sections);
1331   doctokenizerYYlineno = 1;
1332   doctokenizerYYlex();
1333   printlex(yy_flex_debug, FALSE, __FILE__, fileName);
1334 }
1335
1336 void doctokenizerYYinit(const char *input,const char *fileName)
1337 {
1338   g_autoListLevel = 0;
1339   g_inputString = input;
1340   g_inputPos    = 0;
1341   g_fileName    = fileName;
1342   g_insidePre   = FALSE;
1343   BEGIN(St_Para);
1344 }
1345
1346 void doctokenizerYYsetStatePara()
1347 {
1348   BEGIN(St_Para);
1349 }
1350
1351 void doctokenizerYYsetStateTitle()
1352 {
1353   BEGIN(St_Title);
1354 }
1355
1356 void doctokenizerYYsetStateTitleAttrValue()
1357 {
1358   BEGIN(St_TitleV);
1359 }
1360
1361 void doctokenizerYYsetStateCode()
1362 {
1363   g_token->verb="";
1364   g_token->name="";
1365   BEGIN(St_CodeOpt);
1366 }
1367
1368 void doctokenizerYYsetStateXmlCode()
1369 {
1370   g_token->verb="";
1371   g_token->name="";
1372   BEGIN(St_XmlCode);
1373 }
1374
1375 void doctokenizerYYsetStateHtmlOnly()
1376 {
1377   g_token->verb="";
1378   g_token->name="";
1379   BEGIN(St_HtmlOnlyOption);
1380 }
1381
1382 void doctokenizerYYsetStateManOnly()
1383 {
1384   g_token->verb="";
1385   BEGIN(St_ManOnly);
1386 }
1387
1388 void doctokenizerYYsetStateRtfOnly()
1389 {
1390   g_token->verb="";
1391   BEGIN(St_RtfOnly);
1392 }
1393
1394 void doctokenizerYYsetStateXmlOnly()
1395 {
1396   g_token->verb="";
1397   BEGIN(St_XmlOnly);
1398 }
1399
1400 void doctokenizerYYsetStateDbOnly()
1401 {
1402   g_token->verb="";
1403   BEGIN(St_DbOnly);
1404 }
1405
1406 void doctokenizerYYsetStateLatexOnly()
1407 {
1408   g_token->verb="";
1409   BEGIN(St_LatexOnly);
1410 }
1411
1412 void doctokenizerYYsetStateVerbatim()
1413 {
1414   g_token->verb="";
1415   BEGIN(St_Verbatim);
1416 }
1417
1418 void doctokenizerYYsetStateDot()
1419 {
1420   g_token->verb="";
1421   BEGIN(St_Dot);
1422 }
1423
1424 void doctokenizerYYsetStateMsc()
1425 {
1426   g_token->verb="";
1427   BEGIN(St_Msc);
1428 }
1429
1430 void doctokenizerYYsetStatePlantUMLOpt()
1431 {
1432   g_token->verb="";
1433   g_token->sectionId="";
1434   BEGIN(St_PlantUMLOpt);
1435 }
1436
1437 void doctokenizerYYsetStatePlantUML()
1438 {
1439   g_token->verb="";
1440   BEGIN(St_PlantUML);
1441 }
1442
1443 void doctokenizerYYsetStateParam()
1444 {
1445   BEGIN(St_Param);
1446 }
1447
1448 void doctokenizerYYsetStateXRefItem()
1449 {
1450   BEGIN(St_XRefItem);
1451 }
1452
1453 void doctokenizerYYsetStateFile()
1454 {
1455   BEGIN(St_File);
1456 }
1457
1458 void doctokenizerYYsetStatePattern()
1459 {
1460   BEGIN(St_Pattern);
1461 }
1462
1463 void doctokenizerYYsetStateLink()
1464 {
1465   BEGIN(St_Link);
1466 }
1467
1468 void doctokenizerYYsetStateCite()
1469 {
1470   BEGIN(St_Cite);
1471 }
1472
1473 void doctokenizerYYsetStateRef()
1474 {
1475   BEGIN(St_Ref);
1476 }
1477
1478 void doctokenizerYYsetStateInternalRef()
1479 {
1480   BEGIN(St_IntRef);
1481 }
1482
1483 void doctokenizerYYsetStateText()
1484 {
1485   BEGIN(St_Text);
1486 }
1487
1488 void doctokenizerYYsetStateSkipTitle()
1489 {
1490   BEGIN(St_SkipTitle);
1491 }
1492
1493 void doctokenizerYYsetStateAnchor()
1494 {
1495   BEGIN(St_Anchor);
1496 }
1497
1498 void doctokenizerYYsetStateSnippet()
1499 {
1500   BEGIN(St_Snippet);
1501 }
1502
1503 void doctokenizerYYsetStateSetScope()
1504 {
1505   BEGIN(St_SetScope);
1506 }
1507
1508 void doctokenizerYYcleanup()
1509 {
1510   yy_delete_buffer( YY_CURRENT_BUFFER );
1511 }
1512
1513 void doctokenizerYYsetInsidePre(bool b)
1514 {
1515   g_insidePre = b;
1516 }
1517
1518 void doctokenizerYYpushBackHtmlTag(const char *tag)
1519 {
1520   QCString tagName = tag;
1521   int i,l = tagName.length();
1522   unput('>');
1523   for (i=l-1;i>=0;i--)
1524   {
1525     unput(tag[i]);
1526   }
1527   unput('<');
1528 }
1529
1530 void doctokenizerYYstartAutoList()
1531 {
1532   g_autoListLevel++;
1533 }
1534
1535 void doctokenizerYYendAutoList()
1536 {
1537   g_autoListLevel--;
1538 }
1539
1540 //REAL_YY_DECL
1541 //{
1542 //  printlex(yy_flex_debug, TRUE, __FILE__, g_fileName);
1543 //  int retval = LOCAL_YY_DECL;
1544 //  printlex(yy_flex_debug, FALSE, __FILE__, g_fileName);
1545 //  return retval;
1546 //}
1547 #if !defined(YY_FLEX_SUBMINOR_VERSION)
1548 extern "C" { // some bogus code to keep the compiler happy
1549     void doctokenizerYYdummy() { yy_flex_realloc(0,0); }
1550 }
1551 #endif
1552