testing/runtests.py

   1 #!/usr/bin/python
   2
   3 from __future__ import print_function
   4 import argparse, glob, itertools, re, shutil, os, sys
   5 import subprocess
   6 import shlex
   7
   8 config_reg = re.compile('.*\/\/\s*(?P<name>\S+):\s*(?P<value>.*)$')
   9 bkmk_reg = re.compile(r'.*bkmkstart\s+([A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z]).*')
  10 hyper_reg = re.compile(r'.*HYPERLINK\s+[\\l]*\s+"([A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z])".*')
  11 pageref_reg = re.compile(r'.*PAGEREF\s+([A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z][A-Z]).*')
  12
  13
  14 def xopen(fname, mode='r', encoding='utf-8'):
  15         '''Unified file opening for Python 2 an Python 3.
  16
  17         Python 2 does not have the encoding argument. Python 3 has one.
  18         '''
  19
  20         if sys.version_info[0] == 2:
  21                 return open(fname, mode=mode) # Python 2 without encoding
  22         else:
  23                 return open(fname, mode=mode, encoding=encoding) # Python 3 with encoding
  24
  25 def xpopen(cmd, cmd1="",encoding='utf-8-sig', getStderr=False):
  26         '''Unified file pipe opening for Python 2 an Python 3.
  27
  28         Python 2 does not have the encoding argument. Python 3 has one. and
  29         '''
  30
  31         if sys.version_info[0] == 2:
  32                 return os.popen(cmd).read() # Python 2 without encoding
  33         else:
  34                 if (getStderr):
  35                         proc = subprocess.Popen(shlex.split(cmd1),stdout=subprocess.PIPE,stderr=subprocess.PIPE,encoding=encoding) # Python 3 with encoding
  36                         return proc.stderr.read()
  37                 else:
  38                         proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.PIPE,encoding=encoding) # Python 3 with encoding
  39                         return proc.stdout.read()
  40
  41 def clean_header(errmsg):
  42         # messages (due to the usage of more) have a contents like:
  43         # ::::::::::::
  44         # <file name>
  45         # ::::::::::::
  46         # we want to skip these
  47         msg = errmsg.split('\n')
  48         rtnmsg = ""
  49         cnt = -1
  50         for o in msg:
  51                 if (o):
  52                         if (cnt == -1):
  53                                 if o.startswith(":::::::"):
  54                                         cnt = 3
  55                         if (cnt > 0):
  56                                 cnt-=1
  57                         else:
  58                                 rtnmsg+=o
  59         return rtnmsg
  60
  61 class Tester:
  62         def __init__(self,args,test):
  63                 self.args      = args
  64                 self.test      = test
  65                 self.update    = args.updateref
  66                 self.config    = self.get_config()
  67                 self.test_name = '[%s]: %s' % (self.test,self.config['objective'][0])
  68                 self.test_id   = self.test.split('_')[0]
  69                 if self.update:
  70                         self.test_out = self.args.inputdir+'/'+self.test_id
  71                 else:
  72                         self.test_out = self.args.outputdir+'/test_output_'+self.test_id
  73                 self.prepare_test()
  74
  75         def compare_ok(self,got_file,expected_file,name):
  76                 if not os.path.isfile(got_file):
  77                         return (True,'%s absent' % got_file)
  78                 elif not os.path.isfile(expected_file):
  79                         return (True,'%s absent' % expected_file)
  80                 else:
  81                         diff = xpopen('diff -b -w -u %s %s' % (got_file,expected_file))
  82                         if diff and not diff.startswith("No differences"):
  83                                 return (True,'Difference between generated output and reference:\n%s' % diff)
  84                 return (False,'')
  85
  86         def cleanup_xmllint(self,errmsg):
  87                 msg = errmsg.split('\n')
  88                 rtnmsg = ""
  89                 for o in msg:
  90                         if (o):
  91                                 if (o.startswith("I/O error : Attempt")):
  92                                         pass
  93                                 else:
  94                                         if (rtnmsg):
  95                                                 rtnmsg += '\n'
  96                                         rtnmsg += o
  97                 return rtnmsg
  98
  99         def cleanup_xmllint_docbook(self,errmsg):
 100                 # For future work, first get everything valid XML
 101                 msg = self.cleanup_xmllint(errmsg).split('\n')
 102                 rtnmsg = ""
 103                 cnt = 0
 104                 for o in msg:
 105                         if (o):
 106                                 if (cnt):
 107                                         cnt -= 1
 108                                         pass
 109                                 elif (o.endswith("does not validate")):
 110                                         pass
 111                                 elif (o.find("no DTD found!")!=-1):
 112                                         pass
 113                                 elif (o.find("is not an NCName")!=-1):
 114                                         cnt = 2
 115                                 else:
 116                                         if (rtnmsg):
 117                                                 rtnmsg += '\n'
 118                                         rtnmsg += o
 119                 return rtnmsg
 120
 121         def get_config(self):
 122                 config = {}
 123                 with xopen(self.args.inputdir+'/'+self.test,'r') as f:
 124                         for line in f.readlines():
 125                                 m = config_reg.match(line)
 126                                 if m:
 127                                         key   = m.group('name')
 128                                         value = m.group('value')
 129                                         if (key=='config'):
 130                                                 value = value.replace('$INPUTDIR',self.args.inputdir)
 131                                         # print('key=%s value=%s' % (key,value))
 132                                         config.setdefault(key, []).append(value)
 133                 return config
 134
 135         def prepare_test(self):
 136                 # prepare test environment
 137                 shutil.rmtree(self.test_out,ignore_errors=True)
 138                 os.mkdir(self.test_out)
 139                 shutil.copy(self.args.inputdir+'/Doxyfile',self.test_out)
 140                 with xopen(self.test_out+'/Doxyfile','a') as f:
 141                         print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f)
 142                         print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f)
 143                         print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f)
 144                         print('WARN_LOGFILE=%s/warnings.log' % self.test_out, file=f)
 145                         if 'config' in self.config:
 146                                 for option in self.config['config']:
 147                                         print(option, file=f)
 148                         if (self.args.xml or self.args.xmlxsd):
 149                                 print('GENERATE_XML=YES', file=f)
 150                                 print('XML_OUTPUT=%s/out' % self.test_out, file=f)
 151                         else:
 152                                 print('GENERATE_XML=NO', file=f)
 153                         if (self.args.rtf):
 154                                 print('GENERATE_RTF=YES', file=f)
 155                                 print('RTF_HYPERLINKS=YES', file=f)
 156                                 print('RTF_OUTPUT=%s/rtf' % self.test_out, file=f)
 157                         else:
 158                                 print('GENERATE_RTF=NO', file=f)
 159                         if (self.args.docbook):
 160                                 print('GENERATE_DOCBOOK=YES', file=f)
 161                                 print('DOCBOOK_OUTPUT=%s/docbook' % self.test_out, file=f)
 162                         else:
 163                                 print('GENERATE_DOCBOOK=NO', file=f)
 164                         if (self.args.xhtml):
 165                                 print('GENERATE_HTML=YES', file=f)
 166                         # HTML_OUTPUT can also have been set locally
 167                         print('HTML_OUTPUT=%s/html' % self.test_out, file=f)
 168                         print('HTML_FILE_EXTENSION=.xhtml', file=f)
 169                         if (self.args.pdf):
 170                                 print('GENERATE_LATEX=YES', file=f)
 171                                 print('LATEX_BATCHMODE=YES', file=f)
 172                                 print('LATEX_OUTPUT=%s/latex' % self.test_out, file=f)
 173                         if self.args.subdirs:
 174                                 print('CREATE_SUBDIRS=YES', file=f)
 175                         if (self.args.clang):
 176                                 print('CLANG_ASSISTED_PARSING=YES', file=f)
 177                         if (self.args.cfgs):
 178                                 for cfg in self.args.cfgs:
 179                                         if cfg[0].find('=') == -1:
 180                                                 print("Not a doxygen configuration item, missing '=' sign: '%s'."%cfg)
 181                                                 sys.exit(1)
 182                                         print(cfg[0], file=f)
 183
 184                 if 'check' not in self.config or not self.config['check']:
 185                         print('Test doesn\'t specify any files to check')
 186                         sys.exit(1)
 187
 188                 # run doxygen
 189                 if (sys.platform == 'win32'):
 190                         redir=' > nul: 2>&1'
 191                 else:
 192                         redir=' 2> /dev/null > /dev/null'
 193
 194                 if (self.args.noredir):
 195                         redir=''
 196
 197                 if os.system('%s %s/Doxyfile %s' % (self.args.doxygen,self.test_out,redir))!=0:
 198                         print('Error: failed to run %s on %s/Doxyfile' % (self.args.doxygen,self.test_out))
 199                         sys.exit(1)
 200
 201
 202         def check_link_rtf_file(self,fil):
 203                 bkmk_res = []
 204                 hyper_res = []
 205                 pageref_res = []
 206                 with xopen(fil,'r') as f:
 207                         for line in f.readlines():
 208                                 if ("bkmkstart" in line) or ("HYPERLINK" in line) or ("PAGEREF" in line):
 209                                         msg = line.split('}')
 210                                         for m in msg:
 211                                                 if bkmk_reg.match(m):
 212                                                         m1 = re.sub(bkmk_reg, '\\1', m)
 213                                                         bkmk_res.append(m1)
 214                                                 elif hyper_reg.match(m):
 215                                                         m1 = re.sub(hyper_reg, '\\1', m)
 216                                                         hyper_res.append(m1)
 217                                                 elif pageref_reg.match(m):
 218                                                         m1 = re.sub(pageref_reg, '\\1', m)
 219                                                         pageref_res.append(m1)
 220                 # Has been commented out as in the test 57, inline namespace, there is still a small problem.
 221                 #if sorted(bkmk_res) != sorted(set(bkmk_res)):
 222                 #       return (False, "RTF: one (or more) bookmark(s) has(have) been defined multiple times")
 223                 hyper_res = sorted(set(hyper_res))
 224                 for h in hyper_res:
 225                         if h not in bkmk_res:
 226                                 #print(bkmk_res)
 227                                 #print(hyper_res)
 228                                 return (False, "RTF: Not all used hyperlinks have been defined")
 229                 pageref_res = sorted(set(pageref_res))
 230                 for p in pageref_res:
 231                         if p not in bkmk_res:
 232                                 #print(bkmk_res)
 233                                 #print(pageref_res)
 234                                 return (False, "RTF: Not all used page reference bookmarks have been defined")
 235                 return (True,"")
 236
 237
 238         # update the reference data for this test
 239         def update_test(self,testmgr):
 240                 print('Updating reference for %s' % self.test_name)
 241
 242                 if 'check' in self.config:
 243                         for check in self.config['check']:
 244                                 check_file='%s/out/%s' % (self.test_out,check)
 245                                 # check if the file we need to check is actually generated
 246                                 if not os.path.isfile(check_file):
 247                                         print('Non-existing file %s after \'check:\' statement' % check_file)
 248                                         return False
 249                                 # convert output to canonical form
 250                                 data = xpopen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file))
 251                                 if data:
 252                                         # strip version
 253                                         data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
 254                                 else:
 255                                         print('Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out))
 256                                         return False
 257                                 out_file='%s/%s' % (self.test_out,check)
 258                                 with xopen(out_file,'w') as f:
 259                                         print(data,file=f)
 260                 shutil.rmtree(self.test_out+'/out',ignore_errors=True)
 261                 os.remove(self.test_out+'/Doxyfile')
 262                 return True
 263
 264         # check the relevant files of a doxygen run with the reference material
 265         def perform_test(self,testmgr):
 266                 if (sys.platform == 'win32'):
 267                         redir=' > nul:'
 268                         separ='&'
 269                 else:
 270                         redir=' 2> /dev/null'
 271                         separ=';'
 272
 273                 if (self.args.noredir):
 274                         redir=''
 275
 276                 failed_xml=False
 277                 failed_html=False
 278                 failed_latex=False
 279                 failed_docbook=False
 280                 failed_rtf=False
 281                 failed_xmlxsd=False
 282                 msg = ()
 283                 # look for files to check against the reference
 284                 if self.args.xml or self.args.xmlxsd:
 285                         failed_xml=False
 286                         if 'check' in self.config and self.args.xml:
 287                                 failed_xml=True
 288                                 for check in self.config['check']:
 289                                         check_file='%s/out/%s' % (self.test_out,check)
 290                                         # check if the file we need to check is actually generated
 291                                         if not os.path.isfile(check_file):
 292                                                 # try with sub dirs
 293                                                 check_file = glob.glob('%s/out/*/*/%s' % (self.test_out,check))
 294                                                 if not check_file:
 295                                                         check_file='%s/out/%s' % (self.test_out,check)
 296                                                         msg += ('Non-existing file %s after \'check:\' statement' % check_file,)
 297                                                         break
 298                                                 else:
 299                                                         check_file = check_file[0]
 300                                         # convert output to canonical form
 301                                         check_file = check_file.replace('\\','/')
 302                                         data = xpopen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file))
 303                                         if data:
 304                                                 # strip version
 305                                                 data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
 306                                         else:
 307                                                 msg += ('Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out),)
 308                                                 break
 309                                         if self.args.subdirs:
 310                                                 data = re.sub('d[0-9a-f]/d[0-9a-f][0-9a-f]/','',data)
 311                                         out_file='%s/%s' % (self.test_out,check)
 312                                         with xopen(out_file,'w') as f:
 313                                                 print(data,file=f)
 314                                         ref_file='%s/%s/%s' % (self.args.inputdir,self.test_id,check)
 315                                         (failed_xml,xml_msg) = self.compare_ok(out_file,ref_file,self.test_name)
 316                                         if failed_xml:
 317                                                 msg+= (xml_msg,)
 318                                                 break
 319                         failed_xmlxsd=False
 320                         if self.args.xmlxsd:
 321                                 xmlxsd_output='%s/out' % self.test_out
 322                                 if (sys.platform == 'win32'):
 323                                         redirx=' 2> %s/temp >nul:'%xmlxsd_output
 324                                 else:
 325                                         redirx='2>%s/temp >/dev/null'%xmlxsd_output
 326                                 #
 327                                 index_xml = []
 328                                 index_xml.append(glob.glob('%s/index.xml' % (xmlxsd_output)))
 329                                 index_xml.append(glob.glob('%s/*/*/index.xml' % (xmlxsd_output)))
 330                                 index_xml = ' '.join(list(itertools.chain.from_iterable(index_xml))).replace(self.args.outputdir +'/','').replace('\\','/')
 331                                 index_xsd = []
 332                                 index_xsd.append(glob.glob('%s/index.xsd' % (xmlxsd_output)))
 333                                 index_xsd.append(glob.glob('%s/*/*/index.xsd' % (xmlxsd_output)))
 334                                 index_xsd = ' '.join(list(itertools.chain.from_iterable(index_xsd))).replace(self.args.outputdir +'/','').replace('\\','/')
 335                                 exe_string = '%s --noout --schema %s %s' % (self.args.xmllint,index_xsd,index_xml)
 336                                 exe_string1 = exe_string
 337                                 exe_string += ' %s' % (redirx)
 338                                 exe_string += ' %s more "%s/temp"' % (separ,xmlxsd_output)
 339
 340                                 xmllint_out = xpopen(exe_string,exe_string1,getStderr=True)
 341                                 if xmllint_out:
 342                                         xmllint_out = re.sub(r'.*validates','',xmllint_out).rstrip('\n')
 343                                 else:
 344                                         msg += ('Failed to run %s with schema %s for files: %s' % (self.args.xmllint,index_xsd,index_xml),)
 345                                         failed_xmlxsd=True
 346                                 if xmllint_out:
 347                                         xmllint_out  = clean_header(xmllint_out)
 348                                 if xmllint_out:
 349                                         msg += (xmllint_out,)
 350                                         failed_xmlxsd=True
 351                                 #
 352                                 doxyfile_xml = []
 353                                 doxyfile_xml.append(glob.glob('%s/Doxyfile.xml' % (xmlxsd_output)))
 354                                 doxyfile_xml.append(glob.glob('%s/*/*/Doxyfile.xml' % (xmlxsd_output)))
 355                                 doxyfile_xml = ' '.join(list(itertools.chain.from_iterable(doxyfile_xml))).replace(self.args.outputdir +'/','').replace('\\','/')
 356                                 doxyfile_xsd = []
 357                                 doxyfile_xsd.append(glob.glob('%s/doxyfile.xsd' % (xmlxsd_output)))
 358                                 doxyfile_xsd.append(glob.glob('%s/*/*/doxyfile.xsd' % (xmlxsd_output)))
 359                                 doxyfile_xsd = ' '.join(list(itertools.chain.from_iterable(doxyfile_xsd))).replace(self.args.outputdir +'/','').replace('\\','/')
 360                                 exe_string = '%s --noout --schema %s %s' % (self.args.xmllint,doxyfile_xsd,doxyfile_xml)
 361                                 exe_string1 = exe_string
 362                                 exe_string += ' %s' % (redirx)
 363                                 exe_string += ' %s more "%s/temp"' % (separ,xmlxsd_output)
 364
 365                                 xmllint_out = xpopen(exe_string,exe_string1,getStderr=True)
 366                                 if xmllint_out:
 367                                         xmllint_out = re.sub(r'.*validates','',xmllint_out).rstrip('\n')
 368                                 else:
 369                                         msg += ('Failed to run %s with schema %s for files: %s' % (self.args.xmllint,doxyfile_xsd,doxyfile_xml),)
 370                                         failed_xmlxsd=True
 371                                 if xmllint_out:
 372                                         xmllint_out  = clean_header(xmllint_out)
 373                                 if xmllint_out:
 374                                         msg += (xmllint_out,)
 375                                         failed_xmlxsd=True
 376                                 #
 377                                 compound_xml = []
 378                                 compound_xml.append(glob.glob('%s/*.xml' % (xmlxsd_output)))
 379                                 compound_xml.append(glob.glob('%s/*/*/*.xml' % (xmlxsd_output)))
 380                                 compound_xml = ' '.join(list(itertools.chain.from_iterable(compound_xml))).replace(self.args.outputdir +'/','').replace('\\','/')
 381                                 compound_xml = re.sub(r' [^ ]*/index.xml','',compound_xml)
 382                                 compound_xml = re.sub(r'[^ ]*/index.xml ','',compound_xml)
 383                                 compound_xml = re.sub(r' [^ ]*/Doxyfile.xml','',compound_xml)
 384                                 compound_xml = re.sub(r'[^ ]*/Doxyfile.xml ','',compound_xml)
 385
 386                                 compound_xsd = []
 387                                 compound_xsd.append(glob.glob('%s/compound.xsd' % (xmlxsd_output)))
 388                                 compound_xsd.append(glob.glob('%s/*/*/compound.xsd' % (xmlxsd_output)))
 389                                 compound_xsd = ' '.join(list(itertools.chain.from_iterable(compound_xsd))).replace(self.args.outputdir +'/','').replace('\\','/')
 390                                 exe_string = '%s --noout --schema %s %s' % (self.args.xmllint,compound_xsd,compound_xml)
 391                                 exe_string1 = exe_string
 392                                 exe_string += ' %s' % (redirx)
 393                                 exe_string += ' %s more "%s/temp"' % (separ,xmlxsd_output)
 394
 395                                 xmllint_out = xpopen(exe_string,exe_string1,getStderr=True)
 396                                 if xmllint_out:
 397                                         xmllint_out = re.sub(r'.*validates','',xmllint_out).rstrip('\n')
 398                                 else:
 399                                         msg += ('Failed to run %s with schema %s for files: %s' % (self.args.xmllint,compound_xsd,compound_xml),)
 400                                         failed_xmlxsd=True
 401                                 if xmllint_out:
 402                                         xmllint_out  = clean_header(xmllint_out)
 403                                 if xmllint_out:
 404                                         msg += (xmllint_out,)
 405                                         failed_xmlxsd=True
 406
 407                         if not failed_xml and not failed_xmlxsd and not self.args.keep:
 408                                 xml_output='%s/out' % self.test_out
 409                                 shutil.rmtree(xml_output,ignore_errors=True)
 410
 411                 if (self.args.rtf):
 412                         (res, msg1) = self.check_link_rtf_file("%s/rtf/refman.rtf" % self.test_out)
 413                         if not res:
 414                                 #msg += ("RTF: Not all used hyperlinks have been defined",)
 415                                 msg += (msg1,)
 416                                 failed_rtf=True
 417
 418                 if (self.args.docbook):
 419                         docbook_output='%s/docbook' % self.test_out
 420                         if (sys.platform == 'win32'):
 421                                 redirx=' 2> %s/temp >nul:'%docbook_output
 422                         else:
 423                                 redirx='2>%s/temp >/dev/null'%docbook_output
 424                         # For future work, first get everything valid XML
 425                         # exe_string = '%s --relaxng db/docbook.rng --nonet --postvalid %s/*xml %s  % (self.args.xmllint,docbook_output,redirx)
 426                         tests = []
 427                         tests.append(glob.glob('%s/*.xml' % (docbook_output)))
 428                         tests.append(glob.glob('%s/*/*/*.xml' % (docbook_output)))
 429                         tests = ' '.join(list(itertools.chain.from_iterable(tests))).replace(self.args.outputdir +'/','').replace('\\','/')
 430                         exe_string = '%s --noout --nonet --postvalid %s' % (self.args.xmllint,tests)
 431                         exe_string1 = exe_string
 432                         exe_string += ' %s' % (redirx)
 433                         exe_string += ' %s more "%s/temp"' % (separ,docbook_output)
 434
 435                         failed_docbook=False
 436                         xmllint_out = xpopen(exe_string,exe_string1,getStderr=True)
 437                         xmllint_out = self.cleanup_xmllint_docbook(xmllint_out)
 438                         if xmllint_out:
 439                                 xmllint_out  = clean_header(xmllint_out)
 440                         if xmllint_out:
 441                                 msg += (xmllint_out,)
 442                                 failed_docbook=True
 443                         elif not self.args.keep:
 444                                 shutil.rmtree(docbook_output,ignore_errors=True)
 445
 446                 if (self.args.xhtml):
 447                         html_output='%s/html' % self.test_out
 448                         if (sys.platform == 'win32'):
 449                                 redirx=' 2> %s/temp >nul:'%html_output
 450                         else:
 451                                 redirx='2>%s/temp >/dev/null'%html_output
 452                         check_file = []
 453                         check_file.append(glob.glob('%s/*.xhtml' % (html_output)))
 454                         check_file.append(glob.glob('%s/*/*/*.xhtml' % (html_output)))
 455                         check_file = ' '.join(list(itertools.chain.from_iterable(check_file))).replace(self.args.outputdir +'/','').replace('\\','/')
 456                         exe_string = '%s --noout --path dtd --nonet --postvalid %s' % (self.args.xmllint,check_file)
 457                         exe_string1 = exe_string
 458                         exe_string += ' %s' % (redirx)
 459                         exe_string += ' %s more "%s/temp"' % (separ,html_output)
 460                         failed_html=False
 461                         xmllint_out = xpopen(exe_string,exe_string1,getStderr=True)
 462                         xmllint_out = self.cleanup_xmllint(xmllint_out)
 463                         if xmllint_out:
 464                                 xmllint_out  = clean_header(xmllint_out)
 465                         if xmllint_out:
 466                                 msg += (xmllint_out,)
 467                                 failed_html=True
 468                         elif not self.args.keep:
 469                                 shutil.rmtree(html_output,ignore_errors=True)
 470                 if (self.args.pdf):
 471                         failed_latex=False
 472                         latex_output='%s/latex' % self.test_out
 473                         # with languages like Hungarian we had problems with some tests on windows when stderr was used.
 474                         if (sys.platform == 'win32'):
 475                                 outType=False
 476                                 redirl='>nul: 2>temp'
 477                                 mk='make.bat'
 478                         else:
 479                                 outType=True
 480                                 redirl='>/dev/null 2>temp'
 481                                 mk='make'
 482                         cur_directory = os.getcwd()
 483                         os.chdir(latex_output)
 484                         exe_string = mk
 485                         exe_string1 = exe_string
 486                         exe_string += ' %s' % (redirl)
 487                         if outType:
 488                                 exe_string += ' %s more temp' % (separ)
 489                         latex_out = xpopen(exe_string,exe_string1,getStderr=outType)
 490                         os.chdir(cur_directory);
 491                         if (outType and latex_out.find("Error")!=-1):
 492                                 msg += ("PDF generation failed\n  For a description of the problem see 'refman.log' in the latex directory of this test",)
 493                                 failed_latex=True
 494                         elif (not outType and xopen(latex_output + "/temp",'r').read().find("Error")!= -1):
 495                                 msg += ("PDF generation failed\n  For a description of the problem see 'refman.log' in the latex directory of this test",)
 496                                 failed_latex=True
 497                         elif xopen(latex_output + "/refman.log",'r',encoding='ISO-8859-1').read().find("Error")!= -1:
 498                                 msg += ("PDF generation failed\n  For a description of the problem see 'refman.log' in the latex directory of this test",)
 499                                 failed_latex=True
 500                         elif xopen(latex_output + "/refman.log",'r',encoding='ISO-8859-1').read().find("Emergency stop")!= -1:
 501                                 msg += ("PDF generation failed\n  For a description of the problem see 'refman.log' in the latex directory of this test",)
 502                                 failed_latex=True
 503                         elif not self.args.keep:
 504                                 shutil.rmtree(latex_output,ignore_errors=True)
 505
 506                 warnings = xopen(self.test_out + "/warnings.log",'r',encoding='ISO-8859-1').read()
 507                 failed_warn =  len(warnings)!=0
 508                 if failed_warn:
 509                         msg += (warnings,)
 510
 511                 if failed_warn or failed_xml or failed_html or failed_latex or failed_docbook or failed_rtf or failed_xmlxsd:
 512                         testmgr.ok(False,self.test_name,msg)
 513                         return False
 514
 515                 testmgr.ok(True,self.test_name)
 516                 if not self.args.keep:
 517                         shutil.rmtree(self.test_out,ignore_errors=True)
 518                 return True
 519
 520         def run(self,testmgr):
 521                 if self.update:
 522                         return self.update_test(testmgr)
 523                 else:
 524                         return self.perform_test(testmgr)
 525
 526 def do_generation_work(test):
 527         tester = Tester(test[0].args,test[1])
 528         return tester.run(test[0])
 529
 530 class TestManager:
 531         def __init__(self,args,tests):
 532                 self.args  = args
 533                 self.tests = tests
 534                 self.num_tests = len(tests)
 535                 self.count=1
 536                 self.passed=0
 537                 if self.args.xhtml:
 538                         self.prepare_dtd()
 539                 print('1..%d' % self.num_tests)
 540
 541         def ok(self,result,test_name,msg='Ok'):
 542                 if result:
 543                         print('ok - %s' % (test_name))
 544                         self.passed = self.passed + 1
 545                 else:
 546                         print('not ok - %s' % (test_name))
 547                         print('-------------------------------------')
 548                         for o in msg:
 549                                 print(o)
 550                                 print('-------------------------------------')
 551                 self.count = self.count + 1
 552
 553         def result(self):
 554                 if self.passed==self.num_tests:
 555                         print('All tests passed!')
 556                 else:
 557                         print('%d out of %s tests failed' % (self.num_tests-self.passed,self.num_tests))
 558                 return 0 if self.passed==self.num_tests else 1
 559
 560         def perform_tests(self):
 561                 if (self.args.pool == 1):
 562                         passed = 0
 563                         for test in self.tests:
 564                                 tester = Tester(self.args,test)
 565                                 passed += tester.run(self)
 566                         self.passed = passed
 567                 else:
 568                         dl = []
 569                         for test in self.tests:
 570                                 dl += [(self, test)]
 571                         import multiprocessing as mp
 572                         p = mp.Pool(processes=self.args.pool)
 573                         passed = p.map(do_generation_work, dl)
 574                         self.passed = sum(passed)
 575                 res=self.result()
 576                 if self.args.xhtml and self.args.inputdir!='.' and not res and not self.args.keep:
 577                         shutil.rmtree("dtd",ignore_errors=True)
 578                 return 0 if self.args.updateref else res
 579
 580         def prepare_dtd(self):
 581                 if self.args.inputdir!='.':
 582                         shutil.rmtree("dtd",ignore_errors=True)
 583                         shutil.copytree(self.args.inputdir+"/dtd", "dtd")
 584
 585 def split_and_keep(s,sep):
 586     s = s.replace('"','')             # add token separator
 587     s = s.replace(sep,'\0'+sep)             # add token separator
 588     s = s.split('\0')                       # split by null delimiter
 589     s = [x.strip() for x in filter(None,s)] # strip and remove empty elements
 590     s = [z.split(' ',1) for z in s]         # split by first space
 591     s = [i for ss in s for i in ss]         # flatten the list
 592     return s
 593
 594 def main():
 595         # argument handling
 596         parser = argparse.ArgumentParser(description='run doxygen tests')
 597         parser.add_argument('--updateref',help=
 598                 'update the reference files. Should be used in combination with -id to '
 599                 'update the reference file(s) for the given test',action="store_true")
 600         parser.add_argument('--doxygen',nargs='?',default='doxygen',help=
 601                 'path/name of the doxygen executable')
 602         parser.add_argument('--xmllint',nargs='?',default='xmllint',help=
 603                 'path/name of the xmllint executable')
 604         parser.add_argument('--id',nargs='+',dest='ids',action='append',type=int,help=
 605                 'run test with number n only (the option can be specified to run test with '
 606                 'number n only (the option can be specified multiple times')
 607         parser.add_argument('--start_id',dest='start_id',type=int,help=
 608                 'run tests starting with number n')
 609         parser.add_argument('--end_id',dest='end_id',type=int,help=
 610                 'run tests ending with number n')
 611         parser.add_argument('--all',help=
 612                 'can be used in combination with -updateref to update the reference files '
 613                 'for all tests.',action="store_true")
 614         parser.add_argument('--inputdir',nargs='?',default='.',help=
 615                 'input directory containing the tests')
 616         parser.add_argument('--outputdir',nargs='?',default='.',help=
 617                 'output directory to write the doxygen output to')
 618         parser.add_argument('--noredir',help=
 619                 'disable redirection of doxygen warnings',action="store_true")
 620         parser.add_argument('--pool',nargs='?',default='1',type=int,help=
 621                 'pool size of multiprocess tests')
 622         parser.add_argument('--xml',help='create xml output and check',
 623                 action="store_true")
 624         parser.add_argument('--rtf',help=
 625                 'create rtf output',action="store_true")
 626         parser.add_argument('--docbook',help=
 627                 'create docbook output and check with xmllint',action="store_true")
 628         parser.add_argument('--xhtml',help=
 629                 'create xhtml output and check with xmllint',action="store_true")
 630         parser.add_argument('--xmlxsd',help=
 631                 'create xml output and check with xmllint against xsd',action="store_true")
 632         parser.add_argument('--pdf',help='create LaTeX output and create pdf from it',
 633                 action="store_true")
 634         parser.add_argument('--subdirs',help='use the configuration parameter CREATE_SUBDIRS=YES',
 635                 action="store_true")
 636         parser.add_argument('--clang',help='use CLANG_ASSISTED_PARSING, works only when '
 637                 'doxygen has been compiled with "use_libclang"',
 638                 action="store_true")
 639         parser.add_argument('--keep',help='keep result directories',
 640                 action="store_true")
 641         parser.add_argument('--cfg',nargs='+',dest='cfgs',action='append',help=
 642                 'run test with extra doxygen configuration settings '
 643                 '(the option may be specified multiple times')
 644
 645         test_flags = split_and_keep(os.getenv('TEST_FLAGS', default=''), '--')
 646
 647         args = parser.parse_args(test_flags + sys.argv[1:])
 648
 649         # sanity check
 650         if (not args.xml) and (not args.pdf) and (not args.xhtml) and (not args.docbook and (not args.rtf) and (not args.xmlxsd)):
 651                 args.xml=True
 652         if (not args.updateref is None) and (args.ids is None) and (args.all is None):
 653                 parser.error('--updateref requires either --id or --all')
 654
 655         starting_directory = os.getcwd()
 656         os.chdir(args.inputdir)
 657         # find the tests to run
 658         tests = []
 659         if args.start_id:
 660                 if args.end_id:
 661                         for id in range(args.start_id, args.end_id + 1):
 662                                 tests.append(glob.glob('%s_*'%id))
 663                                 tests.append(glob.glob('0%s_*'%id))
 664                                 tests.append(glob.glob('00%s_*'%id))
 665                 else:
 666                         parser.error('--start_id requires --end_id')
 667         elif args.end_id:
 668                 parser.error('--end_id requires --start_id')
 669         if args.ids:  # test ids are given by user
 670                 for id in list(itertools.chain.from_iterable(args.ids)):
 671                         tests.append(glob.glob('%s_*'%id))
 672                         tests.append(glob.glob('0%s_*'%id))
 673                         tests.append(glob.glob('00%s_*'%id))
 674         if (not args.ids and not args.start_id):  # find all tests
 675                 tests = sorted(glob.glob('[0-9][0-9][0-9]_*'))
 676         else:
 677                 tests = list(itertools.chain.from_iterable(tests))
 678         os.chdir(starting_directory)
 679
 680         # create test manager to run the tests
 681         testManager = TestManager(args,tests)
 682         sys.exit(testManager.perform_tests())
 683
 684 if __name__ == '__main__':
 685         main()