testing/runtests.py

   1 #!/usr/bin/python
   2
   3 from __future__ import print_function
   4 import argparse, glob, itertools, re, shutil, os, sys
   5
   6 config_reg = re.compile('.*\/\/\s*(?P<name>\S+):\s*(?P<value>.*)$')
   7
   8 class Tester:
   9         def __init__(self,args,test):
  10                 self.args      = args
  11                 self.test      = test
  12                 self.update    = args.updateref
  13                 self.config    = self.get_config()
  14                 self.test_name = '[%s]: %s' % (self.test,self.config['objective'][0])
  15                 self.test_id   = self.test.split('_')[0]
  16                 if self.update:
  17                         self.test_out = self.args.inputdir+'/'+self.test_id
  18                 else:
  19                         self.test_out = self.args.outputdir+'/test_output_'+self.test_id
  20                 self.prepare_test()
  21
  22         def compare_ok(self,got_file,expected_file,name):
  23                 if not os.path.isfile(got_file):
  24                         return (True,'%s absent' % got_file)
  25                 elif not os.path.isfile(expected_file):
  26                         return (True,'%s absent' % expected_file)
  27                 else:
  28                         diff = os.popen('diff -b -w -u %s %s' % (got_file,expected_file)).read()
  29                         if diff and not diff.startswith("No differences"):
  30                                 return (True,'Difference between generated output and reference:\n%s' % diff)
  31                 return (False,'')
  32
  33         def get_config(self):
  34                 config = {}
  35                 with open(self.args.inputdir+'/'+self.test,'r') as f:
  36                         for line in f.readlines():
  37                                 m = config_reg.match(line)
  38                                 if m:
  39                                         key   = m.group('name')
  40                                         value = m.group('value')
  41                                         if (key=='config'):
  42                                                 value = value.replace('$INPUTDIR',self.args.inputdir)
  43                                         #print('key=%s value=%s' % (key,value))
  44                                         config.setdefault(key, []).append(value)
  45                 return config
  46
  47         def prepare_test(self):
  48                 # prepare test environment
  49                 shutil.rmtree(self.test_out,ignore_errors=True)
  50                 os.mkdir(self.test_out)
  51                 shutil.copy(self.args.inputdir+'/Doxyfile',self.test_out)
  52                 with open(self.test_out+'/Doxyfile','a') as f:
  53                         print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f)
  54                         print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f)
  55                         print('XML_OUTPUT=%s/out' % self.test_out, file=f)
  56                         print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f)
  57                         if 'config' in self.config:
  58                                 for option in self.config['config']:
  59                                         print(option, file=f)
  60
  61                 if 'check' not in self.config or not self.config['check']:
  62                         print('Test doesn\'t specify any files to check')
  63                         sys.exit(1)
  64
  65                 # run doxygen
  66                 if (sys.platform == 'win32'):
  67                         redir=' > nul:'
  68                 else:
  69                         redir=' 2> /dev/null'
  70                 if os.system('%s %s/Doxyfile %s' % (self.args.doxygen,self.test_out, redir))!=0:
  71                         print('Error: failed to run %s on %s/Doxyfile' % (self.args.doxygen,self.test_out));
  72                         sys.exit(1)
  73
  74         # update the reference data for this test
  75         def update_test(self,testmgr):
  76                 print('Updating reference for %s' % self.test_name)
  77
  78                 if 'check' in self.config:
  79                         for check in self.config['check']:
  80                                 check_file='%s/out/%s' % (self.test_out,check)
  81                                 # check if the file we need to check is actually generated
  82                                 if not os.path.isfile(check_file):
  83                                         print('Non-existing file %s after \'check:\' statement' % check_file)
  84                                         return
  85                                 # convert output to canonical form
  86                                 data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
  87                                 if data:
  88                                         # strip version
  89                                         data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
  90                                 else:
  91                                         print('Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out))
  92                                         return
  93                                 out_file='%s/%s' % (self.test_out,check)
  94                                 with open(out_file,'w') as f:
  95                                         print(data,file=f)
  96                 shutil.rmtree(self.test_out+'/out',ignore_errors=True)
  97                 os.remove(self.test_out+'/Doxyfile')
  98
  99         # check the relevant files of a doxygen run with the reference material
 100         def perform_test(self,testmgr):
 101                 # look for files to check against the reference
 102                 if 'check' in self.config:
 103                         for check in self.config['check']:
 104                                 check_file='%s/out/%s' % (self.test_out,check)
 105                                 # check if the file we need to check is actually generated
 106                                 if not os.path.isfile(check_file):
 107                                         testmgr.ok(False,self.test_name,msg='Non-existing file %s after \'check:\' statement' % check_file)
 108                                         return
 109                                 # convert output to canonical form
 110                                 data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
 111                                 if data:
 112                                         # strip version
 113                                         data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
 114                                 else:
 115                                         testmgr.ok(False,self.test_name,msg='Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out))
 116                                         return
 117                                 out_file='%s/%s' % (self.test_out,check)
 118                                 with open(out_file,'w') as f:
 119                                         print(data,file=f)
 120                                 ref_file='%s/%s/%s' % (self.args.inputdir,self.test_id,check)
 121                                 (failed,msg) = self.compare_ok(out_file,ref_file,self.test_name)
 122                                 if failed:
 123                                         testmgr.ok(False,self.test_name,msg)
 124                                         return
 125                 shutil.rmtree(self.test_out,ignore_errors=True)
 126                 testmgr.ok(True,self.test_name)
 127
 128         def run(self,testmgr):
 129                 if self.update:
 130                         self.update_test(testmgr)
 131                 else:
 132                         self.perform_test(testmgr)
 133
 134 class TestManager:
 135         def __init__(self,args,tests):
 136                 self.args  = args
 137                 self.tests = tests
 138                 self.num_tests = len(tests)
 139                 self.count=1
 140                 self.passed=0
 141                 print('1..%d' % self.num_tests)
 142
 143         def ok(self,result,test_name,msg='Ok'):
 144                 if result:
 145                         print('ok %s - %s' % (self.count,test_name))
 146                         self.passed = self.passed + 1
 147                 else:
 148                         print('not ok %s - %s' % (self.count,test_name))
 149                         print('-------------------------------------')
 150                         print(msg)
 151                         print('-------------------------------------')
 152                 self.count = self.count + 1
 153
 154         def result(self):
 155                 if self.passed==self.num_tests:
 156                         print('All tests passed!')
 157                 else:
 158                         print('%d out of %s tests failed' % (self.num_tests-self.passed,self.num_tests))
 159                 return 0 if self.passed==self.num_tests else 1
 160
 161         def perform_tests(self):
 162                 for test in self.tests:
 163                         tester = Tester(self.args,test)
 164                         tester.run(self)
 165                 return 0 if self.args.updateref else self.result()
 166
 167 def main():
 168         # argument handling
 169         parser = argparse.ArgumentParser(description='run doxygen tests')
 170         parser.add_argument('--updateref',help='update the reference data for a test',action="store_true")
 171         parser.add_argument('--doxygen',nargs='?',default='doxygen',help='path/name of the doxygen executable')
 172         parser.add_argument('--xmllint',nargs='?',default='xmllint',help='path/name of the xmllint executable')
 173         parser.add_argument('--id',nargs='+',dest='ids',action='append',type=int,help='id of the test to perform')
 174         parser.add_argument('--all',help='perform all tests',action="store_true")
 175         parser.add_argument('--inputdir',nargs='?',default='.',help='input directory containing the tests')
 176         parser.add_argument('--outputdir',nargs='?',default='.',help='output directory to write the doxygen output to')
 177         args = parser.parse_args()
 178
 179     # sanity check
 180         if (not args.updateref is None) and (args.ids is None) and (args.all is None):
 181                 parser.error('--updateref requires either --id or --all')
 182
 183         starting_directory = os.getcwd()
 184         os.chdir(args.inputdir)
 185     # find the tests to run
 186         if args.ids: # test ids are given by user
 187                 tests = []
 188                 for id in list(itertools.chain.from_iterable(args.ids)):
 189                         tests.append(glob.glob('%s_*'%id))
 190                         tests.append(glob.glob('0%s_*'%id))
 191                         tests.append(glob.glob('00%s_*'%id))
 192                 tests = list(itertools.chain.from_iterable(tests))
 193         else: # find all tests
 194                 tests = glob.glob('[0-9][0-9][0-9]_*')
 195         os.chdir(starting_directory)
 196
 197         # create test manager to run the tests
 198         testManager = TestManager(args,tests)
 199         sys.exit(testManager.perform_tests())
 200
 201 if __name__ == '__main__':
 202         main()