2 import glob, os, string, sys, thread, time
8 # This is a "Work in Progress" attempt at a python script to run the
9 # various regression tests. The rationale for this is that it should be
10 # possible to run this on most major platforms, including those (such as
11 # Windows) which don't support gnu Make.
13 # The script is driven by a parameter file which defines the various tests
14 # to be run, together with the unique settings for each of these tests. A
15 # script for Linux is included (regressions.xml), with comments indicating
16 # the significance of the various parameters. To run the tests under Windows,
17 # edit regressions.xml and remove the comment around the default parameter
18 # "<execpath>" (i.e. make it point to the location of the binary executables).
20 # Note that this current version requires the Python bindings for libxml2 to
21 # have been previously installed and accessible
23 # See Copyright for the status of this software.
24 # William Brack (wbrack@mmm.com.hk)
27 defaultParams = {} # will be used as a dictionary to hold the parsed params
29 # This routine is used for comparing the expected stdout / stdin with the results.
30 # The expected data has already been read in; the result is a file descriptor.
31 # Within the two sets of data, lines may begin with a path string. If so, the
32 # code "relativises" it by removing the path component. The first argument is a
33 # list already read in by a separate thread; the second is a file descriptor.
34 # The two 'base' arguments are to let me "relativise" the results files, allowing
35 # the script to be run from any directory.
36 def compFiles(res, expected, base1, base2):
38 exp = expected.readlines()
40 # the "relativisation" is done here
41 for i in range(len(res)):
42 j = string.find(res[i],base1)
43 if (j == 0) or ((j == 2) and (res[i][0:2] == './')):
44 col = string.find(res[i],':')
46 start = string.rfind(res[i][:col], '/')
48 res[i] = res[i][start+1:]
50 for i in range(len(exp)):
51 j = string.find(exp[i],base2)
52 if (j == 0) or ((j == 2) and (exp[i][0:2] == './')):
53 col = string.find(exp[i],':')
55 start = string.rfind(exp[i][:col], '/')
57 exp[i] = exp[i][start+1:]
60 # ideally we would like to use difflib functions here to do a
61 # nice comparison of the two sets. Unfortunately, during testing
62 # (using python 2.3.3 and 2.3.4) the following code went into
63 # a dead loop under windows. I'll pursue this later.
64 # diff = difflib.ndiff(res, exp)
68 # print string.strip(line)
71 # the following simple compare is fine for when the two data sets
72 # (actual result vs. expected result) are equal, which should be true for
73 # us. Unfortunately, if the test fails it's not nice at all.
77 print 'Length of expected is %d, result is %d' % (el, rl)
79 for i in range(min(el, rl)):
80 if string.strip(res[i]) != string.strip(exp[i]):
81 print '+:%s-:%s' % (res[i], exp[i])
84 for i in range(rl, el):
88 for i in range (el, rl):
93 # Separate threads to handle stdout and stderr are created to run this function
94 def readPfile(file, list, flag):
95 data = file.readlines() # no call by reference, so I cheat
101 # This routine runs the test program (e.g. xmllint)
102 def runOneTest(testDescription, filename, inbase, errbase):
103 if 'execpath' in testDescription:
104 dir = testDescription['execpath'] + '/'
107 cmd = os.path.abspath(dir + testDescription['testprog'])
108 if 'flag' in testDescription:
109 for f in string.split(testDescription['flag']):
111 if 'stdin' not in testDescription:
112 cmd += ' ' + inbase + filename
113 if 'extarg' in testDescription:
114 cmd += ' ' + testDescription['extarg']
118 if 'resext' in testDescription:
119 if testDescription['resext'] == 'None':
122 ext = '.' + testDescription['resext']
127 fname = errbase + filename + ext
128 expout = open(fname, 'rt')
130 print "Can't open result file %s - bypassing test" % fname
134 if 'reserrext' in testDescription:
135 if testDescription['reserrext'] == 'None':
138 if len(testDescription['reserrext'])>0:
139 ext = '.' + testDescription['reserrext']
146 fname = errbase + filename + ext
147 experr = open(fname, 'rt')
153 pin, pout, perr = os.popen3(cmd)
154 if 'stdin' in testDescription:
155 infile = open(inbase + filename, 'rt')
156 pin.writelines(infile.readlines())
160 # popen is great fun, but can lead to the old "deadly embrace", because
161 # synchronizing the writing (by the task being run) of stdout and stderr
162 # with respect to the reading (by this task) is basically impossible. I
163 # tried several ways to cheat, but the only way I have found which works
164 # is to do a *very* elementary multi-threading approach. We can only hope
165 # that Python threads are implemented on the target system (it's okay for
168 th1Flag = [] # flags to show when threads finish
170 outfile = [] # lists to contain the pipe data
172 th1 = thread.start_new_thread(readPfile, (pout, outfile, th1Flag))
173 th2 = thread.start_new_thread(readPfile, (perr, errfile, th2Flag))
174 while (len(th1Flag)==0) or (len(th2Flag)==0):
177 ret = compFiles(outfile, expout, inbase, 'test/')
179 print 'trouble with %s' % cmd
181 if len(outfile) != 0:
184 print 'trouble with %s' % cmd
186 ret = compFiles(errfile, experr, inbase, 'test/')
188 print 'trouble with %s' % cmd
191 if len(errfile) != 0:
194 print 'trouble with %s' % cmd
196 if 'stdin' not in testDescription:
199 # This routine is called by the parameter decoding routine whenever the end of a
200 # 'test' section is encountered. Depending upon file globbing, a large number of
201 # individual tests may be run.
202 def runTest(description):
203 testDescription = defaultParams.copy() # set defaults
204 testDescription.update(description) # override with current ent
205 if 'testname' in testDescription:
206 print "## %s" % testDescription['testname']
207 if not 'file' in testDescription:
208 print "No file specified - can't run this test!"
210 # Set up the source and results directory paths from the decoded params
212 if 'srcdir' in testDescription:
213 dir += testDescription['srcdir'] + '/'
214 if 'srcsub' in testDescription:
215 dir += testDescription['srcsub'] + '/'
218 if 'resdir' in testDescription:
219 rdir += testDescription['resdir'] + '/'
220 if 'ressub' in testDescription:
221 rdir += testDescription['ressub'] + '/'
223 testFiles = glob.glob(os.path.abspath(dir + testDescription['file']))
225 print "No files result from '%s'" % testDescription['file']
228 # Some test programs just don't work (yet). For now we exclude them.
231 if 'exclfile' in testDescription:
232 for f in string.split(testDescription['exclfile']):
233 glb = glob.glob(dir + f)
235 excl.append(os.path.abspath(g))
237 # Run the specified test program
239 if not os.path.isdir(f):
242 runOneTest(testDescription, os.path.basename(f), dir, rdir)
245 # The following classes are used with the xmlreader interface to interpret the
246 # parameter file. Once a test section has been identified, runTest is called
247 # with a dictionary containing the parsed results of the interpretation.
251 curText = '' # accumulates text content of parameter
253 def addToDict(self, key):
254 txt = string.strip(self.curText)
257 if key not in defaultParams:
258 defaultParams[key] = txt
260 defaultParams[key] += ' ' + txt
262 def processNode(self, reader, curClass):
263 if reader.Depth() == 2:
264 if reader.NodeType() == 1:
265 self.curText = '' # clear the working variable
266 elif reader.NodeType() == 15:
267 if (reader.Name() != '#text') and (reader.Name() != '#comment'):
268 self.addToDict(reader.Name())
269 elif reader.Depth() == 3:
270 if reader.Name() == '#text':
271 self.curText += reader.Value()
273 elif reader.NodeType() == 15: # end of element
274 print "Defaults have been set to:"
275 for k in defaultParams.keys():
276 print " %s : '%s'" % (k, defaultParams[k])
277 curClass = rootClass()
283 self.testParams = {} # start with an empty set of params
284 self.curText = '' # and empty text
286 def addToDict(self, key):
287 data = string.strip(self.curText)
288 if key not in self.testParams:
289 self.testParams[key] = data
291 if self.testParams[key] != '':
293 self.testParams[key] += data
295 def processNode(self, reader, curClass):
296 if reader.Depth() == 2:
297 if reader.NodeType() == 1:
298 self.curText = '' # clear the working variable
299 if reader.Name() not in self.testParams:
300 self.testParams[reader.Name()] = ''
301 elif reader.NodeType() == 15:
302 if (reader.Name() != '#text') and (reader.Name() != '#comment'):
303 self.addToDict(reader.Name())
304 elif reader.Depth() == 3:
305 if reader.Name() == '#text':
306 self.curText += reader.Value()
308 elif reader.NodeType() == 15: # end of element
309 runTest(self.testParams)
310 curClass = rootClass()
315 def processNode(self, reader, curClass):
316 if reader.Depth() == 0:
318 if reader.Depth() != 1:
319 print "Unexpected junk: Level %d, type %d, name %s" % (
320 reader.Depth(), reader.NodeType(), reader.Name())
322 if reader.Name() == 'test':
323 curClass = testClass()
324 curClass.testParams = {}
325 elif reader.Name() == 'defaults':
326 curClass = testDefaults()
329 def streamFile(filename):
331 reader = libxml2.newTextReaderFilename(filename)
333 print "unable to open %s" % (filename)
336 curClass = rootClass()
339 curClass = curClass.processNode(reader, curClass)
343 print "%s : failed to parse" % (filename)
345 # OK, we're finished with all the routines. Now for the main program:-
346 if len(sys.argv) != 2:
347 print "Usage: maketest {filename}"
350 streamFile(sys.argv[1])