36465744c37bd9335a71b3243307f1dd3abb7f53
[profile/ivi/opencv.git] / modules / ts / misc / run.py
1 import sys, os, platform, xml, re, tempfile, glob, datetime, getpass
2 from optparse import OptionParser
3 from subprocess import Popen, PIPE
4
5 hostos = os.name # 'nt', 'posix'
6 hostmachine = platform.machine() # 'x86', 'AMD64', 'x86_64'
7 nameprefix = "opencv_perf_"
8
9 parse_patterns = (
10   {'name': "has_perf_tests",     'default': "OFF",      'pattern': re.compile("^BUILD_PERF_TESTS:BOOL=(ON)$")},
11   {'name': "cmake_home",         'default': None,       'pattern': re.compile("^CMAKE_HOME_DIRECTORY:INTERNAL=(.+)$")},
12   {'name': "opencv_home",        'default': None,       'pattern': re.compile("^OpenCV_SOURCE_DIR:STATIC=(.+)$")},
13   {'name': "tests_dir",          'default': None,       'pattern': re.compile("^EXECUTABLE_OUTPUT_PATH:PATH=(.+)$")},
14   {'name': "build_type",         'default': "Release",  'pattern': re.compile("^CMAKE_BUILD_TYPE:STRING=(.*)$")},
15   {'name': "svnversion_path",    'default': None,       'pattern': re.compile("^SVNVERSION_PATH:FILEPATH=(.*)$")},
16   {'name': "cxx_flags",          'default': None,       'pattern': re.compile("^CMAKE_CXX_FLAGS:STRING=(.*)$")},
17   {'name': "cxx_flags_debug",    'default': None,       'pattern': re.compile("^CMAKE_CXX_FLAGS_DEBUG:STRING=(.*)$")},
18   {'name': "cxx_flags_release",  'default': None,       'pattern': re.compile("^CMAKE_CXX_FLAGS_RELEASE:STRING=(.*)$")},
19   {'name': "ndk_path",           'default': None,       'pattern': re.compile("^ANDROID_NDK(?:_TOOLCHAIN_ROOT)?:PATH=(.*)$")},
20   {'name': "arm_target",         'default': None,       'pattern': re.compile("^ARM_TARGET:INTERNAL=(.*)$")},
21   {'name': "android_executable", 'default': None,       'pattern': re.compile("^ANDROID_EXECUTABLE:FILEPATH=(.*android.*)$")},
22   {'name': "is_x64",             'default': "OFF",      'pattern': re.compile("^CUDA_64_BIT_DEVICE_CODE:BOOL=(ON)$")},#ugly(
23   {'name': "cmake_generator",    'default': None,       'pattern': re.compile("^CMAKE_GENERATOR:INTERNAL=(.+)$")},
24   {'name': "cxx_compiler",       'default': None,       'pattern': re.compile("^CMAKE_CXX_COMPILER:FILEPATH=(.+)$")},
25   {'name': "with_cuda",          'default': "OFF",      'pattern': re.compile("^WITH_CUDA:BOOL=(ON)$")},
26   {'name': "cuda_library",       'default': None,       'pattern': re.compile("^CUDA_CUDA_LIBRARY:FILEPATH=(.+)$")},
27 )
28
29 def query_yes_no(stdout, question, default="yes"):
30     valid = {"yes":True, "y":True, "ye":True, "no":False, "n":False}
31     if default == None:
32         prompt = " [y/n] "
33     elif default == "yes":
34         prompt = " [Y/n] "
35     elif default == "no":
36         prompt = " [y/N] "
37     else:
38         raise ValueError("invalid default answer: '%s'" % default)
39
40     while True:
41         stdout.write(os.linesep + question + prompt)
42         choice = raw_input().lower()
43         if default is not None and choice == '':
44             return valid[default]
45         elif choice in valid:
46             return valid[choice]
47         else:
48             stdout.write("Please respond with 'yes' or 'no' "\
49                              "(or 'y' or 'n').\n")
50                              
51 def getRunningProcessExePathByName_win32(name):
52     from ctypes import windll, POINTER, pointer, Structure, sizeof
53     from ctypes import c_long , c_int , c_uint , c_char , c_ubyte , c_char_p , c_void_p
54     
55     class PROCESSENTRY32(Structure):
56         _fields_ = [ ( 'dwSize' , c_uint ) , 
57                     ( 'cntUsage' , c_uint) ,
58                     ( 'th32ProcessID' , c_uint) ,
59                     ( 'th32DefaultHeapID' , c_uint) ,
60                     ( 'th32ModuleID' , c_uint) ,
61                     ( 'cntThreads' , c_uint) ,
62                     ( 'th32ParentProcessID' , c_uint) ,
63                     ( 'pcPriClassBase' , c_long) ,
64                     ( 'dwFlags' , c_uint) ,
65                     ( 'szExeFile' , c_char * 260 ) , 
66                     ( 'th32MemoryBase' , c_long) ,
67                     ( 'th32AccessKey' , c_long ) ]
68                     
69     class MODULEENTRY32(Structure):
70         _fields_ = [ ( 'dwSize' , c_long ) , 
71                     ( 'th32ModuleID' , c_long ),
72                     ( 'th32ProcessID' , c_long ),
73                     ( 'GlblcntUsage' , c_long ),
74                     ( 'ProccntUsage' , c_long ) ,
75                     ( 'modBaseAddr' , c_long ) ,
76                     ( 'modBaseSize' , c_long ) , 
77                     ( 'hModule' , c_void_p ) ,
78                     ( 'szModule' , c_char * 256 ),
79                     ( 'szExePath' , c_char * 260 ) ]
80                 
81     TH32CS_SNAPPROCESS = 2
82     TH32CS_SNAPMODULE = 0x00000008
83     
84     ## CreateToolhelp32Snapshot
85     CreateToolhelp32Snapshot= windll.kernel32.CreateToolhelp32Snapshot
86     CreateToolhelp32Snapshot.reltype = c_long
87     CreateToolhelp32Snapshot.argtypes = [ c_int , c_int ]
88     ## Process32First
89     Process32First = windll.kernel32.Process32First
90     Process32First.argtypes = [ c_void_p , POINTER( PROCESSENTRY32 ) ]
91     Process32First.rettype = c_int
92     ## Process32Next
93     Process32Next = windll.kernel32.Process32Next
94     Process32Next.argtypes = [ c_void_p , POINTER(PROCESSENTRY32) ]
95     Process32Next.rettype = c_int
96     ## CloseHandle
97     CloseHandle = windll.kernel32.CloseHandle
98     CloseHandle.argtypes = [ c_void_p ]
99     CloseHandle.rettype = c_int
100     ## Module32First
101     Module32First = windll.kernel32.Module32First
102     Module32First.argtypes = [ c_void_p , POINTER(MODULEENTRY32) ]
103     Module32First.rettype = c_int
104                     
105     hProcessSnap = c_void_p(0)
106     hProcessSnap = CreateToolhelp32Snapshot( TH32CS_SNAPPROCESS , 0 )
107
108     pe32 = PROCESSENTRY32()
109     pe32.dwSize = sizeof( PROCESSENTRY32 )
110     ret = Process32First( hProcessSnap , pointer( pe32 ) )
111     path = None
112
113     while ret :
114         if name + ".exe" == pe32.szExeFile:
115             hModuleSnap = c_void_p(0)
116             me32 = MODULEENTRY32()
117             me32.dwSize = sizeof( MODULEENTRY32 )
118             hModuleSnap = CreateToolhelp32Snapshot( TH32CS_SNAPMODULE, pe32.th32ProcessID )
119
120             ret = Module32First( hModuleSnap, pointer(me32) )
121             path = me32.szExePath
122             CloseHandle( hModuleSnap )
123             if path:
124                 break
125         ret = Process32Next( hProcessSnap, pointer(pe32) )
126     CloseHandle( hProcessSnap )
127     return path
128
129 def getRunningProcessExePathByName_posix(name):
130     pids= [pid for pid in os.listdir('/proc') if pid.isdigit()]
131     for pid in pids:
132         try:
133             path = os.readlink(os.path.join('/proc', pid, 'exe'))
134             if path and path.endswith(name):  
135                 return path
136         except:
137             pass
138
139 def getRunningProcessExePathByName(name):
140     try:
141         if hostos == "nt":
142             return getRunningProcessExePathByName_win32(name)
143         elif hostos == "posix":
144             return getRunningProcessExePathByName_posix(name)
145         else:
146             return None
147     except:
148         return None
149         
150 class RunInfo(object):
151     def __init__(self, path, configuration = None):
152         self.path = path
153         self.error = None
154         for p in parse_patterns:
155             setattr(self, p["name"], p["default"])
156         cachefile = open(os.path.join(path, "CMakeCache.txt"), "rt")
157         try:
158             for l in cachefile.readlines():
159                 ll = l.strip()
160                 if not ll or ll.startswith("#"):
161                     continue
162                 for p in parse_patterns:
163                     match = p["pattern"].match(ll)
164                     if match:
165                         value = match.groups()[0]
166                         if value and not value.endswith("-NOTFOUND"):
167                             setattr(self, p["name"], value)
168         except:
169             pass
170         cachefile.close()
171         
172         # fix empty tests dir
173         if not self.tests_dir:
174             self.tests_dir = self.path
175         # add path to adb
176         if self.android_executable:
177             self.adb = os.path.join(os.path.dirname(os.path.dirname(self.android_executable)), ("platform-tools/adb","platform-tools/adb.exe")[hostos == 'nt'])
178             if not os.path.isfile(self.adb) or not os.access(self.adb, os.X_OK):
179                 self.adb = None
180         else:
181             self.adb = None
182
183         # detect target platform    
184         if self.android_executable or self.arm_target or self.ndk_path:
185             self.targetos = "android"
186         else:
187             self.targetos = hostos
188             
189         if self.targetos == "android":
190             # fix adb tool location
191             if not self.adb:
192                 self.adb = getRunningProcessExePathByName("adb")
193             if not self.adb:
194                 try:
195                     output = Popen(["adb", "devices"], stdout=PIPE, stderr=PIPE).communicate()
196                     self.adb = "adb"
197                 except OSError:
198                     pass
199             else:
200                 try:
201                     output = Popen([self.adb, "devices"], stdout=PIPE, stderr=PIPE).communicate()
202                 except OSError:
203                     self.adb = None
204
205         # fix has_perf_tests param
206         self.has_perf_tests = self.has_perf_tests == "ON"
207         # fix is_x64 flag
208         self.is_x64 = self.is_x64 == "ON"
209         if not self.is_x64 and ("X64" in "%s %s %s" % (self.cxx_flags, self.cxx_flags_release, self.cxx_flags_debug) or "Win64" in self.cmake_generator):
210             self.is_x64 = True
211
212         # fix test path
213         if "Visual Studio" in self.cmake_generator:
214             if configuration:
215                 self.tests_dir = os.path.join(self.tests_dir, configuration)
216             else:
217                 self.tests_dir = os.path.join(self.tests_dir, self.build_type)
218         elif not self.is_x64 and self.cxx_compiler:
219             #one more attempt to detect x64 compiler
220             try:
221                 output = Popen([self.cxx_compiler, "-v"], stdout=PIPE, stderr=PIPE).communicate()
222                 if not output[0] and "x86_64" in output[1]:
223                     self.is_x64 = True
224             except OSError:
225                 pass
226
227         # detect target arch
228         if self.targetos == "android":
229             self.targetarch = "arm"
230         elif self.is_x64 and hostmachine in ["AMD64", "x86_64"]:
231             self.targetarch = "x64"
232         elif hostmachine in ["x86", "AMD64", "x86_64"]:
233             self.targetarch = "x86"
234         else:
235             self.targetarch = "unknown"
236             
237         # fix CUDA attributes
238         self.with_cuda = self.with_cuda == "ON"
239         if self.cuda_library and self.cuda_library.endswith("-NOTFOUND"):
240             self.cuda_library = None
241         self.has_cuda = self.with_cuda and self.cuda_library and self.targetarch in ["x86", "x64"]
242             
243         self.hardware = None
244         
245         self.getSvnVersion(self.cmake_home, "cmake_home_svn")
246         if self.opencv_home == self.cmake_home:
247             self.opencv_home_svn = self.cmake_home_svn
248         else:
249             self.getSvnVersion(self.opencv_home, "opencv_home_svn")
250             
251         self.tests = self.getAvailableTestApps()
252         
253     def getSvnVersion(self, path, name):
254         if not path:
255             setattr(self, name, None)
256             return
257         if not self.svnversion_path and hostos == 'nt':
258             self.tryGetSvnVersionWithTortoise(path, name)
259         else:
260             svnversion = self.svnversion_path
261             if not svnversion:
262                 svnversion = "svnversion"
263             try:
264                 output = Popen([svnversion, "-n", path], stdout=PIPE, stderr=PIPE).communicate()
265                 if not output[1]:
266                     setattr(self, name, output[0])
267                 else:
268                     setattr(self, name, None)
269             except OSError:
270                 setattr(self, name, None)
271         
272     def tryGetSvnVersionWithTortoise(self, path, name):
273         try:
274             wcrev = "SubWCRev.exe"
275             dir = tempfile.mkdtemp()
276             #print dir
277             tmpfilename = os.path.join(dir, "svn.tmp")
278             tmpfilename2 = os.path.join(dir, "svn_out.tmp")
279             tmpfile = open(tmpfilename, "w")
280             tmpfile.write("$WCRANGE$$WCMODS?M:$")
281             tmpfile.close();
282             output = Popen([wcrev, path, tmpfilename, tmpfilename2, "-f"], stdout=PIPE, stderr=PIPE).communicate()
283             if "is not a working copy" in output[0]:
284                 version = "exported"
285             else:
286                 tmpfile = open(tmpfilename2, "r")
287                 version = tmpfile.read()
288                 tmpfile.close()
289             setattr(self, name, version)
290         except:
291             setattr(self, name, None)
292         finally:
293             if dir:
294                 import shutil
295                 shutil.rmtree(dir)
296                 
297     def isTest(self, fullpath):
298         if not os.path.isfile(fullpath):
299             return False
300         if hostos == self.targetos:
301             return os.access(fullpath, os.X_OK)
302         return True
303                 
304     def getAvailableTestApps(self):
305         if self.tests_dir and os.path.isdir(self.tests_dir):
306             files = glob.glob(os.path.join(self.tests_dir, nameprefix + "*"))
307             if self.targetos == hostos:
308                 files = [f for f in files if self.isTest(f)]
309             return files
310         return []
311     
312     def getLogName(self, app, timestamp):
313         app = os.path.basename(app)
314         if app.endswith(".exe"):
315             app = app[:-4]
316         if app.startswith(nameprefix):
317             app = app[len(nameprefix):]
318         if self.cmake_home_svn:
319             if self.cmake_home_svn == self.opencv_home_svn:
320                 rev = self.cmake_home_svn
321             elif self.opencv_home_svn:
322                 rev = self.cmake_home_svn + "-" + self.opencv_home_svn
323             else:
324                 rev = self.cmake_home_svn
325         else:
326             rev = None
327         if rev:
328             rev = rev.replace(":","to") + "_" 
329         else:
330             rev = ""
331         if self.hardware:
332             hw = str(self.hardware).replace(" ", "_") + "_"
333         elif self.has_cuda:
334             hw = "CUDA_"
335         else:
336             hw = ""
337         #stamp = timestamp.strftime("%Y%m%dT%H%M%S")
338         stamp = timestamp.strftime("%Y-%m-%d--%H-%M-%S")
339         return "%s_%s_%s_%s%s%s.xml" %(app, self.targetos, self.targetarch, hw, rev, stamp)
340         
341     def getTest(self, name):
342         # full path
343         if self.isTest(name):
344             return name
345         
346         # name only
347         fullname = os.path.join(self.tests_dir, name)
348         if self.isTest(fullname):
349             return fullname
350         
351         # name without extension
352         fullname += ".exe"
353         if self.isTest(fullname):
354             return fullname
355         
356         # short name for OpenCV tests
357         for t in self.tests:
358             if t == name:
359                 return t
360             fname = os.path.basename(t)
361             if fname == name:
362                 return t
363             if fname.endswith(".exe"):
364                 fname = fname[:-4]
365             if fname == name:
366                 return t
367             if fname.startswith(nameprefix):
368                 fname = fname[len(nameprefix):]
369             if fname == name:
370                 return t
371         return None
372     
373     def runAdb(self, *args):
374         cmd = [self.adb]
375         cmd.extend(args)
376         try:
377             output = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()
378             if not output[1]:
379                 return output[0]
380             self.error = output[1]
381             print self.error
382         except OSError:
383             pass
384         return None
385     
386     def isRunnable(self):
387         #if not self.has_perf_tests or not self.tests:
388             #self.error = "Performance tests are not built (at %s)" % self.path
389             #return False
390         if self.targetarch == "x64" and hostmachine == "x86":
391             self.error = "Target architecture is incompatible with current platform (at %s)" % self.path
392             return False
393         if self.targetos == "android":
394             if not self.adb:
395                 self.error = "Could not find adb executable (for %s)" % self.path
396                 return False
397             adb_res = self.runAdb("devices")
398             if not adb_res:
399                 self.error = "Could not run adb command: %s (for %s)" % (self.error, self.path)
400                 return False
401             connected_devices = len(re.findall(r"^[^ \t]+[ \t]+device\r?$", adb_res, re.MULTILINE))
402             if connected_devices == 0:
403                 self.error = "No Android device connected (for %s)" % self.path
404                 return False
405             if connected_devices > 1:
406                 self.error = "Too many (%s) devices are connected. Single device is required. (for %s)" % (connected_devices, self.path)
407                 return False
408             if "armeabi-v7a" in self.arm_target:
409                 adb_res = self.runAdb("shell", "cat /proc/cpuinfo")
410                 if not adb_res:
411                     self.error = "Could not get info about Android platform: %s (for %s)" % (self.error, self.path)
412                     return False
413                 if "ARMv7" not in adb_res:
414                     self.error = "Android device does not support ARMv7 commands, but tests are built for armeabi-v7a (for %s)" % self.path
415                     return False
416                 if "NEON" in self.arm_target and "neon" not in adb_res:
417                     self.error = "Android device has no NEON, but tests are built for %s (for %s)" % (self.arm_target, self.path)
418                     return False
419                 hw = re.search(r"^Hardware[ \t]*:[ \t]*(.*?)$", adb_res, re.MULTILINE)
420                 if hw:
421                     self.hardware = hw.groups()[0].strip()
422         return True
423     
424     def runTest(self, path, workingDir, _stdout, _stderr, args = []):
425         if self.error:
426             return
427         args = args[:]
428         timestamp = datetime.datetime.now()
429         logfile = self.getLogName(path, timestamp)
430         exe = os.path.abspath(path)
431         
432         userlog = [a for a in args if a.startswith("--gtest_output=")]
433         if len(userlog) == 0:
434             args.append("--gtest_output=xml:" + logfile)
435         else:
436             logfile = userlog[userlog[0].find(":")+1:]
437         
438         if self.targetos == "android":
439             try:
440                 andoidcwd = "/data/bin/" + getpass.getuser().replace(" ","") + "_perf/"
441                 exename = os.path.basename(exe)
442                 androidexe = andoidcwd + exename
443                 #upload
444                 print >> _stderr, "Uploading", exename, "to device..."
445                 output = Popen([self.adb, "push", exe, androidexe], stdout=_stdout, stderr=_stderr).wait()
446                 if output != 0:
447                     print >> _stderr, "adb finishes unexpectedly with error code", output
448                     return
449                 #chmod
450                 print >> _stderr, "Changing mode of ", androidexe
451                 output = Popen([self.adb, "shell", "chmod 777 " + androidexe], stdout=_stdout, stderr=_stderr).wait()
452                 if output != 0:
453                     print >> _stderr, "adb finishes unexpectedly with error code", output
454                     return
455                 #run
456                 command = exename + " " + " ".join(args)
457                 print >> _stderr, "Running:", command
458                 Popen([self.adb, "shell", "export OPENCV_TEST_DATA_PATH=" + self.test_data_path + "&& cd " + andoidcwd + "&& ./" + command], stdout=_stdout, stderr=_stderr).wait()
459                 # try get log
460                 print >> _stderr, "Pulling", logfile, "from device..."
461                 hostlogpath = os.path.join(workingDir, logfile)
462                 output = Popen([self.adb, "pull", andoidcwd + logfile, hostlogpath], stdout=_stdout, stderr=_stderr).wait()
463                 if output != 0:
464                     print >> _stderr, "adb finishes unexpectedly with error code", output
465                     return
466                 #rm log
467                 Popen([self.adb, "shell", "rm " + andoidcwd + logfile], stdout=_stdout, stderr=_stderr).wait()
468             except OSError:
469                 pass
470             if os.path.isfile(hostlogpath):
471                 return hostlogpath
472             return None
473         else:
474             cmd = [exe]
475             cmd.extend(args)
476             print >> _stderr, "Running:", " ".join(cmd)
477             try: 
478                 Popen(cmd, stdout=_stdout, stderr=_stderr, cwd = workingDir).wait()
479             except OSError:
480                 pass
481             
482             logpath = os.path.join(workingDir, logfile)
483             if os.path.isfile(logpath):
484                 return logpath
485             return None
486             
487     def runTests(self, tests, _stdout, _stderr, workingDir, args = []):
488         if self.error:
489             return []
490         if not tests:
491             tests = self.tests
492         logs = []
493         for test in tests:
494             t = self.getTest(test)
495             if t:
496                 logfile = self.runTest(t, workingDir, _stdout, _stderr, args)
497                 if logfile:
498                     logs.append(os.path.relpath(logfile, "."))
499             else:
500                 print >> _stderr, "Error: Test \"%s\" is not found in %s" % (test, self.tests_dir)
501         return logs
502
503 if __name__ == "__main__":
504     test_args = [a for a in sys.argv if a.startswith("--perf_") or a.startswith("--gtest_")]
505     argv =      [a for a in sys.argv if not(a.startswith("--perf_") or a.startswith("--gtest_"))]
506     
507     parser = OptionParser()
508     parser.add_option("-t", "--tests", dest="tests", help="comma-separated list of modules to test", metavar="SUITS", default="")
509     parser.add_option("-w", "--cwd", dest="cwd", help="working directory for tests", metavar="PATH", default=".")
510     parser.add_option("", "--android_test_data_path", dest="test_data_path", help="OPENCV_TEST_DATA_PATH for Android run", metavar="PATH", default="/sdcard/opencv_testdata/")
511     parser.add_option("", "--configuration", dest="configuration", help="force Debug or Release donfiguration", metavar="CFG", default="")
512     
513     (options, args) = parser.parse_args(argv)
514     
515     run_args = []
516     
517     for path in args:
518         path = os.path.abspath(path)
519         while (True):
520             if os.path.isdir(path) and os.path.isfile(os.path.join(path, "CMakeCache.txt")):
521                 run_args.append(path)
522                 break
523             npath = os.path.dirname(path)
524             if npath == path:
525                 break
526             path = npath
527     
528     if len(run_args) == 0:
529         print >> sys.stderr, "Usage:\n", os.path.basename(sys.argv[0]), "<build_path>"
530         exit(1)
531         
532     tests = [s.strip() for s in options.tests.split(",") if s]
533             
534     if len(tests) != 1 or len(run_args) != 1:
535         #remove --gtest_output from params
536         test_args = [a for a in test_args if not a.startswith("--gtest_output=")]
537     
538     logs = []
539     for path in run_args:
540         info = RunInfo(path, options.configuration)
541         #print vars(info),"\n"
542         if not info.isRunnable():
543             print >> sys.stderr, "Error:", info.error
544         else:
545             info.test_data_path = options.test_data_path
546             logs.extend(info.runTests(tests, sys.stdout, sys.stderr, options.cwd, test_args))
547
548     if logs:            
549         print >> sys.stderr, "Collected:", " ".join(logs)