1 import sys, os, platform, xml, re, tempfile, glob, datetime, getpass
2 from optparse import OptionParser
3 from subprocess import Popen, PIPE
5 hostos = os.name # 'nt', 'posix'
6 hostmachine = platform.machine() # 'x86', 'AMD64', 'x86_64'
7 nameprefix = "opencv_perf_"
10 {'name': "has_perf_tests", 'default': "OFF", 'pattern': re.compile("^BUILD_PERF_TESTS:BOOL=(ON)$")},
11 {'name': "cmake_home", 'default': None, 'pattern': re.compile("^CMAKE_HOME_DIRECTORY:INTERNAL=(.+)$")},
12 {'name': "opencv_home", 'default': None, 'pattern': re.compile("^OpenCV_SOURCE_DIR:STATIC=(.+)$")},
13 {'name': "tests_dir", 'default': None, 'pattern': re.compile("^EXECUTABLE_OUTPUT_PATH:PATH=(.+)$")},
14 {'name': "build_type", 'default': "Release", 'pattern': re.compile("^CMAKE_BUILD_TYPE:STRING=(.*)$")},
15 {'name': "svnversion_path", 'default': None, 'pattern': re.compile("^SVNVERSION_PATH:FILEPATH=(.*)$")},
16 {'name': "cxx_flags", 'default': None, 'pattern': re.compile("^CMAKE_CXX_FLAGS:STRING=(.*)$")},
17 {'name': "cxx_flags_debug", 'default': None, 'pattern': re.compile("^CMAKE_CXX_FLAGS_DEBUG:STRING=(.*)$")},
18 {'name': "cxx_flags_release", 'default': None, 'pattern': re.compile("^CMAKE_CXX_FLAGS_RELEASE:STRING=(.*)$")},
19 {'name': "ndk_path", 'default': None, 'pattern': re.compile("^ANDROID_NDK(?:_TOOLCHAIN_ROOT)?:PATH=(.*)$")},
20 {'name': "arm_target", 'default': None, 'pattern': re.compile("^ARM_TARGET:INTERNAL=(.*)$")},
21 {'name': "android_executable", 'default': None, 'pattern': re.compile("^ANDROID_EXECUTABLE:FILEPATH=(.*android.*)$")},
22 {'name': "is_x64", 'default': "OFF", 'pattern': re.compile("^CUDA_64_BIT_DEVICE_CODE:BOOL=(ON)$")},#ugly(
23 {'name': "cmake_generator", 'default': None, 'pattern': re.compile("^CMAKE_GENERATOR:INTERNAL=(.+)$")},
24 {'name': "cxx_compiler", 'default': None, 'pattern': re.compile("^CMAKE_CXX_COMPILER:FILEPATH=(.+)$")},
25 {'name': "with_cuda", 'default': "OFF", 'pattern': re.compile("^WITH_CUDA:BOOL=(ON)$")},
26 {'name': "cuda_library", 'default': None, 'pattern': re.compile("^CUDA_CUDA_LIBRARY:FILEPATH=(.+)$")},
29 def query_yes_no(stdout, question, default="yes"):
30 valid = {"yes":True, "y":True, "ye":True, "no":False, "n":False}
33 elif default == "yes":
38 raise ValueError("invalid default answer: '%s'" % default)
41 stdout.write(os.linesep + question + prompt)
42 choice = raw_input().lower()
43 if default is not None and choice == '':
48 stdout.write("Please respond with 'yes' or 'no' "\
51 def getRunningProcessExePathByName_win32(name):
52 from ctypes import windll, POINTER, pointer, Structure, sizeof
53 from ctypes import c_long , c_int , c_uint , c_char , c_ubyte , c_char_p , c_void_p
55 class PROCESSENTRY32(Structure):
56 _fields_ = [ ( 'dwSize' , c_uint ) ,
57 ( 'cntUsage' , c_uint) ,
58 ( 'th32ProcessID' , c_uint) ,
59 ( 'th32DefaultHeapID' , c_uint) ,
60 ( 'th32ModuleID' , c_uint) ,
61 ( 'cntThreads' , c_uint) ,
62 ( 'th32ParentProcessID' , c_uint) ,
63 ( 'pcPriClassBase' , c_long) ,
64 ( 'dwFlags' , c_uint) ,
65 ( 'szExeFile' , c_char * 260 ) ,
66 ( 'th32MemoryBase' , c_long) ,
67 ( 'th32AccessKey' , c_long ) ]
69 class MODULEENTRY32(Structure):
70 _fields_ = [ ( 'dwSize' , c_long ) ,
71 ( 'th32ModuleID' , c_long ),
72 ( 'th32ProcessID' , c_long ),
73 ( 'GlblcntUsage' , c_long ),
74 ( 'ProccntUsage' , c_long ) ,
75 ( 'modBaseAddr' , c_long ) ,
76 ( 'modBaseSize' , c_long ) ,
77 ( 'hModule' , c_void_p ) ,
78 ( 'szModule' , c_char * 256 ),
79 ( 'szExePath' , c_char * 260 ) ]
81 TH32CS_SNAPPROCESS = 2
82 TH32CS_SNAPMODULE = 0x00000008
84 ## CreateToolhelp32Snapshot
85 CreateToolhelp32Snapshot= windll.kernel32.CreateToolhelp32Snapshot
86 CreateToolhelp32Snapshot.reltype = c_long
87 CreateToolhelp32Snapshot.argtypes = [ c_int , c_int ]
89 Process32First = windll.kernel32.Process32First
90 Process32First.argtypes = [ c_void_p , POINTER( PROCESSENTRY32 ) ]
91 Process32First.rettype = c_int
93 Process32Next = windll.kernel32.Process32Next
94 Process32Next.argtypes = [ c_void_p , POINTER(PROCESSENTRY32) ]
95 Process32Next.rettype = c_int
97 CloseHandle = windll.kernel32.CloseHandle
98 CloseHandle.argtypes = [ c_void_p ]
99 CloseHandle.rettype = c_int
101 Module32First = windll.kernel32.Module32First
102 Module32First.argtypes = [ c_void_p , POINTER(MODULEENTRY32) ]
103 Module32First.rettype = c_int
105 hProcessSnap = c_void_p(0)
106 hProcessSnap = CreateToolhelp32Snapshot( TH32CS_SNAPPROCESS , 0 )
108 pe32 = PROCESSENTRY32()
109 pe32.dwSize = sizeof( PROCESSENTRY32 )
110 ret = Process32First( hProcessSnap , pointer( pe32 ) )
114 if name + ".exe" == pe32.szExeFile:
115 hModuleSnap = c_void_p(0)
116 me32 = MODULEENTRY32()
117 me32.dwSize = sizeof( MODULEENTRY32 )
118 hModuleSnap = CreateToolhelp32Snapshot( TH32CS_SNAPMODULE, pe32.th32ProcessID )
120 ret = Module32First( hModuleSnap, pointer(me32) )
121 path = me32.szExePath
122 CloseHandle( hModuleSnap )
125 ret = Process32Next( hProcessSnap, pointer(pe32) )
126 CloseHandle( hProcessSnap )
129 def getRunningProcessExePathByName_posix(name):
130 pids= [pid for pid in os.listdir('/proc') if pid.isdigit()]
133 path = os.readlink(os.path.join('/proc', pid, 'exe'))
134 if path and path.endswith(name):
139 def getRunningProcessExePathByName(name):
142 return getRunningProcessExePathByName_win32(name)
143 elif hostos == "posix":
144 return getRunningProcessExePathByName_posix(name)
150 class RunInfo(object):
151 def __init__(self, path, configuration = None):
154 for p in parse_patterns:
155 setattr(self, p["name"], p["default"])
156 cachefile = open(os.path.join(path, "CMakeCache.txt"), "rt")
158 for l in cachefile.readlines():
160 if not ll or ll.startswith("#"):
162 for p in parse_patterns:
163 match = p["pattern"].match(ll)
165 value = match.groups()[0]
166 if value and not value.endswith("-NOTFOUND"):
167 setattr(self, p["name"], value)
172 # fix empty tests dir
173 if not self.tests_dir:
174 self.tests_dir = self.path
176 if self.android_executable:
177 self.adb = os.path.join(os.path.dirname(os.path.dirname(self.android_executable)), ("platform-tools/adb","platform-tools/adb.exe")[hostos == 'nt'])
178 if not os.path.isfile(self.adb) or not os.access(self.adb, os.X_OK):
183 # detect target platform
184 if self.android_executable or self.arm_target or self.ndk_path:
185 self.targetos = "android"
187 self.targetos = hostos
189 if self.targetos == "android":
190 # fix adb tool location
192 self.adb = getRunningProcessExePathByName("adb")
195 output = Popen(["adb", "devices"], stdout=PIPE, stderr=PIPE).communicate()
201 output = Popen([self.adb, "devices"], stdout=PIPE, stderr=PIPE).communicate()
205 # fix has_perf_tests param
206 self.has_perf_tests = self.has_perf_tests == "ON"
208 self.is_x64 = self.is_x64 == "ON"
209 if not self.is_x64 and ("X64" in "%s %s %s" % (self.cxx_flags, self.cxx_flags_release, self.cxx_flags_debug) or "Win64" in self.cmake_generator):
213 if "Visual Studio" in self.cmake_generator:
215 self.tests_dir = os.path.join(self.tests_dir, configuration)
217 self.tests_dir = os.path.join(self.tests_dir, self.build_type)
218 elif not self.is_x64 and self.cxx_compiler:
219 #one more attempt to detect x64 compiler
221 output = Popen([self.cxx_compiler, "-v"], stdout=PIPE, stderr=PIPE).communicate()
222 if not output[0] and "x86_64" in output[1]:
228 if self.targetos == "android":
229 self.targetarch = "arm"
230 elif self.is_x64 and hostmachine in ["AMD64", "x86_64"]:
231 self.targetarch = "x64"
232 elif hostmachine in ["x86", "AMD64", "x86_64"]:
233 self.targetarch = "x86"
235 self.targetarch = "unknown"
237 # fix CUDA attributes
238 self.with_cuda = self.with_cuda == "ON"
239 if self.cuda_library and self.cuda_library.endswith("-NOTFOUND"):
240 self.cuda_library = None
241 self.has_cuda = self.with_cuda and self.cuda_library and self.targetarch in ["x86", "x64"]
245 self.getSvnVersion(self.cmake_home, "cmake_home_svn")
246 if self.opencv_home == self.cmake_home:
247 self.opencv_home_svn = self.cmake_home_svn
249 self.getSvnVersion(self.opencv_home, "opencv_home_svn")
251 self.tests = self.getAvailableTestApps()
253 def getSvnVersion(self, path, name):
255 setattr(self, name, None)
257 if not self.svnversion_path and hostos == 'nt':
258 self.tryGetSvnVersionWithTortoise(path, name)
260 svnversion = self.svnversion_path
262 svnversion = "svnversion"
264 output = Popen([svnversion, "-n", path], stdout=PIPE, stderr=PIPE).communicate()
266 setattr(self, name, output[0])
268 setattr(self, name, None)
270 setattr(self, name, None)
272 def tryGetSvnVersionWithTortoise(self, path, name):
274 wcrev = "SubWCRev.exe"
275 dir = tempfile.mkdtemp()
277 tmpfilename = os.path.join(dir, "svn.tmp")
278 tmpfilename2 = os.path.join(dir, "svn_out.tmp")
279 tmpfile = open(tmpfilename, "w")
280 tmpfile.write("$WCRANGE$$WCMODS?M:$")
282 output = Popen([wcrev, path, tmpfilename, tmpfilename2, "-f"], stdout=PIPE, stderr=PIPE).communicate()
283 if "is not a working copy" in output[0]:
286 tmpfile = open(tmpfilename2, "r")
287 version = tmpfile.read()
289 setattr(self, name, version)
291 setattr(self, name, None)
297 def isTest(self, fullpath):
298 if not os.path.isfile(fullpath):
300 if hostos == self.targetos:
301 return os.access(fullpath, os.X_OK)
304 def getAvailableTestApps(self):
305 if self.tests_dir and os.path.isdir(self.tests_dir):
306 files = glob.glob(os.path.join(self.tests_dir, nameprefix + "*"))
307 if self.targetos == hostos:
308 files = [f for f in files if self.isTest(f)]
312 def getLogName(self, app, timestamp):
313 app = os.path.basename(app)
314 if app.endswith(".exe"):
316 if app.startswith(nameprefix):
317 app = app[len(nameprefix):]
318 if self.cmake_home_svn:
319 if self.cmake_home_svn == self.opencv_home_svn:
320 rev = self.cmake_home_svn
321 elif self.opencv_home_svn:
322 rev = self.cmake_home_svn + "-" + self.opencv_home_svn
324 rev = self.cmake_home_svn
328 rev = rev.replace(":","to") + "_"
332 hw = str(self.hardware).replace(" ", "_") + "_"
337 #stamp = timestamp.strftime("%Y%m%dT%H%M%S")
338 stamp = timestamp.strftime("%Y-%m-%d--%H-%M-%S")
339 return "%s_%s_%s_%s%s%s.xml" %(app, self.targetos, self.targetarch, hw, rev, stamp)
341 def getTest(self, name):
343 if self.isTest(name):
347 fullname = os.path.join(self.tests_dir, name)
348 if self.isTest(fullname):
351 # name without extension
353 if self.isTest(fullname):
356 # short name for OpenCV tests
360 fname = os.path.basename(t)
363 if fname.endswith(".exe"):
367 if fname.startswith(nameprefix):
368 fname = fname[len(nameprefix):]
373 def runAdb(self, *args):
377 output = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()
380 self.error = output[1]
386 def isRunnable(self):
387 #if not self.has_perf_tests or not self.tests:
388 #self.error = "Performance tests are not built (at %s)" % self.path
390 if self.targetarch == "x64" and hostmachine == "x86":
391 self.error = "Target architecture is incompatible with current platform (at %s)" % self.path
393 if self.targetos == "android":
395 self.error = "Could not find adb executable (for %s)" % self.path
397 adb_res = self.runAdb("devices")
399 self.error = "Could not run adb command: %s (for %s)" % (self.error, self.path)
401 connected_devices = len(re.findall(r"^[^ \t]+[ \t]+device\r?$", adb_res, re.MULTILINE))
402 if connected_devices == 0:
403 self.error = "No Android device connected (for %s)" % self.path
405 if connected_devices > 1:
406 self.error = "Too many (%s) devices are connected. Single device is required. (for %s)" % (connected_devices, self.path)
408 if "armeabi-v7a" in self.arm_target:
409 adb_res = self.runAdb("shell", "cat /proc/cpuinfo")
411 self.error = "Could not get info about Android platform: %s (for %s)" % (self.error, self.path)
413 if "ARMv7" not in adb_res:
414 self.error = "Android device does not support ARMv7 commands, but tests are built for armeabi-v7a (for %s)" % self.path
416 if "NEON" in self.arm_target and "neon" not in adb_res:
417 self.error = "Android device has no NEON, but tests are built for %s (for %s)" % (self.arm_target, self.path)
419 hw = re.search(r"^Hardware[ \t]*:[ \t]*(.*?)$", adb_res, re.MULTILINE)
421 self.hardware = hw.groups()[0].strip()
424 def runTest(self, path, workingDir, _stdout, _stderr, args = []):
428 timestamp = datetime.datetime.now()
429 logfile = self.getLogName(path, timestamp)
430 exe = os.path.abspath(path)
432 userlog = [a for a in args if a.startswith("--gtest_output=")]
433 if len(userlog) == 0:
434 args.append("--gtest_output=xml:" + logfile)
436 logfile = userlog[userlog[0].find(":")+1:]
438 if self.targetos == "android":
440 andoidcwd = "/data/bin/" + getpass.getuser().replace(" ","") + "_perf/"
441 exename = os.path.basename(exe)
442 androidexe = andoidcwd + exename
444 print >> _stderr, "Uploading", exename, "to device..."
445 output = Popen([self.adb, "push", exe, androidexe], stdout=_stdout, stderr=_stderr).wait()
447 print >> _stderr, "adb finishes unexpectedly with error code", output
450 print >> _stderr, "Changing mode of ", androidexe
451 output = Popen([self.adb, "shell", "chmod 777 " + androidexe], stdout=_stdout, stderr=_stderr).wait()
453 print >> _stderr, "adb finishes unexpectedly with error code", output
456 command = exename + " " + " ".join(args)
457 print >> _stderr, "Running:", command
458 Popen([self.adb, "shell", "export OPENCV_TEST_DATA_PATH=" + self.test_data_path + "&& cd " + andoidcwd + "&& ./" + command], stdout=_stdout, stderr=_stderr).wait()
460 print >> _stderr, "Pulling", logfile, "from device..."
461 hostlogpath = os.path.join(workingDir, logfile)
462 output = Popen([self.adb, "pull", andoidcwd + logfile, hostlogpath], stdout=_stdout, stderr=_stderr).wait()
464 print >> _stderr, "adb finishes unexpectedly with error code", output
467 Popen([self.adb, "shell", "rm " + andoidcwd + logfile], stdout=_stdout, stderr=_stderr).wait()
470 if os.path.isfile(hostlogpath):
476 print >> _stderr, "Running:", " ".join(cmd)
478 Popen(cmd, stdout=_stdout, stderr=_stderr, cwd = workingDir).wait()
482 logpath = os.path.join(workingDir, logfile)
483 if os.path.isfile(logpath):
487 def runTests(self, tests, _stdout, _stderr, workingDir, args = []):
494 t = self.getTest(test)
496 logfile = self.runTest(t, workingDir, _stdout, _stderr, args)
498 logs.append(os.path.relpath(logfile, "."))
500 print >> _stderr, "Error: Test \"%s\" is not found in %s" % (test, self.tests_dir)
503 if __name__ == "__main__":
504 test_args = [a for a in sys.argv if a.startswith("--perf_") or a.startswith("--gtest_")]
505 argv = [a for a in sys.argv if not(a.startswith("--perf_") or a.startswith("--gtest_"))]
507 parser = OptionParser()
508 parser.add_option("-t", "--tests", dest="tests", help="comma-separated list of modules to test", metavar="SUITS", default="")
509 parser.add_option("-w", "--cwd", dest="cwd", help="working directory for tests", metavar="PATH", default=".")
510 parser.add_option("", "--android_test_data_path", dest="test_data_path", help="OPENCV_TEST_DATA_PATH for Android run", metavar="PATH", default="/sdcard/opencv_testdata/")
511 parser.add_option("", "--configuration", dest="configuration", help="force Debug or Release donfiguration", metavar="CFG", default="")
513 (options, args) = parser.parse_args(argv)
518 path = os.path.abspath(path)
520 if os.path.isdir(path) and os.path.isfile(os.path.join(path, "CMakeCache.txt")):
521 run_args.append(path)
523 npath = os.path.dirname(path)
528 if len(run_args) == 0:
529 print >> sys.stderr, "Usage:\n", os.path.basename(sys.argv[0]), "<build_path>"
532 tests = [s.strip() for s in options.tests.split(",") if s]
534 if len(tests) != 1 or len(run_args) != 1:
535 #remove --gtest_output from params
536 test_args = [a for a in test_args if not a.startswith("--gtest_output=")]
539 for path in run_args:
540 info = RunInfo(path, options.configuration)
541 #print vars(info),"\n"
542 if not info.isRunnable():
543 print >> sys.stderr, "Error:", info.error
545 info.test_data_path = options.test_data_path
546 logs.extend(info.runTests(tests, sys.stdout, sys.stderr, options.cwd, test_args))
549 print >> sys.stderr, "Collected:", " ".join(logs)