3 import os, sys, fnmatch, re
5 sys.path.append("../modules/python/src2/")
6 sys.path.append("../modules/java/generator")
8 import hdr_parser as hp
9 import rst_parser as rp
11 rp.show_warnings = False
12 rp.show_errors = False
14 allmodules = rp.allmodules
15 DOCUMENTED_MARKER = "verified"
17 ERROR_001_NOTACLASS = 1
18 ERROR_002_NOTASTRUCT = 2
19 ERROR_003_INCORRECTBASE = 3
20 ERROR_004_MISSEDNAMESPACE = 4
21 ERROR_005_MISSINGPYFUNC = 5
22 ERROR_006_INVALIDPYOLDDOC = 6
23 ERROR_007_INVALIDPYDOC = 7
24 ERROR_008_CFUNCISNOTGLOBAL = 8
25 ERROR_009_OVERLOADNOTFOUND = 9
26 ERROR_010_UNKNOWNCLASS = 10
27 ERROR_011_UNKNOWNFUNC = 11
29 do_python_crosscheck = True
30 errors_disabled = [ERROR_004_MISSEDNAMESPACE]
32 doc_signatures_whitelist = [
34 "Matx", "Vec", "SparseMat_", "Scalar_", "Mat_", "Ptr", "Size_", "Point_", "Rect_", "Point3_",
35 "DataType", "detail::RotationWarperBase", "flann::Index_", "CalonderDescriptorExtractor",
36 "cuda::PtrStepSz", "cuda::PtrStep", "cuda::PtrElemStep_",
38 "CvArr", "CvFileStorage",
40 "InputArray", "OutputArray",
43 defines = ["cvGraphEdgeIdx", "cvFree", "CV_Assert", "cvSqrt", "cvGetGraphVtx", "cvGraphVtxIdx",
44 "cvCaptureFromFile", "cvCaptureFromCAM", "cvCalcBackProjectPatch", "cvCalcBackProject",
45 "cvGetHistValue_1D", "cvGetHistValue_2D", "cvGetHistValue_3D", "cvGetHistValue_nD",
46 "cvQueryHistValue_1D", "cvQueryHistValue_2D", "cvQueryHistValue_3D", "cvQueryHistValue_nD",
47 # not a real function but behaves as function
49 # ugly "virtual" functions from ml module
50 "CvStatModel::train", "CvStatModel::predict",
56 "StarDetector" : ["StarFeatureDetector"],
57 "MSER" : ["MserFeatureDetector"],
58 "GFTTDetector" : ["GoodFeaturesToTrackDetector"],
59 "cvCaptureFromFile" : ["cvCreateFileCapture"],
60 "cvCaptureFromCAM" : ["cvCreateCameraCapture"],
61 "cvCalcArrBackProjectPatch" : ["cvCalcBackProjectPatch"],
62 "cvCalcArrBackProject" : ["cvCalcBackProject"],
63 "InputArray" : ["_InputArray"],
64 "OutputArray" : ["_OutputArray"],
67 if do_python_crosscheck:
71 print "Could not load cv2"
72 do_python_crosscheck = False
74 def get_cv2_object(name):
75 if name.startswith("cv2."):
77 if name.startswith("cv."):
79 if name == "Algorithm":
80 return cv2.Algorithm__create("Feature2D.ORB"), name
81 elif name == "FeatureDetector":
82 return cv2.FeatureDetector_create("ORB"), name
83 elif name == "DescriptorExtractor":
84 return cv2.DescriptorExtractor_create("ORB"), name
85 elif name == "BackgroundSubtractor":
86 return cv2.createBackgroundSubtractorMOG(), name
87 elif name == "StatModel":
88 return cv2.KNearest(), name
91 obj = getattr(cv2, name)()
92 except AttributeError:
93 obj = getattr(cv2, "create" + name)()
96 def compareSignatures(f, s):
99 return False, "name mismatch"
101 stype = (s[1] or "void")
103 stype = re.sub(r"\b(cv|std)::", "", stype)
105 ftype = re.sub(r"\b(cv|std)::", "", ftype)
106 if ftype and ftype != stype:
107 return False, "return type mismatch"
108 if ("\C" in f[2]) ^ ("\C" in s[2]):
109 return False, "const qualifier mismatch"
110 if ("\S" in f[2]) ^ ("\S" in s[2]):
111 return False, "static qualifier mismatch"
112 if ("\V" in f[2]) ^ ("\V" in s[2]):
113 return False, "virtual qualifier mismatch"
114 if ("\A" in f[2]) ^ ("\A" in s[2]):
115 return False, "abstract qualifier mismatch"
116 if len(f[3]) != len(s[3]):
117 return False, "different number of arguments"
118 for idx, arg in enumerate(zip(f[3], s[3])):
121 ftype = re.sub(r"\b(cv|std)::", "", (farg[0] or ""))
122 stype = re.sub(r"\b(cv|std)::", "", (sarg[0] or ""))
123 ftype = re.sub(r"\s+(\*|&)$", "\\1", ftype)
124 stype = re.sub(r"\s+(\*|&)$", "\\1", stype)
126 return False, "type of argument #" + str(idx+1) + " mismatch"
127 fname = farg[1] or "arg" + str(idx)
128 sname = sarg[1] or "arg" + str(idx)
130 return False, "name of argument #" + str(idx+1) + " mismatch"
131 fdef = re.sub(r"\b(cv|std)::", "", (farg[2] or ""))
132 sdef = re.sub(r"\b(cv|std)::", "", (sarg[2] or ""))
134 return False, "default value of argument #" + str(idx+1) + " mismatch"
137 def formatSignature(s):
146 if not bool(re.match(r"(\w+\.)*(?P<cls>\w+)\.(?P=cls)", s[0])):
148 if s[0].startswith("cv."):
149 _str += s[0][3:].replace(".", "::")
151 _str += s[0].replace(".", "::")
156 for idx, arg in enumerate(s[3]):
159 argtype = re.sub(r"\bcv::", "", arg[0])
160 argtype = re.sub(r"\s+(\*|&)$", "\\1", arg[0])
161 bidx = argtype.find('[')
165 _str += argtype[:bidx]
170 _str += "arg" + str(idx)
172 _str += argtype[bidx:]
174 _str += "=" + re.sub(r"\bcv::", "", arg[2])
183 def logerror(code, message, doc = None):
184 if code in errors_disabled:
187 print doc["file"] + ":" + str(doc["line"]),
188 print "error %03d: %s" % (code, message)
191 def process_module(module, path):
192 hppparser = hp.CppHeaderParser()
193 rstparser = rp.RstParser(hppparser)
195 rstparser.parse(module, path)
196 rst = rstparser.definitions
199 for root, dirs, files in os.walk(os.path.join(path, "include")):
200 for filename in fnmatch.filter(files, "*.h*"):
201 hdrlist.append(os.path.join(root, filename))
204 hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "cuda_types.hpp"))
205 hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "cuda.hpp"))
206 hdrlist.append(os.path.join(path, "..", "core", "include", "opencv2", "core", "cuda_stream_accessor.hpp"))
209 for hname in hdrlist:
210 if not "ts_gtest.h" in hname:
211 decls += hppparser.parse(hname, wmode=False)
214 # not really needed to hardcode all the namespaces. Normally all they are collected automatically
215 namespaces = ['cv', 'cv.cuda', 'cvflann', 'cvflann.anyimpl', 'cvflann.lsh', 'cv.flann', 'cv.linemod', 'cv.detail', 'cvtest', 'perf', 'cv.videostab']
219 # collect namespaces and classes/structs
221 if decl[0].startswith("const"):
223 elif decl[0].startswith("class") or decl[0].startswith("struct"):
224 if decl[0][0] == 'c':
228 dotIdx = decl[0].rfind('.')
230 namespace = decl[0][decl[0].find(' ')+1:dotIdx]
231 if not [c for c in classes if c[0].endswith(namespace)] and not [s for s in structs if s[0].endswith(namespace)]:
232 if namespace not in namespaces:
233 namespaces.append(namespace)
240 name = cl[0][cl[0].find(' ')+1:]
241 if name.find('.') < 0 and not name.startswith("Cv"):
242 logerror(ERROR_004_MISSEDNAMESPACE, "class " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
243 clsnamespaces.append(name)
244 if do_python_crosscheck and not name.startswith("cv.") and name.startswith("Cv"):
245 clsnamespaces.append("cv." + name[2:])
246 if name.startswith("cv."):
248 name = name.replace(".", "::")
249 sns = synonims.get(name, [])
254 #TODO: class is not documented
256 doc[DOCUMENTED_MARKER] = True
257 # verify class marker
258 if not doc.get("isclass"):
259 logerror(ERROR_001_NOTACLASS, "class " + name + " is not marked as \"class\" in documentation", doc)
262 signature = doc.get("class", "")
263 signature = signature.replace(" public ", " ")
264 namespaceIdx = signature.rfind("::")
266 signature = ("class " + signature).strip()
267 hdrsignature = ("class " + name + " " + cl[1]).replace(".", "::").replace("cv::","").strip()
268 if signature != hdrsignature:
269 logerror(ERROR_003_INCORRECTBASE, "invalid base class documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc)
273 name = st[0][st[0].find(' ')+1:]
274 if name.find('.') < 0 and not name.startswith("Cv"):
275 logerror(ERROR_004_MISSEDNAMESPACE, "struct " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
276 clsnamespaces.append(name)
277 if name.startswith("cv."):
279 name = name.replace(".", "::")
282 #TODO: struct is not documented
284 doc[DOCUMENTED_MARKER] = True
285 # verify struct marker
286 if not doc.get("isstruct"):
287 logerror(ERROR_002_NOTASTRUCT, "struct " + name + " is not marked as \"struct\" in documentation", doc)
290 signature = doc.get("class", "")
291 signature = signature.replace(", public ", " ").replace(" public ", " ")
292 signature = signature.replace(", protected ", " ").replace(" protected ", " ")
293 signature = signature.replace(", private ", " ").replace(" private ", " ")
294 signature = ("struct " + signature).strip()
295 hdrsignature = (st[0] + " " + st[1]).replace("struct cv.", "struct ").replace(".", "::").strip()
296 if signature != hdrsignature:
297 logerror(ERROR_003_INCORRECTBASE, "invalid base struct documentation\ndocumented: " + signature + "\nactual: " + hdrsignature, doc)
300 # process functions and methods
306 for cl in clsnamespaces:
307 if name.startswith(cl + "."):
308 if cl.startswith(parent or ""):
311 name = name[len(parent) + 1:]
312 for nm in namespaces:
313 if parent.startswith(nm + "."):
314 if nm.startswith(namespace or ""):
317 parent = parent[len(namespace) + 1:]
319 for nm in namespaces:
320 if name.startswith(nm + "."):
321 if nm.startswith(namespace or ""):
324 name = name[len(namespace) + 1:]
325 #print namespace, parent, name, fn[0]
326 if not namespace and not parent and not name.startswith("cv") and not name.startswith("icv") and not name.startswith("CV_"):
327 logerror(ERROR_004_MISSEDNAMESPACE, "function " + name + " from opencv_" + module + " is placed in global namespace but violates C-style naming convention")
329 fdescr = (namespace, parent, name, fn)
330 flookup_entry = flookup.get(fn[0], [])
331 flookup_entry.append(fdescr)
332 flookup[fn[0]] = flookup_entry
334 if do_python_crosscheck:
335 pyclsnamespaces = ["cv." + x[3:].replace(".", "_") for x in clsnamespaces]
336 for name, doc in rst.iteritems():
337 decls = doc.get("decls")
340 for signature in decls:
341 if signature[0] == "Python1":
342 pname = signature[1][:signature[1].find('(')]
344 fn = getattr(cv2.cv, pname[3:])
345 docstr = "cv." + fn.__doc__
346 except AttributeError:
347 logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function: cv2." + pname, doc)
351 signature.append(DOCUMENTED_MARKER)
352 # convert old signature to pydoc style
353 if docstring.endswith("*"):
354 docstring = docstring[:-1]
358 sign = re.sub(r"^(.*\(.*)\(.*?\)(.*\) *->)", "\\1_\\2", sign)
362 sign = re.sub(r"\s*,\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", " [, \\1\\2])", sign)
363 sign = re.sub(r"\(\s*([^,]+)\s*=\s*[^,]+\s*(( \[.*\])?)\)", "([\\1\\2])", sign)
365 sign = re.sub(r"\)\s*->\s*", ") -> ", sign)
366 sign = sign.replace("-> convexHull", "-> CvSeq")
367 sign = sign.replace("-> lines", "-> CvSeq")
368 sign = sign.replace("-> boundingRects", "-> CvSeq")
369 sign = sign.replace("-> contours", "-> CvSeq")
370 sign = sign.replace("-> retval", "-> int")
371 sign = sign.replace("-> detectedObjects", "-> CvSeqOfCvAvgComp")
373 def retvalRplace(match):
375 m = m.replace("CvScalar", "scalar")
376 m = m.replace("CvMemStorage", "memstorage")
377 m = m.replace("ROIplImage", "image")
378 m = m.replace("IplImage", "image")
379 m = m.replace("ROCvMat", "mat")
380 m = m.replace("CvMat", "mat")
381 m = m.replace("double", "float")
382 m = m.replace("CvSubdiv2DPoint", "point")
383 m = m.replace("CvBox2D", "Box2D")
384 m = m.replace("IplConvKernel", "kernel")
385 m = m.replace("CvHistogram", "hist")
386 m = m.replace("CvSize", "width,height")
387 m = m.replace("cvmatnd", "matND")
388 m = m.replace("CvSeqOfCvConvexityDefect", "convexityDefects")
391 return "(" + ", ".join(mm) + ")"
395 docstring = re.sub(r"(?<=-> )(.*)$", retvalRplace, docstring)
396 docstring = docstring.replace("( [, ", "([")
398 if sign != docstring:
399 logerror(ERROR_006_INVALIDPYOLDDOC, "old-style documentation differs from pydoc\npydoc: " + docstring + "\nfixup: " + sign + "\ncvdoc: " + signature[1], doc)
400 elif signature[0] == "Python2":
401 pname = signature[1][4:signature[1].find('(')]
402 cvname = "cv." + pname
404 for cl in pyclsnamespaces:
405 if cvname.startswith(cl + "."):
406 if cl.startswith(parent or ""):
410 instance, clsname = get_cv2_object(parent)
411 fn = getattr(instance, cvname[len(parent)+1:])
413 docprefix = "cv2." + clsname + "."
415 fn = getattr(cv2, pname)
418 except AttributeError:
420 logerror(ERROR_005_MISSINGPYFUNC, "could not load documented member of " + parent + " class: cv2." + pname, doc)
422 logerror(ERROR_005_MISSINGPYFUNC, "could not load documented function cv2." + pname, doc)
423 signature.append(DOCUMENTED_MARKER) # stop subsequent errors
425 docstrings = [docprefix + s.replace("([, ", "([") for s in docstr.split(" or ")]
426 if not signature[1] in docstrings:
427 pydocs = "\npydoc: ".join(docstrings)
428 logerror(ERROR_007_INVALIDPYDOC, "documentation differs from pydoc\npydoc: " + pydocs + "\ncvdoc: " + signature[1], doc)
429 signature.append(DOCUMENTED_MARKER)
431 # verify C/C++ signatures
432 for name, doc in rst.iteritems():
433 decls = doc.get("decls")
436 for signature in decls:
437 if signature[0] == "C" or signature[0] == "C++":
438 if "template" in (signature[2][1] or ""):
439 # TODO find a way to validate templates
440 signature.append(DOCUMENTED_MARKER)
442 fd = flookup.get(signature[2][0])
444 if signature[2][0].startswith("cv."):
445 fd = flookup.get(signature[2][0][3:])
449 signature[2][0] = signature[2][0][3:]
450 if signature[0] == "C":
451 ffd = [f for f in fd if not f[0] and not f[1]] # filter out C++ stuff
454 logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually member of " + fd[0][1] + " class", doc)
456 logerror(ERROR_008_CFUNCISNOTGLOBAL, "function " + fd[0][2] + " is documented as C function but is actually placed in " + fd[0][0] + " namespace", doc)
460 match, error = compareSignatures(signature[2], f[3])
462 signature.append(DOCUMENTED_MARKER)
464 if signature[-1] != DOCUMENTED_MARKER:
465 candidates = "\n\t".join([formatSignature(f[3]) for f in fd])
466 logerror(ERROR_009_OVERLOADNOTFOUND, signature[0] + " function " + signature[2][0].replace(".","::") + " is documented but misses in headers (" + error + ").\nDocumented as:\n\t" + signature[1] + "\nCandidates are:\n\t" + candidates, doc)
467 signature.append(DOCUMENTED_MARKER) # to stop subsequent error on this function
469 # verify that all signatures was found in the library headers
470 for name, doc in rst.iteritems():
471 # if doc.get(DOCUMENTED_MARKER, False):
472 # continue # this class/struct was found
473 if not doc.get(DOCUMENTED_MARKER, False) and (doc.get("isclass", False) or doc.get("isstruct", False)):
474 if name in doc_signatures_whitelist:
476 logerror(ERROR_010_UNKNOWNCLASS, "class/struct " + name + " is mentioned in documentation but is not found in OpenCV headers", doc)
477 for d in doc.get("decls", []):
478 if d[-1] != DOCUMENTED_MARKER:
479 if d[0] == "C" or d[0] =="C++" or (do_python_crosscheck and d[0].startswith("Python")):
481 sname = d[2][0][3:].replace(".", "::")
483 #TODO: need to find a way to verify #define's
486 sname = d[1][:d[1].find("(")]
487 prefixes = [x for x in doc_signatures_whitelist if sname.startswith(x)]
489 # TODO: member of template class
491 logerror(ERROR_011_UNKNOWNFUNC, d[0] + " function " + sname + " is documented but is not found in OpenCV headers. It is documented as:\n\t" + d[1], doc)
492 # end of process_module
494 if __name__ == "__main__":
495 if len(sys.argv) < 2:
496 print "Usage:\n", os.path.basename(sys.argv[0]), " <module path>"
499 modules = sys.argv[1:]
500 if modules[0] == "all":
503 for module in modules:
504 selfpath = os.path.dirname(os.path.abspath(sys.argv[0]))
505 module_path = os.path.join(selfpath, "..", "modules", module)
507 if not os.path.isdir(module_path):
508 print "Module \"" + module + "\" could not be found."
511 process_module(module, module_path)