3 This file is part of Mbed TLS (https://tls.mbed.org)
5 Copyright (c) 2018, Arm Limited, All Rights Reserved
9 This script checks the current state of the source code for minor issues,
10 including incorrect file permissions, presence of tabs, non-Unix line endings,
11 trailing whitespace, presence of UTF-8 BOM, and TODO comments.
12 Note: requires python 3, must be run from Mbed TLS root.
22 class FileIssueTracker(object):
23 """Base class for file-wide issue tracking.
25 To implement a checker that processes a file as a whole, inherit from
26 this class and implement `check_file_for_issue` and define ``heading``.
28 ``files_exemptions``: files whose name ends with a string in this set
31 ``heading``: human-readable description of the issue
34 files_exemptions = frozenset()
35 # heading must be defined in derived classes.
36 # pylint: disable=no-member
39 self.files_with_issues = {}
41 def should_check_file(self, filepath):
42 for files_exemption in self.files_exemptions:
43 if filepath.endswith(files_exemption):
47 def check_file_for_issue(self, filepath):
48 raise NotImplementedError
50 def record_issue(self, filepath, line_number):
51 if filepath not in list(self.files_with_issues.keys()):
52 self.files_with_issues[filepath] = []
53 self.files_with_issues[filepath].append(line_number)
55 def output_file_issues(self, logger):
56 if list(self.files_with_issues.values()):
57 logger.info(self.heading)
58 for filename, lines in sorted(self.files_with_issues.items()):
60 logger.info("{}: {}".format(
61 filename, ", ".join(str(x) for x in lines)
67 class LineIssueTracker(FileIssueTracker):
68 """Base class for line-by-line issue tracking.
70 To implement a checker that processes files line by line, inherit from
71 this class and implement `line_with_issue`.
74 def issue_with_line(self, line, filepath):
75 raise NotImplementedError
77 def check_file_line(self, filepath, line, line_number):
78 if self.issue_with_line(line, filepath):
79 self.record_issue(filepath, line_number)
81 def check_file_for_issue(self, filepath):
82 with open(filepath, "rb") as f:
83 for i, line in enumerate(iter(f.readline, b"")):
84 self.check_file_line(filepath, line, i + 1)
86 class PermissionIssueTracker(FileIssueTracker):
87 """Track files with bad permissions.
89 Files that are not executable scripts must not be executable."""
91 heading = "Incorrect permissions:"
93 def check_file_for_issue(self, filepath):
94 is_executable = os.access(filepath, os.X_OK)
95 should_be_executable = filepath.endswith((".sh", ".pl", ".py"))
96 if is_executable != should_be_executable:
97 self.files_with_issues[filepath] = None
100 class EndOfFileNewlineIssueTracker(FileIssueTracker):
101 """Track files that end with an incomplete line
102 (no newline character at the end of the last line)."""
104 heading = "Missing newline at end of file:"
106 def check_file_for_issue(self, filepath):
107 with open(filepath, "rb") as f:
108 if not f.read().endswith(b"\n"):
109 self.files_with_issues[filepath] = None
112 class Utf8BomIssueTracker(FileIssueTracker):
113 """Track files that start with a UTF-8 BOM.
114 Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM."""
116 heading = "UTF-8 BOM present:"
118 def check_file_for_issue(self, filepath):
119 with open(filepath, "rb") as f:
120 if f.read().startswith(codecs.BOM_UTF8):
121 self.files_with_issues[filepath] = None
124 class LineEndingIssueTracker(LineIssueTracker):
125 """Track files with non-Unix line endings (i.e. files with CR)."""
127 heading = "Non Unix line endings:"
129 def issue_with_line(self, line, _filepath):
133 class TrailingWhitespaceIssueTracker(LineIssueTracker):
134 """Track lines with trailing whitespace."""
136 heading = "Trailing whitespace:"
137 files_exemptions = frozenset(".md")
139 def issue_with_line(self, line, _filepath):
140 return line.rstrip(b"\r\n") != line.rstrip()
143 class TabIssueTracker(LineIssueTracker):
144 """Track lines with tabs."""
146 heading = "Tabs present:"
147 files_exemptions = frozenset([
149 "generate_visualc_files.pl",
152 def issue_with_line(self, line, _filepath):
156 class MergeArtifactIssueTracker(LineIssueTracker):
157 """Track lines with merge artifacts.
158 These are leftovers from a ``git merge`` that wasn't fully edited."""
160 heading = "Merge artifact:"
162 def issue_with_line(self, line, _filepath):
163 # Detect leftover git conflict markers.
164 if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '):
166 if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3
168 if line.rstrip(b'\r\n') == b'=======' and \
169 not _filepath.endswith('.md'):
173 class TodoIssueTracker(LineIssueTracker):
174 """Track lines containing ``TODO``."""
176 heading = "TODO present:"
177 files_exemptions = frozenset([
178 os.path.basename(__file__),
180 "pull_request_template.md",
183 def issue_with_line(self, line, _filepath):
184 return b"todo" in line.lower()
187 class IntegrityChecker(object):
188 """Sanity-check files under the current directory."""
190 def __init__(self, log_file):
191 """Instantiate the sanity checker.
192 Check files under the current directory.
193 Write a report of issues to log_file."""
194 self.check_repo_path()
196 self.setup_logger(log_file)
197 self.files_to_check = (
198 ".c", ".h", ".sh", ".pl", ".py", ".md", ".function", ".data",
199 "Makefile", "CMakeLists.txt", "ChangeLog"
201 self.excluded_directories = ['.git', 'mbed-os']
202 self.excluded_paths = list(map(os.path.normpath, [
206 self.issues_to_check = [
207 PermissionIssueTracker(),
208 EndOfFileNewlineIssueTracker(),
209 Utf8BomIssueTracker(),
210 LineEndingIssueTracker(),
211 TrailingWhitespaceIssueTracker(),
213 MergeArtifactIssueTracker(),
218 def check_repo_path():
219 if not all(os.path.isdir(d) for d in ["include", "library", "tests"]):
220 raise Exception("Must be run from Mbed TLS root")
222 def setup_logger(self, log_file, level=logging.INFO):
223 self.logger = logging.getLogger()
224 self.logger.setLevel(level)
226 handler = logging.FileHandler(log_file)
227 self.logger.addHandler(handler)
229 console = logging.StreamHandler()
230 self.logger.addHandler(console)
232 def prune_branch(self, root, d):
233 if d in self.excluded_directories:
235 if os.path.normpath(os.path.join(root, d)) in self.excluded_paths:
239 def check_files(self):
240 for root, dirs, files in os.walk("."):
241 dirs[:] = sorted(d for d in dirs if not self.prune_branch(root, d))
242 for filename in sorted(files):
243 filepath = os.path.join(root, filename)
244 if not filepath.endswith(self.files_to_check):
246 for issue_to_check in self.issues_to_check:
247 if issue_to_check.should_check_file(filepath):
248 issue_to_check.check_file_for_issue(filepath)
250 def output_issues(self):
251 integrity_return_code = 0
252 for issue_to_check in self.issues_to_check:
253 if issue_to_check.files_with_issues:
254 integrity_return_code = 1
255 issue_to_check.output_file_issues(self.logger)
256 return integrity_return_code
260 parser = argparse.ArgumentParser(
262 "This script checks the current state of the source code for "
263 "minor issues, including incorrect file permissions, "
264 "presence of tabs, non-Unix line endings, trailing whitespace, "
265 "presence of UTF-8 BOM, and TODO comments. "
266 "Note: requires python 3, must be run from Mbed TLS root."
270 "-l", "--log_file", type=str, help="path to optional output log",
272 check_args = parser.parse_args()
273 integrity_check = IntegrityChecker(check_args.log_file)
274 integrity_check.check_files()
275 return_code = integrity_check.output_issues()
276 sys.exit(return_code)
279 if __name__ == "__main__":