extlibs/mbedtls/mbedtls/tests/scripts/check-files.py

   1 #!/usr/bin/env python3
   2 """
   3 This file is part of Mbed TLS (https://tls.mbed.org)
   4
   5 Copyright (c) 2018, Arm Limited, All Rights Reserved
   6
   7 Purpose
   8
   9 This script checks the current state of the source code for minor issues,
  10 including incorrect file permissions, presence of tabs, non-Unix line endings,
  11 trailing whitespace, presence of UTF-8 BOM, and TODO comments.
  12 Note: requires python 3, must be run from Mbed TLS root.
  13 """
  14
  15 import os
  16 import argparse
  17 import logging
  18 import codecs
  19 import sys
  20
  21
  22 class FileIssueTracker(object):
  23     """Base class for file-wide issue tracking.
  24
  25     To implement a checker that processes a file as a whole, inherit from
  26     this class and implement `check_file_for_issue` and define ``heading``.
  27
  28     ``files_exemptions``: files whose name ends with a string in this set
  29      will not be checked.
  30
  31     ``heading``: human-readable description of the issue
  32     """
  33
  34     files_exemptions = frozenset()
  35     # heading must be defined in derived classes.
  36     # pylint: disable=no-member
  37
  38     def __init__(self):
  39         self.files_with_issues = {}
  40
  41     def should_check_file(self, filepath):
  42         for files_exemption in self.files_exemptions:
  43             if filepath.endswith(files_exemption):
  44                 return False
  45         return True
  46
  47     def check_file_for_issue(self, filepath):
  48         raise NotImplementedError
  49
  50     def record_issue(self, filepath, line_number):
  51         if filepath not in list(self.files_with_issues.keys()):
  52             self.files_with_issues[filepath] = []
  53         self.files_with_issues[filepath].append(line_number)
  54
  55     def output_file_issues(self, logger):
  56         if list(self.files_with_issues.values()):
  57             logger.info(self.heading)
  58             for filename, lines in sorted(self.files_with_issues.items()):
  59                 if lines:
  60                     logger.info("{}: {}".format(
  61                         filename, ", ".join(str(x) for x in lines)
  62                     ))
  63                 else:
  64                     logger.info(filename)
  65             logger.info("")
  66
  67 class LineIssueTracker(FileIssueTracker):
  68     """Base class for line-by-line issue tracking.
  69
  70     To implement a checker that processes files line by line, inherit from
  71     this class and implement `line_with_issue`.
  72     """
  73
  74     def issue_with_line(self, line, filepath):
  75         raise NotImplementedError
  76
  77     def check_file_line(self, filepath, line, line_number):
  78         if self.issue_with_line(line, filepath):
  79             self.record_issue(filepath, line_number)
  80
  81     def check_file_for_issue(self, filepath):
  82         with open(filepath, "rb") as f:
  83             for i, line in enumerate(iter(f.readline, b"")):
  84                 self.check_file_line(filepath, line, i + 1)
  85
  86 class PermissionIssueTracker(FileIssueTracker):
  87     """Track files with bad permissions.
  88
  89     Files that are not executable scripts must not be executable."""
  90
  91     heading = "Incorrect permissions:"
  92
  93     def check_file_for_issue(self, filepath):
  94         is_executable = os.access(filepath, os.X_OK)
  95         should_be_executable = filepath.endswith((".sh", ".pl", ".py"))
  96         if is_executable != should_be_executable:
  97             self.files_with_issues[filepath] = None
  98
  99
 100 class EndOfFileNewlineIssueTracker(FileIssueTracker):
 101     """Track files that end with an incomplete line
 102     (no newline character at the end of the last line)."""
 103
 104     heading = "Missing newline at end of file:"
 105
 106     def check_file_for_issue(self, filepath):
 107         with open(filepath, "rb") as f:
 108             if not f.read().endswith(b"\n"):
 109                 self.files_with_issues[filepath] = None
 110
 111
 112 class Utf8BomIssueTracker(FileIssueTracker):
 113     """Track files that start with a UTF-8 BOM.
 114     Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM."""
 115
 116     heading = "UTF-8 BOM present:"
 117
 118     def check_file_for_issue(self, filepath):
 119         with open(filepath, "rb") as f:
 120             if f.read().startswith(codecs.BOM_UTF8):
 121                 self.files_with_issues[filepath] = None
 122
 123
 124 class LineEndingIssueTracker(LineIssueTracker):
 125     """Track files with non-Unix line endings (i.e. files with CR)."""
 126
 127     heading = "Non Unix line endings:"
 128
 129     def issue_with_line(self, line, _filepath):
 130         return b"\r" in line
 131
 132
 133 class TrailingWhitespaceIssueTracker(LineIssueTracker):
 134     """Track lines with trailing whitespace."""
 135
 136     heading = "Trailing whitespace:"
 137     files_exemptions = frozenset(".md")
 138
 139     def issue_with_line(self, line, _filepath):
 140         return line.rstrip(b"\r\n") != line.rstrip()
 141
 142
 143 class TabIssueTracker(LineIssueTracker):
 144     """Track lines with tabs."""
 145
 146     heading = "Tabs present:"
 147     files_exemptions = frozenset([
 148         "Makefile",
 149         "generate_visualc_files.pl",
 150     ])
 151
 152     def issue_with_line(self, line, _filepath):
 153         return b"\t" in line
 154
 155
 156 class MergeArtifactIssueTracker(LineIssueTracker):
 157     """Track lines with merge artifacts.
 158     These are leftovers from a ``git merge`` that wasn't fully edited."""
 159
 160     heading = "Merge artifact:"
 161
 162     def issue_with_line(self, line, _filepath):
 163         # Detect leftover git conflict markers.
 164         if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '):
 165             return True
 166         if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3
 167             return True
 168         if line.rstrip(b'\r\n') == b'=======' and \
 169            not _filepath.endswith('.md'):
 170             return True
 171         return False
 172
 173 class TodoIssueTracker(LineIssueTracker):
 174     """Track lines containing ``TODO``."""
 175
 176     heading = "TODO present:"
 177     files_exemptions = frozenset([
 178         os.path.basename(__file__),
 179         "benchmark.c",
 180         "pull_request_template.md",
 181     ])
 182
 183     def issue_with_line(self, line, _filepath):
 184         return b"todo" in line.lower()
 185
 186
 187 class IntegrityChecker(object):
 188     """Sanity-check files under the current directory."""
 189
 190     def __init__(self, log_file):
 191         """Instantiate the sanity checker.
 192         Check files under the current directory.
 193         Write a report of issues to log_file."""
 194         self.check_repo_path()
 195         self.logger = None
 196         self.setup_logger(log_file)
 197         self.files_to_check = (
 198             ".c", ".h", ".sh", ".pl", ".py", ".md", ".function", ".data",
 199             "Makefile", "CMakeLists.txt", "ChangeLog"
 200         )
 201         self.excluded_directories = ['.git', 'mbed-os']
 202         self.excluded_paths = list(map(os.path.normpath, [
 203             'cov-int',
 204             'examples',
 205         ]))
 206         self.issues_to_check = [
 207             PermissionIssueTracker(),
 208             EndOfFileNewlineIssueTracker(),
 209             Utf8BomIssueTracker(),
 210             LineEndingIssueTracker(),
 211             TrailingWhitespaceIssueTracker(),
 212             TabIssueTracker(),
 213             MergeArtifactIssueTracker(),
 214             TodoIssueTracker(),
 215         ]
 216
 217     @staticmethod
 218     def check_repo_path():
 219         if not all(os.path.isdir(d) for d in ["include", "library", "tests"]):
 220             raise Exception("Must be run from Mbed TLS root")
 221
 222     def setup_logger(self, log_file, level=logging.INFO):
 223         self.logger = logging.getLogger()
 224         self.logger.setLevel(level)
 225         if log_file:
 226             handler = logging.FileHandler(log_file)
 227             self.logger.addHandler(handler)
 228         else:
 229             console = logging.StreamHandler()
 230             self.logger.addHandler(console)
 231
 232     def prune_branch(self, root, d):
 233         if d in self.excluded_directories:
 234             return True
 235         if os.path.normpath(os.path.join(root, d)) in self.excluded_paths:
 236             return True
 237         return False
 238
 239     def check_files(self):
 240         for root, dirs, files in os.walk("."):
 241             dirs[:] = sorted(d for d in dirs if not self.prune_branch(root, d))
 242             for filename in sorted(files):
 243                 filepath = os.path.join(root, filename)
 244                 if not filepath.endswith(self.files_to_check):
 245                     continue
 246                 for issue_to_check in self.issues_to_check:
 247                     if issue_to_check.should_check_file(filepath):
 248                         issue_to_check.check_file_for_issue(filepath)
 249
 250     def output_issues(self):
 251         integrity_return_code = 0
 252         for issue_to_check in self.issues_to_check:
 253             if issue_to_check.files_with_issues:
 254                 integrity_return_code = 1
 255             issue_to_check.output_file_issues(self.logger)
 256         return integrity_return_code
 257
 258
 259 def run_main():
 260     parser = argparse.ArgumentParser(
 261         description=(
 262             "This script checks the current state of the source code for "
 263             "minor issues, including incorrect file permissions, "
 264             "presence of tabs, non-Unix line endings, trailing whitespace, "
 265             "presence of UTF-8 BOM, and TODO comments. "
 266             "Note: requires python 3, must be run from Mbed TLS root."
 267         )
 268     )
 269     parser.add_argument(
 270         "-l", "--log_file", type=str, help="path to optional output log",
 271     )
 272     check_args = parser.parse_args()
 273     integrity_check = IntegrityChecker(check_args.log_file)
 274     integrity_check.check_files()
 275     return_code = integrity_check.output_issues()
 276     sys.exit(return_code)
 277
 278
 279 if __name__ == "__main__":
 280     run_main()