third_party/pigweed/repo/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright 2020 The Pigweed Authors
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
   6 # use this file except in compliance with the License. You may obtain a copy of
   7 # the License at
   8 #
   9 #     https://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14 # License for the specific language governing permissions and limitations under
  15 # the License.
  16 """Runs the local presubmit checks for the Pigweed repository."""
  17
  18 import argparse
  19 import logging
  20 import os
  21 from pathlib import Path
  22 import re
  23 import sys
  24 from typing import Sequence, IO, Tuple, Optional
  25
  26 try:
  27     import pw_presubmit
  28 except ImportError:
  29     # Append the pw_presubmit package path to the module search path to allow
  30     # running this module without installing the pw_presubmit package.
  31     sys.path.append(os.path.dirname(os.path.dirname(
  32         os.path.abspath(__file__))))
  33     import pw_presubmit
  34
  35 import pw_package.pigweed_packages
  36
  37 from pw_presubmit import build, cli, environment, format_code, git_repo
  38 from pw_presubmit import call, filter_paths, plural, PresubmitContext
  39 from pw_presubmit import PresubmitFailure, Programs
  40 from pw_presubmit.install_hook import install_hook
  41
  42 _LOG = logging.getLogger(__name__)
  43
  44 pw_package.pigweed_packages.initialize()
  45
  46
  47 #
  48 # Initialization
  49 #
  50 def init_cipd(ctx: PresubmitContext):
  51     environment.init_cipd(ctx.root, ctx.output_dir)
  52
  53
  54 def init_virtualenv(ctx: PresubmitContext):
  55     environment.init_virtualenv(
  56         ctx.root,
  57         ctx.output_dir,
  58         gn_targets=(
  59             f'{ctx.root}#:python.install',
  60             f'{ctx.root}#:target_support_packages.install',
  61         ),
  62     )
  63
  64
  65 # Trigger builds if files with these extensions change.
  66 _BUILD_EXTENSIONS = ('.py', '.rst', '.gn', '.gni',
  67                      *format_code.C_FORMAT.extensions)
  68
  69
  70 def _at_all_optimization_levels(target):
  71     for level in ['debug', 'size_optimized', 'speed_optimized']:
  72         yield f'{target}_{level}'
  73
  74
  75 #
  76 # Build presubmit checks
  77 #
  78 def gn_clang_build(ctx: PresubmitContext):
  79     build.gn_gen(ctx.root, ctx.output_dir)
  80     build.ninja(ctx.output_dir, *_at_all_optimization_levels('host_clang'))
  81
  82
  83 @filter_paths(endswith=_BUILD_EXTENSIONS)
  84 def gn_quick_build_check(ctx: PresubmitContext):
  85     build.gn_gen(ctx.root, ctx.output_dir)
  86     build.ninja(ctx.output_dir, 'host_clang_size_optimized',
  87                 'stm32f429i_size_optimized', 'python.tests', 'python.lint')
  88
  89
  90 @filter_paths(endswith=_BUILD_EXTENSIONS)
  91 def gn_gcc_build(ctx: PresubmitContext):
  92     build.gn_gen(ctx.root, ctx.output_dir)
  93
  94     # Skip optimized host GCC builds for now, since GCC sometimes emits spurious
  95     # warnings.
  96     #
  97     #   -02: GCC 9.3 emits spurious maybe-uninitialized warnings
  98     #   -0s: GCC 8.1 (Mingw-w64) emits a spurious nonnull warning
  99     #
 100     # TODO(pwbug/255): Enable optimized GCC builds when this is fixed.
 101     build.ninja(ctx.output_dir, 'host_gcc_debug')
 102
 103
 104 @filter_paths(endswith=_BUILD_EXTENSIONS)
 105 def gn_arm_build(ctx: PresubmitContext):
 106     build.gn_gen(ctx.root, ctx.output_dir)
 107     build.ninja(ctx.output_dir, *_at_all_optimization_levels('stm32f429i'))
 108
 109
 110 @filter_paths(endswith=_BUILD_EXTENSIONS)
 111 def gn_nanopb_build(ctx: PresubmitContext):
 112     build.install_package(ctx.package_root, 'nanopb')
 113     build.gn_gen(ctx.root,
 114                  ctx.output_dir,
 115                  dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root /
 116                                                          'nanopb'))
 117     build.ninja(
 118         ctx.output_dir,
 119         *_at_all_optimization_levels('stm32f429i'),
 120         *_at_all_optimization_levels('host_clang'),
 121     )
 122
 123
 124 @filter_paths(endswith=_BUILD_EXTENSIONS)
 125 def gn_teensy_build(ctx: PresubmitContext):
 126     build.install_package(ctx.package_root, 'teensy')
 127     build.gn_gen(ctx.root,
 128                  ctx.output_dir,
 129                  pw_arduino_build_CORE_PATH='"{}"'.format(str(
 130                      ctx.package_root)),
 131                  pw_arduino_build_CORE_NAME='teensy',
 132                  pw_arduino_build_PACKAGE_NAME='teensy/avr',
 133                  pw_arduino_build_BOARD='teensy40')
 134     build.ninja(ctx.output_dir, *_at_all_optimization_levels('arduino'))
 135
 136
 137 @filter_paths(endswith=_BUILD_EXTENSIONS)
 138 def gn_qemu_build(ctx: PresubmitContext):
 139     build.gn_gen(ctx.root, ctx.output_dir)
 140     build.ninja(ctx.output_dir, *_at_all_optimization_levels('qemu_gcc'))
 141
 142
 143 @filter_paths(endswith=_BUILD_EXTENSIONS)
 144 def gn_qemu_clang_build(ctx: PresubmitContext):
 145     build.gn_gen(ctx.root, ctx.output_dir)
 146     build.ninja(ctx.output_dir, *_at_all_optimization_levels('qemu_clang'))
 147
 148
 149 def gn_docs_build(ctx: PresubmitContext):
 150     build.gn_gen(ctx.root, ctx.output_dir)
 151     build.ninja(ctx.output_dir, 'docs')
 152
 153
 154 def gn_host_tools(ctx: PresubmitContext):
 155     build.gn_gen(ctx.root, ctx.output_dir, pw_build_HOST_TOOLS=True)
 156     build.ninja(ctx.output_dir, 'host')
 157
 158
 159 @filter_paths(endswith=format_code.C_FORMAT.extensions)
 160 def oss_fuzz_build(ctx: PresubmitContext):
 161     build.gn_gen(ctx.root, ctx.output_dir, pw_toolchain_OSS_FUZZ_ENABLED=True)
 162     build.ninja(ctx.output_dir, "fuzzers")
 163
 164
 165 @filter_paths(endswith='.py')
 166 def python_checks(ctx: PresubmitContext):
 167     build.gn_gen(ctx.root, ctx.output_dir)
 168     build.ninja(
 169         ctx.output_dir,
 170         ':python.lint',
 171         ':python.tests',
 172         ':target_support_packages.lint',
 173         ':target_support_packages.tests',
 174     )
 175
 176
 177 def _run_cmake(ctx: PresubmitContext) -> None:
 178     build.install_package(ctx.package_root, 'nanopb')
 179
 180     toolchain = ctx.root / 'pw_toolchain' / 'host_clang' / 'toolchain.cmake'
 181     build.cmake(ctx.root,
 182                 ctx.output_dir,
 183                 f'-DCMAKE_TOOLCHAIN_FILE={toolchain}',
 184                 '-DCMAKE_EXPORT_COMPILE_COMMANDS=1',
 185                 f'-Ddir_pw_third_party_nanopb={ctx.package_root / "nanopb"}',
 186                 env=build.env_with_clang_vars())
 187
 188
 189 @filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.cmake',
 190                         'CMakeLists.txt'))
 191 def cmake_tests(ctx: PresubmitContext):
 192     _run_cmake(ctx)
 193     build.ninja(ctx.output_dir, 'pw_apps', 'pw_run_tests.modules')
 194
 195
 196 @filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.bzl', 'BUILD'))
 197 def bazel_test(ctx: PresubmitContext):
 198     try:
 199         call('bazel',
 200              'test',
 201              '//...',
 202              '--verbose_failures',
 203              '--verbose_explanations',
 204              '--worker_verbose',
 205              '--symlink_prefix',
 206              ctx.output_dir.joinpath('bazel-'),
 207              cwd=ctx.root,
 208              env=build.env_with_clang_vars())
 209     except:
 210         _LOG.info('If the Bazel build inexplicably fails while the '
 211                   'other builds are passing, try deleting the Bazel cache:\n'
 212                   '    rm -rf ~/.cache/bazel')
 213         raise
 214
 215
 216 #
 217 # General presubmit checks
 218 #
 219
 220 # TODO(pwbug/45) Probably want additional checks.
 221 _CLANG_TIDY_CHECKS = ('modernize-use-override', )
 222
 223
 224 @filter_paths(endswith=format_code.C_FORMAT.extensions)
 225 def clang_tidy(ctx: PresubmitContext):
 226     build.gn_gen(ctx.root, ctx.output_dir, '--export-compile-commands')
 227     build.ninja(ctx.output_dir)
 228     build.ninja(ctx.output_dir, '-t', 'compdb', 'objcxx', 'cxx')
 229
 230     run_clang_tidy = None
 231     for var in ('PW_PIGWEED_CIPD_INSTALL_DIR', 'PW_CIPD_INSTALL_DIR'):
 232         if var in os.environ:
 233             possibility = os.path.join(os.environ[var],
 234                                        'share/clang/run-clang-tidy.py')
 235             if os.path.isfile(possibility):
 236                 run_clang_tidy = possibility
 237                 break
 238
 239     checks = ','.join(_CLANG_TIDY_CHECKS)
 240     call(
 241         run_clang_tidy,
 242         f'-p={ctx.output_dir}',
 243         f'-checks={checks}',
 244         # TODO(pwbug/45) not sure if this is needed.
 245         # f'-extra-arg-before=-warnings-as-errors={checks}',
 246         *ctx.paths)
 247
 248
 249 # The first line must be regex because of the '20\d\d' date
 250 COPYRIGHT_FIRST_LINE = r'Copyright 20\d\d The Pigweed Authors'
 251 COPYRIGHT_COMMENTS = r'(#|//| \*|REM|::)'
 252 COPYRIGHT_BLOCK_COMMENTS = (
 253     # HTML comments
 254     (r'<!--', r'-->'), )
 255
 256 COPYRIGHT_FIRST_LINE_EXCEPTIONS = (
 257     '#!',
 258     '/*',
 259     '@echo off',
 260     '# -*-',
 261     ':',
 262 )
 263
 264 COPYRIGHT_LINES = tuple("""\
 265
 266 Licensed under the Apache License, Version 2.0 (the "License"); you may not
 267 use this file except in compliance with the License. You may obtain a copy of
 268 the License at
 269
 270     https://www.apache.org/licenses/LICENSE-2.0
 271
 272 Unless required by applicable law or agreed to in writing, software
 273 distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 274 WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 275 License for the specific language governing permissions and limitations under
 276 the License.
 277 """.splitlines())
 278
 279 _EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = (
 280     # Configuration
 281     r'^(?:.+/)?\..+$',
 282     r'\bPW_PLUGINS$',
 283     # Metadata
 284     r'^docker/tag$',
 285     r'\bAUTHORS$',
 286     r'\bLICENSE$',
 287     r'\bOWNERS$',
 288     r'\brequirements.txt$',
 289     r'\bgo.(mod|sum)$',
 290     r'\bpackage.json$',
 291     r'\byarn.lock$',
 292     # Data files
 293     r'\.elf$',
 294     r'\.gif$',
 295     r'\.jpg$',
 296     r'\.json$',
 297     r'\.png$',
 298     # Documentation
 299     r'\.md$',
 300     r'\.rst$',
 301     # Generated protobuf files
 302     r'\.pb\.h$',
 303     r'\.pb\.c$',
 304     r'\_pb2.pyi?$',
 305     # Diff/Patch files
 306     r'\.diff$',
 307     r'\.patch$',
 308 )
 309
 310
 311 def match_block_comment_start(line: str) -> Optional[str]:
 312     """Matches the start of a block comment and returns the end."""
 313     for block_comment in COPYRIGHT_BLOCK_COMMENTS:
 314         if re.match(block_comment[0], line):
 315             # Return the end of the block comment
 316             return block_comment[1]
 317     return None
 318
 319
 320 def copyright_read_first_line(
 321         file: IO) -> Tuple[Optional[str], Optional[str], Optional[str]]:
 322     """Reads the file until it reads a valid first copyright line.
 323
 324     Returns (comment, block_comment, line). comment and block_comment are
 325     mutually exclusive and refer to the comment character sequence and whether
 326     they form a block comment or a line comment. line is the first line of
 327     the copyright, and is used for error reporting.
 328     """
 329     line = file.readline()
 330     first_line_matcher = re.compile(COPYRIGHT_COMMENTS + ' ' +
 331                                     COPYRIGHT_FIRST_LINE)
 332     while line:
 333         end_block_comment = match_block_comment_start(line)
 334         if end_block_comment:
 335             next_line = file.readline()
 336             copyright_line = re.match(COPYRIGHT_FIRST_LINE, next_line)
 337             if not copyright_line:
 338                 return (None, None, line)
 339             return (None, end_block_comment, line)
 340
 341         first_line = first_line_matcher.match(line)
 342         if first_line:
 343             return (first_line.group(1), None, line)
 344
 345         if (line.strip()
 346                 and not line.startswith(COPYRIGHT_FIRST_LINE_EXCEPTIONS)):
 347             return (None, None, line)
 348
 349         line = file.readline()
 350     return (None, None, None)
 351
 352
 353 @filter_paths(exclude=_EXCLUDE_FROM_COPYRIGHT_NOTICE)
 354 def copyright_notice(ctx: PresubmitContext):
 355     """Checks that the Pigweed copyright notice is present."""
 356     errors = []
 357
 358     for path in ctx.paths:
 359
 360         if path.stat().st_size == 0:
 361             continue  # Skip empty files
 362
 363         with path.open() as file:
 364             (comment, end_block_comment,
 365              line) = copyright_read_first_line(file)
 366
 367             if not line:
 368                 _LOG.warning('%s: invalid first line', path)
 369                 errors.append(path)
 370                 continue
 371
 372             if not (comment or end_block_comment):
 373                 _LOG.warning('%s: invalid first line %r', path, line)
 374                 errors.append(path)
 375                 continue
 376
 377             if end_block_comment:
 378                 expected_lines = COPYRIGHT_LINES + (end_block_comment, )
 379             else:
 380                 expected_lines = COPYRIGHT_LINES
 381
 382             for expected, actual in zip(expected_lines, file):
 383                 if end_block_comment:
 384                     expected_line = expected + '\n'
 385                 elif comment:
 386                     expected_line = (comment + ' ' + expected).rstrip() + '\n'
 387
 388                 if expected_line != actual:
 389                     _LOG.warning('  bad line: %r', actual)
 390                     _LOG.warning('  expected: %r', expected_line)
 391                     errors.append(path)
 392                     break
 393
 394     if errors:
 395         _LOG.warning('%s with a missing or incorrect copyright notice:\n%s',
 396                      plural(errors, 'file'), '\n'.join(str(e) for e in errors))
 397         raise PresubmitFailure
 398
 399
 400 _BAZEL_SOURCES_IN_BUILD = tuple(format_code.C_FORMAT.extensions)
 401 _GN_SOURCES_IN_BUILD = '.rst', '.py', *_BAZEL_SOURCES_IN_BUILD
 402
 403
 404 @filter_paths(endswith=(*_GN_SOURCES_IN_BUILD, 'BUILD', '.bzl', '.gn', '.gni'))
 405 def source_is_in_build_files(ctx: PresubmitContext):
 406     """Checks that source files are in the GN and Bazel builds."""
 407     missing = build.check_builds_for_files(
 408         _BAZEL_SOURCES_IN_BUILD,
 409         _GN_SOURCES_IN_BUILD,
 410         ctx.paths,
 411         bazel_dirs=[ctx.root],
 412         gn_build_files=git_repo.list_files(
 413             pathspecs=['BUILD.gn', '*BUILD.gn']))
 414
 415     if missing:
 416         _LOG.warning(
 417             'All source files must appear in BUILD and BUILD.gn files')
 418         raise PresubmitFailure
 419
 420     _run_cmake(ctx)
 421     cmake_missing = build.check_compile_commands_for_files(
 422         ctx.output_dir / 'compile_commands.json',
 423         (f for f in ctx.paths if f.suffix in ('.c', '.cc')))
 424     if cmake_missing:
 425         _LOG.warning('The CMake build is missing %d files', len(cmake_missing))
 426         _LOG.warning('Files missing from CMake:\n%s',
 427                      '\n'.join(str(f) for f in cmake_missing))
 428         # TODO(hepler): Many files are missing from the CMake build. Make this
 429         #     check an error when the missing files are fixed.
 430         # raise PresubmitFailure
 431
 432
 433 def build_env_setup(ctx: PresubmitContext):
 434     if 'PW_CARGO_SETUP' not in os.environ:
 435         _LOG.warning(
 436             'Skipping build_env_setup since PW_CARGO_SETUP is not set')
 437         return
 438
 439     tmpl = ctx.root.joinpath('pw_env_setup', 'py', 'pyoxidizer.bzl.tmpl')
 440     out = ctx.output_dir.joinpath('pyoxidizer.bzl')
 441
 442     with open(tmpl, 'r') as ins:
 443         cfg = ins.read().replace('${PW_ROOT}', str(ctx.root))
 444         with open(out, 'w') as outs:
 445             outs.write(cfg)
 446
 447     call('pyoxidizer', 'build', cwd=ctx.output_dir)
 448
 449
 450 def commit_message_format(_: PresubmitContext):
 451     """Checks that the top commit's message is correctly formatted."""
 452     lines = git_repo.commit_message().splitlines()
 453
 454     # Show limits and current commit message in log.
 455     _LOG.debug('%-25s%+25s%+22s', 'Line limits', '72|', '72|')
 456     for line in lines:
 457         _LOG.debug(line)
 458
 459     if not lines:
 460         _LOG.error('The commit message is too short!')
 461         raise PresubmitFailure
 462
 463     errors = 0
 464
 465     if len(lines[0]) > 72:
 466         _LOG.warning("The commit message's first line must be no longer than "
 467                      '72 characters.')
 468         _LOG.warning('The first line is %d characters:\n  %s', len(lines[0]),
 469                      lines[0])
 470         errors += 1
 471
 472     if lines[0].endswith('.'):
 473         _LOG.warning(
 474             "The commit message's first line must not end with a period:\n %s",
 475             lines[0])
 476         errors += 1
 477
 478     if len(lines) > 1 and lines[1]:
 479         _LOG.warning("The commit message's second line must be blank.")
 480         _LOG.warning('The second line has %d characters:\n  %s', len(lines[1]),
 481                      lines[1])
 482         errors += 1
 483
 484     # Check that the lines are 72 characters or less, but skip any lines that
 485     # might possibly have a URL, path, or metadata in them. Also skip any lines
 486     # with non-ASCII characters.
 487     for i, line in enumerate(lines[2:], 3):
 488         if any(c in line for c in ':/>') or not line.isascii():
 489             continue
 490
 491         if len(line) > 72:
 492             _LOG.warning(
 493                 'Commit message lines must be no longer than 72 characters.')
 494             _LOG.warning('Line %d has %d characters:\n  %s', i, len(line),
 495                          line)
 496             errors += 1
 497
 498     if errors:
 499         _LOG.error('Found %s in the commit message', plural(errors, 'error'))
 500         raise PresubmitFailure
 501
 502
 503 #
 504 # Presubmit check programs
 505 #
 506
 507 BROKEN = (
 508     # TODO(pwbug/45): Remove clang-tidy from BROKEN when it passes.
 509     clang_tidy,
 510     # Build that attempts to duplicate the build OSS-Fuzz does. Currently
 511     # failing.
 512     oss_fuzz_build,
 513     bazel_test,
 514     cmake_tests,
 515     gn_nanopb_build,
 516 )
 517
 518 QUICK = (
 519     commit_message_format,
 520     source_is_in_build_files,
 521     copyright_notice,
 522     format_code.presubmit_checks(),
 523     pw_presubmit.pragma_once,
 524     gn_quick_build_check,
 525     # TODO(pwbug/141): Re-enable CMake and Bazel for Mac after we have fixed the
 526     # the clang issues. The problem is that all clang++ invocations need the
 527     # two extra flags: "-nostdc++" and "${clang_prefix}/../lib/libc++.a".
 528     cmake_tests if sys.platform != 'darwin' else (),
 529 )
 530
 531 FULL = (
 532     commit_message_format,
 533     init_cipd,
 534     init_virtualenv,
 535     copyright_notice,
 536     format_code.presubmit_checks(),
 537     pw_presubmit.pragma_once,
 538     gn_clang_build,
 539     gn_arm_build,
 540     gn_docs_build,
 541     gn_host_tools,
 542     # On Mac OS, system 'gcc' is a symlink to 'clang' by default, so skip GCC
 543     # host builds on Mac for now.
 544     gn_gcc_build if sys.platform != 'darwin' else (),
 545     # Windows doesn't support QEMU yet.
 546     gn_qemu_build if sys.platform != 'win32' else (),
 547     gn_qemu_clang_build if sys.platform != 'win32' else (),
 548     source_is_in_build_files,
 549     python_checks,
 550     build_env_setup,
 551     # Skip gn_teensy_build if running on Windows. The Teensycore installer is
 552     # an exe that requires an admin role.
 553     gn_teensy_build if sys.platform in ['linux', 'darwin'] else (),
 554 )
 555
 556 PROGRAMS = Programs(broken=BROKEN, quick=QUICK, full=FULL)
 557
 558
 559 def parse_args() -> argparse.Namespace:
 560     """Creates an argument parser and parses arguments."""
 561
 562     parser = argparse.ArgumentParser(description=__doc__)
 563     cli.add_arguments(parser, PROGRAMS, 'quick')
 564     parser.add_argument(
 565         '--install',
 566         action='store_true',
 567         help='Install the presubmit as a Git pre-push hook and exit.')
 568
 569     return parser.parse_args()
 570
 571
 572 def run(install: bool, **presubmit_args) -> int:
 573     """Entry point for presubmit."""
 574
 575     if install:
 576         install_hook(__file__, 'pre-push',
 577                      ['--base', 'origin/master..HEAD', '--program', 'quick'],
 578                      Path.cwd())
 579         return 0
 580
 581     return cli.run(**presubmit_args)
 582
 583
 584 def main() -> int:
 585     """Run the presubmit for the Pigweed repository."""
 586     return run(**vars(parse_args()))
 587
 588
 589 if __name__ == '__main__':
 590     try:
 591         # If pw_cli is available, use it to initialize logs.
 592         from pw_cli import log
 593
 594         log.install(logging.INFO)
 595     except ImportError:
 596         # If pw_cli isn't available, display log messages like a simple print.
 597         logging.basicConfig(format='%(message)s', level=logging.INFO)
 598
 599     sys.exit(main())