tools/run_tests/sanity/check_bazel_workspace.py

   1 #!/usr/bin/env python
   2
   3 # Copyright 2016 gRPC authors.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 from __future__ import print_function
  18
  19 import ast
  20 import os
  21 import re
  22 import subprocess
  23 import sys
  24
  25 os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..'))
  26
  27 git_hash_pattern = re.compile('[0-9a-f]{40}')
  28
  29 # Parse git hashes from submodules
  30 git_submodules = subprocess.check_output('git submodule',
  31                                          shell=True).strip().split('\n')
  32 git_submodule_hashes = {
  33     re.search(git_hash_pattern, s).group() for s in git_submodules
  34 }
  35
  36 _BAZEL_SKYLIB_DEP_NAME = 'bazel_skylib'
  37 _BAZEL_TOOLCHAINS_DEP_NAME = 'bazel_toolchains'
  38 _BAZEL_COMPDB_DEP_NAME = 'bazel_compdb'
  39 _TWISTED_TWISTED_DEP_NAME = 'com_github_twisted_twisted'
  40 _YAML_PYYAML_DEP_NAME = 'com_github_yaml_pyyaml'
  41 _TWISTED_INCREMENTAL_DEP_NAME = 'com_github_twisted_incremental'
  42 _ZOPEFOUNDATION_ZOPE_INTERFACE_DEP_NAME = 'com_github_zopefoundation_zope_interface'
  43 _TWISTED_CONSTANTLY_DEP_NAME = 'com_github_twisted_constantly'
  44
  45 _GRPC_DEP_NAMES = [
  46     'upb',
  47     'boringssl',
  48     'zlib',
  49     'com_google_protobuf',
  50     'com_google_googletest',
  51     'rules_cc',
  52     'com_github_google_benchmark',
  53     'com_github_cares_cares',
  54     'com_google_absl',
  55     'io_opencensus_cpp',
  56     'envoy_api',
  57     _BAZEL_SKYLIB_DEP_NAME,
  58     _BAZEL_TOOLCHAINS_DEP_NAME,
  59     _BAZEL_COMPDB_DEP_NAME,
  60     _TWISTED_TWISTED_DEP_NAME,
  61     _YAML_PYYAML_DEP_NAME,
  62     _TWISTED_INCREMENTAL_DEP_NAME,
  63     _ZOPEFOUNDATION_ZOPE_INTERFACE_DEP_NAME,
  64     _TWISTED_CONSTANTLY_DEP_NAME,
  65     'io_bazel_rules_go',
  66     'build_bazel_rules_apple',
  67     'build_bazel_apple_support',
  68     'libuv',
  69     'com_github_google_re2',
  70 ]
  71
  72 _GRPC_BAZEL_ONLY_DEPS = [
  73     'upb',  # third_party/upb is checked in locally
  74     'rules_cc',
  75     'com_google_absl',
  76     'io_opencensus_cpp',
  77     _BAZEL_SKYLIB_DEP_NAME,
  78     _BAZEL_TOOLCHAINS_DEP_NAME,
  79     _BAZEL_COMPDB_DEP_NAME,
  80     _TWISTED_TWISTED_DEP_NAME,
  81     _YAML_PYYAML_DEP_NAME,
  82     _TWISTED_INCREMENTAL_DEP_NAME,
  83     _ZOPEFOUNDATION_ZOPE_INTERFACE_DEP_NAME,
  84     _TWISTED_CONSTANTLY_DEP_NAME,
  85     'io_bazel_rules_go',
  86     'build_bazel_rules_apple',
  87     'build_bazel_apple_support',
  88 ]
  89
  90
  91 class BazelEvalState(object):
  92
  93     def __init__(self, names_and_urls, overridden_name=None):
  94         self.names_and_urls = names_and_urls
  95         self.overridden_name = overridden_name
  96
  97     def http_archive(self, **args):
  98         self.archive(**args)
  99
 100     def new_http_archive(self, **args):
 101         self.archive(**args)
 102
 103     def bind(self, **args):
 104         pass
 105
 106     def existing_rules(self):
 107         if self.overridden_name:
 108             return [self.overridden_name]
 109         return []
 110
 111     def archive(self, **args):
 112         assert self.names_and_urls.get(args['name']) is None
 113         if args['name'] in _GRPC_BAZEL_ONLY_DEPS:
 114             self.names_and_urls[args['name']] = 'dont care'
 115             return
 116         url = args.get('url', None)
 117         if not url:
 118             # we will only be looking for git commit hashes, so concatenating
 119             # the urls is fine.
 120             url = ' '.join(args['urls'])
 121         self.names_and_urls[args['name']] = url
 122
 123     def git_repository(self, **args):
 124         assert self.names_and_urls.get(args['name']) is None
 125         if args['name'] in _GRPC_BAZEL_ONLY_DEPS:
 126             self.names_and_urls[args['name']] = 'dont care'
 127             return
 128         self.names_and_urls[args['name']] = args['remote']
 129
 130     def grpc_python_deps(self):
 131         pass
 132
 133
 134 # Parse git hashes from bazel/grpc_deps.bzl {new_}http_archive rules
 135 with open(os.path.join('bazel', 'grpc_deps.bzl'), 'r') as f:
 136     names_and_urls = {}
 137     eval_state = BazelEvalState(names_and_urls)
 138     bazel_file = f.read()
 139
 140 # grpc_deps.bzl only defines 'grpc_deps' and 'grpc_test_only_deps', add these
 141 # lines to call them.
 142 bazel_file += '\ngrpc_deps()\n'
 143 bazel_file += '\ngrpc_test_only_deps()\n'
 144 build_rules = {
 145     'native': eval_state,
 146     'http_archive': lambda **args: eval_state.http_archive(**args),
 147     'load': lambda a, b: None,
 148     'git_repository': lambda **args: eval_state.git_repository(**args),
 149     'grpc_python_deps': lambda: None,
 150 }
 151 exec(bazel_file) in build_rules
 152 for name in _GRPC_DEP_NAMES:
 153     assert name in names_and_urls.keys()
 154 if len(_GRPC_DEP_NAMES) != len(names_and_urls.keys()):
 155     assert False, "Diff: " + (str(set(_GRPC_DEP_NAMES) - set(names_and_urls)) +
 156                               "," +
 157                               str(set(names_and_urls) - set(_GRPC_DEP_NAMES)))
 158
 159 # There are some "bazel-only" deps that are exceptions to this sanity check,
 160 # we don't require that there is a corresponding git module for these.
 161 names_without_bazel_only_deps = names_and_urls.keys()
 162 for dep_name in _GRPC_BAZEL_ONLY_DEPS:
 163     names_without_bazel_only_deps.remove(dep_name)
 164 archive_urls = [names_and_urls[name] for name in names_without_bazel_only_deps]
 165 workspace_git_hashes = {
 166     re.search(git_hash_pattern, url).group() for url in archive_urls
 167 }
 168 if len(workspace_git_hashes) == 0:
 169     print("(Likely) parse error, did not find any bazel git dependencies.")
 170     sys.exit(1)
 171
 172 # Validate the equivalence of the git submodules and Bazel git dependencies. The
 173 # condition we impose is that there is a git submodule for every dependency in
 174 # the workspace, but not necessarily conversely. E.g. Bloaty is a dependency
 175 # not used by any of the targets built by Bazel.
 176 if len(workspace_git_hashes - git_submodule_hashes) > 0:
 177     print(
 178         "Found discrepancies between git submodules and Bazel WORKSPACE dependencies"
 179     )
 180     print("workspace_git_hashes: %s" % workspace_git_hashes)
 181     print("git_submodule_hashes: %s" % git_submodule_hashes)
 182     print("workspace_git_hashes - git_submodule_hashes: %s" %
 183           (workspace_git_hashes - git_submodule_hashes))
 184     sys.exit(1)
 185
 186 # Also check that we can override each dependency
 187 for name in _GRPC_DEP_NAMES:
 188     names_and_urls_with_overridden_name = {}
 189     state = BazelEvalState(names_and_urls_with_overridden_name,
 190                            overridden_name=name)
 191     rules = {
 192         'native': state,
 193         'http_archive': lambda **args: state.http_archive(**args),
 194         'load': lambda a, b: None,
 195         'git_repository': lambda **args: state.git_repository(**args),
 196         'grpc_python_deps': lambda *args, **kwargs: None,
 197     }
 198     exec(bazel_file) in rules
 199     assert name not in names_and_urls_with_overridden_name.keys()
 200
 201 sys.exit(0)