test/core/end2end/generate_tests.bzl

   1 #!/usr/bin/env python2.7
   2 # Copyright 2015 gRPC authors.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """Generates the appropriate build.json data for all the end2end tests."""
  17
  18 load("//bazel:grpc_build_system.bzl", "grpc_cc_binary", "grpc_cc_library")
  19
  20 POLLERS = ["epollex", "epoll1", "poll"]
  21
  22 def _fixture_options(
  23         fullstack = True,
  24         includes_proxy = False,
  25         dns_resolver = True,
  26         name_resolution = True,
  27         secure = True,
  28         tracing = False,
  29         _platforms = ["windows", "linux", "mac", "posix"],
  30         is_inproc = False,
  31         is_http2 = True,
  32         supports_proxy_auth = False,
  33         supports_write_buffering = True,
  34         client_channel = True,
  35         supports_msvc = True):
  36     return struct(
  37         fullstack = fullstack,
  38         includes_proxy = includes_proxy,
  39         dns_resolver = dns_resolver,
  40         name_resolution = name_resolution,
  41         secure = secure,
  42         tracing = tracing,
  43         is_inproc = is_inproc,
  44         is_http2 = is_http2,
  45         supports_proxy_auth = supports_proxy_auth,
  46         supports_write_buffering = supports_write_buffering,
  47         client_channel = client_channel,
  48         supports_msvc = supports_msvc,
  49         #_platforms=_platforms,
  50     )
  51
  52 # maps fixture name to whether it requires the security library
  53 END2END_FIXTURES = {
  54     "h2_compress": _fixture_options(),
  55     "h2_census": _fixture_options(),
  56     # TODO(juanlishen): This is disabled for now, but should be considered to re-enable once we have
  57     # decided how the load reporting service should be enabled.
  58     #'h2_load_reporting': _fixture_options(),
  59     "h2_fakesec": _fixture_options(),
  60     "h2_fd": _fixture_options(
  61         dns_resolver = False,
  62         fullstack = False,
  63         client_channel = False,
  64         _platforms = ["linux", "mac", "posix"],
  65     ),
  66     "h2_full": _fixture_options(),
  67     "h2_full+pipe": _fixture_options(_platforms = ["linux"]),
  68     "h2_full+trace": _fixture_options(tracing = True),
  69     "h2_full+workarounds": _fixture_options(),
  70     "h2_http_proxy": _fixture_options(supports_proxy_auth = True),
  71     "h2_oauth2": _fixture_options(),
  72     "h2_proxy": _fixture_options(includes_proxy = True),
  73     "h2_sockpair_1byte": _fixture_options(
  74         fullstack = False,
  75         dns_resolver = False,
  76         client_channel = False,
  77     ),
  78     "h2_sockpair": _fixture_options(
  79         fullstack = False,
  80         dns_resolver = False,
  81         client_channel = False,
  82     ),
  83     "h2_sockpair+trace": _fixture_options(
  84         fullstack = False,
  85         dns_resolver = False,
  86         tracing = True,
  87         client_channel = False,
  88     ),
  89     "h2_ssl": _fixture_options(secure = True),
  90     "h2_ssl_cred_reload": _fixture_options(secure = True),
  91     "h2_spiffe": _fixture_options(secure = True),
  92     "h2_local_uds": _fixture_options(secure = True, dns_resolver = False, _platforms = ["linux", "mac", "posix"]),
  93     "h2_local_ipv4": _fixture_options(secure = True, dns_resolver = False, _platforms = ["linux", "mac", "posix"]),
  94     "h2_local_ipv6": _fixture_options(secure = True, dns_resolver = False, _platforms = ["linux", "mac", "posix"]),
  95     "h2_ssl_proxy": _fixture_options(includes_proxy = True, secure = True),
  96     "h2_uds": _fixture_options(
  97         dns_resolver = False,
  98         _platforms = ["linux", "mac", "posix"],
  99     ),
 100     "inproc": _fixture_options(
 101         secure = True,
 102         fullstack = False,
 103         dns_resolver = False,
 104         name_resolution = False,
 105         is_inproc = True,
 106         is_http2 = False,
 107         supports_write_buffering = False,
 108         client_channel = False,
 109     ),
 110 }
 111
 112 # maps fixture name to whether it requires the security library
 113 END2END_NOSEC_FIXTURES = {
 114     "h2_compress": _fixture_options(secure = False),
 115     "h2_census": _fixture_options(secure = False),
 116     # TODO(juanlishen): This is disabled for now, but should be considered to re-enable once we have
 117     # decided how the load reporting service should be enabled.
 118     #'h2_load_reporting': _fixture_options(),
 119     "h2_fakesec": _fixture_options(),
 120     "h2_fd": _fixture_options(
 121         dns_resolver = False,
 122         fullstack = False,
 123         client_channel = False,
 124         secure = False,
 125         _platforms = ["linux", "mac", "posix"],
 126         supports_msvc = False,
 127     ),
 128     "h2_full": _fixture_options(secure = False),
 129     "h2_full+pipe": _fixture_options(secure = False, _platforms = ["linux"], supports_msvc = False),
 130     "h2_full+trace": _fixture_options(secure = False, tracing = True, supports_msvc = False),
 131     "h2_full+workarounds": _fixture_options(secure = False),
 132     "h2_http_proxy": _fixture_options(secure = False, supports_proxy_auth = True),
 133     "h2_proxy": _fixture_options(secure = False, includes_proxy = True),
 134     "h2_sockpair_1byte": _fixture_options(
 135         fullstack = False,
 136         dns_resolver = False,
 137         client_channel = False,
 138         secure = False,
 139     ),
 140     "h2_sockpair": _fixture_options(
 141         fullstack = False,
 142         dns_resolver = False,
 143         client_channel = False,
 144         secure = False,
 145     ),
 146     "h2_sockpair+trace": _fixture_options(
 147         fullstack = False,
 148         dns_resolver = False,
 149         tracing = True,
 150         secure = False,
 151         client_channel = False,
 152     ),
 153     "h2_ssl": _fixture_options(secure = False),
 154     "h2_ssl_cred_reload": _fixture_options(secure = False),
 155     "h2_ssl_proxy": _fixture_options(includes_proxy = True, secure = False),
 156     "h2_uds": _fixture_options(
 157         dns_resolver = False,
 158         _platforms = ["linux", "mac", "posix"],
 159         secure = False,
 160         supports_msvc = False,
 161     ),
 162 }
 163
 164 def _test_options(
 165         needs_fullstack = False,
 166         needs_dns = False,
 167         needs_names = False,
 168         proxyable = True,
 169         secure = False,
 170         traceable = False,
 171         exclude_inproc = False,
 172         needs_http2 = False,
 173         needs_proxy_auth = False,
 174         needs_write_buffering = False,
 175         needs_client_channel = False):
 176     return struct(
 177         needs_fullstack = needs_fullstack,
 178         needs_dns = needs_dns,
 179         needs_names = needs_names,
 180         proxyable = proxyable,
 181         secure = secure,
 182         traceable = traceable,
 183         exclude_inproc = exclude_inproc,
 184         needs_http2 = needs_http2,
 185         needs_proxy_auth = needs_proxy_auth,
 186         needs_write_buffering = needs_write_buffering,
 187         needs_client_channel = needs_client_channel,
 188     )
 189
 190 # maps test names to options
 191 END2END_TESTS = {
 192     "bad_hostname": _test_options(needs_names = True),
 193     "bad_ping": _test_options(needs_fullstack = True, proxyable = False),
 194     "binary_metadata": _test_options(),
 195     "resource_quota_server": _test_options(proxyable = False),
 196     "call_creds": _test_options(secure = True),
 197     "call_host_override": _test_options(
 198         needs_fullstack = True,
 199         needs_dns = True,
 200         needs_names = True,
 201     ),
 202     "cancel_after_accept": _test_options(),
 203     "cancel_after_client_done": _test_options(),
 204     "cancel_after_invoke": _test_options(),
 205     "cancel_after_round_trip": _test_options(),
 206     "cancel_before_invoke": _test_options(),
 207     "cancel_in_a_vacuum": _test_options(),
 208     "cancel_with_status": _test_options(),
 209     "compressed_payload": _test_options(proxyable = False, exclude_inproc = True),
 210     "connectivity": _test_options(
 211         needs_fullstack = True,
 212         needs_names = True,
 213         proxyable = False,
 214     ),
 215     "channelz": _test_options(),
 216     "default_host": _test_options(
 217         needs_fullstack = True,
 218         needs_dns = True,
 219         needs_names = True,
 220     ),
 221     "disappearing_server": _test_options(needs_fullstack = True, needs_names = True),
 222     "empty_batch": _test_options(),
 223     "filter_causes_close": _test_options(),
 224     "filter_call_init_fails": _test_options(),
 225     "filter_context": _test_options(),
 226     "graceful_server_shutdown": _test_options(exclude_inproc = True),
 227     "hpack_size": _test_options(
 228         proxyable = False,
 229         traceable = False,
 230         exclude_inproc = True,
 231     ),
 232     "high_initial_seqno": _test_options(),
 233     "idempotent_request": _test_options(),
 234     "invoke_large_request": _test_options(),
 235     "keepalive_timeout": _test_options(proxyable = False, needs_http2 = True),
 236     "large_metadata": _test_options(),
 237     "max_concurrent_streams": _test_options(
 238         proxyable = False,
 239         exclude_inproc = True,
 240     ),
 241     "max_connection_age": _test_options(exclude_inproc = True),
 242     "max_connection_idle": _test_options(needs_fullstack = True, proxyable = False),
 243     "max_message_length": _test_options(),
 244     "negative_deadline": _test_options(),
 245     "no_error_on_hotpath": _test_options(proxyable = False),
 246     "no_logging": _test_options(traceable = False),
 247     "no_op": _test_options(),
 248     "payload": _test_options(),
 249     # TODO(juanlishen): This is disabled for now because it depends on some generated functions in
 250     # end2end_tests.cc, which are not generated because they would depend on OpenCensus while
 251     # OpenCensus can only be built via Bazel so far.
 252     # 'load_reporting_hook': _test_options(),
 253     "ping_pong_streaming": _test_options(),
 254     "ping": _test_options(needs_fullstack = True, proxyable = False),
 255     "proxy_auth": _test_options(needs_proxy_auth = True),
 256     "registered_call": _test_options(),
 257     "request_with_flags": _test_options(proxyable = False),
 258     "request_with_payload": _test_options(),
 259     # TODO(roth): Remove proxyable=False for all retry tests once we
 260     # have a way for the proxy to propagate the fact that trailing
 261     # metadata is available when initial metadata is returned.
 262     # See https://github.com/grpc/grpc/issues/14467 for context.
 263     "retry": _test_options(needs_client_channel = True, proxyable = False),
 264     "retry_cancellation": _test_options(
 265         needs_client_channel = True,
 266         proxyable = False,
 267     ),
 268     "retry_disabled": _test_options(needs_client_channel = True, proxyable = False),
 269     "retry_exceeds_buffer_size_in_initial_batch": _test_options(
 270         needs_client_channel = True,
 271         proxyable = False,
 272     ),
 273     "retry_exceeds_buffer_size_in_subsequent_batch": _test_options(
 274         needs_client_channel = True,
 275         proxyable = False,
 276     ),
 277     "retry_non_retriable_status": _test_options(
 278         needs_client_channel = True,
 279         proxyable = False,
 280     ),
 281     "retry_non_retriable_status_before_recv_trailing_metadata_started": _test_options(needs_client_channel = True, proxyable = False),
 282     "retry_recv_initial_metadata": _test_options(
 283         needs_client_channel = True,
 284         proxyable = False,
 285     ),
 286     "retry_recv_message": _test_options(
 287         needs_client_channel = True,
 288         proxyable = False,
 289     ),
 290     "retry_server_pushback_delay": _test_options(
 291         needs_client_channel = True,
 292         proxyable = False,
 293     ),
 294     "retry_server_pushback_disabled": _test_options(
 295         needs_client_channel = True,
 296         proxyable = False,
 297     ),
 298     "retry_streaming": _test_options(needs_client_channel = True, proxyable = False),
 299     "retry_streaming_after_commit": _test_options(
 300         needs_client_channel = True,
 301         proxyable = False,
 302     ),
 303     "retry_streaming_succeeds_before_replay_finished": _test_options(
 304         needs_client_channel = True,
 305         proxyable = False,
 306     ),
 307     "retry_throttled": _test_options(
 308         needs_client_channel = True,
 309         proxyable = False,
 310     ),
 311     "retry_too_many_attempts": _test_options(
 312         needs_client_channel = True,
 313         proxyable = False,
 314     ),
 315     "server_finishes_request": _test_options(),
 316     "shutdown_finishes_calls": _test_options(),
 317     "shutdown_finishes_tags": _test_options(),
 318     "simple_cacheable_request": _test_options(),
 319     "simple_delayed_request": _test_options(needs_fullstack = True),
 320     "simple_metadata": _test_options(),
 321     "simple_request": _test_options(),
 322     "streaming_error_response": _test_options(),
 323     "stream_compression_compressed_payload": _test_options(
 324         proxyable = False,
 325         exclude_inproc = True,
 326     ),
 327     "stream_compression_payload": _test_options(exclude_inproc = True),
 328     "stream_compression_ping_pong_streaming": _test_options(exclude_inproc = True),
 329     "trailing_metadata": _test_options(),
 330     "authority_not_supported": _test_options(),
 331     "filter_latency": _test_options(),
 332     "filter_status_code": _test_options(),
 333     "workaround_cronet_compression": _test_options(),
 334     "write_buffering": _test_options(needs_write_buffering = True),
 335     "write_buffering_at_end": _test_options(needs_write_buffering = True),
 336 }
 337
 338 def _compatible(fopt, topt):
 339     if topt.needs_fullstack:
 340         if not fopt.fullstack:
 341             return False
 342     if topt.needs_dns:
 343         if not fopt.dns_resolver:
 344             return False
 345     if topt.needs_names:
 346         if not fopt.name_resolution:
 347             return False
 348     if not topt.proxyable:
 349         if fopt.includes_proxy:
 350             return False
 351     if not topt.traceable:
 352         if fopt.tracing:
 353             return False
 354     if topt.exclude_inproc:
 355         if fopt.is_inproc:
 356             return False
 357     if topt.needs_http2:
 358         if not fopt.is_http2:
 359             return False
 360     if topt.needs_proxy_auth:
 361         if not fopt.supports_proxy_auth:
 362             return False
 363     if topt.needs_write_buffering:
 364         if not fopt.supports_write_buffering:
 365             return False
 366     if topt.needs_client_channel:
 367         if not fopt.client_channel:
 368             return False
 369     return True
 370
 371 def grpc_end2end_tests():
 372     grpc_cc_library(
 373         name = "end2end_tests",
 374         srcs = ["end2end_tests.cc", "end2end_test_utils.cc"] + [
 375             "tests/%s.cc" % t
 376             for t in sorted(END2END_TESTS.keys())
 377         ],
 378         hdrs = [
 379             "tests/cancel_test_helpers.h",
 380             "end2end_tests.h",
 381         ],
 382         language = "C++",
 383         deps = [
 384             ":cq_verifier",
 385             ":ssl_test_data",
 386             ":http_proxy",
 387             ":proxy",
 388             ":local_util",
 389         ],
 390         tags = ["no_windows"],
 391     )
 392
 393     for f, fopt in END2END_FIXTURES.items():
 394         grpc_cc_binary(
 395             name = "%s_test" % f,
 396             srcs = ["fixtures/%s.cc" % f],
 397             language = "C++",
 398             deps = [
 399                 ":end2end_tests",
 400                 "//test/core/util:grpc_test_util",
 401                 "//:grpc",
 402                 "//:gpr",
 403             ],
 404             tags = ["no_windows"],
 405         )
 406         for t, topt in END2END_TESTS.items():
 407             #print(_compatible(fopt, topt), f, t, fopt, topt)
 408             if not _compatible(fopt, topt):
 409                 continue
 410             for poller in POLLERS:
 411                 native.sh_test(
 412                     name = "%s_test@%s@poller=%s" % (f, t, poller),
 413                     data = [":%s_test" % f],
 414                     srcs = ["end2end_test.sh"],
 415                     args = [
 416                         "$(location %s_test)" % f,
 417                         t,
 418                         poller,
 419                     ],
 420                     tags = ["no_windows"],
 421                 )
 422
 423 def grpc_end2end_nosec_tests():
 424     grpc_cc_library(
 425         name = "end2end_nosec_tests",
 426         srcs = ["end2end_nosec_tests.cc", "end2end_test_utils.cc"] + [
 427             "tests/%s.cc" % t
 428             for t in sorted(END2END_TESTS.keys())
 429             if not END2END_TESTS[t].secure
 430         ],
 431         hdrs = [
 432             "tests/cancel_test_helpers.h",
 433             "end2end_tests.h",
 434         ],
 435         language = "C++",
 436         deps = [
 437             ":cq_verifier",
 438             ":ssl_test_data",
 439             ":http_proxy",
 440             ":proxy",
 441             ":local_util",
 442         ],
 443         tags = ["no_windows"],
 444     )
 445
 446     for f, fopt in END2END_NOSEC_FIXTURES.items():
 447         if fopt.secure:
 448             continue
 449         grpc_cc_binary(
 450             name = "%s_nosec_test" % f,
 451             srcs = ["fixtures/%s.cc" % f],
 452             language = "C++",
 453             deps = [
 454                 ":end2end_nosec_tests",
 455                 "//test/core/util:grpc_test_util_unsecure",
 456                 "//:grpc_unsecure",
 457                 "//:gpr",
 458             ],
 459             tags = ["no_windows"],
 460         )
 461         for t, topt in END2END_TESTS.items():
 462             #print(_compatible(fopt, topt), f, t, fopt, topt)
 463             if not _compatible(fopt, topt):
 464                 continue
 465             if topt.secure:
 466                 continue
 467             for poller in POLLERS:
 468                 native.sh_test(
 469                     name = "%s_nosec_test@%s@poller=%s" % (f, t, poller),
 470                     data = [":%s_nosec_test" % f],
 471                     srcs = ["end2end_test.sh"],
 472                     args = [
 473                         "$(location %s_nosec_test)" % f,
 474                         t,
 475                         poller,
 476                     ],
 477                     tags = ["no_windows"],
 478                 )