tools/run_tests/performance/scenario_config.py

   1 # Copyright 2016 gRPC authors.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # performance scenario configuration for various languages
  16
  17 import math
  18
  19 WARMUP_SECONDS = 5
  20 JAVA_WARMUP_SECONDS = 15  # Java needs more warmup time for JIT to kick in.
  21 BENCHMARK_SECONDS = 30
  22
  23 SMOKETEST = 'smoketest'
  24 SCALABLE = 'scalable'
  25 INPROC = 'inproc'
  26 SWEEP = 'sweep'
  27 DEFAULT_CATEGORIES = [SCALABLE, SMOKETEST]
  28
  29 SECURE_SECARGS = {
  30     'use_test_ca': True,
  31     'server_host_override': 'foo.test.google.fr'
  32 }
  33
  34 HISTOGRAM_PARAMS = {
  35     'resolution': 0.01,
  36     'max_possible': 60e9,
  37 }
  38
  39 # target number of RPCs outstanding on across all client channels in
  40 # non-ping-pong tests (since we can only specify per-channel numbers, the
  41 # actual target will be slightly higher)
  42 OUTSTANDING_REQUESTS = {'async': 6400, 'async-limited': 800, 'sync': 1000}
  43
  44 # wide is the number of client channels in multi-channel tests (1 otherwise)
  45 WIDE = 64
  46
  47
  48 def _get_secargs(is_secure):
  49     if is_secure:
  50         return SECURE_SECARGS
  51     else:
  52         return None
  53
  54
  55 def remove_nonproto_fields(scenario):
  56     """Remove special-purpose that contains some extra info about the scenario
  57   but don't belong to the ScenarioConfig protobuf message"""
  58     scenario.pop('CATEGORIES', None)
  59     scenario.pop('CLIENT_LANGUAGE', None)
  60     scenario.pop('SERVER_LANGUAGE', None)
  61     scenario.pop('EXCLUDED_POLL_ENGINES', None)
  62     return scenario
  63
  64
  65 def geometric_progression(start, stop, step):
  66     n = start
  67     while n < stop:
  68         yield int(round(n))
  69         n *= step
  70
  71
  72 def _payload_type(use_generic_payload, req_size, resp_size):
  73     r = {}
  74     sizes = {
  75         'req_size': req_size,
  76         'resp_size': resp_size,
  77     }
  78     if use_generic_payload:
  79         r['bytebuf_params'] = sizes
  80     else:
  81         r['simple_params'] = sizes
  82     return r
  83
  84
  85 def _load_params(offered_load):
  86     r = {}
  87     if offered_load is None:
  88         r['closed_loop'] = {}
  89     else:
  90         load = {}
  91         load['offered_load'] = offered_load
  92         r['poisson'] = load
  93     return r
  94
  95
  96 def _add_channel_arg(config, key, value):
  97     if 'channel_args' in config:
  98         channel_args = config['channel_args']
  99     else:
 100         channel_args = []
 101         config['channel_args'] = channel_args
 102     arg = {'name': key}
 103     if isinstance(value, int):
 104         arg['int_value'] = value
 105     else:
 106         arg['str_value'] = value
 107     channel_args.append(arg)
 108
 109
 110 def _ping_pong_scenario(name,
 111                         rpc_type,
 112                         client_type,
 113                         server_type,
 114                         secure=True,
 115                         use_generic_payload=False,
 116                         req_size=0,
 117                         resp_size=0,
 118                         unconstrained_client=None,
 119                         client_language=None,
 120                         server_language=None,
 121                         async_server_threads=0,
 122                         server_threads_per_cq=0,
 123                         client_threads_per_cq=0,
 124                         warmup_seconds=WARMUP_SECONDS,
 125                         categories=DEFAULT_CATEGORIES,
 126                         channels=None,
 127                         outstanding=None,
 128                         num_clients=None,
 129                         resource_quota_size=None,
 130                         messages_per_stream=None,
 131                         excluded_poll_engines=[],
 132                         minimal_stack=False,
 133                         offered_load=None):
 134     """Creates a basic ping pong scenario."""
 135     scenario = {
 136         'name': name,
 137         'num_servers': 1,
 138         'num_clients': 1,
 139         'client_config': {
 140             'client_type': client_type,
 141             'security_params': _get_secargs(secure),
 142             'outstanding_rpcs_per_channel': 1,
 143             'client_channels': 1,
 144             'async_client_threads': 1,
 145             'threads_per_cq': client_threads_per_cq,
 146             'rpc_type': rpc_type,
 147             'histogram_params': HISTOGRAM_PARAMS,
 148             'channel_args': [],
 149         },
 150         'server_config': {
 151             'server_type': server_type,
 152             'security_params': _get_secargs(secure),
 153             'async_server_threads': async_server_threads,
 154             'threads_per_cq': server_threads_per_cq,
 155             'channel_args': [],
 156         },
 157         'warmup_seconds': warmup_seconds,
 158         'benchmark_seconds': BENCHMARK_SECONDS
 159     }
 160     if resource_quota_size:
 161         scenario['server_config']['resource_quota_size'] = resource_quota_size
 162     if use_generic_payload:
 163         if server_type != 'ASYNC_GENERIC_SERVER':
 164             raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.')
 165         scenario['server_config']['payload_config'] = _payload_type(
 166             use_generic_payload, req_size, resp_size)
 167
 168     scenario['client_config']['payload_config'] = _payload_type(
 169         use_generic_payload, req_size, resp_size)
 170
 171     # Optimization target of 'throughput' does not work well with epoll1 polling
 172     # engine. Use the default value of 'blend'
 173     optimization_target = 'throughput'
 174
 175     if unconstrained_client:
 176         outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[
 177             unconstrained_client]
 178         # clamp buffer usage to something reasonable (16 gig for now)
 179         MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024
 180         if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE:
 181             outstanding_calls = max(1,
 182                                     MAX_MEMORY_USE / max(req_size, resp_size))
 183         wide = channels if channels is not None else WIDE
 184         deep = int(math.ceil(1.0 * outstanding_calls / wide))
 185
 186         scenario[
 187             'num_clients'] = num_clients if num_clients is not None else 0  # use as many clients as available.
 188         scenario['client_config']['outstanding_rpcs_per_channel'] = deep
 189         scenario['client_config']['client_channels'] = wide
 190         scenario['client_config']['async_client_threads'] = 0
 191         if offered_load is not None:
 192             optimization_target = 'latency'
 193     else:
 194         scenario['client_config']['outstanding_rpcs_per_channel'] = 1
 195         scenario['client_config']['client_channels'] = 1
 196         scenario['client_config']['async_client_threads'] = 1
 197         optimization_target = 'latency'
 198
 199     scenario['client_config']['load_params'] = _load_params(offered_load)
 200
 201     optimization_channel_arg = {
 202         'name': 'grpc.optimization_target',
 203         'str_value': optimization_target
 204     }
 205     scenario['client_config']['channel_args'].append(optimization_channel_arg)
 206     scenario['server_config']['channel_args'].append(optimization_channel_arg)
 207
 208     if minimal_stack:
 209         _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1)
 210         _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1)
 211
 212     if messages_per_stream:
 213         scenario['client_config']['messages_per_stream'] = messages_per_stream
 214     if client_language:
 215         # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py
 216         scenario['CLIENT_LANGUAGE'] = client_language
 217     if server_language:
 218         # the SERVER_LANGUAGE field is recognized by run_performance_tests.py
 219         scenario['SERVER_LANGUAGE'] = server_language
 220     if categories:
 221         scenario['CATEGORIES'] = categories
 222     if len(excluded_poll_engines):
 223         # The polling engines for which this scenario is excluded
 224         scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines
 225     return scenario
 226
 227
 228 class CXXLanguage:
 229
 230     def __init__(self):
 231         self.safename = 'cxx'
 232
 233     def worker_cmdline(self):
 234         return ['cmake/build/qps_worker']
 235
 236     def worker_port_offset(self):
 237         return 0
 238
 239     def scenarios(self):
 240         # TODO(ctiller): add 70% load latency test
 241         yield _ping_pong_scenario(
 242             'cpp_protobuf_async_unary_1channel_100rpcs_1MB',
 243             rpc_type='UNARY',
 244             client_type='ASYNC_CLIENT',
 245             server_type='ASYNC_SERVER',
 246             req_size=1024 * 1024,
 247             resp_size=1024 * 1024,
 248             unconstrained_client='async',
 249             outstanding=100,
 250             channels=1,
 251             num_clients=1,
 252             secure=False,
 253             categories=[INPROC] + [SCALABLE])
 254
 255         yield _ping_pong_scenario(
 256             'cpp_protobuf_async_streaming_from_client_1channel_1MB',
 257             rpc_type='STREAMING_FROM_CLIENT',
 258             client_type='ASYNC_CLIENT',
 259             server_type='ASYNC_SERVER',
 260             req_size=1024 * 1024,
 261             resp_size=1024 * 1024,
 262             unconstrained_client='async',
 263             outstanding=1,
 264             channels=1,
 265             num_clients=1,
 266             secure=False,
 267             categories=[SMOKETEST] + [INPROC] + [SCALABLE])
 268
 269         yield _ping_pong_scenario(
 270             'cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp',
 271             rpc_type='UNARY',
 272             client_type='ASYNC_CLIENT',
 273             server_type='ASYNC_SERVER',
 274             req_size=300,
 275             resp_size=50,
 276             unconstrained_client='async',
 277             outstanding=30000,
 278             channels=300,
 279             offered_load=37500,
 280             secure=False,
 281             async_server_threads=16,
 282             server_threads_per_cq=1,
 283             categories=[SCALABLE])
 284
 285         for secure in [True, False]:
 286             secstr = 'secure' if secure else 'insecure'
 287             smoketest_categories = ([SMOKETEST] if secure else [])
 288             inproc_categories = ([INPROC] if not secure else [])
 289
 290             yield _ping_pong_scenario(
 291                 'cpp_generic_async_streaming_ping_pong_%s' % secstr,
 292                 rpc_type='STREAMING',
 293                 client_type='ASYNC_CLIENT',
 294                 server_type='ASYNC_GENERIC_SERVER',
 295                 use_generic_payload=True,
 296                 async_server_threads=1,
 297                 secure=secure,
 298                 categories=smoketest_categories + inproc_categories +
 299                 [SCALABLE])
 300
 301             yield _ping_pong_scenario(
 302                 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr,
 303                 rpc_type='STREAMING',
 304                 client_type='ASYNC_CLIENT',
 305                 server_type='ASYNC_GENERIC_SERVER',
 306                 unconstrained_client='async',
 307                 use_generic_payload=True,
 308                 secure=secure,
 309                 minimal_stack=not secure,
 310                 categories=smoketest_categories + inproc_categories +
 311                 [SCALABLE])
 312
 313             for mps in geometric_progression(1, 20, 10):
 314                 yield _ping_pong_scenario(
 315                     'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' %
 316                     (mps, secstr),
 317                     rpc_type='STREAMING',
 318                     client_type='ASYNC_CLIENT',
 319                     server_type='ASYNC_GENERIC_SERVER',
 320                     unconstrained_client='async',
 321                     use_generic_payload=True,
 322                     secure=secure,
 323                     messages_per_stream=mps,
 324                     minimal_stack=not secure,
 325                     categories=smoketest_categories + inproc_categories +
 326                     [SCALABLE])
 327
 328             for mps in geometric_progression(1, 200, math.sqrt(10)):
 329                 yield _ping_pong_scenario(
 330                     'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' %
 331                     (mps, secstr),
 332                     rpc_type='STREAMING',
 333                     client_type='ASYNC_CLIENT',
 334                     server_type='ASYNC_GENERIC_SERVER',
 335                     unconstrained_client='async',
 336                     use_generic_payload=True,
 337                     secure=secure,
 338                     messages_per_stream=mps,
 339                     minimal_stack=not secure,
 340                     categories=[SWEEP])
 341
 342             yield _ping_pong_scenario(
 343                 'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr,
 344                 rpc_type='STREAMING',
 345                 req_size=1024 * 1024,
 346                 resp_size=1024 * 1024,
 347                 client_type='ASYNC_CLIENT',
 348                 server_type='ASYNC_GENERIC_SERVER',
 349                 unconstrained_client='async',
 350                 use_generic_payload=True,
 351                 secure=secure,
 352                 minimal_stack=not secure,
 353                 categories=inproc_categories + [SCALABLE],
 354                 channels=1,
 355                 outstanding=100)
 356
 357             yield _ping_pong_scenario(
 358                 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' %
 359                 secstr,
 360                 rpc_type='STREAMING',
 361                 req_size=64 * 1024,
 362                 resp_size=64 * 1024,
 363                 client_type='ASYNC_CLIENT',
 364                 server_type='ASYNC_GENERIC_SERVER',
 365                 unconstrained_client='async',
 366                 use_generic_payload=True,
 367                 secure=secure,
 368                 minimal_stack=not secure,
 369                 categories=inproc_categories + [SCALABLE])
 370
 371             yield _ping_pong_scenario(
 372                 'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr,
 373                 rpc_type='STREAMING',
 374                 client_type='ASYNC_CLIENT',
 375                 server_type='ASYNC_GENERIC_SERVER',
 376                 unconstrained_client='async-limited',
 377                 use_generic_payload=True,
 378                 secure=secure,
 379                 client_threads_per_cq=1000000,
 380                 server_threads_per_cq=1000000,
 381                 categories=smoketest_categories + inproc_categories +
 382                 [SCALABLE])
 383
 384             yield _ping_pong_scenario(
 385                 'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_%s'
 386                 % secstr,
 387                 rpc_type='STREAMING',
 388                 client_type='ASYNC_CLIENT',
 389                 server_type='ASYNC_GENERIC_SERVER',
 390                 unconstrained_client='async',
 391                 use_generic_payload=True,
 392                 secure=secure,
 393                 client_threads_per_cq=2,
 394                 server_threads_per_cq=2,
 395                 categories=inproc_categories + [SCALABLE])
 396
 397             yield _ping_pong_scenario(
 398                 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' %
 399                 secstr,
 400                 rpc_type='STREAMING',
 401                 client_type='ASYNC_CLIENT',
 402                 server_type='ASYNC_SERVER',
 403                 unconstrained_client='async-limited',
 404                 secure=secure,
 405                 client_threads_per_cq=1000000,
 406                 server_threads_per_cq=1000000,
 407                 categories=inproc_categories + [SCALABLE])
 408
 409             yield _ping_pong_scenario(
 410                 'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_%s'
 411                 % secstr,
 412                 rpc_type='STREAMING',
 413                 client_type='ASYNC_CLIENT',
 414                 server_type='ASYNC_SERVER',
 415                 unconstrained_client='async',
 416                 secure=secure,
 417                 client_threads_per_cq=2,
 418                 server_threads_per_cq=2,
 419                 categories=inproc_categories + [SCALABLE])
 420
 421             yield _ping_pong_scenario(
 422                 'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr,
 423                 rpc_type='UNARY',
 424                 client_type='ASYNC_CLIENT',
 425                 server_type='ASYNC_SERVER',
 426                 unconstrained_client='async-limited',
 427                 secure=secure,
 428                 client_threads_per_cq=1000000,
 429                 server_threads_per_cq=1000000,
 430                 categories=smoketest_categories + inproc_categories +
 431                 [SCALABLE])
 432
 433             yield _ping_pong_scenario(
 434                 'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_%s' %
 435                 secstr,
 436                 rpc_type='UNARY',
 437                 client_type='ASYNC_CLIENT',
 438                 server_type='ASYNC_SERVER',
 439                 unconstrained_client='async',
 440                 secure=secure,
 441                 client_threads_per_cq=2,
 442                 server_threads_per_cq=2,
 443                 categories=inproc_categories + [SCALABLE])
 444
 445             yield _ping_pong_scenario(
 446                 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr,
 447                 rpc_type='STREAMING',
 448                 client_type='ASYNC_CLIENT',
 449                 server_type='ASYNC_GENERIC_SERVER',
 450                 unconstrained_client='async-limited',
 451                 use_generic_payload=True,
 452                 async_server_threads=1,
 453                 minimal_stack=not secure,
 454                 secure=secure)
 455
 456             yield _ping_pong_scenario(
 457                 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s'
 458                 % (secstr),
 459                 rpc_type='UNARY',
 460                 client_type='ASYNC_CLIENT',
 461                 server_type='SYNC_SERVER',
 462                 unconstrained_client='async',
 463                 secure=secure,
 464                 minimal_stack=not secure,
 465                 categories=smoketest_categories + inproc_categories +
 466                 [SCALABLE])
 467
 468             yield _ping_pong_scenario(
 469                 'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s'
 470                 % (secstr),
 471                 rpc_type='UNARY',
 472                 client_type='ASYNC_CLIENT',
 473                 server_type='ASYNC_SERVER',
 474                 channels=1,
 475                 outstanding=64,
 476                 req_size=128,
 477                 resp_size=8 * 1024 * 1024,
 478                 secure=secure,
 479                 minimal_stack=not secure,
 480                 categories=inproc_categories + [SCALABLE])
 481
 482             yield _ping_pong_scenario(
 483                 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s'
 484                 % secstr,
 485                 rpc_type='STREAMING',
 486                 client_type='ASYNC_CLIENT',
 487                 server_type='SYNC_SERVER',
 488                 unconstrained_client='async',
 489                 secure=secure,
 490                 minimal_stack=not secure,
 491                 categories=smoketest_categories + inproc_categories +
 492                 [SCALABLE])
 493
 494             yield _ping_pong_scenario(
 495                 'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr,
 496                 rpc_type='UNARY',
 497                 client_type='ASYNC_CLIENT',
 498                 server_type='ASYNC_SERVER',
 499                 req_size=1024 * 1024,
 500                 resp_size=1024 * 1024,
 501                 secure=secure,
 502                 minimal_stack=not secure,
 503                 categories=smoketest_categories + inproc_categories +
 504                 [SCALABLE])
 505
 506             for rpc_type in [
 507                     'unary', 'streaming', 'streaming_from_client',
 508                     'streaming_from_server'
 509             ]:
 510                 for synchronicity in ['sync', 'async']:
 511                     yield _ping_pong_scenario(
 512                         'cpp_protobuf_%s_%s_ping_pong_%s' %
 513                         (synchronicity, rpc_type, secstr),
 514                         rpc_type=rpc_type.upper(),
 515                         client_type='%s_CLIENT' % synchronicity.upper(),
 516                         server_type='%s_SERVER' % synchronicity.upper(),
 517                         async_server_threads=1,
 518                         minimal_stack=not secure,
 519                         secure=secure)
 520
 521                     for size in geometric_progression(1, 1024 * 1024 * 1024 + 1,
 522                                                       8):
 523                         yield _ping_pong_scenario(
 524                             'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' %
 525                             (synchronicity, rpc_type, secstr, size),
 526                             rpc_type=rpc_type.upper(),
 527                             req_size=size,
 528                             resp_size=size,
 529                             client_type='%s_CLIENT' % synchronicity.upper(),
 530                             server_type='%s_SERVER' % synchronicity.upper(),
 531                             unconstrained_client=synchronicity,
 532                             secure=secure,
 533                             minimal_stack=not secure,
 534                             categories=[SWEEP])
 535
 536                     yield _ping_pong_scenario(
 537                         'cpp_protobuf_%s_%s_qps_unconstrained_%s' %
 538                         (synchronicity, rpc_type, secstr),
 539                         rpc_type=rpc_type.upper(),
 540                         client_type='%s_CLIENT' % synchronicity.upper(),
 541                         server_type='%s_SERVER' % synchronicity.upper(),
 542                         unconstrained_client=synchronicity,
 543                         secure=secure,
 544                         minimal_stack=not secure,
 545                         server_threads_per_cq=3,
 546                         client_threads_per_cq=3,
 547                         categories=inproc_categories + [SCALABLE])
 548
 549                     # TODO(vjpai): Re-enable this test. It has a lot of timeouts
 550                     # and hasn't yet been conclusively identified as a test failure
 551                     # or race in the library
 552                     # yield _ping_pong_scenario(
 553                     #     'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr),
 554                     #     rpc_type=rpc_type.upper(),
 555                     #     client_type='%s_CLIENT' % synchronicity.upper(),
 556                     #     server_type='%s_SERVER' % synchronicity.upper(),
 557                     #     unconstrained_client=synchronicity,
 558                     #     secure=secure,
 559                     #     categories=smoketest_categories+[SCALABLE],
 560                     #     resource_quota_size=500*1024)
 561
 562                     if rpc_type == 'streaming':
 563                         for mps in geometric_progression(1, 20, 10):
 564                             yield _ping_pong_scenario(
 565                                 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s'
 566                                 % (synchronicity, rpc_type, mps, secstr),
 567                                 rpc_type=rpc_type.upper(),
 568                                 client_type='%s_CLIENT' % synchronicity.upper(),
 569                                 server_type='%s_SERVER' % synchronicity.upper(),
 570                                 unconstrained_client=synchronicity,
 571                                 secure=secure,
 572                                 messages_per_stream=mps,
 573                                 minimal_stack=not secure,
 574                                 categories=inproc_categories + [SCALABLE])
 575
 576                         for mps in geometric_progression(1, 200, math.sqrt(10)):
 577                             yield _ping_pong_scenario(
 578                                 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s'
 579                                 % (synchronicity, rpc_type, mps, secstr),
 580                                 rpc_type=rpc_type.upper(),
 581                                 client_type='%s_CLIENT' % synchronicity.upper(),
 582                                 server_type='%s_SERVER' % synchronicity.upper(),
 583                                 unconstrained_client=synchronicity,
 584                                 secure=secure,
 585                                 messages_per_stream=mps,
 586                                 minimal_stack=not secure,
 587                                 categories=[SWEEP])
 588
 589                     for channels in geometric_progression(
 590                             1, 20000, math.sqrt(10)):
 591                         for outstanding in geometric_progression(
 592                                 1, 200000, math.sqrt(10)):
 593                             if synchronicity == 'sync' and outstanding > 1200:
 594                                 continue
 595                             if outstanding < channels: continue
 596                             yield _ping_pong_scenario(
 597                                 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding'
 598                                 % (synchronicity, rpc_type, secstr, channels,
 599                                    outstanding),
 600                                 rpc_type=rpc_type.upper(),
 601                                 client_type='%s_CLIENT' % synchronicity.upper(),
 602                                 server_type='%s_SERVER' % synchronicity.upper(),
 603                                 unconstrained_client=synchronicity,
 604                                 secure=secure,
 605                                 minimal_stack=not secure,
 606                                 categories=[SWEEP],
 607                                 channels=channels,
 608                                 outstanding=outstanding)
 609
 610     def __str__(self):
 611         return 'c++'
 612
 613
 614 class CSharpLanguage:
 615
 616     def __init__(self):
 617         self.safename = str(self)
 618
 619     def worker_cmdline(self):
 620         return ['tools/run_tests/performance/run_worker_csharp.sh']
 621
 622     def worker_port_offset(self):
 623         return 100
 624
 625     def scenarios(self):
 626         yield _ping_pong_scenario('csharp_generic_async_streaming_ping_pong',
 627                                   rpc_type='STREAMING',
 628                                   client_type='ASYNC_CLIENT',
 629                                   server_type='ASYNC_GENERIC_SERVER',
 630                                   use_generic_payload=True,
 631                                   categories=[SMOKETEST, SCALABLE])
 632
 633         yield _ping_pong_scenario(
 634             'csharp_generic_async_streaming_ping_pong_insecure_1MB',
 635             rpc_type='STREAMING',
 636             client_type='ASYNC_CLIENT',
 637             server_type='ASYNC_GENERIC_SERVER',
 638             req_size=1024 * 1024,
 639             resp_size=1024 * 1024,
 640             use_generic_payload=True,
 641             secure=False,
 642             categories=[SMOKETEST, SCALABLE])
 643
 644         yield _ping_pong_scenario(
 645             'csharp_generic_async_streaming_qps_unconstrained_insecure',
 646             rpc_type='STREAMING',
 647             client_type='ASYNC_CLIENT',
 648             server_type='ASYNC_GENERIC_SERVER',
 649             unconstrained_client='async',
 650             use_generic_payload=True,
 651             secure=False,
 652             categories=[SMOKETEST, SCALABLE])
 653
 654         yield _ping_pong_scenario('csharp_protobuf_async_streaming_ping_pong',
 655                                   rpc_type='STREAMING',
 656                                   client_type='ASYNC_CLIENT',
 657                                   server_type='ASYNC_SERVER')
 658
 659         yield _ping_pong_scenario('csharp_protobuf_async_unary_ping_pong',
 660                                   rpc_type='UNARY',
 661                                   client_type='ASYNC_CLIENT',
 662                                   server_type='ASYNC_SERVER',
 663                                   categories=[SMOKETEST, SCALABLE])
 664
 665         yield _ping_pong_scenario(
 666             'csharp_protobuf_sync_to_async_unary_ping_pong',
 667             rpc_type='UNARY',
 668             client_type='SYNC_CLIENT',
 669             server_type='ASYNC_SERVER')
 670
 671         yield _ping_pong_scenario(
 672             'csharp_protobuf_async_unary_qps_unconstrained',
 673             rpc_type='UNARY',
 674             client_type='ASYNC_CLIENT',
 675             server_type='ASYNC_SERVER',
 676             unconstrained_client='async',
 677             categories=[SMOKETEST, SCALABLE])
 678
 679         yield _ping_pong_scenario(
 680             'csharp_protobuf_async_streaming_qps_unconstrained',
 681             rpc_type='STREAMING',
 682             client_type='ASYNC_CLIENT',
 683             server_type='ASYNC_SERVER',
 684             unconstrained_client='async',
 685             categories=[SCALABLE])
 686
 687         yield _ping_pong_scenario('csharp_to_cpp_protobuf_sync_unary_ping_pong',
 688                                   rpc_type='UNARY',
 689                                   client_type='SYNC_CLIENT',
 690                                   server_type='SYNC_SERVER',
 691                                   server_language='c++',
 692                                   async_server_threads=1,
 693                                   categories=[SMOKETEST, SCALABLE])
 694
 695         yield _ping_pong_scenario(
 696             'csharp_to_cpp_protobuf_async_streaming_ping_pong',
 697             rpc_type='STREAMING',
 698             client_type='ASYNC_CLIENT',
 699             server_type='ASYNC_SERVER',
 700             server_language='c++',
 701             async_server_threads=1)
 702
 703         yield _ping_pong_scenario(
 704             'csharp_to_cpp_protobuf_async_unary_qps_unconstrained',
 705             rpc_type='UNARY',
 706             client_type='ASYNC_CLIENT',
 707             server_type='ASYNC_SERVER',
 708             unconstrained_client='async',
 709             server_language='c++',
 710             categories=[SCALABLE])
 711
 712         yield _ping_pong_scenario(
 713             'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained',
 714             rpc_type='UNARY',
 715             client_type='SYNC_CLIENT',
 716             server_type='ASYNC_SERVER',
 717             unconstrained_client='sync',
 718             server_language='c++',
 719             categories=[SCALABLE])
 720
 721         yield _ping_pong_scenario(
 722             'cpp_to_csharp_protobuf_async_unary_qps_unconstrained',
 723             rpc_type='UNARY',
 724             client_type='ASYNC_CLIENT',
 725             server_type='ASYNC_SERVER',
 726             unconstrained_client='async',
 727             client_language='c++',
 728             categories=[SCALABLE])
 729
 730         yield _ping_pong_scenario('csharp_protobuf_async_unary_ping_pong_1MB',
 731                                   rpc_type='UNARY',
 732                                   client_type='ASYNC_CLIENT',
 733                                   server_type='ASYNC_SERVER',
 734                                   req_size=1024 * 1024,
 735                                   resp_size=1024 * 1024,
 736                                   categories=[SMOKETEST, SCALABLE])
 737
 738     def __str__(self):
 739         return 'csharp'
 740
 741
 742 class PythonLanguage:
 743
 744     def __init__(self):
 745         self.safename = 'python'
 746
 747     def worker_cmdline(self):
 748         return ['tools/run_tests/performance/run_worker_python.sh']
 749
 750     def worker_port_offset(self):
 751         return 500
 752
 753     def scenarios(self):
 754         yield _ping_pong_scenario('python_generic_sync_streaming_ping_pong',
 755                                   rpc_type='STREAMING',
 756                                   client_type='SYNC_CLIENT',
 757                                   server_type='ASYNC_GENERIC_SERVER',
 758                                   use_generic_payload=True,
 759                                   categories=[SMOKETEST, SCALABLE])
 760
 761         yield _ping_pong_scenario('python_protobuf_sync_streaming_ping_pong',
 762                                   rpc_type='STREAMING',
 763                                   client_type='SYNC_CLIENT',
 764                                   server_type='ASYNC_SERVER')
 765
 766         yield _ping_pong_scenario('python_protobuf_async_unary_ping_pong',
 767                                   rpc_type='UNARY',
 768                                   client_type='ASYNC_CLIENT',
 769                                   server_type='ASYNC_SERVER')
 770
 771         yield _ping_pong_scenario('python_protobuf_sync_unary_ping_pong',
 772                                   rpc_type='UNARY',
 773                                   client_type='SYNC_CLIENT',
 774                                   server_type='ASYNC_SERVER',
 775                                   categories=[SMOKETEST, SCALABLE])
 776
 777         yield _ping_pong_scenario(
 778             'python_protobuf_sync_unary_qps_unconstrained',
 779             rpc_type='UNARY',
 780             client_type='SYNC_CLIENT',
 781             server_type='ASYNC_SERVER',
 782             unconstrained_client='sync')
 783
 784         yield _ping_pong_scenario(
 785             'python_protobuf_sync_streaming_qps_unconstrained',
 786             rpc_type='STREAMING',
 787             client_type='SYNC_CLIENT',
 788             server_type='ASYNC_SERVER',
 789             unconstrained_client='sync')
 790
 791         yield _ping_pong_scenario('python_to_cpp_protobuf_sync_unary_ping_pong',
 792                                   rpc_type='UNARY',
 793                                   client_type='SYNC_CLIENT',
 794                                   server_type='ASYNC_SERVER',
 795                                   server_language='c++',
 796                                   async_server_threads=1,
 797                                   categories=[SMOKETEST, SCALABLE])
 798
 799         yield _ping_pong_scenario(
 800             'python_to_cpp_protobuf_sync_streaming_ping_pong',
 801             rpc_type='STREAMING',
 802             client_type='SYNC_CLIENT',
 803             server_type='ASYNC_SERVER',
 804             server_language='c++',
 805             async_server_threads=1)
 806
 807         yield _ping_pong_scenario('python_protobuf_sync_unary_ping_pong_1MB',
 808                                   rpc_type='UNARY',
 809                                   client_type='SYNC_CLIENT',
 810                                   server_type='ASYNC_SERVER',
 811                                   req_size=1024 * 1024,
 812                                   resp_size=1024 * 1024,
 813                                   categories=[SMOKETEST, SCALABLE])
 814
 815     def __str__(self):
 816         return 'python'
 817
 818
 819 class RubyLanguage:
 820
 821     def __init__(self):
 822         pass
 823         self.safename = str(self)
 824
 825     def worker_cmdline(self):
 826         return ['tools/run_tests/performance/run_worker_ruby.sh']
 827
 828     def worker_port_offset(self):
 829         return 300
 830
 831     def scenarios(self):
 832         yield _ping_pong_scenario('ruby_protobuf_sync_streaming_ping_pong',
 833                                   rpc_type='STREAMING',
 834                                   client_type='SYNC_CLIENT',
 835                                   server_type='SYNC_SERVER',
 836                                   categories=[SMOKETEST, SCALABLE])
 837
 838         yield _ping_pong_scenario('ruby_protobuf_unary_ping_pong',
 839                                   rpc_type='UNARY',
 840                                   client_type='SYNC_CLIENT',
 841                                   server_type='SYNC_SERVER',
 842                                   categories=[SMOKETEST, SCALABLE])
 843
 844         yield _ping_pong_scenario('ruby_protobuf_sync_unary_qps_unconstrained',
 845                                   rpc_type='UNARY',
 846                                   client_type='SYNC_CLIENT',
 847                                   server_type='SYNC_SERVER',
 848                                   unconstrained_client='sync')
 849
 850         yield _ping_pong_scenario(
 851             'ruby_protobuf_sync_streaming_qps_unconstrained',
 852             rpc_type='STREAMING',
 853             client_type='SYNC_CLIENT',
 854             server_type='SYNC_SERVER',
 855             unconstrained_client='sync')
 856
 857         yield _ping_pong_scenario('ruby_to_cpp_protobuf_sync_unary_ping_pong',
 858                                   rpc_type='UNARY',
 859                                   client_type='SYNC_CLIENT',
 860                                   server_type='SYNC_SERVER',
 861                                   server_language='c++',
 862                                   async_server_threads=1)
 863
 864         yield _ping_pong_scenario(
 865             'ruby_to_cpp_protobuf_sync_streaming_ping_pong',
 866             rpc_type='STREAMING',
 867             client_type='SYNC_CLIENT',
 868             server_type='SYNC_SERVER',
 869             server_language='c++',
 870             async_server_threads=1)
 871
 872         yield _ping_pong_scenario('ruby_protobuf_unary_ping_pong_1MB',
 873                                   rpc_type='UNARY',
 874                                   client_type='SYNC_CLIENT',
 875                                   server_type='SYNC_SERVER',
 876                                   req_size=1024 * 1024,
 877                                   resp_size=1024 * 1024,
 878                                   categories=[SMOKETEST, SCALABLE])
 879
 880     def __str__(self):
 881         return 'ruby'
 882
 883
 884 class Php7Language:
 885
 886     def __init__(self, php7_protobuf_c=False):
 887         pass
 888         self.php7_protobuf_c = php7_protobuf_c
 889         self.safename = str(self)
 890
 891     def worker_cmdline(self):
 892         if self.php7_protobuf_c:
 893             return [
 894                 'tools/run_tests/performance/run_worker_php.sh',
 895                 '--use_protobuf_c_extension'
 896             ]
 897         return ['tools/run_tests/performance/run_worker_php.sh']
 898
 899     def worker_port_offset(self):
 900         if self.php7_protobuf_c:
 901             return 900
 902         return 800
 903
 904     def scenarios(self):
 905         php7_extension_mode = 'php7_protobuf_php_extension'
 906         if self.php7_protobuf_c:
 907             php7_extension_mode = 'php7_protobuf_c_extension'
 908
 909         yield _ping_pong_scenario('%s_to_cpp_protobuf_sync_unary_ping_pong' %
 910                                   php7_extension_mode,
 911                                   rpc_type='UNARY',
 912                                   client_type='SYNC_CLIENT',
 913                                   server_type='SYNC_SERVER',
 914                                   server_language='c++',
 915                                   async_server_threads=1)
 916
 917         yield _ping_pong_scenario(
 918             '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php7_extension_mode,
 919             rpc_type='STREAMING',
 920             client_type='SYNC_CLIENT',
 921             server_type='SYNC_SERVER',
 922             server_language='c++',
 923             async_server_threads=1)
 924
 925         # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs
 926         # better than async_server_threads=0/CPU usage 490%.
 927         yield _ping_pong_scenario(
 928             '%s_to_cpp_protobuf_sync_unary_qps_unconstrained' %
 929             php7_extension_mode,
 930             rpc_type='UNARY',
 931             client_type='SYNC_CLIENT',
 932             server_type='ASYNC_SERVER',
 933             server_language='c++',
 934             outstanding=1,
 935             async_server_threads=1,
 936             unconstrained_client='sync')
 937
 938         yield _ping_pong_scenario(
 939             '%s_to_cpp_protobuf_sync_streaming_qps_unconstrained' %
 940             php7_extension_mode,
 941             rpc_type='STREAMING',
 942             client_type='SYNC_CLIENT',
 943             server_type='ASYNC_SERVER',
 944             server_language='c++',
 945             outstanding=1,
 946             async_server_threads=1,
 947             unconstrained_client='sync')
 948
 949     def __str__(self):
 950         if self.php7_protobuf_c:
 951             return 'php7_protobuf_c'
 952         return 'php7'
 953
 954
 955 class JavaLanguage:
 956
 957     def __init__(self):
 958         pass
 959         self.safename = str(self)
 960
 961     def worker_cmdline(self):
 962         return ['tools/run_tests/performance/run_worker_java.sh']
 963
 964     def worker_port_offset(self):
 965         return 400
 966
 967     def scenarios(self):
 968         for secure in [True, False]:
 969             secstr = 'secure' if secure else 'insecure'
 970             smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
 971
 972             yield _ping_pong_scenario(
 973                 'java_generic_async_streaming_ping_pong_%s' % secstr,
 974                 rpc_type='STREAMING',
 975                 client_type='ASYNC_CLIENT',
 976                 server_type='ASYNC_GENERIC_SERVER',
 977                 use_generic_payload=True,
 978                 async_server_threads=1,
 979                 secure=secure,
 980                 warmup_seconds=JAVA_WARMUP_SECONDS,
 981                 categories=smoketest_categories)
 982
 983             yield _ping_pong_scenario(
 984                 'java_protobuf_async_streaming_ping_pong_%s' % secstr,
 985                 rpc_type='STREAMING',
 986                 client_type='ASYNC_CLIENT',
 987                 server_type='ASYNC_SERVER',
 988                 async_server_threads=1,
 989                 secure=secure,
 990                 warmup_seconds=JAVA_WARMUP_SECONDS)
 991
 992             yield _ping_pong_scenario('java_protobuf_async_unary_ping_pong_%s' %
 993                                       secstr,
 994                                       rpc_type='UNARY',
 995                                       client_type='ASYNC_CLIENT',
 996                                       server_type='ASYNC_SERVER',
 997                                       async_server_threads=1,
 998                                       secure=secure,
 999                                       warmup_seconds=JAVA_WARMUP_SECONDS,
1000                                       categories=smoketest_categories)
1001
1002             yield _ping_pong_scenario('java_protobuf_unary_ping_pong_%s' %
1003                                       secstr,
1004                                       rpc_type='UNARY',
1005                                       client_type='SYNC_CLIENT',
1006                                       server_type='SYNC_SERVER',
1007                                       async_server_threads=1,
1008                                       secure=secure,
1009                                       warmup_seconds=JAVA_WARMUP_SECONDS)
1010
1011             yield _ping_pong_scenario(
1012                 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr,
1013                 rpc_type='UNARY',
1014                 client_type='ASYNC_CLIENT',
1015                 server_type='ASYNC_SERVER',
1016                 unconstrained_client='async',
1017                 secure=secure,
1018                 warmup_seconds=JAVA_WARMUP_SECONDS,
1019                 categories=smoketest_categories + [SCALABLE])
1020
1021             yield _ping_pong_scenario(
1022                 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr,
1023                 rpc_type='STREAMING',
1024                 client_type='ASYNC_CLIENT',
1025                 server_type='ASYNC_SERVER',
1026                 unconstrained_client='async',
1027                 secure=secure,
1028                 warmup_seconds=JAVA_WARMUP_SECONDS,
1029                 categories=[SCALABLE])
1030
1031             yield _ping_pong_scenario(
1032                 'java_generic_async_streaming_qps_unconstrained_%s' % secstr,
1033                 rpc_type='STREAMING',
1034                 client_type='ASYNC_CLIENT',
1035                 server_type='ASYNC_GENERIC_SERVER',
1036                 unconstrained_client='async',
1037                 use_generic_payload=True,
1038                 secure=secure,
1039                 warmup_seconds=JAVA_WARMUP_SECONDS,
1040                 categories=[SCALABLE])
1041
1042             yield _ping_pong_scenario(
1043                 'java_generic_async_streaming_qps_one_server_core_%s' % secstr,
1044                 rpc_type='STREAMING',
1045                 client_type='ASYNC_CLIENT',
1046                 server_type='ASYNC_GENERIC_SERVER',
1047                 unconstrained_client='async-limited',
1048                 use_generic_payload=True,
1049                 async_server_threads=1,
1050                 secure=secure,
1051                 warmup_seconds=JAVA_WARMUP_SECONDS)
1052
1053             # TODO(jtattermusch): add scenarios java vs C++
1054
1055     def __str__(self):
1056         return 'java'
1057
1058
1059 class GoLanguage:
1060
1061     def __init__(self):
1062         pass
1063         self.safename = str(self)
1064
1065     def worker_cmdline(self):
1066         return ['tools/run_tests/performance/run_worker_go.sh']
1067
1068     def worker_port_offset(self):
1069         return 600
1070
1071     def scenarios(self):
1072         for secure in [True, False]:
1073             secstr = 'secure' if secure else 'insecure'
1074             smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
1075
1076             # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
1077             # but that's mostly because of lack of better name of the enum value.
1078             yield _ping_pong_scenario('go_generic_sync_streaming_ping_pong_%s' %
1079                                       secstr,
1080                                       rpc_type='STREAMING',
1081                                       client_type='SYNC_CLIENT',
1082                                       server_type='ASYNC_GENERIC_SERVER',
1083                                       use_generic_payload=True,
1084                                       async_server_threads=1,
1085                                       secure=secure,
1086                                       categories=smoketest_categories)
1087
1088             yield _ping_pong_scenario(
1089                 'go_protobuf_sync_streaming_ping_pong_%s' % secstr,
1090                 rpc_type='STREAMING',
1091                 client_type='SYNC_CLIENT',
1092                 server_type='SYNC_SERVER',
1093                 async_server_threads=1,
1094                 secure=secure)
1095
1096             yield _ping_pong_scenario('go_protobuf_sync_unary_ping_pong_%s' %
1097                                       secstr,
1098                                       rpc_type='UNARY',
1099                                       client_type='SYNC_CLIENT',
1100                                       server_type='SYNC_SERVER',
1101                                       async_server_threads=1,
1102                                       secure=secure,
1103                                       categories=smoketest_categories)
1104
1105             # unconstrained_client='async' is intended (client uses goroutines)
1106             yield _ping_pong_scenario(
1107                 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr,
1108                 rpc_type='UNARY',
1109                 client_type='SYNC_CLIENT',
1110                 server_type='SYNC_SERVER',
1111                 unconstrained_client='async',
1112                 secure=secure,
1113                 categories=smoketest_categories + [SCALABLE])
1114
1115             # unconstrained_client='async' is intended (client uses goroutines)
1116             yield _ping_pong_scenario(
1117                 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr,
1118                 rpc_type='STREAMING',
1119                 client_type='SYNC_CLIENT',
1120                 server_type='SYNC_SERVER',
1121                 unconstrained_client='async',
1122                 secure=secure,
1123                 categories=[SCALABLE])
1124
1125             # unconstrained_client='async' is intended (client uses goroutines)
1126             # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
1127             # but that's mostly because of lack of better name of the enum value.
1128             yield _ping_pong_scenario(
1129                 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr,
1130                 rpc_type='STREAMING',
1131                 client_type='SYNC_CLIENT',
1132                 server_type='ASYNC_GENERIC_SERVER',
1133                 unconstrained_client='async',
1134                 use_generic_payload=True,
1135                 secure=secure,
1136                 categories=[SCALABLE])
1137
1138             # TODO(jtattermusch): add scenarios go vs C++
1139
1140     def __str__(self):
1141         return 'go'
1142
1143
1144 class NodeLanguage:
1145
1146     def __init__(self, node_purejs=False):
1147         pass
1148         self.node_purejs = node_purejs
1149         self.safename = str(self)
1150
1151     def worker_cmdline(self):
1152         fixture = 'native_js' if self.node_purejs else 'native_native'
1153         return [
1154             'tools/run_tests/performance/run_worker_node.sh', fixture,
1155             '--benchmark_impl=grpc'
1156         ]
1157
1158     def worker_port_offset(self):
1159         if self.node_purejs:
1160             return 1100
1161         return 1000
1162
1163     def scenarios(self):
1164         node_implementation = 'node_purejs' if self.node_purejs else 'node'
1165         for secure in [True, False]:
1166             secstr = 'secure' if secure else 'insecure'
1167             smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
1168
1169             yield _ping_pong_scenario(
1170                 '%s_to_node_generic_async_streaming_ping_pong_%s' %
1171                 (node_implementation, secstr),
1172                 rpc_type='STREAMING',
1173                 client_type='ASYNC_CLIENT',
1174                 server_type='ASYNC_GENERIC_SERVER',
1175                 server_language='node',
1176                 use_generic_payload=True,
1177                 async_server_threads=1,
1178                 secure=secure,
1179                 categories=smoketest_categories)
1180
1181             yield _ping_pong_scenario(
1182                 '%s_to_node_protobuf_async_streaming_ping_pong_%s' %
1183                 (node_implementation, secstr),
1184                 rpc_type='STREAMING',
1185                 client_type='ASYNC_CLIENT',
1186                 server_type='ASYNC_SERVER',
1187                 server_language='node',
1188                 async_server_threads=1,
1189                 secure=secure)
1190
1191             yield _ping_pong_scenario(
1192                 '%s_to_node_protobuf_async_unary_ping_pong_%s' %
1193                 (node_implementation, secstr),
1194                 rpc_type='UNARY',
1195                 client_type='ASYNC_CLIENT',
1196                 server_type='ASYNC_SERVER',
1197                 server_language='node',
1198                 async_server_threads=1,
1199                 secure=secure,
1200                 categories=smoketest_categories)
1201
1202             yield _ping_pong_scenario(
1203                 '%s_to_node_protobuf_async_unary_qps_unconstrained_%s' %
1204                 (node_implementation, secstr),
1205                 rpc_type='UNARY',
1206                 client_type='ASYNC_CLIENT',
1207                 server_type='ASYNC_SERVER',
1208                 server_language='node',
1209                 unconstrained_client='async',
1210                 secure=secure,
1211                 categories=smoketest_categories + [SCALABLE])
1212
1213             yield _ping_pong_scenario(
1214                 '%s_to_node_protobuf_async_streaming_qps_unconstrained_%s' %
1215                 (node_implementation, secstr),
1216                 rpc_type='STREAMING',
1217                 client_type='ASYNC_CLIENT',
1218                 server_type='ASYNC_SERVER',
1219                 server_language='node',
1220                 unconstrained_client='async',
1221                 secure=secure,
1222                 categories=[SCALABLE])
1223
1224             yield _ping_pong_scenario(
1225                 '%s_to_node_generic_async_streaming_qps_unconstrained_%s' %
1226                 (node_implementation, secstr),
1227                 rpc_type='STREAMING',
1228                 client_type='ASYNC_CLIENT',
1229                 server_type='ASYNC_GENERIC_SERVER',
1230                 server_language='node',
1231                 unconstrained_client='async',
1232                 use_generic_payload=True,
1233                 secure=secure,
1234                 categories=[SCALABLE])
1235
1236             # TODO(murgatroid99): add scenarios node vs C++
1237
1238     def __str__(self):
1239         if self.node_purejs:
1240             return 'node_purejs'
1241         return 'node'
1242
1243
1244 LANGUAGES = {
1245     'c++': CXXLanguage(),
1246     'csharp': CSharpLanguage(),
1247     'ruby': RubyLanguage(),
1248     'php7': Php7Language(),
1249     'php7_protobuf_c': Php7Language(php7_protobuf_c=True),
1250     'java': JavaLanguage(),
1251     'python': PythonLanguage(),
1252     'go': GoLanguage(),
1253     'node': NodeLanguage(),
1254     'node_purejs': NodeLanguage(node_purejs=True)
1255 }