3 * Copyright 2017 gRPC authors.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
19 #include <grpc/support/port_platform.h>
21 #include <grpc/grpc.h>
22 #include <grpc/impl/codegen/grpc_types.h>
23 #include <grpc/support/alloc.h>
24 #include <grpc/support/log.h>
25 #include <grpc/support/string_util.h>
26 #include <grpc/support/sync.h>
27 #include <grpc/support/time.h>
33 #include <gflags/gflags.h>
34 #include <gmock/gmock.h>
38 #include "test/cpp/util/subprocess.h"
39 #include "test/cpp/util/test_config.h"
41 #include "src/core/ext/filters/client_channel/client_channel.h"
42 #include "src/core/ext/filters/client_channel/resolver.h"
43 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
44 #include "src/core/ext/filters/client_channel/resolver_registry.h"
45 #include "src/core/ext/filters/client_channel/server_address.h"
46 #include "src/core/lib/channel/channel_args.h"
47 #include "src/core/lib/gpr/env.h"
48 #include "src/core/lib/gpr/host_port.h"
49 #include "src/core/lib/gpr/string.h"
50 #include "src/core/lib/gprpp/orphanable.h"
51 #include "src/core/lib/iomgr/combiner.h"
52 #include "src/core/lib/iomgr/executor.h"
53 #include "src/core/lib/iomgr/iomgr.h"
54 #include "src/core/lib/iomgr/resolve_address.h"
55 #include "src/core/lib/iomgr/sockaddr_utils.h"
56 #include "test/core/util/port.h"
57 #include "test/core/util/test_config.h"
59 // TODO: pull in different headers when enabling this
60 // test on windows. Also set BAD_SOCKET_RETURN_VAL
61 // to INVALID_SOCKET on windows.
63 #include "src/core/lib/iomgr/sockaddr_windows.h"
64 #include "src/core/lib/iomgr/socket_windows.h"
65 #include "src/core/lib/iomgr/tcp_windows.h"
66 #define BAD_SOCKET_RETURN_VAL INVALID_SOCKET
68 #include "src/core/lib/iomgr/sockaddr_posix.h"
69 #define BAD_SOCKET_RETURN_VAL -1
72 using grpc::SubProcess;
74 using testing::UnorderedElementsAreArray;
76 // Hack copied from "test/cpp/end2end/server_crash_test_client.cc"!
77 // In some distros, gflags is in the namespace google, and in some others,
78 // in gflags. This hack is enabling us to find both.
81 using namespace google;
82 using namespace gflags;
84 DEFINE_string(target_name, "", "Target name to resolve.");
85 DEFINE_string(expected_addrs, "",
86 "List of expected backend or balancer addresses in the form "
87 "'<ip0:port0>,<is_balancer0>;<ip1:port1>,<is_balancer1>;...'. "
88 "'is_balancer' should be bool, i.e. true or false.");
89 DEFINE_string(expected_chosen_service_config, "",
90 "Expected service config json string that gets chosen (no "
91 "whitespace). Empty for none.");
93 local_dns_server_address, "",
94 "Optional. This address is placed as the uri authority if present.");
96 enable_srv_queries, "",
97 "Whether or not to enable SRV queries for the ares resolver instance."
98 "It would be better if this arg could be bool, but the way that we "
100 "the python script runner doesn't allow us to pass a gflags bool to this "
103 enable_txt_queries, "",
104 "Whether or not to enable TXT queries for the ares resolver instance."
105 "It would be better if this arg could be bool, but the way that we "
107 "the python script runner doesn't allow us to pass a gflags bool to this "
109 DEFINE_string(expected_lb_policy, "",
110 "Expected lb policy name that appears in resolver result channel "
111 "arg. Empty for none.");
115 class GrpcLBAddress final {
117 GrpcLBAddress(std::string address, bool is_balancer)
118 : is_balancer(is_balancer), address(std::move(address)) {}
120 bool operator==(const GrpcLBAddress& other) const {
121 return this->is_balancer == other.is_balancer &&
122 this->address == other.address;
125 bool operator!=(const GrpcLBAddress& other) const {
126 return !(*this == other);
133 vector<GrpcLBAddress> ParseExpectedAddrs(std::string expected_addrs) {
134 std::vector<GrpcLBAddress> out;
135 while (expected_addrs.size() != 0) {
136 // get the next <ip>,<port> (v4 or v6)
137 size_t next_comma = expected_addrs.find(',');
138 if (next_comma == std::string::npos) {
140 "Missing ','. Expected_addrs arg should be a semicolon-separated "
141 "list of <ip-port>,<bool> pairs. Left-to-be-parsed arg is |%s|",
142 expected_addrs.c_str());
145 std::string next_addr = expected_addrs.substr(0, next_comma);
146 expected_addrs = expected_addrs.substr(next_comma + 1, std::string::npos);
147 // get the next is_balancer 'bool' associated with this address
148 size_t next_semicolon = expected_addrs.find(';');
150 gpr_is_true(expected_addrs.substr(0, next_semicolon).c_str());
151 out.emplace_back(GrpcLBAddress(next_addr, is_balancer));
152 if (next_semicolon == std::string::npos) {
156 expected_addrs.substr(next_semicolon + 1, std::string::npos);
158 if (out.size() == 0) {
160 "expected_addrs arg should be a semicolon-separated list of "
161 "<ip-port>,<bool> pairs");
167 gpr_timespec TestDeadline(void) {
168 return grpc_timeout_seconds_to_deadline(100);
175 grpc_pollset* pollset;
176 grpc_pollset_set* pollset_set;
178 grpc_channel_args* channel_args;
179 vector<GrpcLBAddress> expected_addrs;
180 std::string expected_service_config_string;
181 std::string expected_lb_policy;
184 void ArgsInit(ArgsStruct* args) {
185 gpr_event_init(&args->ev);
186 args->pollset = (grpc_pollset*)gpr_zalloc(grpc_pollset_size());
187 grpc_pollset_init(args->pollset, &args->mu);
188 args->pollset_set = grpc_pollset_set_create();
189 grpc_pollset_set_add_pollset(args->pollset_set, args->pollset);
190 args->lock = grpc_combiner_create();
191 gpr_atm_rel_store(&args->done_atm, 0);
192 args->channel_args = nullptr;
195 void DoNothing(void* arg, grpc_error* error) {}
197 void ArgsFinish(ArgsStruct* args) {
198 GPR_ASSERT(gpr_event_wait(&args->ev, TestDeadline()));
199 grpc_pollset_set_del_pollset(args->pollset_set, args->pollset);
200 grpc_pollset_set_destroy(args->pollset_set);
201 grpc_closure DoNothing_cb;
202 GRPC_CLOSURE_INIT(&DoNothing_cb, DoNothing, nullptr,
203 grpc_schedule_on_exec_ctx);
204 grpc_pollset_shutdown(args->pollset, &DoNothing_cb);
205 // exec_ctx needs to be flushed before calling grpc_pollset_destroy()
206 grpc_channel_args_destroy(args->channel_args);
207 grpc_core::ExecCtx::Get()->Flush();
208 grpc_pollset_destroy(args->pollset);
209 gpr_free(args->pollset);
210 GRPC_COMBINER_UNREF(args->lock, nullptr);
213 gpr_timespec NSecondDeadline(int seconds) {
214 return gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
215 gpr_time_from_seconds(seconds, GPR_TIMESPAN));
218 void PollPollsetUntilRequestDone(ArgsStruct* args) {
219 gpr_timespec deadline = NSecondDeadline(10);
221 bool done = gpr_atm_acq_load(&args->done_atm) != 0;
225 gpr_timespec time_left =
226 gpr_time_sub(deadline, gpr_now(GPR_CLOCK_REALTIME));
227 gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRId64 ".%09d", done,
228 time_left.tv_sec, time_left.tv_nsec);
229 GPR_ASSERT(gpr_time_cmp(time_left, gpr_time_0(GPR_TIMESPAN)) >= 0);
230 grpc_pollset_worker* worker = nullptr;
231 grpc_core::ExecCtx exec_ctx;
232 gpr_mu_lock(args->mu);
233 GRPC_LOG_IF_ERROR("pollset_work",
234 grpc_pollset_work(args->pollset, &worker,
235 grpc_timespec_to_millis_round_up(
236 NSecondDeadline(1))));
237 gpr_mu_unlock(args->mu);
239 gpr_event_set(&args->ev, (void*)1);
242 void CheckServiceConfigResultLocked(const char* service_config_json,
244 if (args->expected_service_config_string != "") {
245 GPR_ASSERT(service_config_json != nullptr);
246 EXPECT_EQ(service_config_json, args->expected_service_config_string);
248 GPR_ASSERT(service_config_json == nullptr);
252 void CheckLBPolicyResultLocked(const grpc_channel_args* channel_args,
254 const grpc_arg* lb_policy_arg =
255 grpc_channel_args_find(channel_args, GRPC_ARG_LB_POLICY_NAME);
256 if (args->expected_lb_policy != "") {
257 GPR_ASSERT(lb_policy_arg != nullptr);
258 GPR_ASSERT(lb_policy_arg->type == GRPC_ARG_STRING);
259 EXPECT_EQ(lb_policy_arg->value.string, args->expected_lb_policy);
261 GPR_ASSERT(lb_policy_arg == nullptr);
266 void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) {
268 memset(&addr, 0, sizeof(addr));
269 addr.sin6_family = AF_INET6;
270 addr.sin6_port = htons(dummy_port);
271 ((char*)&addr.sin6_addr)[15] = 1;
273 if (gpr_event_get(done_ev)) {
276 std::vector<int> sockets;
277 for (size_t i = 0; i < 50; i++) {
278 SOCKET s = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, nullptr, 0,
279 WSA_FLAG_OVERLAPPED);
280 ASSERT_TRUE(s != BAD_SOCKET_RETURN_VAL)
281 << "Failed to create TCP ipv6 socket";
282 gpr_log(GPR_DEBUG, "Opened socket: %d", s);
284 ASSERT_TRUE(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) !=
286 << "Failed to set socketopt reuseaddr. WSA error: " +
287 std::to_string(WSAGetLastError());
288 ASSERT_TRUE(grpc_tcp_set_non_block(s) == GRPC_ERROR_NONE)
289 << "Failed to set socket non-blocking";
290 ASSERT_TRUE(bind(s, (const sockaddr*)&addr, sizeof(addr)) != SOCKET_ERROR)
291 << "Failed to bind socket " + std::to_string(s) +
292 " to [::1]:" + std::to_string(dummy_port) +
293 ". WSA error: " + std::to_string(WSAGetLastError());
294 ASSERT_TRUE(listen(s, 1) != SOCKET_ERROR)
295 << "Failed to listen on socket " + std::to_string(s) +
296 ". WSA error: " + std::to_string(WSAGetLastError());
297 sockets.push_back(s);
299 // Do a non-blocking accept followed by a close on all of those sockets.
300 // Do this in a separate loop to try to induce a time window to hit races.
301 for (size_t i = 0; i < sockets.size(); i++) {
302 gpr_log(GPR_DEBUG, "non-blocking accept then close on %d", sockets[i]);
303 ASSERT_TRUE(accept(sockets[i], nullptr, nullptr) == INVALID_SOCKET)
304 << "Accept on dummy socket unexpectedly accepted actual connection.";
305 ASSERT_TRUE(WSAGetLastError() == WSAEWOULDBLOCK)
306 << "OpenAndCloseSocketsStressLoop accept on socket " +
307 std::to_string(sockets[i]) +
309 "an unexpected way. "
311 std::to_string(WSAGetLastError()) +
312 ". Socket use-after-close bugs are likely.";
313 ASSERT_TRUE(closesocket(sockets[i]) != SOCKET_ERROR)
314 << "Failed to close socket: " + std::to_string(sockets[i]) +
315 ". WSA error: " + std::to_string(WSAGetLastError());
321 void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) {
322 // The goal of this loop is to catch socket
323 // "use after close" bugs within the c-ares resolver by acting
324 // like some separate thread doing I/O.
325 // It's goal is to try to hit race conditions whereby:
326 // 1) The c-ares resolver closes a socket.
327 // 2) This loop opens a socket with (coincidentally) the same handle.
328 // 3) the c-ares resolver mistakenly uses that same socket without
329 // realizing that its closed.
330 // 4) This loop performs an operation on that socket that should
331 // succeed but instead fails because of what the c-ares
332 // resolver did in the meantime.
334 memset(&addr, 0, sizeof(addr));
335 addr.sin6_family = AF_INET6;
336 addr.sin6_port = htons(dummy_port);
337 ((char*)&addr.sin6_addr)[15] = 1;
339 if (gpr_event_get(done_ev)) {
342 std::vector<int> sockets;
343 // First open a bunch of sockets, bind and listen
344 // '50' is an arbitrary number that, experimentally,
345 // has a good chance of catching bugs.
346 for (size_t i = 0; i < 50; i++) {
347 int s = socket(AF_INET6, SOCK_STREAM, 0);
349 ASSERT_TRUE(setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)) ==
351 << "Failed to set socketopt reuseport";
352 ASSERT_TRUE(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) ==
354 << "Failed to set socket reuseaddr";
355 ASSERT_TRUE(fcntl(s, F_SETFL, O_NONBLOCK) == 0)
356 << "Failed to set socket non-blocking";
357 ASSERT_TRUE(s != BAD_SOCKET_RETURN_VAL)
358 << "Failed to create TCP ipv6 socket";
359 gpr_log(GPR_DEBUG, "Opened fd: %d", s);
360 ASSERT_TRUE(bind(s, (const sockaddr*)&addr, sizeof(addr)) == 0)
361 << "Failed to bind socket " + std::to_string(s) +
362 " to [::1]:" + std::to_string(dummy_port) +
363 ". errno: " + std::to_string(errno);
364 ASSERT_TRUE(listen(s, 1) == 0) << "Failed to listen on socket " +
366 ". errno: " + std::to_string(errno);
367 sockets.push_back(s);
369 // Do a non-blocking accept followed by a close on all of those sockets.
370 // Do this in a separate loop to try to induce a time window to hit races.
371 for (size_t i = 0; i < sockets.size(); i++) {
372 gpr_log(GPR_DEBUG, "non-blocking accept then close on %d", sockets[i]);
373 if (accept(sockets[i], nullptr, nullptr)) {
374 // If e.g. a "shutdown" was called on this fd from another thread,
375 // then this accept call should fail with an unexpected error.
376 ASSERT_TRUE(errno == EAGAIN || errno == EWOULDBLOCK)
377 << "OpenAndCloseSocketsStressLoop accept on socket " +
378 std::to_string(sockets[i]) +
380 "an unexpected way. "
382 std::to_string(errno) +
383 ". Socket use-after-close bugs are likely.";
385 ASSERT_TRUE(close(sockets[i]) == 0)
386 << "Failed to close socket: " + std::to_string(sockets[i]) +
387 ". errno: " + std::to_string(errno);
393 class ResultHandler : public grpc_core::Resolver::ResultHandler {
395 static grpc_core::UniquePtr<grpc_core::Resolver::ResultHandler> Create(
397 return grpc_core::UniquePtr<grpc_core::Resolver::ResultHandler>(
398 grpc_core::New<ResultHandler>(args));
401 explicit ResultHandler(ArgsStruct* args) : args_(args) {}
403 void ReturnResult(grpc_core::Resolver::Result result) override {
405 gpr_atm_rel_store(&args_->done_atm, 1);
406 gpr_mu_lock(args_->mu);
407 GRPC_LOG_IF_ERROR("pollset_kick",
408 grpc_pollset_kick(args_->pollset, nullptr));
409 gpr_mu_unlock(args_->mu);
412 void ReturnError(grpc_error* error) override {
413 gpr_log(GPR_ERROR, "resolver returned error: %s", grpc_error_string(error));
417 virtual void CheckResult(const grpc_core::Resolver::Result& result) {}
420 ArgsStruct* args_struct() const { return args_; }
426 class CheckingResultHandler : public ResultHandler {
428 static grpc_core::UniquePtr<grpc_core::Resolver::ResultHandler> Create(
430 return grpc_core::UniquePtr<grpc_core::Resolver::ResultHandler>(
431 grpc_core::New<CheckingResultHandler>(args));
434 explicit CheckingResultHandler(ArgsStruct* args) : ResultHandler(args) {}
436 void CheckResult(const grpc_core::Resolver::Result& result) override {
437 ArgsStruct* args = args_struct();
438 gpr_log(GPR_INFO, "num addrs found: %" PRIdPTR ". expected %" PRIdPTR,
439 result.addresses.size(), args->expected_addrs.size());
440 GPR_ASSERT(result.addresses.size() == args->expected_addrs.size());
441 std::vector<GrpcLBAddress> found_lb_addrs;
442 for (size_t i = 0; i < result.addresses.size(); i++) {
443 const grpc_core::ServerAddress& addr = result.addresses[i];
445 grpc_sockaddr_to_string(&str, &addr.address(), 1 /* normalize */);
446 gpr_log(GPR_INFO, "%s", str);
447 found_lb_addrs.emplace_back(
448 GrpcLBAddress(std::string(str), addr.IsBalancer()));
451 if (args->expected_addrs.size() != found_lb_addrs.size()) {
453 "found lb addrs size is: %" PRIdPTR
454 ". expected addrs size is %" PRIdPTR,
455 found_lb_addrs.size(), args->expected_addrs.size());
458 EXPECT_THAT(args->expected_addrs,
459 UnorderedElementsAreArray(found_lb_addrs));
460 const char* service_config_json =
461 result.service_config == nullptr
463 : result.service_config->service_config_json();
464 CheckServiceConfigResultLocked(service_config_json, args);
465 if (args->expected_service_config_string == "") {
466 CheckLBPolicyResultLocked(result.args, args);
471 void RunResolvesRelevantRecordsTest(
472 grpc_core::UniquePtr<grpc_core::Resolver::ResultHandler> (
473 *CreateResultHandler)(ArgsStruct* args)) {
474 grpc_core::ExecCtx exec_ctx;
477 args.expected_addrs = ParseExpectedAddrs(FLAGS_expected_addrs);
478 args.expected_service_config_string = FLAGS_expected_chosen_service_config;
479 args.expected_lb_policy = FLAGS_expected_lb_policy;
480 // maybe build the address with an authority
481 char* whole_uri = nullptr;
482 GPR_ASSERT(gpr_asprintf(&whole_uri, "dns://%s/%s",
483 FLAGS_local_dns_server_address.c_str(),
484 FLAGS_target_name.c_str()));
485 gpr_log(GPR_DEBUG, "resolver_component_test: --enable_srv_queries: %s",
486 FLAGS_enable_srv_queries.c_str());
487 grpc_channel_args* resolver_args = nullptr;
488 // By default, SRV queries are disabled, so tests that expect no SRV query
489 // should avoid setting any channel arg. Test cases that do rely on the SRV
490 // query must explicitly enable SRV though.
491 if (FLAGS_enable_srv_queries == "True") {
492 grpc_arg srv_queries_arg = grpc_channel_arg_integer_create(
493 const_cast<char*>(GRPC_ARG_DNS_ENABLE_SRV_QUERIES), true);
495 grpc_channel_args_copy_and_add(nullptr, &srv_queries_arg, 1);
496 } else if (FLAGS_enable_srv_queries != "False") {
497 gpr_log(GPR_DEBUG, "Invalid value for --enable_srv_queries.");
500 gpr_log(GPR_DEBUG, "resolver_component_test: --enable_txt_queries: %s",
501 FLAGS_enable_txt_queries.c_str());
502 // By default, TXT queries are disabled, so tests that expect no TXT query
503 // should avoid setting any channel arg. Test cases that do rely on the TXT
504 // query must explicitly enable TXT though.
505 if (FLAGS_enable_txt_queries == "True") {
506 // Unlike SRV queries, there isn't a channel arg specific to TXT records.
507 // Rather, we use the resolver-agnostic "service config" resolution option,
508 // for which c-ares has its own specific default value, which isn't
509 // necessarily shared by other resolvers.
510 grpc_arg txt_queries_arg = grpc_channel_arg_integer_create(
511 const_cast<char*>(GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION), false);
512 grpc_channel_args* tmp_args =
513 grpc_channel_args_copy_and_add(resolver_args, &txt_queries_arg, 1);
514 grpc_channel_args_destroy(resolver_args);
515 resolver_args = tmp_args;
516 } else if (FLAGS_enable_txt_queries != "False") {
517 gpr_log(GPR_DEBUG, "Invalid value for --enable_txt_queries.");
520 // create resolver and resolve
521 grpc_core::OrphanablePtr<grpc_core::Resolver> resolver =
522 grpc_core::ResolverRegistry::CreateResolver(whole_uri, resolver_args,
523 args.pollset_set, args.lock,
524 CreateResultHandler(&args));
525 grpc_channel_args_destroy(resolver_args);
527 resolver->StartLocked();
528 grpc_core::ExecCtx::Get()->Flush();
529 PollPollsetUntilRequestDone(&args);
533 TEST(ResolverComponentTest, TestResolvesRelevantRecords) {
534 RunResolvesRelevantRecordsTest(CheckingResultHandler::Create);
537 TEST(ResolverComponentTest, TestResolvesRelevantRecordsWithConcurrentFdStress) {
538 // Start up background stress thread
539 int dummy_port = grpc_pick_unused_port_or_die();
541 gpr_event_init(&done_ev);
542 std::thread socket_stress_thread(OpenAndCloseSocketsStressLoop, dummy_port,
544 // Run the resolver test
545 RunResolvesRelevantRecordsTest(ResultHandler::Create);
546 // Shutdown and join stress thread
547 gpr_event_set(&done_ev, (void*)1);
548 socket_stress_thread.join();
553 int main(int argc, char** argv) {
555 grpc::testing::TestEnvironment env(argc, argv);
556 ::testing::InitGoogleTest(&argc, argv);
557 ParseCommandLineFlags(&argc, &argv, true);
558 if (FLAGS_target_name == "") {
559 gpr_log(GPR_ERROR, "Missing target_name param.");
562 if (FLAGS_local_dns_server_address != "") {
563 gpr_log(GPR_INFO, "Specifying authority in uris to: %s",
564 FLAGS_local_dns_server_address.c_str());
566 auto result = RUN_ALL_TESTS();