3 * Copyright 2015 gRPC authors.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
19 #include <grpc/support/port_platform.h>
21 #include "src/core/lib/iomgr/port.h"
23 #ifdef GRPC_POSIX_SOCKET_TCP
25 #include "src/core/lib/iomgr/tcp_posix.h"
29 #include <netinet/in.h>
30 #include <netinet/tcp.h>
35 #include <sys/socket.h>
36 #include <sys/types.h>
40 #include <grpc/slice.h>
41 #include <grpc/support/alloc.h>
42 #include <grpc/support/log.h>
43 #include <grpc/support/string_util.h>
44 #include <grpc/support/sync.h>
45 #include <grpc/support/time.h>
47 #include "src/core/lib/channel/channel_args.h"
48 #include "src/core/lib/debug/stats.h"
49 #include "src/core/lib/debug/trace.h"
50 #include "src/core/lib/gpr/string.h"
51 #include "src/core/lib/gpr/useful.h"
52 #include "src/core/lib/iomgr/buffer_list.h"
53 #include "src/core/lib/iomgr/ev_posix.h"
54 #include "src/core/lib/iomgr/executor.h"
55 #include "src/core/lib/profiling/timers.h"
56 #include "src/core/lib/slice/slice_internal.h"
57 #include "src/core/lib/slice/slice_string_helpers.h"
60 #define SOL_TCP IPPROTO_TCP
65 #define TCP_CM_INQ TCP_INQ
68 #ifdef GRPC_HAVE_MSG_NOSIGNAL
69 #define SENDMSG_FLAGS MSG_NOSIGNAL
71 #define SENDMSG_FLAGS 0
74 #ifdef GRPC_MSG_IOVLEN_TYPE
75 typedef GRPC_MSG_IOVLEN_TYPE msg_iovlen_type;
77 typedef size_t msg_iovlen_type;
80 extern grpc_core::TraceFlag grpc_tcp_trace;
87 /* Used by the endpoint read function to distinguish the very first read call
91 double bytes_read_this_round;
92 gpr_refcount refcount;
93 gpr_atm shutdown_count;
95 int min_read_chunk_size;
96 int max_read_chunk_size;
98 /* garbage after the last read */
99 grpc_slice_buffer last_read_buffer;
101 grpc_slice_buffer* incoming_buffer;
102 int inq; /* bytes pending on the socket from the last read. */
103 bool inq_capable; /* cache whether kernel supports inq */
105 grpc_slice_buffer* outgoing_buffer;
106 /* byte within outgoing_buffer->slices[0] to write next */
107 size_t outgoing_byte_idx;
109 grpc_closure* read_cb;
110 grpc_closure* write_cb;
111 grpc_closure* release_fd_cb;
114 grpc_closure read_done_closure;
115 grpc_closure write_done_closure;
116 grpc_closure error_closure;
120 grpc_resource_user* resource_user;
121 grpc_resource_user_slice_allocator slice_allocator;
123 grpc_core::TracedBuffer* tb_head; /* List of traced buffers */
124 gpr_mu tb_mu; /* Lock for access to list of traced buffers */
126 /* grpc_endpoint_write takes an argument which if non-null means that the
127 * transport layer wants the TCP layer to collect timestamps for this write.
128 * This arg is forwarded to the timestamps callback function when the ACK
129 * timestamp is received from the kernel. This arg is a (void *) which allows
130 * users of this API to pass in a pointer to any kind of structure. This
131 * structure could actually be a tag or any book-keeping object that the user
132 * can use to distinguish between different traced writes. The only
133 * requirement from the TCP endpoint layer is that this arg should be non-null
134 * if the user wants timestamps for the write. */
135 void* outgoing_buffer_arg;
136 /* A counter which starts at 0. It is initialized the first time the socket
137 * options for collecting timestamps are set, and is incremented with each
140 bool socket_ts_enabled; /* True if timestamping options are set on the socket
142 bool ts_capable; /* Cache whether we can set timestamping options */
143 gpr_atm stop_error_notification; /* Set to 1 if we do not want to be notified
147 struct backup_poller {
149 grpc_closure run_poller;
154 #define BACKUP_POLLER_POLLSET(b) ((grpc_pollset*)((b) + 1))
156 static gpr_atm g_uncovered_notifications_pending;
157 static gpr_atm g_backup_poller; /* backup_poller* */
159 static void tcp_handle_read(void* arg /* grpc_tcp */, grpc_error* error);
160 static void tcp_handle_write(void* arg /* grpc_tcp */, grpc_error* error);
161 static void tcp_drop_uncovered_then_handle_write(void* arg /* grpc_tcp */,
164 static void done_poller(void* bp, grpc_error* error_ignored) {
165 backup_poller* p = static_cast<backup_poller*>(bp);
166 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
167 gpr_log(GPR_INFO, "BACKUP_POLLER:%p destroy", p);
169 grpc_pollset_destroy(BACKUP_POLLER_POLLSET(p));
173 static void run_poller(void* bp, grpc_error* error_ignored) {
174 backup_poller* p = static_cast<backup_poller*>(bp);
175 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
176 gpr_log(GPR_INFO, "BACKUP_POLLER:%p run", p);
178 gpr_mu_lock(p->pollset_mu);
179 grpc_millis deadline = grpc_core::ExecCtx::Get()->Now() + 10 * GPR_MS_PER_SEC;
180 GRPC_STATS_INC_TCP_BACKUP_POLLER_POLLS();
182 "backup_poller:pollset_work",
183 grpc_pollset_work(BACKUP_POLLER_POLLSET(p), nullptr, deadline));
184 gpr_mu_unlock(p->pollset_mu);
185 /* last "uncovered" notification is the ref that keeps us polling, if we get
186 * there try a cas to release it */
187 if (gpr_atm_no_barrier_load(&g_uncovered_notifications_pending) == 1 &&
188 gpr_atm_full_cas(&g_uncovered_notifications_pending, 1, 0)) {
189 gpr_mu_lock(p->pollset_mu);
190 bool cas_ok = gpr_atm_full_cas(&g_backup_poller, (gpr_atm)p, 0);
191 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
192 gpr_log(GPR_INFO, "BACKUP_POLLER:%p done cas_ok=%d", p, cas_ok);
194 gpr_mu_unlock(p->pollset_mu);
195 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
196 gpr_log(GPR_INFO, "BACKUP_POLLER:%p shutdown", p);
198 grpc_pollset_shutdown(BACKUP_POLLER_POLLSET(p),
199 GRPC_CLOSURE_INIT(&p->run_poller, done_poller, p,
200 grpc_schedule_on_exec_ctx));
202 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
203 gpr_log(GPR_INFO, "BACKUP_POLLER:%p reschedule", p);
205 GRPC_CLOSURE_SCHED(&p->run_poller, GRPC_ERROR_NONE);
209 static void drop_uncovered(grpc_tcp* tcp) {
210 backup_poller* p = (backup_poller*)gpr_atm_acq_load(&g_backup_poller);
212 gpr_atm_full_fetch_add(&g_uncovered_notifications_pending, -1);
213 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
214 gpr_log(GPR_INFO, "BACKUP_POLLER:%p uncover cnt %d->%d", p,
215 static_cast<int>(old_count), static_cast<int>(old_count) - 1);
217 GPR_ASSERT(old_count != 1);
220 // gRPC API considers a Write operation to be done the moment it clears ‘flow
221 // control’ i.e., not necessarily sent on the wire. This means that the
222 // application MIGHT not call `grpc_completion_queue_next/pluck` in a timely
223 // manner when its `Write()` API is acked.
225 // We need to ensure that the fd is 'covered' (i.e being monitored by some
226 // polling thread and progress is made) and hence add it to a backup poller here
227 static void cover_self(grpc_tcp* tcp) {
230 gpr_atm_no_barrier_fetch_add(&g_uncovered_notifications_pending, 2);
231 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
232 gpr_log(GPR_INFO, "BACKUP_POLLER: cover cnt %d->%d",
233 static_cast<int>(old_count), 2 + static_cast<int>(old_count));
235 if (old_count == 0) {
236 GRPC_STATS_INC_TCP_BACKUP_POLLERS_CREATED();
237 p = static_cast<backup_poller*>(
238 gpr_zalloc(sizeof(*p) + grpc_pollset_size()));
239 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
240 gpr_log(GPR_INFO, "BACKUP_POLLER:%p create", p);
242 grpc_pollset_init(BACKUP_POLLER_POLLSET(p), &p->pollset_mu);
243 gpr_atm_rel_store(&g_backup_poller, (gpr_atm)p);
244 GRPC_CLOSURE_SCHED(GRPC_CLOSURE_INIT(&p->run_poller, run_poller, p,
245 grpc_core::Executor::Scheduler(
246 grpc_core::ExecutorJobType::LONG)),
249 while ((p = (backup_poller*)gpr_atm_acq_load(&g_backup_poller)) ==
251 // spin waiting for backup poller
254 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
255 gpr_log(GPR_INFO, "BACKUP_POLLER:%p add %p", p, tcp);
257 grpc_pollset_add_fd(BACKUP_POLLER_POLLSET(p), tcp->em_fd);
258 if (old_count != 0) {
263 static void notify_on_read(grpc_tcp* tcp) {
264 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
265 gpr_log(GPR_INFO, "TCP:%p notify_on_read", tcp);
267 grpc_fd_notify_on_read(tcp->em_fd, &tcp->read_done_closure);
270 static void notify_on_write(grpc_tcp* tcp) {
271 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
272 gpr_log(GPR_INFO, "TCP:%p notify_on_write", tcp);
274 if (!grpc_event_engine_run_in_background()) {
277 grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_done_closure);
280 static void tcp_drop_uncovered_then_handle_write(void* arg, grpc_error* error) {
281 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
282 gpr_log(GPR_INFO, "TCP:%p got_write: %s", arg, grpc_error_string(error));
284 drop_uncovered(static_cast<grpc_tcp*>(arg));
285 tcp_handle_write(arg, error);
288 static void add_to_estimate(grpc_tcp* tcp, size_t bytes) {
289 tcp->bytes_read_this_round += static_cast<double>(bytes);
292 static void finish_estimate(grpc_tcp* tcp) {
293 /* If we read >80% of the target buffer in one read loop, increase the size
294 of the target buffer to either the amount read, or twice its previous
296 if (tcp->bytes_read_this_round > tcp->target_length * 0.8) {
298 GPR_MAX(2 * tcp->target_length, tcp->bytes_read_this_round);
301 0.99 * tcp->target_length + 0.01 * tcp->bytes_read_this_round;
303 tcp->bytes_read_this_round = 0;
306 static size_t get_target_read_size(grpc_tcp* tcp) {
307 grpc_resource_quota* rq = grpc_resource_user_quota(tcp->resource_user);
308 double pressure = grpc_resource_quota_get_memory_pressure(rq);
310 tcp->target_length * (pressure > 0.8 ? (1.0 - pressure) / 0.2 : 1.0);
311 size_t sz = ((static_cast<size_t> GPR_CLAMP(target, tcp->min_read_chunk_size,
312 tcp->max_read_chunk_size)) +
314 ~static_cast<size_t>(255);
315 /* don't use more than 1/16th of the overall resource quota for a single read
317 size_t rqmax = grpc_resource_quota_peek_size(rq);
318 if (sz > rqmax / 16 && rqmax > 1024) {
324 static grpc_error* tcp_annotate_error(grpc_error* src_error, grpc_tcp* tcp) {
325 return grpc_error_set_str(
327 grpc_error_set_int(src_error, GRPC_ERROR_INT_FD, tcp->fd),
328 /* All tcp errors are marked with UNAVAILABLE so that application may
329 * choose to retry. */
330 GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE),
331 GRPC_ERROR_STR_TARGET_ADDRESS,
332 grpc_slice_from_copied_string(tcp->peer_string));
335 static void tcp_handle_read(void* arg /* grpc_tcp */, grpc_error* error);
336 static void tcp_handle_write(void* arg /* grpc_tcp */, grpc_error* error);
338 static void tcp_shutdown(grpc_endpoint* ep, grpc_error* why) {
339 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
340 grpc_fd_shutdown(tcp->em_fd, why);
341 grpc_resource_user_shutdown(tcp->resource_user);
344 static void tcp_free(grpc_tcp* tcp) {
345 grpc_fd_orphan(tcp->em_fd, tcp->release_fd_cb, tcp->release_fd,
347 grpc_slice_buffer_destroy_internal(&tcp->last_read_buffer);
348 grpc_resource_user_unref(tcp->resource_user);
349 gpr_free(tcp->peer_string);
350 /* The lock is not really necessary here, since all refs have been released */
351 gpr_mu_lock(&tcp->tb_mu);
352 grpc_core::TracedBuffer::Shutdown(
353 &tcp->tb_head, tcp->outgoing_buffer_arg,
354 GRPC_ERROR_CREATE_FROM_STATIC_STRING("endpoint destroyed"));
355 gpr_mu_unlock(&tcp->tb_mu);
356 tcp->outgoing_buffer_arg = nullptr;
357 gpr_mu_destroy(&tcp->tb_mu);
362 #define TCP_UNREF(tcp, reason) tcp_unref((tcp), (reason), __FILE__, __LINE__)
363 #define TCP_REF(tcp, reason) tcp_ref((tcp), (reason), __FILE__, __LINE__)
364 static void tcp_unref(grpc_tcp* tcp, const char* reason, const char* file,
366 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
367 gpr_atm val = gpr_atm_no_barrier_load(&tcp->refcount.count);
368 gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
369 "TCP unref %p : %s %" PRIdPTR " -> %" PRIdPTR, tcp, reason, val,
372 if (gpr_unref(&tcp->refcount)) {
377 static void tcp_ref(grpc_tcp* tcp, const char* reason, const char* file,
379 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
380 gpr_atm val = gpr_atm_no_barrier_load(&tcp->refcount.count);
381 gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
382 "TCP ref %p : %s %" PRIdPTR " -> %" PRIdPTR, tcp, reason, val,
385 gpr_ref(&tcp->refcount);
388 #define TCP_UNREF(tcp, reason) tcp_unref((tcp))
389 #define TCP_REF(tcp, reason) tcp_ref((tcp))
390 static void tcp_unref(grpc_tcp* tcp) {
391 if (gpr_unref(&tcp->refcount)) {
396 static void tcp_ref(grpc_tcp* tcp) { gpr_ref(&tcp->refcount); }
399 static void tcp_destroy(grpc_endpoint* ep) {
400 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
401 grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer);
402 if (grpc_event_engine_can_track_errors()) {
403 gpr_atm_no_barrier_store(&tcp->stop_error_notification, true);
404 grpc_fd_set_error(tcp->em_fd);
406 TCP_UNREF(tcp, "destroy");
409 static void call_read_cb(grpc_tcp* tcp, grpc_error* error) {
410 grpc_closure* cb = tcp->read_cb;
412 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
413 gpr_log(GPR_INFO, "TCP:%p call_cb %p %p:%p", tcp, cb, cb->cb, cb->cb_arg);
415 const char* str = grpc_error_string(error);
416 gpr_log(GPR_INFO, "READ %p (peer=%s) error=%s", tcp, tcp->peer_string, str);
418 if (gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) {
419 for (i = 0; i < tcp->incoming_buffer->count; i++) {
420 char* dump = grpc_dump_slice(tcp->incoming_buffer->slices[i],
421 GPR_DUMP_HEX | GPR_DUMP_ASCII);
422 gpr_log(GPR_DEBUG, "DATA: %s", dump);
428 tcp->read_cb = nullptr;
429 tcp->incoming_buffer = nullptr;
430 GRPC_CLOSURE_SCHED(cb, error);
433 #define MAX_READ_IOVEC 4
434 static void tcp_do_read(grpc_tcp* tcp) {
435 GPR_TIMER_SCOPE("tcp_do_read", 0);
437 struct iovec iov[MAX_READ_IOVEC];
438 char cmsgbuf[24 /*CMSG_SPACE(sizeof(int))*/];
440 size_t total_read_bytes = 0;
443 std::min<size_t>(MAX_READ_IOVEC, tcp->incoming_buffer->count);
444 for (size_t i = 0; i < iov_len; i++) {
445 iov[i].iov_base = GRPC_SLICE_START_PTR(tcp->incoming_buffer->slices[i]);
446 iov[i].iov_len = GRPC_SLICE_LENGTH(tcp->incoming_buffer->slices[i]);
450 /* Assume there is something on the queue. If we receive TCP_INQ from
451 * kernel, we will update this value, otherwise, we have to assume there is
452 * always something to read until we get EAGAIN. */
455 msg.msg_name = nullptr;
458 msg.msg_iovlen = static_cast<msg_iovlen_type>(iov_len);
459 if (tcp->inq_capable) {
460 msg.msg_control = cmsgbuf;
461 msg.msg_controllen = sizeof(cmsgbuf);
463 msg.msg_control = nullptr;
464 msg.msg_controllen = 0;
468 GRPC_STATS_INC_TCP_READ_OFFER(tcp->incoming_buffer->length);
469 GRPC_STATS_INC_TCP_READ_OFFER_IOV_SIZE(tcp->incoming_buffer->count);
472 GPR_TIMER_SCOPE("recvmsg", 0);
473 GRPC_STATS_INC_SYSCALL_READ();
474 read_bytes = recvmsg(tcp->fd, &msg, 0);
475 } while (read_bytes < 0 && errno == EINTR);
477 /* We have read something in previous reads. We need to deliver those
478 * bytes to the upper layer. */
479 if (read_bytes <= 0 && total_read_bytes > 0) {
484 if (read_bytes < 0) {
485 /* NB: After calling call_read_cb a parallel call of the read handler may
487 if (errno == EAGAIN) {
488 finish_estimate(tcp);
490 /* We've consumed the edge, request a new one */
493 grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
495 tcp_annotate_error(GRPC_OS_ERROR(errno, "recvmsg"), tcp));
496 TCP_UNREF(tcp, "read");
500 if (read_bytes == 0) {
501 /* 0 read size ==> end of stream
503 * We may have read something, i.e., total_read_bytes > 0, but
504 * since the connection is closed we will drop the data here, because we
505 * can't call the callback multiple times. */
506 grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
508 tcp, tcp_annotate_error(
509 GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp));
510 TCP_UNREF(tcp, "read");
514 GRPC_STATS_INC_TCP_READ_SIZE(read_bytes);
515 add_to_estimate(tcp, static_cast<size_t>(read_bytes));
516 GPR_DEBUG_ASSERT((size_t)read_bytes <=
517 tcp->incoming_buffer->length - total_read_bytes);
519 #ifdef GRPC_HAVE_TCP_INQ
520 if (tcp->inq_capable) {
521 GPR_DEBUG_ASSERT(!(msg.msg_flags & MSG_CTRUNC));
522 struct cmsghdr* cmsg = CMSG_FIRSTHDR(&msg);
523 for (; cmsg != nullptr; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
524 if (cmsg->cmsg_level == SOL_TCP && cmsg->cmsg_type == TCP_CM_INQ &&
525 cmsg->cmsg_len == CMSG_LEN(sizeof(int))) {
526 tcp->inq = *reinterpret_cast<int*>(CMSG_DATA(cmsg));
530 #endif /* GRPC_HAVE_TCP_INQ */
532 total_read_bytes += read_bytes;
533 if (tcp->inq == 0 || total_read_bytes == tcp->incoming_buffer->length) {
534 /* We have filled incoming_buffer, and we cannot read any more. */
538 /* We had a partial read, and still have space to read more data.
539 * So, adjust IOVs and try to read more. */
540 size_t remaining = read_bytes;
542 for (size_t i = 0; i < iov_len; i++) {
543 if (remaining >= iov[i].iov_len) {
544 remaining -= iov[i].iov_len;
548 iov[j].iov_base = static_cast<char*>(iov[i].iov_base) + remaining;
549 iov[j].iov_len = iov[i].iov_len - remaining;
552 iov[j].iov_base = iov[i].iov_base;
553 iov[j].iov_len = iov[i].iov_len;
561 finish_estimate(tcp);
564 GPR_DEBUG_ASSERT(total_read_bytes > 0);
565 if (total_read_bytes < tcp->incoming_buffer->length) {
566 grpc_slice_buffer_trim_end(tcp->incoming_buffer,
567 tcp->incoming_buffer->length - total_read_bytes,
568 &tcp->last_read_buffer);
570 call_read_cb(tcp, GRPC_ERROR_NONE);
571 TCP_UNREF(tcp, "read");
574 static void tcp_read_allocation_done(void* tcpp, grpc_error* error) {
575 grpc_tcp* tcp = static_cast<grpc_tcp*>(tcpp);
576 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
577 gpr_log(GPR_INFO, "TCP:%p read_allocation_done: %s", tcp,
578 grpc_error_string(error));
580 if (error != GRPC_ERROR_NONE) {
581 grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
582 grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer);
583 call_read_cb(tcp, GRPC_ERROR_REF(error));
584 TCP_UNREF(tcp, "read");
590 static void tcp_continue_read(grpc_tcp* tcp) {
591 size_t target_read_size = get_target_read_size(tcp);
592 /* Wait for allocation only when there is no buffer left. */
593 if (tcp->incoming_buffer->length == 0 &&
594 tcp->incoming_buffer->count < MAX_READ_IOVEC) {
595 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
596 gpr_log(GPR_INFO, "TCP:%p alloc_slices", tcp);
598 grpc_resource_user_alloc_slices(&tcp->slice_allocator, target_read_size, 1,
599 tcp->incoming_buffer);
601 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
602 gpr_log(GPR_INFO, "TCP:%p do_read", tcp);
608 static void tcp_handle_read(void* arg /* grpc_tcp */, grpc_error* error) {
609 grpc_tcp* tcp = static_cast<grpc_tcp*>(arg);
610 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
611 gpr_log(GPR_INFO, "TCP:%p got_read: %s", tcp, grpc_error_string(error));
614 if (error != GRPC_ERROR_NONE) {
615 grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
616 grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer);
617 call_read_cb(tcp, GRPC_ERROR_REF(error));
618 TCP_UNREF(tcp, "read");
620 tcp_continue_read(tcp);
624 static void tcp_read(grpc_endpoint* ep, grpc_slice_buffer* incoming_buffer,
625 grpc_closure* cb, bool urgent) {
626 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
627 GPR_ASSERT(tcp->read_cb == nullptr);
629 tcp->incoming_buffer = incoming_buffer;
630 grpc_slice_buffer_reset_and_unref_internal(incoming_buffer);
631 grpc_slice_buffer_swap(incoming_buffer, &tcp->last_read_buffer);
632 TCP_REF(tcp, "read");
633 if (tcp->is_first_read) {
634 /* Endpoint read called for the very first time. Register read callback with
635 * the polling engine */
636 tcp->is_first_read = false;
638 } else if (!urgent && tcp->inq == 0) {
639 /* Upper layer asked to read more but we know there is no pending data
640 * to read from previous reads. So, wait for POLLIN.
644 /* Not the first time. We may or may not have more bytes available. In any
645 * case call tcp->read_done_closure (i.e tcp_handle_read()) which does the
646 * right thing (i.e calls tcp_do_read() which either reads the available
647 * bytes or calls notify_on_read() to be notified when new bytes become
649 GRPC_CLOSURE_SCHED(&tcp->read_done_closure, GRPC_ERROR_NONE);
653 /* A wrapper around sendmsg. It sends \a msg over \a fd and returns the number
655 ssize_t tcp_send(int fd, const struct msghdr* msg) {
656 GPR_TIMER_SCOPE("sendmsg", 1);
659 /* TODO(klempner): Cork if this is a partial write */
660 GRPC_STATS_INC_SYSCALL_WRITE();
661 sent_length = sendmsg(fd, msg, SENDMSG_FLAGS);
662 } while (sent_length < 0 && errno == EINTR);
666 /** This is to be called if outgoing_buffer_arg is not null. On linux platforms,
667 * this will call sendmsg with socket options set to collect timestamps inside
668 * the kernel. On return, sent_length is set to the return value of the sendmsg
669 * call. Returns false if setting the socket options failed. This is not
670 * implemented for non-linux platforms currently, and crashes out.
672 static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg,
673 size_t sending_length,
674 ssize_t* sent_length);
676 /** The callback function to be invoked when we get an error on the socket. */
677 static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error);
679 #ifdef GRPC_LINUX_ERRQUEUE
681 static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg,
682 size_t sending_length,
683 ssize_t* sent_length) {
684 if (!tcp->socket_ts_enabled) {
685 uint32_t opt = grpc_core::kTimestampingSocketOptions;
686 if (setsockopt(tcp->fd, SOL_SOCKET, SO_TIMESTAMPING,
687 static_cast<void*>(&opt), sizeof(opt)) != 0) {
688 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
689 gpr_log(GPR_ERROR, "Failed to set timestamping options on the socket.");
693 tcp->bytes_counter = -1;
694 tcp->socket_ts_enabled = true;
696 /* Set control message to indicate that you want timestamps. */
698 char cmsg_buf[CMSG_SPACE(sizeof(uint32_t))];
699 struct cmsghdr align;
701 cmsghdr* cmsg = reinterpret_cast<cmsghdr*>(u.cmsg_buf);
702 cmsg->cmsg_level = SOL_SOCKET;
703 cmsg->cmsg_type = SO_TIMESTAMPING;
704 cmsg->cmsg_len = CMSG_LEN(sizeof(uint32_t));
705 *reinterpret_cast<int*>(CMSG_DATA(cmsg)) =
706 grpc_core::kTimestampingRecordingOptions;
707 msg->msg_control = u.cmsg_buf;
708 msg->msg_controllen = CMSG_SPACE(sizeof(uint32_t));
710 /* If there was an error on sendmsg the logic in tcp_flush will handle it. */
711 ssize_t length = tcp_send(tcp->fd, msg);
712 *sent_length = length;
713 /* Only save timestamps if all the bytes were taken by sendmsg. */
714 if (sending_length == static_cast<size_t>(length)) {
715 gpr_mu_lock(&tcp->tb_mu);
716 grpc_core::TracedBuffer::AddNewEntry(
717 &tcp->tb_head, static_cast<uint32_t>(tcp->bytes_counter + length),
718 tcp->fd, tcp->outgoing_buffer_arg);
719 gpr_mu_unlock(&tcp->tb_mu);
720 tcp->outgoing_buffer_arg = nullptr;
725 /** Reads \a cmsg to derive timestamps from the control messages. If a valid
726 * timestamp is found, the traced buffer list is updated with this timestamp.
727 * The caller of this function should be looping on the control messages found
728 * in \a msg. \a cmsg should point to the control message that the caller wants
730 * On return, a pointer to a control message is returned. On the next iteration,
731 * CMSG_NXTHDR(msg, ret_val) should be passed as \a cmsg. */
732 struct cmsghdr* process_timestamp(grpc_tcp* tcp, msghdr* msg,
733 struct cmsghdr* cmsg) {
734 auto next_cmsg = CMSG_NXTHDR(msg, cmsg);
735 cmsghdr* opt_stats = nullptr;
736 if (next_cmsg == nullptr) {
737 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
738 gpr_log(GPR_ERROR, "Received timestamp without extended error");
743 /* Check if next_cmsg is an OPT_STATS msg */
744 if (next_cmsg->cmsg_level == SOL_SOCKET &&
745 next_cmsg->cmsg_type == SCM_TIMESTAMPING_OPT_STATS) {
746 opt_stats = next_cmsg;
747 next_cmsg = CMSG_NXTHDR(msg, opt_stats);
748 if (next_cmsg == nullptr) {
749 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
750 gpr_log(GPR_ERROR, "Received timestamp without extended error");
756 if (!(next_cmsg->cmsg_level == SOL_IP || next_cmsg->cmsg_level == SOL_IPV6) ||
757 !(next_cmsg->cmsg_type == IP_RECVERR ||
758 next_cmsg->cmsg_type == IPV6_RECVERR)) {
759 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
760 gpr_log(GPR_ERROR, "Unexpected control message");
766 reinterpret_cast<struct grpc_core::scm_timestamping*>(CMSG_DATA(cmsg));
767 auto serr = reinterpret_cast<struct sock_extended_err*>(CMSG_DATA(next_cmsg));
768 if (serr->ee_errno != ENOMSG ||
769 serr->ee_origin != SO_EE_ORIGIN_TIMESTAMPING) {
770 gpr_log(GPR_ERROR, "Unexpected control message");
773 /* The error handling can potentially be done on another thread so we need
774 * to protect the traced buffer list. A lock free list might be better. Using
775 * a simple mutex for now. */
776 gpr_mu_lock(&tcp->tb_mu);
777 grpc_core::TracedBuffer::ProcessTimestamp(&tcp->tb_head, serr, opt_stats,
779 gpr_mu_unlock(&tcp->tb_mu);
783 /** For linux platforms, reads the socket's error queue and processes error
784 * messages from the queue.
786 static void process_errors(grpc_tcp* tcp) {
789 iov.iov_base = nullptr;
792 msg.msg_name = nullptr;
798 /* Allocate enough space so we don't need to keep increasing this as size
799 * of OPT_STATS increase */
800 constexpr size_t cmsg_alloc_space =
801 CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) +
802 CMSG_SPACE(sizeof(sock_extended_err) + sizeof(sockaddr_in)) +
803 CMSG_SPACE(32 * NLA_ALIGN(NLA_HDRLEN + sizeof(uint64_t)));
804 /* Allocate aligned space for cmsgs received along with timestamps */
806 char rbuf[cmsg_alloc_space];
807 struct cmsghdr align;
809 memset(&aligned_buf, 0, sizeof(aligned_buf));
811 msg.msg_control = aligned_buf.rbuf;
812 msg.msg_controllen = sizeof(aligned_buf.rbuf);
816 r = recvmsg(tcp->fd, &msg, MSG_ERRQUEUE);
818 } while (r < 0 && saved_errno == EINTR);
820 if (r == -1 && saved_errno == EAGAIN) {
821 return; /* No more errors to process */
826 if ((msg.msg_flags & MSG_CTRUNC) != 0) {
827 gpr_log(GPR_ERROR, "Error message was truncated.");
830 if (msg.msg_controllen == 0) {
831 /* There was no control message found. It was probably spurious. */
835 for (auto cmsg = CMSG_FIRSTHDR(&msg); cmsg && cmsg->cmsg_len;
836 cmsg = CMSG_NXTHDR(&msg, cmsg)) {
837 if (cmsg->cmsg_level != SOL_SOCKET ||
838 cmsg->cmsg_type != SCM_TIMESTAMPING) {
839 /* Got a control message that is not a timestamp. Don't know how to
841 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
843 "unknown control message cmsg_level:%d cmsg_type:%d",
844 cmsg->cmsg_level, cmsg->cmsg_type);
848 cmsg = process_timestamp(tcp, &msg, cmsg);
857 static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error) {
858 grpc_tcp* tcp = static_cast<grpc_tcp*>(arg);
859 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
860 gpr_log(GPR_INFO, "TCP:%p got_error: %s", tcp, grpc_error_string(error));
863 if (error != GRPC_ERROR_NONE ||
864 static_cast<bool>(gpr_atm_acq_load(&tcp->stop_error_notification))) {
865 /* We aren't going to register to hear on error anymore, so it is safe to
867 TCP_UNREF(tcp, "error-tracking");
871 /* We are still interested in collecting timestamps, so let's try reading
874 /* This might not a timestamps error. Set the read and write closures to be
876 grpc_fd_set_readable(tcp->em_fd);
877 grpc_fd_set_writable(tcp->em_fd);
878 grpc_fd_notify_on_error(tcp->em_fd, &tcp->error_closure);
881 #else /* GRPC_LINUX_ERRQUEUE */
882 static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg,
883 size_t sending_length,
884 ssize_t* sent_length) {
885 gpr_log(GPR_ERROR, "Write with timestamps not supported for this platform");
890 static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error) {
891 gpr_log(GPR_ERROR, "Error handling is not supported for this platform");
894 #endif /* GRPC_LINUX_ERRQUEUE */
896 /* If outgoing_buffer_arg is filled, shuts down the list early, so that any
897 * release operations needed can be performed on the arg */
898 void tcp_shutdown_buffer_list(grpc_tcp* tcp) {
899 if (tcp->outgoing_buffer_arg) {
900 gpr_mu_lock(&tcp->tb_mu);
901 grpc_core::TracedBuffer::Shutdown(
902 &tcp->tb_head, tcp->outgoing_buffer_arg,
903 GRPC_ERROR_CREATE_FROM_STATIC_STRING("TracedBuffer list shutdown"));
904 gpr_mu_unlock(&tcp->tb_mu);
905 tcp->outgoing_buffer_arg = nullptr;
909 /* returns true if done, false if pending; if returning true, *error is set */
910 #if defined(IOV_MAX) && IOV_MAX < 1000
911 #define MAX_WRITE_IOVEC IOV_MAX
913 #define MAX_WRITE_IOVEC 1000
915 static bool tcp_flush(grpc_tcp* tcp, grpc_error** error) {
917 struct iovec iov[MAX_WRITE_IOVEC];
918 msg_iovlen_type iov_size;
919 ssize_t sent_length = 0;
920 size_t sending_length;
922 size_t unwind_slice_idx;
923 size_t unwind_byte_idx;
925 // We always start at zero, because we eagerly unref and trim the slice
926 // buffer as we write
927 size_t outgoing_slice_idx = 0;
931 unwind_slice_idx = outgoing_slice_idx;
932 unwind_byte_idx = tcp->outgoing_byte_idx;
933 for (iov_size = 0; outgoing_slice_idx != tcp->outgoing_buffer->count &&
934 iov_size != MAX_WRITE_IOVEC;
936 iov[iov_size].iov_base =
937 GRPC_SLICE_START_PTR(
938 tcp->outgoing_buffer->slices[outgoing_slice_idx]) +
939 tcp->outgoing_byte_idx;
940 iov[iov_size].iov_len =
941 GRPC_SLICE_LENGTH(tcp->outgoing_buffer->slices[outgoing_slice_idx]) -
942 tcp->outgoing_byte_idx;
943 sending_length += iov[iov_size].iov_len;
944 outgoing_slice_idx++;
945 tcp->outgoing_byte_idx = 0;
947 GPR_ASSERT(iov_size > 0);
949 msg.msg_name = nullptr;
952 msg.msg_iovlen = iov_size;
954 bool tried_sending_message = false;
955 if (tcp->outgoing_buffer_arg != nullptr) {
956 if (!tcp->ts_capable ||
957 !tcp_write_with_timestamps(tcp, &msg, sending_length, &sent_length)) {
958 /* We could not set socket options to collect Fathom timestamps.
959 * Fallback on writing without timestamps. */
960 tcp->ts_capable = false;
961 tcp_shutdown_buffer_list(tcp);
963 tried_sending_message = true;
966 if (!tried_sending_message) {
967 msg.msg_control = nullptr;
968 msg.msg_controllen = 0;
970 GRPC_STATS_INC_TCP_WRITE_SIZE(sending_length);
971 GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(iov_size);
973 sent_length = tcp_send(tcp->fd, &msg);
976 if (sent_length < 0) {
977 if (errno == EAGAIN) {
978 tcp->outgoing_byte_idx = unwind_byte_idx;
979 // unref all and forget about all slices that have been written to this
981 for (size_t idx = 0; idx < unwind_slice_idx; ++idx) {
982 grpc_slice_buffer_remove_first(tcp->outgoing_buffer);
985 } else if (errno == EPIPE) {
986 *error = tcp_annotate_error(GRPC_OS_ERROR(errno, "sendmsg"), tcp);
987 grpc_slice_buffer_reset_and_unref_internal(tcp->outgoing_buffer);
988 tcp_shutdown_buffer_list(tcp);
991 *error = tcp_annotate_error(GRPC_OS_ERROR(errno, "sendmsg"), tcp);
992 grpc_slice_buffer_reset_and_unref_internal(tcp->outgoing_buffer);
993 tcp_shutdown_buffer_list(tcp);
998 GPR_ASSERT(tcp->outgoing_byte_idx == 0);
999 tcp->bytes_counter += sent_length;
1000 trailing = sending_length - static_cast<size_t>(sent_length);
1001 while (trailing > 0) {
1002 size_t slice_length;
1004 outgoing_slice_idx--;
1006 GRPC_SLICE_LENGTH(tcp->outgoing_buffer->slices[outgoing_slice_idx]);
1007 if (slice_length > trailing) {
1008 tcp->outgoing_byte_idx = slice_length - trailing;
1011 trailing -= slice_length;
1014 if (outgoing_slice_idx == tcp->outgoing_buffer->count) {
1015 *error = GRPC_ERROR_NONE;
1016 grpc_slice_buffer_reset_and_unref_internal(tcp->outgoing_buffer);
1022 static void tcp_handle_write(void* arg /* grpc_tcp */, grpc_error* error) {
1023 grpc_tcp* tcp = static_cast<grpc_tcp*>(arg);
1026 if (error != GRPC_ERROR_NONE) {
1028 tcp->write_cb = nullptr;
1029 cb->cb(cb->cb_arg, error);
1030 TCP_UNREF(tcp, "write");
1034 if (!tcp_flush(tcp, &error)) {
1035 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
1036 gpr_log(GPR_INFO, "write: delayed");
1038 notify_on_write(tcp);
1041 tcp->write_cb = nullptr;
1042 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
1043 const char* str = grpc_error_string(error);
1044 gpr_log(GPR_INFO, "write: %s", str);
1046 GRPC_CLOSURE_SCHED(cb, error);
1047 TCP_UNREF(tcp, "write");
1051 static void tcp_write(grpc_endpoint* ep, grpc_slice_buffer* buf,
1052 grpc_closure* cb, void* arg) {
1053 GPR_TIMER_SCOPE("tcp_write", 0);
1054 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1055 grpc_error* error = GRPC_ERROR_NONE;
1057 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
1060 for (i = 0; i < buf->count; i++) {
1061 gpr_log(GPR_INFO, "WRITE %p (peer=%s)", tcp, tcp->peer_string);
1062 if (gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) {
1064 grpc_dump_slice(buf->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII);
1065 gpr_log(GPR_DEBUG, "DATA: %s", data);
1071 GPR_ASSERT(tcp->write_cb == nullptr);
1073 tcp->outgoing_buffer_arg = arg;
1074 if (buf->length == 0) {
1076 cb, grpc_fd_is_shutdown(tcp->em_fd)
1077 ? tcp_annotate_error(
1078 GRPC_ERROR_CREATE_FROM_STATIC_STRING("EOF"), tcp)
1080 tcp_shutdown_buffer_list(tcp);
1083 tcp->outgoing_buffer = buf;
1084 tcp->outgoing_byte_idx = 0;
1086 GPR_ASSERT(grpc_event_engine_can_track_errors());
1089 if (!tcp_flush(tcp, &error)) {
1090 TCP_REF(tcp, "write");
1092 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
1093 gpr_log(GPR_INFO, "write: delayed");
1095 notify_on_write(tcp);
1097 if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
1098 const char* str = grpc_error_string(error);
1099 gpr_log(GPR_INFO, "write: %s", str);
1101 GRPC_CLOSURE_SCHED(cb, error);
1105 static void tcp_add_to_pollset(grpc_endpoint* ep, grpc_pollset* pollset) {
1106 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1107 grpc_pollset_add_fd(pollset, tcp->em_fd);
1110 static void tcp_add_to_pollset_set(grpc_endpoint* ep,
1111 grpc_pollset_set* pollset_set) {
1112 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1113 grpc_pollset_set_add_fd(pollset_set, tcp->em_fd);
1116 static void tcp_delete_from_pollset_set(grpc_endpoint* ep,
1117 grpc_pollset_set* pollset_set) {
1118 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1119 grpc_pollset_set_del_fd(pollset_set, tcp->em_fd);
1122 static char* tcp_get_peer(grpc_endpoint* ep) {
1123 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1124 return gpr_strdup(tcp->peer_string);
1127 static int tcp_get_fd(grpc_endpoint* ep) {
1128 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1132 static grpc_resource_user* tcp_get_resource_user(grpc_endpoint* ep) {
1133 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1134 return tcp->resource_user;
1137 static bool tcp_can_track_err(grpc_endpoint* ep) {
1138 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1139 if (!grpc_event_engine_can_track_errors()) {
1142 struct sockaddr addr;
1143 socklen_t len = sizeof(addr);
1144 if (getsockname(tcp->fd, &addr, &len) < 0) {
1147 if (addr.sa_family == AF_INET || addr.sa_family == AF_INET6) {
1153 static const grpc_endpoint_vtable vtable = {tcp_read,
1156 tcp_add_to_pollset_set,
1157 tcp_delete_from_pollset_set,
1160 tcp_get_resource_user,
1165 #define MAX_CHUNK_SIZE 32 * 1024 * 1024
1167 grpc_endpoint* grpc_tcp_create(grpc_fd* em_fd,
1168 const grpc_channel_args* channel_args,
1169 const char* peer_string) {
1170 int tcp_read_chunk_size = GRPC_TCP_DEFAULT_READ_SLICE_SIZE;
1171 int tcp_max_read_chunk_size = 4 * 1024 * 1024;
1172 int tcp_min_read_chunk_size = 256;
1173 grpc_resource_quota* resource_quota = grpc_resource_quota_create(nullptr);
1174 if (channel_args != nullptr) {
1175 for (size_t i = 0; i < channel_args->num_args; i++) {
1177 strcmp(channel_args->args[i].key, GRPC_ARG_TCP_READ_CHUNK_SIZE)) {
1178 grpc_integer_options options = {tcp_read_chunk_size, 1, MAX_CHUNK_SIZE};
1179 tcp_read_chunk_size =
1180 grpc_channel_arg_get_integer(&channel_args->args[i], options);
1181 } else if (0 == strcmp(channel_args->args[i].key,
1182 GRPC_ARG_TCP_MIN_READ_CHUNK_SIZE)) {
1183 grpc_integer_options options = {tcp_read_chunk_size, 1, MAX_CHUNK_SIZE};
1184 tcp_min_read_chunk_size =
1185 grpc_channel_arg_get_integer(&channel_args->args[i], options);
1186 } else if (0 == strcmp(channel_args->args[i].key,
1187 GRPC_ARG_TCP_MAX_READ_CHUNK_SIZE)) {
1188 grpc_integer_options options = {tcp_read_chunk_size, 1, MAX_CHUNK_SIZE};
1189 tcp_max_read_chunk_size =
1190 grpc_channel_arg_get_integer(&channel_args->args[i], options);
1192 strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) {
1193 grpc_resource_quota_unref_internal(resource_quota);
1195 grpc_resource_quota_ref_internal(static_cast<grpc_resource_quota*>(
1196 channel_args->args[i].value.pointer.p));
1201 if (tcp_min_read_chunk_size > tcp_max_read_chunk_size) {
1202 tcp_min_read_chunk_size = tcp_max_read_chunk_size;
1204 tcp_read_chunk_size = GPR_CLAMP(tcp_read_chunk_size, tcp_min_read_chunk_size,
1205 tcp_max_read_chunk_size);
1207 grpc_tcp* tcp = static_cast<grpc_tcp*>(gpr_malloc(sizeof(grpc_tcp)));
1208 tcp->base.vtable = &vtable;
1209 tcp->peer_string = gpr_strdup(peer_string);
1210 tcp->fd = grpc_fd_wrapped_fd(em_fd);
1211 tcp->read_cb = nullptr;
1212 tcp->write_cb = nullptr;
1213 tcp->release_fd_cb = nullptr;
1214 tcp->release_fd = nullptr;
1215 tcp->incoming_buffer = nullptr;
1216 tcp->target_length = static_cast<double>(tcp_read_chunk_size);
1217 tcp->min_read_chunk_size = tcp_min_read_chunk_size;
1218 tcp->max_read_chunk_size = tcp_max_read_chunk_size;
1219 tcp->bytes_read_this_round = 0;
1220 /* Will be set to false by the very first endpoint read function */
1221 tcp->is_first_read = true;
1222 tcp->bytes_counter = -1;
1223 tcp->socket_ts_enabled = false;
1224 tcp->ts_capable = true;
1225 tcp->outgoing_buffer_arg = nullptr;
1226 /* paired with unref in grpc_tcp_destroy */
1227 gpr_ref_init(&tcp->refcount, 1);
1228 gpr_atm_no_barrier_store(&tcp->shutdown_count, 0);
1230 grpc_slice_buffer_init(&tcp->last_read_buffer);
1231 tcp->resource_user = grpc_resource_user_create(resource_quota, peer_string);
1232 grpc_resource_user_slice_allocator_init(
1233 &tcp->slice_allocator, tcp->resource_user, tcp_read_allocation_done, tcp);
1234 grpc_resource_quota_unref_internal(resource_quota);
1235 gpr_mu_init(&tcp->tb_mu);
1236 tcp->tb_head = nullptr;
1237 GRPC_CLOSURE_INIT(&tcp->read_done_closure, tcp_handle_read, tcp,
1238 grpc_schedule_on_exec_ctx);
1239 if (grpc_event_engine_run_in_background()) {
1240 // If there is a polling engine always running in the background, there is
1241 // no need to run the backup poller.
1242 GRPC_CLOSURE_INIT(&tcp->write_done_closure, tcp_handle_write, tcp,
1243 grpc_schedule_on_exec_ctx);
1245 GRPC_CLOSURE_INIT(&tcp->write_done_closure,
1246 tcp_drop_uncovered_then_handle_write, tcp,
1247 grpc_schedule_on_exec_ctx);
1249 /* Always assume there is something on the queue to read. */
1251 #ifdef GRPC_HAVE_TCP_INQ
1253 if (setsockopt(tcp->fd, SOL_TCP, TCP_INQ, &one, sizeof(one)) == 0) {
1254 tcp->inq_capable = true;
1256 gpr_log(GPR_DEBUG, "cannot set inq fd=%d errno=%d", tcp->fd, errno);
1257 tcp->inq_capable = false;
1260 tcp->inq_capable = false;
1261 #endif /* GRPC_HAVE_TCP_INQ */
1262 /* Start being notified on errors if event engine can track errors. */
1263 if (grpc_event_engine_can_track_errors()) {
1264 /* Grab a ref to tcp so that we can safely access the tcp struct when
1265 * processing errors. We unref when we no longer want to track errors
1267 TCP_REF(tcp, "error-tracking");
1268 gpr_atm_rel_store(&tcp->stop_error_notification, 0);
1269 GRPC_CLOSURE_INIT(&tcp->error_closure, tcp_handle_error, tcp,
1270 grpc_schedule_on_exec_ctx);
1271 grpc_fd_notify_on_error(tcp->em_fd, &tcp->error_closure);
1277 int grpc_tcp_fd(grpc_endpoint* ep) {
1278 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1279 GPR_ASSERT(ep->vtable == &vtable);
1280 return grpc_fd_wrapped_fd(tcp->em_fd);
1283 void grpc_tcp_destroy_and_release_fd(grpc_endpoint* ep, int* fd,
1284 grpc_closure* done) {
1285 grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
1286 GPR_ASSERT(ep->vtable == &vtable);
1287 tcp->release_fd = fd;
1288 tcp->release_fd_cb = done;
1289 grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer);
1290 if (grpc_event_engine_can_track_errors()) {
1291 /* Stop errors notification. */
1292 gpr_atm_no_barrier_store(&tcp->stop_error_notification, true);
1293 grpc_fd_set_error(tcp->em_fd);
1295 TCP_UNREF(tcp, "destroy");
1298 #endif /* GRPC_POSIX_SOCKET_TCP */