3 * Copyright 2016 gRPC authors.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
19 #ifndef GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H
20 #define GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H
22 #include <grpc/support/port_platform.h>
24 #include <grpc/slice.h>
25 #include <grpc/slice_buffer.h>
28 #include "src/core/lib/gpr/murmur_hash.h"
29 #include "src/core/lib/gprpp/ref_counted.h"
30 #include "src/core/lib/transport/static_metadata.h"
32 // Interned slices have specific fast-path operations for hashing. To inline
33 // these operations, we need to forward declare them here.
34 extern uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
35 extern uint32_t g_hash_seed;
37 // grpc_slice_refcount : A reference count for grpc_slice.
39 // Non-inlined grpc_slice objects are refcounted. Historically this was
40 // implemented via grpc_slice_refcount, a C-style polymorphic class using a
41 // manually managed vtable of operations. Subclasses would define their own
42 // vtable; the 'virtual' methods (ref, unref, equals and hash) would simply call
43 // the function pointers in the vtable as necessary.
45 // Unfortunately, this leads to some inefficiencies in the generated code that
46 // can be improved upon. For example, equality checking for interned slices is a
47 // simple equality check on the refcount pointer. With the vtable approach, this
48 // would translate to roughly the following (high-level) instructions:
50 // grpc_slice_equals(slice1, slice2):
51 // load vtable->eq -> eq_func
52 // call eq_func(slice1, slice2)
54 // interned_slice_equals(slice1, slice2)
55 // load slice1.ref -> r1
56 // load slice2.ref -> r2
57 // cmp r1, r2 -> retval
60 // This leads to a function call for a function defined in another translation
61 // unit, which imposes memory barriers, which reduces the compiler's ability to
62 // optimize (in addition to the added overhead of call/ret). Additionally, it
63 // may be harder to reason about branch prediction when we're jumping to
64 // essentially arbitrarily provided function pointers.
66 // In addition, it is arguable that while virtualization was helpful for
67 // Equals()/Hash() methods, that it was fundamentally unnecessary for
70 // Instead, grpc_slice_refcount provides the same functionality as the C-style
71 // virtual class, but in a de-virtualized manner - Eq(), Hash(), Ref() and
72 // Unref() are provided within this header file. Fastpaths for Eq()/Hash()
73 // (interned and static metadata slices), as well as the Ref() operation, can
74 // all be inlined without any memory barriers.
77 // 1. Using grpc_core::RefCount<> (header-only) for Ref/Unref. Two special cases
78 // need support: No-op ref/unref (eg. static metadata slices) and stream
79 // slice references (where all the slices share the streamref). This is in
80 // addition to the normal case of '1 slice, 1 ref'.
81 // To support these cases, we explicitly track a nullable pointer to the
82 // underlying RefCount<>. No-op ref/unref is used by checking the pointer for
83 // null, and doing nothing if it is. Both stream slice refs and 'normal'
84 // slices use the same path for Ref/Unref (by targeting the non-null
87 // 2. introducing the notion of grpc_slice_refcount::Type. This describes if a
88 // slice ref is used by a static metadata slice, an interned slice, or other
89 // slices. We switch on the slice ref type in order to provide fastpaths for
90 // Equals() and Hash().
92 // In total, this saves us roughly 1-2% latency for unary calls, with smaller
93 // calls benefitting. The effect is present, but not as useful, for larger calls
94 // where the cost of sending the data dominates.
95 struct grpc_slice_refcount {
98 STATIC, // Refcount for a static metadata slice.
99 INTERNED, // Refcount for an interned slice.
100 REGULAR // Refcount for non-static-metadata, non-interned slices.
102 typedef void (*DestroyerFn)(void*);
104 grpc_slice_refcount() = default;
106 explicit grpc_slice_refcount(grpc_slice_refcount* sub) : sub_refcount_(sub) {}
107 // Regular constructor for grpc_slice_refcount.
110 // 1. grpc_slice_refcount::Type type
111 // Whether we are the refcount for a static
112 // metadata slice, an interned slice, or any other kind of slice.
115 // The pointer to the actual underlying grpc_core::RefCount. Rather than
116 // performing struct offset computations as in the original implementation to
117 // get to the refcount, which requires a virtual method, we devirtualize by
118 // using a nullable pointer to allow a single pair of Ref/Unref methods.
120 // 3. DestroyerFn destroyer_fn
121 // Called when the refcount goes to 0, with destroyer_arg as parameter.
123 // 4. void* destroyer_arg
124 // Argument for the virtualized destructor.
126 // 5. grpc_slice_refcount* sub
127 // Argument used for interned slices.
128 grpc_slice_refcount(grpc_slice_refcount::Type type, grpc_core::RefCount* ref,
129 DestroyerFn destroyer_fn, void* destroyer_arg,
130 grpc_slice_refcount* sub)
134 dest_fn_(destroyer_fn),
135 destroy_fn_arg_(destroyer_arg) {}
136 // Initializer for static refcounts.
137 grpc_slice_refcount(grpc_slice_refcount* sub, Type type)
138 : ref_type_(type), sub_refcount_(sub) {}
140 Type GetType() const { return ref_type_; }
142 int Eq(const grpc_slice& a, const grpc_slice& b);
144 uint32_t Hash(const grpc_slice& slice);
146 if (ref_ == nullptr) return;
150 if (ref_ == nullptr) return;
152 dest_fn_(destroy_fn_arg_);
156 grpc_slice_refcount* sub_refcount() const { return sub_refcount_; }
159 grpc_core::RefCount* ref_ = nullptr;
160 const Type ref_type_ = Type::REGULAR;
161 grpc_slice_refcount* sub_refcount_ = this;
162 DestroyerFn dest_fn_ = nullptr;
163 void* destroy_fn_arg_ = nullptr;
166 namespace grpc_core {
168 struct InternedSliceRefcount {
169 static void Destroy(void* arg) {
170 auto* rc = static_cast<InternedSliceRefcount*>(arg);
171 rc->~InternedSliceRefcount();
175 InternedSliceRefcount(size_t length, uint32_t hash,
176 InternedSliceRefcount* bucket_next)
177 : base(grpc_slice_refcount::Type::INTERNED, &refcnt, Destroy, this, &sub),
178 sub(grpc_slice_refcount::Type::REGULAR, &refcnt, Destroy, this, &sub),
181 bucket_next(bucket_next) {}
183 ~InternedSliceRefcount();
185 grpc_slice_refcount base;
186 grpc_slice_refcount sub;
190 InternedSliceRefcount* bucket_next;
193 } // namespace grpc_core
195 inline int grpc_slice_refcount::Eq(const grpc_slice& a, const grpc_slice& b) {
198 return GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b);
200 return a.refcount == b.refcount;
204 if (GRPC_SLICE_LENGTH(a) != GRPC_SLICE_LENGTH(b)) return false;
205 if (GRPC_SLICE_LENGTH(a) == 0) return true;
206 return 0 == memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b),
207 GRPC_SLICE_LENGTH(a));
210 inline uint32_t grpc_slice_refcount::Hash(const grpc_slice& slice) {
213 return ::grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(
216 return reinterpret_cast<grpc_core::InternedSliceRefcount*>(slice.refcount)
221 return gpr_murmur_hash3(GRPC_SLICE_START_PTR(slice), GRPC_SLICE_LENGTH(slice),
225 inline const grpc_slice& grpc_slice_ref_internal(const grpc_slice& slice) {
226 if (slice.refcount) {
227 slice.refcount->Ref();
232 inline void grpc_slice_unref_internal(const grpc_slice& slice) {
233 if (slice.refcount) {
234 slice.refcount->Unref();
238 void grpc_slice_buffer_reset_and_unref_internal(grpc_slice_buffer* sb);
239 void grpc_slice_buffer_partial_unref_internal(grpc_slice_buffer* sb,
241 void grpc_slice_buffer_destroy_internal(grpc_slice_buffer* sb);
243 /* Check if a slice is interned */
244 bool grpc_slice_is_interned(const grpc_slice& slice);
246 void grpc_slice_intern_init(void);
247 void grpc_slice_intern_shutdown(void);
248 void grpc_test_only_set_slice_hash_seed(uint32_t key);
249 // if slice matches a static slice, returns the static slice
250 // otherwise returns the passed in slice (without reffing it)
251 // used for surface boundaries where we might receive an un-interned static
253 grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
254 bool* returned_slice_is_different);
255 uint32_t grpc_static_slice_hash(grpc_slice s);
256 int grpc_static_slice_eq(grpc_slice a, grpc_slice b);
258 // Returns the memory used by this slice, not counting the slice structure
259 // itself. This means that inlined and slices from static strings will return
260 // 0. All other slices will return the size of the allocated chars.
261 size_t grpc_slice_memory_usage(grpc_slice s);
263 #endif /* GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H */