1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "src/code-factory.h"
6 #include "src/code-stubs.h"
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/js-generic-lowering.h"
9 #include "src/compiler/js-graph.h"
10 #include "src/compiler/machine-operator.h"
11 #include "src/compiler/node-matchers.h"
12 #include "src/compiler/node-properties.h"
13 #include "src/compiler/operator-properties.h"
14 #include "src/unique.h"
20 JSGenericLowering::JSGenericLowering(bool is_typing_enabled, JSGraph* jsgraph)
21 : is_typing_enabled_(is_typing_enabled), jsgraph_(jsgraph) {}
24 JSGenericLowering::~JSGenericLowering() {}
27 Reduction JSGenericLowering::Reduce(Node* node) {
28 switch (node->opcode()) {
29 #define DECLARE_CASE(x) \
30 case IrOpcode::k##x: \
33 JS_OP_LIST(DECLARE_CASE)
35 case IrOpcode::kBranch:
36 // TODO(mstarzinger): If typing is enabled then simplified lowering will
37 // have inserted the correct ChangeBoolToBit, otherwise we need to perform
38 // poor-man's representation inference here and insert manual change.
39 if (!is_typing_enabled_) {
40 Node* condition = node->InputAt(0);
41 if (condition->opcode() != IrOpcode::kAlways) {
42 Node* test = graph()->NewNode(machine()->WordEqual(), condition,
43 jsgraph()->TrueConstant());
44 node->ReplaceInput(0, test);
57 #define REPLACE_BINARY_OP_IC_CALL(op, token) \
58 void JSGenericLowering::Lower##op(Node* node) { \
59 ReplaceWithStubCall(node, CodeFactory::BinaryOpIC(isolate(), token), \
60 CallDescriptor::kPatchableCallSiteWithNop); \
62 REPLACE_BINARY_OP_IC_CALL(JSBitwiseOr, Token::BIT_OR)
63 REPLACE_BINARY_OP_IC_CALL(JSBitwiseXor, Token::BIT_XOR)
64 REPLACE_BINARY_OP_IC_CALL(JSBitwiseAnd, Token::BIT_AND)
65 REPLACE_BINARY_OP_IC_CALL(JSShiftLeft, Token::SHL)
66 REPLACE_BINARY_OP_IC_CALL(JSShiftRight, Token::SAR)
67 REPLACE_BINARY_OP_IC_CALL(JSShiftRightLogical, Token::SHR)
68 REPLACE_BINARY_OP_IC_CALL(JSAdd, Token::ADD)
69 REPLACE_BINARY_OP_IC_CALL(JSSubtract, Token::SUB)
70 REPLACE_BINARY_OP_IC_CALL(JSMultiply, Token::MUL)
71 REPLACE_BINARY_OP_IC_CALL(JSDivide, Token::DIV)
72 REPLACE_BINARY_OP_IC_CALL(JSModulus, Token::MOD)
73 #undef REPLACE_BINARY_OP_IC_CALL
76 #define REPLACE_COMPARE_IC_CALL(op, token) \
77 void JSGenericLowering::Lower##op(Node* node) { \
78 ReplaceWithCompareIC(node, token); \
80 REPLACE_COMPARE_IC_CALL(JSEqual, Token::EQ)
81 REPLACE_COMPARE_IC_CALL(JSNotEqual, Token::NE)
82 REPLACE_COMPARE_IC_CALL(JSStrictEqual, Token::EQ_STRICT)
83 REPLACE_COMPARE_IC_CALL(JSStrictNotEqual, Token::NE_STRICT)
84 REPLACE_COMPARE_IC_CALL(JSLessThan, Token::LT)
85 REPLACE_COMPARE_IC_CALL(JSGreaterThan, Token::GT)
86 REPLACE_COMPARE_IC_CALL(JSLessThanOrEqual, Token::LTE)
87 REPLACE_COMPARE_IC_CALL(JSGreaterThanOrEqual, Token::GTE)
88 #undef REPLACE_COMPARE_IC_CALL
91 #define REPLACE_RUNTIME_CALL(op, fun) \
92 void JSGenericLowering::Lower##op(Node* node) { \
93 ReplaceWithRuntimeCall(node, fun); \
95 REPLACE_RUNTIME_CALL(JSTypeOf, Runtime::kTypeof)
96 REPLACE_RUNTIME_CALL(JSCreate, Runtime::kAbort)
97 REPLACE_RUNTIME_CALL(JSCreateFunctionContext, Runtime::kNewFunctionContext)
98 REPLACE_RUNTIME_CALL(JSCreateWithContext, Runtime::kPushWithContext)
99 REPLACE_RUNTIME_CALL(JSCreateBlockContext, Runtime::kPushBlockContext)
100 REPLACE_RUNTIME_CALL(JSCreateModuleContext, Runtime::kPushModuleContext)
101 REPLACE_RUNTIME_CALL(JSCreateScriptContext, Runtime::kAbort)
102 #undef REPLACE_RUNTIME
105 #define REPLACE_UNIMPLEMENTED(op) \
106 void JSGenericLowering::Lower##op(Node* node) { UNIMPLEMENTED(); }
107 REPLACE_UNIMPLEMENTED(JSYield)
108 #undef REPLACE_UNIMPLEMENTED
111 static CallDescriptor::Flags FlagsForNode(Node* node) {
112 CallDescriptor::Flags result = CallDescriptor::kNoFlags;
113 if (OperatorProperties::GetFrameStateInputCount(node->op()) > 0) {
114 result |= CallDescriptor::kNeedsFrameState;
120 void JSGenericLowering::ReplaceWithCompareIC(Node* node, Token::Value token) {
121 Callable callable = CodeFactory::CompareIC(isolate(), token);
122 CallDescriptor* desc_compare = Linkage::GetStubCallDescriptor(
123 isolate(), zone(), callable.descriptor(), 0,
124 CallDescriptor::kPatchableCallSiteWithNop | FlagsForNode(node),
125 Operator::kNoProperties, kMachInt32);
127 // Create a new call node asking a CompareIC for help.
128 NodeVector inputs(zone());
129 inputs.reserve(node->InputCount() + 1);
130 inputs.push_back(jsgraph()->HeapConstant(callable.code()));
131 inputs.push_back(NodeProperties::GetValueInput(node, 0));
132 inputs.push_back(NodeProperties::GetValueInput(node, 1));
133 inputs.push_back(NodeProperties::GetContextInput(node));
134 if (node->op()->HasProperty(Operator::kPure)) {
135 // A pure (strict) comparison doesn't have an effect, control or frame
136 // state. But for the graph, we need to add control and effect inputs.
137 DCHECK(OperatorProperties::GetFrameStateInputCount(node->op()) == 0);
138 inputs.push_back(graph()->start());
139 inputs.push_back(graph()->start());
141 DCHECK((OperatorProperties::GetFrameStateInputCount(node->op()) == 1) ==
142 FLAG_turbo_deoptimization);
143 if (FLAG_turbo_deoptimization) {
144 inputs.push_back(NodeProperties::GetFrameStateInput(node, 0));
146 inputs.push_back(NodeProperties::GetEffectInput(node));
147 inputs.push_back(NodeProperties::GetControlInput(node));
150 graph()->NewNode(common()->Call(desc_compare),
151 static_cast<int>(inputs.size()), &inputs.front());
152 NodeProperties::SetBounds(
153 compare, Bounds(Type::None(zone()), Type::UntaggedSigned(zone())));
155 // Decide how the return value from the above CompareIC can be converted into
156 // a JavaScript boolean oddball depending on the given token.
157 Node* false_value = jsgraph()->FalseConstant();
158 Node* true_value = jsgraph()->TrueConstant();
159 const Operator* op = nullptr;
161 case Token::EQ: // a == 0
162 case Token::EQ_STRICT:
163 op = machine()->WordEqual();
165 case Token::NE: // a != 0 becomes !(a == 0)
166 case Token::NE_STRICT:
167 op = machine()->WordEqual();
168 std::swap(true_value, false_value);
170 case Token::LT: // a < 0
171 op = machine()->IntLessThan();
173 case Token::GT: // a > 0 becomes !(a <= 0)
174 op = machine()->IntLessThanOrEqual();
175 std::swap(true_value, false_value);
177 case Token::LTE: // a <= 0
178 op = machine()->IntLessThanOrEqual();
180 case Token::GTE: // a >= 0 becomes !(a < 0)
181 op = machine()->IntLessThan();
182 std::swap(true_value, false_value);
187 Node* booleanize = graph()->NewNode(op, compare, jsgraph()->ZeroConstant());
189 // Finally patch the original node to select a boolean.
190 NodeProperties::ReplaceWithValue(node, node, compare);
191 // TODO(mstarzinger): Just a work-around because SelectLowering might
192 // otherwise introduce a Phi without any uses, making Scheduler unhappy.
193 if (node->UseCount() == 0) return;
194 node->TrimInputCount(3);
195 node->ReplaceInput(0, booleanize);
196 node->ReplaceInput(1, true_value);
197 node->ReplaceInput(2, false_value);
198 node->set_op(common()->Select(kMachAnyTagged));
202 void JSGenericLowering::ReplaceWithStubCall(Node* node, Callable callable,
203 CallDescriptor::Flags flags) {
204 Operator::Properties properties = node->op()->properties();
205 flags |= FlagsForNode(node);
206 CallDescriptor* desc = Linkage::GetStubCallDescriptor(
207 isolate(), zone(), callable.descriptor(), 0, flags, properties);
208 const Operator* new_op = common()->Call(desc);
210 // Take care of frame states.
211 int old_frame_state_count =
212 OperatorProperties::GetFrameStateInputCount(node->op());
213 int new_frame_state_count =
214 (flags & CallDescriptor::kNeedsFrameState) ? 1 : 0;
215 DCHECK_GE(old_frame_state_count, new_frame_state_count);
216 // If there are extra frame states, get rid of them.
217 for (int i = new_frame_state_count; i < old_frame_state_count; i++) {
218 node->RemoveInput(NodeProperties::FirstFrameStateIndex(node) +
219 new_frame_state_count);
222 Node* stub_code = jsgraph()->HeapConstant(callable.code());
223 node->InsertInput(zone(), 0, stub_code);
224 node->set_op(new_op);
228 void JSGenericLowering::ReplaceWithBuiltinCall(Node* node,
229 Builtins::JavaScript id,
231 Operator::Properties properties = node->op()->properties();
233 CodeFactory::CallFunction(isolate(), nargs - 1, NO_CALL_FUNCTION_FLAGS);
234 CallDescriptor* desc =
235 Linkage::GetStubCallDescriptor(isolate(), zone(), callable.descriptor(),
236 nargs, FlagsForNode(node), properties);
237 Node* global_object = graph()->NewNode(
238 machine()->Load(kMachAnyTagged), NodeProperties::GetContextInput(node),
239 jsgraph()->IntPtrConstant(
240 Context::SlotOffset(Context::GLOBAL_OBJECT_INDEX)),
241 NodeProperties::GetEffectInput(node), graph()->start());
242 Node* builtins_object = graph()->NewNode(
243 machine()->Load(kMachAnyTagged), global_object,
244 jsgraph()->IntPtrConstant(GlobalObject::kBuiltinsOffset - kHeapObjectTag),
245 NodeProperties::GetEffectInput(node), graph()->start());
246 Node* function = graph()->NewNode(
247 machine()->Load(kMachAnyTagged), builtins_object,
248 jsgraph()->IntPtrConstant(JSBuiltinsObject::OffsetOfFunctionWithId(id) -
250 NodeProperties::GetEffectInput(node), graph()->start());
251 Node* stub_code = jsgraph()->HeapConstant(callable.code());
252 node->InsertInput(zone(), 0, stub_code);
253 node->InsertInput(zone(), 1, function);
254 node->set_op(common()->Call(desc));
258 void JSGenericLowering::ReplaceWithRuntimeCall(Node* node,
259 Runtime::FunctionId f,
260 int nargs_override) {
261 Operator::Properties properties = node->op()->properties();
262 const Runtime::Function* fun = Runtime::FunctionForId(f);
263 int nargs = (nargs_override < 0) ? fun->nargs : nargs_override;
264 CallDescriptor* desc =
265 Linkage::GetRuntimeCallDescriptor(zone(), f, nargs, properties);
266 Node* ref = jsgraph()->ExternalConstant(ExternalReference(f, isolate()));
267 Node* arity = jsgraph()->Int32Constant(nargs);
268 node->InsertInput(zone(), 0, jsgraph()->CEntryStubConstant(fun->result_size));
269 node->InsertInput(zone(), nargs + 1, ref);
270 node->InsertInput(zone(), nargs + 2, arity);
271 node->set_op(common()->Call(desc));
275 void JSGenericLowering::LowerJSUnaryNot(Node* node) {
276 Callable callable = CodeFactory::ToBoolean(
277 isolate(), ToBooleanStub::RESULT_AS_INVERSE_ODDBALL);
278 ReplaceWithStubCall(node, callable, CallDescriptor::kPatchableCallSite);
282 void JSGenericLowering::LowerJSToBoolean(Node* node) {
284 CodeFactory::ToBoolean(isolate(), ToBooleanStub::RESULT_AS_ODDBALL);
285 ReplaceWithStubCall(node, callable, CallDescriptor::kPatchableCallSite);
289 void JSGenericLowering::LowerJSToNumber(Node* node) {
290 Callable callable = CodeFactory::ToNumber(isolate());
291 ReplaceWithStubCall(node, callable, FlagsForNode(node));
295 void JSGenericLowering::LowerJSToString(Node* node) {
296 ReplaceWithBuiltinCall(node, Builtins::TO_STRING, 1);
300 void JSGenericLowering::LowerJSToName(Node* node) {
301 ReplaceWithBuiltinCall(node, Builtins::TO_NAME, 1);
305 void JSGenericLowering::LowerJSToObject(Node* node) {
306 ReplaceWithBuiltinCall(node, Builtins::TO_OBJECT, 1);
310 void JSGenericLowering::LowerJSLoadProperty(Node* node) {
311 const LoadPropertyParameters& p = LoadPropertyParametersOf(node->op());
313 CodeFactory::KeyedLoadICInOptimizedCode(isolate(), UNINITIALIZED);
314 if (FLAG_vector_ics) {
315 node->InsertInput(zone(), 2, jsgraph()->SmiConstant(p.feedback().index()));
316 node->InsertInput(zone(), 3,
317 jsgraph()->HeapConstant(p.feedback().vector()));
319 ReplaceWithStubCall(node, callable, CallDescriptor::kPatchableCallSite);
323 void JSGenericLowering::LowerJSLoadNamed(Node* node) {
324 const LoadNamedParameters& p = LoadNamedParametersOf(node->op());
325 Callable callable = CodeFactory::LoadICInOptimizedCode(
326 isolate(), p.contextual_mode(), UNINITIALIZED);
327 node->InsertInput(zone(), 1, jsgraph()->HeapConstant(p.name()));
328 if (FLAG_vector_ics) {
329 node->InsertInput(zone(), 2, jsgraph()->SmiConstant(p.feedback().index()));
330 node->InsertInput(zone(), 3,
331 jsgraph()->HeapConstant(p.feedback().vector()));
333 ReplaceWithStubCall(node, callable, CallDescriptor::kPatchableCallSite);
337 void JSGenericLowering::LowerJSStoreProperty(Node* node) {
338 LanguageMode language_mode = OpParameter<LanguageMode>(node);
339 Callable callable = CodeFactory::KeyedStoreICInOptimizedCode(
340 isolate(), language_mode, UNINITIALIZED);
341 ReplaceWithStubCall(node, callable, CallDescriptor::kPatchableCallSite);
345 void JSGenericLowering::LowerJSStoreNamed(Node* node) {
346 const StoreNamedParameters& p = StoreNamedParametersOf(node->op());
347 Callable callable = CodeFactory::StoreIC(isolate(), p.language_mode());
348 node->InsertInput(zone(), 1, jsgraph()->HeapConstant(p.name()));
349 ReplaceWithStubCall(node, callable, CallDescriptor::kPatchableCallSite);
353 void JSGenericLowering::LowerJSDeleteProperty(Node* node) {
354 LanguageMode language_mode = OpParameter<LanguageMode>(node);
355 ReplaceWithBuiltinCall(node, Builtins::DELETE, 3);
356 node->InsertInput(zone(), 4, jsgraph()->SmiConstant(language_mode));
360 void JSGenericLowering::LowerJSHasProperty(Node* node) {
361 ReplaceWithBuiltinCall(node, Builtins::IN, 2);
365 void JSGenericLowering::LowerJSInstanceOf(Node* node) {
366 InstanceofStub::Flags flags = static_cast<InstanceofStub::Flags>(
367 InstanceofStub::kReturnTrueFalseObject |
368 InstanceofStub::kArgsInRegisters);
369 InstanceofStub stub(isolate(), flags);
370 CallInterfaceDescriptor d = stub.GetCallInterfaceDescriptor();
371 CallDescriptor* desc = Linkage::GetStubCallDescriptor(isolate(), zone(), d, 0,
373 Node* stub_code = jsgraph()->HeapConstant(stub.GetCode());
374 node->InsertInput(zone(), 0, stub_code);
375 node->set_op(common()->Call(desc));
379 void JSGenericLowering::LowerJSLoadContext(Node* node) {
380 const ContextAccess& access = ContextAccessOf(node->op());
381 for (size_t i = 0; i < access.depth(); ++i) {
383 0, graph()->NewNode(machine()->Load(kMachAnyTagged),
384 NodeProperties::GetValueInput(node, 0),
385 jsgraph()->Int32Constant(
386 Context::SlotOffset(Context::PREVIOUS_INDEX)),
387 NodeProperties::GetEffectInput(node),
390 node->ReplaceInput(1, jsgraph()->Int32Constant(Context::SlotOffset(
391 static_cast<int>(access.index()))));
392 node->AppendInput(zone(), graph()->start());
393 node->set_op(machine()->Load(kMachAnyTagged));
397 void JSGenericLowering::LowerJSStoreContext(Node* node) {
398 const ContextAccess& access = ContextAccessOf(node->op());
399 for (size_t i = 0; i < access.depth(); ++i) {
401 0, graph()->NewNode(machine()->Load(kMachAnyTagged),
402 NodeProperties::GetValueInput(node, 0),
403 jsgraph()->Int32Constant(
404 Context::SlotOffset(Context::PREVIOUS_INDEX)),
405 NodeProperties::GetEffectInput(node),
408 node->ReplaceInput(2, NodeProperties::GetValueInput(node, 1));
409 node->ReplaceInput(1, jsgraph()->Int32Constant(Context::SlotOffset(
410 static_cast<int>(access.index()))));
412 machine()->Store(StoreRepresentation(kMachAnyTagged, kFullWriteBarrier)));
416 void JSGenericLowering::LowerJSCreateCatchContext(Node* node) {
417 Unique<String> name = OpParameter<Unique<String>>(node);
418 node->InsertInput(zone(), 0, jsgraph()->HeapConstant(name));
419 ReplaceWithRuntimeCall(node, Runtime::kPushCatchContext);
423 void JSGenericLowering::LowerJSCallConstruct(Node* node) {
424 int arity = OpParameter<int>(node);
425 CallConstructStub stub(isolate(), NO_CALL_CONSTRUCTOR_FLAGS);
426 CallInterfaceDescriptor d = stub.GetCallInterfaceDescriptor();
427 CallDescriptor* desc = Linkage::GetStubCallDescriptor(
428 isolate(), zone(), d, arity, FlagsForNode(node));
429 Node* stub_code = jsgraph()->HeapConstant(stub.GetCode());
430 Node* construct = NodeProperties::GetValueInput(node, 0);
431 node->InsertInput(zone(), 0, stub_code);
432 node->InsertInput(zone(), 1, jsgraph()->Int32Constant(arity - 1));
433 node->InsertInput(zone(), 2, construct);
434 node->InsertInput(zone(), 3, jsgraph()->UndefinedConstant());
435 node->set_op(common()->Call(desc));
439 bool JSGenericLowering::TryLowerDirectJSCall(Node* node) {
440 // Lower to a direct call to a constant JSFunction if legal.
441 const CallFunctionParameters& p = CallFunctionParametersOf(node->op());
442 int arg_count = static_cast<int>(p.arity() - 2);
444 // Check the function is a constant and is really a JSFunction.
445 HeapObjectMatcher<Object> function_const(node->InputAt(0));
446 if (!function_const.HasValue()) return false; // not a constant.
447 Handle<Object> func = function_const.Value().handle();
448 if (!func->IsJSFunction()) return false; // not a function.
449 Handle<JSFunction> function = Handle<JSFunction>::cast(func);
450 if (arg_count != function->shared()->internal_formal_parameter_count()) {
454 // Check the receiver doesn't need to be wrapped.
455 Node* receiver = node->InputAt(1);
456 if (!NodeProperties::IsTyped(receiver)) return false;
457 Type* ok_receiver = Type::Union(Type::Undefined(), Type::Receiver(), zone());
458 if (!NodeProperties::GetBounds(receiver).upper->Is(ok_receiver)) return false;
460 int index = NodeProperties::FirstContextIndex(node);
462 // TODO(titzer): total hack to share function context constants.
463 // Remove this when the JSGraph canonicalizes heap constants.
464 Node* context = node->InputAt(index);
465 HeapObjectMatcher<Context> context_const(context);
466 if (!context_const.HasValue() ||
467 *(context_const.Value().handle()) != function->context()) {
468 context = jsgraph()->HeapConstant(Handle<Context>(function->context()));
470 node->ReplaceInput(index, context);
471 CallDescriptor* desc = Linkage::GetJSCallDescriptor(
472 zone(), false, 1 + arg_count, FlagsForNode(node));
473 node->set_op(common()->Call(desc));
478 void JSGenericLowering::LowerJSCallFunction(Node* node) {
479 // Fast case: call function directly.
480 if (TryLowerDirectJSCall(node)) return;
482 // General case: CallFunctionStub.
483 const CallFunctionParameters& p = CallFunctionParametersOf(node->op());
484 int arg_count = static_cast<int>(p.arity() - 2);
485 CallFunctionStub stub(isolate(), arg_count, p.flags());
486 CallInterfaceDescriptor d = stub.GetCallInterfaceDescriptor();
487 CallDescriptor* desc = Linkage::GetStubCallDescriptor(
488 isolate(), zone(), d, static_cast<int>(p.arity() - 1),
490 Node* stub_code = jsgraph()->HeapConstant(stub.GetCode());
491 node->InsertInput(zone(), 0, stub_code);
492 node->set_op(common()->Call(desc));
496 void JSGenericLowering::LowerJSCallRuntime(Node* node) {
497 const CallRuntimeParameters& p = CallRuntimeParametersOf(node->op());
498 ReplaceWithRuntimeCall(node, p.id(), static_cast<int>(p.arity()));
502 void JSGenericLowering::LowerJSStackCheck(Node* node) {
503 Node* effect = NodeProperties::GetEffectInput(node);
504 Node* control = NodeProperties::GetControlInput(node);
506 Node* limit = graph()->NewNode(
507 machine()->Load(kMachPtr),
508 jsgraph()->ExternalConstant(
509 ExternalReference::address_of_stack_limit(isolate())),
510 jsgraph()->IntPtrConstant(0), effect, control);
511 Node* pointer = graph()->NewNode(machine()->LoadStackPointer());
513 Node* check = graph()->NewNode(machine()->UintLessThan(), limit, pointer);
515 graph()->NewNode(common()->Branch(BranchHint::kTrue), check, control);
517 Node* if_true = graph()->NewNode(common()->IfTrue(), branch);
518 Node* etrue = effect;
520 Node* if_false = graph()->NewNode(common()->IfFalse(), branch);
521 NodeProperties::ReplaceControlInput(node, if_false);
524 Node* merge = graph()->NewNode(common()->Merge(2), if_true, if_false);
525 Node* ephi = graph()->NewNode(common()->EffectPhi(2), etrue, efalse, merge);
527 // Relax controls of {node}, i.e. make it free floating.
528 NodeProperties::ReplaceWithValue(node, node, ephi, merge);
529 NodeProperties::ReplaceEffectInput(ephi, efalse, 1);
531 // Turn the stack check into a runtime call.
532 ReplaceWithRuntimeCall(node, Runtime::kStackGuard);
536 Zone* JSGenericLowering::zone() const { return graph()->zone(); }
539 Isolate* JSGenericLowering::isolate() const { return jsgraph()->isolate(); }
542 Graph* JSGenericLowering::graph() const { return jsgraph()->graph(); }
545 CommonOperatorBuilder* JSGenericLowering::common() const {
546 return jsgraph()->common();
550 MachineOperatorBuilder* JSGenericLowering::machine() const {
551 return jsgraph()->machine();
554 } // namespace compiler
555 } // namespace internal