7#if defined(TARGET_ARCH_X64)
32#define __ compiler->assembler()->
33#define Z (compiler->zone())
41 const Instruction* instr,
42 LocationSummary* locs) {
74LocationSummary* LoadIndexedUnsafeInstr::MakeLocationSummary(Zone* zone,
76 const intptr_t kNumInputs = 1;
77 const intptr_t kNumTemps = 0;
78 LocationSummary* locs =
new (zone)
97void LoadIndexedUnsafeInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
103#if defined(DART_COMPRESSED_POINTERS)
114 case kUnboxedInt64: {
115 const auto out = locs()->out(0).reg();
119 case kUnboxedDouble: {
120 const auto out = locs()->out(0).fpu_reg();
132 ASSERT(instr->RequiredInputRepresentation(
134#if defined(DART_COMPRESSED_POINTERS)
140 __ movsxd(index, index);
142 __ movq(compiler::Address(instr->base_reg(), index,
TIMES_4, instr->offset()),
149DEFINE_BACKEND(TailCall, (NoLocation, Fixed<Register, ARGS_DESC_REG>)) {
150 compiler->EmitTailCallToStub(instr->code());
156 __ set_constant_pool_allowed(
true);
159LocationSummary* MemoryCopyInstr::MakeLocationSummary(Zone* zone,
167 const intptr_t kNumInputs = 5;
168 const intptr_t kNumTemps = 2;
169 LocationSummary* locs =
new (zone)
173 const bool needs_writable_inputs =
174 (((element_size_ == 1) && !unboxed_inputs_) ||
175 ((element_size_ == 16) && unboxed_inputs_));
177 needs_writable_inputs
181 needs_writable_inputs
184 if (
length()->BindsToSmiConstant() &&
length()->BoundSmiConstant() <= 4) {
188 length()->definition()->OriginalDefinition()->AsConstant()));
198static inline intptr_t SizeOfMemoryCopyElements(intptr_t
element_size) {
199 return Utils::Minimum<intptr_t>(
element_size, compiler::target::kWordSize);
204 compiler::Label*
done) {
205 const intptr_t mov_size = SizeOfMemoryCopyElements(element_size_);
214 __ SmiUntag(length_reg);
215 }
else if (shift > 0) {
216 __ OBJ(shl)(length_reg, compiler::Immediate(shift));
218 __ ExtendNonNegativeSmi(length_reg);
226 compiler::Label*
done,
227 compiler::Label* copy_forwards) {
228 const intptr_t mov_size = SizeOfMemoryCopyElements(element_size_);
229 const bool reversed = copy_forwards !=
nullptr;
233 __ BranchIfZero(length_reg,
done);
237 __ leaq(
TMP, compiler::Address(src_reg, length_reg,
scale, -mov_size));
238 __ CompareRegisters(dest_reg,
TMP);
239#if defined(USING_MEMORY_SANITIZER)
247 __ movq(src_reg,
TMP);
249 compiler::Address(dest_reg, length_reg,
scale, -mov_size));
252#if defined(USING_MEMORY_SANITIZER)
255 __ movq(
TMP, length_reg);
258 __ MulImmediate(
TMP, mov_size);
261 __ MsanUnpoison(dest_reg,
TMP);
284#if defined(USING_MEMORY_SANITIZER)
286 __ MsanUnpoison(dest_reg,
TMP);
291void MemoryCopyInstr::EmitComputeStartPointer(FlowGraphCompiler*
compiler,
298 if (array_rep != kTagged) {
309 case kOneByteStringCid:
313 case kTwoByteStringCid:
322 ASSERT(start_loc.IsRegister() || start_loc.IsConstant());
323 if (start_loc.IsConstant()) {
324 const auto& constant = start_loc.constant();
325 ASSERT(constant.IsInteger());
326 const int64_t start_value = Integer::Cast(constant).AsInt64Value();
328 Utils::MulWithWrapAround<intptr_t>(start_value, element_size_),
offset);
329 __ leaq(payload_reg, compiler::Address(array_reg, add_value));
333 const Register start_reg = start_loc.reg();
334 bool index_unboxed = unboxed_inputs_;
337 if (element_size_ == 1 && !index_unboxed) {
339 __ SmiUntag(start_reg);
340 index_unboxed =
true;
341 }
else if (element_size_ == 16 && index_unboxed) {
344 __ SmiTag(start_reg);
345 index_unboxed =
false;
346 }
else if (!index_unboxed) {
347 __ ExtendNonNegativeSmi(start_reg);
350 __ leaq(payload_reg, compiler::Address(array_reg, start_reg,
scale,
offset));
353LocationSummary* CalculateElementAddressInstr::MakeLocationSummary(
356 const intptr_t kNumInputs = 3;
357 const intptr_t kNumTemps = 0;
358 auto*
const summary =
new (zone)
376void CalculateElementAddressInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
380 const Register result_reg = locs()->out(0).reg();
382 if (index_loc.IsConstant()) {
383 if (offset_loc.IsConstant()) {
384 ASSERT_EQUAL(Smi::Cast(index_loc.constant()).Value(), 0);
385 ASSERT(Smi::Cast(offset_loc.constant()).Value() != 0);
387 const int32_t offset_value = Smi::Cast(offset_loc.constant()).Value();
388 __ leaq(result_reg, compiler::Address(base_reg, offset_value));
392 const int32_t scaled_index =
393 Smi::Cast(index_loc.constant()).Value() *
index_scale();
394 __ leaq(result_reg, compiler::Address(base_reg, offset_loc.reg(),
TIMES_1,
398 Register index_reg = index_loc.reg();
407 __ MoveAndSmiTagRegister(result_reg, index_reg);
408 index_reg = result_reg;
409 index_unboxed =
false;
412 if (offset_loc.IsConstant()) {
413 const int32_t offset_value = Smi::Cast(offset_loc.constant()).Value();
415 compiler::Address(base_reg, index_reg,
scale, offset_value));
419 __ leaq(result_reg, compiler::Address(base_reg, index_reg,
scale,
421 __ AddRegisters(result_reg, offset_loc.reg());
426LocationSummary* MoveArgumentInstr::MakeLocationSummary(Zone* zone,
428 const intptr_t kNumInputs = 1;
429 const intptr_t kNumTemps = 0;
430 LocationSummary* locs =
new (zone)
442void MoveArgumentInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
447 if (
value.IsRegister()) {
449 }
else if (
value.IsConstant()) {
450 __ StoreObject(dst,
value.constant());
451 }
else if (
value.IsFpuRegister()) {
452 __ movsd(dst,
value.fpu_reg());
461 const intptr_t kNumInputs = 1;
462 const intptr_t kNumTemps = 0;
463 LocationSummary*
locs =
new (zone)
493 if (
locs()->in(0).IsRegister()) {
496 }
else if (
locs()->in(0).IsPairLocation()) {
507 if (
compiler->parsed_function().function().IsAsyncFunction() ||
508 compiler->parsed_function().function().IsAsyncGenerator()) {
510 const Code& stub = GetReturnStub(
compiler);
515 if (!
compiler->flow_graph().graph_entry()->NeedsFrame()) {
521 __ Comment(
"Stack Check");
522 compiler::Label
done;
523 const intptr_t fp_sp_dist =
524 (compiler::target::frame_layout.first_local_from_fp + 1 -
530 __ CompareImmediate(
RDI, compiler::Immediate(fp_sp_dist));
540 __ set_constant_pool_allowed(
true);
543static const RegisterSet kCalleeSaveRegistersSet(
561 __ popq(old_exit_frame_reg);
563 __ popq(old_exit_through_ffi_reg);
568 compiler::Address(
THR, compiler::target::Thread::top_resource_offset()),
574 __ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg,
575 old_exit_through_ffi_reg,
579 __ PopRegisters(kCalleeSaveRegistersSet);
581#if defined(DART_TARGET_OS_FUCHSIA) && defined(USING_SHADOW_CALL_STACK)
594 __ set_constant_pool_allowed(
true);
598static bool IsPowerOfTwoKind(intptr_t v1, intptr_t
v2) {
622 BranchLabels labels = {
nullptr,
nullptr,
nullptr};
626 const bool is_power_of_two_kind = IsPowerOfTwoKind(if_true_, if_false_);
628 intptr_t true_value = if_true_;
629 intptr_t false_value = if_false_;
631 if (is_power_of_two_kind) {
632 if (true_value == 0) {
637 if (true_value == 0) {
639 intptr_t temp = true_value;
640 true_value = false_value;
647 __ setcc(true_condition,
DL);
649 if (is_power_of_two_kind) {
650 const intptr_t shift =
657 if (false_value != 0) {
663LocationSummary* LoadLocalInstr::MakeLocationSummary(Zone* zone,
665 const intptr_t kNumInputs = 0;
666 const intptr_t stack_index =
667 compiler::target::frame_layout.FrameSlotForVariable(&
local());
673void LoadLocalInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
678LocationSummary* StoreLocalInstr::MakeLocationSummary(Zone* zone,
680 const intptr_t kNumInputs = 1;
685void StoreLocalInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
689 __ movq(compiler::Address(
690 RBP, compiler::target::FrameOffsetInBytesForVariable(&
local())),
694LocationSummary* ConstantInstr::MakeLocationSummary(Zone* zone,
696 const intptr_t kNumInputs = 0;
701 LocationSummary::kNoCall);
704void ConstantInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
708 if (
out.IsRegister()) {
717 intptr_t pair_index) {
719 if (destination.IsRegister()) {
721 const int64_t
value = Integer::Cast(value_).AsInt64Value();
723 __ xorl(destination.reg(), destination.reg());
725 __ movq(destination.reg(), compiler::Immediate(
value));
728 ASSERT(representation() == kTagged);
729 __ LoadObject(destination.reg(), value_);
731 }
else if (destination.IsFpuRegister()) {
732 switch (representation()) {
734 __ LoadSImmediate(destination.fpu_reg(), Double::Cast(value_).value());
737 __ LoadDImmediate(destination.fpu_reg(), Double::Cast(value_).value());
739 case kUnboxedFloat64x2:
740 __ LoadQImmediate(destination.fpu_reg(),
741 Float64x2::Cast(value_).value());
743 case kUnboxedFloat32x4:
744 __ LoadQImmediate(destination.fpu_reg(),
745 Float32x4::Cast(value_).value());
747 case kUnboxedInt32x4:
748 __ LoadQImmediate(destination.fpu_reg(), Int32x4::Cast(value_).value());
753 }
else if (destination.IsDoubleStackSlot()) {
754 ASSERT(representation() == kUnboxedDouble);
757 }
else if (destination.IsQuadStackSlot()) {
758 switch (representation()) {
759 case kUnboxedFloat64x2:
762 case kUnboxedFloat32x4:
765 case kUnboxedInt32x4:
773 ASSERT(destination.IsStackSlot());
775 const int64_t
value = Integer::Cast(value_).AsInt64Value();
777 compiler::Immediate(
value));
778 }
else if (representation() == kUnboxedFloat) {
780 bit_cast<int32_t, float>(Double::Cast(value_).
value());
784 ASSERT(representation() == kTagged);
790LocationSummary* UnboxedConstantInstr::MakeLocationSummary(Zone* zone,
792 const bool is_unboxed_int =
795 compiler::target::kWordSize);
796 const intptr_t kNumInputs = 0;
797 const intptr_t kNumTemps = is_unboxed_int ? 0 : 1;
798 LocationSummary* locs =
new (zone)
800 if (is_unboxed_int) {
816void UnboxedConstantInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
818 if (!locs()->
out(0).IsInvalid()) {
822 : locs()->temp(0).reg();
827LocationSummary* AssertAssignableInstr::MakeLocationSummary(Zone* zone,
829 auto const dst_type_loc =
839 const intptr_t kNonChangeableInputRegs =
841 ((dst_type_loc.IsRegister() ? 1 : 0) << TypeTestABI::kDstTypeReg) |
842 (1 << TypeTestABI::kInstantiatorTypeArgumentsReg) |
843 (1 << TypeTestABI::kFunctionTypeArgumentsReg);
851 const intptr_t kCpuRegistersToPreserve =
853 const intptr_t kFpuRegistersToPreserve =
859 LocationSummary* summary =
new (zone) LocationSummary(
872 intptr_t next_temp = 0;
874 const bool should_preserve = ((1 << i) & kCpuRegistersToPreserve) != 0;
875 if (should_preserve) {
876 summary->set_temp(next_temp++,
882 const bool should_preserve = ((1 << i) & kFpuRegistersToPreserve) != 0;
883 if (should_preserve) {
892void AssertBooleanInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
893 ASSERT(locs()->always_calls());
895 auto object_store =
compiler->isolate_group()->object_store();
896 const auto& assert_boolean_stub =
899 compiler::Label
done;
905 UntaggedPcDescriptors::kOther, locs(),
930LocationSummary* EqualityCompareInstr::MakeLocationSummary(Zone* zone,
932 const intptr_t kNumInputs = 2;
933 if (operation_cid() == kDoubleCid) {
934 const intptr_t kNumTemps = 0;
935 LocationSummary* locs =
new (zone)
942 if (operation_cid() == kSmiCid || operation_cid() == kMintCid ||
943 operation_cid() == kIntegerCid) {
944 const intptr_t kNumTemps = 0;
945 LocationSummary* locs =
new (zone)
955 locs->set_in(1, locs->in(0).IsConstant()
966static void LoadValueCid(FlowGraphCompiler*
compiler,
969 compiler::Label* value_is_smi =
nullptr) {
970 compiler::Label
done;
971 if (value_is_smi ==
nullptr) {
972 __ LoadImmediate(value_cid_reg, compiler::Immediate(kSmiCid));
975 if (value_is_smi ==
nullptr) {
980 __ LoadClassId(value_cid_reg, value_reg);
1012static void EmitBranchOnCondition(
1015 BranchLabels labels,
1018 if (labels.fall_through == labels.false_label) {
1020 __ j(true_condition, labels.true_label, jump_distance);
1024 __ j(false_condition, labels.false_label, jump_distance);
1027 if (labels.fall_through != labels.true_label) {
1028 __ jmp(labels.true_label, jump_distance);
1034 const LocationSummary& locs,
1040 Condition true_condition = TokenKindToIntCondition(kind);
1041 if (
left.IsConstant() ||
right.IsConstant()) {
1043 ConstantInstr* constant =
nullptr;
1044 if (
left.IsConstant()) {
1045 constant =
left.constant_instruction();
1049 true_condition = FlipCondition(true_condition);
1051 constant =
right.constant_instruction();
1058 __ OBJ(cmp)(
left.reg(), compiler::Immediate(value));
1060 ASSERT(constant->representation() == kTagged);
1063 }
else if (
right.IsStackSlot()) {
1068 return true_condition;
1072 const LocationSummary& locs,
1078 Condition true_condition = TokenKindToIntCondition(kind);
1079 if (
left.IsConstant() ||
right.IsConstant()) {
1081 ConstantInstr* constant =
nullptr;
1082 if (
left.IsConstant()) {
1083 constant =
left.constant_instruction();
1087 true_condition = FlipCondition(true_condition);
1089 constant =
right.constant_instruction();
1096 __ cmpq(
left.reg(), compiler::Immediate(value));
1100 }
else if (
right.IsStackSlot()) {
1105 return true_condition;
1109 const LocationSummary& locs,
1111 BranchLabels labels) {
1112 ASSERT((kind == Token::kEQ) || (kind == Token::kNE));
1115 const Condition true_condition = TokenKindToIntCondition(kind);
1116 compiler::Label* equal_result =
1117 (true_condition ==
EQUAL) ? labels.true_label : labels.false_label;
1118 compiler::Label* not_equal_result =
1119 (true_condition ==
EQUAL) ? labels.false_label : labels.true_label;
1127 __ BranchIfSmi(
TMP, not_equal_result);
1128 __ CompareClassId(
left, kMintCid);
1130 __ CompareClassId(
right, kMintCid);
1134 return true_condition;
1158 const LocationSummary& locs,
1160 BranchLabels labels) {
1166 Condition true_condition = TokenKindToDoubleCondition(kind);
1167 compiler::Label* nan_result =
1168 (true_condition ==
NOT_EQUAL) ? labels.true_label : labels.false_label;
1170 return true_condition;
1174 BranchLabels labels) {
1176 ASSERT(operation_cid() == kMintCid);
1177 return EmitNullAwareInt64ComparisonOp(
compiler, *locs(), kind(), labels);
1179 if (operation_cid() == kSmiCid) {
1180 return EmitSmiComparisonOp(
compiler, *locs(), kind());
1181 }
else if (operation_cid() == kMintCid || operation_cid() == kIntegerCid) {
1182 return EmitInt64ComparisonOp(
compiler, *locs(), kind());
1184 ASSERT(operation_cid() == kDoubleCid);
1185 return EmitDoubleComparisonOp(
compiler, *locs(), kind(), labels);
1190 compiler::Label is_true, is_false;
1191 BranchLabels labels = {&is_true, &is_false, &is_false};
1196 EmitBranchOnCondition(
compiler, true_condition, labels,
1202 compiler::Label
done;
1212 BranchInstr* branch) {
1213 BranchLabels labels =
compiler->CreateBranchLabels(branch);
1216 EmitBranchOnCondition(
compiler, true_condition, labels);
1220LocationSummary* TestSmiInstr::MakeLocationSummary(Zone* zone,
bool opt)
const {
1221 const intptr_t kNumInputs = 2;
1222 const intptr_t kNumTemps = 0;
1223 LocationSummary* locs =
new (zone)
1233 BranchLabels labels) {
1234 Register left_reg = locs()->in(0).reg();
1236 if (
right.IsConstant()) {
1239 __ TestImmediate(left_reg, compiler::Immediate(imm),
1245 return true_condition;
1248LocationSummary* TestCidsInstr::MakeLocationSummary(Zone* zone,
1250 const intptr_t kNumInputs = 1;
1251 const intptr_t kNumTemps = 1;
1252 LocationSummary* locs =
new (zone)
1261 BranchLabels labels) {
1262 ASSERT((kind() == Token::kIS) || (kind() == Token::kISNOT));
1263 Register val_reg = locs()->in(0).reg();
1264 Register cid_reg = locs()->temp(0).reg();
1266 compiler::Label* deopt =
1268 ?
compiler->AddDeoptStub(deopt_id(), ICData::kDeoptTestCids)
1271 const intptr_t true_result = (kind() == Token::kIS) ? 1 : 0;
1276 __ j(
ZERO,
result ? labels.true_label : labels.false_label);
1277 __ LoadClassId(cid_reg, val_reg);
1278 for (intptr_t i = 2; i <
data.length(); i += 2) {
1279 const intptr_t test_cid =
data[i];
1280 ASSERT(test_cid != kSmiCid);
1282 __ cmpq(cid_reg, compiler::Immediate(test_cid));
1283 __ j(
EQUAL,
result ? labels.true_label : labels.false_label);
1286 if (deopt ==
nullptr) {
1290 compiler::Label*
target =
result ? labels.false_label : labels.true_label;
1291 if (
target != labels.fall_through) {
1302LocationSummary* RelationalOpInstr::MakeLocationSummary(Zone* zone,
1304 const intptr_t kNumInputs = 2;
1305 const intptr_t kNumTemps = 0;
1306 if (operation_cid() == kDoubleCid) {
1307 LocationSummary* summary =
new (zone)
1314 if (operation_cid() == kSmiCid || operation_cid() == kMintCid) {
1315 LocationSummary* summary =
new (zone)
1320 summary->set_in(1, summary->in(0).IsConstant()
1331 BranchLabels labels) {
1332 if (operation_cid() == kSmiCid) {
1333 return EmitSmiComparisonOp(
compiler, *locs(), kind());
1334 }
else if (operation_cid() == kMintCid) {
1335 return EmitInt64ComparisonOp(
compiler, *locs(), kind());
1337 ASSERT(operation_cid() == kDoubleCid);
1338 return EmitDoubleComparisonOp(
compiler, *locs(), kind(), labels);
1351 __ LoadImmediate(
R10, compiler::Immediate(argc_tag));
1354 stub = &StubCode::CallBootstrapNative();
1356 __ LoadNativeEntry(
RBX, &label,
1359 source(), *stub, UntaggedPcDescriptors::kOther,
locs(),
1363 stub = &StubCode::CallBootstrapNative();
1365 stub = &StubCode::CallAutoScopeNative();
1367 stub = &StubCode::CallNoScopeNative();
1369 const compiler::ExternalLabel label(
1371 __ LoadNativeEntry(
RBX, &label,
1375 compiler->GenerateNonLazyDeoptableStubCall(
1376 source(), *stub, UntaggedPcDescriptors::kOther,
locs(),
1383#define R(r) (1 << r)
1386 bool is_optimizing)
const {
1389 return MakeLocationSummaryInternal(
1390 zone, is_optimizing,
1421 __ pushq(compiler::Immediate(0));
1425 __ LoadObject(
CODE_REG, Code::null_object());
1426 __ set_constant_pool_allowed(
false);
1427 __ EnterDartFrame(0,
PP);
1431 intptr_t stack_space = marshaller_.RequiredStackSpaceInBytes();
1432 __ ReserveAlignedFrameSpace(stack_space);
1433#if defined(USING_MEMORY_SANITIZER)
1438 __ PushRegisters(kVolatileRegisterSet);
1446 __ MsanUnpoison(temp,
RAX);
1450 __ MsanUnpoison(is_leaf_ ?
FPREG : saved_fp,
1455 __ CallCFunction(compiler::Address(
1456 THR, kMsanUnpoisonParamRuntimeEntry.OffsetFromThread()));
1458 __ PopRegisters(kVolatileRegisterSet);
1469 __ Comment(is_leaf_ ?
"Leaf Call" :
"Call");
1473#if !defined(PRODUCT)
1476 __ movq(compiler::Address(
1477 THR, compiler::target::Thread::top_exit_frame_info_offset()),
1482 if (marshaller_.contains_varargs() &&
1488 __ CallCFunction(target_address,
true);
1490#if !defined(PRODUCT)
1492 compiler::Immediate(compiler::target::Thread::vm_tag_dart_id()));
1493 __ movq(compiler::Address(
1494 THR, compiler::target::Thread::top_exit_frame_info_offset()),
1495 compiler::Immediate(0));
1504 UntaggedPcDescriptors::Kind::kOther,
locs(),
1511 __ movq(temp, compiler::Immediate(
1512 compiler::target::Thread::exit_through_ffi()));
1514 __ TransitionGeneratedToNative(target_address,
FPREG, temp,
1517 if (marshaller_.contains_varargs() &&
1521 __ CallCFunction(target_address,
true);
1524 __ TransitionNativeToGenerated(
true);
1532 THR, compiler::target::Thread::
1533 call_native_through_safepoint_entry_point_offset()));
1536 __ movq(
RBX, target_address);
1537 if (marshaller_.contains_varargs() &&
1545 __ Comment(
"Check Dart_Handle for Error.");
1546 compiler::Label not_error;
1549 compiler::target::LocalHandle::ptr_offset()));
1550 __ BranchIfSmi(temp, ¬_error);
1551 __ LoadClassId(temp, temp);
1556 __ Comment(
"Slow path: call Dart_PropagateError through stub.");
1559 THR, compiler::target::Thread::
1560 call_native_through_safepoint_entry_point_offset()));
1561 __ movq(
RBX, compiler::Address(
1562 THR, kPropagateErrorRuntimeEntry.OffsetFromThread()));
1570 __ Bind(¬_error);
1581 __ LeaveDartFrame();
1584 if (FLAG_precompiled_mode) {
1587 __ set_constant_pool_allowed(
true);
1595void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
1602#if defined(DART_TARGET_OS_FUCHSIA) && defined(USING_SHADOW_CALL_STACK)
1612 __ PushImmediate(compiler::Immediate(0));
1616 __ PushImmediate(compiler::Immediate(0));
1620 __ PushImmediate(compiler::Immediate(0));
1623 __ PushRegisters(kCalleeSaveRegistersSet);
1631 compiler::Address(
THR, compiler::target::Thread::top_resource_offset()));
1633 compiler::Address(
THR, compiler::target::Thread::top_resource_offset()),
1634 compiler::Immediate(0));
1636 __ pushq(compiler::Address(
1637 THR, compiler::target::Thread::exit_through_ffi_offset()));
1640 __ pushq(compiler::Address(
1641 THR, compiler::target::Thread::top_exit_frame_info_offset()));
1645 __ EmitEntryFrameVerification();
1648 __ TransitionNativeToGenerated(
false);
1651 const Function& target_function = marshaller_.dart_signature();
1652 const intptr_t callback_id = target_function.FfiCallbackId();
1653 __ movq(
RAX, compiler::Address(
1654 THR, compiler::target::Thread::isolate_group_offset()));
1655 __ movq(
RAX, compiler::Address(
1656 RAX, compiler::target::IsolateGroup::object_store_offset()));
1659 RAX, compiler::target::ObjectStore::ffi_callback_code_offset()));
1661 RAX, compiler::FieldAddress(
1662 RAX, compiler::target::GrowableObjectArray::data_offset()));
1665 compiler::FieldAddress(
1666 RAX, compiler::target::Array::data_offset() +
1667 callback_id * compiler::target::kCompressedWordSize));
1670 __ movq(compiler::Address(
FPREG,
1674 if (FLAG_precompiled_mode) {
1677 THR, compiler::target::Thread::global_object_pool_offset()));
1689 THR, compiler::target::Thread::invoke_dart_code_stub_offset()));
1690 __ pushq(compiler::FieldAddress(
1691 RAX, compiler::target::Code::entry_point_offset()));
1694 FunctionEntryInstr::EmitNativeCode(
compiler);
1697#define R(r) (1 << r)
1701 bool is_optimizing)
const {
1714 __ MoveRegister(saved_fp,
FPREG);
1716 const intptr_t frame_space = native_calling_convention_.
StackTopInBytes();
1717 __ EnterCFrame(frame_space);
1722 __ CallCFunction(target_address);
1724 compiler::Immediate(VMTag::kDartTagId));
1729LocationSummary* OneByteStringFromCharCodeInstr::MakeLocationSummary(
1732 const intptr_t kNumInputs = 1;
1738void OneByteStringFromCharCodeInstr::EmitNativeCode(
1749 compiler::Address(
THR, Thread::predefined_symbols_address_offset()));
1752 TIMES_HALF_WORD_SIZE,
1756LocationSummary* StringToCharCodeInstr::MakeLocationSummary(Zone* zone,
1758 const intptr_t kNumInputs = 1;
1763void StringToCharCodeInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
1764 ASSERT(cid_ == kOneByteStringCid);
1767 compiler::Label is_one,
done;
1780LocationSummary* Utf8ScanInstr::MakeLocationSummary(Zone* zone,
1782 const intptr_t kNumInputs = 5;
1783 const intptr_t kNumTemps = 1;
1784 LocationSummary* summary =
new (zone)
1796void Utf8ScanInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
1797 const Register bytes_reg = locs()->in(1).reg();
1798 const Register start_reg = locs()->in(2).reg();
1799 const Register end_reg = locs()->in(3).reg();
1800 const Register table_reg = locs()->in(4).reg();
1801 const Register size_reg = locs()->out(0).reg();
1803 const Register bytes_ptr_reg = start_reg;
1804 const Register bytes_end_reg = end_reg;
1805 const Register bytes_end_minus_16_reg = bytes_reg;
1806 const Register flags_reg = locs()->temp(0).reg();
1810 const intptr_t kSizeMask = 0x03;
1811 const intptr_t kFlagsMask = 0x3C;
1813 compiler::Label scan_ascii, ascii_loop, ascii_loop_in, nonascii_loop;
1814 compiler::Label rest, rest_loop, rest_loop_in,
done;
1817 __ LoadFromSlot(bytes_reg, bytes_reg, Slot::PointerBase_data());
1820 __ leaq(bytes_ptr_reg, compiler::Address(bytes_reg, start_reg,
TIMES_1, 0));
1821 __ leaq(bytes_end_reg, compiler::Address(bytes_reg, end_reg,
TIMES_1, 0));
1822 __ leaq(bytes_end_minus_16_reg, compiler::Address(bytes_end_reg, -16));
1825 __ xorq(size_reg, size_reg);
1826 __ xorq(flags_reg, flags_reg);
1835 __ Bind(&ascii_loop);
1836 __ addq(bytes_ptr_reg, compiler::Immediate(16));
1837 __ Bind(&ascii_loop_in);
1840 __ cmpq(bytes_ptr_reg, bytes_end_minus_16_reg);
1847 __ movups(vector_reg, compiler::Address(bytes_ptr_reg, 0));
1848 __ pmovmskb(temp_reg, vector_reg);
1849 __ bsfq(temp_reg, temp_reg);
1853 __ addq(bytes_ptr_reg, temp_reg);
1854 __ addq(size_reg, bytes_ptr_reg);
1857 __ movzxb(temp_reg, compiler::Address(bytes_ptr_reg, 0));
1860 __ Bind(&nonascii_loop);
1861 __ addq(bytes_ptr_reg, compiler::Immediate(1));
1864 __ movzxb(temp_reg, compiler::FieldAddress(
1866 compiler::target::OneByteString::data_offset()));
1867 __ orq(flags_reg, temp_reg);
1868 __ andq(temp_reg, compiler::Immediate(kSizeMask));
1869 __ addq(size_reg, temp_reg);
1872 __ cmpq(bytes_ptr_reg, bytes_end_reg);
1876 __ movzxb(temp_reg, compiler::Address(bytes_ptr_reg, 0));
1877 __ testq(temp_reg, compiler::Immediate(0x80));
1881 __ Bind(&scan_ascii);
1882 __ subq(size_reg, bytes_ptr_reg);
1883 __ jmp(&ascii_loop_in);
1889 __ addq(size_reg, bytes_ptr_reg);
1892 __ Bind(&rest_loop);
1895 __ movzxb(temp_reg, compiler::Address(bytes_ptr_reg, 0));
1896 __ addq(bytes_ptr_reg, compiler::Immediate(1));
1899 __ movzxb(temp_reg, compiler::FieldAddress(
1901 compiler::target::OneByteString::data_offset()));
1902 __ orq(flags_reg, temp_reg);
1903 __ andq(temp_reg, compiler::Immediate(kSizeMask));
1904 __ addq(size_reg, temp_reg);
1907 __ Bind(&rest_loop_in);
1908 __ cmpq(bytes_ptr_reg, bytes_end_reg);
1913 __ andq(flags_reg, compiler::Immediate(kFlagsMask));
1915 __ SmiTag(flags_reg);
1918 const Location decoder_location = locs()->in(0);
1919 if (decoder_location.IsStackSlot()) {
1921 decoder_reg = temp_reg;
1923 decoder_reg = decoder_location.reg();
1925 const auto scan_flags_field_offset = scan_flags_field_.offset_in_bytes();
1927 __ OBJ(or)(compiler::FieldAddress(decoder_reg, scan_flags_field_offset),
1930 __ orq(compiler::FieldAddress(decoder_reg, scan_flags_field_offset),
1935LocationSummary* LoadIndexedInstr::MakeLocationSummary(Zone* zone,
1942 const intptr_t kNumInputs = 2;
1943 const intptr_t kNumTemps = 0;
1944 LocationSummary* locs =
new (zone)
1950 const bool need_writable_index_register =
1953 const bool can_be_constant =
1961 : (need_writable_index_register ?
Location::WritableRegister()
1975void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
1980 bool index_unboxed = index_unboxed_;
1981 if (
index.IsRegister()) {
1982 if (index_scale_ == 1 && !index_unboxed) {
1984 index_unboxed =
true;
1985 }
else if (index_scale_ == 16 && index_unboxed) {
1988 index_unboxed =
false;
1989 }
else if (!index_unboxed) {
1993 __ ExtendNonNegativeSmi(
index.reg());
1999 compiler::Address element_address =
2003 :
compiler::Assembler::ElementAddressForIntIndex(
2015 if (rep == kUnboxedFloat) {
2018 }
else if (rep == kUnboxedDouble) {
2021 ASSERT(rep == kUnboxedInt32x4 || rep == kUnboxedFloat32x4 ||
2022 rep == kUnboxedFloat64x2);
2023 __ movups(
result, element_address);
2030 __ LoadCompressed(
result, element_address);
2034LocationSummary* LoadCodeUnitsInstr::MakeLocationSummary(Zone* zone,
2036 const intptr_t kNumInputs = 2;
2037 const intptr_t kNumTemps = 0;
2038 LocationSummary* summary =
new (zone)
2049void LoadCodeUnitsInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2051 const Register str = locs()->in(0).reg();
2054 bool index_unboxed =
false;
2057 index_unboxed =
true;
2059 __ ExtendNonNegativeSmi(
index);
2061 compiler::Address element_address =
2067 case kOneByteStringCid:
2070 __ movzxb(
result, element_address);
2073 __ movzxw(
result, element_address);
2084 case kTwoByteStringCid:
2087 __ movzxw(
result, element_address);
2104LocationSummary* StoreIndexedInstr::MakeLocationSummary(Zone* zone,
2111 const intptr_t kNumInputs = 3;
2112 const intptr_t kNumTemps =
2114 LocationSummary* locs =
new (zone)
2120 const bool need_writable_index_register =
2123 const bool can_be_constant =
2130 : (need_writable_index_register ?
Location::WritableRegister()
2135 if (rep == kUnboxedUint8 || rep == kUnboxedInt8) {
2145 }
else if (
class_id() == kArrayCid) {
2159void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2164 bool index_unboxed = index_unboxed_;
2165 if (
index.IsRegister()) {
2166 if (index_scale_ == 1 && !index_unboxed) {
2168 index_unboxed =
true;
2169 }
else if (index_scale_ == 16 && index_unboxed) {
2172 index_unboxed =
false;
2173 }
else if (!index_unboxed) {
2177 __ ExtendNonNegativeSmi(
index.reg());
2183 compiler::Address element_address =
2187 :
compiler::Assembler::ElementAddressForIntIndex(
2195 ASSERT(rep == kUnboxedUint8);
2197 const Smi& constant = Smi::Cast(locs()->in(2).constant());
2202 }
else if (
value < 0) {
2205 __ movb(element_address, compiler::Immediate(
static_cast<int8_t
>(
value)));
2207 const Register storedValueReg = locs()->in(2).reg();
2208 compiler::Label store_value, store_0xff;
2209 __ CompareImmediate(storedValueReg, compiler::Immediate(0xFF));
2213 __ xorq(storedValueReg, storedValueReg);
2215 __ Bind(&store_0xff);
2216 __ LoadImmediate(storedValueReg, compiler::Immediate(0xFF));
2217 __ Bind(&store_value);
2221 if (rep == kUnboxedUint8 || rep == kUnboxedInt8) {
2223 const Smi& constant = Smi::Cast(locs()->in(2).constant());
2224 __ movb(element_address,
2225 compiler::Immediate(
static_cast<int8_t
>(constant.Value())));
2234 if (rep == kUnboxedFloat) {
2235 __ movss(element_address, locs()->in(2).fpu_reg());
2236 }
else if (rep == kUnboxedDouble) {
2237 __ movsd(element_address, locs()->in(2).fpu_reg());
2239 ASSERT(rep == kUnboxedInt32x4 || rep == kUnboxedFloat32x4 ||
2240 rep == kUnboxedFloat64x2);
2241 __ movups(element_address, locs()->in(2).fpu_reg());
2243 }
else if (
class_id() == kArrayCid) {
2247 Register slot = locs()->temp(0).reg();
2248 __ leaq(slot, element_address);
2249 __ StoreCompressedIntoArray(
array, slot,
value, CanValueBeSmi());
2251 const Object& constant = locs()->in(2).constant();
2252 __ StoreCompressedObjectIntoObjectNoBarrier(
array, element_address,
2256 __ StoreCompressedIntoObjectNoBarrier(
array, element_address,
value);
2262#if defined(USING_MEMORY_SANITIZER)
2263 __ leaq(
TMP, element_address);
2266 __ MsanUnpoison(
TMP, length_in_bytes);
2270LocationSummary* GuardFieldClassInstr::MakeLocationSummary(Zone* zone,
2272 const intptr_t kNumInputs = 1;
2277 const bool emit_full_guard = !opt || (field_cid ==
kIllegalCid);
2278 const bool needs_value_cid_temp_reg =
2279 (value_cid ==
kDynamicCid) && (emit_full_guard || (field_cid != kSmiCid));
2280 const bool needs_field_temp_reg = emit_full_guard;
2282 intptr_t num_temps = 0;
2283 if (needs_value_cid_temp_reg) {
2286 if (needs_field_temp_reg) {
2290 LocationSummary* summary =
new (zone)
2294 for (intptr_t i = 0; i < num_temps; i++) {
2301void GuardFieldClassInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2302 ASSERT(compiler::target::UntaggedObject::kClassIdTagSize == 20);
2303 ASSERT(
sizeof(UntaggedField::guarded_cid_) == 4);
2304 ASSERT(
sizeof(UntaggedField::is_nullable_) == 4);
2314 const bool emit_full_guard =
2317 const bool needs_value_cid_temp_reg =
2318 (value_cid ==
kDynamicCid) && (emit_full_guard || (field_cid != kSmiCid));
2320 const bool needs_field_temp_reg = emit_full_guard;
2322 const Register value_reg = locs()->in(0).reg();
2325 needs_value_cid_temp_reg ? locs()->temp(0).reg() :
kNoRegister;
2327 const Register field_reg = needs_field_temp_reg
2328 ? locs()->temp(locs()->temp_count() - 1).reg()
2331 compiler::Label
ok, fail_label;
2333 compiler::Label* deopt =
nullptr;
2335 deopt =
compiler->AddDeoptStub(deopt_id(), ICData::kDeoptGuardField);
2338 compiler::Label*
fail = (deopt !=
nullptr) ? deopt : &fail_label;
2340 if (emit_full_guard) {
2343 compiler::FieldAddress field_cid_operand(field_reg,
2345 compiler::FieldAddress field_nullability_operand(
2349 LoadValueCid(
compiler, value_cid_reg, value_reg);
2351 __ cmpl(value_cid_reg, field_cid_operand);
2353 __ cmpl(value_cid_reg, field_nullability_operand);
2354 }
else if (value_cid ==
kNullCid) {
2355 __ cmpl(field_nullability_operand, compiler::Immediate(value_cid));
2357 __ cmpl(field_cid_operand, compiler::Immediate(value_cid));
2366 const bool is_complicated_field =
2369 if (!is_complicated_field) {
2372 __ cmpl(field_cid_operand, compiler::Immediate(
kIllegalCid));
2376 __ movl(field_cid_operand, value_cid_reg);
2377 __ movl(field_nullability_operand, value_cid_reg);
2380 __ movl(field_cid_operand, compiler::Immediate(value_cid));
2381 __ movl(field_nullability_operand, compiler::Immediate(value_cid));
2387 if (deopt ==
nullptr) {
2394 __ pushq(field_reg);
2395 __ pushq(value_reg);
2397 __ CallRuntime(kUpdateFieldCidRuntimeEntry, 2);
2404 ASSERT(deopt !=
nullptr);
2411 if (field_cid != kSmiCid) {
2413 __ LoadClassId(value_cid_reg, value_reg);
2414 __ CompareImmediate(value_cid_reg, compiler::Immediate(field_cid));
2419 __ CompareObject(value_reg, Object::null_object());
2423 }
else if (value_cid == field_cid) {
2428 ASSERT(value_cid != nullability);
2435LocationSummary* GuardFieldLengthInstr::MakeLocationSummary(Zone* zone,
2437 const intptr_t kNumInputs = 1;
2439 const intptr_t kNumTemps = 3;
2440 LocationSummary* summary =
new (zone)
2449 LocationSummary* summary =
new (zone)
2457void GuardFieldLengthInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2462 compiler::Label* deopt =
2464 ?
compiler->AddDeoptStub(deopt_id(), ICData::kDeoptGuardField)
2467 const Register value_reg = locs()->in(0).reg();
2471 const Register field_reg = locs()->temp(0).reg();
2472 const Register offset_reg = locs()->temp(1).reg();
2473 const Register length_reg = locs()->temp(2).reg();
2481 compiler::FieldAddress(
2483 __ LoadCompressedSmi(
2487 __ cmpq(offset_reg, compiler::Immediate(0));
2494 __ OBJ(cmp)(length_reg,
2495 compiler::Address(value_reg, offset_reg,
TIMES_1, 0));
2497 if (deopt ==
nullptr) {
2500 __ pushq(field_reg);
2501 __ pushq(value_reg);
2503 __ CallRuntime(kUpdateFieldCidRuntimeEntry, 2);
2513 ASSERT(
field().guarded_list_length_in_object_offset() !=
2516 __ CompareImmediate(
2517 compiler::FieldAddress(value_reg,
2518 field().guarded_list_length_in_object_offset()),
2524LocationSummary* GuardFieldTypeInstr::MakeLocationSummary(Zone* zone,
2526 const intptr_t kNumInputs = 1;
2527 const intptr_t kNumTemps = 1;
2528 LocationSummary* summary =
new (zone)
2535void GuardFieldTypeInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2537 ASSERT(
field().static_type_exactness_state().IsTracking());
2538 if (!
field().static_type_exactness_state().NeedsFieldGuard()) {
2545 compiler::Label* deopt =
2547 ?
compiler->AddDeoptStub(deopt_id(), ICData::kDeoptGuardField)
2552 const Register value_reg = locs()->in(0).reg();
2553 const Register temp = locs()->temp(0).reg();
2562 const Field& original =
2564 __ LoadObject(temp, original);
2565 __ movsxb(temp, compiler::FieldAddress(
2576 compiler::Label call_runtime;
2577 if (
field().static_type_exactness_state().IsUninitialized()) {
2585 __ movq(temp, compiler::FieldAddress(value_reg, temp,
2591 .GetInstanceTypeArguments(
compiler->thread())));
2592 if (deopt !=
nullptr) {
2597 __ Bind(&call_runtime);
2598 __ PushObject(original);
2599 __ pushq(value_reg);
2601 __ CallRuntime(kUpdateFieldCidRuntimeEntry, 2);
2608LocationSummary* StoreStaticFieldInstr::MakeLocationSummary(Zone* zone,
2610 const intptr_t kNumInputs = 1;
2611 const intptr_t kNumTemps = 1;
2612 LocationSummary* locs =
new (zone)
2619void StoreStaticFieldInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2621 Register temp = locs()->temp(0).reg();
2627 THR, compiler::target::Thread::field_table_values_offset()));
2630 compiler::Address(temp, compiler::target::FieldTable::OffsetOf(
field())),
2634LocationSummary* InstanceOfInstr::MakeLocationSummary(Zone* zone,
2636 const intptr_t kNumInputs = 3;
2637 const intptr_t kNumTemps = 0;
2638 LocationSummary* summary =
new (zone)
2649void InstanceOfInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2662 const intptr_t kNumInputs = 2;
2663 const intptr_t kNumTemps = 0;
2664 LocationSummary*
locs =
new (zone)
2675static void InlineArrayAllocation(FlowGraphCompiler*
compiler,
2676 intptr_t num_elements,
2677 compiler::Label* slow_path,
2678 compiler::Label*
done) {
2679 const int kInlineArraySize = 12;
2682 __ TryAllocateArray(kArrayCid, instance_size, slow_path,
2690 __ StoreCompressedIntoObjectNoBarrier(
2697 __ StoreCompressedIntoObjectNoBarrier(
2708 if (num_elements > 0) {
2709 const intptr_t array_size = instance_size -
sizeof(UntaggedArray);
2710 __ LoadObject(
R12, Object::null_object());
2712 sizeof(UntaggedArray)));
2714 intptr_t current_offset = 0;
2715 while (current_offset < array_size) {
2716 __ StoreCompressedIntoObjectNoBarrier(
2718 compiler::Address(
RDI, current_offset),
R12);
2722 compiler::Label init_loop;
2723 __ Bind(&init_loop);
2725 compiler::Address(
RDI, 0),
R12);
2735 TypeUsageInfo* type_usage_info =
compiler->thread()->type_usage_info();
2736 if (type_usage_info !=
nullptr) {
2737 const Class& list_class =
2743 compiler::Label slow_path,
done;
2744 if (!FLAG_use_slow_path && FLAG_inline_alloc) {
2745 if (
compiler->is_optimizing() && !FLAG_precompiled_mode &&
2756 __ Bind(&slow_path);
2757 auto object_store =
compiler->isolate_group()->object_store();
2758 const auto& allocate_array_stub =
2770 const intptr_t kNumInputs = 0;
2771 const intptr_t kNumTemps = 2;
2772 LocationSummary*
locs =
new (zone) LocationSummary(
2780class AllocateContextSlowPath
2781 :
public TemplateSlowPathCode<AllocateUninitializedContextInstr> {
2783 explicit AllocateContextSlowPath(
2784 AllocateUninitializedContextInstr* instruction)
2785 : TemplateSlowPathCode(instruction) {}
2787 virtual void EmitNativeCode(FlowGraphCompiler*
compiler) {
2788 __ Comment(
"AllocateContextSlowPath");
2789 __ Bind(entry_label());
2791 LocationSummary* locs = instruction()->locs();
2792 locs->live_registers()->Remove(locs->out(0));
2796 auto slow_path_env =
compiler->SlowPathEnvironmentFor(
2798 ASSERT(slow_path_env !=
nullptr);
2800 auto object_store =
compiler->isolate_group()->object_store();
2801 const auto& allocate_context_stub = Code::ZoneHandle(
2802 compiler->zone(), object_store->allocate_context_stub());
2805 R10, compiler::Immediate(instruction()->num_context_variables()));
2806 compiler->GenerateStubCall(instruction()->
source(), allocate_context_stub,
2807 UntaggedPcDescriptors::kOther, locs,
2808 instruction()->deopt_id(), slow_path_env);
2809 ASSERT(instruction()->locs()->
out(0).reg() == RAX);
2811 compiler->RestoreLiveRegisters(instruction()->locs());
2812 __ jmp(exit_label());
2822 AllocateContextSlowPath* slow_path =
new AllocateContextSlowPath(
this);
2823 compiler->AddSlowPathCode(slow_path);
2826 if (!FLAG_use_slow_path && FLAG_inline_alloc) {
2827 __ TryAllocateArray(kContextCid, instance_size, slow_path->entry_label(),
2831 locs()->temp(1).reg());
2837 __ Jump(slow_path->entry_label());
2840 __ Bind(slow_path->exit_label());
2845 const intptr_t kNumInputs = 0;
2846 const intptr_t kNumTemps = 1;
2847 LocationSummary*
locs =
new (zone)
2858 auto object_store =
compiler->isolate_group()->object_store();
2859 const auto& allocate_context_stub =
2868LocationSummary* CloneContextInstr::MakeLocationSummary(Zone* zone,
2870 const intptr_t kNumInputs = 1;
2871 const intptr_t kNumTemps = 0;
2872 LocationSummary* locs =
new (zone)
2879void CloneContextInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2883 auto object_store =
compiler->isolate_group()->object_store();
2884 const auto& clone_context_stub =
2887 UntaggedPcDescriptors::kOther, locs(),
2891LocationSummary* CatchBlockEntryInstr::MakeLocationSummary(Zone* zone,
2896void CatchBlockEntryInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
2898 compiler->AddExceptionHandler(
this);
2905 const intptr_t fp_sp_dist =
2906 (compiler::target::frame_layout.first_local_from_fp + 1 -
2910 __ leaq(
RSP, compiler::Address(
RBP, fp_sp_dist));
2913 if (raw_exception_var_ !=
nullptr) {
2914 __ movq(compiler::Address(
RBP,
2915 compiler::target::FrameOffsetInBytesForVariable(
2916 raw_exception_var_)),
2919 if (raw_stacktrace_var_ !=
nullptr) {
2920 __ movq(compiler::Address(
RBP,
2921 compiler::target::FrameOffsetInBytesForVariable(
2922 raw_stacktrace_var_)),
2928LocationSummary* CheckStackOverflowInstr::MakeLocationSummary(Zone* zone,
2930 const intptr_t kNumInputs = 0;
2931 const intptr_t kNumTemps = 1;
2933 LocationSummary* summary =
new (zone)
2934 LocationSummary(zone, kNumInputs, kNumTemps,
2936 : LocationSummary::kCallOnSlowPath);
2941class CheckStackOverflowSlowPath
2942 :
public TemplateSlowPathCode<CheckStackOverflowInstr> {
2944 static constexpr intptr_t kNumSlowPathArgs = 0;
2946 explicit CheckStackOverflowSlowPath(CheckStackOverflowInstr* instruction)
2947 : TemplateSlowPathCode(instruction) {}
2949 virtual void EmitNativeCode(FlowGraphCompiler*
compiler) {
2950 if (
compiler->isolate_group()->use_osr() && osr_entry_label()->IsLinked()) {
2951 __ Comment(
"CheckStackOverflowSlowPathOsr");
2952 __ Bind(osr_entry_label());
2953 __ movq(compiler::Address(THR, Thread::stack_overflow_flags_offset()),
2954 compiler::Immediate(Thread::kOsrRequest));
2956 __ Comment(
"CheckStackOverflowSlowPath");
2957 __ Bind(entry_label());
2958 const bool using_shared_stub =
2959 instruction()->locs()->call_on_shared_slow_path();
2960 if (!using_shared_stub) {
2961 compiler->SaveLiveRegisters(instruction()->locs());
2967 compiler->SlowPathEnvironmentFor(instruction(), kNumSlowPathArgs);
2970 const bool has_frame =
compiler->flow_graph().graph_entry()->NeedsFrame();
2971 if (using_shared_stub) {
2973 ASSERT(
__ constant_pool_allowed());
2974 __ set_constant_pool_allowed(
false);
2975 __ EnterDartFrame(0);
2977 const uword entry_point_offset =
2978 Thread::stack_overflow_shared_stub_entry_point_offset(
2979 instruction()->locs()->live_registers()->FpuRegisterCount() > 0);
2980 __ call(compiler::Address(THR, entry_point_offset));
2981 compiler->RecordSafepoint(instruction()->locs(), kNumSlowPathArgs);
2983 compiler->AddCurrentDescriptor(UntaggedPcDescriptors::kOther,
2984 instruction()->deopt_id(),
2985 instruction()->
source());
2987 __ LeaveDartFrame();
2988 __ set_constant_pool_allowed(
true);
2992 __ CallRuntime(kInterruptOrStackOverflowRuntimeEntry, kNumSlowPathArgs);
2994 instruction()->
source(), instruction()->deopt_id(),
2995 UntaggedPcDescriptors::kOther, instruction()->locs(),
env);
2999 instruction()->in_loop()) {
3001 compiler->AddCurrentDescriptor(UntaggedPcDescriptors::kOsrEntry,
3002 instruction()->deopt_id(),
3003 InstructionSource());
3005 compiler->pending_deoptimization_env_ =
nullptr;
3006 if (!using_shared_stub) {
3007 compiler->RestoreLiveRegisters(instruction()->locs());
3009 __ jmp(exit_label());
3012 compiler::Label* osr_entry_label() {
3013 ASSERT(IsolateGroup::Current()->use_osr());
3014 return &osr_entry_label_;
3018 compiler::Label osr_entry_label_;
3021void CheckStackOverflowInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
3022 CheckStackOverflowSlowPath* slow_path =
new CheckStackOverflowSlowPath(
this);
3023 compiler->AddSlowPathCode(slow_path);
3025 Register temp = locs()->temp(0).reg();
3033 __ LoadObject(temp,
compiler->parsed_function().function());
3034 const intptr_t configured_optimization_counter_threshold =
3035 compiler->thread()->isolate_group()->optimization_counter_threshold();
3036 const int32_t threshold =
3037 configured_optimization_counter_threshold * (
loop_depth() + 1);
3038 __ incl(compiler::FieldAddress(temp, Function::usage_counter_offset()));
3039 __ cmpl(compiler::FieldAddress(temp, Function::usage_counter_offset()),
3040 compiler::Immediate(threshold));
3043 if (
compiler->ForceSlowPathForStackOverflow()) {
3044 __ jmp(slow_path->entry_label());
3046 __ Bind(slow_path->exit_label());
3049static void EmitSmiShiftLeft(FlowGraphCompiler*
compiler,
3050 BinarySmiOpInstr* shift_left) {
3051 const LocationSummary& locs = *shift_left->locs();
3055 compiler::Label* deopt =
3056 shift_left->CanDeoptimize()
3057 ?
compiler->AddDeoptStub(shift_left->deopt_id(),
3058 ICData::kDeoptBinarySmiOp)
3060 if (locs.in(1).IsConstant()) {
3061 const Object& constant = locs.in(1).constant();
3062 ASSERT(constant.IsSmi());
3064#if !defined(DART_COMPRESSED_POINTERS)
3065 const intptr_t kCountLimit = 0x3F;
3067 const intptr_t kCountLimit = 0x1F;
3069 const intptr_t
value = Smi::Cast(constant).Value();
3070 ASSERT((0 < value) && (value < kCountLimit));
3071 if (shift_left->can_overflow()) {
3074 __ OBJ(shl)(
left, compiler::Immediate(1));
3079 Register temp = locs.temp(0).reg();
3081 __ OBJ(shl)(
left, compiler::Immediate(value));
3082 __ OBJ(sar)(
left, compiler::Immediate(value));
3087 __ OBJ(shl)(
left, compiler::Immediate(value));
3093 Range* right_range = shift_left->right_range();
3094 if (shift_left->left()->BindsToConstant() && shift_left->can_overflow()) {
3097 const Object& obj = shift_left->left()->BoundConstant();
3099 const intptr_t left_int = Smi::Cast(obj).Value();
3100 if (left_int == 0) {
3101 __ CompareImmediate(
right, compiler::Immediate(0),
3107 const bool right_needs_check =
3109 if (right_needs_check) {
3119 const bool right_needs_check =
3122 if (!shift_left->can_overflow()) {
3123 if (right_needs_check) {
3124 const bool right_may_be_negative =
3125 (right_range ==
nullptr) || !right_range->IsPositive();
3126 if (right_may_be_negative) {
3127 ASSERT(shift_left->CanDeoptimize());
3128 __ CompareImmediate(
right, compiler::Immediate(0),
3132 compiler::Label
done, is_not_zero;
3137 __ Bind(&is_not_zero);
3146 if (right_needs_check) {
3147 ASSERT(shift_left->CanDeoptimize());
3152 Register temp = locs.temp(0).reg();
3163 ASSERT(!shift_left->is_truncating());
3167static bool CanBeImmediate(
const Object& constant) {
3168 return constant.IsSmi() &&
3173static bool IsSmiValue(
const Object& constant, intptr_t value) {
3174 return constant.IsSmi() && (Smi::Cast(constant).Value() ==
value);
3177LocationSummary* BinarySmiOpInstr::MakeLocationSummary(Zone* zone,
3179 const intptr_t kNumInputs = 2;
3182 if ((right_constant !=
nullptr) && (
op_kind() != Token::kTRUNCDIV) &&
3184#
if defined(DART_COMPRESSED_POINTERS)
3185 (
op_kind() != Token::kUSHR) &&
3188 CanBeImmediate(right_constant->value())) {
3189 const intptr_t kNumTemps = 0;
3190 LocationSummary* summary =
new (zone)
3198 if (
op_kind() == Token::kTRUNCDIV) {
3199 const intptr_t kNumTemps = 1;
3200 LocationSummary* summary =
new (zone)
3217 }
else if (
op_kind() == Token::kMOD) {
3218 const intptr_t kNumTemps = 1;
3219 LocationSummary* summary =
new (zone)
3228 }
else if ((
op_kind() == Token::kSHR)
3229#if !defined(DART_COMPRESSED_POINTERS)
3230 || (
op_kind() == Token::kUSHR)
3233 const intptr_t kNumTemps = 0;
3234 LocationSummary* summary =
new (zone)
3240#if defined(DART_COMPRESSED_POINTERS)
3241 }
else if (
op_kind() == Token::kUSHR) {
3242 const intptr_t kNumTemps = 1;
3243 LocationSummary* summary =
new (zone)
3246 if ((right_constant !=
nullptr) &&
3247 CanBeImmediate(right_constant->value())) {
3256 }
else if (
op_kind() == Token::kSHL) {
3258 const bool shiftBy1 =
3259 (right_constant !=
nullptr) &&
IsSmiValue(right_constant->value(), 1);
3260 const intptr_t kNumTemps = (
can_overflow() && !shiftBy1) ? 1 : 0;
3261 LocationSummary* summary =
new (zone)
3265 if (kNumTemps == 1) {
3271 const intptr_t kNumTemps = 0;
3272 LocationSummary* summary =
new (zone)
3276 if (constant !=
nullptr) {
3286void BinarySmiOpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
3287 if (
op_kind() == Token::kSHL) {
3295 compiler::Label* deopt =
nullptr;
3296 if (CanDeoptimize()) {
3297 deopt =
compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinarySmiOp);
3301 const Object& constant = locs()->in(1).constant();
3302 ASSERT(constant.IsSmi());
3317 const intptr_t
value = Smi::Cast(constant).Value();
3318 __ MulImmediate(
left, compiler::Immediate(value),
3323 case Token::kTRUNCDIV: {
3324 const intptr_t
value = Smi::Cast(constant).Value();
3327 const intptr_t shift_count =
3330 Register temp = locs()->temp(0).reg();
3332#if !defined(DART_COMPRESSED_POINTERS)
3333 __ sarq(temp, compiler::Immediate(63));
3335 __ sarl(temp, compiler::Immediate(31));
3338#if !defined(DART_COMPRESSED_POINTERS)
3339 __ shrq(temp, compiler::Immediate(64 - shift_count));
3341 __ shrl(temp, compiler::Immediate(32 - shift_count));
3345 __ OBJ(sar)(
left, compiler::Immediate(shift_count));
3352 case Token::kBIT_AND: {
3354 __ AndImmediate(
left, compiler::Immediate(imm));
3357 case Token::kBIT_OR: {
3359 __ OrImmediate(
left, compiler::Immediate(imm));
3362 case Token::kBIT_XOR: {
3364 __ XorImmediate(
left, compiler::Immediate(imm));
3370#if !defined(DART_COMPRESSED_POINTERS)
3371 const intptr_t kCountLimit = 0x3F;
3373 const intptr_t kCountLimit = 0x1F;
3375 const intptr_t
value = Smi::Cast(constant).Value();
3382 case Token::kUSHR: {
3386 const intptr_t kCountLimit = 0x3F;
3387 const intptr_t
value = Smi::Cast(constant).Value();
3388 ASSERT((value >= 0) && (value <= kCountLimit));
3389 __ SmiUntagAndSignExtend(
left);
3390 __ shrq(
left, compiler::Immediate(value));
3391 __ shlq(
left, compiler::Immediate(1));
3392 if (deopt !=
nullptr) {
3394#if defined(DART_COMPRESSED_POINTERS)
3395 const Register temp = locs()->temp(0).reg();
3411 if (locs()->in(1).IsStackSlot()) {
3430 case Token::kBIT_AND: {
3435 case Token::kBIT_OR: {
3440 case Token::kBIT_XOR: {
3471 case Token::kBIT_AND: {
3476 case Token::kBIT_OR: {
3481 case Token::kBIT_XOR: {
3486 case Token::kTRUNCDIV: {
3487 compiler::Label not_32bit,
done;
3489 Register temp = locs()->temp(0).reg();
3499#if !defined(DART_COMPRESSED_POINTERS)
3521 __ Bind(¬_32bit);
3529 __ CompareImmediate(
result, compiler::Immediate(0x4000000000000000));
3542 __ cmpl(
result, compiler::Immediate(0x40000000));
3552 compiler::Label not_32bit, div_done;
3554 Register temp = locs()->temp(0).reg();
3564#if !defined(DART_COMPRESSED_POINTERS)
3584#if !defined(DART_COMPRESSED_POINTERS)
3588 __ Bind(¬_32bit);
3604 compiler::Label all_done;
3628 if (CanDeoptimize()) {
3629 __ CompareImmediate(
right, compiler::Immediate(0),
3635#if !defined(DART_COMPRESSED_POINTERS)
3636 const intptr_t kCountLimit = 0x3F;
3638 const intptr_t kCountLimit = 0x1F;
3641 __ CompareImmediate(
right, compiler::Immediate(kCountLimit));
3642 compiler::Label count_ok;
3644 __ LoadImmediate(
right, compiler::Immediate(kCountLimit));
3653 case Token::kUSHR: {
3654 if (deopt !=
nullptr) {
3655 __ CompareImmediate(
right, compiler::Immediate(0),
3661 const intptr_t kCountLimit = 0x3F;
3663 compiler::Label
done;
3665 __ CompareImmediate(
right, compiler::Immediate(kCountLimit),
3667 compiler::Label count_ok;
3674 __ SmiUntagAndSignExtend(
left);
3676 __ shlq(
left, compiler::Immediate(1));
3677 if (deopt !=
nullptr) {
3679#if defined(DART_COMPRESSED_POINTERS)
3680 const Register temp = locs()->temp(0).reg();
3708LocationSummary* CheckEitherNonSmiInstr::MakeLocationSummary(Zone* zone,
3712 ASSERT((left_cid != kDoubleCid) && (right_cid != kDoubleCid));
3713 const intptr_t kNumInputs = 2;
3715 (left_cid != kSmiCid) && (right_cid != kSmiCid);
3716 const intptr_t kNumTemps = need_temp ? 1 : 0;
3717 LocationSummary* summary =
new (zone)
3725void CheckEitherNonSmiInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
3726 compiler::Label* deopt =
3727 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinaryDoubleOp);
3732 if (this->
left()->definition() == this->
right()->definition()) {
3734 }
else if (left_cid == kSmiCid) {
3736 }
else if (right_cid == kSmiCid) {
3739 Register temp = locs()->temp(0).reg();
3747LocationSummary* BoxInstr::MakeLocationSummary(Zone* zone,
bool opt)
const {
3748 const intptr_t kNumInputs = 1;
3749 const intptr_t kNumTemps = 1;
3750 LocationSummary* summary =
new (zone) LocationSummary(
3758void BoxInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
3759 Register out_reg = locs()->out(0).reg();
3760 Register temp = locs()->temp(0).reg();
3768 case kUnboxedDouble:
3769 __ movsd(compiler::FieldAddress(out_reg, ValueOffset()),
value);
3771 case kUnboxedFloat: {
3773 __ movsd(compiler::FieldAddress(out_reg, ValueOffset()),
FpuTMP);
3776 case kUnboxedFloat32x4:
3777 case kUnboxedFloat64x2:
3778 case kUnboxedInt32x4:
3779 __ movups(compiler::FieldAddress(out_reg, ValueOffset()),
value);
3787LocationSummary* UnboxInstr::MakeLocationSummary(Zone* zone,
bool opt)
const {
3789 const intptr_t kNumInputs = 1;
3790 const intptr_t kNumTemps = 0;
3791 const bool needs_writable_input =
3794 LocationSummary* summary =
new (zone)
3799#if !defined(DART_COMPRESSED_POINTERS)
3810void UnboxInstr::EmitLoadFromBox(FlowGraphCompiler*
compiler) {
3811 const Register box = locs()->in(0).reg();
3814 case kUnboxedInt64: {
3816 __ movq(
result, compiler::FieldAddress(box, ValueOffset()));
3820 case kUnboxedDouble: {
3822 __ movsd(
result, compiler::FieldAddress(box, ValueOffset()));
3826 case kUnboxedFloat: {
3828 __ movsd(
result, compiler::FieldAddress(box, ValueOffset()));
3833 case kUnboxedFloat32x4:
3834 case kUnboxedFloat64x2:
3835 case kUnboxedInt32x4: {
3837 __ movups(
result, compiler::FieldAddress(box, ValueOffset()));
3847void UnboxInstr::EmitSmiConversion(FlowGraphCompiler*
compiler) {
3848 const Register box = locs()->in(0).reg();
3851 case kUnboxedInt32: {
3856 case kUnboxedInt64: {
3858 __ SmiUntagAndSignExtend(
result, box);
3861 case kUnboxedDouble: {
3874void UnboxInstr::EmitLoadInt32FromBoxOrSmi(FlowGraphCompiler*
compiler) {
3880void UnboxInstr::EmitLoadInt64FromBoxOrSmi(FlowGraphCompiler*
compiler) {
3886LocationSummary* UnboxInteger32Instr::MakeLocationSummary(Zone* zone,
3888 const intptr_t kNumInputs = 1;
3889 const intptr_t kNumTemps = (!
is_truncating() && CanDeoptimize()) ? 1 : 0;
3890 LocationSummary* summary =
new (zone)
3894 if (kNumTemps > 0) {
3900void UnboxInteger32Instr::EmitNativeCode(FlowGraphCompiler*
compiler) {
3903 compiler::Label* deopt =
3905 ?
compiler->AddDeoptStub(GetDeoptId(), ICData::kDeoptUnboxInteger)
3909 if (value_cid == kSmiCid) {
3911 }
else if (value_cid == kMintCid) {
3913 }
else if (!CanDeoptimize()) {
3918 compiler::Label
done;
3919#if !defined(DART_COMPRESSED_POINTERS)
3930 compiler::Label not_smi;
3932 __ SmiUntagAndSignExtend(
value);
3940 compiler::Label
done;
3941#if !defined(DART_COMPRESSED_POINTERS)
3952 compiler::Label not_smi;
3954 __ SmiUntagAndSignExtend(
value);
3957 __ CompareClassId(
value, kMintCid);
3968 Register temp = locs()->temp(0).reg();
3975LocationSummary* BoxInteger32Instr::MakeLocationSummary(Zone* zone,
3979#if !defined(DART_COMPRESSED_POINTERS)
3982 const bool kMayAllocateMint =
false;
3986 const intptr_t kNumInputs = 1;
3987 const intptr_t kNumTemps = kMayAllocateMint ? 1 : 0;
3988 LocationSummary* summary =
new (zone)
3989 LocationSummary(zone, kNumInputs, kNumTemps,
3991 : LocationSummary::kNoCall);
3994 if (kMayAllocateMint) {
4000void BoxInteger32Instr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4005#if !defined(DART_COMPRESSED_POINTERS)
4015 compiler::Label
done;
4019 __ movsxd(out, out);
4030 __ TestImmediate(
value, compiler::Immediate(0xC0000000LL));
4037 const Register temp = locs()->temp(0).reg();
4050LocationSummary* BoxInt64Instr::MakeLocationSummary(Zone* zone,
4052 const intptr_t kNumInputs = 1;
4058 const bool stubs_in_vm_isolate =
4059 object_store->allocate_mint_with_fpu_regs_stub()
4061 ->InVMIsolateHeap() ||
4062 object_store->allocate_mint_without_fpu_regs_stub()
4064 ->InVMIsolateHeap();
4065 const bool shared_slow_path_call =
4066 SlowPathSharingSupported(opt) && !stubs_in_vm_isolate;
4067 LocationSummary* summary =
new (zone) LocationSummary(
4068 zone, kNumInputs, kNumTemps,
4076 }
else if (shared_slow_path_call) {
4087void BoxInt64Instr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4090#if !defined(DART_COMPRESSED_POINTERS)
4098 compiler::Label
done;
4099 const Register temp = locs()->temp(0).reg();
4106 compiler::Label
done;
4107 const Register temp = locs()->temp(0).reg();
4109 __ sarq(temp, compiler::Immediate(30));
4110 __ addq(temp, compiler::Immediate(1));
4111 __ cmpq(temp, compiler::Immediate(2));
4117 compiler->intrinsic_slow_path_label(),
4119 }
else if (locs()->call_on_shared_slow_path()) {
4120 const bool has_frame =
compiler->flow_graph().graph_entry()->NeedsFrame();
4122 ASSERT(
__ constant_pool_allowed());
4123 __ set_constant_pool_allowed(
false);
4124 __ EnterDartFrame(0);
4126 auto object_store =
compiler->isolate_group()->object_store();
4127 const bool live_fpu_regs = locs()->live_registers()->FpuRegisterCount() > 0;
4130 live_fpu_regs ? object_store->allocate_mint_with_fpu_regs_stub()
4131 : object_store->allocate_mint_without_fpu_regs_stub());
4133 ASSERT(!locs()->live_registers()->ContainsRegister(
4135 auto extended_env =
compiler->SlowPathEnvironmentFor(
this, 0);
4136 compiler->GenerateStubCall(
source(), stub, UntaggedPcDescriptors::kOther,
4139 __ LeaveDartFrame();
4140 __ set_constant_pool_allowed(
true);
4151LocationSummary* BinaryDoubleOpInstr::MakeLocationSummary(Zone* zone,
4153 const intptr_t kNumInputs = 2;
4154 const intptr_t kNumTemps = 0;
4155 LocationSummary* summary =
new (zone)
4163void BinaryDoubleOpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4187LocationSummary* DoubleTestOpInstr::MakeLocationSummary(Zone* zone,
4189 const intptr_t kNumInputs = 1;
4190 const intptr_t kNumTemps =
4191 op_kind() == MethodRecognizer::kDouble_getIsNegative
4193 : (
op_kind() == MethodRecognizer::kDouble_getIsInfinite ? 1 : 0);
4194 LocationSummary* summary =
new (zone)
4197 if (kNumTemps > 0) {
4199 if (
op_kind() == MethodRecognizer::kDouble_getIsNegative) {
4208 BranchLabels labels) {
4211 const bool is_negated = kind() != Token::kEQ;
4214 case MethodRecognizer::kDouble_getIsNaN: {
4218 case MethodRecognizer::kDouble_getIsInfinite: {
4219 const Register temp = locs()->temp(0).reg();
4222 __ movq(temp, compiler::Address(
RSP, 0));
4225 __ AndImmediate(temp, compiler::Immediate(0x7FFFFFFFFFFFFFFFLL));
4227 __ CompareImmediate(temp, compiler::Immediate(0x7FF0000000000000LL));
4230 case MethodRecognizer::kDouble_getIsNegative: {
4231 const Register temp = locs()->temp(0).reg();
4232 const FpuRegister temp_fpu = locs()->temp(1).fpu_reg();
4233 compiler::Label not_zero;
4234 __ xorpd(temp_fpu, temp_fpu);
4237 __ j(
PARITY_EVEN, is_negated ? labels.true_label : labels.false_label);
4240 __ TestImmediate(temp, compiler::Immediate(1));
4250#define DEFINE_EMIT(Name, Args) \
4251 static void Emit##Name(FlowGraphCompiler* compiler, SimdOpInstr* instr, \
4252 PP_APPLY(PP_UNPACK, Args))
4254#define SIMD_OP_FLOAT_ARITH(V, Name, op) \
4255 V(Float32x4##Name, op##ps) \
4256 V(Float64x2##Name, op##pd)
4258#define SIMD_OP_SIMPLE_BINARY(V) \
4259 SIMD_OP_FLOAT_ARITH(V, Add, add) \
4260 SIMD_OP_FLOAT_ARITH(V, Sub, sub) \
4261 SIMD_OP_FLOAT_ARITH(V, Mul, mul) \
4262 SIMD_OP_FLOAT_ARITH(V, Div, div) \
4263 SIMD_OP_FLOAT_ARITH(V, Min, min) \
4264 SIMD_OP_FLOAT_ARITH(V, Max, max) \
4265 V(Int32x4Add, addpl) \
4266 V(Int32x4Sub, subpl) \
4267 V(Int32x4BitAnd, andps) \
4268 V(Int32x4BitOr, orps) \
4269 V(Int32x4BitXor, xorps) \
4270 V(Float32x4Equal, cmppseq) \
4271 V(Float32x4NotEqual, cmppsneq) \
4272 V(Float32x4LessThan, cmppslt) \
4273 V(Float32x4LessThanOrEqual, cmppsle)
4275DEFINE_EMIT(SimdBinaryOp,
4277 switch (instr->kind()) {
4278#define EMIT(Name, op) \
4279 case SimdOpInstr::k##Name: \
4280 __ op(left, right); \
4282 SIMD_OP_SIMPLE_BINARY(EMIT)
4284 case SimdOpInstr::kFloat32x4Scale:
4286 __ shufps(
left,
left, compiler::Immediate(0x00));
4289 case SimdOpInstr::kFloat32x4ShuffleMix:
4290 case SimdOpInstr::kInt32x4ShuffleMix:
4291 __ shufps(
left,
right, compiler::Immediate(instr->mask()));
4293 case SimdOpInstr::kFloat64x2FromDoubles:
4299 case SimdOpInstr::kFloat64x2Scale:
4303 case SimdOpInstr::kFloat64x2WithX:
4304 case SimdOpInstr::kFloat64x2WithY: {
4307 (SimdOpInstr::kFloat64x2WithX + 1));
4308 const intptr_t lane_index = instr->kind() - SimdOpInstr::kFloat64x2WithX;
4309 ASSERT(0 <= lane_index && lane_index < 2);
4312 __ movups(compiler::Address(
RSP, 0),
left);
4314 __ movups(
left, compiler::Address(
RSP, 0));
4318 case SimdOpInstr::kFloat32x4WithX:
4319 case SimdOpInstr::kFloat32x4WithY:
4320 case SimdOpInstr::kFloat32x4WithZ:
4321 case SimdOpInstr::kFloat32x4WithW: {
4326 SimdOpInstr::kFloat32x4WithY == (SimdOpInstr::kFloat32x4WithX + 1) &&
4327 SimdOpInstr::kFloat32x4WithZ == (SimdOpInstr::kFloat32x4WithX + 2) &&
4328 SimdOpInstr::kFloat32x4WithW == (SimdOpInstr::kFloat32x4WithX + 3));
4329 const intptr_t lane_index = instr->kind() - SimdOpInstr::kFloat32x4WithX;
4330 ASSERT(0 <= lane_index && lane_index < 4);
4335 __ movups(
left, compiler::Address(
RSP, 0));
4345#define SIMD_OP_SIMPLE_UNARY(V) \
4346 SIMD_OP_FLOAT_ARITH(V, Sqrt, sqrt) \
4347 SIMD_OP_FLOAT_ARITH(V, Negate, negate) \
4348 SIMD_OP_FLOAT_ARITH(V, Abs, abs) \
4349 V(Float32x4Reciprocal, rcpps) \
4350 V(Float32x4ReciprocalSqrt, rsqrtps)
4352DEFINE_EMIT(SimdUnaryOp, (SameAsFirstInput,
XmmRegister value)) {
4355 switch (instr->kind()) {
4356#define EMIT(Name, op) \
4357 case SimdOpInstr::k##Name: \
4358 __ op(value, value); \
4360 SIMD_OP_SIMPLE_UNARY(EMIT)
4362 case SimdOpInstr::kFloat32x4GetX:
4364 __ cvtss2sd(value, value);
4366 case SimdOpInstr::kFloat32x4GetY:
4367 __ shufps(value, value, compiler::Immediate(0x55));
4368 __ cvtss2sd(value, value);
4370 case SimdOpInstr::kFloat32x4GetZ:
4371 __ shufps(value, value, compiler::Immediate(0xAA));
4372 __ cvtss2sd(value, value);
4374 case SimdOpInstr::kFloat32x4GetW:
4375 __ shufps(value, value, compiler::Immediate(0xFF));
4376 __ cvtss2sd(value, value);
4378 case SimdOpInstr::kFloat32x4Shuffle:
4379 case SimdOpInstr::kInt32x4Shuffle:
4380 __ shufps(value, value, compiler::Immediate(instr->mask()));
4382 case SimdOpInstr::kFloat32x4Splat:
4384 __ cvtsd2ss(value, value);
4386 __ shufps(value, value, compiler::Immediate(0x00));
4388 case SimdOpInstr::kFloat32x4ToFloat64x2:
4389 __ cvtps2pd(value, value);
4391 case SimdOpInstr::kFloat64x2ToFloat32x4:
4392 __ cvtpd2ps(value, value);
4394 case SimdOpInstr::kInt32x4ToFloat32x4:
4395 case SimdOpInstr::kFloat32x4ToInt32x4:
4400 case SimdOpInstr::kFloat64x2GetX:
4403 case SimdOpInstr::kFloat64x2GetY:
4404 __ shufpd(value, value, compiler::Immediate(0x33));
4406 case SimdOpInstr::kFloat64x2Splat:
4407 __ shufpd(value, value, compiler::Immediate(0x0));
4416 switch (instr->kind()) {
4417 case SimdOpInstr::kFloat32x4GetSignMask:
4418 case SimdOpInstr::kInt32x4GetSignMask:
4419 __ movmskps(out, value);
4421 case SimdOpInstr::kFloat64x2GetSignMask:
4422 __ movmskpd(out, value);
4431 Float32x4FromDoubles,
4437 for (intptr_t i = 0; i < 4; i++) {
4438 __ cvtsd2ss(out, instr->locs()->in(i).fpu_reg());
4441 __ movups(out, compiler::Address(
RSP, 0));
4446 __ xorps(value, value);
4450 __ xorpd(value, value);
4453DEFINE_EMIT(Float32x4Clamp,
4458 __ minps(value, upper);
4459 __ maxps(value, lower);
4462DEFINE_EMIT(Float64x2Clamp,
4467 __ minpd(value, upper);
4468 __ maxpd(value, lower);
4471DEFINE_EMIT(Int32x4FromInts,
4475 for (intptr_t i = 0; i < 4; i++) {
4476 __ movl(compiler::Address(
RSP, i *
kInt32Size), instr->locs()->in(i).reg());
4482DEFINE_EMIT(Int32x4FromBools,
4488 Temp<Register> temp)) {
4491 for (intptr_t i = 0; i < 4; i++) {
4492 compiler::Label
done, load_false;
4493 __ xorq(temp, temp);
4494 __ CompareObject(instr->locs()->in(i).reg(),
Bool::True());
4507 __ movzxb(out, out);
4509 compiler::Address(
THR, out,
TIMES_8, Thread::bool_true_offset()));
4512DEFINE_EMIT(Int32x4GetFlagZorW,
4514 __ movhlps(temp, value);
4516 if (instr->kind() == SimdOpInstr::kInt32x4GetFlagW) {
4517 __ shrq(out, compiler::Immediate(32));
4523 __ movq(out, value);
4524 if (instr->kind() == SimdOpInstr::kInt32x4GetFlagY) {
4525 __ shrq(out, compiler::Immediate(32));
4535 SimdOpInstr::kInt32x4WithFlagY == (SimdOpInstr::kInt32x4WithFlagX + 1) &&
4536 SimdOpInstr::kInt32x4WithFlagZ == (SimdOpInstr::kInt32x4WithFlagX + 2) &&
4537 SimdOpInstr::kInt32x4WithFlagW == (SimdOpInstr::kInt32x4WithFlagX + 3));
4538 const intptr_t lane_index = instr->kind() - SimdOpInstr::kInt32x4WithFlagX;
4539 ASSERT(0 <= lane_index && lane_index < 4);
4541 __ movups(compiler::Address(
RSP, 0), mask);
4544 __ xorq(temp, temp);
4550 __ movups(mask, compiler::Address(
RSP, 0));
4554DEFINE_EMIT(Int32x4Select,
4559 Temp<XmmRegister> temp)) {
4561 __ movaps(temp, mask);
4563 __ notps(temp, temp);
4565 __ andps(mask, trueValue);
4567 __ andps(temp, falseValue);
4569 __ orps(mask, temp);
4578#define SIMD_OP_VARIANTS(CASE, ____, SIMPLE) \
4579 SIMD_OP_SIMPLE_BINARY(CASE) \
4580 CASE(Float32x4Scale) \
4581 CASE(Float32x4ShuffleMix) \
4582 CASE(Int32x4ShuffleMix) \
4583 CASE(Float64x2FromDoubles) \
4584 CASE(Float64x2Scale) \
4585 CASE(Float64x2WithX) \
4586 CASE(Float64x2WithY) \
4587 CASE(Float32x4WithX) \
4588 CASE(Float32x4WithY) \
4589 CASE(Float32x4WithZ) \
4590 CASE(Float32x4WithW) \
4591 ____(SimdBinaryOp) \
4592 SIMD_OP_SIMPLE_UNARY(CASE) \
4593 CASE(Float32x4GetX) \
4594 CASE(Float32x4GetY) \
4595 CASE(Float32x4GetZ) \
4596 CASE(Float32x4GetW) \
4597 CASE(Float32x4Shuffle) \
4598 CASE(Int32x4Shuffle) \
4599 CASE(Float32x4Splat) \
4600 CASE(Float32x4ToFloat64x2) \
4601 CASE(Float64x2ToFloat32x4) \
4602 CASE(Int32x4ToFloat32x4) \
4603 CASE(Float32x4ToInt32x4) \
4604 CASE(Float64x2GetX) \
4605 CASE(Float64x2GetY) \
4606 CASE(Float64x2Splat) \
4608 CASE(Float32x4GetSignMask) \
4609 CASE(Int32x4GetSignMask) \
4610 CASE(Float64x2GetSignMask) \
4611 ____(SimdGetSignMask) \
4612 SIMPLE(Float32x4FromDoubles) \
4613 SIMPLE(Int32x4FromInts) \
4614 SIMPLE(Int32x4FromBools) \
4615 SIMPLE(Float32x4Zero) \
4616 SIMPLE(Float64x2Zero) \
4617 SIMPLE(Float32x4Clamp) \
4618 SIMPLE(Float64x2Clamp) \
4619 CASE(Int32x4GetFlagX) \
4620 CASE(Int32x4GetFlagY) \
4621 ____(Int32x4GetFlagXorY) \
4622 CASE(Int32x4GetFlagZ) \
4623 CASE(Int32x4GetFlagW) \
4624 ____(Int32x4GetFlagZorW) \
4625 CASE(Int32x4WithFlagX) \
4626 CASE(Int32x4WithFlagY) \
4627 CASE(Int32x4WithFlagZ) \
4628 CASE(Int32x4WithFlagW) \
4629 ____(Int32x4WithFlag) \
4630 SIMPLE(Int32x4Select)
4634#define CASE(Name, ...) case k##Name:
4636 return MakeLocationSummaryFromEmitter(zone, this, &Emit##Name);
4637#define SIMPLE(Name) CASE(Name) EMIT(Name)
4642 case SimdOpInstr::kFloat32x4GreaterThan:
4643 case SimdOpInstr::kFloat32x4GreaterThanOrEqual:
4653#define CASE(Name, ...) case k##Name:
4655 InvokeEmitter(compiler, this, &Emit##Name); \
4657#define SIMPLE(Name) CASE(Name) EMIT(Name)
4662 case SimdOpInstr::kFloat32x4GreaterThan:
4663 case SimdOpInstr::kFloat32x4GreaterThanOrEqual:
4672LocationSummary* CaseInsensitiveCompareInstr::MakeLocationSummary(
4675 const intptr_t kNumTemps = 0;
4676 LocationSummary* summary =
new (zone)
4686void CaseInsensitiveCompareInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4687 compiler::LeafRuntimeScope rt(
compiler->assembler(),
4694LocationSummary* UnarySmiOpInstr::MakeLocationSummary(Zone* zone,
4696 const intptr_t kNumInputs = 1;
4701void UnarySmiOpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4705 case Token::kNEGATE: {
4706 compiler::Label* deopt =
4707 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptUnaryOp);
4712 case Token::kBIT_NOT:
4722LocationSummary* UnaryDoubleOpInstr::MakeLocationSummary(Zone* zone,
4724 const intptr_t kNumInputs = 1;
4725 const intptr_t kNumTemps = 0;
4726 LocationSummary* summary =
new (zone)
4729 if (
op_kind() == Token::kSQUARE) {
4737void UnaryDoubleOpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4742 case Token::kNEGATE:
4748 case Token::kSQUARE:
4752 case Token::kTRUNCATE:
4758 case Token::kCEILING:
4766LocationSummary* MathMinMaxInstr::MakeLocationSummary(Zone* zone,
4769 const intptr_t kNumInputs = 2;
4770 const intptr_t kNumTemps = 1;
4771 LocationSummary* summary =
new (zone)
4781 const intptr_t kNumInputs = 2;
4782 const intptr_t kNumTemps = 0;
4783 LocationSummary* summary =
new (zone)
4792void MathMinMaxInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4794 (
op_kind() == MethodRecognizer::kMathMax));
4795 const bool is_min =
op_kind() == MethodRecognizer::kMathMin;
4801 Register temp = locs()->temp(0).reg();
4806 is_min ? TokenKindToDoubleCondition(Token::kLT)
4807 : TokenKindToDoubleCondition(Token::kGT);
4813 __ Bind(&returns_nan);
4814 __ movq(temp, compiler::Address(
THR, Thread::double_nan_address_offset()));
4815 __ movsd(
result, compiler::Address(temp, 0));
4819 compiler::Label left_is_negative;
4827 __ testq(temp, compiler::Immediate(1));
4855LocationSummary* Int32ToDoubleInstr::MakeLocationSummary(Zone* zone,
4857 const intptr_t kNumInputs = 1;
4858 const intptr_t kNumTemps = 0;
4859 LocationSummary*
result =
new (zone)
4866void Int32ToDoubleInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4872LocationSummary* SmiToDoubleInstr::MakeLocationSummary(Zone* zone,
4874 const intptr_t kNumInputs = 1;
4875 const intptr_t kNumTemps = 0;
4876 LocationSummary*
result =
new (zone)
4883void SmiToDoubleInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4894LocationSummary* DoubleToIntegerInstr::MakeLocationSummary(Zone* zone,
4896 const intptr_t kNumInputs = 1;
4897 const intptr_t kNumTemps = 1;
4898 LocationSummary*
result =
new (zone) LocationSummary(
4906void DoubleToIntegerInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4908 const Register temp = locs()->temp(0).reg();
4909 XmmRegister value_double = locs()->in(0).fpu_reg();
4912 DoubleToIntegerSlowPath* slow_path =
4913 new DoubleToIntegerSlowPath(
this, value_double);
4914 compiler->AddSlowPathCode(slow_path);
4923 if (FLAG_use_slow_path) {
4924 __ jmp(slow_path->entry_label());
4925 __ Bind(slow_path->exit_label());
4931 compiler::target::Thread::double_truncate_round_supported_offset()),
4932 compiler::Immediate(0));
4933 __ j(
EQUAL, slow_path->entry_label());
4937 case MethodRecognizer::kDoubleFloorToInt:
4940 case MethodRecognizer::kDoubleCeilToInt:
4953 __ OBJ(shl)(temp, compiler::Immediate(1));
4956 __ Bind(slow_path->exit_label());
4959LocationSummary* DoubleToSmiInstr::MakeLocationSummary(Zone* zone,
4961 const intptr_t kNumInputs = 1;
4962 const intptr_t kNumTemps = 1;
4963 LocationSummary*
result =
new (zone)
4971void DoubleToSmiInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
4972 compiler::Label* deopt =
4973 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptDoubleToSmi);
4976 Register temp = locs()->temp(0).reg();
4980 compiler::Label do_call,
done;
4983 __ OBJ(shl)(temp, compiler::Immediate(1));
4988LocationSummary* DoubleToFloatInstr::MakeLocationSummary(Zone* zone,
4990 const intptr_t kNumInputs = 1;
4991 const intptr_t kNumTemps = 0;
4992 LocationSummary*
result =
new (zone)
4999void DoubleToFloatInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5000 __ cvtsd2ss(locs()->
out(0).fpu_reg(), locs()->in(0).fpu_reg());
5003LocationSummary* FloatToDoubleInstr::MakeLocationSummary(Zone* zone,
5005 const intptr_t kNumInputs = 1;
5006 const intptr_t kNumTemps = 0;
5007 LocationSummary*
result =
new (zone)
5014void FloatToDoubleInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5015 __ cvtss2sd(locs()->
out(0).fpu_reg(), locs()->in(0).fpu_reg());
5018LocationSummary* FloatCompareInstr::MakeLocationSummary(Zone* zone,
5024void FloatCompareInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5037 const intptr_t kNumTemps = 4;
5038 LocationSummary*
result =
new (zone)
5053 const intptr_t kNumTemps = 1;
5054 LocationSummary*
result =
new (zone)
5080static void InvokeDoublePow(FlowGraphCompiler*
compiler,
5081 InvokeMathCFunctionInstr* instr) {
5082 ASSERT(instr->recognized_kind() == MethodRecognizer::kMathDoublePow);
5083 const intptr_t kInputCount = 2;
5084 ASSERT(instr->InputCount() == kInputCount);
5085 LocationSummary* locs = instr->locs();
5093 __ xorps(zero_temp, zero_temp);
5096 compiler::Label check_base, skip_call;
5098 __ comisd(exp, zero_temp);
5104 compiler::Label return_base;
5108 __ LoadDImmediate(
XMM0, 2.0);
5110 compiler::Label return_base_times_2;
5114 __ LoadDImmediate(
XMM0, 3.0);
5124 __ Bind(&return_base);
5128 __ Bind(&return_base_times_2);
5133 __ Bind(&check_base);
5136 compiler::Label return_nan;
5144 compiler::Label try_sqrt;
5147 __ Bind(&return_nan);
5151 compiler::Label do_pow, return_zero;
5165 __ comisd(
base, zero_temp);
5171 __ Bind(&return_zero);
5177 compiler::LeafRuntimeScope rt(
compiler->assembler(),
5180 __ movaps(
XMM0, locs->in(0).fpu_reg());
5182 rt.Call(instr->TargetFunction(), kInputCount);
5183 __ movaps(locs->out(0).fpu_reg(),
XMM0);
5185 __ Bind(&skip_call);
5194 compiler::LeafRuntimeScope rt(
compiler->assembler(),
5205LocationSummary* ExtractNthOutputInstr::MakeLocationSummary(Zone* zone,
5209 const intptr_t kNumInputs = 1;
5210 LocationSummary* summary =
5237void ExtractNthOutputInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5238 ASSERT(locs()->in(0).IsPairLocation());
5239 PairLocation* pair = locs()->in(0).AsPairLocation();
5253LocationSummary* UnboxLaneInstr::MakeLocationSummary(Zone* zone,
5259void UnboxLaneInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5263LocationSummary* BoxLanesInstr::MakeLocationSummary(Zone* zone,
5269void BoxLanesInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5273LocationSummary* TruncDivModInstr::MakeLocationSummary(Zone* zone,
5275 const intptr_t kNumInputs = 2;
5276 const intptr_t kNumTemps = 0;
5277 LocationSummary* summary =
new (zone)
5287void TruncDivModInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5289 compiler::Label* deopt =
5290 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinarySmiOp);
5293 ASSERT(locs()->
out(0).IsPairLocation());
5294 PairLocation* pair = locs()->out(0).AsPairLocation();
5295 Register result1 = pair->At(0).reg();
5296 Register result2 = pair->At(1).reg();
5297 compiler::Label not_32bit,
done;
5308#if !defined(DART_COMPRESSED_POINTERS)
5331 __ Bind(¬_32bit);
5338 __ CompareImmediate(
RAX, compiler::Immediate(0x4000000000000000));
5351 __ cmpl(
RAX, compiler::Immediate(0x40000000));
5366 compiler::Label all_done;
5367 __ cmpq(
RDX, compiler::Immediate(0));
5370 if ((divisor_range() ==
nullptr) || divisor_range()->Overlaps(-1, 1)) {
5372 __ cmpq(
right, compiler::Immediate(0));
5378 }
else if (divisor_range()->IsPositive()) {
5394static void EmitHashIntegerCodeSequence(FlowGraphCompiler*
compiler) {
5395 __ movq(
RDX, compiler::Immediate(0x2d51));
5399 __ shrq(
RDX, compiler::Immediate(32));
5401 __ andq(
RAX, compiler::Immediate(0x3fffffff));
5404LocationSummary* HashDoubleOpInstr::MakeLocationSummary(Zone* zone,
5406 const intptr_t kNumInputs = 1;
5407 const intptr_t kNumTemps = 2;
5408 LocationSummary* summary =
new (zone)
5417void HashDoubleOpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5421 const FpuRegister temp_fpu_reg = locs()->temp(1).fpu_reg();
5423 compiler::Label hash_double;
5426 __ cvtsi2sdq(temp_fpu_reg,
RAX);
5427 __ comisd(
value, temp_fpu_reg);
5432 EmitHashIntegerCodeSequence(
compiler);
5434 compiler::Label
done;
5437 __ Bind(&hash_double);
5441 __ shrq(
RDX, compiler::Immediate(32));
5443 __ andq(
RAX, compiler::Immediate(compiler::target::kSmiMax));
5448LocationSummary* HashIntegerOpInstr::MakeLocationSummary(Zone* zone,
5450 const intptr_t kNumInputs = 1;
5451 const intptr_t kNumTemps = 1;
5452 LocationSummary* summary =
new (zone)
5460void HashIntegerOpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5463 Register temp = locs()->temp(0).reg();
5469 __ SmiUntagAndSignExtend(
RAX);
5474 EmitHashIntegerCodeSequence(
compiler);
5489LocationSummary* CheckClassInstr::MakeLocationSummary(Zone* zone,
5491 const intptr_t kNumInputs = 1;
5492 const bool need_mask_temp =
IsBitTest();
5493 const intptr_t kNumTemps = !
IsNullCheck() ? (need_mask_temp ? 2 : 1) : 0;
5494 LocationSummary* summary =
new (zone)
5499 if (need_mask_temp) {
5507 compiler::Label* deopt) {
5508 __ CompareObject(locs()->in(0).reg(), Object::null_object());
5517 compiler::Label* deopt) {
5518 Register biased_cid = locs()->temp(0).reg();
5519 __ subq(biased_cid, compiler::Immediate(
min));
5520 __ cmpq(biased_cid, compiler::Immediate(
max -
min));
5523 Register mask_reg = locs()->temp(1).reg();
5524 __ movq(mask_reg, compiler::Immediate(mask));
5525 __ btq(mask_reg, biased_cid);
5529int CheckClassInstr::EmitCheckCid(FlowGraphCompiler*
compiler,
5534 compiler::Label* is_ok,
5535 compiler::Label* deopt,
5536 bool use_near_jump) {
5537 Register biased_cid = locs()->temp(0).reg();
5539 if (cid_start == cid_end) {
5540 __ cmpl(biased_cid, compiler::Immediate(cid_start - bias));
5546 __ addl(biased_cid, compiler::Immediate(bias - cid_start));
5548 __ cmpl(biased_cid, compiler::Immediate(cid_end - cid_start));
5554 __ j(no_match, deopt);
5556 if (use_near_jump) {
5565LocationSummary* CheckSmiInstr::MakeLocationSummary(Zone* zone,
5567 const intptr_t kNumInputs = 1;
5568 const intptr_t kNumTemps = 0;
5569 LocationSummary* summary =
new (zone)
5575void CheckSmiInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5577 compiler::Label* deopt =
5578 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptCheckSmi);
5579 __ BranchIfNotSmi(
value, deopt);
5582void CheckNullInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5583 ThrowErrorSlowPathCode* slow_path =
new NullErrorSlowPath(
this);
5584 compiler->AddSlowPathCode(slow_path);
5586 Register value_reg = locs()->in(0).reg();
5589 __ CompareObject(value_reg, Object::null_object());
5590 __ BranchIf(
EQUAL, slow_path->entry_label());
5593LocationSummary* CheckClassIdInstr::MakeLocationSummary(Zone* zone,
5595 const intptr_t kNumInputs = 1;
5596 const intptr_t kNumTemps = 0;
5597 LocationSummary* summary =
new (zone)
5604void CheckClassIdInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5606 compiler::Label* deopt =
5607 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptCheckClass);
5608 if (cids_.IsSingleCid()) {
5620LocationSummary* CheckArrayBoundInstr::MakeLocationSummary(Zone* zone,
5622 const intptr_t kNumInputs = 2;
5623 const intptr_t kNumTemps = 0;
5624 LocationSummary* locs =
new (zone)
5631void CheckArrayBoundInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5633 compiler::Label* deopt =
5634 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptCheckArrayBound,
flags);
5639 if (length_loc.IsConstant() && index_loc.IsConstant()) {
5640 ASSERT((Smi::Cast(length_loc.constant()).Value() <=
5641 Smi::Cast(index_loc.constant()).Value()) ||
5642 (Smi::Cast(index_loc.constant()).Value() < 0));
5650 if (index_loc.IsConstant()) {
5652 const Smi&
index = Smi::Cast(index_loc.constant());
5655 }
else if (length_loc.IsConstant()) {
5656 const Smi&
length = Smi::Cast(length_loc.constant());
5658 if (index_cid != kSmiCid) {
5659 __ BranchIfNotSmi(
index, deopt);
5671 if (index_cid != kSmiCid) {
5672 __ BranchIfNotSmi(
index, deopt);
5679LocationSummary* CheckWritableInstr::MakeLocationSummary(Zone* zone,
5681 const intptr_t kNumInputs = 1;
5682 const intptr_t kNumTemps = 0;
5683 LocationSummary* locs =
new (zone) LocationSummary(
5684 zone, kNumInputs, kNumTemps,
5691void CheckWritableInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5692 WriteErrorSlowPath* slow_path =
new WriteErrorSlowPath(
this);
5693 compiler->AddSlowPathCode(slow_path);
5694 __ movq(
TMP, compiler::FieldAddress(locs()->in(0).reg(),
5695 compiler::target::Object::tags_offset()));
5696 __ testq(
TMP, compiler::Immediate(
5697 1 << compiler::target::UntaggedObject::kImmutableBit));
5701class Int64DivideSlowPath :
public ThrowErrorSlowPathCode {
5703 Int64DivideSlowPath(BinaryInt64OpInstr* instruction,
5705 Range* divisor_range)
5706 : ThrowErrorSlowPathCode(instruction,
5707 kIntegerDivisionByZeroExceptionRuntimeEntry),
5708 is_mod_(instruction->op_kind() == Token::kMOD),
5710 divisor_range_(divisor_range),
5711 div_by_minus_one_label_(),
5712 adjust_sign_label_() {}
5714 void EmitNativeCode(FlowGraphCompiler*
compiler)
override {
5716 if (has_divide_by_zero()) {
5717 ThrowErrorSlowPathCode::EmitNativeCode(
compiler);
5719 __ Bind(entry_label());
5720 if (compiler::Assembler::EmittingComments()) {
5721 __ Comment(
"slow path %s operation (no throw)",
name());
5726 if (has_divide_by_minus_one()) {
5727 __ Bind(div_by_minus_one_label());
5733 __ jmp(exit_label());
5740 if (has_adjust_sign()) {
5741 __ Bind(adjust_sign_label());
5742 if (RangeUtils::Overlaps(divisor_range_, -1, 1)) {
5745 __ testq(divisor_, divisor_);
5746 __ j(LESS, &
subtract, compiler::Assembler::kNearJump);
5747 __ addq(RDX, divisor_);
5748 __ jmp(exit_label());
5750 __ subq(RDX, divisor_);
5751 }
else if (divisor_range_->IsPositive()) {
5753 __ addq(RDX, divisor_);
5756 __ subq(RDX, divisor_);
5758 __ jmp(exit_label());
5762 const char*
name()
override {
return "int64 divide"; }
5764 bool has_divide_by_zero() {
return RangeUtils::CanBeZero(divisor_range_); }
5766 bool has_divide_by_minus_one() {
5767 return RangeUtils::Overlaps(divisor_range_, -1, -1);
5770 bool has_adjust_sign() {
return is_mod_; }
5773 return has_divide_by_zero() || has_divide_by_minus_one() ||
5777 compiler::Label* div_by_minus_one_label() {
5778 ASSERT(has_divide_by_minus_one());
5779 return &div_by_minus_one_label_;
5782 compiler::Label* adjust_sign_label() {
5783 ASSERT(has_adjust_sign());
5784 return &adjust_sign_label_;
5790 Range* divisor_range_;
5791 compiler::Label div_by_minus_one_label_;
5792 compiler::Label adjust_sign_label_;
5795static void EmitInt64ModTruncDiv(FlowGraphCompiler*
compiler,
5796 BinaryInt64OpInstr* instruction,
5802 ASSERT(op_kind == Token::kMOD || op_kind == Token::kTRUNCDIV);
5808 if (
auto c = instruction->right()->definition()->AsConstant()) {
5809 if (c->value().IsInteger()) {
5810 const int64_t divisor = Integer::Cast(c->value()).AsInt64Value();
5811 if (divisor <= -2 || divisor >= 2) {
5813 compiler::Label
pos;
5820 __ LoadImmediate(
RAX, compiler::Immediate(magic));
5823 if (divisor > 0 && magic < 0) {
5825 }
else if (divisor < 0 && magic > 0) {
5830 __ sarq(
RDX, compiler::Immediate(shift));
5834 __ shrq(
RDX, compiler::Immediate(63));
5837 if (op_kind == Token::kTRUNCDIV) {
5843 __ LoadImmediate(
TMP, compiler::Immediate(divisor));
5863 Range* right_range = instruction->right()->definition()->range();
5864 Int64DivideSlowPath* slow_path =
5865 new (
Z) Int64DivideSlowPath(instruction,
right, right_range);
5868 if (slow_path->has_divide_by_zero()) {
5870 __ j(
EQUAL, slow_path->entry_label());
5875 if (slow_path->has_divide_by_minus_one()) {
5876 __ cmpq(
right, compiler::Immediate(-1));
5877 __ j(
EQUAL, slow_path->div_by_minus_one_label());
5894 compiler::Label div_64;
5895 compiler::Label div_merge;
5904 __ movsxd(out, out);
5909 __ Bind(&div_merge);
5910 if (op_kind == Token::kMOD) {
5916 __ j(
LESS, slow_path->adjust_sign_label());
5922 if (slow_path->is_needed()) {
5923 __ Bind(slow_path->exit_label());
5924 compiler->AddSlowPathCode(slow_path);
5928template <
typename OperandType>
5929static void EmitInt64Arithmetic(FlowGraphCompiler*
compiler,
5932 const OperandType&
right) {
5940 case Token::kBIT_AND:
5943 case Token::kBIT_OR:
5946 case Token::kBIT_XOR:
5957LocationSummary* BinaryInt64OpInstr::MakeLocationSummary(Zone* zone,
5961 case Token::kTRUNCDIV: {
5962 const intptr_t kNumInputs = 2;
5963 const intptr_t kNumTemps = 1;
5964 LocationSummary* summary =
new (zone) LocationSummary(
5977 const intptr_t kNumInputs = 2;
5978 const intptr_t kNumTemps = 0;
5979 LocationSummary* summary =
new (zone) LocationSummary(
5989void BinaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
5994 ASSERT(!CanDeoptimize());
5997 const Location temp = locs()->temp(0);
5999 temp.reg(),
out.reg());
6000 }
else if (
right.IsConstant()) {
6006 compiler::Immediate(value));
6013LocationSummary* UnaryInt64OpInstr::MakeLocationSummary(Zone* zone,
6015 const intptr_t kNumInputs = 1;
6016 const intptr_t kNumTemps = 0;
6017 LocationSummary* summary =
new (zone)
6024void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6029 case Token::kBIT_NOT:
6032 case Token::kNEGATE:
6040static void EmitShiftInt64ByConstant(FlowGraphCompiler*
compiler,
6043 const Object&
right) {
6044 const int64_t shift = Integer::Cast(
right).AsInt64Value();
6048 __ sarq(
left, compiler::Immediate(
6053 __ shrq(
left, compiler::Immediate(shift));
6057 __ shlq(
left, compiler::Immediate(shift));
6065static void EmitShiftInt64ByRCX(FlowGraphCompiler*
compiler,
6073 case Token::kUSHR: {
6086static void EmitShiftUint32ByConstant(FlowGraphCompiler*
compiler,
6089 const Object&
right) {
6090 const int64_t shift = Integer::Cast(
right).AsInt64Value();
6097 case Token::kUSHR: {
6098 __ shrl(
left, compiler::Immediate(shift));
6102 __ shll(
left, compiler::Immediate(shift));
6111static void EmitShiftUint32ByRCX(FlowGraphCompiler*
compiler,
6116 case Token::kUSHR: {
6129class ShiftInt64OpSlowPath :
public ThrowErrorSlowPathCode {
6131 explicit ShiftInt64OpSlowPath(ShiftInt64OpInstr* instruction)
6132 : ThrowErrorSlowPathCode(instruction,
6133 kArgumentErrorUnboxedInt64RuntimeEntry) {}
6135 const char*
name()
override {
return "int64 shift"; }
6137 void EmitCodeAtSlowPathEntry(FlowGraphCompiler*
compiler)
override {
6138 const Register out = instruction()->locs()->out(0).reg();
6139 ASSERT(out == instruction()->locs()->in(0).reg());
6141 compiler::Label throw_error;
6143 __ j(LESS, &throw_error);
6145 switch (instruction()->AsShiftInt64Op()->op_kind()) {
6147 __ sarq(out, compiler::Immediate(kBitsPerInt64 - 1));
6156 __ jmp(exit_label());
6158 __ Bind(&throw_error);
6165 __ movq(compiler::Address(
6166 THR, compiler::target::Thread::unboxed_runtime_arg_offset()),
6171LocationSummary* ShiftInt64OpInstr::MakeLocationSummary(Zone* zone,
6173 const intptr_t kNumInputs = 2;
6174 const intptr_t kNumTemps = 0;
6175 LocationSummary* summary =
new (zone) LocationSummary(
6185void ShiftInt64OpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6193 locs()->in(1).constant());
6199 ShiftInt64OpSlowPath* slow_path =
nullptr;
6201 slow_path =
new (
Z) ShiftInt64OpSlowPath(
this);
6202 compiler->AddSlowPathCode(slow_path);
6204 __ cmpq(
RCX, compiler::Immediate(kShiftCountLimit));
6205 __ j(
ABOVE, slow_path->entry_label());
6210 if (slow_path !=
nullptr) {
6211 __ Bind(slow_path->exit_label());
6216LocationSummary* SpeculativeShiftInt64OpInstr::MakeLocationSummary(
6219 const intptr_t kNumInputs = 2;
6220 const intptr_t kNumTemps = 0;
6221 LocationSummary* summary =
new (zone)
6229void SpeculativeShiftInt64OpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6237 locs()->in(1).constant());
6245 compiler::Label* deopt =
6246 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinaryInt64Op);
6248 __ cmpq(
RCX, compiler::Immediate(kShiftCountLimit));
6256class ShiftUint32OpSlowPath :
public ThrowErrorSlowPathCode {
6258 explicit ShiftUint32OpSlowPath(ShiftUint32OpInstr* instruction)
6259 : ThrowErrorSlowPathCode(instruction,
6260 kArgumentErrorUnboxedInt64RuntimeEntry) {}
6262 const char*
name()
override {
return "uint32 shift"; }
6264 void EmitCodeAtSlowPathEntry(FlowGraphCompiler*
compiler)
override {
6265 const Register out = instruction()->locs()->out(0).reg();
6266 ASSERT(out == instruction()->locs()->in(0).reg());
6268 compiler::Label throw_error;
6270 __ j(LESS, &throw_error);
6273 __ jmp(exit_label());
6275 __ Bind(&throw_error);
6282 __ movq(compiler::Address(
6283 THR, compiler::target::Thread::unboxed_runtime_arg_offset()),
6288LocationSummary* ShiftUint32OpInstr::MakeLocationSummary(Zone* zone,
6290 const intptr_t kNumInputs = 2;
6291 const intptr_t kNumTemps = 0;
6292 LocationSummary* summary =
new (zone) LocationSummary(
6302void ShiftUint32OpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6309 locs()->in(1).constant());
6315 ShiftUint32OpSlowPath* slow_path =
nullptr;
6317 slow_path =
new (
Z) ShiftUint32OpSlowPath(
this);
6318 compiler->AddSlowPathCode(slow_path);
6320 __ cmpq(
RCX, compiler::Immediate(kUint32ShiftCountLimit));
6321 __ j(
ABOVE, slow_path->entry_label());
6326 if (slow_path !=
nullptr) {
6327 __ Bind(slow_path->exit_label());
6332LocationSummary* SpeculativeShiftUint32OpInstr::MakeLocationSummary(
6335 const intptr_t kNumInputs = 2;
6336 const intptr_t kNumTemps = 0;
6337 LocationSummary* summary =
new (zone)
6345void SpeculativeShiftUint32OpInstr::EmitNativeCode(
6353 locs()->in(1).constant());
6362 compiler::Label* deopt =
6363 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinaryInt64Op);
6369 compiler::Label cont;
6370 __ OBJ(cmp)(
RCX, compiler::Immediate(kUint32ShiftCountLimit));
6382LocationSummary* BinaryUint32OpInstr::MakeLocationSummary(Zone* zone,
6384 const intptr_t kNumInputs = 2;
6385 const intptr_t kNumTemps = 0;
6386 LocationSummary* summary =
new (zone)
6394template <
typename OperandType>
6395static void EmitIntegerArithmetic(FlowGraphCompiler*
compiler,
6398 const OperandType&
right) {
6406 case Token::kBIT_AND:
6409 case Token::kBIT_OR:
6412 case Token::kBIT_XOR:
6423void BinaryUint32OpInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6429 case Token::kBIT_AND:
6430 case Token::kBIT_OR:
6431 case Token::kBIT_XOR:
6443 ASSERT(instr->op_kind() == Token::kBIT_NOT);
6449LocationSummary* IntConverterInstr::MakeLocationSummary(Zone* zone,
6451 const intptr_t kNumInputs = 1;
6452 const intptr_t kNumTemps = 0;
6453 LocationSummary* summary =
new (zone)
6455 if (
from() == kUntagged ||
to() == kUntagged) {
6458 ASSERT(!CanDeoptimize());
6459 }
else if (
from() == kUnboxedInt64) {
6460 ASSERT(
to() == kUnboxedUint32 ||
to() == kUnboxedInt32);
6461 }
else if (
to() == kUnboxedInt64) {
6464 ASSERT(
to() == kUnboxedUint32 ||
to() == kUnboxedInt32);
6474void IntConverterInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6475 const bool is_nop_conversion =
6478 if (is_nop_conversion) {
6479 ASSERT(locs()->in(0).reg() == locs()->
out(0).reg());
6483 if (
from() == kUnboxedInt32 &&
to() == kUnboxedUint32) {
6491 }
else if (
from() == kUnboxedUint32 &&
to() == kUnboxedInt32) {
6496 if (CanDeoptimize()) {
6497 compiler::Label* deopt =
6498 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptUnboxInteger);
6502 }
else if (
from() == kUnboxedInt64) {
6503 ASSERT(
to() == kUnboxedUint32 ||
to() == kUnboxedInt32);
6506 if (!CanDeoptimize()) {
6510 compiler::Label* deopt =
6511 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptUnboxInteger);
6519 }
else if (
to() == kUnboxedInt64) {
6523 if (
from() == kUnboxedUint32) {
6536LocationSummary* StopInstr::MakeLocationSummary(Zone* zone,
bool opt)
const {
6540void StopInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6544void GraphEntryInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6546 if (entry !=
nullptr) {
6547 if (!
compiler->CanFallThroughTo(entry)) {
6548 FATAL(
"Checked function entry must have no offset");
6552 if (!
compiler->CanFallThroughTo(entry)) {
6558LocationSummary* GotoInstr::MakeLocationSummary(Zone* zone,
bool opt)
const {
6562void GotoInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6564 if (FLAG_reorder_basic_blocks) {
6569 compiler->AddCurrentDescriptor(UntaggedPcDescriptors::kDeopt, GetDeoptId(),
6570 InstructionSource());
6583LocationSummary* IndirectGotoInstr::MakeLocationSummary(Zone* zone,
6585 const intptr_t kNumInputs = 1;
6586 const intptr_t kNumTemps = 1;
6588 LocationSummary* summary =
new (zone)
6597void IndirectGotoInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6598 Register index_reg = locs()->in(0).reg();
6599 Register offset_reg = locs()->temp(0).reg();
6605 __ ExtendNonNegativeSmi(index_reg);
6606 __ LoadObject(offset_reg, offsets_);
6608 false, kTypedDataInt32ArrayCid,
6610 false, offset_reg, index_reg));
6613 const intptr_t kRIPRelativeLeaqSize = 7;
6614 const intptr_t entry_to_rip_offset =
__ CodeSize() + kRIPRelativeLeaqSize;
6616 ASSERT(
__ CodeSize() == entry_to_rip_offset);
6619 __ addq(
TMP, offset_reg);
6625LocationSummary* StrictCompareInstr::MakeLocationSummary(Zone* zone,
6627 const intptr_t kNumInputs = 2;
6628 const intptr_t kNumTemps = 0;
6630 LocationSummary* locs =
new (zone)
6637 LocationSummary* locs =
new (zone)
6642 locs->set_in(1, locs->in(0).IsConstant()
6649Condition StrictCompareInstr::EmitComparisonCodeRegConstant(
6651 BranchLabels labels,
6653 const Object& obj) {
6660 const intptr_t kNumInputs = 1;
6661 const intptr_t kNumTemps = 0;
6662 LocationSummary* summary =
new (zone)
6672 const Array& arguments_descriptor =
6676 if (FLAG_precompiled_mode) {
6679 __ movq(
RCX, compiler::FieldAddress(
6680 RAX, compiler::target::Closure::entry_point_offset()));
6686 FUNCTION_REG, compiler::target::Function::code_offset()));
6688 __ movq(
RCX, compiler::FieldAddress(
6690 compiler::target::Function::entry_point_offset()));
6695 if (!FLAG_precompiled_mode) {
6701 UntaggedPcDescriptors::kOther,
locs(),
env());
6705LocationSummary* BooleanNegateInstr::MakeLocationSummary(Zone* zone,
6711void BooleanNegateInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6712 Register input = locs()->in(0).reg();
6715 __ xorq(
result, compiler::Immediate(
6719LocationSummary* BoolToIntInstr::MakeLocationSummary(Zone* zone,
6725void BoolToIntInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6729LocationSummary* IntToBoolInstr::MakeLocationSummary(Zone* zone,
6735void IntToBoolInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6741 const intptr_t kNumInputs = (
type_arguments() !=
nullptr) ? 1 : 0;
6742 const intptr_t kNumTemps = 0;
6743 LocationSummary*
locs =
new (zone)
6755 TypeUsageInfo* type_usage_info =
compiler->thread()->type_usage_info();
6756 if (type_usage_info !=
nullptr) {
6763 compiler->GenerateStubCall(
source(), stub, UntaggedPcDescriptors::kOther,
6767void DebugStepCheckInstr::EmitNativeCode(FlowGraphCompiler*
compiler) {
6772 __ CallPatchable(StubCode::DebugStepCheck());
static void done(const char *config, const char *src, const char *srcOptions, const char *name)
static void fail(const SkString &err)
static bool match(const char *needle, const char *haystack)
static bool are_equal(skiatest::Reporter *reporter, const SkMatrix &a, const SkMatrix &b)
static int float_bits(float f)
static bool ok(int result)
static bool left(const SkPoint &p0, const SkPoint &p1)
static bool right(const SkPoint &p0, const SkPoint &p1)
static bool subtract(const R &a, const R &b, R *out)
#define ASSERT_EQUAL(expected, actual)
#define RELEASE_ASSERT(cond)
#define COMPILE_ASSERT(expr)
intptr_t num_context_variables() const
Value * type_arguments() const
const Class & cls() const
intptr_t num_context_variables() const
static intptr_t type_arguments_offset()
static intptr_t InstanceSize()
static constexpr bool IsValidLength(intptr_t len)
static intptr_t length_offset()
Token::Kind op_kind() const
bool can_overflow() const
Token::Kind op_kind() const
bool RightIsPowerOfTwoConstant() const
Range * right_range() const
ParallelMoveInstr * parallel_move() const
bool HasParallelMove() const
BlockEntryInstr(intptr_t block_id, intptr_t try_index, intptr_t deopt_id, intptr_t stack_depth)
static const Bool & False()
static const Bool & True()
static void Allocate(FlowGraphCompiler *compiler, Instruction *instruction, const Class &cls, Register result, Register temp)
Representation from_representation() const
virtual bool ValueFitsSmi() const
ComparisonInstr * comparison() const
virtual Representation RequiredInputRepresentation(intptr_t idx) const
intptr_t index_scale() const
static constexpr Register kSecondReturnReg
static constexpr Register kVarArgFpuRegisterCount
static constexpr intptr_t kCalleeSaveCpuRegisters
static constexpr RegList kVolatileXmmRegisters
static constexpr intptr_t kVolatileCpuRegisters
static constexpr intptr_t kFpuArgumentRegisters
static constexpr FpuRegister kReturnFpuReg
static constexpr Register kFfiAnyNonAbiRegister
static constexpr Register kArg3Reg
static constexpr Register kArg1Reg
static constexpr Register kReturnReg
static constexpr Register kSecondNonArgumentRegister
static constexpr Register kArg2Reg
static constexpr Register kArg4Reg
static constexpr intptr_t kCalleeSaveXmmRegisters
const RuntimeEntry & TargetFunction() const
bool IsDeoptIfNull() const
DECLARE_INSTRUCTION_SERIALIZABLE_FIELDS(CheckClassInstr, TemplateInstruction, FIELD_LIST) private void EmitBitTest(FlowGraphCompiler *compiler, intptr_t min, intptr_t max, intptr_t mask, compiler::Label *deopt)
void EmitNullCheck(FlowGraphCompiler *compiler, compiler::Label *deopt)
virtual bool UseSharedSlowPathStub(bool is_optimizing) const
intptr_t loop_depth() const
virtual void EmitBranchCode(FlowGraphCompiler *compiler, BranchInstr *branch)
virtual void EmitNativeCode(FlowGraphCompiler *compiler)
virtual Condition EmitComparisonCode(FlowGraphCompiler *compiler, BranchLabels labels)=0
static CompilerState & Current()
const Object & value() const
void EmitMoveToLocation(FlowGraphCompiler *compiler, const Location &destination, Register tmp=kNoRegister, intptr_t pair_index=0)
static intptr_t num_variables_offset()
static intptr_t InstanceSize()
Value * type_arguments() const
virtual Value * num_elements() const
virtual Representation representation() const
static constexpr intptr_t kNone
MethodRecognizer::Kind op_kind() const
MethodRecognizer::Kind recognized_kind() const
bool is_null_aware() const
void EmitReturnMoves(FlowGraphCompiler *compiler, const Register temp0, const Register temp1)
DECLARE_INSTRUCTION_SERIALIZABLE_FIELDS(FfiCallInstr, VariadicDefinition, FIELD_LIST) private void EmitParamMoves(FlowGraphCompiler *compiler, const Register saved_fp, const Register temp0, const Register temp1)
static bool CanExecuteGeneratedCodeInSafepoint()
intptr_t TargetAddressIndex() const
static intptr_t guarded_cid_offset()
static intptr_t static_type_exactness_state_offset()
bool needs_length_check() const
static intptr_t guarded_list_length_in_object_offset_offset()
StaticTypeExactnessState static_type_exactness_state() const
intptr_t guarded_cid() const
static intptr_t is_nullable_offset()
static intptr_t guarded_list_length_offset()
ParallelMoveInstr * parallel_move() const
BlockEntryInstr * block() const
bool HasParallelMove() const
JoinEntryInstr * successor() const
FunctionEntryInstr * normal_entry() const
OsrEntryInstr * osr_entry() const
const Field & field() const
ComparisonInstr * comparison() const
virtual Representation RequiredInputRepresentation(intptr_t idx) const
const AbstractType & type() const
Environment * env() const
virtual LocationSummary * MakeLocationSummary(Zone *zone, bool is_optimizing) const =0
virtual void EmitNativeCode(FlowGraphCompiler *compiler)
void InitializeLocationSummary(Zone *zone, bool optimizing)
virtual Representation representation() const
InstructionSource source() const
intptr_t deopt_id() const
static LocationSummary * MakeCallSummary(Zone *zone, const Instruction *instr, LocationSummary *locs=nullptr)
Representation to() const
Representation from() const
const RuntimeEntry & TargetFunction() const
MethodRecognizer::Kind recognized_kind() const
static constexpr intptr_t kDoubleTempIndex
ObjectStore * object_store() const
static IsolateGroup * Current()
intptr_t TargetAddressIndex() const
void EmitParamMoves(FlowGraphCompiler *compiler, Register saved_fp, Register temp0)
LocationSummary * MakeLocationSummaryInternal(Zone *zone, const RegList temps) const
intptr_t index_scale() const
bool can_pack_into_smi() const
intptr_t element_count() const
intptr_t class_id() const
intptr_t class_id() const
intptr_t index_scale() const
Representation representation() const
virtual Representation RequiredInputRepresentation(intptr_t index) const
Register base_reg() const
virtual Representation representation() const
const LocalVariable & local() const
Location temp(intptr_t index) const
Location out(intptr_t index) const
static LocationSummary * Make(Zone *zone, intptr_t input_count, Location out, ContainsCall contains_call)
void set_temp(intptr_t index, Location loc)
void set_out(intptr_t index, Location loc)
bool always_calls() const
Location in(intptr_t index) const
void set_in(intptr_t index, Location loc)
static Location StackSlot(intptr_t stack_index, Register base)
static Location NoLocation()
static Location SameAsFirstInput()
static Location Pair(Location first, Location second)
static Location FpuRegisterLocation(FpuRegister reg)
static Location WritableRegister()
static Location RegisterLocation(Register reg)
static Location PrefersRegister()
PairLocation * AsPairLocation() const
static Location RequiresRegister()
static Location RequiresFpuRegister()
FpuRegister fpu_reg() const
static Location Constant(const ConstantInstr *obj, int pair_index=0)
intptr_t result_cid() const
MethodRecognizer::Kind op_kind() const
bool unboxed_inputs() const
Value * src_start() const
void EmitLoopCopy(FlowGraphCompiler *compiler, Register dest_reg, Register src_reg, Register length_reg, compiler::Label *done, compiler::Label *copy_forwards=nullptr)
void PrepareLengthRegForLoop(FlowGraphCompiler *compiler, Register length_reg, compiler::Label *done)
Value * dest_start() const
static intptr_t value_offset()
virtual Representation representation() const
Location location() const
static int ComputeArgcTag(const Function &function)
bool is_auto_scope() const
bool is_bootstrap_native() const
const Function & function() const
NativeFunction native_c_function() const
static uword LinkNativeCallEntry()
static Object & ZoneHandle()
Value * char_code() const
static intptr_t data_offset()
Location At(intptr_t i) const
static bool Overlaps(Range *range, intptr_t min, intptr_t max)
static bool OnlyLessThanOrEqualTo(Range *range, intptr_t value)
static bool IsWithin(const Range *range, int64_t min, int64_t max)
static bool IsPositive(Range *range)
static bool CanBeZero(Range *range)
DECLARE_INSTRUCTION_SERIALIZABLE_FIELDS(ShiftIntegerOpInstr, BinaryIntegerOpInstr, FIELD_LIST) protected bool IsShiftCountInRange(int64_t max=kShiftCountLimit) const
Range * shift_range() const
static constexpr intptr_t kBits
static SmiPtr New(intptr_t value)
static constexpr intptr_t kMaxValue
static intptr_t RawValue(intptr_t value)
static constexpr int8_t kUninitialized
bool IsUninitialized() const
const char * message() const
bool ShouldEmitStoreBarrier() const
virtual Representation RequiredInputRepresentation(intptr_t idx) const
intptr_t class_id() const
intptr_t index_scale() const
const LocalVariable & local() const
const Field & field() const
bool needs_number_check() const
static intptr_t length_offset()
static CodePtr GetAllocationStubForClass(const Class &cls)
static constexpr int kNullCharCodeSymbolOffset
intptr_t ArgumentCount() const
ArrayPtr GetArgumentsDescriptor() const
virtual intptr_t InputCount() const
const ZoneGrowableArray< intptr_t > & cid_results() const
static intptr_t stack_limit_offset()
static intptr_t global_object_pool_offset()
virtual Representation representation() const
Token::Kind op_kind() const
Token::Kind op_kind() const
virtual Representation representation() const
bool is_truncating() const
virtual Representation representation() const
bool IsScanFlagsUnboxed() const
static void CalculateMagicAndShiftForDivRem(int64_t divisor, int64_t *magic, int64_t *shift)
static constexpr T Maximum(T x, T y)
static constexpr int ShiftForPowerOfTwo(T x)
static T Minimum(T x, T y)
static T AddWithWrapAround(T a, T b)
static constexpr int CountOneBits64(uint64_t x)
static constexpr size_t HighestBit(int64_t v)
static constexpr bool IsPowerOfTwo(T x)
bool BindsToConstant() const
Definition * definition() const
Value(Definition *definition)
intptr_t InputCount() const
static Address AddressRIPRelative(int32_t disp)
void static bool EmittingComments()
Address ElementAddressForRegIndex(bool is_load, bool is_external, intptr_t cid, intptr_t index_scale, bool index_unboxed, Register array, Register index)
static Address VMTagAddress()
static bool AddressCanHoldConstantIndex(const Object &constant, bool is_load, bool is_external, intptr_t cid, intptr_t index_scale, bool *needs_base=nullptr)
static bool IsSafe(const Object &object)
intptr_t StackTopInBytes() const
FlutterSemanticsFlag flag
FlutterSemanticsFlag flags
static float max(float r, float g, float b)
static float min(float r, float g, float b)
#define CASE(Arity, Mask, Name, Args, Result)
#define DEFINE_UNIMPLEMENTED_INSTRUCTION(Name)
#define DEFINE_BACKEND(Name, Args)
const intptr_t kResultIndex
bool HasIntegerValue(const dart::Object &object, int64_t *value)
Location LocationAnyOrConstant(Value *value)
Location LocationRegisterOrConstant(Value *value)
const Register kWriteBarrierSlotReg
@ TIMES_COMPRESSED_WORD_SIZE
static Condition InvertCondition(Condition c)
static constexpr int kSavedCallerPcSlotFromFp
bool IsTypedDataBaseClassId(intptr_t index)
constexpr intptr_t kBitsPerWord
static bool IsSmiValue(Value *val, intptr_t *int_val)
const Register kExceptionObjectReg
const Register kWriteBarrierObjectReg
constexpr int32_t kMinInt32
const Register kWriteBarrierValueReg
static constexpr bool IsCalleeSavedRegister(Register reg)
constexpr intptr_t kIntptrMin
static const ClassId kLastErrorCid
constexpr intptr_t kSimd128Size
const Register CALLEE_SAVED_TEMP
static const ClassId kFirstErrorCid
void RegisterTypeArgumentsUse(const Function &function, TypeUsageInfo *type_usage_info, const Class &klass, Definition *type_arguments)
static constexpr int kParamEndSlotFromFp
const Register ARGS_DESC_REG
bool IsClampedTypedDataBaseClassId(intptr_t index)
Location LocationFixedRegisterOrConstant(Value *value, Register reg)
const int kNumberOfFpuRegisters
bool IsExternalPayloadClassId(classid_t cid)
constexpr RegList kDartAvailableCpuRegs
constexpr intptr_t kInt32Size
static constexpr intptr_t kCompressedWordSize
static constexpr int kPcMarkerSlotFromFp
const Register FUNCTION_REG
const Register IC_DATA_REG
constexpr int32_t kMaxInt32
compiler::Address LocationToStackSlotAddress(Location loc)
constexpr intptr_t kWordSize
Location LocationWritableRegisterOrConstant(Value *value)
static bool IsConstant(Definition *def, int64_t *val)
constexpr intptr_t kFloatSize
static constexpr Representation kUnboxedIntPtr
const Register kStackTraceObjectReg
static int8_t data[kExtLength]
constexpr intptr_t kDoubleSize
Location LocationFixedRegisterOrSmiConstant(Value *value, Register reg)
static ScaleFactor ToScaleFactor(intptr_t index_scale, bool index_unboxed)
Location LocationRegisterOrSmiConstant(Value *value, intptr_t min_value, intptr_t max_value)
constexpr intptr_t kBitsPerInt64
ByteRegister ByteRegisterOf(Register reg)
static constexpr Register kResultReg
static constexpr Register kLengthReg
static constexpr Register kTypeArgumentsReg
static constexpr Register kResultReg
static constexpr Register kTempReg
static constexpr Register kTypeArgumentsReg
static constexpr Register kResultReg
static constexpr Register kObjectReg
static constexpr Representation NativeRepresentation(Representation rep)
static constexpr intptr_t kBoolVsNullMask
static constexpr intptr_t kBoolValueMask
static constexpr size_t ValueSize(Representation rep)
static constexpr bool IsUnboxedInteger(Representation rep)
static compiler::OperandSize OperandSize(Representation rep)
static constexpr bool IsUnboxed(Representation rep)
static bool IsUnsignedInteger(Representation rep)
static Representation RepresentationOfArrayElement(classid_t cid)
static constexpr Register kDstTypeReg
static constexpr Register kInstanceReg
static constexpr Register kFunctionTypeArgumentsReg
static constexpr Register kInstantiatorTypeArgumentsReg
@ kResetToBootstrapNative
#define ASSERT_BOOL_FALSE_FOLLOWS_BOOL_TRUE()