15#if defined(DART_PRECOMPILER) && !defined(TARGET_ARCH_IA32)
19 always_generate_trampolines_for_testing,
21 "Generate always trampolines (for testing purposes).");
24 lower_tail_pc_relative_call_distance,
26 "Lower tail call distance.");
28 upper_tail_pc_relative_call_distance,
30 "Upper tail call distance.");
31DEFINE_FLAG(
int, lower_pc_relative_call_distance, -1,
"Lower call distance.");
32DEFINE_FLAG(
int, upper_pc_relative_call_distance, -1,
"Upper call distance.");
34struct TailCallDistanceLimits {
35 static intptr_t Lower() {
36 if (FLAG_lower_tail_pc_relative_call_distance != -1) {
37 return FLAG_lower_tail_pc_relative_call_distance;
39 return PcRelativeTailCallPattern::kLowerCallingRange;
41 static intptr_t Upper() {
42 if (FLAG_upper_tail_pc_relative_call_distance != -1) {
43 return FLAG_upper_tail_pc_relative_call_distance;
45 return PcRelativeTailCallPattern::kUpperCallingRange;
49struct CallDistanceLimits {
50 static intptr_t Lower() {
51 if (FLAG_lower_pc_relative_call_distance != -1) {
52 return FLAG_lower_pc_relative_call_distance;
54 return PcRelativeCallPattern::kLowerCallingRange;
56 static intptr_t Upper() {
57 if (FLAG_upper_pc_relative_call_distance != -1) {
58 return FLAG_upper_pc_relative_call_distance;
60 return PcRelativeCallPattern::kUpperCallingRange;
64const intptr_t kTrampolineSize =
66 compiler::target::Instructions::kBarePayloadAlignment);
68CodeRelocator::CodeRelocator(Thread* thread,
69 GrowableArray<CodePtr>* code_objects,
70 GrowableArray<ImageWriterCommand>* commands)
71 : StackResource(thread),
73 code_objects_(code_objects),
75 kind_type_and_offset_(Smi::Handle(thread->zone())),
76 target_(Object::Handle(thread->zone())),
77 destination_(Code::Handle(thread->zone())) {}
79void CodeRelocator::Relocate(
bool is_vm_isolate) {
80 Zone* zone = Thread::Current()->zone();
81 auto& current_caller = Code::Handle(zone);
82 auto& call_targets = Array::Handle(zone);
84 auto& next_caller = Code::Handle(zone);
85 auto& next_caller_targets = Array::Handle(zone);
88 for (intptr_t i = 0; i < code_objects_->length(); ++i) {
89 current_caller = (*code_objects_)[i];
91 const intptr_t code_text_offset = next_text_offset_;
92 if (!AddInstructionsToText(current_caller.ptr())) {
96 call_targets = current_caller.static_calls_target_table();
97 ScanCallTargets(current_caller, call_targets, code_text_offset);
100 ResolveUnresolvedCallsTargeting(current_caller.instructions());
104 if (i < (code_objects_->length() - 1)) {
105 next_caller = (*code_objects_)[i + 1];
106 next_caller_targets = next_caller.static_calls_target_table();
108 next_caller = Code::null();
109 next_caller_targets = Array::null();
111 BuildTrampolinesForAlmostOutOfRangeCalls(next_caller, next_caller_targets);
117 if (!all_unresolved_calls_.IsEmpty()) {
118 for (
auto call : all_unresolved_calls_) {
119 OS::PrintErr(
"Unresolved call to %s from %s\n",
120 Object::Handle(
call->callee).ToCString(),
121 Object::Handle(
call->caller).ToCString());
128 auto it = trampolines_by_destination_.GetIterator();
130 auto entry = it.Next();
131 if (entry ==
nullptr)
break;
133 UnresolvedTrampolineList* trampoline_list = entry->value;
134 while (!trampoline_list->IsEmpty()) {
135 auto unresolved_trampoline = trampoline_list->RemoveFirst();
136 ResolveTrampoline(unresolved_trampoline);
137 delete unresolved_trampoline;
139 delete trampoline_list;
141 trampolines_by_destination_.Clear();
147bool CodeRelocator::AddInstructionsToText(CodePtr code) {
148 InstructionsPtr instructions = Code::InstructionsOf(code);
152 if (text_offsets_.HasKey(instructions)) {
156 if (Instructions::ShouldBeAligned(instructions) &&
157 !Utils::IsAligned(next_text_offset_, kPreferredLoopAlignment)) {
158 const intptr_t padding_size =
159 Utils::RoundUp(next_text_offset_, kPreferredLoopAlignment) -
162 commands_->Add(ImageWriterCommand(next_text_offset_, padding_size));
163 next_text_offset_ += padding_size;
166 text_offsets_.Insert({instructions, next_text_offset_});
167 commands_->Add(ImageWriterCommand(next_text_offset_, code));
168 next_text_offset_ += ImageWriter::SizeInSnapshot(instructions);
173UnresolvedTrampoline* CodeRelocator::FindTrampolineFor(
174 UnresolvedCall* unresolved_call) {
175 auto destination = Code::InstructionsOf(unresolved_call->callee);
176 auto entry = trampolines_by_destination_.Lookup(destination);
177 if (entry !=
nullptr) {
178 UnresolvedTrampolineList* trampolines = entry->value;
179 ASSERT(!trampolines->IsEmpty());
186 auto it = trampolines->End();
189 UnresolvedTrampoline* trampoline = *it;
190 if (!IsTargetInRangeFor(unresolved_call, trampoline->text_offset)) {
193 if (trampoline->offset_into_target ==
194 unresolved_call->offset_into_target) {
198 }
while (it != trampolines->Begin());
203void CodeRelocator::AddTrampolineToText(InstructionsPtr destination,
204 uint8_t* trampoline_bytes,
205 intptr_t trampoline_length) {
206 commands_->Add(ImageWriterCommand(next_text_offset_, trampoline_bytes,
208 next_text_offset_ += trampoline_length;
211void CodeRelocator::ScanCallTargets(
const Code& code,
212 const Array& call_targets,
213 intptr_t code_text_offset) {
214 if (call_targets.IsNull()) {
218 for (
auto call : calls) {
219 kind_type_and_offset_ =
call.Get<Code::kSCallTableKindAndOffset>();
220 const auto kind = Code::KindField::decode(kind_type_and_offset_.Value());
221 const auto return_pc_offset =
222 Code::OffsetField::decode(kind_type_and_offset_.Value());
223 const auto call_entry_point =
224 Code::EntryPointField::decode(kind_type_and_offset_.Value());
226 if (kind == Code::kCallViaCode) {
230 destination_ = GetTarget(call);
234 int32_t offset_into_target = 0;
236 intptr_t call_instruction_offset;
237 if (kind == Code::kPcRelativeCall || kind == Code::kPcRelativeTTSCall) {
238 call_instruction_offset =
239 return_pc_offset - PcRelativeCallPattern::kLengthInBytes;
240 PcRelativeCallPattern
call(
code.PayloadStart() + call_instruction_offset);
242 offset_into_target =
call.distance();
243 is_tail_call =
false;
245 ASSERT(kind == Code::kPcRelativeTailCall);
246 call_instruction_offset =
247 return_pc_offset - PcRelativeTailCallPattern::kLengthInBytes;
248 PcRelativeTailCallPattern
call(
code.PayloadStart() +
249 call_instruction_offset);
251 offset_into_target =
call.distance();
255 const uword destination_payload = destination_.PayloadStart();
256 const uword entry_point = call_entry_point == Code::kUncheckedEntry
257 ? destination_.UncheckedEntryPoint()
258 : destination_.EntryPoint();
260 offset_into_target += (entry_point - destination_payload);
262 const intptr_t text_offset =
263 code_text_offset + AdjustPayloadOffset(call_instruction_offset);
265 UnresolvedCall unresolved_call(
code.ptr(), call_instruction_offset,
266 text_offset, destination_.ptr(),
267 offset_into_target, is_tail_call);
268 if (!TryResolveBackwardsCall(&unresolved_call)) {
269 EnqueueUnresolvedCall(
new UnresolvedCall(unresolved_call));
274void CodeRelocator::EnqueueUnresolvedCall(UnresolvedCall* unresolved_call) {
276 all_unresolved_calls_.Append(unresolved_call);
279 InstructionsPtr destination = Code::InstructionsOf(unresolved_call->callee);
280 if (!unresolved_calls_by_destination_.HasKey(destination)) {
281 unresolved_calls_by_destination_.Insert(
282 {destination,
new SameDestinationUnresolvedCallsList()});
284 unresolved_calls_by_destination_.LookupValue(destination)
285 ->Append(unresolved_call);
288void CodeRelocator::EnqueueUnresolvedTrampoline(
289 UnresolvedTrampoline* unresolved_trampoline) {
290 auto destination = Code::InstructionsOf(unresolved_trampoline->callee);
291 auto entry = trampolines_by_destination_.Lookup(destination);
293 UnresolvedTrampolineList* trampolines =
nullptr;
294 if (entry ==
nullptr) {
295 trampolines =
new UnresolvedTrampolineList();
296 trampolines_by_destination_.Insert({destination, trampolines});
298 trampolines = entry->value;
300 trampolines->Append(unresolved_trampoline);
303bool CodeRelocator::TryResolveBackwardsCall(UnresolvedCall* unresolved_call) {
304 auto callee = Code::InstructionsOf(unresolved_call->callee);
305 auto map_entry = text_offsets_.Lookup(callee);
306 if (map_entry ==
nullptr)
return false;
308 if (IsTargetInRangeFor(unresolved_call, map_entry->value)) {
309 ResolveCall(unresolved_call);
315void CodeRelocator::ResolveUnresolvedCallsTargeting(
316 const InstructionsPtr instructions) {
317 if (unresolved_calls_by_destination_.HasKey(instructions)) {
318 SameDestinationUnresolvedCallsList* calls =
319 unresolved_calls_by_destination_.LookupValue(instructions);
320 auto it = calls->Begin();
321 while (it != calls->End()) {
322 UnresolvedCall* unresolved_call = *it;
324 ASSERT(Code::InstructionsOf(unresolved_call->callee) == instructions);
325 ResolveCall(unresolved_call);
328 calls->Remove(unresolved_call);
329 all_unresolved_calls_.Remove(unresolved_call);
331 delete unresolved_call;
335 bool ok = unresolved_calls_by_destination_.Remove(instructions);
340void CodeRelocator::ResolveCall(UnresolvedCall* unresolved_call) {
341 const intptr_t destination_text =
342 FindDestinationInText(Code::InstructionsOf(unresolved_call->callee),
343 unresolved_call->offset_into_target);
345 ResolveCallToDestination(unresolved_call, destination_text);
348void CodeRelocator::ResolveCallToDestination(UnresolvedCall* unresolved_call,
349 intptr_t destination_text) {
350 const intptr_t call_text_offset = unresolved_call->text_offset;
351 const intptr_t call_offset = unresolved_call->call_offset;
353 const int32_t
distance = destination_text - call_text_offset;
355 auto const caller = unresolved_call->caller;
356 uword addr = Code::PayloadStartOf(caller) + call_offset;
357 if (unresolved_call->is_tail_call) {
358 PcRelativeTailCallPattern
call(addr);
360 call.set_distance(
static_cast<int32_t
>(distance));
363 PcRelativeCallPattern
call(addr);
365 call.set_distance(
static_cast<int32_t
>(distance));
370 unresolved_call->caller =
nullptr;
371 unresolved_call->callee =
nullptr;
374void CodeRelocator::ResolveTrampoline(
375 UnresolvedTrampoline* unresolved_trampoline) {
376 const intptr_t trampoline_text_offset = unresolved_trampoline->text_offset;
377 const uword trampoline_start =
378 reinterpret_cast<uword>(unresolved_trampoline->trampoline_bytes);
380 auto callee = Code::InstructionsOf(unresolved_trampoline->callee);
381 auto destination_text =
382 FindDestinationInText(callee, unresolved_trampoline->offset_into_target);
383 const int32_t
distance = destination_text - trampoline_text_offset;
385 PcRelativeTrampolineJumpPattern pattern(trampoline_start);
386 pattern.Initialize();
387 pattern.set_distance(distance);
388 ASSERT(pattern.distance() == distance);
391bool CodeRelocator::IsTargetInRangeFor(UnresolvedCall* unresolved_call,
392 intptr_t target_text_offset) {
393 const auto forward_distance =
394 target_text_offset - unresolved_call->text_offset;
395 if (unresolved_call->is_tail_call) {
396 return TailCallDistanceLimits::Lower() <= forward_distance &&
397 forward_distance <= TailCallDistanceLimits::Upper();
399 return CallDistanceLimits::Lower() <= forward_distance &&
400 forward_distance <= CallDistanceLimits::Upper();
404CodePtr CodeRelocator::GetTarget(
const StaticCallsTableEntry& call) {
407 ASSERT(
call.Get<Code::kSCallTableFunctionTarget>() == Function::null());
409 target_ =
call.Get<Code::kSCallTableCodeOrTypeTarget>();
410 if (target_.IsAbstractType()) {
411 target_ = AbstractType::Cast(target_).type_test_stub();
412 destination_ = Code::Cast(target_).ptr();
424 if (destination_.InVMIsolateHeap()) {
425 auto object_store = thread_->isolate_group()->object_store();
427 if (destination_.ptr() == StubCode::DefaultTypeTest().ptr()) {
428 destination_ = object_store->default_tts_stub();
429 }
else if (destination_.ptr() ==
430 StubCode::DefaultNullableTypeTest().ptr()) {
431 destination_ = object_store->default_nullable_tts_stub();
432 }
else if (destination_.ptr() == StubCode::TopTypeTypeTest().ptr()) {
433 destination_ = object_store->top_type_tts_stub();
434 }
else if (destination_.ptr() == StubCode::UnreachableTypeTest().ptr()) {
435 destination_ = object_store->unreachable_tts_stub();
436 }
else if (destination_.ptr() == StubCode::SlowTypeTest().ptr()) {
437 destination_ = object_store->slow_tts_stub();
438 }
else if (destination_.ptr() ==
439 StubCode::NullableTypeParameterTypeTest().ptr()) {
440 destination_ = object_store->nullable_type_parameter_tts_stub();
441 }
else if (destination_.ptr() ==
442 StubCode::TypeParameterTypeTest().ptr()) {
443 destination_ = object_store->type_parameter_tts_stub();
450 destination_ = Code::Cast(target_).ptr();
452 ASSERT(!destination_.InVMIsolateHeap());
453 return destination_.ptr();
456void CodeRelocator::BuildTrampolinesForAlmostOutOfRangeCalls(
457 const Code& next_caller,
458 const Array& next_caller_targets) {
459 const bool all_functions_emitted = next_caller.IsNull();
461 bool next_requires_alignment =
false;
463 uword next_call_count = 0;
464 if (!all_functions_emitted) {
465 next_size = ImageWriter::SizeInSnapshot(next_caller.instructions());
466 next_requires_alignment =
467 Instructions::ShouldBeAligned(next_caller.instructions());
468 if (!next_caller_targets.IsNull()) {
470 next_call_count = calls.Length();
474 while (!all_unresolved_calls_.IsEmpty()) {
475 UnresolvedCall* unresolved_call = all_unresolved_calls_.First();
477 if (!all_functions_emitted) {
482 const intptr_t next_start =
483 next_requires_alignment
484 ? Utils::RoundUp(next_text_offset_, kPreferredLoopAlignment)
486 const intptr_t future_boundary =
487 next_start + next_size +
489 (unresolved_calls_by_destination_.Length() + next_call_count - 1);
490 if (IsTargetInRangeFor(unresolved_call, future_boundary) &&
491 !FLAG_always_generate_trampolines_for_testing) {
503 ASSERT(IsTargetInRangeFor(unresolved_call, next_text_offset_));
506 intptr_t trampoline_text_offset = -1;
507 auto callee = Code::InstructionsOf(unresolved_call->callee);
509 if (!FLAG_always_generate_trampolines_for_testing) {
510 auto old_trampoline_entry = FindTrampolineFor(unresolved_call);
511 if (old_trampoline_entry !=
nullptr) {
512 trampoline_text_offset = old_trampoline_entry->text_offset;
517 if (trampoline_text_offset == -1) {
521 auto trampoline_bytes =
new uint8_t[kTrampolineSize];
522 ASSERT((kTrampolineSize % compiler::target::kWordSize) == 0);
523 for (uint8_t* cur = trampoline_bytes;
524 cur < trampoline_bytes + kTrampolineSize;
525 cur += compiler::target::kWordSize) {
526 *
reinterpret_cast<compiler::target::uword*
>(cur) =
527 kBreakInstructionFiller;
529 auto unresolved_trampoline =
new UnresolvedTrampoline{
530 unresolved_call->callee,
531 unresolved_call->offset_into_target,
535 AddTrampolineToText(callee, trampoline_bytes, kTrampolineSize);
536 EnqueueUnresolvedTrampoline(unresolved_trampoline);
537 trampoline_text_offset = unresolved_trampoline->text_offset;
542 auto destination = Code::InstructionsOf(unresolved_call->callee);
543 ResolveCallToDestination(unresolved_call, trampoline_text_offset);
547 auto calls = unresolved_calls_by_destination_.LookupValue(destination);
548 calls->Remove(unresolved_call);
549 all_unresolved_calls_.Remove(unresolved_call);
550 delete unresolved_call;
553 if (calls->IsEmpty()) {
554 unresolved_calls_by_destination_.Remove(destination);
560intptr_t CodeRelocator::FindDestinationInText(
const InstructionsPtr destination,
561 intptr_t offset_into_target) {
562 auto const destination_offset = text_offsets_.LookupValue(destination);
563 return destination_offset + AdjustPayloadOffset(offset_into_target);
566intptr_t CodeRelocator::AdjustPayloadOffset(intptr_t payload_offset) {
567 if (FLAG_precompiled_mode) {
568 return payload_offset;
570 return compiler::target::Instructions::HeaderSize() + payload_offset;
static bool ok(int result)
#define RELEASE_ASSERT(cond)
static constexpr int kLengthInBytes
static constexpr T RoundUp(T x, uintptr_t alignment, uintptr_t offset=0)
#define DEFINE_FLAG(type, name, default_value, comment)
ArrayOfTuplesView< Code::SCallTableEntry, std::tuple< Smi, Object, Function > > StaticCallsTable