From 6cc3450a52468b2ca4b8872179cfe856bf65b177 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 14 Jun 2022 10:58:06 +0900 Subject: [PATCH] [JITLink][AArch64] Lift fixup functions from aarch64.cpp to aarch64.h. (NFC) Lift fixup functions from aarch64.cpp to aarch64.h so that they have better chance of getting inlined. Also, adds some comments documenting the purpose of functions. Reviewed By: sgraenitz Differential Revision: https://reviews.llvm.org/D127559 --- .../llvm/ExecutionEngine/JITLink/aarch64.h | 161 +++++++++++++++++- llvm/lib/ExecutionEngine/JITLink/aarch64.cpp | 153 ----------------- 2 files changed, 158 insertions(+), 156 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 36403fab9efb..2cebc03310e1 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -45,11 +45,166 @@ enum EdgeKind_aarch64 : Edge::Kind { /// only const char *getEdgeKindName(Edge::Kind K); -bool isLoadStoreImm12(uint32_t Instr); +// Returns whether the Instr is LD/ST (imm12) +inline bool isLoadStoreImm12(uint32_t Instr) { + constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + return (Instr & LoadStoreImm12Mask) == 0x39000000; +} -unsigned getPageOffset12Shift(uint32_t Instr); +// Returns the amount the address operand of LD/ST (imm12) +// should be shifted right by. +// +// The shift value varies by the data size of LD/ST instruction. +// For instance, LDH instructoin needs the address to be shifted +// right by 1. +inline unsigned getPageOffset12Shift(uint32_t Instr) { + constexpr uint32_t Vec128Mask = 0x04800000; -Error applyFixup(LinkGraph &G, Block &B, const Edge &E); + if (isLoadStoreImm12(Instr)) { + uint32_t ImplicitShift = Instr >> 30; + if (ImplicitShift == 0) + if ((Instr & Vec128Mask) == Vec128Mask) + ImplicitShift = 4; + + return ImplicitShift; + } + + return 0; +} + +/// Apply fixup expression for edge to block content. +inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { + using namespace support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch26: { + assert((FixupAddress.getValue() & 0x3) == 0 && + "Branch-inst is not 32-bit aligned"); + + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + + if (static_cast(Value) & 0x3) + return make_error("Branch26 target is not 32-bit " + "aligned"); + + if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(little32_t *)FixupPtr; + assert((RawInstr & 0x7fffffff) == 0x14000000 && + "RawInstr isn't a B or BR immediate instruction"); + uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; + uint32_t FixedInstr = RawInstr | Imm; + *(little32_t *)FixupPtr = FixedInstr; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + if (Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case Page21: { + assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && + "GOTPAGE21 with non-zero addend"); + uint64_t TargetPage = + (E.getTarget().getAddress().getValue() + E.getAddend()) & + ~static_cast(4096 - 1); + uint64_t PCPage = + FixupAddress.getValue() & ~static_cast(4096 - 1); + + int64_t PageDelta = TargetPage - PCPage; + if (!isInt<33>(PageDelta)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xffffffe0) == 0x90000000 && + "RawInstr isn't an ADRP instruction"); + uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; + uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; + uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case PageOffset12: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + unsigned ImmShift = getPageOffset12Shift(RawInstr); + + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error("PAGEOFF12 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case LDRLiteral19: { + assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); + int64_t Delta = E.getTarget().getAddress() - FixupAddress; + if (Delta & 0x3) + return make_error("LDR literal target is not 32-bit " + "aligned"); + if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t EncodedImm = ((static_cast(Delta) >> 2) & 0x7ffff) << 5; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + case TLVPage21: + case GOTPage21: + case TLVPageOffset12: + case GOTPageOffset12: + case PointerToGOT: { + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); + } + default: + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); + } + + return Error::success(); +} /// AArch64 null pointer content. extern const uint8_t NullGOTEntryContent[8]; diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 697c31b65a0b..1397a45df4c2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -18,159 +18,6 @@ namespace llvm { namespace jitlink { namespace aarch64 { -bool isLoadStoreImm12(uint32_t Instr) { - constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; - return (Instr & LoadStoreImm12Mask) == 0x39000000; -} - -unsigned getPageOffset12Shift(uint32_t Instr) { - constexpr uint32_t Vec128Mask = 0x04800000; - - if (isLoadStoreImm12(Instr)) { - uint32_t ImplicitShift = Instr >> 30; - if (ImplicitShift == 0) - if ((Instr & Vec128Mask) == Vec128Mask) - ImplicitShift = 4; - - return ImplicitShift; - } - - return 0; -} - -Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { - using namespace support; - - char *BlockWorkingMem = B.getAlreadyMutableContent().data(); - char *FixupPtr = BlockWorkingMem + E.getOffset(); - orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); - - switch (E.getKind()) { - case Branch26: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "Branch-inst is not 32-bit aligned"); - - int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - - if (static_cast(Value) & 0x3) - return make_error("Branch26 target is not 32-bit " - "aligned"); - - if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(little32_t *)FixupPtr; - assert((RawInstr & 0x7fffffff) == 0x14000000 && - "RawInstr isn't a B or BR immediate instruction"); - uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; - uint32_t FixedInstr = RawInstr | Imm; - *(little32_t *)FixupPtr = FixedInstr; - break; - } - case Pointer32: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - if (Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(ulittle32_t *)FixupPtr = Value; - break; - } - case Pointer64: - case Pointer64Anon: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - *(ulittle64_t *)FixupPtr = Value; - break; - } - case Page21: { - assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && - "GOTPAGE21 with non-zero addend"); - uint64_t TargetPage = - (E.getTarget().getAddress().getValue() + E.getAddend()) & - ~static_cast(4096 - 1); - uint64_t PCPage = - FixupAddress.getValue() & ~static_cast(4096 - 1); - - int64_t PageDelta = TargetPage - PCPage; - if (!isInt<33>(PageDelta)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xffffffe0) == 0x90000000 && - "RawInstr isn't an ADRP instruction"); - uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; - uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; - uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case PageOffset12: { - uint64_t TargetOffset = - (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - unsigned ImmShift = getPageOffset12Shift(RawInstr); - - if (TargetOffset & ((1 << ImmShift) - 1)) - return make_error("PAGEOFF12 target is not aligned"); - - uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case LDRLiteral19: { - assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); - assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); - int64_t Delta = E.getTarget().getAddress() - FixupAddress; - if (Delta & 0x3) - return make_error("LDR literal target is not 32-bit " - "aligned"); - if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t EncodedImm = ((static_cast(Delta) >> 2) & 0x7ffff) << 5; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case Delta32: - case Delta64: - case NegDelta32: - case NegDelta64: { - int64_t Value; - if (E.getKind() == Delta32 || E.getKind() == Delta64) - Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - else - Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); - - if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(little32_t *)FixupPtr = Value; - } else - *(little64_t *)FixupPtr = Value; - break; - } - case TLVPage21: - case GOTPage21: - case TLVPageOffset12: - case GOTPageOffset12: - case PointerToGOT: { - return make_error( - "In graph " + G.getName() + ", section " + B.getSection().getName() + - "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); - } - default: - return make_error( - "In graph " + G.getName() + ", section " + B.getSection().getName() + - "unsupported edge kind" + getEdgeKindName(E.getKind())); - } - - return Error::success(); -} - const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};