[JITLink][AArch64] Lift fixup functions from aarch64.cpp to aarch64.h. (NFC)
Lift fixup functions from aarch64.cpp to aarch64.h so that they have better chance of getting inlined. Also, adds some comments documenting the purpose of functions. Reviewed By: sgraenitz Differential Revision: https://reviews.llvm.org/D127559
This commit is contained in:
Sunho Kim
parent
db37225803
commit
6cc3450a52
|
|
@ -45,11 +45,166 @@ enum EdgeKind_aarch64 : Edge::Kind {
|
|||
/// only
|
||||
const char *getEdgeKindName(Edge::Kind K);
|
||||
|
||||
bool isLoadStoreImm12(uint32_t Instr);
|
||||
// Returns whether the Instr is LD/ST (imm12)
|
||||
inline bool isLoadStoreImm12(uint32_t Instr) {
|
||||
constexpr uint32_t LoadStoreImm12Mask = 0x3b000000;
|
||||
return (Instr & LoadStoreImm12Mask) == 0x39000000;
|
||||
}
|
||||
|
||||
unsigned getPageOffset12Shift(uint32_t Instr);
|
||||
// Returns the amount the address operand of LD/ST (imm12)
|
||||
// should be shifted right by.
|
||||
//
|
||||
// The shift value varies by the data size of LD/ST instruction.
|
||||
// For instance, LDH instructoin needs the address to be shifted
|
||||
// right by 1.
|
||||
inline unsigned getPageOffset12Shift(uint32_t Instr) {
|
||||
constexpr uint32_t Vec128Mask = 0x04800000;
|
||||
|
||||
Error applyFixup(LinkGraph &G, Block &B, const Edge &E);
|
||||
if (isLoadStoreImm12(Instr)) {
|
||||
uint32_t ImplicitShift = Instr >> 30;
|
||||
if (ImplicitShift == 0)
|
||||
if ((Instr & Vec128Mask) == Vec128Mask)
|
||||
ImplicitShift = 4;
|
||||
|
||||
return ImplicitShift;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// Apply fixup expression for edge to block content.
|
||||
inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
|
||||
using namespace support;
|
||||
|
||||
char *BlockWorkingMem = B.getAlreadyMutableContent().data();
|
||||
char *FixupPtr = BlockWorkingMem + E.getOffset();
|
||||
orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset();
|
||||
|
||||
switch (E.getKind()) {
|
||||
case Branch26: {
|
||||
assert((FixupAddress.getValue() & 0x3) == 0 &&
|
||||
"Branch-inst is not 32-bit aligned");
|
||||
|
||||
int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
|
||||
|
||||
if (static_cast<uint64_t>(Value) & 0x3)
|
||||
return make_error<JITLinkError>("Branch26 target is not 32-bit "
|
||||
"aligned");
|
||||
|
||||
if (Value < -(1 << 27) || Value > ((1 << 27) - 1))
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
|
||||
uint32_t RawInstr = *(little32_t *)FixupPtr;
|
||||
assert((RawInstr & 0x7fffffff) == 0x14000000 &&
|
||||
"RawInstr isn't a B or BR immediate instruction");
|
||||
uint32_t Imm = (static_cast<uint32_t>(Value) & ((1 << 28) - 1)) >> 2;
|
||||
uint32_t FixedInstr = RawInstr | Imm;
|
||||
*(little32_t *)FixupPtr = FixedInstr;
|
||||
break;
|
||||
}
|
||||
case Pointer32: {
|
||||
uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
|
||||
if (Value > std::numeric_limits<uint32_t>::max())
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
*(ulittle32_t *)FixupPtr = Value;
|
||||
break;
|
||||
}
|
||||
case Pointer64:
|
||||
case Pointer64Anon: {
|
||||
uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
|
||||
*(ulittle64_t *)FixupPtr = Value;
|
||||
break;
|
||||
}
|
||||
case Page21: {
|
||||
assert((E.getKind() != GOTPage21 || E.getAddend() == 0) &&
|
||||
"GOTPAGE21 with non-zero addend");
|
||||
uint64_t TargetPage =
|
||||
(E.getTarget().getAddress().getValue() + E.getAddend()) &
|
||||
~static_cast<uint64_t>(4096 - 1);
|
||||
uint64_t PCPage =
|
||||
FixupAddress.getValue() & ~static_cast<uint64_t>(4096 - 1);
|
||||
|
||||
int64_t PageDelta = TargetPage - PCPage;
|
||||
if (!isInt<33>(PageDelta))
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
|
||||
uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
|
||||
assert((RawInstr & 0xffffffe0) == 0x90000000 &&
|
||||
"RawInstr isn't an ADRP instruction");
|
||||
uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3;
|
||||
uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff;
|
||||
uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5);
|
||||
*(ulittle32_t *)FixupPtr = FixedInstr;
|
||||
break;
|
||||
}
|
||||
case PageOffset12: {
|
||||
uint64_t TargetOffset =
|
||||
(E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff;
|
||||
|
||||
uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
|
||||
unsigned ImmShift = getPageOffset12Shift(RawInstr);
|
||||
|
||||
if (TargetOffset & ((1 << ImmShift) - 1))
|
||||
return make_error<JITLinkError>("PAGEOFF12 target is not aligned");
|
||||
|
||||
uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;
|
||||
uint32_t FixedInstr = RawInstr | EncodedImm;
|
||||
*(ulittle32_t *)FixupPtr = FixedInstr;
|
||||
break;
|
||||
}
|
||||
case LDRLiteral19: {
|
||||
assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned");
|
||||
assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend");
|
||||
uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
|
||||
assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal");
|
||||
int64_t Delta = E.getTarget().getAddress() - FixupAddress;
|
||||
if (Delta & 0x3)
|
||||
return make_error<JITLinkError>("LDR literal target is not 32-bit "
|
||||
"aligned");
|
||||
if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1))
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
|
||||
uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5;
|
||||
uint32_t FixedInstr = RawInstr | EncodedImm;
|
||||
*(ulittle32_t *)FixupPtr = FixedInstr;
|
||||
break;
|
||||
}
|
||||
case Delta32:
|
||||
case Delta64:
|
||||
case NegDelta32:
|
||||
case NegDelta64: {
|
||||
int64_t Value;
|
||||
if (E.getKind() == Delta32 || E.getKind() == Delta64)
|
||||
Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
|
||||
else
|
||||
Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
|
||||
|
||||
if (E.getKind() == Delta32 || E.getKind() == NegDelta32) {
|
||||
if (Value < std::numeric_limits<int32_t>::min() ||
|
||||
Value > std::numeric_limits<int32_t>::max())
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
*(little32_t *)FixupPtr = Value;
|
||||
} else
|
||||
*(little64_t *)FixupPtr = Value;
|
||||
break;
|
||||
}
|
||||
case TLVPage21:
|
||||
case GOTPage21:
|
||||
case TLVPageOffset12:
|
||||
case GOTPageOffset12:
|
||||
case PointerToGOT: {
|
||||
return make_error<JITLinkError>(
|
||||
"In graph " + G.getName() + ", section " + B.getSection().getName() +
|
||||
"GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind()));
|
||||
}
|
||||
default:
|
||||
return make_error<JITLinkError>(
|
||||
"In graph " + G.getName() + ", section " + B.getSection().getName() +
|
||||
"unsupported edge kind" + getEdgeKindName(E.getKind()));
|
||||
}
|
||||
|
||||
return Error::success();
|
||||
}
|
||||
|
||||
/// AArch64 null pointer content.
|
||||
extern const uint8_t NullGOTEntryContent[8];
|
||||
|
|
|
|||
|
|
@ -18,159 +18,6 @@ namespace llvm {
|
|||
namespace jitlink {
|
||||
namespace aarch64 {
|
||||
|
||||
bool isLoadStoreImm12(uint32_t Instr) {
|
||||
constexpr uint32_t LoadStoreImm12Mask = 0x3b000000;
|
||||
return (Instr & LoadStoreImm12Mask) == 0x39000000;
|
||||
}
|
||||
|
||||
unsigned getPageOffset12Shift(uint32_t Instr) {
|
||||
constexpr uint32_t Vec128Mask = 0x04800000;
|
||||
|
||||
if (isLoadStoreImm12(Instr)) {
|
||||
uint32_t ImplicitShift = Instr >> 30;
|
||||
if (ImplicitShift == 0)
|
||||
if ((Instr & Vec128Mask) == Vec128Mask)
|
||||
ImplicitShift = 4;
|
||||
|
||||
return ImplicitShift;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
|
||||
using namespace support;
|
||||
|
||||
char *BlockWorkingMem = B.getAlreadyMutableContent().data();
|
||||
char *FixupPtr = BlockWorkingMem + E.getOffset();
|
||||
orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset();
|
||||
|
||||
switch (E.getKind()) {
|
||||
case Branch26: {
|
||||
assert((FixupAddress.getValue() & 0x3) == 0 &&
|
||||
"Branch-inst is not 32-bit aligned");
|
||||
|
||||
int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
|
||||
|
||||
if (static_cast<uint64_t>(Value) & 0x3)
|
||||
return make_error<JITLinkError>("Branch26 target is not 32-bit "
|
||||
"aligned");
|
||||
|
||||
if (Value < -(1 << 27) || Value > ((1 << 27) - 1))
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
|
||||
uint32_t RawInstr = *(little32_t *)FixupPtr;
|
||||
assert((RawInstr & 0x7fffffff) == 0x14000000 &&
|
||||
"RawInstr isn't a B or BR immediate instruction");
|
||||
uint32_t Imm = (static_cast<uint32_t>(Value) & ((1 << 28) - 1)) >> 2;
|
||||
uint32_t FixedInstr = RawInstr | Imm;
|
||||
*(little32_t *)FixupPtr = FixedInstr;
|
||||
break;
|
||||
}
|
||||
case Pointer32: {
|
||||
uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
|
||||
if (Value > std::numeric_limits<uint32_t>::max())
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
*(ulittle32_t *)FixupPtr = Value;
|
||||
break;
|
||||
}
|
||||
case Pointer64:
|
||||
case Pointer64Anon: {
|
||||
uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
|
||||
*(ulittle64_t *)FixupPtr = Value;
|
||||
break;
|
||||
}
|
||||
case Page21: {
|
||||
assert((E.getKind() != GOTPage21 || E.getAddend() == 0) &&
|
||||
"GOTPAGE21 with non-zero addend");
|
||||
uint64_t TargetPage =
|
||||
(E.getTarget().getAddress().getValue() + E.getAddend()) &
|
||||
~static_cast<uint64_t>(4096 - 1);
|
||||
uint64_t PCPage =
|
||||
FixupAddress.getValue() & ~static_cast<uint64_t>(4096 - 1);
|
||||
|
||||
int64_t PageDelta = TargetPage - PCPage;
|
||||
if (!isInt<33>(PageDelta))
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
|
||||
uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
|
||||
assert((RawInstr & 0xffffffe0) == 0x90000000 &&
|
||||
"RawInstr isn't an ADRP instruction");
|
||||
uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3;
|
||||
uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff;
|
||||
uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5);
|
||||
*(ulittle32_t *)FixupPtr = FixedInstr;
|
||||
break;
|
||||
}
|
||||
case PageOffset12: {
|
||||
uint64_t TargetOffset =
|
||||
(E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff;
|
||||
|
||||
uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
|
||||
unsigned ImmShift = getPageOffset12Shift(RawInstr);
|
||||
|
||||
if (TargetOffset & ((1 << ImmShift) - 1))
|
||||
return make_error<JITLinkError>("PAGEOFF12 target is not aligned");
|
||||
|
||||
uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;
|
||||
uint32_t FixedInstr = RawInstr | EncodedImm;
|
||||
*(ulittle32_t *)FixupPtr = FixedInstr;
|
||||
break;
|
||||
}
|
||||
case LDRLiteral19: {
|
||||
assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned");
|
||||
assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend");
|
||||
uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
|
||||
assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal");
|
||||
int64_t Delta = E.getTarget().getAddress() - FixupAddress;
|
||||
if (Delta & 0x3)
|
||||
return make_error<JITLinkError>("LDR literal target is not 32-bit "
|
||||
"aligned");
|
||||
if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1))
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
|
||||
uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5;
|
||||
uint32_t FixedInstr = RawInstr | EncodedImm;
|
||||
*(ulittle32_t *)FixupPtr = FixedInstr;
|
||||
break;
|
||||
}
|
||||
case Delta32:
|
||||
case Delta64:
|
||||
case NegDelta32:
|
||||
case NegDelta64: {
|
||||
int64_t Value;
|
||||
if (E.getKind() == Delta32 || E.getKind() == Delta64)
|
||||
Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
|
||||
else
|
||||
Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
|
||||
|
||||
if (E.getKind() == Delta32 || E.getKind() == NegDelta32) {
|
||||
if (Value < std::numeric_limits<int32_t>::min() ||
|
||||
Value > std::numeric_limits<int32_t>::max())
|
||||
return makeTargetOutOfRangeError(G, B, E);
|
||||
*(little32_t *)FixupPtr = Value;
|
||||
} else
|
||||
*(little64_t *)FixupPtr = Value;
|
||||
break;
|
||||
}
|
||||
case TLVPage21:
|
||||
case GOTPage21:
|
||||
case TLVPageOffset12:
|
||||
case GOTPageOffset12:
|
||||
case PointerToGOT: {
|
||||
return make_error<JITLinkError>(
|
||||
"In graph " + G.getName() + ", section " + B.getSection().getName() +
|
||||
"GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind()));
|
||||
}
|
||||
default:
|
||||
return make_error<JITLinkError>(
|
||||
"In graph " + G.getName() + ", section " + B.getSection().getName() +
|
||||
"unsupported edge kind" + getEdgeKindName(E.getKind()));
|
||||
}
|
||||
|
||||
return Error::success();
|
||||
}
|
||||
|
||||
const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue