From 3a12a6f36cc46f07cd44d85b1dc1bc26f78f2156 Mon Sep 17 00:00:00 2001 From: Kamil Rytarowski Date: Thu, 10 Aug 2017 01:43:54 +0200 Subject: [PATCH] jit1 --- .../llvm/ExecutionEngine/Orc/IndirectionUtils.h | 16 +- .../ExecutionEngine/Orc/OrcRemoteTargetClient.h | 22 +- .../ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h | 3 +- .../ExecutionEngine/Orc/OrcRemoteTargetServer.h | 34 +-- .../llvm/ExecutionEngine/SectionMemoryManager.h | 11 +- include/llvm/Support/Memory.h | 68 +----- lib/ExecutionEngine/Orc/OrcABISupport.cpp | 12 +- lib/ExecutionEngine/SectionMemoryManager.cpp | 30 +-- lib/Support/Unix/Memory.inc | 269 +++++---------------- tools/lli/lli.cpp | 3 - tools/llvm-rtdyld/llvm-rtdyld.cpp | 29 ++- 11 files changed, 146 insertions(+), 351 deletions(-) diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index e038093d762..950628c8f10 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -167,15 +167,15 @@ public: std::error_code EC; ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( TargetT::ResolverCodeSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + sys::Memory::ProtectionMode::MF_WRITABLE, EC)); assert(!EC && "Failed to allocate resolver block"); TargetT::writeResolverCode(static_cast(ResolverBlock.base()), &reenter, this); - EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), - sys::Memory::MF_READ | - sys::Memory::MF_EXEC); + EC = sys::Memory::protectMappedMemory( + ResolverBlock.getMemoryBlock(), + sys::Memory::ProtectionMode::MF_EXECUTABLE); assert(!EC && "Failed to mprotect resolver block"); } @@ -195,7 +195,7 @@ private: auto TrampolineBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( sys::Process::getPageSize(), nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + sys::Memory::ProtectionMode::MF_WRITABLE, EC)); assert(!EC && "Failed to allocate trampoline block"); unsigned NumTrampolines = @@ -211,9 +211,9 @@ private: static_cast(reinterpret_cast( TrampolineMem + (I * TargetT::TrampolineSize)))); - EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(), - sys::Memory::MF_READ | - sys::Memory::MF_EXEC); + EC = sys::Memory::protectMappedMemory( + TrampolineBlock.getMemoryBlock(), + sys::Memory::ProtectionMode::MF_EXECUTABLE); assert(!EC && "Failed to mprotect trampoline block"); TrampolineBlocks.push_back(std::move(TrampolineBlock)); diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h index da02250ba16..f8d09e76169 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -248,9 +248,9 @@ public: if (ObjAllocs.RemoteCodeAddr) { DEBUG(dbgs() << " setting R-X permissions on code block: " << format("0x%016x", ObjAllocs.RemoteCodeAddr) << "\n"); - if (auto Err = Client.setProtections(Id, ObjAllocs.RemoteCodeAddr, - sys::Memory::MF_READ | - sys::Memory::MF_EXEC)) { + if (auto Err = Client.setProtections( + Id, ObjAllocs.RemoteCodeAddr, + sys::Memory::ProtectionMode::MF_EXECUTABLE)) { // FIXME: Replace this once finalizeMemory can return an Error. handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { if (ErrMsg) { @@ -285,8 +285,9 @@ public: DEBUG(dbgs() << " setting R-- permissions on ro-data block: " << format("0x%016x", ObjAllocs.RemoteRODataAddr) << "\n"); - if (auto Err = Client.setProtections(Id, ObjAllocs.RemoteRODataAddr, - sys::Memory::MF_READ)) { + if (auto Err = Client.setProtections( + Id, ObjAllocs.RemoteRODataAddr, + sys::Memory::ProtectionMode::MF_READABLE)) { // FIXME: Replace this once finalizeMemory can return an Error. handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { if (ErrMsg) { @@ -321,9 +322,9 @@ public: DEBUG(dbgs() << " setting RW- permissions on rw-data block: " << format("0x%016x", ObjAllocs.RemoteRWDataAddr) << "\n"); - if (auto Err = Client.setProtections(Id, ObjAllocs.RemoteRWDataAddr, - sys::Memory::MF_READ | - sys::Memory::MF_WRITE)) { + if (auto Err = Client.setProtections( + Id, ObjAllocs.RemoteRWDataAddr, + sys::Memory::ProtectionMode::MF_WRITABLE)) { // FIXME: Replace this once finalizeMemory can return an Error. handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { if (ErrMsg) { @@ -736,8 +737,9 @@ private: } Error setProtections(ResourceIdMgr::ResourceId Id, - JITTargetAddress RemoteSegAddr, unsigned ProtFlags) { - return callB(Id, RemoteSegAddr, ProtFlags); + JITTargetAddress RemoteSegAddr, + sys::Memory::ProtectionMode Mode) { + return callB(Id, RemoteSegAddr, Mode); } Error writeMem(JITTargetAddress Addr, const char *Src, uint64_t Size) { diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h index 07ae7f04d1a..3392985e51e 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h @@ -239,7 +239,8 @@ public: class SetProtections : public rpc::Function { + JITTargetAddress Dst, + sys::Memory::ProtectionMode Mode)> { public: static const char *getName() { return "SetProtections"; } }; diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h index e7b6d64931b..e9076b4cfa1 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h @@ -117,7 +117,7 @@ private: Error allocate(void *&Addr, size_t Size, uint32_t Align) { std::error_code EC; sys::MemoryBlock MB = sys::Memory::allocateMappedMemory( - Size, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); + Size, nullptr, sys::Memory::ProtectionMode::MF_WRITABLE, EC); if (EC) return errorCodeToError(EC); @@ -127,12 +127,12 @@ private: return Error::success(); } - Error setProtections(void *block, unsigned Flags) { + Error setProtections(void *block, sys::Memory::ProtectionMode Mode) { auto I = Allocs.find(block); if (I == Allocs.end()) return errorCodeToError(orcError(OrcErrorCode::RemoteMProtectAddrUnrecognized)); return errorCodeToError( - sys::Memory::protectMappedMemory(I->second, Flags)); + sys::Memory::protectMappedMemory(I->second, Mode)); } private: @@ -280,7 +280,7 @@ private: std::error_code EC; ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( TargetT::ResolverCodeSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + sys::Memory::ProtectionMode::MF_WRITABLE, EC)); if (EC) return errorCodeToError(EC); @@ -289,7 +289,7 @@ private: return errorCodeToError(sys::Memory::protectMappedMemory( ResolverBlock.getMemoryBlock(), - sys::Memory::MF_READ | sys::Memory::MF_EXEC)); + sys::Memory::ProtectionMode::MF_EXECUTABLE)); } Expected> handleEmitTrampolineBlock() { @@ -297,7 +297,7 @@ private: auto TrampolineBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( sys::Process::getPageSize(), nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + sys::Memory::ProtectionMode::MF_WRITABLE, EC)); if (EC) return errorCodeToError(EC); @@ -309,9 +309,9 @@ private: TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(), NumTrampolines); - EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(), - sys::Memory::MF_READ | - sys::Memory::MF_EXEC); + EC = sys::Memory::protectMappedMemory( + TrampolineBlock.getMemoryBlock(), + sys::Memory::ProtectionMode::MF_EXECUTABLE); TrampolineBlocks.push_back(std::move(TrampolineBlock)); @@ -389,18 +389,22 @@ private: } Error handleSetProtections(ResourceIdMgr::ResourceId Id, - JITTargetAddress Addr, uint32_t Flags) { + JITTargetAddress Addr, + sys::Memory::ProtectionMode Mode) { auto I = Allocators.find(Id); if (I == Allocators.end()) return errorCodeToError( orcError(OrcErrorCode::RemoteAllocatorDoesNotExist)); auto &Allocator = I->second; void *LocalAddr = reinterpret_cast(static_cast(Addr)); - DEBUG(dbgs() << " Allocator " << Id << " set permissions on " << LocalAddr - << " to " << (Flags & sys::Memory::MF_READ ? 'R' : '-') - << (Flags & sys::Memory::MF_WRITE ? 'W' : '-') - << (Flags & sys::Memory::MF_EXEC ? 'X' : '-') << "\n"); - return Allocator.setProtections(LocalAddr, Flags); + DEBUG(dbgs() + << " Allocator " << Id << " set permissions on " << LocalAddr + << " to " + << (Mode == sys::Memory::ProtectionMode::MF_READABLE ? "R--" : "") + << (Mode == sys::Memory::ProtectionMode::MF_WRITABLE ? "RW-" : "") + << (Mode == sys::Memory::ProtectionMode::MF_READABLE ? "R-X" : "") + << "\n"); + return Allocator.setProtections(LocalAddr, Mode); } Error handleTerminateSession() { diff --git a/include/llvm/ExecutionEngine/SectionMemoryManager.h b/include/llvm/ExecutionEngine/SectionMemoryManager.h index 3b2af11cdaf..aa094028719 100644 --- a/include/llvm/ExecutionEngine/SectionMemoryManager.h +++ b/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -76,15 +76,6 @@ public: /// \returns true if an error occurred, false otherwise. bool finalizeMemory(std::string *ErrMsg = nullptr) override; - /// \brief Invalidate instruction cache for code sections. - /// - /// Some platforms with separate data cache and instruction cache require - /// explicit cache flush, otherwise JIT code manipulations (like resolved - /// relocations) will get to the data cache but not to the instruction cache. - /// - /// This method is called from finalizeMemory. - virtual void invalidateInstructionCache(); - private: struct FreeMemBlock { // The actual block of free memory @@ -114,7 +105,7 @@ private: unsigned Alignment); std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, - unsigned Permissions); + sys::Memory::ProtectionMode Mode); MemoryGroup CodeMem; MemoryGroup RWDataMem; diff --git a/include/llvm/Support/Memory.h b/include/llvm/Support/Memory.h index 8103aea2fa2..5d4ff32457c 100644 --- a/include/llvm/Support/Memory.h +++ b/include/llvm/Support/Memory.h @@ -45,10 +45,10 @@ namespace sys { /// @brief An abstraction for memory operations. class Memory { public: - enum ProtectionFlags { - MF_READ = 0x1000000, - MF_WRITE = 0x2000000, - MF_EXEC = 0x4000000 + enum class ProtectionMode { + MF_READABLE = 0x1000000, ///< (R--) + MF_WRITABLE = 0x2000000, ///< (RW-) + MF_EXECUTABLE = 0x4000000 ///< (R-X) }; /// This method allocates a block of memory that is suitable for loading @@ -58,7 +58,7 @@ namespace sys { /// an attempt is made to allocate more memory near the existing block. /// The actual allocated address is not guaranteed to be near the requested /// address. - /// \p Flags is used to set the initial protection flags for the block + /// \p Mode is used to set the initial protection mode for the block /// of the memory. /// \p EC [out] returns an object describing any error that occurs. /// @@ -77,7 +77,7 @@ namespace sys { /// @brief Allocate mapped memory. static MemoryBlock allocateMappedMemory(size_t NumBytes, const MemoryBlock *const NearBlock, - unsigned Flags, + ProtectionMode Mode, std::error_code &EC); /// This method releases a block of memory that was allocated with the @@ -92,68 +92,18 @@ namespace sys { static std::error_code releaseMappedMemory(MemoryBlock &Block); /// This method sets the protection flags for a block of memory to the - /// state specified by /p Flags. The behavior is not specified if the + /// state specified by /p Mode. The behavior is not specified if the /// memory was not allocated using the allocateMappedMemory method. /// \p Block describes the memory block to be protected. - /// \p Flags specifies the new protection state to be assigned to the block. + /// \p Mode specifies the new protection state to be assigned to the block. /// \p ErrMsg [out] returns a string describing any error that occurred. /// - /// If \p Flags is MF_WRITE, the actual behavior varies - /// with the operating system (i.e. MF_READ | MF_WRITE on Windows) and the - /// target architecture (i.e. MF_WRITE -> MF_READ | MF_WRITE on i386). - /// /// \r error_success if the function was successful, or an error_code /// describing the failure if an error occurred. /// /// @brief Set memory protection state. static std::error_code protectMappedMemory(const MemoryBlock &Block, - unsigned Flags); - - /// This method allocates a block of Read/Write/Execute memory that is - /// suitable for executing dynamically generated code (e.g. JIT). An - /// attempt to allocate \p NumBytes bytes of virtual memory is made. - /// \p NearBlock may point to an existing allocation in which case - /// an attempt is made to allocate more memory near the existing block. - /// - /// On success, this returns a non-null memory block, otherwise it returns - /// a null memory block and fills in *ErrMsg. - /// - /// @brief Allocate Read/Write/Execute memory. - static MemoryBlock AllocateRWX(size_t NumBytes, - const MemoryBlock *NearBlock, - std::string *ErrMsg = nullptr); - - /// This method releases a block of Read/Write/Execute memory that was - /// allocated with the AllocateRWX method. It should not be used to - /// release any memory block allocated any other way. - /// - /// On success, this returns false, otherwise it returns true and fills - /// in *ErrMsg. - /// @brief Release Read/Write/Execute memory. - static bool ReleaseRWX(MemoryBlock &block, std::string *ErrMsg = nullptr); - - /// InvalidateInstructionCache - Before the JIT can run a block of code - /// that has been emitted it must invalidate the instruction cache on some - /// platforms. - static void InvalidateInstructionCache(const void *Addr, size_t Len); - - /// setExecutable - Before the JIT can run a block of code, it has to be - /// given read and executable privilege. Return true if it is already r-x - /// or the system is able to change its previlege. - static bool setExecutable(MemoryBlock &M, std::string *ErrMsg = nullptr); - - /// setWritable - When adding to a block of code, the JIT may need - /// to mark a block of code as RW since the protections are on page - /// boundaries, and the JIT internal allocations are not page aligned. - static bool setWritable(MemoryBlock &M, std::string *ErrMsg = nullptr); - - /// setRangeExecutable - Mark the page containing a range of addresses - /// as executable. - static bool setRangeExecutable(const void *Addr, size_t Size); - - /// setRangeWritable - Mark the page containing a range of addresses - /// as writable. - static bool setRangeWritable(const void *Addr, size_t Size); + ProtectionMode Mode); }; /// Owning version of MemoryBlock. diff --git a/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/lib/ExecutionEngine/Orc/OrcABISupport.cpp index 9869b6c7050..334ff53c32e 100644 --- a/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -157,7 +157,7 @@ Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, std::error_code EC; auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 2 * NumPages * PageSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + sys::Memory::ProtectionMode::MF_WRITABLE, EC)); if (EC) return errorCodeToError(EC); @@ -177,7 +177,7 @@ Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, Stub[I] = 0xd61f020058000010 | PtrOffsetField; if (auto EC = sys::Memory::protectMappedMemory( - StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + StubsBlock, sys::Memory::ProtectionMode::MF_EXECUTABLE)) return errorCodeToError(EC); // Initialize all pointers to point at FailureAddress. @@ -239,7 +239,7 @@ Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, std::error_code EC; auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 2 * NumPages * PageSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + sys::Memory::ProtectionMode::MF_WRITABLE, EC)); if (EC) return errorCodeToError(EC); @@ -258,7 +258,7 @@ Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, Stub[I] = 0xF1C40000000025ff | PtrOffsetField; if (auto EC = sys::Memory::protectMappedMemory( - StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + StubsBlock, sys::Memory::ProtectionMode::MF_EXECUTABLE)) return errorCodeToError(EC); // Initialize all pointers to point at FailureAddress. @@ -507,7 +507,7 @@ Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, std::error_code EC; auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 2 * NumPages * PageSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + sys::Memory::ProtectionMode::MF_WRITABLE, EC)); if (EC) return errorCodeToError(EC); @@ -525,7 +525,7 @@ Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, Stub[I] = 0xF1C40000000025ff | (PtrAddr << 16); if (auto EC = sys::Memory::protectMappedMemory( - StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + StubsBlock, sys::Memory::ProtectionMode::MF_EXECUTABLE)) return errorCodeToError(EC); // Initialize all pointers to point at FailureAddress. diff --git a/lib/ExecutionEngine/SectionMemoryManager.cpp b/lib/ExecutionEngine/SectionMemoryManager.cpp index 8904475f084..36a2bd53e7c 100644 --- a/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -84,11 +84,9 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // FIXME: Initialize the Near member for each memory group to avoid // interleaving. std::error_code ec; - sys::MemoryBlock MB = sys::Memory::allocateMappedMemory(RequiredSize, - &MemGroup.Near, - sys::Memory::MF_READ | - sys::Memory::MF_WRITE, - ec); + sys::MemoryBlock MB = sys::Memory::allocateMappedMemory( + RequiredSize, &MemGroup.Near, sys::Memory::ProtectionMode::MF_WRITABLE, + ec); if (ec) { // FIXME: Add error propagation to the interface. return nullptr; @@ -129,7 +127,7 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) // Make code memory executable. ec = applyMemoryGroupPermissions(CodeMem, - sys::Memory::MF_READ | sys::Memory::MF_EXEC); + sys::Memory::ProtectionMode::MF_EXECUTABLE); if (ec) { if (ErrMsg) { *ErrMsg = ec.message(); @@ -139,7 +137,7 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) // Make read-only data memory read-only. ec = applyMemoryGroupPermissions(RODataMem, - sys::Memory::MF_READ | sys::Memory::MF_EXEC); + sys::Memory::ProtectionMode::MF_READABLE); if (ec) { if (ErrMsg) { *ErrMsg = ec.message(); @@ -149,11 +147,6 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) // Read-write data memory already has the correct permissions - // Some platforms with separate data cache and instruction cache require - // explicit cache flush, otherwise JIT code manipulations (like resolved - // relocations) will get to the data cache but not to the instruction cache. - invalidateInstructionCache(); - return false; } @@ -176,12 +169,10 @@ static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { return Trimmed; } - -std::error_code -SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, - unsigned Permissions) { +std::error_code SectionMemoryManager::applyMemoryGroupPermissions( + MemoryGroup &MemGroup, sys::Memory::ProtectionMode Mode) { for (sys::MemoryBlock &MB : MemGroup.PendingMem) - if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions)) + if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Mode)) return EC; MemGroup.PendingMem.clear(); @@ -203,11 +194,6 @@ SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, return std::error_code(); } -void SectionMemoryManager::invalidateInstructionCache() { - for (sys::MemoryBlock &Block : CodeMem.PendingMem) - sys::Memory::InvalidateInstructionCache(Block.base(), Block.size()); -} - SectionMemoryManager::~SectionMemoryManager() { for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { for (sys::MemoryBlock &Block : Group->AllocatedMem) diff --git a/lib/Support/Unix/Memory.inc b/lib/Support/Unix/Memory.inc index dd39ef935bf..5f9038e4ed5 100644 --- a/lib/Support/Unix/Memory.inc +++ b/lib/Support/Unix/Memory.inc @@ -40,39 +40,64 @@ extern "C" void __clear_cache(void *, void*); namespace { -int getPosixProtectionFlags(unsigned Flags) { - switch (Flags) { - case llvm::sys::Memory::MF_READ: +int getPosixProtectionFlags(llvm::sys::Memory::ProtectionMode Mode) { + switch (Mode) { + case llvm::sys::Memory::ProtectionMode::MF_READABLE: return PROT_READ; - case llvm::sys::Memory::MF_WRITE: - return PROT_WRITE; - case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_WRITE: + case llvm::sys::Memory::ProtectionMode::MF_WRITABLE: return PROT_READ | PROT_WRITE; - case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_EXEC: + case llvm::sys::Memory::ProtectionMode::MF_EXECUTABLE: return PROT_READ | PROT_EXEC; - case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE | - llvm::sys::Memory::MF_EXEC: - return PROT_READ | PROT_WRITE | PROT_EXEC; - case llvm::sys::Memory::MF_EXEC: -#if defined(__FreeBSD__) - // On PowerPC, having an executable page that has no read permission - // can have unintended consequences. The function InvalidateInstruction- - // Cache uses instructions dcbf and icbi, both of which are treated by - // the processor as loads. If the page has no read permissions, - // executing these instructions will result in a segmentation fault. - // Somehow, this problem is not present on Linux, but it does happen - // on FreeBSD. - return PROT_READ | PROT_EXEC; -#else - return PROT_EXEC; -#endif - default: - llvm_unreachable("Illegal memory protection flag specified!"); } // Provide a default return value as required by some compilers. return PROT_NONE; } +/// InvalidateInstructionCache - Before the JIT can run a block of code +/// that has been emitted it must invalidate the instruction cache on some +/// platforms. +void InvalidateInstructionCache(const void *Addr, + size_t Len) { + +// icache invalidation for PPC and ARM. +#if defined(__APPLE__) + +# if (defined(__POWERPC__) || defined (__ppc__) || \ + defined(_POWER) || defined(_ARCH_PPC) || defined(__arm__) || \ + defined(__arm64__)) + sys_icache_invalidate(const_cast(Addr), Len); +# endif + +#else + +# if (defined(__POWERPC__) || defined (__ppc__) || \ + defined(_POWER) || defined(_ARCH_PPC)) && defined(__GNUC__) + const size_t LineSize = 32; + + const intptr_t Mask = ~(LineSize - 1); + const intptr_t StartLine = ((intptr_t) Addr) & Mask; + const intptr_t EndLine = ((intptr_t) Addr + Len + LineSize - 1) & Mask; + + for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize) + asm volatile("dcbf 0, %0" : : "r"(Line)); + asm volatile("sync"); + + for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize) + asm volatile("icbi 0, %0" : : "r"(Line)); + asm volatile("isync"); +# elif (defined(__arm__) || defined(__aarch64__) || defined(__mips__)) && \ + defined(__GNUC__) + // FIXME: Can we safely always call this for __GNUC__ everywhere? + const char *Start = static_cast(Addr); + const char *End = Start + Len; + __clear_cache(const_cast(Start), const_cast(End)); +# endif + +#endif // end apple + + llvm::sys::ValgrindDiscardTranslations(Addr, Len); +} + } // anonymous namespace namespace llvm { @@ -81,7 +106,7 @@ namespace sys { MemoryBlock Memory::allocateMappedMemory(size_t NumBytes, const MemoryBlock *const NearBlock, - unsigned PFlags, + ProtectionMode Mode, std::error_code &EC) { EC = std::error_code(); if (NumBytes == 0) @@ -100,7 +125,12 @@ Memory::allocateMappedMemory(size_t NumBytes, #endif ; // Ends statement above - int Protect = getPosixProtectionFlags(PFlags); + int Protect = getPosixProtectionFlags(Mode); + + // Allow swapping memory protection in future on NetBSD with PaX MPROTECT +#if defined(__NetBSD__) && defined(PROT_MPROTECT) + Protect |= PROT_MPROTECT(PROT_READ | PROT_WRITE | PROT_EXEC); +#endif // Use any near hint and the page size to set a page-aligned starting address uintptr_t Start = NearBlock ? reinterpret_cast(NearBlock->base()) + @@ -112,7 +142,7 @@ Memory::allocateMappedMemory(size_t NumBytes, Protect, MMFlags, fd, 0); if (Addr == MAP_FAILED) { if (NearBlock) //Try again without a near hint - return allocateMappedMemory(NumBytes, nullptr, PFlags, EC); + return allocateMappedMemory(NumBytes, nullptr, Mode, EC); EC = std::error_code(errno, std::generic_category()); return MemoryBlock(); @@ -122,8 +152,8 @@ Memory::allocateMappedMemory(size_t NumBytes, Result.Address = Addr; Result.Size = NumPages*PageSize; - if (PFlags & MF_EXEC) - Memory::InvalidateInstructionCache(Result.Address, Result.Size); + if (Mode == ProtectionMode::MF_EXECUTABLE) + InvalidateInstructionCache(Result.Address, Result.Size); return Result; } @@ -143,15 +173,12 @@ Memory::releaseMappedMemory(MemoryBlock &M) { } std::error_code -Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { +Memory::protectMappedMemory(const MemoryBlock &M, ProtectionMode Mode) { static const size_t PageSize = Process::getPageSize(); if (M.Address == nullptr || M.Size == 0) return std::error_code(); - if (!Flags) - return std::error_code(EINVAL, std::generic_category()); - - int Protect = getPosixProtectionFlags(Flags); + int Protect = getPosixProtectionFlags(Mode); uintptr_t Start = alignAddr((uint8_t *)M.Address - PageSize + 1, PageSize); uintptr_t End = alignAddr((uint8_t *)M.Address + M.Size, PageSize); @@ -160,179 +187,11 @@ Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { if (Result != 0) return std::error_code(errno, std::generic_category()); - if (Flags & MF_EXEC) - Memory::InvalidateInstructionCache(M.Address, M.Size); + if (Mode == ProtectionMode::MF_EXECUTABLE) + InvalidateInstructionCache(M.Address, M.Size); return std::error_code(); } -/// AllocateRWX - Allocate a slab of memory with read/write/execute -/// permissions. This is typically used for JIT applications where we want -/// to emit code to the memory then jump to it. Getting this type of memory -/// is very OS specific. -/// -MemoryBlock -Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock, - std::string *ErrMsg) { - if (NumBytes == 0) return MemoryBlock(); - - static const size_t PageSize = Process::getPageSize(); - size_t NumPages = (NumBytes+PageSize-1)/PageSize; - - int fd = -1; - - int flags = MAP_PRIVATE | -#ifdef MAP_ANONYMOUS - MAP_ANONYMOUS -#else - MAP_ANON -#endif - ; - - void* start = NearBlock ? (unsigned char*)NearBlock->base() + - NearBlock->size() : nullptr; - -#if defined(__APPLE__) && (defined(__arm__) || defined(__arm64__)) - void *pa = ::mmap(start, PageSize*NumPages, PROT_READ|PROT_EXEC, - flags, fd, 0); -#elif defined(__NetBSD__) && defined(PROT_MPROTECT) - void *pa = - ::mmap(start, PageSize * NumPages, - PROT_READ | PROT_WRITE | PROT_MPROTECT(PROT_EXEC), flags, fd, 0); -#else - void *pa = ::mmap(start, PageSize*NumPages, PROT_READ|PROT_WRITE|PROT_EXEC, - flags, fd, 0); -#endif - if (pa == MAP_FAILED) { - if (NearBlock) //Try again without a near hint - return AllocateRWX(NumBytes, nullptr); - - MakeErrMsg(ErrMsg, "Can't allocate RWX Memory"); - return MemoryBlock(); - } - -#if defined(__APPLE__) && (defined(__arm__) || defined(__arm64__)) - kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)pa, - (vm_size_t)(PageSize*NumPages), 0, - VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY); - if (KERN_SUCCESS != kr) { - MakeErrMsg(ErrMsg, "vm_protect max RX failed"); - return MemoryBlock(); - } - - kr = vm_protect(mach_task_self(), (vm_address_t)pa, - (vm_size_t)(PageSize*NumPages), 0, - VM_PROT_READ | VM_PROT_WRITE); - if (KERN_SUCCESS != kr) { - MakeErrMsg(ErrMsg, "vm_protect RW failed"); - return MemoryBlock(); - } -#endif - - MemoryBlock result; - result.Address = pa; - result.Size = NumPages*PageSize; - - return result; -} - -bool Memory::ReleaseRWX(MemoryBlock &M, std::string *ErrMsg) { - if (M.Address == nullptr || M.Size == 0) return false; - if (0 != ::munmap(M.Address, M.Size)) - return MakeErrMsg(ErrMsg, "Can't release RWX Memory"); - return false; -} - -bool Memory::setWritable (MemoryBlock &M, std::string *ErrMsg) { -#if defined(__APPLE__) && (defined(__arm__) || defined(__arm64__)) - if (M.Address == 0 || M.Size == 0) return false; - Memory::InvalidateInstructionCache(M.Address, M.Size); - kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)M.Address, - (vm_size_t)M.Size, 0, VM_PROT_READ | VM_PROT_WRITE); - return KERN_SUCCESS == kr; -#else - return true; -#endif -} - -bool Memory::setExecutable (MemoryBlock &M, std::string *ErrMsg) { - if (M.Address == nullptr || M.Size == 0) return false; - Memory::InvalidateInstructionCache(M.Address, M.Size); -#if defined(__APPLE__) && (defined(__arm__) || defined(__arm64__)) - kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)M.Address, - (vm_size_t)M.Size, 0, VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY); - return KERN_SUCCESS == kr; -#else - return true; -#endif -} - -bool Memory::setRangeWritable(const void *Addr, size_t Size) { -#if defined(__APPLE__) && (defined(__arm__) || defined(__arm64__)) - kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)Addr, - (vm_size_t)Size, 0, - VM_PROT_READ | VM_PROT_WRITE); - return KERN_SUCCESS == kr; -#else - return true; -#endif -} - -bool Memory::setRangeExecutable(const void *Addr, size_t Size) { -#if defined(__APPLE__) && (defined(__arm__) || defined(__arm64__)) - kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)Addr, - (vm_size_t)Size, 0, - VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY); - return KERN_SUCCESS == kr; -#else - return true; -#endif -} - -/// InvalidateInstructionCache - Before the JIT can run a block of code -/// that has been emitted it must invalidate the instruction cache on some -/// platforms. -void Memory::InvalidateInstructionCache(const void *Addr, - size_t Len) { - -// icache invalidation for PPC and ARM. -#if defined(__APPLE__) - -# if (defined(__POWERPC__) || defined (__ppc__) || \ - defined(_POWER) || defined(_ARCH_PPC) || defined(__arm__) || \ - defined(__arm64__)) - sys_icache_invalidate(const_cast(Addr), Len); -# endif - -#else - -# if (defined(__POWERPC__) || defined (__ppc__) || \ - defined(_POWER) || defined(_ARCH_PPC)) && defined(__GNUC__) - const size_t LineSize = 32; - - const intptr_t Mask = ~(LineSize - 1); - const intptr_t StartLine = ((intptr_t) Addr) & Mask; - const intptr_t EndLine = ((intptr_t) Addr + Len + LineSize - 1) & Mask; - - for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize) - asm volatile("dcbf 0, %0" : : "r"(Line)); - asm volatile("sync"); - - for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize) - asm volatile("icbi 0, %0" : : "r"(Line)); - asm volatile("isync"); -# elif (defined(__arm__) || defined(__aarch64__) || defined(__mips__)) && \ - defined(__GNUC__) - // FIXME: Can we safely always call this for __GNUC__ everywhere? - const char *Start = static_cast(Addr); - const char *End = Start + Len; - __clear_cache(const_cast(Start), const_cast(End)); -# endif - -#endif // end apple - - ValgrindDiscardTranslations(Addr, Len); -} - } // namespace sys } // namespace llvm diff --git a/tools/lli/lli.cpp b/tools/lli/lli.cpp index cce549d0a25..a32a81cd35a 100644 --- a/tools/lli/lli.cpp +++ b/tools/lli/lli.cpp @@ -568,9 +568,6 @@ int main(int argc, char **argv, char * const *envp) { // Trigger compilation separately so code regions that need to be // invalidated will be known. (void)EE->getPointerToFunction(EntryFn); - // Clear instruction cache before code will be executed. - if (RTDyldMM) - static_cast(RTDyldMM)->invalidateInstructionCache(); // Run main. Result = EE->runFunctionAsMain(EntryFn, InputArgv, envp); diff --git a/tools/llvm-rtdyld/llvm-rtdyld.cpp b/tools/llvm-rtdyld/llvm-rtdyld.cpp index daafcdfcc95..c154700b870 100644 --- a/tools/llvm-rtdyld/llvm-rtdyld.cpp +++ b/tools/llvm-rtdyld/llvm-rtdyld.cpp @@ -178,10 +178,11 @@ public: void deregisterEHFrames() override {} void preallocateSlab(uint64_t Size) { - std::string Err; - sys::MemoryBlock MB = sys::Memory::AllocateRWX(Size, nullptr, &Err); + std::error_code EC; + sys::MemoryBlock MB = sys::Memory::allocateMappedMemory( + Size, nullptr, sys::Memory::ProtectionMode::MF_WRITABLE, EC); if (!MB.base()) - report_fatal_error("Can't allocate enough memory: " + Err); + report_fatal_error("Can't allocate enough memory"); PreallocSlab = MB; UsePreallocation = true; @@ -222,10 +223,11 @@ uint8_t *TrivialMemoryManager::allocateCodeSection(uintptr_t Size, if (UsePreallocation) return allocateFromSlab(Size, Alignment, true /* isCode */); - std::string Err; - sys::MemoryBlock MB = sys::Memory::AllocateRWX(Size, nullptr, &Err); + std::error_code EC; + sys::MemoryBlock MB = sys::Memory::allocateMappedMemory( + Size, nullptr, sys::Memory::ProtectionMode::MF_WRITABLE, EC); if (!MB.base()) - report_fatal_error("MemoryManager allocation failed: " + Err); + report_fatal_error("MemoryManager allocation failed"); FunctionMemory.push_back(MB); return (uint8_t*)MB.base(); } @@ -242,10 +244,11 @@ uint8_t *TrivialMemoryManager::allocateDataSection(uintptr_t Size, if (UsePreallocation) return allocateFromSlab(Size, Alignment, false /* isCode */); - std::string Err; - sys::MemoryBlock MB = sys::Memory::AllocateRWX(Size, nullptr, &Err); + std::error_code EC; + sys::MemoryBlock MB = sys::Memory::allocateMappedMemory( + Size, nullptr, sys::Memory::ProtectionMode::MF_WRITABLE, EC); if (!MB.base()) - report_fatal_error("MemoryManager allocation failed: " + Err); + report_fatal_error("MemoryManager allocation failed"); DataMemory.push_back(MB); return (uint8_t*)MB.base(); } @@ -453,9 +456,11 @@ static int executeInput() { // Make sure the memory is executable. // setExecutable will call InvalidateInstructionCache. - std::string ErrorStr; - if (!sys::Memory::setExecutable(FM, &ErrorStr)) - ErrorAndExit("unable to mark function executable: '" + ErrorStr + "'"); + std::error_code EC; + EC = sys::Memory::protectMappedMemory( + FM, sys::Memory::ProtectionMode::MF_EXECUTABLE); + if (EC) + ErrorAndExit("unable to mark function executable"); } // Dispatch to _main(). -- 2.13.2