diff options
Diffstat (limited to 'include')
482 files changed, 30176 insertions, 10255 deletions
diff --git a/include/llvm-c/Analysis.h b/include/llvm-c/Analysis.h index f0bdddc50ab7..36dcb89e0e08 100644 --- a/include/llvm-c/Analysis.h +++ b/include/llvm-c/Analysis.h @@ -19,7 +19,7 @@ #ifndef LLVM_C_ANALYSIS_H #define LLVM_C_ANALYSIS_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { diff --git a/include/llvm-c/BitReader.h b/include/llvm-c/BitReader.h index f3b388bc4fb4..d1fc302767ba 100644 --- a/include/llvm-c/BitReader.h +++ b/include/llvm-c/BitReader.h @@ -19,7 +19,7 @@ #ifndef LLVM_C_BITREADER_H #define LLVM_C_BITREADER_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { @@ -34,36 +34,45 @@ extern "C" { /* Builds a module from the bitcode in the specified memory buffer, returning a reference to the module via the OutModule parameter. Returns 0 on success. - Optionally returns a human-readable error message via OutMessage. */ -LLVMBool LLVMParseBitcode(LLVMMemoryBufferRef MemBuf, - LLVMModuleRef *OutModule, char **OutMessage); + Optionally returns a human-readable error message via OutMessage. + This is deprecated. Use LLVMParseBitcode2. */ +LLVMBool LLVMParseBitcode(LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutModule, + char **OutMessage); + +/* Builds a module from the bitcode in the specified memory buffer, returning a + reference to the module via the OutModule parameter. Returns 0 on success. */ +LLVMBool LLVMParseBitcode2(LLVMMemoryBufferRef MemBuf, + LLVMModuleRef *OutModule); + +/* This is deprecated. Use LLVMParseBitcodeInContext2. */ LLVMBool LLVMParseBitcodeInContext(LLVMContextRef ContextRef, LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutModule, char **OutMessage); +LLVMBool LLVMParseBitcodeInContext2(LLVMContextRef ContextRef, + LLVMMemoryBufferRef MemBuf, + LLVMModuleRef *OutModule); + /** Reads a module from the specified path, returning via the OutMP parameter a module provider which performs lazy deserialization. Returns 0 on success. - Optionally returns a human-readable error message via OutMessage. */ + Optionally returns a human-readable error message via OutMessage. + This is deprecated. Use LLVMGetBitcodeModuleInContext2. */ LLVMBool LLVMGetBitcodeModuleInContext(LLVMContextRef ContextRef, LLVMMemoryBufferRef MemBuf, - LLVMModuleRef *OutM, - char **OutMessage); + LLVMModuleRef *OutM, char **OutMessage); +/** Reads a module from the specified path, returning via the OutMP parameter a + * module provider which performs lazy deserialization. Returns 0 on success. */ +LLVMBool LLVMGetBitcodeModuleInContext2(LLVMContextRef ContextRef, + LLVMMemoryBufferRef MemBuf, + LLVMModuleRef *OutM); + +/* This is deprecated. Use LLVMGetBitcodeModule2. */ LLVMBool LLVMGetBitcodeModule(LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutM, char **OutMessage); - -/** Deprecated: Use LLVMGetBitcodeModuleInContext instead. */ -LLVMBool LLVMGetBitcodeModuleProviderInContext(LLVMContextRef ContextRef, - LLVMMemoryBufferRef MemBuf, - LLVMModuleProviderRef *OutMP, - char **OutMessage); - -/** Deprecated: Use LLVMGetBitcodeModule instead. */ -LLVMBool LLVMGetBitcodeModuleProvider(LLVMMemoryBufferRef MemBuf, - LLVMModuleProviderRef *OutMP, - char **OutMessage); +LLVMBool LLVMGetBitcodeModule2(LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutM); /** * @} diff --git a/include/llvm-c/BitWriter.h b/include/llvm-c/BitWriter.h index f25ad3a445f5..797d03179ab3 100644 --- a/include/llvm-c/BitWriter.h +++ b/include/llvm-c/BitWriter.h @@ -19,7 +19,7 @@ #ifndef LLVM_C_BITWRITER_H #define LLVM_C_BITWRITER_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { diff --git a/include/llvm-c/Core.h b/include/llvm-c/Core.h index 9dbcbfea387f..c8fda15c5ed6 100644 --- a/include/llvm-c/Core.h +++ b/include/llvm-c/Core.h @@ -15,7 +15,8 @@ #ifndef LLVM_C_CORE_H #define LLVM_C_CORE_H -#include "llvm-c/Support.h" +#include "llvm-c/ErrorHandling.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { @@ -40,15 +41,6 @@ extern "C" { * the LLVM intermediate representation as well as other related types * and utilities. * - * LLVM uses a polymorphic type hierarchy which C cannot represent, therefore - * parameters must be passed as base types. Despite the declared types, most - * of the functions provided operate only on branches of the type hierarchy. - * The declared parameter names are descriptive and specify which type is - * required. Additionally, each type hierarchy is documented along with the - * functions that operate upon it. For more detail, refer to LLVM's C++ code. - * If in doubt, refer to Core.cpp, which performs parameter downcasts in the - * form unwrap<RequiredType>(Param). - * * Many exotic languages can interoperate with C code but have a harder time * with C++ due to name mangling. So in addition to C, this interface enables * tools written in such languages. @@ -62,74 +54,6 @@ extern "C" { * @{ */ -/* Opaque types. */ - -/** - * The top-level container for all LLVM global data. See the LLVMContext class. - */ -typedef struct LLVMOpaqueContext *LLVMContextRef; - -/** - * The top-level container for all other LLVM Intermediate Representation (IR) - * objects. - * - * @see llvm::Module - */ -typedef struct LLVMOpaqueModule *LLVMModuleRef; - -/** - * Each value in the LLVM IR has a type, an LLVMTypeRef. - * - * @see llvm::Type - */ -typedef struct LLVMOpaqueType *LLVMTypeRef; - -/** - * Represents an individual value in LLVM IR. - * - * This models llvm::Value. - */ -typedef struct LLVMOpaqueValue *LLVMValueRef; - -/** - * Represents a basic block of instructions in LLVM IR. - * - * This models llvm::BasicBlock. - */ -typedef struct LLVMOpaqueBasicBlock *LLVMBasicBlockRef; - -/** - * Represents an LLVM basic block builder. - * - * This models llvm::IRBuilder. - */ -typedef struct LLVMOpaqueBuilder *LLVMBuilderRef; - -/** - * Interface used to provide a module to JIT or interpreter. - * This is now just a synonym for llvm::Module, but we have to keep using the - * different type to keep binary compatibility. - */ -typedef struct LLVMOpaqueModuleProvider *LLVMModuleProviderRef; - -/** @see llvm::PassManagerBase */ -typedef struct LLVMOpaquePassManager *LLVMPassManagerRef; - -/** @see llvm::PassRegistry */ -typedef struct LLVMOpaquePassRegistry *LLVMPassRegistryRef; - -/** - * Used to get the users and usees of a Value. - * - * @see llvm::Use */ -typedef struct LLVMOpaqueUse *LLVMUseRef; - - -/** - * @see llvm::DiagnosticInfo - */ -typedef struct LLVMOpaqueDiagnosticInfo *LLVMDiagnosticInfoRef; - typedef enum { LLVMZExtAttribute = 1<<0, LLVMSExtAttribute = 1<<1, @@ -248,8 +172,12 @@ typedef enum { /* Exception Handling Operators */ LLVMResume = 58, - LLVMLandingPad = 59 - + LLVMLandingPad = 59, + LLVMCleanupRet = 61, + LLVMCatchRet = 62, + LLVMCatchPad = 63, + LLVMCleanupPad = 64, + LLVMCatchSwitch = 65 } LLVMOpcode; typedef enum { @@ -268,7 +196,8 @@ typedef enum { LLVMPointerTypeKind, /**< Pointers */ LLVMVectorTypeKind, /**< SIMD 'packed' format, or other vector type */ LLVMMetadataTypeKind, /**< Metadata */ - LLVMX86_MMXTypeKind /**< X86 MMX */ + LLVMX86_MMXTypeKind, /**< X86 MMX */ + LLVMTokenTypeKind /**< Tokens */ } LLVMTypeKind; typedef enum { @@ -428,36 +357,11 @@ void LLVMInitializeCore(LLVMPassRegistryRef R); @see ManagedStatic */ void LLVMShutdown(void); - /*===-- Error handling ----------------------------------------------------===*/ char *LLVMCreateMessage(const char *Message); void LLVMDisposeMessage(char *Message); -typedef void (*LLVMFatalErrorHandler)(const char *Reason); - -/** - * Install a fatal error handler. By default, if LLVM detects a fatal error, it - * will call exit(1). This may not be appropriate in many contexts. For example, - * doing exit(1) will bypass many crash reporting/tracing system tools. This - * function allows you to install a callback that will be invoked prior to the - * call to exit(1). - */ -void LLVMInstallFatalErrorHandler(LLVMFatalErrorHandler Handler); - -/** - * Reset the fatal error handler. This resets LLVM's fatal error handling - * behavior to the default. - */ -void LLVMResetFatalErrorHandler(void); - -/** - * Enable LLVM's built-in stack trace code. This intercepts the OS's crash - * signals and prints which component of LLVM you were in at the time if the - * crash. - */ -void LLVMEnablePrettyStackTrace(void); - /** * @defgroup LLVMCCoreContext Contexts * @@ -808,6 +712,7 @@ LLVMTypeRef LLVMInt8TypeInContext(LLVMContextRef C); LLVMTypeRef LLVMInt16TypeInContext(LLVMContextRef C); LLVMTypeRef LLVMInt32TypeInContext(LLVMContextRef C); LLVMTypeRef LLVMInt64TypeInContext(LLVMContextRef C); +LLVMTypeRef LLVMInt128TypeInContext(LLVMContextRef C); LLVMTypeRef LLVMIntTypeInContext(LLVMContextRef C, unsigned NumBits); /** @@ -819,6 +724,7 @@ LLVMTypeRef LLVMInt8Type(void); LLVMTypeRef LLVMInt16Type(void); LLVMTypeRef LLVMInt32Type(void); LLVMTypeRef LLVMInt64Type(void); +LLVMTypeRef LLVMInt128Type(void); LLVMTypeRef LLVMIntType(unsigned NumBits); unsigned LLVMGetIntTypeWidth(LLVMTypeRef IntegerTy); @@ -1022,7 +928,6 @@ LLVMBool LLVMIsOpaqueStruct(LLVMTypeRef StructTy); * @} */ - /** * @defgroup LLVMCCoreTypeSequential Sequential Types * @@ -1178,6 +1083,7 @@ LLVMTypeRef LLVMX86MMXType(void); macro(ConstantInt) \ macro(ConstantPointerNull) \ macro(ConstantStruct) \ + macro(ConstantTokenNone) \ macro(ConstantVector) \ macro(GlobalValue) \ macro(GlobalAlias) \ @@ -1215,6 +1121,11 @@ LLVMTypeRef LLVMX86MMXType(void); macro(SwitchInst) \ macro(UnreachableInst) \ macro(ResumeInst) \ + macro(CleanupReturnInst) \ + macro(CatchReturnInst) \ + macro(FuncletPadInst) \ + macro(CatchPadInst) \ + macro(CleanupPadInst) \ macro(UnaryInstruction) \ macro(AllocaInst) \ macro(CastInst) \ @@ -1950,7 +1861,7 @@ void LLVMSetGC(LLVMValueRef Fn, const char *Name); void LLVMAddFunctionAttr(LLVMValueRef Fn, LLVMAttribute PA); /** - * Add a target-dependent attribute to a fuction + * Add a target-dependent attribute to a function * @see llvm::AttrBuilder::addAttribute() */ void LLVMAddTargetDependentFunctionAttr(LLVMValueRef Fn, const char *A, @@ -2427,7 +2338,7 @@ void LLVMInstructionEraseFromParent(LLVMValueRef Inst); * * @see llvm::Instruction::getOpCode() */ -LLVMOpcode LLVMGetInstructionOpcode(LLVMValueRef Inst); +LLVMOpcode LLVMGetInstructionOpcode(LLVMValueRef Inst); /** * Obtain the predicate of an instruction. @@ -2780,6 +2691,8 @@ LLVMValueRef LLVMBuildGlobalStringPtr(LLVMBuilderRef B, const char *Str, const char *Name); LLVMBool LLVMGetVolatile(LLVMValueRef MemoryAccessInst); void LLVMSetVolatile(LLVMValueRef MemoryAccessInst, LLVMBool IsVolatile); +LLVMAtomicOrdering LLVMGetOrdering(LLVMValueRef MemoryAccessInst); +void LLVMSetOrdering(LLVMValueRef MemoryAccessInst, LLVMAtomicOrdering Ordering); /* Casts */ LLVMValueRef LLVMBuildTrunc(LLVMBuilderRef, LLVMValueRef Val, @@ -3020,6 +2933,6 @@ LLVMBool LLVMIsMultithreaded(void); #ifdef __cplusplus } -#endif /* !defined(__cplusplus) */ +#endif -#endif /* !defined(LLVM_C_CORE_H) */ +#endif /* LLVM_C_CORE_H */ diff --git a/include/llvm-c/ErrorHandling.h b/include/llvm-c/ErrorHandling.h new file mode 100644 index 000000000000..5a80bc5e654f --- /dev/null +++ b/include/llvm-c/ErrorHandling.h @@ -0,0 +1,51 @@ +/*===-- llvm-c/ErrorHandling.h - Error Handling C Interface -------*- C -*-===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This file defines the C interface to LLVM's error handling mechanism. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_ERROR_HANDLING_H +#define LLVM_C_ERROR_HANDLING_H + +#include "llvm-c/Types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*LLVMFatalErrorHandler)(const char *Reason); + +/** + * Install a fatal error handler. By default, if LLVM detects a fatal error, it + * will call exit(1). This may not be appropriate in many contexts. For example, + * doing exit(1) will bypass many crash reporting/tracing system tools. This + * function allows you to install a callback that will be invoked prior to the + * call to exit(1). + */ +void LLVMInstallFatalErrorHandler(LLVMFatalErrorHandler Handler); + +/** + * Reset the fatal error handler. This resets LLVM's fatal error handling + * behavior to the default. + */ +void LLVMResetFatalErrorHandler(void); + +/** + * Enable LLVM's built-in stack trace code. This intercepts the OS's crash + * signals and prints which component of LLVM you were in at the time if the + * crash. + */ +void LLVMEnablePrettyStackTrace(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/llvm-c/ExecutionEngine.h b/include/llvm-c/ExecutionEngine.h index eb3ecabfa8a8..b72a91a8b137 100644 --- a/include/llvm-c/ExecutionEngine.h +++ b/include/llvm-c/ExecutionEngine.h @@ -19,7 +19,7 @@ #ifndef LLVM_C_EXECUTIONENGINE_H #define LLVM_C_EXECUTIONENGINE_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #include "llvm-c/Target.h" #include "llvm-c/TargetMachine.h" @@ -110,22 +110,6 @@ LLVMBool LLVMCreateMCJITCompilerForModule( struct LLVMMCJITCompilerOptions *Options, size_t SizeOfOptions, char **OutError); -/** Deprecated: Use LLVMCreateExecutionEngineForModule instead. */ -LLVMBool LLVMCreateExecutionEngine(LLVMExecutionEngineRef *OutEE, - LLVMModuleProviderRef MP, - char **OutError); - -/** Deprecated: Use LLVMCreateInterpreterForModule instead. */ -LLVMBool LLVMCreateInterpreter(LLVMExecutionEngineRef *OutInterp, - LLVMModuleProviderRef MP, - char **OutError); - -/** Deprecated: Use LLVMCreateJITCompilerForModule instead. */ -LLVMBool LLVMCreateJITCompiler(LLVMExecutionEngineRef *OutJIT, - LLVMModuleProviderRef MP, - unsigned OptLevel, - char **OutError); - void LLVMDisposeExecutionEngine(LLVMExecutionEngineRef EE); void LLVMRunStaticConstructors(LLVMExecutionEngineRef EE); @@ -144,17 +128,9 @@ void LLVMFreeMachineCodeForFunction(LLVMExecutionEngineRef EE, LLVMValueRef F); void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M); -/** Deprecated: Use LLVMAddModule instead. */ -void LLVMAddModuleProvider(LLVMExecutionEngineRef EE, LLVMModuleProviderRef MP); - LLVMBool LLVMRemoveModule(LLVMExecutionEngineRef EE, LLVMModuleRef M, LLVMModuleRef *OutMod, char **OutError); -/** Deprecated: Use LLVMRemoveModule instead. */ -LLVMBool LLVMRemoveModuleProvider(LLVMExecutionEngineRef EE, - LLVMModuleProviderRef MP, - LLVMModuleRef *OutMod, char **OutError); - LLVMBool LLVMFindFunction(LLVMExecutionEngineRef EE, const char *Name, LLVMValueRef *OutFn); diff --git a/include/llvm-c/IRReader.h b/include/llvm-c/IRReader.h index 5001afb7ed7d..5b58d9921fb0 100644 --- a/include/llvm-c/IRReader.h +++ b/include/llvm-c/IRReader.h @@ -14,7 +14,7 @@ #ifndef LLVM_C_IRREADER_H #define LLVM_C_IRREADER_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { diff --git a/include/llvm-c/Initialization.h b/include/llvm-c/Initialization.h index 44194f8ea311..90c8396f7ad3 100644 --- a/include/llvm-c/Initialization.h +++ b/include/llvm-c/Initialization.h @@ -16,7 +16,7 @@ #ifndef LLVM_C_INITIALIZATION_H #define LLVM_C_INITIALIZATION_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { diff --git a/include/llvm-c/Linker.h b/include/llvm-c/Linker.h index 9f98a3342d0b..4d9bd46a259b 100644 --- a/include/llvm-c/Linker.h +++ b/include/llvm-c/Linker.h @@ -14,7 +14,7 @@ #ifndef LLVM_C_LINKER_H #define LLVM_C_LINKER_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { @@ -27,17 +27,27 @@ typedef enum { should not be used. */ } LLVMLinkerMode; -/* Links the source module into the destination module, taking ownership - * of the source module away from the caller. Optionally returns a - * human-readable description of any errors that occurred in linking. - * OutMessage must be disposed with LLVMDisposeMessage. The return value - * is true if an error occurred, false otherwise. +/* Links the source module into the destination module. The source module is + * damaged. The only thing that can be done is destroy it. Optionally returns a + * human-readable description of any errors that occurred in linking. OutMessage + * must be disposed with LLVMDisposeMessage. The return value is true if an + * error occurred, false otherwise. * * Note that the linker mode parameter \p Unused is no longer used, and has - * no effect. */ + * no effect. + * + * This function is deprecated. Use LLVMLinkModules2 instead. + */ LLVMBool LLVMLinkModules(LLVMModuleRef Dest, LLVMModuleRef Src, LLVMLinkerMode Unused, char **OutMessage); +/* Links the source module into the destination module. The source module is + * destroyed. + * The return value is true if an error occurred, false otherwise. + * Use the diagnostic handler to get any diagnostic message. +*/ +LLVMBool LLVMLinkModules2(LLVMModuleRef Dest, LLVMModuleRef Src); + #ifdef __cplusplus } #endif diff --git a/include/llvm-c/Object.h b/include/llvm-c/Object.h index 9cab5c426c45..a2980e89fe3d 100644 --- a/include/llvm-c/Object.h +++ b/include/llvm-c/Object.h @@ -19,7 +19,7 @@ #ifndef LLVM_C_OBJECT_H #define LLVM_C_OBJECT_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #include "llvm/Config/llvm-config.h" #ifdef __cplusplus diff --git a/include/llvm-c/OrcBindings.h b/include/llvm-c/OrcBindings.h new file mode 100644 index 000000000000..f6aff916999a --- /dev/null +++ b/include/llvm-c/OrcBindings.h @@ -0,0 +1,134 @@ +/*===----------- llvm-c/OrcBindings.h - Orc Lib C Iface ---------*- C++ -*-===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header declares the C interface to libLLVMOrcJIT.a, which implements *| +|* JIT compilation of LLVM IR. *| +|* *| +|* Many exotic languages can interoperate with C code but have a harder time *| +|* with C++ due to name mangling. So in addition to C, this interface enables *| +|* tools written in such languages. *| +|* *| +|* Note: This interface is experimental. It is *NOT* stable, and may be *| +|* changed without warning. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_ORCBINDINGS_H +#define LLVM_C_ORCBINDINGS_H + +#include "llvm-c/Object.h" +#include "llvm-c/Support.h" +#include "llvm-c/TargetMachine.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct LLVMOrcOpaqueJITStack *LLVMOrcJITStackRef; +typedef uint32_t LLVMOrcModuleHandle; +typedef uint64_t LLVMOrcTargetAddress; +typedef uint64_t (*LLVMOrcSymbolResolverFn)(const char *Name, + void *LookupCtx); +typedef uint64_t (*LLVMOrcLazyCompileCallbackFn)(LLVMOrcJITStackRef JITStack, + void *CallbackCtx); + +/** + * Create an ORC JIT stack. + * + * The client owns the resulting stack, and must call OrcDisposeInstance(...) + * to destroy it and free its memory. The JIT stack will take ownership of the + * TargetMachine, which will be destroyed when the stack is destroyed. The + * client should not attempt to dispose of the Target Machine, or it will result + * in a double-free. + */ +LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM); + +/** + * Mangle the given symbol. + * Memory will be allocated for MangledSymbol to hold the result. The client + */ +void LLVMOrcGetMangledSymbol(LLVMOrcJITStackRef JITStack, char **MangledSymbol, + const char *Symbol); + +/** + * Dispose of a mangled symbol. + */ + +void LLVMOrcDisposeMangledSymbol(char *MangledSymbol); + +/** + * Create a lazy compile callback. + */ +LLVMOrcTargetAddress +LLVMOrcCreateLazyCompileCallback(LLVMOrcJITStackRef JITStack, + LLVMOrcLazyCompileCallbackFn Callback, + void *CallbackCtx); + +/** + * Create a named indirect call stub. + */ +void LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress InitAddr); + +/** + * Set the pointer for the given indirect stub. + */ +void LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, + const char *StubName, + LLVMOrcTargetAddress NewAddr); + +/** + * Add module to be eagerly compiled. + */ +LLVMOrcModuleHandle +LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx); + +/** + * Add module to be lazily compiled one function at a time. + */ +LLVMOrcModuleHandle +LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, LLVMModuleRef Mod, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx); + +/** + * Add an object file. + */ +LLVMOrcModuleHandle +LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, LLVMObjectFileRef Obj, + LLVMOrcSymbolResolverFn SymbolResolver, + void *SymbolResolverCtx); + +/** + * Remove a module set from the JIT. + * + * This works for all modules that can be added via OrcAdd*, including object + * files. + */ +void LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, LLVMOrcModuleHandle H); + +/** + * Get symbol address from JIT instance. + */ +LLVMOrcTargetAddress LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, + const char *SymbolName); + +/** + * Dispose of an ORC JIT stack. + */ +void LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack); + +#ifdef __cplusplus +} +#endif /* extern "C" */ + +#endif /* LLVM_C_ORCBINDINGS_H */ diff --git a/include/llvm-c/Support.h b/include/llvm-c/Support.h index eca3b7a42037..735d1fbc78cc 100644 --- a/include/llvm-c/Support.h +++ b/include/llvm-c/Support.h @@ -15,31 +15,13 @@ #define LLVM_C_SUPPORT_H #include "llvm/Support/DataTypes.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { #endif /** - * @defgroup LLVMCSupportTypes Types and Enumerations - * - * @{ - */ - -typedef int LLVMBool; - -/** - * Used to pass regions of memory through LLVM interfaces. - * - * @see llvm::MemoryBuffer - */ -typedef struct LLVMOpaqueMemoryBuffer *LLVMMemoryBufferRef; - -/** - * @} - */ - -/** * This function permanently loads the dynamic library at the given path. * It is safe to call this function multiple times for the same library. * diff --git a/include/llvm-c/Target.h b/include/llvm-c/Target.h index b465b4b88db5..24d2cb4c9598 100644 --- a/include/llvm-c/Target.h +++ b/include/llvm-c/Target.h @@ -19,7 +19,7 @@ #ifndef LLVM_C_TARGET_H #define LLVM_C_TARGET_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #include "llvm/Config/llvm-config.h" #if defined(_MSC_VER) && !defined(inline) diff --git a/include/llvm-c/TargetMachine.h b/include/llvm-c/TargetMachine.h index 8cf1f43cb3c5..303708093653 100644 --- a/include/llvm-c/TargetMachine.h +++ b/include/llvm-c/TargetMachine.h @@ -19,7 +19,7 @@ #ifndef LLVM_C_TARGETMACHINE_H #define LLVM_C_TARGETMACHINE_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #include "llvm-c/Target.h" #ifdef __cplusplus @@ -115,7 +115,7 @@ char *LLVMGetTargetMachineCPU(LLVMTargetMachineRef T); LLVMDisposeMessage. */ char *LLVMGetTargetMachineFeatureString(LLVMTargetMachineRef T); -/** Deprecated: use LLVMGetDataLayout(LLVMModuleRef M) instead. */ +/** Returns the llvm::DataLayout used for this llvm:TargetMachine. */ LLVMTargetDataRef LLVMGetTargetMachineData(LLVMTargetMachineRef T); /** Set the target machine's ASM verbosity. */ diff --git a/include/llvm-c/Transforms/IPO.h b/include/llvm-c/Transforms/IPO.h index 448078012eac..3af7425dd268 100644 --- a/include/llvm-c/Transforms/IPO.h +++ b/include/llvm-c/Transforms/IPO.h @@ -15,7 +15,7 @@ #ifndef LLVM_C_TRANSFORMS_IPO_H #define LLVM_C_TRANSFORMS_IPO_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { diff --git a/include/llvm-c/Transforms/PassManagerBuilder.h b/include/llvm-c/Transforms/PassManagerBuilder.h index 3d7a9d677eab..69786b341ab4 100644 --- a/include/llvm-c/Transforms/PassManagerBuilder.h +++ b/include/llvm-c/Transforms/PassManagerBuilder.h @@ -14,7 +14,7 @@ #ifndef LLVM_C_TRANSFORMS_PASSMANAGERBUILDER_H #define LLVM_C_TRANSFORMS_PASSMANAGERBUILDER_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" typedef struct LLVMOpaquePassManagerBuilder *LLVMPassManagerBuilderRef; diff --git a/include/llvm-c/Transforms/Scalar.h b/include/llvm-c/Transforms/Scalar.h index 48c19a6e3117..c989ee86b9f7 100644 --- a/include/llvm-c/Transforms/Scalar.h +++ b/include/llvm-c/Transforms/Scalar.h @@ -19,7 +19,7 @@ #ifndef LLVM_C_TRANSFORMS_SCALAR_H #define LLVM_C_TRANSFORMS_SCALAR_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { diff --git a/include/llvm-c/Transforms/Vectorize.h b/include/llvm-c/Transforms/Vectorize.h index c9102da60297..a82ef49cb167 100644 --- a/include/llvm-c/Transforms/Vectorize.h +++ b/include/llvm-c/Transforms/Vectorize.h @@ -20,7 +20,7 @@ #ifndef LLVM_C_TRANSFORMS_VECTORIZE_H #define LLVM_C_TRANSFORMS_VECTORIZE_H -#include "llvm-c/Core.h" +#include "llvm-c/Types.h" #ifdef __cplusplus extern "C" { @@ -51,4 +51,3 @@ void LLVMAddSLPVectorizePass(LLVMPassManagerRef PM); #endif /* defined(__cplusplus) */ #endif - diff --git a/include/llvm-c/Types.h b/include/llvm-c/Types.h new file mode 100644 index 000000000000..19029584efcc --- /dev/null +++ b/include/llvm-c/Types.h @@ -0,0 +1,124 @@ +/*===-- llvm-c/Support.h - C Interface Types declarations ---------*- C -*-===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This file defines types used by the the C interface to LLVM. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_TYPES_H +#define LLVM_C_TYPES_H + +#include "llvm/Support/DataTypes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @defgroup LLVMCSupportTypes Types and Enumerations + * + * @{ + */ + +typedef int LLVMBool; + +/* Opaque types. */ + +/** + * LLVM uses a polymorphic type hierarchy which C cannot represent, therefore + * parameters must be passed as base types. Despite the declared types, most + * of the functions provided operate only on branches of the type hierarchy. + * The declared parameter names are descriptive and specify which type is + * required. Additionally, each type hierarchy is documented along with the + * functions that operate upon it. For more detail, refer to LLVM's C++ code. + * If in doubt, refer to Core.cpp, which performs parameter downcasts in the + * form unwrap<RequiredType>(Param). + */ + +/** + * Used to pass regions of memory through LLVM interfaces. + * + * @see llvm::MemoryBuffer + */ +typedef struct LLVMOpaqueMemoryBuffer *LLVMMemoryBufferRef; + +/** + * The top-level container for all LLVM global data. See the LLVMContext class. + */ +typedef struct LLVMOpaqueContext *LLVMContextRef; + +/** + * The top-level container for all other LLVM Intermediate Representation (IR) + * objects. + * + * @see llvm::Module + */ +typedef struct LLVMOpaqueModule *LLVMModuleRef; + +/** + * Each value in the LLVM IR has a type, an LLVMTypeRef. + * + * @see llvm::Type + */ +typedef struct LLVMOpaqueType *LLVMTypeRef; + +/** + * Represents an individual value in LLVM IR. + * + * This models llvm::Value. + */ +typedef struct LLVMOpaqueValue *LLVMValueRef; + +/** + * Represents a basic block of instructions in LLVM IR. + * + * This models llvm::BasicBlock. + */ +typedef struct LLVMOpaqueBasicBlock *LLVMBasicBlockRef; + +/** + * Represents an LLVM basic block builder. + * + * This models llvm::IRBuilder. + */ +typedef struct LLVMOpaqueBuilder *LLVMBuilderRef; + +/** + * Interface used to provide a module to JIT or interpreter. + * This is now just a synonym for llvm::Module, but we have to keep using the + * different type to keep binary compatibility. + */ +typedef struct LLVMOpaqueModuleProvider *LLVMModuleProviderRef; + +/** @see llvm::PassManagerBase */ +typedef struct LLVMOpaquePassManager *LLVMPassManagerRef; + +/** @see llvm::PassRegistry */ +typedef struct LLVMOpaquePassRegistry *LLVMPassRegistryRef; + +/** + * Used to get the users and usees of a Value. + * + * @see llvm::Use */ +typedef struct LLVMOpaqueUse *LLVMUseRef; + +/** + * @see llvm::DiagnosticInfo + */ +typedef struct LLVMOpaqueDiagnosticInfo *LLVMDiagnosticInfoRef; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/llvm-c/lto.h b/include/llvm-c/lto.h index cb3a69160454..691a0cd3f55c 100644 --- a/include/llvm-c/lto.h +++ b/include/llvm-c/lto.h @@ -374,8 +374,8 @@ extern lto_bool_t lto_codegen_add_module(lto_code_gen_t cg, lto_module_t mod); /** - * Sets the object module for code generation. This will transfer the ownship of - * the module to code generator. + * Sets the object module for code generation. This will transfer the ownership + * of the module to the code generator. * * \c cg and \c mod must both be in the same context. * diff --git a/include/llvm/ADT/APFloat.h b/include/llvm/ADT/APFloat.h index 76615affb253..3fe04060fd59 100644 --- a/include/llvm/ADT/APFloat.h +++ b/include/llvm/ADT/APFloat.h @@ -142,6 +142,9 @@ public: /// @} static unsigned int semanticsPrecision(const fltSemantics &); + static ExponentType semanticsMinExponent(const fltSemantics &); + static ExponentType semanticsMaxExponent(const fltSemantics &); + static unsigned int semanticsSizeInBits(const fltSemantics &); /// IEEE-754R 5.11: Floating Point Comparison Relations. enum cmpResult { @@ -296,7 +299,7 @@ public: /// IEEE remainder. opStatus remainder(const APFloat &); /// C fmod, or llvm frem. - opStatus mod(const APFloat &, roundingMode); + opStatus mod(const APFloat &); opStatus fusedMultiplyAdd(const APFloat &, const APFloat &, roundingMode); opStatus roundToIntegral(roundingMode); /// IEEE-754R 5.3.1: nextUp/nextDown. @@ -445,6 +448,9 @@ public: /// Returns true if and only if the number has the largest possible finite /// magnitude in the current semantics. bool isLargest() const; + + /// Returns true if and only if the number is an exact integer. + bool isInteger() const; /// @} diff --git a/include/llvm/ADT/APInt.h b/include/llvm/ADT/APInt.h index 5013f295f5c7..e2a0cb5e69dc 100644 --- a/include/llvm/ADT/APInt.h +++ b/include/llvm/ADT/APInt.h @@ -294,11 +294,12 @@ public: delete[] pVal; } - /// \brief Default constructor that creates an uninitialized APInt. + /// \brief Default constructor that creates an uninteresting APInt + /// representing a 1-bit zero value. /// /// This is useful for object deserialization (pair this with the static /// method Read). - explicit APInt() : BitWidth(1) {} + explicit APInt() : BitWidth(1), VAL(0) {} /// \brief Returns whether this instance allocated memory. bool needsCleanup() const { return !isSingleWord(); } @@ -1528,7 +1529,7 @@ public: /// \returns the nearest log base 2 of this APInt. Ties round up. /// /// NOTE: When we have a BitWidth of 1, we define: - /// + /// /// log2(0) = UINT32_MAX /// log2(1) = 0 /// diff --git a/include/llvm/ADT/APSInt.h b/include/llvm/ADT/APSInt.h index a187515f8592..a6552d0a2f36 100644 --- a/include/llvm/ADT/APSInt.h +++ b/include/llvm/ADT/APSInt.h @@ -21,6 +21,7 @@ namespace llvm { class APSInt : public APInt { bool IsUnsigned; + public: /// Default constructor that creates an uninitialized APInt. explicit APSInt() : IsUnsigned(false) {} @@ -246,8 +247,7 @@ public: return this->operator|(RHS); } - - APSInt operator^(const APSInt& RHS) const { + APSInt operator^(const APSInt &RHS) const { assert(IsUnsigned == RHS.IsUnsigned && "Signedness mismatch!"); return APSInt(static_cast<const APInt&>(*this) ^ RHS, IsUnsigned); } @@ -286,7 +286,7 @@ public: } /// \brief Determine if two APSInts have the same value, zero- or - /// sign-extending as needed. + /// sign-extending as needed. static bool isSameValue(const APSInt &I1, const APSInt &I2) { return !compareValues(I1, I2); } diff --git a/include/llvm/ADT/ArrayRef.h b/include/llvm/ADT/ArrayRef.h index c8795fd89e33..517ba39849e1 100644 --- a/include/llvm/ADT/ArrayRef.h +++ b/include/llvm/ADT/ArrayRef.h @@ -10,6 +10,7 @@ #ifndef LLVM_ADT_ARRAYREF_H #define LLVM_ADT_ARRAYREF_H +#include "llvm/ADT/Hashing.h" #include "llvm/ADT/None.h" #include "llvm/ADT/SmallVector.h" #include <vector> @@ -85,7 +86,7 @@ namespace llvm { /// Construct an ArrayRef from a std::initializer_list. /*implicit*/ ArrayRef(const std::initializer_list<T> &Vec) - : Data(Vec.begin() == Vec.end() ? (T*)0 : Vec.begin()), + : Data(Vec.begin() == Vec.end() ? (T*)nullptr : Vec.begin()), Length(Vec.size()) {} /// Construct an ArrayRef<const T*> from ArrayRef<T*>. This uses SFINAE to @@ -148,7 +149,7 @@ namespace llvm { // copy - Allocate copy in Allocator and return ArrayRef<T> to it. template <typename Allocator> ArrayRef<T> copy(Allocator &A) { T *Buff = A.template Allocate<T>(Length); - std::copy(begin(), end(), Buff); + std::uninitialized_copy(begin(), end(), Buff); return ArrayRef<T>(Buff, Length); } @@ -156,8 +157,6 @@ namespace llvm { bool equals(ArrayRef RHS) const { if (Length != RHS.Length) return false; - if (Length == 0) - return true; return std::equal(begin(), end(), RHS.begin()); } @@ -339,6 +338,16 @@ namespace llvm { return Vec; } + /// Construct an ArrayRef from an ArrayRef (no-op) (const) + template <typename T> ArrayRef<T> makeArrayRef(const ArrayRef<T> &Vec) { + return Vec; + } + + /// Construct an ArrayRef from an ArrayRef (no-op) + template <typename T> ArrayRef<T> &makeArrayRef(ArrayRef<T> &Vec) { + return Vec; + } + /// Construct an ArrayRef from a C array. template<typename T, size_t N> ArrayRef<T> makeArrayRef(const T (&Arr)[N]) { @@ -366,6 +375,10 @@ namespace llvm { template <typename T> struct isPodLike<ArrayRef<T> > { static const bool value = true; }; + + template <typename T> hash_code hash_value(ArrayRef<T> S) { + return hash_combine_range(S.begin(), S.end()); + } } #endif diff --git a/include/llvm/ADT/BitVector.h b/include/llvm/ADT/BitVector.h index f58dd7356c7d..ad00d51f99e9 100644 --- a/include/llvm/ADT/BitVector.h +++ b/include/llvm/ADT/BitVector.h @@ -34,7 +34,7 @@ class BitVector { BitWord *Bits; // Actual bits. unsigned Size; // Size of bitvector in bits. - unsigned Capacity; // Size of allocated memory in BitWord. + unsigned Capacity; // Number of BitWords allocated in the Bits array. public: typedef unsigned size_type; @@ -566,8 +566,16 @@ private: if (AddBits) clear_unused_bits(); } + +public: + /// Return the size (in bytes) of the bit vector. + size_t getMemorySize() const { return Capacity * sizeof(BitWord); } }; +static inline size_t capacity_in_bytes(const BitVector &X) { + return X.getMemorySize(); +} + } // End llvm namespace namespace std { diff --git a/include/llvm/ADT/DeltaAlgorithm.h b/include/llvm/ADT/DeltaAlgorithm.h index 21bc1e80c9d8..a26f37dfdc7d 100644 --- a/include/llvm/ADT/DeltaAlgorithm.h +++ b/include/llvm/ADT/DeltaAlgorithm.h @@ -68,7 +68,7 @@ private: /// \return - True on success. bool Search(const changeset_ty &Changes, const changesetlist_ty &Sets, changeset_ty &Res); - + protected: /// UpdatedSearchState - Callback used when the search state changes. virtual void UpdatedSearchState(const changeset_ty &Changes, diff --git a/include/llvm/ADT/DenseMap.h b/include/llvm/ADT/DenseMap.h index 27f73157a29f..6ee1960b5c82 100644 --- a/include/llvm/ADT/DenseMap.h +++ b/include/llvm/ADT/DenseMap.h @@ -282,7 +282,7 @@ protected: "# initial buckets must be a power of two!"); const KeyT EmptyKey = getEmptyKey(); for (BucketT *B = getBuckets(), *E = getBucketsEnd(); B != E; ++B) - new (&B->getFirst()) KeyT(EmptyKey); + ::new (&B->getFirst()) KeyT(EmptyKey); } void moveFromOldBuckets(BucketT *OldBucketsBegin, BucketT *OldBucketsEnd) { @@ -300,7 +300,7 @@ protected: (void)FoundVal; // silence warning. assert(!FoundVal && "Key already in new map?"); DestBucket->getFirst() = std::move(B->getFirst()); - new (&DestBucket->getSecond()) ValueT(std::move(B->getSecond())); + ::new (&DestBucket->getSecond()) ValueT(std::move(B->getSecond())); incrementNumEntries(); // Free the value. @@ -324,11 +324,11 @@ protected: getNumBuckets() * sizeof(BucketT)); else for (size_t i = 0; i < getNumBuckets(); ++i) { - new (&getBuckets()[i].getFirst()) + ::new (&getBuckets()[i].getFirst()) KeyT(other.getBuckets()[i].getFirst()); if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) && !KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey())) - new (&getBuckets()[i].getSecond()) + ::new (&getBuckets()[i].getSecond()) ValueT(other.getBuckets()[i].getSecond()); } } @@ -402,7 +402,7 @@ private: TheBucket = InsertIntoBucketImpl(Key, TheBucket); TheBucket->getFirst() = Key; - new (&TheBucket->getSecond()) ValueT(Value); + ::new (&TheBucket->getSecond()) ValueT(Value); return TheBucket; } @@ -411,7 +411,7 @@ private: TheBucket = InsertIntoBucketImpl(Key, TheBucket); TheBucket->getFirst() = Key; - new (&TheBucket->getSecond()) ValueT(std::move(Value)); + ::new (&TheBucket->getSecond()) ValueT(std::move(Value)); return TheBucket; } @@ -419,7 +419,7 @@ private: TheBucket = InsertIntoBucketImpl(Key, TheBucket); TheBucket->getFirst() = std::move(Key); - new (&TheBucket->getSecond()) ValueT(std::move(Value)); + ::new (&TheBucket->getSecond()) ValueT(std::move(Value)); return TheBucket; } @@ -766,10 +766,10 @@ public: // Swap separately and handle any assymetry. std::swap(LHSB->getFirst(), RHSB->getFirst()); if (hasLHSValue) { - new (&RHSB->getSecond()) ValueT(std::move(LHSB->getSecond())); + ::new (&RHSB->getSecond()) ValueT(std::move(LHSB->getSecond())); LHSB->getSecond().~ValueT(); } else if (hasRHSValue) { - new (&LHSB->getSecond()) ValueT(std::move(RHSB->getSecond())); + ::new (&LHSB->getSecond()) ValueT(std::move(RHSB->getSecond())); RHSB->getSecond().~ValueT(); } } @@ -795,11 +795,11 @@ public: for (unsigned i = 0, e = InlineBuckets; i != e; ++i) { BucketT *NewB = &LargeSide.getInlineBuckets()[i], *OldB = &SmallSide.getInlineBuckets()[i]; - new (&NewB->getFirst()) KeyT(std::move(OldB->getFirst())); + ::new (&NewB->getFirst()) KeyT(std::move(OldB->getFirst())); OldB->getFirst().~KeyT(); if (!KeyInfoT::isEqual(NewB->getFirst(), EmptyKey) && !KeyInfoT::isEqual(NewB->getFirst(), TombstoneKey)) { - new (&NewB->getSecond()) ValueT(std::move(OldB->getSecond())); + ::new (&NewB->getSecond()) ValueT(std::move(OldB->getSecond())); OldB->getSecond().~ValueT(); } } @@ -866,8 +866,8 @@ public: !KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { assert(size_t(TmpEnd - TmpBegin) < InlineBuckets && "Too many inline buckets!"); - new (&TmpEnd->getFirst()) KeyT(std::move(P->getFirst())); - new (&TmpEnd->getSecond()) ValueT(std::move(P->getSecond())); + ::new (&TmpEnd->getFirst()) KeyT(std::move(P->getFirst())); + ::new (&TmpEnd->getSecond()) ValueT(std::move(P->getSecond())); ++TmpEnd; P->getSecond().~ValueT(); } diff --git a/include/llvm/ADT/DenseMapInfo.h b/include/llvm/ADT/DenseMapInfo.h index b0a053072079..a844ebcccf5b 100644 --- a/include/llvm/ADT/DenseMapInfo.h +++ b/include/llvm/ADT/DenseMapInfo.h @@ -14,6 +14,7 @@ #ifndef LLVM_ADT_DENSEMAPINFO_H #define LLVM_ADT_DENSEMAPINFO_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/PointerLikeTypeTraits.h" @@ -58,7 +59,7 @@ template<> struct DenseMapInfo<char> { return LHS == RHS; } }; - + // Provide DenseMapInfo for unsigned ints. template<> struct DenseMapInfo<unsigned> { static inline unsigned getEmptyKey() { return ~0U; } @@ -190,6 +191,31 @@ template <> struct DenseMapInfo<StringRef> { } }; +// Provide DenseMapInfo for ArrayRefs. +template <typename T> struct DenseMapInfo<ArrayRef<T>> { + static inline ArrayRef<T> getEmptyKey() { + return ArrayRef<T>(reinterpret_cast<const T *>(~static_cast<uintptr_t>(0)), + size_t(0)); + } + static inline ArrayRef<T> getTombstoneKey() { + return ArrayRef<T>(reinterpret_cast<const T *>(~static_cast<uintptr_t>(1)), + size_t(0)); + } + static unsigned getHashValue(ArrayRef<T> Val) { + assert(Val.data() != getEmptyKey().data() && "Cannot hash the empty key!"); + assert(Val.data() != getTombstoneKey().data() && + "Cannot hash the tombstone key!"); + return (unsigned)(hash_value(Val)); + } + static bool isEqual(ArrayRef<T> LHS, ArrayRef<T> RHS) { + if (RHS.data() == getEmptyKey().data()) + return LHS.data() == getEmptyKey().data(); + if (RHS.data() == getTombstoneKey().data()) + return LHS.data() == getTombstoneKey().data(); + return LHS == RHS; + } +}; + } // end namespace llvm #endif diff --git a/include/llvm/ADT/DenseSet.h b/include/llvm/ADT/DenseSet.h index d34024005dfe..ef09dce37980 100644 --- a/include/llvm/ADT/DenseSet.h +++ b/include/llvm/ADT/DenseSet.h @@ -42,6 +42,7 @@ class DenseSet { static_assert(sizeof(typename MapTy::value_type) == sizeof(ValueT), "DenseMap buckets unexpectedly large!"); MapTy TheMap; + public: typedef ValueT key_type; typedef ValueT value_type; @@ -79,6 +80,7 @@ public: class Iterator { typename MapTy::iterator I; friend class DenseSet; + public: typedef typename MapTy::iterator::difference_type difference_type; typedef ValueT value_type; @@ -99,6 +101,7 @@ public: class ConstIterator { typename MapTy::const_iterator I; friend class DenseSet; + public: typedef typename MapTy::const_iterator::difference_type difference_type; typedef ValueT value_type; @@ -148,7 +151,7 @@ public: detail::DenseSetEmpty Empty; return TheMap.insert(std::make_pair(V, Empty)); } - + // Range insertion of values. template<typename InputIt> void insert(InputIt I, InputIt E) { diff --git a/include/llvm/ADT/DepthFirstIterator.h b/include/llvm/ADT/DepthFirstIterator.h index d79b9acacfa9..c9317b8539b3 100644 --- a/include/llvm/ADT/DepthFirstIterator.h +++ b/include/llvm/ADT/DepthFirstIterator.h @@ -58,7 +58,6 @@ public: SetType &Visited; }; - // Generic Depth First Iterator template<class GraphT, class SetType = llvm::SmallPtrSet<typename GraphTraits<GraphT>::NodeType*, 8>, @@ -76,21 +75,22 @@ class df_iterator : public std::iterator<std::forward_iterator_tag, // VisitStack - Used to maintain the ordering. Top = current block // First element is node pointer, second is the 'next child' to visit // if the int in PointerIntTy is 0, the 'next child' to visit is invalid - std::vector<std::pair<PointerIntTy, ChildItTy> > VisitStack; + std::vector<std::pair<PointerIntTy, ChildItTy>> VisitStack; + private: inline df_iterator(NodeType *Node) { this->Visited.insert(Node); - VisitStack.push_back(std::make_pair(PointerIntTy(Node, 0), - GT::child_begin(Node))); + VisitStack.push_back( + std::make_pair(PointerIntTy(Node, 0), GT::child_begin(Node))); } - inline df_iterator() { - // End is when stack is empty + inline df_iterator() { + // End is when stack is empty } inline df_iterator(NodeType *Node, SetType &S) : df_iterator_storage<SetType, ExtStorage>(S) { if (!S.count(Node)) { - VisitStack.push_back(std::make_pair(PointerIntTy(Node, 0), - GT::child_begin(Node))); + VisitStack.push_back( + std::make_pair(PointerIntTy(Node, 0), GT::child_begin(Node))); this->Visited.insert(Node); } } @@ -115,8 +115,8 @@ private: // Has our next sibling been visited? if (Next && this->Visited.insert(Next).second) { // No, do it now. - VisitStack.push_back(std::make_pair(PointerIntTy(Next, 0), - GT::child_begin(Next))); + VisitStack.push_back( + std::make_pair(PointerIntTy(Next, 0), GT::child_begin(Next))); return; } } @@ -195,7 +195,6 @@ public: } }; - // Provide global constructors that automatically figure out correct types... // template <class T> @@ -237,7 +236,6 @@ iterator_range<df_ext_iterator<T, SetTy>> depth_first_ext(const T& G, return make_range(df_ext_begin(G, S), df_ext_end(G, S)); } - // Provide global definitions of inverse depth first iterators... template <class T, class SetTy = llvm::SmallPtrSet<typename GraphTraits<T>::NodeType*, 8>, diff --git a/include/llvm/ADT/FoldingSet.h b/include/llvm/ADT/FoldingSet.h index 52d10c1c1245..c9205396591b 100644 --- a/include/llvm/ADT/FoldingSet.h +++ b/include/llvm/ADT/FoldingSet.h @@ -122,9 +122,10 @@ protected: /// is greater than twice the number of buckets. unsigned NumNodes; - ~FoldingSetImpl(); - explicit FoldingSetImpl(unsigned Log2InitSize = 6); + FoldingSetImpl(FoldingSetImpl &&Arg); + FoldingSetImpl &operator=(FoldingSetImpl &&RHS); + ~FoldingSetImpl(); public: //===--------------------------------------------------------------------===// @@ -137,7 +138,6 @@ public: void *NextInFoldingSetBucket; public: - Node() : NextInFoldingSetBucket(nullptr) {} // Accessors @@ -182,13 +182,11 @@ public: bool empty() const { return NumNodes == 0; } private: - /// GrowHashTable - Double the size of the hash table and rehash everything. /// void GrowHashTable(); protected: - /// GetNodeProfile - Instantiations of the FoldingSet template implement /// this function to gather data bits for the given node. virtual void GetNodeProfile(Node *N, FoldingSetNodeID &ID) const = 0; @@ -269,6 +267,7 @@ template<typename T, typename Ctx> struct ContextualFoldingSetTrait class FoldingSetNodeIDRef { const unsigned *Data; size_t Size; + public: FoldingSetNodeIDRef() : Data(nullptr), Size(0) {} FoldingSetNodeIDRef(const unsigned *D, size_t S) : Data(D), Size(S) {} @@ -393,6 +392,10 @@ DefaultContextualFoldingSetTrait<T, Ctx>::ComputeHash(T &X, /// implementation of the folding set to the node class T. T must be a /// subclass of FoldingSetNode and implement a Profile function. /// +/// Note that this set type is movable and move-assignable. However, its +/// moved-from state is not a valid state for anything other than +/// move-assigning and destroying. This is primarily to enable movable APIs +/// that incorporate these objects. template <class T> class FoldingSet final : public FoldingSetImpl { private: /// GetNodeProfile - Each instantiatation of the FoldingSet needs to provide a @@ -417,8 +420,13 @@ private: public: explicit FoldingSet(unsigned Log2InitSize = 6) - : FoldingSetImpl(Log2InitSize) - {} + : FoldingSetImpl(Log2InitSize) {} + + FoldingSet(FoldingSet &&Arg) : FoldingSetImpl(std::move(Arg)) {} + FoldingSet &operator=(FoldingSet &&RHS) { + (void)FoldingSetImpl::operator=(std::move(RHS)); + return *this; + } typedef FoldingSetIterator<T> iterator; iterator begin() { return iterator(Buckets); } @@ -498,7 +506,6 @@ public: Ctx getContext() const { return Context; } - typedef FoldingSetIterator<T> iterator; iterator begin() { return iterator(Buckets); } iterator end() { return iterator(Buckets+NumBuckets); } @@ -614,9 +621,7 @@ public: } }; - -template<class T> -class FoldingSetIterator : public FoldingSetIteratorImpl { +template <class T> class FoldingSetIterator : public FoldingSetIteratorImpl { public: explicit FoldingSetIterator(void **Bucket) : FoldingSetIteratorImpl(Bucket) {} @@ -666,8 +671,7 @@ public: } }; - -template<class T> +template <class T> class FoldingSetBucketIterator : public FoldingSetBucketIteratorImpl { public: explicit FoldingSetBucketIterator(void **Bucket) : @@ -694,6 +698,7 @@ public: template <typename T> class FoldingSetNodeWrapper : public FoldingSetNode { T data; + public: template <typename... Ts> explicit FoldingSetNodeWrapper(Ts &&... Args) @@ -716,12 +721,12 @@ public: /// information that would otherwise only be required for recomputing an ID. class FastFoldingSetNode : public FoldingSetNode { FoldingSetNodeID FastID; + protected: explicit FastFoldingSetNode(const FoldingSetNodeID &ID) : FastID(ID) {} + public: - void Profile(FoldingSetNodeID &ID) const { - ID.AddNodeID(FastID); - } + void Profile(FoldingSetNodeID &ID) const { ID.AddNodeID(FastID); } }; //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/ImmutableList.h b/include/llvm/ADT/ImmutableList.h index 748d3e4bf9ff..a1d26bd97045 100644 --- a/include/llvm/ADT/ImmutableList.h +++ b/include/llvm/ADT/ImmutableList.h @@ -28,7 +28,7 @@ class ImmutableListImpl : public FoldingSetNode { T Head; const ImmutableListImpl* Tail; - ImmutableListImpl(const T& head, const ImmutableListImpl* tail = 0) + ImmutableListImpl(const T& head, const ImmutableListImpl* tail = nullptr) : Head(head), Tail(tail) {} friend class ImmutableListFactory<T>; @@ -72,7 +72,7 @@ public: // This constructor should normally only be called by ImmutableListFactory<T>. // There may be cases, however, when one needs to extract the internal pointer // and reconstruct a list object from that pointer. - ImmutableList(const ImmutableListImpl<T>* x = 0) : X(x) {} + ImmutableList(const ImmutableListImpl<T>* x = nullptr) : X(x) {} const ImmutableListImpl<T>* getInternalPointer() const { return X; @@ -81,7 +81,7 @@ public: class iterator { const ImmutableListImpl<T>* L; public: - iterator() : L(0) {} + iterator() : L(nullptr) {} iterator(ImmutableList l) : L(l.getInternalPointer()) {} iterator& operator++() { L = L->getTail(); return *this; } @@ -128,7 +128,7 @@ public: /// getTail - Returns the tail of the list, which is another (possibly empty) /// ImmutableList. ImmutableList getTail() { - return X ? X->getTail() : 0; + return X ? X->getTail() : nullptr; } void Profile(FoldingSetNodeID& ID) const { @@ -190,7 +190,7 @@ public: } ImmutableList<T> getEmptyList() const { - return ImmutableList<T>(0); + return ImmutableList<T>(nullptr); } ImmutableList<T> create(const T& X) { @@ -226,4 +226,4 @@ struct isPodLike<ImmutableList<T> > { static const bool value = true; }; } // end llvm namespace -#endif +#endif // LLVM_ADT_IMMUTABLELIST_H diff --git a/include/llvm/ADT/ImmutableMap.h b/include/llvm/ADT/ImmutableMap.h index 438dec2333c5..7480cd73da61 100644 --- a/include/llvm/ADT/ImmutableMap.h +++ b/include/llvm/ADT/ImmutableMap.h @@ -55,7 +55,6 @@ struct ImutKeyValueInfo { } }; - template <typename KeyT, typename ValT, typename ValInfo = ImutKeyValueInfo<KeyT,ValT> > class ImmutableMap { @@ -79,9 +78,11 @@ public: explicit ImmutableMap(const TreeTy* R) : Root(const_cast<TreeTy*>(R)) { if (Root) { Root->retain(); } } + ImmutableMap(const ImmutableMap &X) : Root(X.Root) { if (Root) { Root->retain(); } } + ImmutableMap &operator=(const ImmutableMap &X) { if (Root != X.Root) { if (X.Root) { X.Root->retain(); } @@ -90,6 +91,7 @@ public: } return *this; } + ~ImmutableMap() { if (Root) { Root->release(); } } @@ -99,11 +101,10 @@ public: const bool Canonicalize; public: - Factory(bool canonicalize = true) - : Canonicalize(canonicalize) {} - - Factory(BumpPtrAllocator& Alloc, bool canonicalize = true) - : F(Alloc), Canonicalize(canonicalize) {} + Factory(bool canonicalize = true) : Canonicalize(canonicalize) {} + + Factory(BumpPtrAllocator &Alloc, bool canonicalize = true) + : F(Alloc), Canonicalize(canonicalize) {} ImmutableMap getEmptyMap() { return ImmutableMap(F.getEmptyTree()); } @@ -143,14 +144,12 @@ public: return Root; } - TreeTy *getRootWithoutRetain() const { - return Root; - } - + TreeTy *getRootWithoutRetain() const { return Root; } + void manualRetain() { if (Root) Root->retain(); } - + void manualRelease() { if (Root) Root->release(); } @@ -224,7 +223,7 @@ public: return nullptr; } - + /// getMaxElement - Returns the <key,value> pair in the ImmutableMap for /// which key is the highest in the ordering of keys in the map. This /// method returns NULL if the map is empty. @@ -260,20 +259,21 @@ public: typedef typename ValInfo::data_type_ref data_type_ref; typedef ImutAVLTree<ValInfo> TreeTy; typedef typename TreeTy::Factory FactoryTy; - + protected: TreeTy *Root; FactoryTy *Factory; - + public: /// Constructs a map from a pointer to a tree root. In general one /// should use a Factory object to create maps instead of directly /// invoking the constructor, but there are cases where make this /// constructor public is useful. - explicit ImmutableMapRef(const TreeTy* R, FactoryTy *F) - : Root(const_cast<TreeTy*>(R)), - Factory(F) { - if (Root) { Root->retain(); } + explicit ImmutableMapRef(const TreeTy *R, FactoryTy *F) + : Root(const_cast<TreeTy *>(R)), Factory(F) { + if (Root) { + Root->retain(); + } } explicit ImmutableMapRef(const ImmutableMap<KeyT, ValT> &X, @@ -282,21 +282,21 @@ public: Factory(F.getTreeFactory()) { if (Root) { Root->retain(); } } - - ImmutableMapRef(const ImmutableMapRef &X) - : Root(X.Root), - Factory(X.Factory) { - if (Root) { Root->retain(); } + + ImmutableMapRef(const ImmutableMapRef &X) : Root(X.Root), Factory(X.Factory) { + if (Root) { + Root->retain(); + } } ImmutableMapRef &operator=(const ImmutableMapRef &X) { if (Root != X.Root) { if (X.Root) X.Root->retain(); - + if (Root) Root->release(); - + Root = X.Root; Factory = X.Factory; } @@ -307,7 +307,7 @@ public: if (Root) Root->release(); } - + static inline ImmutableMapRef getEmptyMap(FactoryTy *F) { return ImmutableMapRef(0, F); } @@ -329,31 +329,34 @@ public: TreeTy *NewT = Factory->remove(Root, K); return ImmutableMapRef(NewT, Factory); } - + bool contains(key_type_ref K) const { return Root ? Root->contains(K) : false; } - + ImmutableMap<KeyT, ValT> asImmutableMap() const { return ImmutableMap<KeyT, ValT>(Factory->getCanonicalTree(Root)); } - + bool operator==(const ImmutableMapRef &RHS) const { return Root && RHS.Root ? Root->isEqual(*RHS.Root) : Root == RHS.Root; } - + bool operator!=(const ImmutableMapRef &RHS) const { return Root && RHS.Root ? Root->isNotEqual(*RHS.Root) : Root != RHS.Root; } - + bool isEmpty() const { return !Root; } - + //===--------------------------------------------------===// // For testing. //===--------------------------------------------------===// - - void verify() const { if (Root) Root->verify(); } - + + void verify() const { + if (Root) + Root->verify(); + } + //===--------------------------------------------------===// // Iterators. //===--------------------------------------------------===// @@ -370,38 +373,36 @@ public: iterator begin() const { return iterator(Root); } iterator end() const { return iterator(); } - - data_type* lookup(key_type_ref K) const { + + data_type *lookup(key_type_ref K) const { if (Root) { TreeTy* T = Root->find(K); if (T) return &T->getValue().second; } - - return 0; + + return nullptr; } - + /// getMaxElement - Returns the <key,value> pair in the ImmutableMap for /// which key is the highest in the ordering of keys in the map. This /// method returns NULL if the map is empty. value_type* getMaxElement() const { return Root ? &(Root->getMaxElement()->getValue()) : 0; } - + //===--------------------------------------------------===// // Utility methods. //===--------------------------------------------------===// - + unsigned getHeight() const { return Root ? Root->getHeight() : 0; } - - static inline void Profile(FoldingSetNodeID& ID, const ImmutableMapRef &M) { + + static inline void Profile(FoldingSetNodeID &ID, const ImmutableMapRef &M) { ID.AddPointer(M.Root); } - - inline void Profile(FoldingSetNodeID& ID) const { - return Profile(ID, *this); - } + + inline void Profile(FoldingSetNodeID &ID) const { return Profile(ID, *this); } }; - + } // end namespace llvm -#endif +#endif // LLVM_ADT_IMMUTABLEMAP_H diff --git a/include/llvm/ADT/IntrusiveRefCntPtr.h b/include/llvm/ADT/IntrusiveRefCntPtr.h index 65b2da793d7c..8057ec10be00 100644 --- a/include/llvm/ADT/IntrusiveRefCntPtr.h +++ b/include/llvm/ADT/IntrusiveRefCntPtr.h @@ -154,7 +154,7 @@ public: template <class X> IntrusiveRefCntPtr(IntrusiveRefCntPtr<X>&& S) : Obj(S.get()) { - S.Obj = 0; + S.Obj = nullptr; } template <class X> @@ -190,7 +190,7 @@ public: } void resetWithoutRelease() { - Obj = 0; + Obj = nullptr; } private: diff --git a/include/llvm/ADT/Optional.h b/include/llvm/ADT/Optional.h index 855ab890392e..d9acaf6d23b0 100644 --- a/include/llvm/ADT/Optional.h +++ b/include/llvm/ADT/Optional.h @@ -159,6 +159,25 @@ template <typename T> struct isPodLike<Optional<T> > { template<typename T, typename U> void operator==(const Optional<T> &X, const Optional<U> &Y); +template<typename T> +bool operator==(const Optional<T> &X, NoneType) { + return !X.hasValue(); +} + +template<typename T> +bool operator==(NoneType, const Optional<T> &X) { + return X == None; +} + +template<typename T> +bool operator!=(const Optional<T> &X, NoneType) { + return !(X == None); +} + +template<typename T> +bool operator!=(NoneType, const Optional<T> &X) { + return X != None; +} /// \brief Poison comparison between two \c Optional objects. Clients needs to /// explicitly compare the underlying values and account for empty \c Optional /// objects. diff --git a/include/llvm/ADT/PackedVector.h b/include/llvm/ADT/PackedVector.h index 1ae2a77e7eaf..09267173fd77 100644 --- a/include/llvm/ADT/PackedVector.h +++ b/include/llvm/ADT/PackedVector.h @@ -83,9 +83,9 @@ public: PackedVector &Vec; const unsigned Idx; - reference(); // Undefined + reference(); // Undefined public: - reference(PackedVector &vec, unsigned idx) : Vec(vec), Idx(idx) { } + reference(PackedVector &vec, unsigned idx) : Vec(vec), Idx(idx) {} reference &operator=(T val) { Vec.setValue(Vec.Bits, Idx, val); @@ -96,16 +96,16 @@ public: } }; - PackedVector() { } + PackedVector() = default; explicit PackedVector(unsigned size) : Bits(size << (BitNum-1)) { } bool empty() const { return Bits.empty(); } - unsigned size() const { return Bits.size() >> (BitNum-1); } - + unsigned size() const { return Bits.size() >> (BitNum - 1); } + void clear() { Bits.clear(); } - - void resize(unsigned N) { Bits.resize(N << (BitNum-1)); } + + void resize(unsigned N) { Bits.resize(N << (BitNum - 1)); } void reserve(unsigned N) { Bits.reserve(N << (BitNum-1)); } @@ -135,24 +135,14 @@ public: return Bits != RHS.Bits; } - const PackedVector &operator=(const PackedVector &RHS) { - Bits = RHS.Bits; - return *this; - } - PackedVector &operator|=(const PackedVector &RHS) { Bits |= RHS.Bits; return *this; } - - void swap(PackedVector &RHS) { - Bits.swap(RHS.Bits); - } }; -// Leave BitNum=0 undefined. -template <typename T> -class PackedVector<T, 0>; +// Leave BitNum=0 undefined. +template <typename T> class PackedVector<T, 0>; } // end llvm namespace diff --git a/include/llvm/ADT/PointerIntPair.h b/include/llvm/ADT/PointerIntPair.h index 45a40db85c04..0058d85d1ae4 100644 --- a/include/llvm/ADT/PointerIntPair.h +++ b/include/llvm/ADT/PointerIntPair.h @@ -21,8 +21,10 @@ namespace llvm { -template<typename T> -struct DenseMapInfo; +template <typename T> struct DenseMapInfo; + +template <typename PointerT, unsigned IntBits, typename PtrTraits> +struct PointerIntPairInfo; /// PointerIntPair - This class implements a pair of a pointer and small /// integer. It is designed to represent this in the space required by one @@ -38,83 +40,35 @@ struct DenseMapInfo; /// PointerIntPair<PointerIntPair<void*, 1, bool>, 1, bool> /// ... and the two bools will land in different bits. /// -template <typename PointerTy, unsigned IntBits, typename IntType=unsigned, - typename PtrTraits = PointerLikeTypeTraits<PointerTy> > +template <typename PointerTy, unsigned IntBits, typename IntType = unsigned, + typename PtrTraits = PointerLikeTypeTraits<PointerTy>, + typename Info = PointerIntPairInfo<PointerTy, IntBits, PtrTraits>> class PointerIntPair { intptr_t Value; - static_assert(PtrTraits::NumLowBitsAvailable < - std::numeric_limits<uintptr_t>::digits, - "cannot use a pointer type that has all bits free"); - static_assert(IntBits <= PtrTraits::NumLowBitsAvailable, - "PointerIntPair with integer size too large for pointer"); - enum : uintptr_t { - /// PointerBitMask - The bits that come from the pointer. - PointerBitMask = - ~(uintptr_t)(((intptr_t)1 << PtrTraits::NumLowBitsAvailable)-1), - /// IntShift - The number of low bits that we reserve for other uses, and - /// keep zero. - IntShift = (uintptr_t)PtrTraits::NumLowBitsAvailable-IntBits, - - /// IntMask - This is the unshifted mask for valid bits of the int type. - IntMask = (uintptr_t)(((intptr_t)1 << IntBits)-1), - - // ShiftedIntMask - This is the bits for the integer shifted in place. - ShiftedIntMask = (uintptr_t)(IntMask << IntShift) - }; public: PointerIntPair() : Value(0) {} PointerIntPair(PointerTy PtrVal, IntType IntVal) { setPointerAndInt(PtrVal, IntVal); } - explicit PointerIntPair(PointerTy PtrVal) { - initWithPointer(PtrVal); - } + explicit PointerIntPair(PointerTy PtrVal) { initWithPointer(PtrVal); } - PointerTy getPointer() const { - return PtrTraits::getFromVoidPointer( - reinterpret_cast<void*>(Value & PointerBitMask)); - } + PointerTy getPointer() const { return Info::getPointer(Value); } - IntType getInt() const { - return (IntType)((Value >> IntShift) & IntMask); - } + IntType getInt() const { return (IntType)Info::getInt(Value); } void setPointer(PointerTy PtrVal) { - intptr_t PtrWord - = reinterpret_cast<intptr_t>(PtrTraits::getAsVoidPointer(PtrVal)); - assert((PtrWord & ~PointerBitMask) == 0 && - "Pointer is not sufficiently aligned"); - // Preserve all low bits, just update the pointer. - Value = PtrWord | (Value & ~PointerBitMask); + Value = Info::updatePointer(Value, PtrVal); } - void setInt(IntType IntVal) { - intptr_t IntWord = static_cast<intptr_t>(IntVal); - assert((IntWord & ~IntMask) == 0 && "Integer too large for field"); - - // Preserve all bits other than the ones we are updating. - Value &= ~ShiftedIntMask; // Remove integer field. - Value |= IntWord << IntShift; // Set new integer. - } + void setInt(IntType IntVal) { Value = Info::updateInt(Value, IntVal); } void initWithPointer(PointerTy PtrVal) { - intptr_t PtrWord - = reinterpret_cast<intptr_t>(PtrTraits::getAsVoidPointer(PtrVal)); - assert((PtrWord & ~PointerBitMask) == 0 && - "Pointer is not sufficiently aligned"); - Value = PtrWord; + Value = Info::updatePointer(0, PtrVal); } void setPointerAndInt(PointerTy PtrVal, IntType IntVal) { - intptr_t PtrWord - = reinterpret_cast<intptr_t>(PtrTraits::getAsVoidPointer(PtrVal)); - assert((PtrWord & ~PointerBitMask) == 0 && - "Pointer is not sufficiently aligned"); - intptr_t IntWord = static_cast<intptr_t>(IntVal); - assert((IntWord & ~IntMask) == 0 && "Integer too large for field"); - - Value = PtrWord | (IntWord << IntShift); + Value = Info::updateInt(Info::updatePointer(0, PtrVal), IntVal); } PointerTy const *getAddrOfPointer() const { @@ -128,11 +82,15 @@ public: return reinterpret_cast<PointerTy *>(&Value); } - void *getOpaqueValue() const { return reinterpret_cast<void*>(Value); } - void setFromOpaqueValue(void *Val) { Value = reinterpret_cast<intptr_t>(Val);} + void *getOpaqueValue() const { return reinterpret_cast<void *>(Value); } + void setFromOpaqueValue(void *Val) { + Value = reinterpret_cast<intptr_t>(Val); + } static PointerIntPair getFromOpaqueValue(void *V) { - PointerIntPair P; P.setFromOpaqueValue(V); return P; + PointerIntPair P; + P.setFromOpaqueValue(V); + return P; } // Allow PointerIntPairs to be created from const void * if and only if the @@ -142,23 +100,81 @@ public: return getFromOpaqueValue(const_cast<void *>(V)); } - bool operator==(const PointerIntPair &RHS) const {return Value == RHS.Value;} - bool operator!=(const PointerIntPair &RHS) const {return Value != RHS.Value;} - bool operator<(const PointerIntPair &RHS) const {return Value < RHS.Value;} - bool operator>(const PointerIntPair &RHS) const {return Value > RHS.Value;} - bool operator<=(const PointerIntPair &RHS) const {return Value <= RHS.Value;} - bool operator>=(const PointerIntPair &RHS) const {return Value >= RHS.Value;} + bool operator==(const PointerIntPair &RHS) const { + return Value == RHS.Value; + } + bool operator!=(const PointerIntPair &RHS) const { + return Value != RHS.Value; + } + bool operator<(const PointerIntPair &RHS) const { return Value < RHS.Value; } + bool operator>(const PointerIntPair &RHS) const { return Value > RHS.Value; } + bool operator<=(const PointerIntPair &RHS) const { + return Value <= RHS.Value; + } + bool operator>=(const PointerIntPair &RHS) const { + return Value >= RHS.Value; + } +}; + +template <typename PointerT, unsigned IntBits, typename PtrTraits> +struct PointerIntPairInfo { + static_assert(PtrTraits::NumLowBitsAvailable < + std::numeric_limits<uintptr_t>::digits, + "cannot use a pointer type that has all bits free"); + static_assert(IntBits <= PtrTraits::NumLowBitsAvailable, + "PointerIntPair with integer size too large for pointer"); + enum : uintptr_t { + /// PointerBitMask - The bits that come from the pointer. + PointerBitMask = + ~(uintptr_t)(((intptr_t)1 << PtrTraits::NumLowBitsAvailable) - 1), + + /// IntShift - The number of low bits that we reserve for other uses, and + /// keep zero. + IntShift = (uintptr_t)PtrTraits::NumLowBitsAvailable - IntBits, + + /// IntMask - This is the unshifted mask for valid bits of the int type. + IntMask = (uintptr_t)(((intptr_t)1 << IntBits) - 1), + + // ShiftedIntMask - This is the bits for the integer shifted in place. + ShiftedIntMask = (uintptr_t)(IntMask << IntShift) + }; + + static PointerT getPointer(intptr_t Value) { + return PtrTraits::getFromVoidPointer( + reinterpret_cast<void *>(Value & PointerBitMask)); + } + + static intptr_t getInt(intptr_t Value) { + return (Value >> IntShift) & IntMask; + } + + static intptr_t updatePointer(intptr_t OrigValue, PointerT Ptr) { + intptr_t PtrWord = + reinterpret_cast<intptr_t>(PtrTraits::getAsVoidPointer(Ptr)); + assert((PtrWord & ~PointerBitMask) == 0 && + "Pointer is not sufficiently aligned"); + // Preserve all low bits, just update the pointer. + return PtrWord | (OrigValue & ~PointerBitMask); + } + + static intptr_t updateInt(intptr_t OrigValue, intptr_t Int) { + intptr_t IntWord = static_cast<intptr_t>(Int); + assert((IntWord & ~IntMask) == 0 && "Integer too large for field"); + + // Preserve all bits other than the ones we are updating. + return (OrigValue & ~ShiftedIntMask) | IntWord << IntShift; + } }; template <typename T> struct isPodLike; -template<typename PointerTy, unsigned IntBits, typename IntType> -struct isPodLike<PointerIntPair<PointerTy, IntBits, IntType> > { - static const bool value = true; +template <typename PointerTy, unsigned IntBits, typename IntType> +struct isPodLike<PointerIntPair<PointerTy, IntBits, IntType>> { + static const bool value = true; }; - + // Provide specialization of DenseMapInfo for PointerIntPair. -template<typename PointerTy, unsigned IntBits, typename IntType> -struct DenseMapInfo<PointerIntPair<PointerTy, IntBits, IntType> > { +template <typename PointerTy, unsigned IntBits, typename IntType> +struct DenseMapInfo<PointerIntPair<PointerTy, IntBits, IntType>> { typedef PointerIntPair<PointerTy, IntBits, IntType> Ty; static Ty getEmptyKey() { uintptr_t Val = static_cast<uintptr_t>(-1); @@ -178,10 +194,10 @@ struct DenseMapInfo<PointerIntPair<PointerTy, IntBits, IntType> > { }; // Teach SmallPtrSet that PointerIntPair is "basically a pointer". -template<typename PointerTy, unsigned IntBits, typename IntType, - typename PtrTraits> -class PointerLikeTypeTraits<PointerIntPair<PointerTy, IntBits, IntType, - PtrTraits> > { +template <typename PointerTy, unsigned IntBits, typename IntType, + typename PtrTraits> +class PointerLikeTypeTraits< + PointerIntPair<PointerTy, IntBits, IntType, PtrTraits>> { public: static inline void * getAsVoidPointer(const PointerIntPair<PointerTy, IntBits, IntType> &P) { @@ -195,9 +211,7 @@ public: getFromVoidPointer(const void *P) { return PointerIntPair<PointerTy, IntBits, IntType>::getFromOpaqueValue(P); } - enum { - NumLowBitsAvailable = PtrTraits::NumLowBitsAvailable - IntBits - }; + enum { NumLowBitsAvailable = PtrTraits::NumLowBitsAvailable - IntBits }; }; } // end namespace llvm diff --git a/include/llvm/ADT/PointerUnion.h b/include/llvm/ADT/PointerUnion.h index f27b81113ec5..6b3fe5749ad5 100644 --- a/include/llvm/ADT/PointerUnion.h +++ b/include/llvm/ADT/PointerUnion.h @@ -21,492 +21,454 @@ namespace llvm { - template <typename T> - struct PointerUnionTypeSelectorReturn { - typedef T Return; +template <typename T> struct PointerUnionTypeSelectorReturn { + typedef T Return; +}; + +/// Get a type based on whether two types are the same or not. +/// +/// For: +/// +/// \code +/// typedef typename PointerUnionTypeSelector<T1, T2, EQ, NE>::Return Ret; +/// \endcode +/// +/// Ret will be EQ type if T1 is same as T2 or NE type otherwise. +template <typename T1, typename T2, typename RET_EQ, typename RET_NE> +struct PointerUnionTypeSelector { + typedef typename PointerUnionTypeSelectorReturn<RET_NE>::Return Return; +}; + +template <typename T, typename RET_EQ, typename RET_NE> +struct PointerUnionTypeSelector<T, T, RET_EQ, RET_NE> { + typedef typename PointerUnionTypeSelectorReturn<RET_EQ>::Return Return; +}; + +template <typename T1, typename T2, typename RET_EQ, typename RET_NE> +struct PointerUnionTypeSelectorReturn< + PointerUnionTypeSelector<T1, T2, RET_EQ, RET_NE>> { + typedef + typename PointerUnionTypeSelector<T1, T2, RET_EQ, RET_NE>::Return Return; +}; + +/// Provide PointerLikeTypeTraits for void* that is used by PointerUnion +/// for the two template arguments. +template <typename PT1, typename PT2> class PointerUnionUIntTraits { +public: + static inline void *getAsVoidPointer(void *P) { return P; } + static inline void *getFromVoidPointer(void *P) { return P; } + enum { + PT1BitsAv = (int)(PointerLikeTypeTraits<PT1>::NumLowBitsAvailable), + PT2BitsAv = (int)(PointerLikeTypeTraits<PT2>::NumLowBitsAvailable), + NumLowBitsAvailable = PT1BitsAv < PT2BitsAv ? PT1BitsAv : PT2BitsAv }; - - /// \brief Get a type based on whether two types are the same or not. For: - /// @code - /// typedef typename PointerUnionTypeSelector<T1, T2, EQ, NE>::Return Ret; - /// @endcode - /// Ret will be EQ type if T1 is same as T2 or NE type otherwise. - template <typename T1, typename T2, typename RET_EQ, typename RET_NE> - struct PointerUnionTypeSelector { - typedef typename PointerUnionTypeSelectorReturn<RET_NE>::Return Return; - }; - - template <typename T, typename RET_EQ, typename RET_NE> - struct PointerUnionTypeSelector<T, T, RET_EQ, RET_NE> { - typedef typename PointerUnionTypeSelectorReturn<RET_EQ>::Return Return; - }; - - template <typename T1, typename T2, typename RET_EQ, typename RET_NE> - struct PointerUnionTypeSelectorReturn< - PointerUnionTypeSelector<T1, T2, RET_EQ, RET_NE> > { - typedef typename PointerUnionTypeSelector<T1, T2, RET_EQ, RET_NE>::Return - Return; +}; + +/// A discriminated union of two pointer types, with the discriminator in the +/// low bit of the pointer. +/// +/// This implementation is extremely efficient in space due to leveraging the +/// low bits of the pointer, while exposing a natural and type-safe API. +/// +/// Common use patterns would be something like this: +/// PointerUnion<int*, float*> P; +/// P = (int*)0; +/// printf("%d %d", P.is<int*>(), P.is<float*>()); // prints "1 0" +/// X = P.get<int*>(); // ok. +/// Y = P.get<float*>(); // runtime assertion failure. +/// Z = P.get<double*>(); // compile time failure. +/// P = (float*)0; +/// Y = P.get<float*>(); // ok. +/// X = P.get<int*>(); // runtime assertion failure. +template <typename PT1, typename PT2> class PointerUnion { +public: + typedef PointerIntPair<void *, 1, bool, PointerUnionUIntTraits<PT1, PT2>> + ValTy; + +private: + ValTy Val; + + struct IsPT1 { + static const int Num = 0; }; - - /// Provide PointerLikeTypeTraits for void* that is used by PointerUnion - /// for the two template arguments. - template <typename PT1, typename PT2> - class PointerUnionUIntTraits { - public: - static inline void *getAsVoidPointer(void *P) { return P; } - static inline void *getFromVoidPointer(void *P) { return P; } - enum { - PT1BitsAv = (int)(PointerLikeTypeTraits<PT1>::NumLowBitsAvailable), - PT2BitsAv = (int)(PointerLikeTypeTraits<PT2>::NumLowBitsAvailable), - NumLowBitsAvailable = PT1BitsAv < PT2BitsAv ? PT1BitsAv : PT2BitsAv - }; + struct IsPT2 { + static const int Num = 1; }; + template <typename T> struct UNION_DOESNT_CONTAIN_TYPE {}; + +public: + PointerUnion() {} + + PointerUnion(PT1 V) + : Val(const_cast<void *>( + PointerLikeTypeTraits<PT1>::getAsVoidPointer(V))) {} + PointerUnion(PT2 V) + : Val(const_cast<void *>(PointerLikeTypeTraits<PT2>::getAsVoidPointer(V)), + 1) {} + + /// Test if the pointer held in the union is null, regardless of + /// which type it is. + bool isNull() const { + // Convert from the void* to one of the pointer types, to make sure that + // we recursively strip off low bits if we have a nested PointerUnion. + return !PointerLikeTypeTraits<PT1>::getFromVoidPointer(Val.getPointer()); + } + explicit operator bool() const { return !isNull(); } + + /// Test if the Union currently holds the type matching T. + template <typename T> int is() const { + typedef typename ::llvm::PointerUnionTypeSelector< + PT1, T, IsPT1, ::llvm::PointerUnionTypeSelector< + PT2, T, IsPT2, UNION_DOESNT_CONTAIN_TYPE<T>>>::Return + Ty; + int TyNo = Ty::Num; + return static_cast<int>(Val.getInt()) == TyNo; + } - /// PointerUnion - This implements a discriminated union of two pointer types, - /// and keeps the discriminator bit-mangled into the low bits of the pointer. - /// This allows the implementation to be extremely efficient in space, but - /// permits a very natural and type-safe API. + /// Returns the value of the specified pointer type. /// - /// Common use patterns would be something like this: - /// PointerUnion<int*, float*> P; - /// P = (int*)0; - /// printf("%d %d", P.is<int*>(), P.is<float*>()); // prints "1 0" - /// X = P.get<int*>(); // ok. - /// Y = P.get<float*>(); // runtime assertion failure. - /// Z = P.get<double*>(); // compile time failure. - /// P = (float*)0; - /// Y = P.get<float*>(); // ok. - /// X = P.get<int*>(); // runtime assertion failure. - template <typename PT1, typename PT2> - class PointerUnion { - public: - typedef PointerIntPair<void*, 1, bool, - PointerUnionUIntTraits<PT1,PT2> > ValTy; - private: - ValTy Val; - - struct IsPT1 { - static const int Num = 0; - }; - struct IsPT2 { - static const int Num = 1; - }; - template <typename T> - struct UNION_DOESNT_CONTAIN_TYPE { }; - - public: - PointerUnion() {} - - PointerUnion(PT1 V) : Val( - const_cast<void *>(PointerLikeTypeTraits<PT1>::getAsVoidPointer(V))) { - } - PointerUnion(PT2 V) : Val( - const_cast<void *>(PointerLikeTypeTraits<PT2>::getAsVoidPointer(V)), 1) { - } - - /// isNull - Return true if the pointer held in the union is null, - /// regardless of which type it is. - bool isNull() const { - // Convert from the void* to one of the pointer types, to make sure that - // we recursively strip off low bits if we have a nested PointerUnion. - return !PointerLikeTypeTraits<PT1>::getFromVoidPointer(Val.getPointer()); - } - explicit operator bool() const { return !isNull(); } - - /// is<T>() return true if the Union currently holds the type matching T. - template<typename T> - int is() const { - typedef typename - ::llvm::PointerUnionTypeSelector<PT1, T, IsPT1, - ::llvm::PointerUnionTypeSelector<PT2, T, IsPT2, - UNION_DOESNT_CONTAIN_TYPE<T> > >::Return Ty; - int TyNo = Ty::Num; - return static_cast<int>(Val.getInt()) == TyNo; - } - - /// get<T>() - Return the value of the specified pointer type. If the - /// specified pointer type is incorrect, assert. - template<typename T> - T get() const { - assert(is<T>() && "Invalid accessor called"); - return PointerLikeTypeTraits<T>::getFromVoidPointer(Val.getPointer()); - } + /// If the specified pointer type is incorrect, assert. + template <typename T> T get() const { + assert(is<T>() && "Invalid accessor called"); + return PointerLikeTypeTraits<T>::getFromVoidPointer(Val.getPointer()); + } - /// dyn_cast<T>() - If the current value is of the specified pointer type, - /// return it, otherwise return null. - template<typename T> - T dyn_cast() const { - if (is<T>()) return get<T>(); - return T(); - } + /// Returns the current pointer if it is of the specified pointer type, + /// otherwises returns null. + template <typename T> T dyn_cast() const { + if (is<T>()) + return get<T>(); + return T(); + } - /// \brief If the union is set to the first pointer type get an address - /// pointing to it. - PT1 const *getAddrOfPtr1() const { - return const_cast<PointerUnion *>(this)->getAddrOfPtr1(); - } + /// If the union is set to the first pointer type get an address pointing to + /// it. + PT1 const *getAddrOfPtr1() const { + return const_cast<PointerUnion *>(this)->getAddrOfPtr1(); + } - /// \brief If the union is set to the first pointer type get an address - /// pointing to it. - PT1 *getAddrOfPtr1() { - assert(is<PT1>() && "Val is not the first pointer"); - assert(get<PT1>() == Val.getPointer() && - "Can't get the address because PointerLikeTypeTraits changes the ptr"); - return (PT1 *)Val.getAddrOfPointer(); - } + /// If the union is set to the first pointer type get an address pointing to + /// it. + PT1 *getAddrOfPtr1() { + assert(is<PT1>() && "Val is not the first pointer"); + assert( + get<PT1>() == Val.getPointer() && + "Can't get the address because PointerLikeTypeTraits changes the ptr"); + return (PT1 *)Val.getAddrOfPointer(); + } - /// \brief Assignment from nullptr which just clears the union. - const PointerUnion &operator=(std::nullptr_t) { - Val.initWithPointer(nullptr); - return *this; - } + /// Assignment from nullptr which just clears the union. + const PointerUnion &operator=(std::nullptr_t) { + Val.initWithPointer(nullptr); + return *this; + } - /// Assignment operators - Allow assigning into this union from either - /// pointer type, setting the discriminator to remember what it came from. - const PointerUnion &operator=(const PT1 &RHS) { - Val.initWithPointer( - const_cast<void *>(PointerLikeTypeTraits<PT1>::getAsVoidPointer(RHS))); - return *this; - } - const PointerUnion &operator=(const PT2 &RHS) { - Val.setPointerAndInt( + /// Assignment operators - Allow assigning into this union from either + /// pointer type, setting the discriminator to remember what it came from. + const PointerUnion &operator=(const PT1 &RHS) { + Val.initWithPointer( + const_cast<void *>(PointerLikeTypeTraits<PT1>::getAsVoidPointer(RHS))); + return *this; + } + const PointerUnion &operator=(const PT2 &RHS) { + Val.setPointerAndInt( const_cast<void *>(PointerLikeTypeTraits<PT2>::getAsVoidPointer(RHS)), 1); - return *this; - } - - void *getOpaqueValue() const { return Val.getOpaqueValue(); } - static inline PointerUnion getFromOpaqueValue(void *VP) { - PointerUnion V; - V.Val = ValTy::getFromOpaqueValue(VP); - return V; - } - }; - - template<typename PT1, typename PT2> - static bool operator==(PointerUnion<PT1, PT2> lhs, - PointerUnion<PT1, PT2> rhs) { - return lhs.getOpaqueValue() == rhs.getOpaqueValue(); + return *this; } - template<typename PT1, typename PT2> - static bool operator!=(PointerUnion<PT1, PT2> lhs, - PointerUnion<PT1, PT2> rhs) { - return lhs.getOpaqueValue() != rhs.getOpaqueValue(); + void *getOpaqueValue() const { return Val.getOpaqueValue(); } + static inline PointerUnion getFromOpaqueValue(void *VP) { + PointerUnion V; + V.Val = ValTy::getFromOpaqueValue(VP); + return V; } +}; - template<typename PT1, typename PT2> - static bool operator<(PointerUnion<PT1, PT2> lhs, - PointerUnion<PT1, PT2> rhs) { - return lhs.getOpaqueValue() < rhs.getOpaqueValue(); - } +template <typename PT1, typename PT2> +static bool operator==(PointerUnion<PT1, PT2> lhs, PointerUnion<PT1, PT2> rhs) { + return lhs.getOpaqueValue() == rhs.getOpaqueValue(); +} - // Teach SmallPtrSet that PointerUnion is "basically a pointer", that has - // # low bits available = min(PT1bits,PT2bits)-1. - template<typename PT1, typename PT2> - class PointerLikeTypeTraits<PointerUnion<PT1, PT2> > { - public: - static inline void * - getAsVoidPointer(const PointerUnion<PT1, PT2> &P) { - return P.getOpaqueValue(); - } - static inline PointerUnion<PT1, PT2> - getFromVoidPointer(void *P) { - return PointerUnion<PT1, PT2>::getFromOpaqueValue(P); - } +template <typename PT1, typename PT2> +static bool operator!=(PointerUnion<PT1, PT2> lhs, PointerUnion<PT1, PT2> rhs) { + return lhs.getOpaqueValue() != rhs.getOpaqueValue(); +} + +template <typename PT1, typename PT2> +static bool operator<(PointerUnion<PT1, PT2> lhs, PointerUnion<PT1, PT2> rhs) { + return lhs.getOpaqueValue() < rhs.getOpaqueValue(); +} + +// Teach SmallPtrSet that PointerUnion is "basically a pointer", that has +// # low bits available = min(PT1bits,PT2bits)-1. +template <typename PT1, typename PT2> +class PointerLikeTypeTraits<PointerUnion<PT1, PT2>> { +public: + static inline void *getAsVoidPointer(const PointerUnion<PT1, PT2> &P) { + return P.getOpaqueValue(); + } + static inline PointerUnion<PT1, PT2> getFromVoidPointer(void *P) { + return PointerUnion<PT1, PT2>::getFromOpaqueValue(P); + } - // The number of bits available are the min of the two pointer types. - enum { - NumLowBitsAvailable = - PointerLikeTypeTraits<typename PointerUnion<PT1,PT2>::ValTy> - ::NumLowBitsAvailable - }; + // The number of bits available are the min of the two pointer types. + enum { + NumLowBitsAvailable = PointerLikeTypeTraits< + typename PointerUnion<PT1, PT2>::ValTy>::NumLowBitsAvailable }; +}; +/// A pointer union of three pointer types. See documentation for PointerUnion +/// for usage. +template <typename PT1, typename PT2, typename PT3> class PointerUnion3 { +public: + typedef PointerUnion<PT1, PT2> InnerUnion; + typedef PointerUnion<InnerUnion, PT3> ValTy; - /// PointerUnion3 - This is a pointer union of three pointer types. See - /// documentation for PointerUnion for usage. - template <typename PT1, typename PT2, typename PT3> - class PointerUnion3 { - public: - typedef PointerUnion<PT1, PT2> InnerUnion; - typedef PointerUnion<InnerUnion, PT3> ValTy; - private: - ValTy Val; +private: + ValTy Val; - struct IsInnerUnion { - ValTy Val; - IsInnerUnion(ValTy val) : Val(val) { } - template<typename T> - int is() const { - return Val.template is<InnerUnion>() && - Val.template get<InnerUnion>().template is<T>(); - } - template<typename T> - T get() const { - return Val.template get<InnerUnion>().template get<T>(); - } - }; - - struct IsPT3 { - ValTy Val; - IsPT3(ValTy val) : Val(val) { } - template<typename T> - int is() const { - return Val.template is<T>(); - } - template<typename T> - T get() const { - return Val.template get<T>(); - } - }; - - public: - PointerUnion3() {} - - PointerUnion3(PT1 V) { - Val = InnerUnion(V); - } - PointerUnion3(PT2 V) { - Val = InnerUnion(V); + struct IsInnerUnion { + ValTy Val; + IsInnerUnion(ValTy val) : Val(val) {} + template <typename T> int is() const { + return Val.template is<InnerUnion>() && + Val.template get<InnerUnion>().template is<T>(); } - PointerUnion3(PT3 V) { - Val = V; + template <typename T> T get() const { + return Val.template get<InnerUnion>().template get<T>(); } + }; - /// isNull - Return true if the pointer held in the union is null, - /// regardless of which type it is. - bool isNull() const { return Val.isNull(); } - explicit operator bool() const { return !isNull(); } - - /// is<T>() return true if the Union currently holds the type matching T. - template<typename T> - int is() const { - // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. - typedef typename - ::llvm::PointerUnionTypeSelector<PT1, T, IsInnerUnion, - ::llvm::PointerUnionTypeSelector<PT2, T, IsInnerUnion, IsPT3 > - >::Return Ty; - return Ty(Val).template is<T>(); - } + struct IsPT3 { + ValTy Val; + IsPT3(ValTy val) : Val(val) {} + template <typename T> int is() const { return Val.template is<T>(); } + template <typename T> T get() const { return Val.template get<T>(); } + }; - /// get<T>() - Return the value of the specified pointer type. If the - /// specified pointer type is incorrect, assert. - template<typename T> - T get() const { - assert(is<T>() && "Invalid accessor called"); - // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. - typedef typename - ::llvm::PointerUnionTypeSelector<PT1, T, IsInnerUnion, - ::llvm::PointerUnionTypeSelector<PT2, T, IsInnerUnion, IsPT3 > - >::Return Ty; - return Ty(Val).template get<T>(); - } +public: + PointerUnion3() {} + + PointerUnion3(PT1 V) { Val = InnerUnion(V); } + PointerUnion3(PT2 V) { Val = InnerUnion(V); } + PointerUnion3(PT3 V) { Val = V; } + + /// Test if the pointer held in the union is null, regardless of + /// which type it is. + bool isNull() const { return Val.isNull(); } + explicit operator bool() const { return !isNull(); } + + /// Test if the Union currently holds the type matching T. + template <typename T> int is() const { + // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. + typedef typename ::llvm::PointerUnionTypeSelector< + PT1, T, IsInnerUnion, + ::llvm::PointerUnionTypeSelector<PT2, T, IsInnerUnion, IsPT3>>::Return + Ty; + return Ty(Val).template is<T>(); + } - /// dyn_cast<T>() - If the current value is of the specified pointer type, - /// return it, otherwise return null. - template<typename T> - T dyn_cast() const { - if (is<T>()) return get<T>(); - return T(); - } + /// Returns the value of the specified pointer type. + /// + /// If the specified pointer type is incorrect, assert. + template <typename T> T get() const { + assert(is<T>() && "Invalid accessor called"); + // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. + typedef typename ::llvm::PointerUnionTypeSelector< + PT1, T, IsInnerUnion, + ::llvm::PointerUnionTypeSelector<PT2, T, IsInnerUnion, IsPT3>>::Return + Ty; + return Ty(Val).template get<T>(); + } - /// \brief Assignment from nullptr which just clears the union. - const PointerUnion3 &operator=(std::nullptr_t) { - Val = nullptr; - return *this; - } + /// Returns the current pointer if it is of the specified pointer type, + /// otherwises returns null. + template <typename T> T dyn_cast() const { + if (is<T>()) + return get<T>(); + return T(); + } - /// Assignment operators - Allow assigning into this union from either - /// pointer type, setting the discriminator to remember what it came from. - const PointerUnion3 &operator=(const PT1 &RHS) { - Val = InnerUnion(RHS); - return *this; - } - const PointerUnion3 &operator=(const PT2 &RHS) { - Val = InnerUnion(RHS); - return *this; - } - const PointerUnion3 &operator=(const PT3 &RHS) { - Val = RHS; - return *this; - } + /// Assignment from nullptr which just clears the union. + const PointerUnion3 &operator=(std::nullptr_t) { + Val = nullptr; + return *this; + } - void *getOpaqueValue() const { return Val.getOpaqueValue(); } - static inline PointerUnion3 getFromOpaqueValue(void *VP) { - PointerUnion3 V; - V.Val = ValTy::getFromOpaqueValue(VP); - return V; - } - }; + /// Assignment operators - Allow assigning into this union from either + /// pointer type, setting the discriminator to remember what it came from. + const PointerUnion3 &operator=(const PT1 &RHS) { + Val = InnerUnion(RHS); + return *this; + } + const PointerUnion3 &operator=(const PT2 &RHS) { + Val = InnerUnion(RHS); + return *this; + } + const PointerUnion3 &operator=(const PT3 &RHS) { + Val = RHS; + return *this; + } - // Teach SmallPtrSet that PointerUnion3 is "basically a pointer", that has - // # low bits available = min(PT1bits,PT2bits,PT2bits)-2. - template<typename PT1, typename PT2, typename PT3> - class PointerLikeTypeTraits<PointerUnion3<PT1, PT2, PT3> > { - public: - static inline void * - getAsVoidPointer(const PointerUnion3<PT1, PT2, PT3> &P) { - return P.getOpaqueValue(); - } - static inline PointerUnion3<PT1, PT2, PT3> - getFromVoidPointer(void *P) { - return PointerUnion3<PT1, PT2, PT3>::getFromOpaqueValue(P); - } + void *getOpaqueValue() const { return Val.getOpaqueValue(); } + static inline PointerUnion3 getFromOpaqueValue(void *VP) { + PointerUnion3 V; + V.Val = ValTy::getFromOpaqueValue(VP); + return V; + } +}; + +// Teach SmallPtrSet that PointerUnion3 is "basically a pointer", that has +// # low bits available = min(PT1bits,PT2bits,PT2bits)-2. +template <typename PT1, typename PT2, typename PT3> +class PointerLikeTypeTraits<PointerUnion3<PT1, PT2, PT3>> { +public: + static inline void *getAsVoidPointer(const PointerUnion3<PT1, PT2, PT3> &P) { + return P.getOpaqueValue(); + } + static inline PointerUnion3<PT1, PT2, PT3> getFromVoidPointer(void *P) { + return PointerUnion3<PT1, PT2, PT3>::getFromOpaqueValue(P); + } - // The number of bits available are the min of the two pointer types. - enum { - NumLowBitsAvailable = - PointerLikeTypeTraits<typename PointerUnion3<PT1, PT2, PT3>::ValTy> - ::NumLowBitsAvailable - }; + // The number of bits available are the min of the two pointer types. + enum { + NumLowBitsAvailable = PointerLikeTypeTraits< + typename PointerUnion3<PT1, PT2, PT3>::ValTy>::NumLowBitsAvailable }; +}; + +/// A pointer union of four pointer types. See documentation for PointerUnion +/// for usage. +template <typename PT1, typename PT2, typename PT3, typename PT4> +class PointerUnion4 { +public: + typedef PointerUnion<PT1, PT2> InnerUnion1; + typedef PointerUnion<PT3, PT4> InnerUnion2; + typedef PointerUnion<InnerUnion1, InnerUnion2> ValTy; + +private: + ValTy Val; + +public: + PointerUnion4() {} + + PointerUnion4(PT1 V) { Val = InnerUnion1(V); } + PointerUnion4(PT2 V) { Val = InnerUnion1(V); } + PointerUnion4(PT3 V) { Val = InnerUnion2(V); } + PointerUnion4(PT4 V) { Val = InnerUnion2(V); } + + /// Test if the pointer held in the union is null, regardless of + /// which type it is. + bool isNull() const { return Val.isNull(); } + explicit operator bool() const { return !isNull(); } + + /// Test if the Union currently holds the type matching T. + template <typename T> int is() const { + // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. + typedef typename ::llvm::PointerUnionTypeSelector< + PT1, T, InnerUnion1, ::llvm::PointerUnionTypeSelector< + PT2, T, InnerUnion1, InnerUnion2>>::Return Ty; + return Val.template is<Ty>() && Val.template get<Ty>().template is<T>(); + } - /// PointerUnion4 - This is a pointer union of four pointer types. See - /// documentation for PointerUnion for usage. - template <typename PT1, typename PT2, typename PT3, typename PT4> - class PointerUnion4 { - public: - typedef PointerUnion<PT1, PT2> InnerUnion1; - typedef PointerUnion<PT3, PT4> InnerUnion2; - typedef PointerUnion<InnerUnion1, InnerUnion2> ValTy; - private: - ValTy Val; - public: - PointerUnion4() {} - - PointerUnion4(PT1 V) { - Val = InnerUnion1(V); - } - PointerUnion4(PT2 V) { - Val = InnerUnion1(V); - } - PointerUnion4(PT3 V) { - Val = InnerUnion2(V); - } - PointerUnion4(PT4 V) { - Val = InnerUnion2(V); - } - - /// isNull - Return true if the pointer held in the union is null, - /// regardless of which type it is. - bool isNull() const { return Val.isNull(); } - explicit operator bool() const { return !isNull(); } - - /// is<T>() return true if the Union currently holds the type matching T. - template<typename T> - int is() const { - // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. - typedef typename - ::llvm::PointerUnionTypeSelector<PT1, T, InnerUnion1, - ::llvm::PointerUnionTypeSelector<PT2, T, InnerUnion1, InnerUnion2 > - >::Return Ty; - return Val.template is<Ty>() && - Val.template get<Ty>().template is<T>(); - } + /// Returns the value of the specified pointer type. + /// + /// If the specified pointer type is incorrect, assert. + template <typename T> T get() const { + assert(is<T>() && "Invalid accessor called"); + // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. + typedef typename ::llvm::PointerUnionTypeSelector< + PT1, T, InnerUnion1, ::llvm::PointerUnionTypeSelector< + PT2, T, InnerUnion1, InnerUnion2>>::Return Ty; + return Val.template get<Ty>().template get<T>(); + } - /// get<T>() - Return the value of the specified pointer type. If the - /// specified pointer type is incorrect, assert. - template<typename T> - T get() const { - assert(is<T>() && "Invalid accessor called"); - // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. - typedef typename - ::llvm::PointerUnionTypeSelector<PT1, T, InnerUnion1, - ::llvm::PointerUnionTypeSelector<PT2, T, InnerUnion1, InnerUnion2 > - >::Return Ty; - return Val.template get<Ty>().template get<T>(); - } + /// Returns the current pointer if it is of the specified pointer type, + /// otherwises returns null. + template <typename T> T dyn_cast() const { + if (is<T>()) + return get<T>(); + return T(); + } - /// dyn_cast<T>() - If the current value is of the specified pointer type, - /// return it, otherwise return null. - template<typename T> - T dyn_cast() const { - if (is<T>()) return get<T>(); - return T(); - } + /// Assignment from nullptr which just clears the union. + const PointerUnion4 &operator=(std::nullptr_t) { + Val = nullptr; + return *this; + } - /// \brief Assignment from nullptr which just clears the union. - const PointerUnion4 &operator=(std::nullptr_t) { - Val = nullptr; - return *this; - } + /// Assignment operators - Allow assigning into this union from either + /// pointer type, setting the discriminator to remember what it came from. + const PointerUnion4 &operator=(const PT1 &RHS) { + Val = InnerUnion1(RHS); + return *this; + } + const PointerUnion4 &operator=(const PT2 &RHS) { + Val = InnerUnion1(RHS); + return *this; + } + const PointerUnion4 &operator=(const PT3 &RHS) { + Val = InnerUnion2(RHS); + return *this; + } + const PointerUnion4 &operator=(const PT4 &RHS) { + Val = InnerUnion2(RHS); + return *this; + } - /// Assignment operators - Allow assigning into this union from either - /// pointer type, setting the discriminator to remember what it came from. - const PointerUnion4 &operator=(const PT1 &RHS) { - Val = InnerUnion1(RHS); - return *this; - } - const PointerUnion4 &operator=(const PT2 &RHS) { - Val = InnerUnion1(RHS); - return *this; - } - const PointerUnion4 &operator=(const PT3 &RHS) { - Val = InnerUnion2(RHS); - return *this; - } - const PointerUnion4 &operator=(const PT4 &RHS) { - Val = InnerUnion2(RHS); - return *this; - } + void *getOpaqueValue() const { return Val.getOpaqueValue(); } + static inline PointerUnion4 getFromOpaqueValue(void *VP) { + PointerUnion4 V; + V.Val = ValTy::getFromOpaqueValue(VP); + return V; + } +}; + +// Teach SmallPtrSet that PointerUnion4 is "basically a pointer", that has +// # low bits available = min(PT1bits,PT2bits,PT2bits)-2. +template <typename PT1, typename PT2, typename PT3, typename PT4> +class PointerLikeTypeTraits<PointerUnion4<PT1, PT2, PT3, PT4>> { +public: + static inline void * + getAsVoidPointer(const PointerUnion4<PT1, PT2, PT3, PT4> &P) { + return P.getOpaqueValue(); + } + static inline PointerUnion4<PT1, PT2, PT3, PT4> getFromVoidPointer(void *P) { + return PointerUnion4<PT1, PT2, PT3, PT4>::getFromOpaqueValue(P); + } - void *getOpaqueValue() const { return Val.getOpaqueValue(); } - static inline PointerUnion4 getFromOpaqueValue(void *VP) { - PointerUnion4 V; - V.Val = ValTy::getFromOpaqueValue(VP); - return V; - } + // The number of bits available are the min of the two pointer types. + enum { + NumLowBitsAvailable = PointerLikeTypeTraits< + typename PointerUnion4<PT1, PT2, PT3, PT4>::ValTy>::NumLowBitsAvailable }; +}; - // Teach SmallPtrSet that PointerUnion4 is "basically a pointer", that has - // # low bits available = min(PT1bits,PT2bits,PT2bits)-2. - template<typename PT1, typename PT2, typename PT3, typename PT4> - class PointerLikeTypeTraits<PointerUnion4<PT1, PT2, PT3, PT4> > { - public: - static inline void * - getAsVoidPointer(const PointerUnion4<PT1, PT2, PT3, PT4> &P) { - return P.getOpaqueValue(); - } - static inline PointerUnion4<PT1, PT2, PT3, PT4> - getFromVoidPointer(void *P) { - return PointerUnion4<PT1, PT2, PT3, PT4>::getFromOpaqueValue(P); - } +// Teach DenseMap how to use PointerUnions as keys. +template <typename T, typename U> struct DenseMapInfo<PointerUnion<T, U>> { + typedef PointerUnion<T, U> Pair; + typedef DenseMapInfo<T> FirstInfo; + typedef DenseMapInfo<U> SecondInfo; - // The number of bits available are the min of the two pointer types. - enum { - NumLowBitsAvailable = - PointerLikeTypeTraits<typename PointerUnion4<PT1, PT2, PT3, PT4>::ValTy> - ::NumLowBitsAvailable - }; - }; - - // Teach DenseMap how to use PointerUnions as keys. - template<typename T, typename U> - struct DenseMapInfo<PointerUnion<T, U> > { - typedef PointerUnion<T, U> Pair; - typedef DenseMapInfo<T> FirstInfo; - typedef DenseMapInfo<U> SecondInfo; + static inline Pair getEmptyKey() { return Pair(FirstInfo::getEmptyKey()); } + static inline Pair getTombstoneKey() { + return Pair(FirstInfo::getTombstoneKey()); + } + static unsigned getHashValue(const Pair &PairVal) { + intptr_t key = (intptr_t)PairVal.getOpaqueValue(); + return DenseMapInfo<intptr_t>::getHashValue(key); + } + static bool isEqual(const Pair &LHS, const Pair &RHS) { + return LHS.template is<T>() == RHS.template is<T>() && + (LHS.template is<T>() ? FirstInfo::isEqual(LHS.template get<T>(), + RHS.template get<T>()) + : SecondInfo::isEqual(LHS.template get<U>(), + RHS.template get<U>())); + } +}; - static inline Pair getEmptyKey() { - return Pair(FirstInfo::getEmptyKey()); - } - static inline Pair getTombstoneKey() { - return Pair(FirstInfo::getTombstoneKey()); - } - static unsigned getHashValue(const Pair &PairVal) { - intptr_t key = (intptr_t)PairVal.getOpaqueValue(); - return DenseMapInfo<intptr_t>::getHashValue(key); - } - static bool isEqual(const Pair &LHS, const Pair &RHS) { - return LHS.template is<T>() == RHS.template is<T>() && - (LHS.template is<T>() ? - FirstInfo::isEqual(LHS.template get<T>(), - RHS.template get<T>()) : - SecondInfo::isEqual(LHS.template get<U>(), - RHS.template get<U>())); - } - }; } #endif diff --git a/include/llvm/ADT/PostOrderIterator.h b/include/llvm/ADT/PostOrderIterator.h index 759a2db24f2a..ce343a161b7b 100644 --- a/include/llvm/ADT/PostOrderIterator.h +++ b/include/llvm/ADT/PostOrderIterator.h @@ -215,8 +215,8 @@ struct ipo_iterator : public po_iterator<Inverse<T>, SetType, External > { }; template <class T> -ipo_iterator<T> ipo_begin(const T &G, bool Reverse = false) { - return ipo_iterator<T>::begin(G, Reverse); +ipo_iterator<T> ipo_begin(const T &G) { + return ipo_iterator<T>::begin(G); } template <class T> @@ -225,8 +225,8 @@ ipo_iterator<T> ipo_end(const T &G){ } template <class T> -iterator_range<ipo_iterator<T>> inverse_post_order(const T &G, bool Reverse = false) { - return make_range(ipo_begin(G, Reverse), ipo_end(G)); +iterator_range<ipo_iterator<T>> inverse_post_order(const T &G) { + return make_range(ipo_begin(G), ipo_end(G)); } // Provide global definitions of external inverse postorder iterators... diff --git a/include/llvm/ADT/STLExtras.h b/include/llvm/ADT/STLExtras.h index b68345a1dcf6..d4360fa8d218 100644 --- a/include/llvm/ADT/STLExtras.h +++ b/include/llvm/ADT/STLExtras.h @@ -196,6 +196,41 @@ inline mapped_iterator<ItTy, FuncTy> map_iterator(const ItTy &I, FuncTy F) { return mapped_iterator<ItTy, FuncTy>(I, F); } +/// \brief Metafunction to determine if type T has a member called rbegin(). +template <typename T> struct has_rbegin { + template <typename U> static char(&f(const U &, decltype(&U::rbegin)))[1]; + static char(&f(...))[2]; + const static bool value = sizeof(f(std::declval<T>(), nullptr)) == 1; +}; + +// Returns an iterator_range over the given container which iterates in reverse. +// Note that the container must have rbegin()/rend() methods for this to work. +template <typename ContainerTy> +auto reverse(ContainerTy &&C, + typename std::enable_if<has_rbegin<ContainerTy>::value>::type * = + nullptr) -> decltype(make_range(C.rbegin(), C.rend())) { + return make_range(C.rbegin(), C.rend()); +} + +// Returns a std::reverse_iterator wrapped around the given iterator. +template <typename IteratorTy> +std::reverse_iterator<IteratorTy> make_reverse_iterator(IteratorTy It) { + return std::reverse_iterator<IteratorTy>(It); +} + +// Returns an iterator_range over the given container which iterates in reverse. +// Note that the container must have begin()/end() methods which return +// bidirectional iterators for this to work. +template <typename ContainerTy> +auto reverse( + ContainerTy &&C, + typename std::enable_if<!has_rbegin<ContainerTy>::value>::type * = nullptr) + -> decltype(make_range(llvm::make_reverse_iterator(std::end(C)), + llvm::make_reverse_iterator(std::begin(C)))) { + return make_range(llvm::make_reverse_iterator(std::end(C)), + llvm::make_reverse_iterator(std::begin(C))); +} + //===----------------------------------------------------------------------===// // Extra additions to <utility> //===----------------------------------------------------------------------===// @@ -329,13 +364,28 @@ void DeleteContainerSeconds(Container &C) { } /// Provide wrappers to std::all_of which take ranges instead of having to pass -/// being/end explicitly. +/// begin/end explicitly. template<typename R, class UnaryPredicate> bool all_of(R &&Range, UnaryPredicate &&P) { return std::all_of(Range.begin(), Range.end(), std::forward<UnaryPredicate>(P)); } +/// Provide wrappers to std::any_of which take ranges instead of having to pass +/// begin/end explicitly. +template <typename R, class UnaryPredicate> +bool any_of(R &&Range, UnaryPredicate &&P) { + return std::any_of(Range.begin(), Range.end(), + std::forward<UnaryPredicate>(P)); +} + +/// Provide wrappers to std::find which take ranges instead of having to pass +/// begin/end explicitly. +template<typename R, class T> +auto find(R &&Range, const T &val) -> decltype(Range.begin()) { + return std::find(Range.begin(), Range.end(), val); +} + //===----------------------------------------------------------------------===// // Extra additions to <memory> //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/ScopedHashTable.h b/include/llvm/ADT/ScopedHashTable.h index 5abe76c12259..4af3d6d37e33 100644 --- a/include/llvm/ADT/ScopedHashTable.h +++ b/include/llvm/ADT/ScopedHashTable.h @@ -1,4 +1,4 @@ -//===- ScopedHashTable.h - A simple scoped hash table ---------------------===// +//===- ScopedHashTable.h - A simple scoped hash table -----------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -47,8 +47,8 @@ class ScopedHashTableVal { K Key; V Val; ScopedHashTableVal(const K &key, const V &val) : Key(key), Val(val) {} -public: +public: const K &getKey() const { return Key; } const V &getValue() const { return Val; } V &getValue() { return Val; } @@ -56,7 +56,7 @@ public: ScopedHashTableVal *getNextForKey() { return NextForKey; } const ScopedHashTableVal *getNextForKey() const { return NextForKey; } ScopedHashTableVal *getNextInScope() { return NextInScope; } - + template <typename AllocatorTy> static ScopedHashTableVal *Create(ScopedHashTableVal *nextInScope, ScopedHashTableVal *nextForKey, @@ -66,12 +66,11 @@ public: // Set up the value. new (New) ScopedHashTableVal(key, val); New->NextInScope = nextInScope; - New->NextForKey = nextForKey; + New->NextForKey = nextForKey; return New; } - - template <typename AllocatorTy> - void Destroy(AllocatorTy &Allocator) { + + template <typename AllocatorTy> void Destroy(AllocatorTy &Allocator) { // Free memory referenced by the item. this->~ScopedHashTableVal(); Allocator.Deallocate(this); @@ -90,15 +89,16 @@ class ScopedHashTableScope { /// LastValInScope - This is the last value that was inserted for this scope /// or null if none have been inserted yet. ScopedHashTableVal<K, V> *LastValInScope; - void operator=(ScopedHashTableScope&) = delete; - ScopedHashTableScope(ScopedHashTableScope&) = delete; + void operator=(ScopedHashTableScope &) = delete; + ScopedHashTableScope(ScopedHashTableScope &) = delete; + public: ScopedHashTableScope(ScopedHashTable<K, V, KInfo, AllocatorTy> &HT); ~ScopedHashTableScope(); ScopedHashTableScope *getParentScope() { return PrevScope; } const ScopedHashTableScope *getParentScope() const { return PrevScope; } - + private: friend class ScopedHashTable<K, V, KInfo, AllocatorTy>; ScopedHashTableVal<K, V> *getLastValInScope() { @@ -109,10 +109,10 @@ private: } }; - -template <typename K, typename V, typename KInfo = DenseMapInfo<K> > +template <typename K, typename V, typename KInfo = DenseMapInfo<K>> class ScopedHashTableIterator { ScopedHashTableVal<K, V> *Node; + public: ScopedHashTableIterator(ScopedHashTableVal<K, V> *node) : Node(node) {} @@ -141,7 +141,6 @@ public: } }; - template <typename K, typename V, typename KInfo, typename AllocatorTy> class ScopedHashTable { public: @@ -149,23 +148,24 @@ public: /// to the name of the scope for this hash table. typedef ScopedHashTableScope<K, V, KInfo, AllocatorTy> ScopeTy; typedef unsigned size_type; + private: typedef ScopedHashTableVal<K, V> ValTy; DenseMap<K, ValTy*, KInfo> TopLevelMap; ScopeTy *CurScope; - + AllocatorTy Allocator; - - ScopedHashTable(const ScopedHashTable&); // NOT YET IMPLEMENTED - void operator=(const ScopedHashTable&); // NOT YET IMPLEMENTED + + ScopedHashTable(const ScopedHashTable &); // NOT YET IMPLEMENTED + void operator=(const ScopedHashTable &); // NOT YET IMPLEMENTED friend class ScopedHashTableScope<K, V, KInfo, AllocatorTy>; + public: ScopedHashTable() : CurScope(nullptr) {} ScopedHashTable(AllocatorTy A) : CurScope(0), Allocator(A) {} ~ScopedHashTable() { assert(!CurScope && TopLevelMap.empty() && "Scope imbalance!"); } - /// Access to the allocator. AllocatorTy &getAllocator() { return Allocator; } @@ -180,7 +180,7 @@ public: typename DenseMap<K, ValTy*, KInfo>::iterator I = TopLevelMap.find(Key); if (I != TopLevelMap.end()) return I->second->getValue(); - + return V(); } @@ -198,7 +198,7 @@ public: if (I == TopLevelMap.end()) return end(); return iterator(I->second); } - + ScopeTy *getCurScope() { return CurScope; } const ScopeTy *getCurScope() const { return CurScope; } diff --git a/include/llvm/ADT/SetOperations.h b/include/llvm/ADT/SetOperations.h index 71f5db380f6e..7c9f2fbe066e 100644 --- a/include/llvm/ADT/SetOperations.h +++ b/include/llvm/ADT/SetOperations.h @@ -39,7 +39,7 @@ bool set_union(S1Ty &S1, const S2Ty &S2) { template <class S1Ty, class S2Ty> void set_intersect(S1Ty &S1, const S2Ty &S2) { for (typename S1Ty::iterator I = S1.begin(); I != S1.end();) { - const typename S1Ty::key_type &E = *I; + const auto &E = *I; ++I; if (!S2.count(E)) S1.erase(E); // Erase element if not in S2 } diff --git a/include/llvm/ADT/SetVector.h b/include/llvm/ADT/SetVector.h index a7fd408c854a..bc563570c203 100644 --- a/include/llvm/ADT/SetVector.h +++ b/include/llvm/ADT/SetVector.h @@ -20,6 +20,7 @@ #ifndef LLVM_ADT_SETVECTOR_H #define LLVM_ADT_SETVECTOR_H +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallSet.h" #include <algorithm> #include <cassert> @@ -33,7 +34,7 @@ namespace llvm { /// property of a deterministic iteration order. The order of iteration is the /// order of insertion. template <typename T, typename Vector = std::vector<T>, - typename Set = SmallSet<T, 16> > + typename Set = DenseSet<T>> class SetVector { public: typedef T value_type; @@ -44,6 +45,8 @@ public: typedef Vector vector_type; typedef typename vector_type::const_iterator iterator; typedef typename vector_type::const_iterator const_iterator; + typedef typename vector_type::const_reverse_iterator reverse_iterator; + typedef typename vector_type::const_reverse_iterator const_reverse_iterator; typedef typename vector_type::size_type size_type; /// \brief Construct an empty SetVector @@ -55,6 +58,8 @@ public: insert(Start, End); } + ArrayRef<T> getArrayRef() const { return vector_; } + /// \brief Determine if the SetVector is empty or not. bool empty() const { return vector_.empty(); @@ -85,6 +90,26 @@ public: return vector_.end(); } + /// \brief Get an reverse_iterator to the end of the SetVector. + reverse_iterator rbegin() { + return vector_.rbegin(); + } + + /// \brief Get a const_reverse_iterator to the end of the SetVector. + const_reverse_iterator rbegin() const { + return vector_.rbegin(); + } + + /// \brief Get a reverse_iterator to the beginning of the SetVector. + reverse_iterator rend() { + return vector_.rend(); + } + + /// \brief Get a const_reverse_iterator to the beginning of the SetVector. + const_reverse_iterator rend() const { + return vector_.rend(); + } + /// \brief Return the last element of the SetVector. const T &back() const { assert(!empty() && "Cannot call back() on empty SetVector!"); @@ -150,7 +175,6 @@ public: return true; } - /// \brief Count the number of elements of a given key in the SetVector. /// \returns 0 if the element is not in the SetVector, 1 if it is. size_type count(const key_type &key) const { @@ -169,7 +193,7 @@ public: set_.erase(back()); vector_.pop_back(); } - + T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val() { T Ret = back(); pop_back(); diff --git a/include/llvm/ADT/SmallBitVector.h b/include/llvm/ADT/SmallBitVector.h index ae3d645396fd..4aa3bc217f41 100644 --- a/include/llvm/ADT/SmallBitVector.h +++ b/include/llvm/ADT/SmallBitVector.h @@ -551,19 +551,18 @@ public: } private: - template<bool AddBits, bool InvertMask> + template <bool AddBits, bool InvertMask> void applyMask(const uint32_t *Mask, unsigned MaskWords) { - if (NumBaseBits == 64 && MaskWords >= 2) { - uint64_t M = Mask[0] | (uint64_t(Mask[1]) << 32); - if (InvertMask) M = ~M; - if (AddBits) setSmallBits(getSmallBits() | M); - else setSmallBits(getSmallBits() & ~M); - } else { - uint32_t M = Mask[0]; - if (InvertMask) M = ~M; - if (AddBits) setSmallBits(getSmallBits() | M); - else setSmallBits(getSmallBits() & ~M); - } + assert(MaskWords <= sizeof(uintptr_t) && "Mask is larger than base!"); + uintptr_t M = Mask[0]; + if (NumBaseBits == 64) + M |= uint64_t(Mask[1]) << 32; + if (InvertMask) + M = ~M; + if (AddBits) + setSmallBits(getSmallBits() | M); + else + setSmallBits(getSmallBits() & ~M); } }; diff --git a/include/llvm/ADT/SmallPtrSet.h b/include/llvm/ADT/SmallPtrSet.h index 3e3c9c154ef4..3d98e8fac43b 100644 --- a/include/llvm/ADT/SmallPtrSet.h +++ b/include/llvm/ADT/SmallPtrSet.h @@ -48,6 +48,7 @@ class SmallPtrSetIteratorImpl; /// class SmallPtrSetImplBase { friend class SmallPtrSetIteratorImpl; + protected: /// SmallArray - Points to a fixed size set of buckets, used in 'small mode'. const void **SmallArray; @@ -133,6 +134,7 @@ private: void Grow(unsigned NewSize); void operator=(const SmallPtrSetImplBase &RHS) = delete; + protected: /// swap - Swaps the elements of two sets. /// Note: This method assumes that both sets have the same small size. @@ -148,6 +150,7 @@ class SmallPtrSetIteratorImpl { protected: const void *const *Bucket; const void *const *End; + public: explicit SmallPtrSetIteratorImpl(const void *const *BP, const void*const *E) : Bucket(BP), End(E) { @@ -178,14 +181,14 @@ protected: template<typename PtrTy> class SmallPtrSetIterator : public SmallPtrSetIteratorImpl { typedef PointerLikeTypeTraits<PtrTy> PtrTraits; - + public: typedef PtrTy value_type; typedef PtrTy reference; typedef PtrTy pointer; typedef std::ptrdiff_t difference_type; typedef std::forward_iterator_tag iterator_category; - + explicit SmallPtrSetIterator(const void *const *BP, const void *const *E) : SmallPtrSetIteratorImpl(BP, E) {} @@ -231,7 +234,6 @@ template<unsigned N> struct RoundUpToPowerOfTwo { enum { Val = RoundUpToPowerOfTwoH<N, (N&(N-1)) == 0>::Val }; }; - /// \brief A templated base class for \c SmallPtrSet which provides the /// typesafe interface that is common across all small sizes. @@ -242,7 +244,8 @@ template <typename PtrType> class SmallPtrSetImpl : public SmallPtrSetImplBase { typedef PointerLikeTypeTraits<PtrType> PtrTraits; - SmallPtrSetImpl(const SmallPtrSetImpl&) = delete; + SmallPtrSetImpl(const SmallPtrSetImpl &) = delete; + protected: // Constructors that forward to the base. SmallPtrSetImpl(const void **SmallStorage, const SmallPtrSetImpl &that) @@ -303,6 +306,7 @@ class SmallPtrSet : public SmallPtrSetImpl<PtrType> { enum { SmallSizePowTwo = RoundUpToPowerOfTwo<SmallSize>::Val }; /// SmallStorage - Fixed size storage used in 'small mode'. const void *SmallStorage[SmallSizePowTwo]; + public: SmallPtrSet() : BaseT(SmallStorage, SmallSizePowTwo) {} SmallPtrSet(const SmallPtrSet &that) : BaseT(SmallStorage, that) {} @@ -333,7 +337,6 @@ public: SmallPtrSetImplBase::swap(RHS); } }; - } namespace std { diff --git a/include/llvm/ADT/SmallSet.h b/include/llvm/ADT/SmallSet.h index bc6493554c8b..39a57b87b2a7 100644 --- a/include/llvm/ADT/SmallSet.h +++ b/include/llvm/ADT/SmallSet.h @@ -37,6 +37,7 @@ class SmallSet { std::set<T, C> Set; typedef typename SmallVector<T, N>::const_iterator VIterator; typedef typename SmallVector<T, N>::iterator mutable_iterator; + public: typedef size_t size_type; SmallSet() {} @@ -92,7 +93,7 @@ public: for (; I != E; ++I) insert(*I); } - + bool erase(const T &V) { if (!isSmall()) return Set.erase(V); @@ -108,6 +109,7 @@ public: Vector.clear(); Set.clear(); } + private: bool isSmall() const { return Set.empty(); } diff --git a/include/llvm/ADT/SmallVector.h b/include/llvm/ADT/SmallVector.h index b9384702c3ba..d1062acbbb61 100644 --- a/include/llvm/ADT/SmallVector.h +++ b/include/llvm/ADT/SmallVector.h @@ -109,9 +109,13 @@ public: typedef const T *const_pointer; // forward iterator creation methods. + LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin() { return (iterator)this->BeginX; } + LLVM_ATTRIBUTE_ALWAYS_INLINE const_iterator begin() const { return (const_iterator)this->BeginX; } + LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end() { return (iterator)this->EndX; } + LLVM_ATTRIBUTE_ALWAYS_INLINE const_iterator end() const { return (const_iterator)this->EndX; } protected: iterator capacity_ptr() { return (iterator)this->CapacityX; } @@ -124,6 +128,7 @@ public: reverse_iterator rend() { return reverse_iterator(begin()); } const_reverse_iterator rend() const { return const_reverse_iterator(begin());} + LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const { return end()-begin(); } size_type max_size() const { return size_type(-1) / sizeof(T); } @@ -135,10 +140,12 @@ public: /// Return a pointer to the vector's buffer, even if empty(). const_pointer data() const { return const_pointer(begin()); } + LLVM_ATTRIBUTE_ALWAYS_INLINE reference operator[](size_type idx) { assert(idx < size()); return begin()[idx]; } + LLVM_ATTRIBUTE_ALWAYS_INLINE const_reference operator[](size_type idx) const { assert(idx < size()); return begin()[idx]; diff --git a/include/llvm/ADT/SparseBitVector.h b/include/llvm/ADT/SparseBitVector.h index 20cbe2cddfc2..e6e72413da4e 100644 --- a/include/llvm/ADT/SparseBitVector.h +++ b/include/llvm/ADT/SparseBitVector.h @@ -39,7 +39,6 @@ namespace llvm { /// etc) do not perform as well in practice as a linked list with this iterator /// kept up to date. They are also significantly more memory intensive. - template <unsigned ElementSize = 128> struct SparseBitVectorElement : public ilist_node<SparseBitVectorElement<ElementSize> > { @@ -204,6 +203,7 @@ public: BecameZero = allzero; return changed; } + // Intersect this Element with the complement of RHS and return true if this // one changed. BecameZero is set to true if this element became all-zero // bits. @@ -226,6 +226,7 @@ public: BecameZero = allzero; return changed; } + // Three argument version of intersectWithComplement that intersects // RHS1 & ~RHS2 into this element void intersectWithComplement(const SparseBitVectorElement &RHS1, @@ -408,12 +409,13 @@ class SparseBitVector { // bitmap. return AtEnd == RHS.AtEnd && RHS.BitNumber == BitNumber; } + bool operator!=(const SparseBitVectorIterator &RHS) const { return !(*this == RHS); } - SparseBitVectorIterator(): BitVector(NULL) { - } + SparseBitVectorIterator(): BitVector(nullptr) { + } SparseBitVectorIterator(const SparseBitVector<ElementSize> *RHS, bool end = false):BitVector(RHS) { @@ -453,6 +455,9 @@ public: // Assignment SparseBitVector& operator=(const SparseBitVector& RHS) { + if (this == &RHS) + return *this; + Elements.clear(); ElementListConstIter ElementIter = RHS.Elements.begin(); @@ -559,6 +564,9 @@ public: // Union our bitmap with the RHS and return true if we changed. bool operator|=(const SparseBitVector &RHS) { + if (this == &RHS) + return false; + bool changed = false; ElementListIter Iter1 = Elements.begin(); ElementListConstIter Iter2 = RHS.Elements.begin(); @@ -587,6 +595,9 @@ public: // Intersect our bitmap with the RHS and return true if ours changed. bool operator&=(const SparseBitVector &RHS) { + if (this == &RHS) + return false; + bool changed = false; ElementListIter Iter1 = Elements.begin(); ElementListConstIter Iter2 = RHS.Elements.begin(); @@ -619,9 +630,13 @@ public: ElementListIter IterTmp = Iter1; ++Iter1; Elements.erase(IterTmp); + changed = true; } } - Elements.erase(Iter1, Elements.end()); + if (Iter1 != Elements.end()) { + Elements.erase(Iter1, Elements.end()); + changed = true; + } CurrElementIter = Elements.begin(); return changed; } @@ -629,6 +644,14 @@ public: // Intersect our bitmap with the complement of the RHS and return true // if ours changed. bool intersectWithComplement(const SparseBitVector &RHS) { + if (this == &RHS) { + if (!empty()) { + clear(); + return true; + } + return false; + } + bool changed = false; ElementListIter Iter1 = Elements.begin(); ElementListConstIter Iter2 = RHS.Elements.begin(); @@ -669,12 +692,20 @@ public: return intersectWithComplement(*RHS); } - // Three argument version of intersectWithComplement. // Result of RHS1 & ~RHS2 is stored into this bitmap. void intersectWithComplement(const SparseBitVector<ElementSize> &RHS1, const SparseBitVector<ElementSize> &RHS2) { + if (this == &RHS1) { + intersectWithComplement(RHS2); + return; + } else if (this == &RHS2) { + SparseBitVector RHS2Copy(RHS2); + intersectWithComplement(RHS1, RHS2Copy); + return; + } + Elements.clear(); CurrElementIter = Elements.begin(); ElementListConstIter Iter1 = RHS1.Elements.begin(); @@ -719,8 +750,6 @@ public: Elements.push_back(NewElement); ++Iter1; } - - return; } void intersectWithComplement(const SparseBitVector<ElementSize> *RHS1, @@ -855,9 +884,6 @@ operator-(const SparseBitVector<ElementSize> &LHS, return Result; } - - - // Dump a SparseBitVector to a stream template <unsigned ElementSize> void dump(const SparseBitVector<ElementSize> &LHS, raw_ostream &out) { @@ -875,4 +901,4 @@ void dump(const SparseBitVector<ElementSize> &LHS, raw_ostream &out) { } } // end namespace llvm -#endif +#endif // LLVM_ADT_SPARSEBITVECTOR_H diff --git a/include/llvm/ADT/Statistic.h b/include/llvm/ADT/Statistic.h index d98abc375e8a..7c84e3ef6b4d 100644 --- a/include/llvm/ADT/Statistic.h +++ b/include/llvm/ADT/Statistic.h @@ -28,9 +28,11 @@ #include "llvm/Support/Atomic.h" #include "llvm/Support/Valgrind.h" +#include <memory> namespace llvm { class raw_ostream; +class raw_fd_ostream; class Statistic { public: @@ -170,6 +172,9 @@ void EnableStatistics(); /// \brief Check if statistics are enabled. bool AreStatisticsEnabled(); +/// \brief Return a file stream to print our output on. +std::unique_ptr<raw_fd_ostream> CreateInfoOutputFile(); + /// \brief Print statistics to the file returned by CreateInfoOutputFile(). void PrintStatistics(); diff --git a/include/llvm/ADT/StringMap.h b/include/llvm/ADT/StringMap.h index 9d038560bf92..700bb9e10ef7 100644 --- a/include/llvm/ADT/StringMap.h +++ b/include/llvm/ADT/StringMap.h @@ -30,6 +30,7 @@ namespace llvm { /// StringMapEntryBase - Shared base class of StringMapEntry instances. class StringMapEntryBase { unsigned StrLen; + public: explicit StringMapEntryBase(unsigned Len) : StrLen(Len) {} @@ -48,6 +49,7 @@ protected: unsigned NumItems; unsigned NumTombstones; unsigned ItemSize; + protected: explicit StringMapImpl(unsigned itemSize) : TheTable(nullptr), @@ -85,8 +87,10 @@ protected: /// RemoveKey - Remove the StringMapEntry for the specified key from the /// table, returning it. If the key is not in the table, this returns null. StringMapEntryBase *RemoveKey(StringRef Key); + private: void init(unsigned Size); + public: static StringMapEntryBase *getTombstoneVal() { return (StringMapEntryBase*)-1; @@ -112,6 +116,7 @@ public: template<typename ValueTy> class StringMapEntry : public StringMapEntryBase { StringMapEntry(StringMapEntry &E) = delete; + public: ValueTy second; @@ -205,7 +210,6 @@ public: } }; - /// StringMap - This is an unconventional map that is specialized for handling /// keys that are "strings", which are basically ranges of bytes. This does some /// funky memory allocation and hashing things to make it extremely efficient, @@ -213,9 +217,10 @@ public: template<typename ValueTy, typename AllocatorTy = MallocAllocator> class StringMap : public StringMapImpl { AllocatorTy Allocator; + public: typedef StringMapEntry<ValueTy> MapEntryTy; - + StringMap() : StringMapImpl(static_cast<unsigned>(sizeof(MapEntryTy))) {} explicit StringMap(unsigned InitialSize) : StringMapImpl(InitialSize, static_cast<unsigned>(sizeof(MapEntryTy))) {} @@ -227,6 +232,13 @@ public: : StringMapImpl(InitialSize, static_cast<unsigned>(sizeof(MapEntryTy))), Allocator(A) {} + StringMap(std::initializer_list<std::pair<StringRef, ValueTy>> List) + : StringMapImpl(static_cast<unsigned>(sizeof(MapEntryTy))) { + for (const auto &P : List) { + insert(P); + } + } + StringMap(StringMap &&RHS) : StringMapImpl(std::move(RHS)), Allocator(std::move(RHS.Allocator)) {} @@ -386,11 +398,10 @@ public: } }; - -template<typename ValueTy> -class StringMapConstIterator { +template <typename ValueTy> class StringMapConstIterator { protected: StringMapEntryBase **Ptr; + public: typedef StringMapEntry<ValueTy> value_type; @@ -447,7 +458,6 @@ public: return static_cast<StringMapEntry<ValueTy>*>(*this->Ptr); } }; - } #endif diff --git a/include/llvm/ADT/StringRef.h b/include/llvm/ADT/StringRef.h index 95660a49f1f1..350032b8c4e7 100644 --- a/include/llvm/ADT/StringRef.h +++ b/include/llvm/ADT/StringRef.h @@ -10,6 +10,7 @@ #ifndef LLVM_ADT_STRINGREF_H #define LLVM_ADT_STRINGREF_H +#include "llvm/Support/Compiler.h" #include <algorithm> #include <cassert> #include <cstring> @@ -53,6 +54,7 @@ namespace llvm { // Workaround memcmp issue with null pointers (undefined behavior) // by providing a specialized version + LLVM_ATTRIBUTE_ALWAYS_INLINE static int compareMemory(const char *Lhs, const char *Rhs, size_t Length) { if (Length == 0) { return 0; } return ::memcmp(Lhs,Rhs,Length); @@ -73,6 +75,7 @@ namespace llvm { } /// Construct a string ref from a pointer and length. + LLVM_ATTRIBUTE_ALWAYS_INLINE /*implicit*/ StringRef(const char *data, size_t length) : Data(data), Length(length) { assert((data || length == 0) && @@ -80,6 +83,7 @@ namespace llvm { } /// Construct a string ref from an std::string. + LLVM_ATTRIBUTE_ALWAYS_INLINE /*implicit*/ StringRef(const std::string &Str) : Data(Str.data()), Length(Str.length()) {} @@ -104,12 +108,15 @@ namespace llvm { /// data - Get a pointer to the start of the string (which may not be null /// terminated). + LLVM_ATTRIBUTE_ALWAYS_INLINE const char *data() const { return Data; } /// empty - Check if the string is empty. + LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const { return Length == 0; } /// size - Get the string size. + LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const { return Length; } /// front - Get the first character in the string. @@ -133,6 +140,7 @@ namespace llvm { /// equals - Check for string equality, this is more efficient than /// compare() when the relative ordering of inequal strings isn't needed. + LLVM_ATTRIBUTE_ALWAYS_INLINE bool equals(StringRef RHS) const { return (Length == RHS.Length && compareMemory(Data, RHS.Data, RHS.Length) == 0); @@ -145,6 +153,7 @@ namespace llvm { /// compare - Compare two strings; the result is -1, 0, or 1 if this string /// is lexicographically less than, equal to, or greater than the \p RHS. + LLVM_ATTRIBUTE_ALWAYS_INLINE int compare(StringRef RHS) const { // Check the prefix for a mismatch. if (int Res = compareMemory(Data, RHS.Data, std::min(Length, RHS.Length))) @@ -212,6 +221,7 @@ namespace llvm { /// @{ /// Check if this string starts with the given \p Prefix. + LLVM_ATTRIBUTE_ALWAYS_INLINE bool startswith(StringRef Prefix) const { return Length >= Prefix.Length && compareMemory(Data, Prefix.Data, Prefix.Length) == 0; @@ -221,6 +231,7 @@ namespace llvm { bool startswith_lower(StringRef Prefix) const; /// Check if this string ends with the given \p Suffix. + LLVM_ATTRIBUTE_ALWAYS_INLINE bool endswith(StringRef Suffix) const { return Length >= Suffix.Length && compareMemory(end() - Suffix.Length, Suffix.Data, Suffix.Length) == 0; @@ -237,6 +248,7 @@ namespace llvm { /// /// \returns The index of the first occurrence of \p C, or npos if not /// found. + LLVM_ATTRIBUTE_ALWAYS_INLINE size_t find(char C, size_t From = 0) const { size_t FindBegin = std::min(From, Length); if (FindBegin < Length) { // Avoid calling memchr with nullptr. @@ -402,6 +414,7 @@ namespace llvm { /// \param N The number of characters to included in the substring. If N /// exceeds the number of characters remaining in the string, the string /// suffix (starting with \p Start) will be returned. + LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef substr(size_t Start, size_t N = npos) const { Start = std::min(Start, Length); return StringRef(Data + Start, std::min(N, Length - Start)); @@ -409,6 +422,7 @@ namespace llvm { /// Return a StringRef equal to 'this' but with the first \p N elements /// dropped. + LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_front(size_t N = 1) const { assert(size() >= N && "Dropping more elements than exist"); return substr(N); @@ -416,6 +430,7 @@ namespace llvm { /// Return a StringRef equal to 'this' but with the last \p N elements /// dropped. + LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_back(size_t N = 1) const { assert(size() >= N && "Dropping more elements than exist"); return substr(0, size()-N); @@ -431,6 +446,7 @@ namespace llvm { /// substring. If this is npos, or less than \p Start, or exceeds the /// number of characters remaining in the string, the string suffix /// (starting with \p Start) will be returned. + LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef slice(size_t Start, size_t End) const { Start = std::min(Start, Length); End = std::min(std::max(Start, End), Length); @@ -474,7 +490,7 @@ namespace llvm { /// Split into substrings around the occurrences of a separator string. /// /// Each substring is stored in \p A. If \p MaxSplit is >= 0, at most - /// \p MaxSplit splits are done and consequently <= \p MaxSplit + /// \p MaxSplit splits are done and consequently <= \p MaxSplit + 1 /// elements are added to A. /// If \p KeepEmpty is false, empty strings are not added to \p A. They /// still count when considering \p MaxSplit @@ -489,6 +505,23 @@ namespace llvm { StringRef Separator, int MaxSplit = -1, bool KeepEmpty = true) const; + /// Split into substrings around the occurrences of a separator character. + /// + /// Each substring is stored in \p A. If \p MaxSplit is >= 0, at most + /// \p MaxSplit splits are done and consequently <= \p MaxSplit + 1 + /// elements are added to A. + /// If \p KeepEmpty is false, empty strings are not added to \p A. They + /// still count when considering \p MaxSplit + /// An useful invariant is that + /// Separator.join(A) == *this if MaxSplit == -1 and KeepEmpty == true + /// + /// \param A - Where to put the substrings. + /// \param Separator - The string to split on. + /// \param MaxSplit - The maximum number of times the string is split. + /// \param KeepEmpty - True if empty substring should be added. + void split(SmallVectorImpl<StringRef> &A, char Separator, int MaxSplit = -1, + bool KeepEmpty = true) const; + /// Split into two substrings around the last occurrence of a separator /// character. /// @@ -530,10 +563,12 @@ namespace llvm { /// @name StringRef Comparison Operators /// @{ + LLVM_ATTRIBUTE_ALWAYS_INLINE inline bool operator==(StringRef LHS, StringRef RHS) { return LHS.equals(RHS); } + LLVM_ATTRIBUTE_ALWAYS_INLINE inline bool operator!=(StringRef LHS, StringRef RHS) { return !(LHS == RHS); } diff --git a/include/llvm/ADT/StringSet.h b/include/llvm/ADT/StringSet.h index 3e0cc200b6dd..08626dc7af84 100644 --- a/include/llvm/ADT/StringSet.h +++ b/include/llvm/ADT/StringSet.h @@ -23,6 +23,11 @@ namespace llvm { class StringSet : public llvm::StringMap<char, AllocatorTy> { typedef llvm::StringMap<char, AllocatorTy> base; public: + StringSet() = default; + StringSet(std::initializer_list<StringRef> S) { + for (StringRef X : S) + insert(X); + } std::pair<typename base::iterator, bool> insert(StringRef Key) { assert(!Key.empty()); diff --git a/include/llvm/ADT/StringSwitch.h b/include/llvm/ADT/StringSwitch.h index 0393a0c373ef..42b0fc4bc441 100644 --- a/include/llvm/ADT/StringSwitch.h +++ b/include/llvm/ADT/StringSwitch.h @@ -14,6 +14,7 @@ #define LLVM_ADT_STRINGSWITCH_H #include "llvm/ADT/StringRef.h" +#include "llvm/Support/Compiler.h" #include <cassert> #include <cstring> @@ -48,10 +49,12 @@ class StringSwitch { const T *Result; public: + LLVM_ATTRIBUTE_ALWAYS_INLINE explicit StringSwitch(StringRef S) : Str(S), Result(nullptr) { } template<unsigned N> + LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& Case(const char (&S)[N], const T& Value) { if (!Result && N-1 == Str.size() && (std::memcmp(S, Str.data(), N-1) == 0)) { @@ -62,6 +65,7 @@ public: } template<unsigned N> + LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& EndsWith(const char (&S)[N], const T &Value) { if (!Result && Str.size() >= N-1 && std::memcmp(S, Str.data() + Str.size() + 1 - N, N-1) == 0) { @@ -72,6 +76,7 @@ public: } template<unsigned N> + LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& StartsWith(const char (&S)[N], const T &Value) { if (!Result && Str.size() >= N-1 && std::memcmp(S, Str.data(), N-1) == 0) { @@ -82,32 +87,66 @@ public: } template<unsigned N0, unsigned N1> + LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& Cases(const char (&S0)[N0], const char (&S1)[N1], const T& Value) { - return Case(S0, Value).Case(S1, Value); + if (!Result && ( + (N0-1 == Str.size() && std::memcmp(S0, Str.data(), N0-1) == 0) || + (N1-1 == Str.size() && std::memcmp(S1, Str.data(), N1-1) == 0))) { + Result = &Value; + } + + return *this; } template<unsigned N0, unsigned N1, unsigned N2> + LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& Cases(const char (&S0)[N0], const char (&S1)[N1], const char (&S2)[N2], const T& Value) { - return Case(S0, Value).Case(S1, Value).Case(S2, Value); + if (!Result && ( + (N0-1 == Str.size() && std::memcmp(S0, Str.data(), N0-1) == 0) || + (N1-1 == Str.size() && std::memcmp(S1, Str.data(), N1-1) == 0) || + (N2-1 == Str.size() && std::memcmp(S2, Str.data(), N2-1) == 0))) { + Result = &Value; + } + + return *this; } template<unsigned N0, unsigned N1, unsigned N2, unsigned N3> + LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& Cases(const char (&S0)[N0], const char (&S1)[N1], const char (&S2)[N2], const char (&S3)[N3], const T& Value) { - return Case(S0, Value).Case(S1, Value).Case(S2, Value).Case(S3, Value); + if (!Result && ( + (N0-1 == Str.size() && std::memcmp(S0, Str.data(), N0-1) == 0) || + (N1-1 == Str.size() && std::memcmp(S1, Str.data(), N1-1) == 0) || + (N2-1 == Str.size() && std::memcmp(S2, Str.data(), N2-1) == 0) || + (N3-1 == Str.size() && std::memcmp(S3, Str.data(), N3-1) == 0))) { + Result = &Value; + } + + return *this; } template<unsigned N0, unsigned N1, unsigned N2, unsigned N3, unsigned N4> + LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& Cases(const char (&S0)[N0], const char (&S1)[N1], const char (&S2)[N2], const char (&S3)[N3], const char (&S4)[N4], const T& Value) { - return Case(S0, Value).Case(S1, Value).Case(S2, Value).Case(S3, Value) - .Case(S4, Value); + if (!Result && ( + (N0-1 == Str.size() && std::memcmp(S0, Str.data(), N0-1) == 0) || + (N1-1 == Str.size() && std::memcmp(S1, Str.data(), N1-1) == 0) || + (N2-1 == Str.size() && std::memcmp(S2, Str.data(), N2-1) == 0) || + (N3-1 == Str.size() && std::memcmp(S3, Str.data(), N3-1) == 0) || + (N4-1 == Str.size() && std::memcmp(S4, Str.data(), N4-1) == 0))) { + Result = &Value; + } + + return *this; } + LLVM_ATTRIBUTE_ALWAYS_INLINE R Default(const T& Value) const { if (Result) return *Result; @@ -115,6 +154,7 @@ public: return Value; } + LLVM_ATTRIBUTE_ALWAYS_INLINE operator R() const { assert(Result && "Fell off the end of a string-switch"); return *Result; diff --git a/include/llvm/ADT/TinyPtrVector.h b/include/llvm/ADT/TinyPtrVector.h index f29608f3d3d1..487aa46cf642 100644 --- a/include/llvm/ADT/TinyPtrVector.h +++ b/include/llvm/ADT/TinyPtrVector.h @@ -15,7 +15,7 @@ #include "llvm/ADT/SmallVector.h" namespace llvm { - + /// TinyPtrVector - This class is specialized for cases where there are /// normally 0 or 1 element in a vector, but is general enough to go beyond that /// when required. @@ -150,7 +150,6 @@ public: return Val.getAddrOfPtr1(); return Val.template get<VecTy *>()->begin(); - } iterator end() { if (Val.template is<EltTy>()) diff --git a/include/llvm/ADT/Triple.h b/include/llvm/ADT/Triple.h index 947812d94ecb..e01db0a61fd5 100644 --- a/include/llvm/ADT/Triple.h +++ b/include/llvm/ADT/Triple.h @@ -50,6 +50,7 @@ public: armeb, // ARM (big endian): armeb aarch64, // AArch64 (little endian): aarch64 aarch64_be, // AArch64 (big endian): aarch64_be + avr, // AVR: Atmel AVR microcontroller bpfel, // eBPF or extended BPF or 64-bit BPF (little endian) bpfeb, // eBPF or extended BPF or 64-bit BPF (big endian) hexagon, // Hexagon: hexagon @@ -75,8 +76,8 @@ public: xcore, // XCore: xcore nvptx, // NVPTX: 32-bit nvptx64, // NVPTX: 64-bit - le32, // le32: generic little-endian 32-bit CPU (PNaCl / Emscripten) - le64, // le64: generic little-endian 64-bit CPU (PNaCl / Emscripten) + le32, // le32: generic little-endian 32-bit CPU (PNaCl) + le64, // le64: generic little-endian 64-bit CPU (PNaCl) amdil, // AMDIL amdil64, // AMDIL with 64-bit pointers hsail, // AMD HSAIL @@ -92,12 +93,14 @@ public: enum SubArchType { NoSubArch, + ARMSubArch_v8_2a, ARMSubArch_v8_1a, ARMSubArch_v8, ARMSubArch_v7, ARMSubArch_v7em, ARMSubArch_v7m, ARMSubArch_v7s, + ARMSubArch_v7k, ARMSubArch_v6, ARMSubArch_v6m, ARMSubArch_v6k, @@ -124,7 +127,8 @@ public: MipsTechnologies, NVIDIA, CSR, - LastVendorType = CSR + Myriad, + LastVendorType = Myriad }; enum OSType { UnknownOS, @@ -153,7 +157,10 @@ public: NVCL, // NVIDIA OpenCL AMDHSA, // AMD HSA Runtime PS4, - LastOSType = PS4 + ELFIAMCU, + TvOS, // Apple tvOS + WatchOS, // Apple watchOS + LastOSType = WatchOS }; enum EnvironmentType { UnknownEnvironment, @@ -170,7 +177,9 @@ public: MSVC, Itanium, Cygnus, - LastEnvironmentType = Cygnus + AMDOpenCL, + CoreCLR, + LastEnvironmentType = CoreCLR }; enum ObjectFormatType { UnknownObjectFormat, @@ -205,7 +214,7 @@ public: /// @name Constructors /// @{ - /// \brief Default constructor is the same as an empty string and leaves all + /// Default constructor is the same as an empty string and leaves all /// triple fields unknown. Triple() : Data(), Arch(), Vendor(), OS(), Environment(), ObjectFormat() {} @@ -231,7 +240,7 @@ public: /// common case in which otherwise valid components are in the wrong order. static std::string normalize(StringRef Str); - /// \brief Return the normalized form of this triple's string. + /// Return the normalized form of this triple's string. std::string normalize() const { return normalize(Data); } /// @} @@ -259,7 +268,7 @@ public: /// getEnvironment - Get the parsed environment type of this triple. EnvironmentType getEnvironment() const { return Environment; } - /// \brief Parse the version number from the OS name component of the + /// Parse the version number from the OS name component of the /// triple, if present. /// /// For example, "fooos1.2.3" would return (1, 2, 3). @@ -295,10 +304,15 @@ public: unsigned &Micro) const; /// getiOSVersion - Parse the version number as with getOSVersion. This should - /// only be called with IOS triples. + /// only be called with IOS or generic triples. void getiOSVersion(unsigned &Major, unsigned &Minor, unsigned &Micro) const; + /// getWatchOSVersion - Parse the version number as with getOSVersion. This + /// should only be called with WatchOS or generic triples. + void getWatchOSVersion(unsigned &Major, unsigned &Minor, + unsigned &Micro) const; + /// @} /// @name Direct Component Access /// @{ @@ -331,7 +345,7 @@ public: /// @name Convenience Predicates /// @{ - /// \brief Test whether the architecture is 64-bit + /// Test whether the architecture is 64-bit /// /// Note that this tests for 64-bit pointer width, and nothing else. Note /// that we intentionally expose only three predicates, 64-bit, 32-bit, and @@ -340,12 +354,12 @@ public: /// system is provided. bool isArch64Bit() const; - /// \brief Test whether the architecture is 32-bit + /// Test whether the architecture is 32-bit /// /// Note that this tests for 32-bit pointer width, and nothing else. bool isArch32Bit() const; - /// \brief Test whether the architecture is 16-bit + /// Test whether the architecture is 16-bit /// /// Note that this tests for 16-bit pointer width, and nothing else. bool isArch16Bit() const; @@ -396,13 +410,27 @@ public: } /// Is this an iOS triple. + /// Note: This identifies tvOS as a variant of iOS. If that ever + /// changes, i.e., if the two operating systems diverge or their version + /// numbers get out of sync, that will need to be changed. + /// watchOS has completely different version numbers so it is not included. bool isiOS() const { - return getOS() == Triple::IOS; + return getOS() == Triple::IOS || isTvOS(); + } + + /// Is this an Apple tvOS triple. + bool isTvOS() const { + return getOS() == Triple::TvOS; + } + + /// Is this an Apple watchOS triple. + bool isWatchOS() const { + return getOS() == Triple::WatchOS; } - /// isOSDarwin - Is this a "Darwin" OS (OS X or iOS). + /// isOSDarwin - Is this a "Darwin" OS (OS X, iOS, or watchOS). bool isOSDarwin() const { - return isMacOSX() || isiOS(); + return isMacOSX() || isiOS() || isWatchOS(); } bool isOSNetBSD() const { @@ -427,16 +455,26 @@ public: return getOS() == Triple::Bitrig; } + bool isOSIAMCU() const { + return getOS() == Triple::ELFIAMCU; + } + + /// Checks if the environment could be MSVC. bool isWindowsMSVCEnvironment() const { return getOS() == Triple::Win32 && (getEnvironment() == Triple::UnknownEnvironment || getEnvironment() == Triple::MSVC); } + /// Checks if the environment is MSVC. bool isKnownWindowsMSVCEnvironment() const { return getOS() == Triple::Win32 && getEnvironment() == Triple::MSVC; } + bool isWindowsCoreCLREnvironment() const { + return getOS() == Triple::Win32 && getEnvironment() == Triple::CoreCLR; + } + bool isWindowsItaniumEnvironment() const { return getOS() == Triple::Win32 && getEnvironment() == Triple::Itanium; } @@ -449,60 +487,63 @@ public: return getOS() == Triple::Win32 && getEnvironment() == Triple::GNU; } - /// \brief Tests for either Cygwin or MinGW OS + /// Tests for either Cygwin or MinGW OS bool isOSCygMing() const { return isWindowsCygwinEnvironment() || isWindowsGNUEnvironment(); } - /// \brief Is this a "Windows" OS targeting a "MSVCRT.dll" environment. + /// Is this a "Windows" OS targeting a "MSVCRT.dll" environment. bool isOSMSVCRT() const { return isWindowsMSVCEnvironment() || isWindowsGNUEnvironment() || isWindowsItaniumEnvironment(); } - /// \brief Tests whether the OS is Windows. + /// Tests whether the OS is Windows. bool isOSWindows() const { return getOS() == Triple::Win32; } - /// \brief Tests whether the OS is NaCl (Native Client) + /// Tests whether the OS is NaCl (Native Client) bool isOSNaCl() const { return getOS() == Triple::NaCl; } - /// \brief Tests whether the OS is Linux. + /// Tests whether the OS is Linux. bool isOSLinux() const { return getOS() == Triple::Linux; } - /// \brief Tests whether the OS uses the ELF binary format. + /// Tests whether the OS uses the ELF binary format. bool isOSBinFormatELF() const { return getObjectFormat() == Triple::ELF; } - /// \brief Tests whether the OS uses the COFF binary format. + /// Tests whether the OS uses the COFF binary format. bool isOSBinFormatCOFF() const { return getObjectFormat() == Triple::COFF; } - /// \brief Tests whether the environment is MachO. + /// Tests whether the environment is MachO. bool isOSBinFormatMachO() const { return getObjectFormat() == Triple::MachO; } - /// \brief Tests whether the target is the PS4 CPU + /// Tests whether the target is the PS4 CPU bool isPS4CPU() const { return getArch() == Triple::x86_64 && getVendor() == Triple::SCEI && getOS() == Triple::PS4; } - /// \brief Tests whether the target is the PS4 platform + /// Tests whether the target is the PS4 platform bool isPS4() const { return getVendor() == Triple::SCEI && getOS() == Triple::PS4; } + /// Tests whether the target is Android + bool isAndroid() const { return getEnvironment() == Triple::Android; } + /// @} /// @name Mutators /// @{ @@ -553,7 +594,7 @@ public: /// @name Helpers to build variants of a particular triple. /// @{ - /// \brief Form a triple with a 32-bit variant of the current architecture. + /// Form a triple with a 32-bit variant of the current architecture. /// /// This can be used to move across "families" of architectures where useful. /// @@ -561,7 +602,7 @@ public: /// architecture if no such variant can be found. llvm::Triple get32BitArchVariant() const; - /// \brief Form a triple with a 64-bit variant of the current architecture. + /// Form a triple with a 64-bit variant of the current architecture. /// /// This can be used to move across "families" of architectures where useful. /// @@ -589,7 +630,7 @@ public: /// /// \param Arch the architecture name (e.g., "armv7s"). If it is an empty /// string then the triple's arch name is used. - const char* getARMCPUForArch(StringRef Arch = StringRef()) const; + StringRef getARMCPUForArch(StringRef Arch = StringRef()) const; /// @} /// @name Static helpers for IDs. diff --git a/include/llvm/ADT/UniqueVector.h b/include/llvm/ADT/UniqueVector.h index a9cb2f5709eb..e1ab4b56023f 100644 --- a/include/llvm/ADT/UniqueVector.h +++ b/include/llvm/ADT/UniqueVector.h @@ -11,6 +11,7 @@ #define LLVM_ADT_UNIQUEVECTOR_H #include <cassert> +#include <cstddef> #include <map> #include <vector> diff --git a/include/llvm/ADT/ilist.h b/include/llvm/ADT/ilist.h index a7b9306b3a73..3044a6c435f1 100644 --- a/include/llvm/ADT/ilist.h +++ b/include/llvm/ADT/ilist.h @@ -104,6 +104,53 @@ struct ilist_sentinel_traits { } }; +template <typename NodeTy> class ilist_half_node; +template <typename NodeTy> class ilist_node; + +/// Traits with an embedded ilist_node as a sentinel. +/// +/// FIXME: The downcast in createSentinel() is UB. +template <typename NodeTy> struct ilist_embedded_sentinel_traits { + /// Get hold of the node that marks the end of the list. + NodeTy *createSentinel() const { + // Since i(p)lists always publicly derive from their corresponding traits, + // placing a data member in this class will augment the i(p)list. But since + // the NodeTy is expected to be publicly derive from ilist_node<NodeTy>, + // there is a legal viable downcast from it to NodeTy. We use this trick to + // superimpose an i(p)list with a "ghostly" NodeTy, which becomes the + // sentinel. Dereferencing the sentinel is forbidden (save the + // ilist_node<NodeTy>), so no one will ever notice the superposition. + return static_cast<NodeTy *>(&Sentinel); + } + static void destroySentinel(NodeTy *) {} + + NodeTy *provideInitialHead() const { return createSentinel(); } + NodeTy *ensureHead(NodeTy *) const { return createSentinel(); } + static void noteHead(NodeTy *, NodeTy *) {} + +private: + mutable ilist_node<NodeTy> Sentinel; +}; + +/// Trait with an embedded ilist_half_node as a sentinel. +/// +/// FIXME: The downcast in createSentinel() is UB. +template <typename NodeTy> struct ilist_half_embedded_sentinel_traits { + /// Get hold of the node that marks the end of the list. + NodeTy *createSentinel() const { + // See comment in ilist_embedded_sentinel_traits::createSentinel(). + return static_cast<NodeTy *>(&Sentinel); + } + static void destroySentinel(NodeTy *) {} + + NodeTy *provideInitialHead() const { return createSentinel(); } + NodeTy *ensureHead(NodeTy *) const { return createSentinel(); } + static void noteHead(NodeTy *, NodeTy *) {} + +private: + mutable ilist_half_node<NodeTy> Sentinel; +}; + /// ilist_node_traits - A fragment for template traits for intrusive list /// that provides default node related operations. /// @@ -173,8 +220,8 @@ private: template<class T> void operator-(T) const; public: - ilist_iterator(pointer NP) : NodePtr(NP) {} - ilist_iterator(reference NR) : NodePtr(&NR) {} + explicit ilist_iterator(pointer NP) : NodePtr(NP) {} + explicit ilist_iterator(reference NR) : NodePtr(&NR) {} ilist_iterator() : NodePtr(nullptr) {} // This is templated so that we can allow constructing a const iterator from @@ -191,8 +238,10 @@ public: return *this; } + void reset(pointer NP) { NodePtr = NP; } + // Accessors... - operator pointer() const { + explicit operator pointer() const { return NodePtr; } @@ -202,11 +251,11 @@ public: pointer operator->() const { return &operator*(); } // Comparison operators - bool operator==(const ilist_iterator &RHS) const { - return NodePtr == RHS.NodePtr; + template <class Y> bool operator==(const ilist_iterator<Y> &RHS) const { + return NodePtr == RHS.getNodePtrUnchecked(); } - bool operator!=(const ilist_iterator &RHS) const { - return NodePtr != RHS.NodePtr; + template <class Y> bool operator!=(const ilist_iterator<Y> &RHS) const { + return NodePtr != RHS.getNodePtrUnchecked(); } // Increment and decrement operators... @@ -422,7 +471,7 @@ public: this->setPrev(CurNode, New); this->addNodeToList(New); // Notify traits that we added a node... - return New; + return iterator(New); } iterator insertAfter(iterator where, NodeTy *New) { @@ -443,7 +492,7 @@ public: else Head = NextNode; this->setPrev(NextNode, PrevNode); - IT = NextNode; + IT.reset(NextNode); this->removeNodeFromList(Node); // Notify traits that we removed a node... // Set the next/prev pointers of the current node to null. This isn't @@ -461,12 +510,18 @@ public: return remove(MutIt); } + NodeTy *remove(NodeTy *IT) { return remove(iterator(IT)); } + NodeTy *remove(NodeTy &IT) { return remove(iterator(IT)); } + // erase - remove a node from the controlled sequence... and delete it. iterator erase(iterator where) { this->deleteNode(remove(where)); return where; } + iterator erase(NodeTy *IT) { return erase(iterator(IT)); } + iterator erase(NodeTy &IT) { return erase(iterator(IT)); } + /// Remove all nodes from the list like clear(), but do not call /// removeNodeFromList() or deleteNode(). /// @@ -522,7 +577,7 @@ private: this->setNext(Last, PosNext); this->setPrev(PosNext, Last); - this->transferNodesFromList(L2, First, PosNext); + this->transferNodesFromList(L2, iterator(First), iterator(PosNext)); // Now that everything is set, restore the pointers to the list sentinels. L2.setTail(L2Sentinel); @@ -579,6 +634,83 @@ public: void splice(iterator where, iplist &L2, iterator first, iterator last) { if (first != last) transfer(where, L2, first, last); } + void splice(iterator where, iplist &L2, NodeTy &N) { + splice(where, L2, iterator(N)); + } + void splice(iterator where, iplist &L2, NodeTy *N) { + splice(where, L2, iterator(N)); + } + + template <class Compare> + void merge(iplist &Right, Compare comp) { + if (this == &Right) + return; + iterator First1 = begin(), Last1 = end(); + iterator First2 = Right.begin(), Last2 = Right.end(); + while (First1 != Last1 && First2 != Last2) { + if (comp(*First2, *First1)) { + iterator Next = First2; + transfer(First1, Right, First2, ++Next); + First2 = Next; + } else { + ++First1; + } + } + if (First2 != Last2) + transfer(Last1, Right, First2, Last2); + } + void merge(iplist &Right) { return merge(Right, op_less); } + + template <class Compare> + void sort(Compare comp) { + // The list is empty, vacuously sorted. + if (empty()) + return; + // The list has a single element, vacuously sorted. + if (std::next(begin()) == end()) + return; + // Find the split point for the list. + iterator Center = begin(), End = begin(); + while (End != end() && std::next(End) != end()) { + Center = std::next(Center); + End = std::next(std::next(End)); + } + // Split the list into two. + iplist RightHalf; + RightHalf.splice(RightHalf.begin(), *this, Center, end()); + + // Sort the two sublists. + sort(comp); + RightHalf.sort(comp); + + // Merge the two sublists back together. + merge(RightHalf, comp); + } + void sort() { sort(op_less); } + + /// \brief Get the previous node, or \c nullptr for the list head. + NodeTy *getPrevNode(NodeTy &N) const { + auto I = N.getIterator(); + if (I == begin()) + return nullptr; + return &*std::prev(I); + } + /// \brief Get the previous node, or \c nullptr for the list head. + const NodeTy *getPrevNode(const NodeTy &N) const { + return getPrevNode(const_cast<NodeTy &>(N)); + } + + /// \brief Get the next node, or \c nullptr for the list tail. + NodeTy *getNextNode(NodeTy &N) const { + auto Next = std::next(N.getIterator()); + if (Next == end()) + return nullptr; + return &*Next; + } + /// \brief Get the next node, or \c nullptr for the list tail. + const NodeTy *getNextNode(const NodeTy &N) const { + return getNextNode(const_cast<NodeTy &>(N)); + } }; diff --git a/include/llvm/ADT/ilist_node.h b/include/llvm/ADT/ilist_node.h index 26d0b55e4093..7e5a0e0e5ad8 100644 --- a/include/llvm/ADT/ilist_node.h +++ b/include/llvm/ADT/ilist_node.h @@ -19,12 +19,15 @@ namespace llvm { template<typename NodeTy> struct ilist_traits; +template <typename NodeTy> struct ilist_embedded_sentinel_traits; +template <typename NodeTy> struct ilist_half_embedded_sentinel_traits; /// ilist_half_node - Base class that provides prev services for sentinels. /// template<typename NodeTy> class ilist_half_node { friend struct ilist_traits<NodeTy>; + friend struct ilist_half_embedded_sentinel_traits<NodeTy>; NodeTy *Prev; protected: NodeTy *getPrev() { return Prev; } @@ -36,6 +39,8 @@ protected: template<typename NodeTy> struct ilist_nextprev_traits; +template <typename NodeTy> class ilist_iterator; + /// ilist_node - Base class that provides next/prev services for nodes /// that use ilist_nextprev_traits or ilist_default_traits. /// @@ -43,6 +48,8 @@ template<typename NodeTy> class ilist_node : private ilist_half_node<NodeTy> { friend struct ilist_nextprev_traits<NodeTy>; friend struct ilist_traits<NodeTy>; + friend struct ilist_half_embedded_sentinel_traits<NodeTy>; + friend struct ilist_embedded_sentinel_traits<NodeTy>; NodeTy *Next; NodeTy *getNext() { return Next; } const NodeTy *getNext() const { return Next; } @@ -51,53 +58,63 @@ protected: ilist_node() : Next(nullptr) {} public: + ilist_iterator<NodeTy> getIterator() { + // FIXME: Stop downcasting to create the iterator (potential UB). + return ilist_iterator<NodeTy>(static_cast<NodeTy *>(this)); + } + ilist_iterator<const NodeTy> getIterator() const { + // FIXME: Stop downcasting to create the iterator (potential UB). + return ilist_iterator<const NodeTy>(static_cast<const NodeTy *>(this)); + } +}; + +/// An ilist node that can access its parent list. +/// +/// Requires \c NodeTy to have \a getParent() to find the parent node, and the +/// \c ParentTy to have \a getSublistAccess() to get a reference to the list. +template <typename NodeTy, typename ParentTy> +class ilist_node_with_parent : public ilist_node<NodeTy> { +protected: + ilist_node_with_parent() = default; + +private: + /// Forward to NodeTy::getParent(). + /// + /// Note: do not use the name "getParent()". We want a compile error + /// (instead of recursion) when the subclass fails to implement \a + /// getParent(). + const ParentTy *getNodeParent() const { + return static_cast<const NodeTy *>(this)->getParent(); + } + +public: /// @name Adjacent Node Accessors /// @{ - - /// \brief Get the previous node, or 0 for the list head. + /// \brief Get the previous node, or \c nullptr for the list head. NodeTy *getPrevNode() { - NodeTy *Prev = this->getPrev(); - - // Check for sentinel. - if (!Prev->getNext()) - return nullptr; - - return Prev; + // Should be separated to a reused function, but then we couldn't use auto + // (and would need the type of the list). + const auto &List = + getNodeParent()->*(ParentTy::getSublistAccess((NodeTy *)nullptr)); + return List.getPrevNode(*static_cast<NodeTy *>(this)); } - - /// \brief Get the previous node, or 0 for the list head. + /// \brief Get the previous node, or \c nullptr for the list head. const NodeTy *getPrevNode() const { - const NodeTy *Prev = this->getPrev(); - - // Check for sentinel. - if (!Prev->getNext()) - return nullptr; - - return Prev; + return const_cast<ilist_node_with_parent *>(this)->getPrevNode(); } - /// \brief Get the next node, or 0 for the list tail. + /// \brief Get the next node, or \c nullptr for the list tail. NodeTy *getNextNode() { - NodeTy *Next = getNext(); - - // Check for sentinel. - if (!Next->getNext()) - return nullptr; - - return Next; + // Should be separated to a reused function, but then we couldn't use auto + // (and would need the type of the list). + const auto &List = + getNodeParent()->*(ParentTy::getSublistAccess((NodeTy *)nullptr)); + return List.getNextNode(*static_cast<NodeTy *>(this)); } - - /// \brief Get the next node, or 0 for the list tail. + /// \brief Get the next node, or \c nullptr for the list tail. const NodeTy *getNextNode() const { - const NodeTy *Next = getNext(); - - // Check for sentinel. - if (!Next->getNext()) - return nullptr; - - return Next; + return const_cast<ilist_node_with_parent *>(this)->getNextNode(); } - /// @} }; diff --git a/include/llvm/ADT/iterator_range.h b/include/llvm/ADT/iterator_range.h index 523a86f02e08..3dd679bd9b79 100644 --- a/include/llvm/ADT/iterator_range.h +++ b/include/llvm/ADT/iterator_range.h @@ -20,6 +20,7 @@ #define LLVM_ADT_ITERATOR_RANGE_H #include <utility> +#include <iterator> namespace llvm { @@ -32,6 +33,12 @@ class iterator_range { IteratorT begin_iterator, end_iterator; public: + //TODO: Add SFINAE to test that the Container's iterators match the range's + // iterators. + template <typename Container> + iterator_range(Container &&c) + //TODO: Consider ADL/non-member begin/end calls. + : begin_iterator(c.begin()), end_iterator(c.end()) {} iterator_range(IteratorT begin_iterator, IteratorT end_iterator) : begin_iterator(std::move(begin_iterator)), end_iterator(std::move(end_iterator)) {} @@ -51,6 +58,11 @@ template <class T> iterator_range<T> make_range(T x, T y) { template <typename T> iterator_range<T> make_range(std::pair<T, T> p) { return iterator_range<T>(std::move(p.first), std::move(p.second)); } + +template<typename T> +iterator_range<decltype(begin(std::declval<T>()))> drop_begin(T &&t, int n) { + return make_range(std::next(begin(t), n), end(t)); +} } #endif diff --git a/include/llvm/Analysis/AliasAnalysis.h b/include/llvm/Analysis/AliasAnalysis.h index 36f8199a0322..5cc840a64a62 100644 --- a/include/llvm/Analysis/AliasAnalysis.h +++ b/include/llvm/Analysis/AliasAnalysis.h @@ -41,10 +41,11 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Metadata.h" +#include "llvm/IR/PassManager.h" #include "llvm/Analysis/MemoryLocation.h" namespace llvm { - +class BasicAAResult; class LoadInst; class StoreInst; class VAArgInst; @@ -55,6 +56,7 @@ class AnalysisUsage; class MemTransferInst; class MemIntrinsic; class DominatorTree; +class OrderedBasicBlock; /// The possible results of an alias query. /// @@ -84,462 +86,871 @@ enum AliasResult { MustAlias, }; -class AliasAnalysis { -protected: - const DataLayout *DL; - const TargetLibraryInfo *TLI; +/// Flags indicating whether a memory access modifies or references memory. +/// +/// This is no access at all, a modification, a reference, or both +/// a modification and a reference. These are specifically structured such that +/// they form a two bit matrix and bit-tests for 'mod' or 'ref' work with any +/// of the possible values. +enum ModRefInfo { + /// The access neither references nor modifies the value stored in memory. + MRI_NoModRef = 0, + /// The access references the value stored in memory. + MRI_Ref = 1, + /// The access modifies the value stored in memory. + MRI_Mod = 2, + /// The access both references and modifies the value stored in memory. + MRI_ModRef = MRI_Ref | MRI_Mod +}; -private: - AliasAnalysis *AA; // Previous Alias Analysis to chain to. +/// The locations at which a function might access memory. +/// +/// These are primarily used in conjunction with the \c AccessKind bits to +/// describe both the nature of access and the locations of access for a +/// function call. +enum FunctionModRefLocation { + /// Base case is no access to memory. + FMRL_Nowhere = 0, + /// Access to memory via argument pointers. + FMRL_ArgumentPointees = 4, + /// Access to any memory. + FMRL_Anywhere = 8 | FMRL_ArgumentPointees +}; -protected: - /// InitializeAliasAnalysis - Subclasses must call this method to initialize - /// the AliasAnalysis interface before any other methods are called. This is - /// typically called by the run* methods of these subclasses. This may be - /// called multiple times. +/// Summary of how a function affects memory in the program. +/// +/// Loads from constant globals are not considered memory accesses for this +/// interface. Also, functions may freely modify stack space local to their +/// invocation without having to report it through these interfaces. +enum FunctionModRefBehavior { + /// This function does not perform any non-local loads or stores to memory. /// - void InitializeAliasAnalysis(Pass *P, const DataLayout *DL); - - /// getAnalysisUsage - All alias analysis implementations should invoke this - /// directly (using AliasAnalysis::getAnalysisUsage(AU)). - virtual void getAnalysisUsage(AnalysisUsage &AU) const; + /// This property corresponds to the GCC 'const' attribute. + /// This property corresponds to the LLVM IR 'readnone' attribute. + /// This property corresponds to the IntrNoMem LLVM intrinsic flag. + FMRB_DoesNotAccessMemory = FMRL_Nowhere | MRI_NoModRef, -public: - static char ID; // Class identification, replacement for typeinfo - AliasAnalysis() : DL(nullptr), TLI(nullptr), AA(nullptr) {} - virtual ~AliasAnalysis(); // We want to be subclassed + /// The only memory references in this function (if it has any) are + /// non-volatile loads from objects pointed to by its pointer-typed + /// arguments, with arbitrary offsets. + /// + /// This property corresponds to the IntrReadArgMem LLVM intrinsic flag. + FMRB_OnlyReadsArgumentPointees = FMRL_ArgumentPointees | MRI_Ref, - /// getTargetLibraryInfo - Return a pointer to the current TargetLibraryInfo - /// object, or null if no TargetLibraryInfo object is available. + /// The only memory references in this function (if it has any) are + /// non-volatile loads and stores from objects pointed to by its + /// pointer-typed arguments, with arbitrary offsets. /// - const TargetLibraryInfo *getTargetLibraryInfo() const { return TLI; } + /// This property corresponds to the IntrReadWriteArgMem LLVM intrinsic flag. + FMRB_OnlyAccessesArgumentPointees = FMRL_ArgumentPointees | MRI_ModRef, - /// getTypeStoreSize - Return the DataLayout store size for the given type, - /// if known, or a conservative value otherwise. + /// This function does not perform any non-local stores or volatile loads, + /// but may read from any memory location. /// - uint64_t getTypeStoreSize(Type *Ty); + /// This property corresponds to the GCC 'pure' attribute. + /// This property corresponds to the LLVM IR 'readonly' attribute. + /// This property corresponds to the IntrReadMem LLVM intrinsic flag. + FMRB_OnlyReadsMemory = FMRL_Anywhere | MRI_Ref, + + /// This indicates that the function could not be classified into one of the + /// behaviors above. + FMRB_UnknownModRefBehavior = FMRL_Anywhere | MRI_ModRef +}; + +class AAResults { +public: + // Make these results default constructable and movable. We have to spell + // these out because MSVC won't synthesize them. + AAResults() {} + AAResults(AAResults &&Arg); + AAResults &operator=(AAResults &&Arg); + ~AAResults(); + + /// Register a specific AA result. + template <typename AAResultT> void addAAResult(AAResultT &AAResult) { + // FIXME: We should use a much lighter weight system than the usual + // polymorphic pattern because we don't own AAResult. It should + // ideally involve two pointers and no separate allocation. + AAs.emplace_back(new Model<AAResultT>(AAResult, *this)); + } //===--------------------------------------------------------------------===// - /// Alias Queries... - /// + /// \name Alias Queries + /// @{ - /// alias - The main low level interface to the alias analysis implementation. + /// The main low level interface to the alias analysis implementation. /// Returns an AliasResult indicating whether the two pointers are aliased to - /// each other. This is the interface that must be implemented by specific + /// each other. This is the interface that must be implemented by specific /// alias analysis implementations. - virtual AliasResult alias(const MemoryLocation &LocA, - const MemoryLocation &LocB); + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); - /// alias - A convenience wrapper. - AliasResult alias(const Value *V1, uint64_t V1Size, - const Value *V2, uint64_t V2Size) { + /// A convenience wrapper around the primary \c alias interface. + AliasResult alias(const Value *V1, uint64_t V1Size, const Value *V2, + uint64_t V2Size) { return alias(MemoryLocation(V1, V1Size), MemoryLocation(V2, V2Size)); } - /// alias - A convenience wrapper. + /// A convenience wrapper around the primary \c alias interface. AliasResult alias(const Value *V1, const Value *V2) { return alias(V1, MemoryLocation::UnknownSize, V2, MemoryLocation::UnknownSize); } - /// isNoAlias - A trivial helper function to check to see if the specified - /// pointers are no-alias. + /// A trivial helper function to check to see if the specified pointers are + /// no-alias. bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) { return alias(LocA, LocB) == NoAlias; } - /// isNoAlias - A convenience wrapper. - bool isNoAlias(const Value *V1, uint64_t V1Size, - const Value *V2, uint64_t V2Size) { + /// A convenience wrapper around the \c isNoAlias helper interface. + bool isNoAlias(const Value *V1, uint64_t V1Size, const Value *V2, + uint64_t V2Size) { return isNoAlias(MemoryLocation(V1, V1Size), MemoryLocation(V2, V2Size)); } - - /// isNoAlias - A convenience wrapper. + + /// A convenience wrapper around the \c isNoAlias helper interface. bool isNoAlias(const Value *V1, const Value *V2) { return isNoAlias(MemoryLocation(V1), MemoryLocation(V2)); } - - /// isMustAlias - A convenience wrapper. + + /// A trivial helper function to check to see if the specified pointers are + /// must-alias. bool isMustAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) { return alias(LocA, LocB) == MustAlias; } - /// isMustAlias - A convenience wrapper. + /// A convenience wrapper around the \c isMustAlias helper interface. bool isMustAlias(const Value *V1, const Value *V2) { return alias(V1, 1, V2, 1) == MustAlias; } - - /// pointsToConstantMemory - If the specified memory location is - /// known to be constant, return true. If OrLocal is true and the - /// specified memory location is known to be "local" (derived from - /// an alloca), return true. Otherwise return false. - virtual bool pointsToConstantMemory(const MemoryLocation &Loc, - bool OrLocal = false); - /// pointsToConstantMemory - A convenient wrapper. + /// Checks whether the given location points to constant memory, or if + /// \p OrLocal is true whether it points to a local alloca. + bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false); + + /// A convenience wrapper around the primary \c pointsToConstantMemory + /// interface. bool pointsToConstantMemory(const Value *P, bool OrLocal = false) { return pointsToConstantMemory(MemoryLocation(P), OrLocal); } + /// @} //===--------------------------------------------------------------------===// - /// Simple mod/ref information... - /// - - /// ModRefResult - Represent the result of a mod/ref query. Mod and Ref are - /// bits which may be or'd together. - /// - enum ModRefResult { NoModRef = 0, Ref = 1, Mod = 2, ModRef = 3 }; - - /// These values define additional bits used to define the - /// ModRefBehavior values. - enum { Nowhere = 0, ArgumentPointees = 4, Anywhere = 8 | ArgumentPointees }; - - /// ModRefBehavior - Summary of how a function affects memory in the program. - /// Loads from constant globals are not considered memory accesses for this - /// interface. Also, functions may freely modify stack space local to their - /// invocation without having to report it through these interfaces. - enum ModRefBehavior { - /// DoesNotAccessMemory - This function does not perform any non-local loads - /// or stores to memory. - /// - /// This property corresponds to the GCC 'const' attribute. - /// This property corresponds to the LLVM IR 'readnone' attribute. - /// This property corresponds to the IntrNoMem LLVM intrinsic flag. - DoesNotAccessMemory = Nowhere | NoModRef, - - /// OnlyReadsArgumentPointees - The only memory references in this function - /// (if it has any) are non-volatile loads from objects pointed to by its - /// pointer-typed arguments, with arbitrary offsets. - /// - /// This property corresponds to the LLVM IR 'argmemonly' attribute combined - /// with 'readonly' attribute. - /// This property corresponds to the IntrReadArgMem LLVM intrinsic flag. - OnlyReadsArgumentPointees = ArgumentPointees | Ref, - - /// OnlyAccessesArgumentPointees - The only memory references in this - /// function (if it has any) are non-volatile loads and stores from objects - /// pointed to by its pointer-typed arguments, with arbitrary offsets. - /// - /// This property corresponds to the LLVM IR 'argmemonly' attribute. - /// This property corresponds to the IntrReadWriteArgMem LLVM intrinsic flag. - OnlyAccessesArgumentPointees = ArgumentPointees | ModRef, - - /// OnlyReadsMemory - This function does not perform any non-local stores or - /// volatile loads, but may read from any memory location. - /// - /// This property corresponds to the GCC 'pure' attribute. - /// This property corresponds to the LLVM IR 'readonly' attribute. - /// This property corresponds to the IntrReadMem LLVM intrinsic flag. - OnlyReadsMemory = Anywhere | Ref, - - /// UnknownModRefBehavior - This indicates that the function could not be - /// classified into one of the behaviors above. - UnknownModRefBehavior = Anywhere | ModRef - }; + /// \name Simple mod/ref information + /// @{ /// Get the ModRef info associated with a pointer argument of a callsite. The /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note /// that these bits do not necessarily account for the overall behavior of /// the function, but rather only provide additional per-argument /// information. - virtual ModRefResult getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx); + ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx); - /// getModRefBehavior - Return the behavior when calling the given call site. - virtual ModRefBehavior getModRefBehavior(ImmutableCallSite CS); + /// Return the behavior of the given call site. + FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS); - /// getModRefBehavior - Return the behavior when calling the given function. - /// For use when the call site is not known. - virtual ModRefBehavior getModRefBehavior(const Function *F); + /// Return the behavior when calling the given function. + FunctionModRefBehavior getModRefBehavior(const Function *F); - /// doesNotAccessMemory - If the specified call is known to never read or - /// write memory, return true. If the call only reads from known-constant - /// memory, it is also legal to return true. Calls that unwind the stack - /// are legal for this predicate. + /// Checks if the specified call is known to never read or write memory. + /// + /// Note that if the call only reads from known-constant memory, it is also + /// legal to return true. Also, calls that unwind the stack are legal for + /// this predicate. /// /// Many optimizations (such as CSE and LICM) can be performed on such calls /// without worrying about aliasing properties, and many calls have this /// property (e.g. calls to 'sin' and 'cos'). /// /// This property corresponds to the GCC 'const' attribute. - /// bool doesNotAccessMemory(ImmutableCallSite CS) { - return getModRefBehavior(CS) == DoesNotAccessMemory; + return getModRefBehavior(CS) == FMRB_DoesNotAccessMemory; } - /// doesNotAccessMemory - If the specified function is known to never read or - /// write memory, return true. For use when the call site is not known. + /// Checks if the specified function is known to never read or write memory. + /// + /// Note that if the function only reads from known-constant memory, it is + /// also legal to return true. Also, function that unwind the stack are legal + /// for this predicate. + /// + /// Many optimizations (such as CSE and LICM) can be performed on such calls + /// to such functions without worrying about aliasing properties, and many + /// functions have this property (e.g. 'sin' and 'cos'). /// + /// This property corresponds to the GCC 'const' attribute. bool doesNotAccessMemory(const Function *F) { - return getModRefBehavior(F) == DoesNotAccessMemory; + return getModRefBehavior(F) == FMRB_DoesNotAccessMemory; } - /// onlyReadsMemory - If the specified call is known to only read from - /// non-volatile memory (or not access memory at all), return true. Calls - /// that unwind the stack are legal for this predicate. + /// Checks if the specified call is known to only read from non-volatile + /// memory (or not access memory at all). + /// + /// Calls that unwind the stack are legal for this predicate. /// /// This property allows many common optimizations to be performed in the /// absence of interfering store instructions, such as CSE of strlen calls. /// /// This property corresponds to the GCC 'pure' attribute. - /// bool onlyReadsMemory(ImmutableCallSite CS) { return onlyReadsMemory(getModRefBehavior(CS)); } - /// onlyReadsMemory - If the specified function is known to only read from - /// non-volatile memory (or not access memory at all), return true. For use - /// when the call site is not known. + /// Checks if the specified function is known to only read from non-volatile + /// memory (or not access memory at all). /// - bool onlyReadsMemory(const Function *F) { - return onlyReadsMemory(getModRefBehavior(F)); - } - - /// onlyReadsMemory - Return true if functions with the specified behavior are - /// known to only read from non-volatile memory (or not access memory at all). + /// Functions that unwind the stack are legal for this predicate. /// - static bool onlyReadsMemory(ModRefBehavior MRB) { - return !(MRB & Mod); - } - - /// onlyAccessesArgPointees - Return true if functions with the specified - /// behavior are known to read and write at most from objects pointed to by - /// their pointer-typed arguments (with arbitrary offsets). - /// - static bool onlyAccessesArgPointees(ModRefBehavior MRB) { - return !(MRB & Anywhere & ~ArgumentPointees); - } - - /// doesAccessArgPointees - Return true if functions with the specified - /// behavior are known to potentially read or write from objects pointed - /// to be their pointer-typed arguments (with arbitrary offsets). + /// This property allows many common optimizations to be performed in the + /// absence of interfering store instructions, such as CSE of strlen calls. /// - static bool doesAccessArgPointees(ModRefBehavior MRB) { - return (MRB & ModRef) && (MRB & ArgumentPointees); + /// This property corresponds to the GCC 'pure' attribute. + bool onlyReadsMemory(const Function *F) { + return onlyReadsMemory(getModRefBehavior(F)); } - /// getModRefInfo - Return information about whether or not an - /// instruction may read or write memory (without regard to a - /// specific location) - ModRefResult getModRefInfo(const Instruction *I) { - if (auto CS = ImmutableCallSite(I)) { - auto MRB = getModRefBehavior(CS); - if (MRB & ModRef) - return ModRef; - else if (MRB & Ref) - return Ref; - else if (MRB & Mod) - return Mod; - return NoModRef; - } - - return getModRefInfo(I, MemoryLocation()); + /// Checks if functions with the specified behavior are known to only read + /// from non-volatile memory (or not access memory at all). + static bool onlyReadsMemory(FunctionModRefBehavior MRB) { + return !(MRB & MRI_Mod); } - /// getModRefInfo - Return information about whether or not an instruction may - /// read or write the specified memory location. An instruction - /// that doesn't read or write memory may be trivially LICM'd for example. - ModRefResult getModRefInfo(const Instruction *I, const MemoryLocation &Loc) { - switch (I->getOpcode()) { - case Instruction::VAArg: return getModRefInfo((const VAArgInst*)I, Loc); - case Instruction::Load: return getModRefInfo((const LoadInst*)I, Loc); - case Instruction::Store: return getModRefInfo((const StoreInst*)I, Loc); - case Instruction::Fence: return getModRefInfo((const FenceInst*)I, Loc); - case Instruction::AtomicCmpXchg: - return getModRefInfo((const AtomicCmpXchgInst*)I, Loc); - case Instruction::AtomicRMW: - return getModRefInfo((const AtomicRMWInst*)I, Loc); - case Instruction::Call: return getModRefInfo((const CallInst*)I, Loc); - case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc); - default: return NoModRef; - } + /// Checks if functions with the specified behavior are known to read and + /// write at most from objects pointed to by their pointer-typed arguments + /// (with arbitrary offsets). + static bool onlyAccessesArgPointees(FunctionModRefBehavior MRB) { + return !(MRB & FMRL_Anywhere & ~FMRL_ArgumentPointees); } - /// getModRefInfo - A convenience wrapper. - ModRefResult getModRefInfo(const Instruction *I, - const Value *P, uint64_t Size) { - return getModRefInfo(I, MemoryLocation(P, Size)); + /// Checks if functions with the specified behavior are known to potentially + /// read or write from objects pointed to be their pointer-typed arguments + /// (with arbitrary offsets). + static bool doesAccessArgPointees(FunctionModRefBehavior MRB) { + return (MRB & MRI_ModRef) && (MRB & FMRL_ArgumentPointees); } /// getModRefInfo (for call sites) - Return information about whether /// a particular call site modifies or reads the specified memory location. - virtual ModRefResult getModRefInfo(ImmutableCallSite CS, - const MemoryLocation &Loc); + ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); /// getModRefInfo (for call sites) - A convenience wrapper. - ModRefResult getModRefInfo(ImmutableCallSite CS, - const Value *P, uint64_t Size) { + ModRefInfo getModRefInfo(ImmutableCallSite CS, const Value *P, + uint64_t Size) { return getModRefInfo(CS, MemoryLocation(P, Size)); } /// getModRefInfo (for calls) - Return information about whether /// a particular call modifies or reads the specified memory location. - ModRefResult getModRefInfo(const CallInst *C, const MemoryLocation &Loc) { + ModRefInfo getModRefInfo(const CallInst *C, const MemoryLocation &Loc) { return getModRefInfo(ImmutableCallSite(C), Loc); } /// getModRefInfo (for calls) - A convenience wrapper. - ModRefResult getModRefInfo(const CallInst *C, const Value *P, uint64_t Size) { + ModRefInfo getModRefInfo(const CallInst *C, const Value *P, uint64_t Size) { return getModRefInfo(C, MemoryLocation(P, Size)); } /// getModRefInfo (for invokes) - Return information about whether /// a particular invoke modifies or reads the specified memory location. - ModRefResult getModRefInfo(const InvokeInst *I, const MemoryLocation &Loc) { + ModRefInfo getModRefInfo(const InvokeInst *I, const MemoryLocation &Loc) { return getModRefInfo(ImmutableCallSite(I), Loc); } /// getModRefInfo (for invokes) - A convenience wrapper. - ModRefResult getModRefInfo(const InvokeInst *I, - const Value *P, uint64_t Size) { + ModRefInfo getModRefInfo(const InvokeInst *I, const Value *P, uint64_t Size) { return getModRefInfo(I, MemoryLocation(P, Size)); } /// getModRefInfo (for loads) - Return information about whether /// a particular load modifies or reads the specified memory location. - ModRefResult getModRefInfo(const LoadInst *L, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc); /// getModRefInfo (for loads) - A convenience wrapper. - ModRefResult getModRefInfo(const LoadInst *L, const Value *P, uint64_t Size) { + ModRefInfo getModRefInfo(const LoadInst *L, const Value *P, uint64_t Size) { return getModRefInfo(L, MemoryLocation(P, Size)); } /// getModRefInfo (for stores) - Return information about whether /// a particular store modifies or reads the specified memory location. - ModRefResult getModRefInfo(const StoreInst *S, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc); /// getModRefInfo (for stores) - A convenience wrapper. - ModRefResult getModRefInfo(const StoreInst *S, const Value *P, uint64_t Size){ + ModRefInfo getModRefInfo(const StoreInst *S, const Value *P, uint64_t Size) { return getModRefInfo(S, MemoryLocation(P, Size)); } /// getModRefInfo (for fences) - Return information about whether /// a particular store modifies or reads the specified memory location. - ModRefResult getModRefInfo(const FenceInst *S, const MemoryLocation &Loc) { + ModRefInfo getModRefInfo(const FenceInst *S, const MemoryLocation &Loc) { // Conservatively correct. (We could possibly be a bit smarter if // Loc is a alloca that doesn't escape.) - return ModRef; + return MRI_ModRef; } /// getModRefInfo (for fences) - A convenience wrapper. - ModRefResult getModRefInfo(const FenceInst *S, const Value *P, uint64_t Size){ + ModRefInfo getModRefInfo(const FenceInst *S, const Value *P, uint64_t Size) { return getModRefInfo(S, MemoryLocation(P, Size)); } /// getModRefInfo (for cmpxchges) - Return information about whether /// a particular cmpxchg modifies or reads the specified memory location. - ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX, - const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX, + const MemoryLocation &Loc); /// getModRefInfo (for cmpxchges) - A convenience wrapper. - ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX, - const Value *P, unsigned Size) { + ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX, const Value *P, + unsigned Size) { return getModRefInfo(CX, MemoryLocation(P, Size)); } /// getModRefInfo (for atomicrmws) - Return information about whether /// a particular atomicrmw modifies or reads the specified memory location. - ModRefResult getModRefInfo(const AtomicRMWInst *RMW, - const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc); /// getModRefInfo (for atomicrmws) - A convenience wrapper. - ModRefResult getModRefInfo(const AtomicRMWInst *RMW, - const Value *P, unsigned Size) { + ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const Value *P, + unsigned Size) { return getModRefInfo(RMW, MemoryLocation(P, Size)); } /// getModRefInfo (for va_args) - Return information about whether /// a particular va_arg modifies or reads the specified memory location. - ModRefResult getModRefInfo(const VAArgInst *I, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const VAArgInst *I, const MemoryLocation &Loc); /// getModRefInfo (for va_args) - A convenience wrapper. - ModRefResult getModRefInfo(const VAArgInst* I, const Value* P, uint64_t Size){ + ModRefInfo getModRefInfo(const VAArgInst *I, const Value *P, uint64_t Size) { return getModRefInfo(I, MemoryLocation(P, Size)); } - /// getModRefInfo - Return information about whether a call and an instruction - /// may refer to the same memory locations. - ModRefResult getModRefInfo(Instruction *I, - ImmutableCallSite Call); - /// getModRefInfo - Return information about whether two call sites may refer - /// to the same set of memory locations. See - /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo - /// for details. - virtual ModRefResult getModRefInfo(ImmutableCallSite CS1, - ImmutableCallSite CS2); + /// getModRefInfo (for catchpads) - Return information about whether + /// a particular catchpad modifies or reads the specified memory location. + ModRefInfo getModRefInfo(const CatchPadInst *I, const MemoryLocation &Loc); - /// callCapturesBefore - Return information about whether a particular call - /// site modifies or reads the specified memory location. - ModRefResult callCapturesBefore(const Instruction *I, - const MemoryLocation &MemLoc, - DominatorTree *DT); + /// getModRefInfo (for catchpads) - A convenience wrapper. + ModRefInfo getModRefInfo(const CatchPadInst *I, const Value *P, + uint64_t Size) { + return getModRefInfo(I, MemoryLocation(P, Size)); + } - /// callCapturesBefore - A convenience wrapper. - ModRefResult callCapturesBefore(const Instruction *I, const Value *P, - uint64_t Size, DominatorTree *DT) { - return callCapturesBefore(I, MemoryLocation(P, Size), DT); + /// getModRefInfo (for catchrets) - Return information about whether + /// a particular catchret modifies or reads the specified memory location. + ModRefInfo getModRefInfo(const CatchReturnInst *I, const MemoryLocation &Loc); + + /// getModRefInfo (for catchrets) - A convenience wrapper. + ModRefInfo getModRefInfo(const CatchReturnInst *I, const Value *P, + uint64_t Size) { + return getModRefInfo(I, MemoryLocation(P, Size)); } - //===--------------------------------------------------------------------===// - /// Higher level methods for querying mod/ref information. + /// Check whether or not an instruction may read or write memory (without + /// regard to a specific location). /// + /// For function calls, this delegates to the alias-analysis specific + /// call-site mod-ref behavior queries. Otherwise it delegates to the generic + /// mod ref information query without a location. + ModRefInfo getModRefInfo(const Instruction *I) { + if (auto CS = ImmutableCallSite(I)) { + auto MRB = getModRefBehavior(CS); + if (MRB & MRI_ModRef) + return MRI_ModRef; + else if (MRB & MRI_Ref) + return MRI_Ref; + else if (MRB & MRI_Mod) + return MRI_Mod; + return MRI_NoModRef; + } + + return getModRefInfo(I, MemoryLocation()); + } + + /// Check whether or not an instruction may read or write the specified + /// memory location. + /// + /// An instruction that doesn't read or write memory may be trivially LICM'd + /// for example. + /// + /// This primarily delegates to specific helpers above. + ModRefInfo getModRefInfo(const Instruction *I, const MemoryLocation &Loc) { + switch (I->getOpcode()) { + case Instruction::VAArg: return getModRefInfo((const VAArgInst*)I, Loc); + case Instruction::Load: return getModRefInfo((const LoadInst*)I, Loc); + case Instruction::Store: return getModRefInfo((const StoreInst*)I, Loc); + case Instruction::Fence: return getModRefInfo((const FenceInst*)I, Loc); + case Instruction::AtomicCmpXchg: + return getModRefInfo((const AtomicCmpXchgInst*)I, Loc); + case Instruction::AtomicRMW: + return getModRefInfo((const AtomicRMWInst*)I, Loc); + case Instruction::Call: return getModRefInfo((const CallInst*)I, Loc); + case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc); + case Instruction::CatchPad: + return getModRefInfo((const CatchPadInst *)I, Loc); + case Instruction::CatchRet: + return getModRefInfo((const CatchReturnInst *)I, Loc); + default: + return MRI_NoModRef; + } + } + + /// A convenience wrapper for constructing the memory location. + ModRefInfo getModRefInfo(const Instruction *I, const Value *P, + uint64_t Size) { + return getModRefInfo(I, MemoryLocation(P, Size)); + } + + /// Return information about whether a call and an instruction may refer to + /// the same memory locations. + ModRefInfo getModRefInfo(Instruction *I, ImmutableCallSite Call); + + /// Return information about whether two call sites may refer to the same set + /// of memory locations. See the AA documentation for details: + /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo + ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); + + /// \brief Return information about whether a particular call site modifies + /// or reads the specified memory location \p MemLoc before instruction \p I + /// in a BasicBlock. A ordered basic block \p OBB can be used to speed up + /// instruction ordering queries inside the BasicBlock containing \p I. + ModRefInfo callCapturesBefore(const Instruction *I, + const MemoryLocation &MemLoc, DominatorTree *DT, + OrderedBasicBlock *OBB = nullptr); + + /// \brief A convenience wrapper to synthesize a memory location. + ModRefInfo callCapturesBefore(const Instruction *I, const Value *P, + uint64_t Size, DominatorTree *DT, + OrderedBasicBlock *OBB = nullptr) { + return callCapturesBefore(I, MemoryLocation(P, Size), DT, OBB); + } + + /// @} + //===--------------------------------------------------------------------===// + /// \name Higher level methods for querying mod/ref information. + /// @{ - /// canBasicBlockModify - Return true if it is possible for execution of the - /// specified basic block to modify the location Loc. + /// Check if it is possible for execution of the specified basic block to + /// modify the location Loc. bool canBasicBlockModify(const BasicBlock &BB, const MemoryLocation &Loc); - /// canBasicBlockModify - A convenience wrapper. - bool canBasicBlockModify(const BasicBlock &BB, const Value *P, uint64_t Size){ + /// A convenience wrapper synthesizing a memory location. + bool canBasicBlockModify(const BasicBlock &BB, const Value *P, + uint64_t Size) { return canBasicBlockModify(BB, MemoryLocation(P, Size)); } - /// canInstructionRangeModRef - Return true if it is possible for the - /// execution of the specified instructions to mod\ref (according to the - /// mode) the location Loc. The instructions to consider are all - /// of the instructions in the range of [I1,I2] INCLUSIVE. - /// I1 and I2 must be in the same basic block. + /// Check if it is possible for the execution of the specified instructions + /// to mod\ref (according to the mode) the location Loc. + /// + /// The instructions to consider are all of the instructions in the range of + /// [I1,I2] INCLUSIVE. I1 and I2 must be in the same basic block. bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2, const MemoryLocation &Loc, - const ModRefResult Mode); + const ModRefInfo Mode); - /// canInstructionRangeModRef - A convenience wrapper. - bool canInstructionRangeModRef(const Instruction &I1, - const Instruction &I2, const Value *Ptr, - uint64_t Size, const ModRefResult Mode) { + /// A convenience wrapper synthesizing a memory location. + bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2, + const Value *Ptr, uint64_t Size, + const ModRefInfo Mode) { return canInstructionRangeModRef(I1, I2, MemoryLocation(Ptr, Size), Mode); } +private: + class Concept; + template <typename T> class Model; + + template <typename T> friend class AAResultBase; + + std::vector<std::unique_ptr<Concept>> AAs; +}; + +/// Temporary typedef for legacy code that uses a generic \c AliasAnalysis +/// pointer or reference. +typedef AAResults AliasAnalysis; + +/// A private abstract base class describing the concept of an individual alias +/// analysis implementation. +/// +/// This interface is implemented by any \c Model instantiation. It is also the +/// interface which a type used to instantiate the model must provide. +/// +/// All of these methods model methods by the same name in the \c +/// AAResults class. Only differences and specifics to how the +/// implementations are called are documented here. +class AAResults::Concept { +public: + virtual ~Concept() = 0; + + /// An update API used internally by the AAResults to provide + /// a handle back to the top level aggregation. + virtual void setAAResults(AAResults *NewAAR) = 0; + //===--------------------------------------------------------------------===// - /// Methods that clients should call when they transform the program to allow - /// alias analyses to update their internal data structures. Note that these - /// methods may be called on any instruction, regardless of whether or not - /// they have pointer-analysis implications. - /// + /// \name Alias Queries + /// @{ - /// deleteValue - This method should be called whenever an LLVM Value is - /// deleted from the program, for example when an instruction is found to be - /// redundant and is eliminated. - /// - virtual void deleteValue(Value *V); + /// The main low level interface to the alias analysis implementation. + /// Returns an AliasResult indicating whether the two pointers are aliased to + /// each other. This is the interface that must be implemented by specific + /// alias analysis implementations. + virtual AliasResult alias(const MemoryLocation &LocA, + const MemoryLocation &LocB) = 0; - /// addEscapingUse - This method should be used whenever an escaping use is - /// added to a pointer value. Analysis implementations may either return - /// conservative responses for that value in the future, or may recompute - /// some or all internal state to continue providing precise responses. - /// - /// Escaping uses are considered by anything _except_ the following: - /// - GEPs or bitcasts of the pointer - /// - Loads through the pointer - /// - Stores through (but not of) the pointer - virtual void addEscapingUse(Use &U); - - /// replaceWithNewValue - This method is the obvious combination of the two - /// above, and it provided as a helper to simplify client code. + /// Checks whether the given location points to constant memory, or if + /// \p OrLocal is true whether it points to a local alloca. + virtual bool pointsToConstantMemory(const MemoryLocation &Loc, + bool OrLocal) = 0; + + /// @} + //===--------------------------------------------------------------------===// + /// \name Simple mod/ref information + /// @{ + + /// Get the ModRef info associated with a pointer argument of a callsite. The + /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note + /// that these bits do not necessarily account for the overall behavior of + /// the function, but rather only provide additional per-argument + /// information. + virtual ModRefInfo getArgModRefInfo(ImmutableCallSite CS, + unsigned ArgIdx) = 0; + + /// Return the behavior of the given call site. + virtual FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) = 0; + + /// Return the behavior when calling the given function. + virtual FunctionModRefBehavior getModRefBehavior(const Function *F) = 0; + + /// getModRefInfo (for call sites) - Return information about whether + /// a particular call site modifies or reads the specified memory location. + virtual ModRefInfo getModRefInfo(ImmutableCallSite CS, + const MemoryLocation &Loc) = 0; + + /// Return information about whether two call sites may refer to the same set + /// of memory locations. See the AA documentation for details: + /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo + virtual ModRefInfo getModRefInfo(ImmutableCallSite CS1, + ImmutableCallSite CS2) = 0; + + /// @} +}; + +/// A private class template which derives from \c Concept and wraps some other +/// type. +/// +/// This models the concept by directly forwarding each interface point to the +/// wrapped type which must implement a compatible interface. This provides +/// a type erased binding. +template <typename AAResultT> class AAResults::Model final : public Concept { + AAResultT &Result; + +public: + explicit Model(AAResultT &Result, AAResults &AAR) : Result(Result) { + Result.setAAResults(&AAR); + } + ~Model() override {} + + void setAAResults(AAResults *NewAAR) override { Result.setAAResults(NewAAR); } + + AliasResult alias(const MemoryLocation &LocA, + const MemoryLocation &LocB) override { + return Result.alias(LocA, LocB); + } + + bool pointsToConstantMemory(const MemoryLocation &Loc, + bool OrLocal) override { + return Result.pointsToConstantMemory(Loc, OrLocal); + } + + ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) override { + return Result.getArgModRefInfo(CS, ArgIdx); + } + + FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override { + return Result.getModRefBehavior(CS); + } + + FunctionModRefBehavior getModRefBehavior(const Function *F) override { + return Result.getModRefBehavior(F); + } + + ModRefInfo getModRefInfo(ImmutableCallSite CS, + const MemoryLocation &Loc) override { + return Result.getModRefInfo(CS, Loc); + } + + ModRefInfo getModRefInfo(ImmutableCallSite CS1, + ImmutableCallSite CS2) override { + return Result.getModRefInfo(CS1, CS2); + } +}; + +/// A CRTP-driven "mixin" base class to help implement the function alias +/// analysis results concept. +/// +/// Because of the nature of many alias analysis implementations, they often +/// only implement a subset of the interface. This base class will attempt to +/// implement the remaining portions of the interface in terms of simpler forms +/// of the interface where possible, and otherwise provide conservatively +/// correct fallback implementations. +/// +/// Implementors of an alias analysis should derive from this CRTP, and then +/// override specific methods that they wish to customize. There is no need to +/// use virtual anywhere, the CRTP base class does static dispatch to the +/// derived type passed into it. +template <typename DerivedT> class AAResultBase { + // Expose some parts of the interface only to the AAResults::Model + // for wrapping. Specifically, this allows the model to call our + // setAAResults method without exposing it as a fully public API. + friend class AAResults::Model<DerivedT>; + + /// A pointer to the AAResults object that this AAResult is + /// aggregated within. May be null if not aggregated. + AAResults *AAR; + + /// Helper to dispatch calls back through the derived type. + DerivedT &derived() { return static_cast<DerivedT &>(*this); } + + /// A setter for the AAResults pointer, which is used to satisfy the + /// AAResults::Model contract. + void setAAResults(AAResults *NewAAR) { AAR = NewAAR; } + +protected: + /// This proxy class models a common pattern where we delegate to either the + /// top-level \c AAResults aggregation if one is registered, or to the + /// current result if none are registered. + class AAResultsProxy { + AAResults *AAR; + DerivedT &CurrentResult; + + public: + AAResultsProxy(AAResults *AAR, DerivedT &CurrentResult) + : AAR(AAR), CurrentResult(CurrentResult) {} + + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { + return AAR ? AAR->alias(LocA, LocB) : CurrentResult.alias(LocA, LocB); + } + + bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal) { + return AAR ? AAR->pointsToConstantMemory(Loc, OrLocal) + : CurrentResult.pointsToConstantMemory(Loc, OrLocal); + } + + ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) { + return AAR ? AAR->getArgModRefInfo(CS, ArgIdx) : CurrentResult.getArgModRefInfo(CS, ArgIdx); + } + + FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) { + return AAR ? AAR->getModRefBehavior(CS) : CurrentResult.getModRefBehavior(CS); + } + + FunctionModRefBehavior getModRefBehavior(const Function *F) { + return AAR ? AAR->getModRefBehavior(F) : CurrentResult.getModRefBehavior(F); + } + + ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc) { + return AAR ? AAR->getModRefInfo(CS, Loc) + : CurrentResult.getModRefInfo(CS, Loc); + } + + ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) { + return AAR ? AAR->getModRefInfo(CS1, CS2) : CurrentResult.getModRefInfo(CS1, CS2); + } + }; + + const TargetLibraryInfo &TLI; + + explicit AAResultBase(const TargetLibraryInfo &TLI) : TLI(TLI) {} + + // Provide all the copy and move constructors so that derived types aren't + // constrained. + AAResultBase(const AAResultBase &Arg) : TLI(Arg.TLI) {} + AAResultBase(AAResultBase &&Arg) : TLI(Arg.TLI) {} + + /// Get a proxy for the best AA result set to query at this time. /// - void replaceWithNewValue(Value *Old, Value *New) { - deleteValue(Old); + /// When this result is part of a larger aggregation, this will proxy to that + /// aggregation. When this result is used in isolation, it will just delegate + /// back to the derived class's implementation. + AAResultsProxy getBestAAResults() { return AAResultsProxy(AAR, derived()); } + +public: + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { + return MayAlias; + } + + bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal) { + return false; + } + + ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) { + return MRI_ModRef; + } + + FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) { + if (!CS.hasOperandBundles()) + // If CS has operand bundles then aliasing attributes from the function it + // calls do not directly apply to the CallSite. This can be made more + // precise in the future. + if (const Function *F = CS.getCalledFunction()) + return getBestAAResults().getModRefBehavior(F); + + return FMRB_UnknownModRefBehavior; + } + + FunctionModRefBehavior getModRefBehavior(const Function *F) { + return FMRB_UnknownModRefBehavior; } + + ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + + ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); }; +/// Synthesize \c ModRefInfo for a call site and memory location by examining +/// the general behavior of the call site and any specific information for its +/// arguments. +/// +/// This essentially, delegates across the alias analysis interface to collect +/// information which may be enough to (conservatively) fulfill the query. +template <typename DerivedT> +ModRefInfo AAResultBase<DerivedT>::getModRefInfo(ImmutableCallSite CS, + const MemoryLocation &Loc) { + auto MRB = getBestAAResults().getModRefBehavior(CS); + if (MRB == FMRB_DoesNotAccessMemory) + return MRI_NoModRef; + + ModRefInfo Mask = MRI_ModRef; + if (AAResults::onlyReadsMemory(MRB)) + Mask = MRI_Ref; + + if (AAResults::onlyAccessesArgPointees(MRB)) { + bool DoesAlias = false; + ModRefInfo AllArgsMask = MRI_NoModRef; + if (AAResults::doesAccessArgPointees(MRB)) { + for (ImmutableCallSite::arg_iterator AI = CS.arg_begin(), + AE = CS.arg_end(); + AI != AE; ++AI) { + const Value *Arg = *AI; + if (!Arg->getType()->isPointerTy()) + continue; + unsigned ArgIdx = std::distance(CS.arg_begin(), AI); + MemoryLocation ArgLoc = MemoryLocation::getForArgument(CS, ArgIdx, TLI); + AliasResult ArgAlias = getBestAAResults().alias(ArgLoc, Loc); + if (ArgAlias != NoAlias) { + ModRefInfo ArgMask = getBestAAResults().getArgModRefInfo(CS, ArgIdx); + DoesAlias = true; + AllArgsMask = ModRefInfo(AllArgsMask | ArgMask); + } + } + } + if (!DoesAlias) + return MRI_NoModRef; + Mask = ModRefInfo(Mask & AllArgsMask); + } + + // If Loc is a constant memory location, the call definitely could not + // modify the memory location. + if ((Mask & MRI_Mod) && + getBestAAResults().pointsToConstantMemory(Loc, /*OrLocal*/ false)) + Mask = ModRefInfo(Mask & ~MRI_Mod); + + return Mask; +} + +/// Synthesize \c ModRefInfo for two call sites by examining the general +/// behavior of the call site and any specific information for its arguments. +/// +/// This essentially, delegates across the alias analysis interface to collect +/// information which may be enough to (conservatively) fulfill the query. +template <typename DerivedT> +ModRefInfo AAResultBase<DerivedT>::getModRefInfo(ImmutableCallSite CS1, + ImmutableCallSite CS2) { + // If CS1 or CS2 are readnone, they don't interact. + auto CS1B = getBestAAResults().getModRefBehavior(CS1); + if (CS1B == FMRB_DoesNotAccessMemory) + return MRI_NoModRef; + + auto CS2B = getBestAAResults().getModRefBehavior(CS2); + if (CS2B == FMRB_DoesNotAccessMemory) + return MRI_NoModRef; + + // If they both only read from memory, there is no dependence. + if (AAResults::onlyReadsMemory(CS1B) && AAResults::onlyReadsMemory(CS2B)) + return MRI_NoModRef; + + ModRefInfo Mask = MRI_ModRef; + + // If CS1 only reads memory, the only dependence on CS2 can be + // from CS1 reading memory written by CS2. + if (AAResults::onlyReadsMemory(CS1B)) + Mask = ModRefInfo(Mask & MRI_Ref); + + // If CS2 only access memory through arguments, accumulate the mod/ref + // information from CS1's references to the memory referenced by + // CS2's arguments. + if (AAResults::onlyAccessesArgPointees(CS2B)) { + ModRefInfo R = MRI_NoModRef; + if (AAResults::doesAccessArgPointees(CS2B)) { + for (ImmutableCallSite::arg_iterator I = CS2.arg_begin(), + E = CS2.arg_end(); + I != E; ++I) { + const Value *Arg = *I; + if (!Arg->getType()->isPointerTy()) + continue; + unsigned CS2ArgIdx = std::distance(CS2.arg_begin(), I); + auto CS2ArgLoc = MemoryLocation::getForArgument(CS2, CS2ArgIdx, TLI); + + // ArgMask indicates what CS2 might do to CS2ArgLoc, and the dependence + // of CS1 on that location is the inverse. + ModRefInfo ArgMask = + getBestAAResults().getArgModRefInfo(CS2, CS2ArgIdx); + if (ArgMask == MRI_Mod) + ArgMask = MRI_ModRef; + else if (ArgMask == MRI_Ref) + ArgMask = MRI_Mod; + + ArgMask = ModRefInfo(ArgMask & + getBestAAResults().getModRefInfo(CS1, CS2ArgLoc)); + + R = ModRefInfo((R | ArgMask) & Mask); + if (R == Mask) + break; + } + } + return R; + } + + // If CS1 only accesses memory through arguments, check if CS2 references + // any of the memory referenced by CS1's arguments. If not, return NoModRef. + if (AAResults::onlyAccessesArgPointees(CS1B)) { + ModRefInfo R = MRI_NoModRef; + if (AAResults::doesAccessArgPointees(CS1B)) { + for (ImmutableCallSite::arg_iterator I = CS1.arg_begin(), + E = CS1.arg_end(); + I != E; ++I) { + const Value *Arg = *I; + if (!Arg->getType()->isPointerTy()) + continue; + unsigned CS1ArgIdx = std::distance(CS1.arg_begin(), I); + auto CS1ArgLoc = MemoryLocation::getForArgument(CS1, CS1ArgIdx, TLI); + + // ArgMask indicates what CS1 might do to CS1ArgLoc; if CS1 might Mod + // CS1ArgLoc, then we care about either a Mod or a Ref by CS2. If CS1 + // might Ref, then we care only about a Mod by CS2. + ModRefInfo ArgMask = getBestAAResults().getArgModRefInfo(CS1, CS1ArgIdx); + ModRefInfo ArgR = getBestAAResults().getModRefInfo(CS2, CS1ArgLoc); + if (((ArgMask & MRI_Mod) != MRI_NoModRef && + (ArgR & MRI_ModRef) != MRI_NoModRef) || + ((ArgMask & MRI_Ref) != MRI_NoModRef && + (ArgR & MRI_Mod) != MRI_NoModRef)) + R = ModRefInfo((R | ArgMask) & Mask); + + if (R == Mask) + break; + } + } + return R; + } + + return Mask; +} + /// isNoAliasCall - Return true if this pointer is returned by a noalias /// function. bool isNoAliasCall(const Value *V); @@ -564,6 +975,98 @@ bool isIdentifiedObject(const Value *V); /// IdentifiedObjects. bool isIdentifiedFunctionLocal(const Value *V); +/// A manager for alias analyses. +/// +/// This class can have analyses registered with it and when run, it will run +/// all of them and aggregate their results into single AA results interface +/// that dispatches across all of the alias analysis results available. +/// +/// Note that the order in which analyses are registered is very significant. +/// That is the order in which the results will be aggregated and queried. +/// +/// This manager effectively wraps the AnalysisManager for registering alias +/// analyses. When you register your alias analysis with this manager, it will +/// ensure the analysis itself is registered with its AnalysisManager. +class AAManager { +public: + typedef AAResults Result; + + // This type hase value semantics. We have to spell these out because MSVC + // won't synthesize them. + AAManager() {} + AAManager(AAManager &&Arg) + : FunctionResultGetters(std::move(Arg.FunctionResultGetters)) {} + AAManager(const AAManager &Arg) + : FunctionResultGetters(Arg.FunctionResultGetters) {} + AAManager &operator=(AAManager &&RHS) { + FunctionResultGetters = std::move(RHS.FunctionResultGetters); + return *this; + } + AAManager &operator=(const AAManager &RHS) { + FunctionResultGetters = RHS.FunctionResultGetters; + return *this; + } + + /// Register a specific AA result. + template <typename AnalysisT> void registerFunctionAnalysis() { + FunctionResultGetters.push_back(&getFunctionAAResultImpl<AnalysisT>); + } + + Result run(Function &F, AnalysisManager<Function> &AM) { + Result R; + for (auto &Getter : FunctionResultGetters) + (*Getter)(F, AM, R); + return R; + } + +private: + SmallVector<void (*)(Function &F, AnalysisManager<Function> &AM, + AAResults &AAResults), + 4> FunctionResultGetters; + + template <typename AnalysisT> + static void getFunctionAAResultImpl(Function &F, + AnalysisManager<Function> &AM, + AAResults &AAResults) { + AAResults.addAAResult(AM.template getResult<AnalysisT>(F)); + } +}; + +/// A wrapper pass to provide the legacy pass manager access to a suitably +/// prepared AAResults object. +class AAResultsWrapperPass : public FunctionPass { + std::unique_ptr<AAResults> AAR; + +public: + static char ID; + + AAResultsWrapperPass(); + + AAResults &getAAResults() { return *AAR; } + const AAResults &getAAResults() const { return *AAR; } + + bool runOnFunction(Function &F) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +FunctionPass *createAAResultsWrapperPass(); + +/// A wrapper pass around a callback which can be used to populate the +/// AAResults in the AAResultsWrapperPass from an external AA. +/// +/// The callback provided here will be used each time we prepare an AAResults +/// object, and will receive a reference to the function wrapper pass, the +/// function, and the AAResults object to populate. This should be used when +/// setting up a custom pass pipeline to inject a hook into the AA results. +ImmutablePass *createExternalAAWrapperPass( + std::function<void(Pass &, Function &, AAResults &)> Callback); + +/// A helper for the legacy pass manager to create a \c AAResults +/// object populated to the best of our ability for a particular function when +/// inside of a \c ModulePass or a \c CallGraphSCCPass. +AAResults createLegacyPMAAResults(Pass &P, Function &F, BasicAAResult &BAR); + } // End llvm namespace #endif diff --git a/include/llvm/Analysis/AliasSetTracker.h b/include/llvm/Analysis/AliasSetTracker.h index 881699d09225..37fd69b081cc 100644 --- a/include/llvm/Analysis/AliasSetTracker.h +++ b/include/llvm/Analysis/AliasSetTracker.h @@ -20,13 +20,13 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/ValueHandle.h" #include <vector> namespace llvm { -class AliasAnalysis; class LoadInst; class StoreInst; class VAArgInst; @@ -42,13 +42,14 @@ class AliasSet : public ilist_node<AliasSet> { AliasSet *AS; uint64_t Size; AAMDNodes AAInfo; + public: PointerRec(Value *V) : Val(V), PrevInList(nullptr), NextInList(nullptr), AS(nullptr), Size(0), AAInfo(DenseMapInfo<AAMDNodes>::getEmptyKey()) {} Value *getValue() const { return Val; } - + PointerRec *getNext() const { return NextInList; } bool hasAliasSet() const { return AS != nullptr; } @@ -156,7 +157,7 @@ class AliasSet : public ilist_node<AliasSet> { assert(i < UnknownInsts.size()); return UnknownInsts[i]; } - + public: /// Accessors... bool isRef() const { return Access & RefAccess; } @@ -190,6 +191,7 @@ public: class iterator : public std::iterator<std::forward_iterator_tag, PointerRec, ptrdiff_t> { PointerRec *CurNode; + public: explicit iterator(PointerRec *CN = nullptr) : CurNode(CN) {} @@ -282,14 +284,14 @@ inline raw_ostream& operator<<(raw_ostream &OS, const AliasSet &AS) { return OS; } - class AliasSetTracker { /// CallbackVH - A CallbackVH to arrange for AliasSetTracker to be /// notified whenever a Value is deleted. - class ASTCallbackVH : public CallbackVH { + class ASTCallbackVH final : public CallbackVH { AliasSetTracker *AST; void deleted() override; void allUsesReplacedWith(Value *) override; + public: ASTCallbackVH(Value *V, AliasSetTracker *AST = nullptr); ASTCallbackVH &operator=(Value *V); @@ -347,7 +349,7 @@ public: bool remove(Instruction *I); void remove(AliasSet &AS); bool removeUnknown(Instruction *I); - + void clear(); /// getAliasSets - Return the alias sets that are active. @@ -398,7 +400,6 @@ public: /// void copyValue(Value *From, Value *To); - typedef ilist<AliasSet>::iterator iterator; typedef ilist<AliasSet>::const_iterator const_iterator; diff --git a/include/llvm/Analysis/AssumptionCache.h b/include/llvm/Analysis/AssumptionCache.h index 1f00b691b305..b903f96d55b2 100644 --- a/include/llvm/Analysis/AssumptionCache.h +++ b/include/llvm/Analysis/AssumptionCache.h @@ -66,7 +66,7 @@ public: /// \brief Add an @llvm.assume intrinsic to this function's cache. /// - /// The call passed in must be an instruction within this fuction and must + /// The call passed in must be an instruction within this function and must /// not already be in the cache. void registerAssumption(CallInst *CI); @@ -79,7 +79,7 @@ public: } /// \brief Access the list of assumption handles currently tracked for this - /// fuction. + /// function. /// /// Note that these produce weak handles that may be null. The caller must /// handle that case. @@ -140,7 +140,7 @@ public: class AssumptionCacheTracker : public ImmutablePass { /// A callback value handle applied to function objects, which we use to /// delete our cache of intrinsics for a function when it is deleted. - class FunctionCallbackVH : public CallbackVH { + class FunctionCallbackVH final : public CallbackVH { AssumptionCacheTracker *ACT; void deleted() override; diff --git a/include/llvm/Analysis/BasicAliasAnalysis.h b/include/llvm/Analysis/BasicAliasAnalysis.h new file mode 100644 index 000000000000..181a9327024c --- /dev/null +++ b/include/llvm/Analysis/BasicAliasAnalysis.h @@ -0,0 +1,223 @@ +//===- BasicAliasAnalysis.h - Stateless, local Alias Analysis ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This is the interface for LLVM's primary stateless and local alias analysis. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_BASICALIASANALYSIS_H +#define LLVM_ANALYSIS_BASICALIASANALYSIS_H + +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GetElementPtrTypeIterator.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Support/ErrorHandling.h" + +namespace llvm { +class AssumptionCache; +class DominatorTree; +class LoopInfo; + +/// This is the AA result object for the basic, local, and stateless alias +/// analysis. It implements the AA query interface in an entirely stateless +/// manner. As one consequence, it is never invalidated. While it does retain +/// some storage, that is used as an optimization and not to preserve +/// information from query to query. +class BasicAAResult : public AAResultBase<BasicAAResult> { + friend AAResultBase<BasicAAResult>; + + const DataLayout &DL; + AssumptionCache &AC; + DominatorTree *DT; + LoopInfo *LI; + +public: + BasicAAResult(const DataLayout &DL, const TargetLibraryInfo &TLI, + AssumptionCache &AC, DominatorTree *DT = nullptr, + LoopInfo *LI = nullptr) + : AAResultBase(TLI), DL(DL), AC(AC), DT(DT), LI(LI) {} + + BasicAAResult(const BasicAAResult &Arg) + : AAResultBase(Arg), DL(Arg.DL), AC(Arg.AC), DT(Arg.DT), LI(Arg.LI) {} + BasicAAResult(BasicAAResult &&Arg) + : AAResultBase(std::move(Arg)), DL(Arg.DL), AC(Arg.AC), DT(Arg.DT), + LI(Arg.LI) {} + + /// Handle invalidation events from the new pass manager. + /// + /// By definition, this result is stateless and so remains valid. + bool invalidate(Function &, const PreservedAnalyses &) { return false; } + + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); + + ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + + ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); + + /// Chases pointers until we find a (constant global) or not. + bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal); + + /// Get the location associated with a pointer argument of a callsite. + ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx); + + /// Returns the behavior when calling the given call site. + FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS); + + /// Returns the behavior when calling the given function. For use when the + /// call site is not known. + FunctionModRefBehavior getModRefBehavior(const Function *F); + +private: + // A linear transformation of a Value; this class represents ZExt(SExt(V, + // SExtBits), ZExtBits) * Scale + Offset. + struct VariableGEPIndex { + + // An opaque Value - we can't decompose this further. + const Value *V; + + // We need to track what extensions we've done as we consider the same Value + // with different extensions as different variables in a GEP's linear + // expression; + // e.g.: if V == -1, then sext(x) != zext(x). + unsigned ZExtBits; + unsigned SExtBits; + + int64_t Scale; + + bool operator==(const VariableGEPIndex &Other) const { + return V == Other.V && ZExtBits == Other.ZExtBits && + SExtBits == Other.SExtBits && Scale == Other.Scale; + } + + bool operator!=(const VariableGEPIndex &Other) const { + return !operator==(Other); + } + }; + + /// Track alias queries to guard against recursion. + typedef std::pair<MemoryLocation, MemoryLocation> LocPair; + typedef SmallDenseMap<LocPair, AliasResult, 8> AliasCacheTy; + AliasCacheTy AliasCache; + + /// Tracks phi nodes we have visited. + /// + /// When interpret "Value" pointer equality as value equality we need to make + /// sure that the "Value" is not part of a cycle. Otherwise, two uses could + /// come from different "iterations" of a cycle and see different values for + /// the same "Value" pointer. + /// + /// The following example shows the problem: + /// %p = phi(%alloca1, %addr2) + /// %l = load %ptr + /// %addr1 = gep, %alloca2, 0, %l + /// %addr2 = gep %alloca2, 0, (%l + 1) + /// alias(%p, %addr1) -> MayAlias ! + /// store %l, ... + SmallPtrSet<const BasicBlock *, 8> VisitedPhiBBs; + + /// Tracks instructions visited by pointsToConstantMemory. + SmallPtrSet<const Value *, 16> Visited; + + static const Value * + GetLinearExpression(const Value *V, APInt &Scale, APInt &Offset, + unsigned &ZExtBits, unsigned &SExtBits, + const DataLayout &DL, unsigned Depth, AssumptionCache *AC, + DominatorTree *DT, bool &NSW, bool &NUW); + + static const Value * + DecomposeGEPExpression(const Value *V, int64_t &BaseOffs, + SmallVectorImpl<VariableGEPIndex> &VarIndices, + bool &MaxLookupReached, const DataLayout &DL, + AssumptionCache *AC, DominatorTree *DT); + /// \brief A Heuristic for aliasGEP that searches for a constant offset + /// between the variables. + /// + /// GetLinearExpression has some limitations, as generally zext(%x + 1) + /// != zext(%x) + zext(1) if the arithmetic overflows. GetLinearExpression + /// will therefore conservatively refuse to decompose these expressions. + /// However, we know that, for all %x, zext(%x) != zext(%x + 1), even if + /// the addition overflows. + bool + constantOffsetHeuristic(const SmallVectorImpl<VariableGEPIndex> &VarIndices, + uint64_t V1Size, uint64_t V2Size, int64_t BaseOffset, + AssumptionCache *AC, DominatorTree *DT); + + bool isValueEqualInPotentialCycles(const Value *V1, const Value *V2); + + void GetIndexDifference(SmallVectorImpl<VariableGEPIndex> &Dest, + const SmallVectorImpl<VariableGEPIndex> &Src); + + AliasResult aliasGEP(const GEPOperator *V1, uint64_t V1Size, + const AAMDNodes &V1AAInfo, const Value *V2, + uint64_t V2Size, const AAMDNodes &V2AAInfo, + const Value *UnderlyingV1, const Value *UnderlyingV2); + + AliasResult aliasPHI(const PHINode *PN, uint64_t PNSize, + const AAMDNodes &PNAAInfo, const Value *V2, + uint64_t V2Size, const AAMDNodes &V2AAInfo); + + AliasResult aliasSelect(const SelectInst *SI, uint64_t SISize, + const AAMDNodes &SIAAInfo, const Value *V2, + uint64_t V2Size, const AAMDNodes &V2AAInfo); + + AliasResult aliasCheck(const Value *V1, uint64_t V1Size, AAMDNodes V1AATag, + const Value *V2, uint64_t V2Size, AAMDNodes V2AATag); +}; + +/// Analysis pass providing a never-invalidated alias analysis result. +class BasicAA { +public: + typedef BasicAAResult Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + BasicAAResult run(Function &F, AnalysisManager<Function> *AM); + + /// \brief Provide access to a name for this pass for debugging purposes. + static StringRef name() { return "BasicAliasAnalysis"; } + +private: + static char PassID; +}; + +/// Legacy wrapper pass to provide the BasicAAResult object. +class BasicAAWrapperPass : public FunctionPass { + std::unique_ptr<BasicAAResult> Result; + + virtual void anchor(); + +public: + static char ID; + + BasicAAWrapperPass(); + + BasicAAResult &getResult() { return *Result; } + const BasicAAResult &getResult() const { return *Result; } + + bool runOnFunction(Function &F) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +FunctionPass *createBasicAAWrapperPass(); + +/// A helper for the legacy pass manager to create a \c BasicAAResult object +/// populated to the best of our ability for a particular function when inside +/// of a \c ModulePass or a \c CallGraphSCCPass. +BasicAAResult createLegacyPMBasicAAResult(Pass &P, Function &F); +} + +#endif diff --git a/include/llvm/Analysis/BlockFrequencyInfo.h b/include/llvm/Analysis/BlockFrequencyInfo.h index f27c32df9283..6f2a2b522769 100644 --- a/include/llvm/Analysis/BlockFrequencyInfo.h +++ b/include/llvm/Analysis/BlockFrequencyInfo.h @@ -21,26 +21,20 @@ namespace llvm { class BranchProbabilityInfo; +class LoopInfo; template <class BlockT> class BlockFrequencyInfoImpl; /// BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to /// estimate IR basic block frequencies. -class BlockFrequencyInfo : public FunctionPass { +class BlockFrequencyInfo { typedef BlockFrequencyInfoImpl<BasicBlock> ImplType; std::unique_ptr<ImplType> BFI; public: - static char ID; - BlockFrequencyInfo(); + BlockFrequencyInfo(const Function &F, const BranchProbabilityInfo &BPI, + const LoopInfo &LI); - ~BlockFrequencyInfo() override; - - void getAnalysisUsage(AnalysisUsage &AU) const override; - - bool runOnFunction(Function &F) override; - void releaseMemory() override; - void print(raw_ostream &O, const Module *M) const override; const Function *getFunction() const; void view() const; @@ -51,6 +45,13 @@ public: /// floating points. BlockFrequency getBlockFreq(const BasicBlock *BB) const; + // Set the frequency of the given basic block. + void setBlockFreq(const BasicBlock *BB, uint64_t Freq); + + /// calculate - compute block frequency info for the given function. + void calculate(const Function &F, const BranchProbabilityInfo &BPI, + const LoopInfo &LI); + // Print the block frequency Freq to OS using the current functions entry // frequency to convert freq into a relative decimal form. raw_ostream &printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const; @@ -60,7 +61,28 @@ public: raw_ostream &printBlockFreq(raw_ostream &OS, const BasicBlock *BB) const; uint64_t getEntryFreq() const; + void releaseMemory(); + void print(raw_ostream &OS) const; +}; +/// \brief Legacy analysis pass which computes \c BlockFrequencyInfo. +class BlockFrequencyInfoWrapperPass : public FunctionPass { + BlockFrequencyInfo BFI; + +public: + static char ID; + + BlockFrequencyInfoWrapperPass(); + ~BlockFrequencyInfoWrapperPass() override; + + BlockFrequencyInfo &getBFI() { return BFI; } + const BlockFrequencyInfo &getBFI() const { return BFI; } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnFunction(Function &F) override; + void releaseMemory() override; + void print(raw_ostream &OS, const Module *M) const override; }; } diff --git a/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/include/llvm/Analysis/BlockFrequencyInfoImpl.h index 32d96090f456..387e9a887d93 100644 --- a/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -84,7 +84,7 @@ public: /// \brief Add another mass. /// /// Adds another mass, saturating at \a isFull() rather than overflowing. - BlockMass &operator+=(const BlockMass &X) { + BlockMass &operator+=(BlockMass X) { uint64_t Sum = Mass + X.Mass; Mass = Sum < Mass ? UINT64_MAX : Sum; return *this; @@ -94,23 +94,23 @@ public: /// /// Subtracts another mass, saturating at \a isEmpty() rather than /// undeflowing. - BlockMass &operator-=(const BlockMass &X) { + BlockMass &operator-=(BlockMass X) { uint64_t Diff = Mass - X.Mass; Mass = Diff > Mass ? 0 : Diff; return *this; } - BlockMass &operator*=(const BranchProbability &P) { + BlockMass &operator*=(BranchProbability P) { Mass = P.scale(Mass); return *this; } - bool operator==(const BlockMass &X) const { return Mass == X.Mass; } - bool operator!=(const BlockMass &X) const { return Mass != X.Mass; } - bool operator<=(const BlockMass &X) const { return Mass <= X.Mass; } - bool operator>=(const BlockMass &X) const { return Mass >= X.Mass; } - bool operator<(const BlockMass &X) const { return Mass < X.Mass; } - bool operator>(const BlockMass &X) const { return Mass > X.Mass; } + bool operator==(BlockMass X) const { return Mass == X.Mass; } + bool operator!=(BlockMass X) const { return Mass != X.Mass; } + bool operator<=(BlockMass X) const { return Mass <= X.Mass; } + bool operator>=(BlockMass X) const { return Mass >= X.Mass; } + bool operator<(BlockMass X) const { return Mass < X.Mass; } + bool operator>(BlockMass X) const { return Mass > X.Mass; } /// \brief Convert to scaled number. /// @@ -122,20 +122,20 @@ public: raw_ostream &print(raw_ostream &OS) const; }; -inline BlockMass operator+(const BlockMass &L, const BlockMass &R) { +inline BlockMass operator+(BlockMass L, BlockMass R) { return BlockMass(L) += R; } -inline BlockMass operator-(const BlockMass &L, const BlockMass &R) { +inline BlockMass operator-(BlockMass L, BlockMass R) { return BlockMass(L) -= R; } -inline BlockMass operator*(const BlockMass &L, const BranchProbability &R) { +inline BlockMass operator*(BlockMass L, BranchProbability R) { return BlockMass(L) *= R; } -inline BlockMass operator*(const BranchProbability &L, const BlockMass &R) { +inline BlockMass operator*(BranchProbability L, BlockMass R) { return BlockMass(R) *= L; } -inline raw_ostream &operator<<(raw_ostream &OS, const BlockMass &X) { +inline raw_ostream &operator<<(raw_ostream &OS, BlockMass X) { return X.print(OS); } @@ -477,6 +477,8 @@ public: BlockFrequency getBlockFreq(const BlockNode &Node) const; + void setBlockFreq(const BlockNode &Node, uint64_t Freq); + raw_ostream &printBlockFreq(raw_ostream &OS, const BlockNode &Node) const; raw_ostream &printBlockFreq(raw_ostream &OS, const BlockFrequency &Freq) const; @@ -905,14 +907,15 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase { public: const FunctionT *getFunction() const { return F; } - void doFunction(const FunctionT *F, const BranchProbabilityInfoT *BPI, - const LoopInfoT *LI); + void calculate(const FunctionT &F, const BranchProbabilityInfoT &BPI, + const LoopInfoT &LI); BlockFrequencyInfoImpl() : BPI(nullptr), LI(nullptr), F(nullptr) {} using BlockFrequencyInfoImplBase::getEntryFreq; BlockFrequency getBlockFreq(const BlockT *BB) const { return BlockFrequencyInfoImplBase::getBlockFreq(getNode(BB)); } + void setBlockFreq(const BlockT *BB, uint64_t Freq); Scaled64 getFloatingBlockFreq(const BlockT *BB) const { return BlockFrequencyInfoImplBase::getFloatingBlockFreq(getNode(BB)); } @@ -938,13 +941,13 @@ public: }; template <class BT> -void BlockFrequencyInfoImpl<BT>::doFunction(const FunctionT *F, - const BranchProbabilityInfoT *BPI, - const LoopInfoT *LI) { +void BlockFrequencyInfoImpl<BT>::calculate(const FunctionT &F, + const BranchProbabilityInfoT &BPI, + const LoopInfoT &LI) { // Save the parameters. - this->BPI = BPI; - this->LI = LI; - this->F = F; + this->BPI = &BPI; + this->LI = &LI; + this->F = &F; // Clean up left-over data structures. BlockFrequencyInfoImplBase::clear(); @@ -952,8 +955,8 @@ void BlockFrequencyInfoImpl<BT>::doFunction(const FunctionT *F, Nodes.clear(); // Initialize. - DEBUG(dbgs() << "\nblock-frequency: " << F->getName() << "\n=================" - << std::string(F->getName().size(), '=') << "\n"); + DEBUG(dbgs() << "\nblock-frequency: " << F.getName() << "\n=================" + << std::string(F.getName().size(), '=') << "\n"); initializeRPOT(); initializeLoops(); @@ -965,8 +968,23 @@ void BlockFrequencyInfoImpl<BT>::doFunction(const FunctionT *F, finalizeMetrics(); } +template <class BT> +void BlockFrequencyInfoImpl<BT>::setBlockFreq(const BlockT *BB, uint64_t Freq) { + if (Nodes.count(BB)) + BlockFrequencyInfoImplBase::setBlockFreq(getNode(BB), Freq); + else { + // If BB is a newly added block after BFI is done, we need to create a new + // BlockNode for it assigned with a new index. The index can be determined + // by the size of Freqs. + BlockNode NewNode(Freqs.size()); + Nodes[BB] = NewNode; + Freqs.emplace_back(); + BlockFrequencyInfoImplBase::setBlockFreq(NewNode, Freq); + } +} + template <class BT> void BlockFrequencyInfoImpl<BT>::initializeRPOT() { - const BlockT *Entry = F->begin(); + const BlockT *Entry = &F->front(); RPOT.reserve(F->size()); std::copy(po_begin(Entry), po_end(Entry), std::back_inserter(RPOT)); std::reverse(RPOT.begin(), RPOT.end()); @@ -1155,6 +1173,13 @@ void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass( updateLoopWithIrreducible(*OuterLoop); } +namespace { +// A helper function that converts a branch probability into weight. +inline uint32_t getWeightFromBranchProb(const BranchProbability Prob) { + return Prob.getNumerator(); +} +} // namespace + template <class BT> bool BlockFrequencyInfoImpl<BT>::propagateMassToSuccessors(LoopData *OuterLoop, @@ -1171,10 +1196,8 @@ BlockFrequencyInfoImpl<BT>::propagateMassToSuccessors(LoopData *OuterLoop, const BlockT *BB = getBlock(Node); for (auto SI = Successor::child_begin(BB), SE = Successor::child_end(BB); SI != SE; ++SI) - // Do not dereference SI, or getEdgeWeight() is linear in the number of - // successors. if (!addToDist(Dist, OuterLoop, Node, getNode(*SI), - BPI->getEdgeWeight(BB, SI))) + getWeightFromBranchProb(BPI->getEdgeProbability(BB, SI)))) // Irreducible backedge. return false; } @@ -1190,10 +1213,11 @@ raw_ostream &BlockFrequencyInfoImpl<BT>::print(raw_ostream &OS) const { if (!F) return OS; OS << "block-frequency-info: " << F->getName() << "\n"; - for (const BlockT &BB : *F) - OS << " - " << bfi_detail::getBlockName(&BB) - << ": float = " << getFloatingBlockFreq(&BB) - << ", int = " << getBlockFreq(&BB).getFrequency() << "\n"; + for (const BlockT &BB : *F) { + OS << " - " << bfi_detail::getBlockName(&BB) << ": float = "; + getFloatingBlockFreq(&BB).print(OS, 5) + << ", int = " << getBlockFreq(&BB).getFrequency() << "\n"; + } // Add an extra newline for readability. OS << "\n"; diff --git a/include/llvm/Analysis/BranchProbabilityInfo.h b/include/llvm/Analysis/BranchProbabilityInfo.h index 9d867567ba29..cfdf218491bd 100644 --- a/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/include/llvm/Analysis/BranchProbabilityInfo.h @@ -25,9 +25,9 @@ namespace llvm { class LoopInfo; class raw_ostream; -/// \brief Analysis pass providing branch probability information. +/// \brief Analysis providing branch probability information. /// -/// This is a function analysis pass which provides information on the relative +/// This is a function analysis which provides information on the relative /// probabilities of each "edge" in the function's CFG where such an edge is /// defined by a pair (PredBlock and an index in the successors). The /// probability of an edge from one block is always relative to the @@ -37,20 +37,14 @@ class raw_ostream; /// identify an edge, since we can have multiple edges from Src to Dst. /// As an example, we can have a switch which jumps to Dst with value 0 and /// value 10. -class BranchProbabilityInfo : public FunctionPass { +class BranchProbabilityInfo { public: - static char ID; - - BranchProbabilityInfo() : FunctionPass(ID) { - initializeBranchProbabilityInfoPass(*PassRegistry::getPassRegistry()); - } - - void getAnalysisUsage(AnalysisUsage &AU) const override; - bool runOnFunction(Function &F) override; + BranchProbabilityInfo() {} + BranchProbabilityInfo(Function &F, const LoopInfo &LI) { calculate(F, LI); } - void releaseMemory() override; + void releaseMemory(); - void print(raw_ostream &OS, const Module *M = nullptr) const override; + void print(raw_ostream &OS) const; /// \brief Get an edge's probability, relative to other out-edges of the Src. /// @@ -67,6 +61,9 @@ public: BranchProbability getEdgeProbability(const BasicBlock *Src, const BasicBlock *Dst) const; + BranchProbability getEdgeProbability(const BasicBlock *Src, + succ_const_iterator Dst) const; + /// \brief Test if an edge is hot relative to other out-edges of the Src. /// /// Check whether this edge out of the source block is 'hot'. We define hot @@ -87,37 +84,22 @@ public: raw_ostream &printEdgeProbability(raw_ostream &OS, const BasicBlock *Src, const BasicBlock *Dst) const; - /// \brief Get the raw edge weight calculated for the edge. + /// \brief Set the raw edge probability for the given edge. /// - /// This returns the raw edge weight. It is guaranteed to fall between 1 and - /// UINT32_MAX. Note that the raw edge weight is not meaningful in isolation. - /// This interface should be very carefully, and primarily by routines that - /// are updating the analysis by later calling setEdgeWeight. - uint32_t getEdgeWeight(const BasicBlock *Src, - unsigned IndexInSuccessors) const; - - /// \brief Get the raw edge weight calculated for the block pair. - /// - /// This returns the sum of all raw edge weights from Src to Dst. - /// It is guaranteed to fall between 1 and UINT32_MAX. - uint32_t getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const; - - uint32_t getEdgeWeight(const BasicBlock *Src, - succ_const_iterator Dst) const; - - /// \brief Set the raw edge weight for a given edge. - /// - /// This allows a pass to explicitly set the edge weight for an edge. It can - /// be used when updating the CFG to update and preserve the branch + /// This allows a pass to explicitly set the edge probability for an edge. It + /// can be used when updating the CFG to update and preserve the branch /// probability information. Read the implementation of how these edge - /// weights are calculated carefully before using! - void setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors, - uint32_t Weight); + /// probabilities are calculated carefully before using! + void setEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors, + BranchProbability Prob); - static uint32_t getBranchWeightStackProtector(bool IsLikely) { - return IsLikely ? (1u << 20) - 1 : 1; + static BranchProbability getBranchProbStackProtector(bool IsLikely) { + static const BranchProbability LikelyProb((1u << 20) - 1, 1u << 20); + return IsLikely ? LikelyProb : LikelyProb.getCompl(); } + void calculate(Function &F, const LoopInfo& LI); + private: // Since we allow duplicate edges from one basic block to another, we use // a pair (PredBlock and an index in the successors) to specify an edge. @@ -131,10 +113,7 @@ private: // weight to just "inherit" the non-zero weight of an adjacent successor. static const uint32_t DEFAULT_WEIGHT = 16; - DenseMap<Edge, uint32_t> Weights; - - /// \brief Handle to the LoopInfo analysis. - LoopInfo *LI; + DenseMap<Edge, BranchProbability> Probs; /// \brief Track the last function we run over for printing. Function *LastF; @@ -145,19 +124,37 @@ private: /// \brief Track the set of blocks that always lead to a cold call. SmallPtrSet<BasicBlock *, 16> PostDominatedByColdCall; - /// \brief Get sum of the block successors' weights. - uint32_t getSumForBlock(const BasicBlock *BB) const; - bool calcUnreachableHeuristics(BasicBlock *BB); bool calcMetadataWeights(BasicBlock *BB); bool calcColdCallHeuristics(BasicBlock *BB); bool calcPointerHeuristics(BasicBlock *BB); - bool calcLoopBranchHeuristics(BasicBlock *BB); + bool calcLoopBranchHeuristics(BasicBlock *BB, const LoopInfo &LI); bool calcZeroHeuristics(BasicBlock *BB); bool calcFloatingPointHeuristics(BasicBlock *BB); bool calcInvokeHeuristics(BasicBlock *BB); }; +/// \brief Legacy analysis pass which computes \c BranchProbabilityInfo. +class BranchProbabilityInfoWrapperPass : public FunctionPass { + BranchProbabilityInfo BPI; + +public: + static char ID; + + BranchProbabilityInfoWrapperPass() : FunctionPass(ID) { + initializeBranchProbabilityInfoWrapperPassPass( + *PassRegistry::getPassRegistry()); + } + + BranchProbabilityInfo &getBPI() { return BPI; } + const BranchProbabilityInfo &getBPI() const { return BPI; } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnFunction(Function &F) override; + void releaseMemory() override; + void print(raw_ostream &OS, const Module *M = nullptr) const override; +}; + } #endif diff --git a/include/llvm/Analysis/CFG.h b/include/llvm/Analysis/CFG.h index 7c4df780198c..35165f4061f1 100644 --- a/include/llvm/Analysis/CFG.h +++ b/include/llvm/Analysis/CFG.h @@ -40,7 +40,7 @@ void FindFunctionBackedges( /// Search for the specified successor of basic block BB and return its position /// in the terminator instruction's list of successors. It is an error to call /// this with a block that is not a successor. -unsigned GetSuccessorNumber(BasicBlock *BB, BasicBlock *Succ); +unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ); /// Return true if the specified edge is a critical edge. Critical edges are /// edges from a block with multiple successors to a block with multiple diff --git a/include/llvm/Analysis/CFLAliasAnalysis.h b/include/llvm/Analysis/CFLAliasAnalysis.h new file mode 100644 index 000000000000..7473a454ab30 --- /dev/null +++ b/include/llvm/Analysis/CFLAliasAnalysis.h @@ -0,0 +1,158 @@ +//===- CFLAliasAnalysis.h - CFL-Based Alias Analysis Interface ---*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This is the interface for LLVM's primary stateless and local alias analysis. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_CFLALIASANALYSIS_H +#define LLVM_ANALYSIS_CFLALIASANALYSIS_H + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Pass.h" +#include <forward_list> + +namespace llvm { + +class CFLAAResult : public AAResultBase<CFLAAResult> { + friend AAResultBase<CFLAAResult>; + + struct FunctionInfo; + +public: + explicit CFLAAResult(const TargetLibraryInfo &TLI); + CFLAAResult(CFLAAResult &&Arg); + + /// Handle invalidation events from the new pass manager. + /// + /// By definition, this result is stateless and so remains valid. + bool invalidate(Function &, const PreservedAnalyses &) { return false; } + + /// \brief Inserts the given Function into the cache. + void scan(Function *Fn); + + void evict(Function *Fn); + + /// \brief Ensures that the given function is available in the cache. + /// Returns the appropriate entry from the cache. + const Optional<FunctionInfo> &ensureCached(Function *Fn); + + AliasResult query(const MemoryLocation &LocA, const MemoryLocation &LocB); + + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { + if (LocA.Ptr == LocB.Ptr) { + if (LocA.Size == LocB.Size) { + return MustAlias; + } else { + return PartialAlias; + } + } + + // Comparisons between global variables and other constants should be + // handled by BasicAA. + // TODO: ConstantExpr handling -- CFLAA may report NoAlias when comparing + // a GlobalValue and ConstantExpr, but every query needs to have at least + // one Value tied to a Function, and neither GlobalValues nor ConstantExprs + // are. + if (isa<Constant>(LocA.Ptr) && isa<Constant>(LocB.Ptr)) { + return AAResultBase::alias(LocA, LocB); + } + + AliasResult QueryResult = query(LocA, LocB); + if (QueryResult == MayAlias) + return AAResultBase::alias(LocA, LocB); + + return QueryResult; + } + +private: + struct FunctionHandle final : public CallbackVH { + FunctionHandle(Function *Fn, CFLAAResult *Result) + : CallbackVH(Fn), Result(Result) { + assert(Fn != nullptr); + assert(Result != nullptr); + } + + void deleted() override { removeSelfFromCache(); } + void allUsesReplacedWith(Value *) override { removeSelfFromCache(); } + + private: + CFLAAResult *Result; + + void removeSelfFromCache() { + assert(Result != nullptr); + auto *Val = getValPtr(); + Result->evict(cast<Function>(Val)); + setValPtr(nullptr); + } + }; + + /// \brief Cached mapping of Functions to their StratifiedSets. + /// If a function's sets are currently being built, it is marked + /// in the cache as an Optional without a value. This way, if we + /// have any kind of recursion, it is discernable from a function + /// that simply has empty sets. + DenseMap<Function *, Optional<FunctionInfo>> Cache; + std::forward_list<FunctionHandle> Handles; + + FunctionInfo buildSetsFrom(Function *F); +}; + +/// Analysis pass providing a never-invalidated alias analysis result. +/// +/// FIXME: We really should refactor CFL to use the analysis more heavily, and +/// in particular to leverage invalidation to trigger re-computation of sets. +class CFLAA { +public: + typedef CFLAAResult Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + CFLAAResult run(Function &F, AnalysisManager<Function> *AM); + + /// \brief Provide access to a name for this pass for debugging purposes. + static StringRef name() { return "CFLAA"; } + +private: + static char PassID; +}; + +/// Legacy wrapper pass to provide the CFLAAResult object. +class CFLAAWrapperPass : public ImmutablePass { + std::unique_ptr<CFLAAResult> Result; + +public: + static char ID; + + CFLAAWrapperPass(); + + CFLAAResult &getResult() { return *Result; } + const CFLAAResult &getResult() const { return *Result; } + + bool doInitialization(Module &M) override; + bool doFinalization(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +//===--------------------------------------------------------------------===// +// +// createCFLAAWrapperPass - This pass implements a set-based approach to +// alias analysis. +// +ImmutablePass *createCFLAAWrapperPass(); +} + +#endif diff --git a/include/llvm/Analysis/CGSCCPassManager.h b/include/llvm/Analysis/CGSCCPassManager.h index 6a406cd24402..e7635eb1ab67 100644 --- a/include/llvm/Analysis/CGSCCPassManager.h +++ b/include/llvm/Analysis/CGSCCPassManager.h @@ -358,7 +358,7 @@ private: /// returned PreservedAnalysis set. class CGSCCAnalysisManagerFunctionProxy { public: - /// \brief Result proxy object for \c ModuleAnalysisManagerFunctionProxy. + /// \brief Result proxy object for \c CGSCCAnalysisManagerFunctionProxy. class Result { public: explicit Result(const CGSCCAnalysisManager &CGAM) : CGAM(&CGAM) {} diff --git a/include/llvm/Analysis/CallGraph.h b/include/llvm/Analysis/CallGraph.h index 662ae0e6363c..5562e9b9465f 100644 --- a/include/llvm/Analysis/CallGraph.h +++ b/include/llvm/Analysis/CallGraph.h @@ -75,7 +75,8 @@ class CallGraphNode; class CallGraph { Module &M; - typedef std::map<const Function *, CallGraphNode *> FunctionMapTy; + typedef std::map<const Function *, std::unique_ptr<CallGraphNode>> + FunctionMapTy; /// \brief A map from \c Function* to \c CallGraphNode*. FunctionMapTy FunctionMap; @@ -90,7 +91,7 @@ class CallGraph { /// \brief This node has edges to it from all functions making indirect calls /// or calling an external function. - CallGraphNode *CallsExternalNode; + std::unique_ptr<CallGraphNode> CallsExternalNode; /// \brief Replace the function represented by this node by another. /// @@ -104,7 +105,8 @@ class CallGraph { void addToCallGraph(Function *F); public: - CallGraph(Module &M); + explicit CallGraph(Module &M); + CallGraph(CallGraph &&Arg); ~CallGraph(); void print(raw_ostream &OS) const; @@ -125,21 +127,23 @@ public: inline const CallGraphNode *operator[](const Function *F) const { const_iterator I = FunctionMap.find(F); assert(I != FunctionMap.end() && "Function not in callgraph!"); - return I->second; + return I->second.get(); } /// \brief Returns the call graph node for the provided function. inline CallGraphNode *operator[](const Function *F) { const_iterator I = FunctionMap.find(F); assert(I != FunctionMap.end() && "Function not in callgraph!"); - return I->second; + return I->second.get(); } /// \brief Returns the \c CallGraphNode which is used to represent /// undetermined calls into the callgraph. CallGraphNode *getExternalCallingNode() const { return ExternalCallingNode; } - CallGraphNode *getCallsExternalNode() const { return CallsExternalNode; } + CallGraphNode *getCallsExternalNode() const { + return CallsExternalNode.get(); + } //===--------------------------------------------------------------------- // Functions to keep a call graph up to date with a function that has been @@ -444,8 +448,10 @@ struct GraphTraits<CallGraph *> : public GraphTraits<CallGraphNode *> { static NodeType *getEntryNode(CallGraph *CGN) { return CGN->getExternalCallingNode(); // Start at the external node! } - typedef std::pair<const Function *, CallGraphNode *> PairTy; - typedef std::pointer_to_unary_function<PairTy, CallGraphNode &> DerefFun; + typedef std::pair<const Function *const, std::unique_ptr<CallGraphNode>> + PairTy; + typedef std::pointer_to_unary_function<const PairTy &, CallGraphNode &> + DerefFun; // nodes_iterator/begin/end - Allow iteration over all nodes in the graph typedef mapped_iterator<CallGraph::iterator, DerefFun> nodes_iterator; @@ -456,7 +462,7 @@ struct GraphTraits<CallGraph *> : public GraphTraits<CallGraphNode *> { return map_iterator(CG->end(), DerefFun(CGdereference)); } - static CallGraphNode &CGdereference(PairTy P) { return *P.second; } + static CallGraphNode &CGdereference(const PairTy &P) { return *P.second; } }; template <> @@ -465,8 +471,9 @@ struct GraphTraits<const CallGraph *> : public GraphTraits< static NodeType *getEntryNode(const CallGraph *CGN) { return CGN->getExternalCallingNode(); // Start at the external node! } - typedef std::pair<const Function *, const CallGraphNode *> PairTy; - typedef std::pointer_to_unary_function<PairTy, const CallGraphNode &> + typedef std::pair<const Function *const, std::unique_ptr<CallGraphNode>> + PairTy; + typedef std::pointer_to_unary_function<const PairTy &, const CallGraphNode &> DerefFun; // nodes_iterator/begin/end - Allow iteration over all nodes in the graph @@ -478,7 +485,9 @@ struct GraphTraits<const CallGraph *> : public GraphTraits< return map_iterator(CG->end(), DerefFun(CGdereference)); } - static const CallGraphNode &CGdereference(PairTy P) { return *P.second; } + static const CallGraphNode &CGdereference(const PairTy &P) { + return *P.second; + } }; } // End llvm namespace diff --git a/include/llvm/Analysis/CallGraphSCCPass.h b/include/llvm/Analysis/CallGraphSCCPass.h index 667e1715775f..9c7f7bd34cce 100644 --- a/include/llvm/Analysis/CallGraphSCCPass.h +++ b/include/llvm/Analysis/CallGraphSCCPass.h @@ -30,7 +30,7 @@ class CallGraphNode; class CallGraph; class PMStack; class CallGraphSCC; - + class CallGraphSCCPass : public Pass { public: explicit CallGraphSCCPass(char &pid) : Pass(PT_CallGraphSCC, pid) {} @@ -79,25 +79,26 @@ public: void getAnalysisUsage(AnalysisUsage &Info) const override; }; -/// CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on. +/// CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on. class CallGraphSCC { void *Context; // The CGPassManager object that is vending this. std::vector<CallGraphNode*> Nodes; + public: CallGraphSCC(void *context) : Context(context) {} - - void initialize(CallGraphNode*const*I, CallGraphNode*const*E) { + + void initialize(CallGraphNode *const *I, CallGraphNode *const *E) { Nodes.assign(I, E); } - + bool isSingular() const { return Nodes.size() == 1; } unsigned size() const { return Nodes.size(); } - + /// ReplaceNode - This informs the SCC and the pass manager that the specified /// Old node has been deleted, and New is to be used in its place. void ReplaceNode(CallGraphNode *Old, CallGraphNode *New); - - typedef std::vector<CallGraphNode*>::const_iterator iterator; + + typedef std::vector<CallGraphNode *>::const_iterator iterator; iterator begin() const { return Nodes.begin(); } iterator end() const { return Nodes.end(); } }; diff --git a/include/llvm/Analysis/CaptureTracking.h b/include/llvm/Analysis/CaptureTracking.h index 8b7c7a90f7c0..8d2c095d8585 100644 --- a/include/llvm/Analysis/CaptureTracking.h +++ b/include/llvm/Analysis/CaptureTracking.h @@ -20,6 +20,7 @@ namespace llvm { class Use; class Instruction; class DominatorTree; + class OrderedBasicBlock; /// PointerMayBeCaptured - Return true if this pointer value may be captured /// by the enclosing function (which is required to exist). This routine can @@ -41,10 +42,12 @@ namespace llvm { /// it or not. The boolean StoreCaptures specified whether storing the value /// (or part of it) into memory anywhere automatically counts as capturing it /// or not. Captures by the provided instruction are considered if the - /// final parameter is true. + /// final parameter is true. An ordered basic block in \p OBB could be used + /// to speed up capture-tracker queries. bool PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures, bool StoreCaptures, const Instruction *I, - DominatorTree *DT, bool IncludeI = false); + DominatorTree *DT, bool IncludeI = false, + OrderedBasicBlock *OBB = nullptr); /// This callback is used in conjunction with PointerMayBeCaptured. In /// addition to the interface here, you'll need to provide your own getters diff --git a/include/llvm/Analysis/DOTGraphTraitsPass.h b/include/llvm/Analysis/DOTGraphTraitsPass.h index cb74e9f32d3d..ca50ee2f829a 100644 --- a/include/llvm/Analysis/DOTGraphTraitsPass.h +++ b/include/llvm/Analysis/DOTGraphTraitsPass.h @@ -36,8 +36,23 @@ public: DOTGraphTraitsViewer(StringRef GraphName, char &ID) : FunctionPass(ID), Name(GraphName) {} + /// @brief Return true if this function should be processed. + /// + /// An implementation of this class my override this function to indicate that + /// only certain functions should be viewed. + /// + /// @param Analysis The current analysis result for this function. + virtual bool processFunction(Function &F, AnalysisT &Analysis) { + return true; + } + bool runOnFunction(Function &F) override { - GraphT Graph = AnalysisGraphTraitsT::getGraph(&getAnalysis<AnalysisT>()); + auto &Analysis = getAnalysis<AnalysisT>(); + + if (!processFunction(F, Analysis)) + return false; + + GraphT Graph = AnalysisGraphTraitsT::getGraph(&Analysis); std::string GraphName = DOTGraphTraits<GraphT>::getGraphName(Graph); std::string Title = GraphName + " for '" + F.getName().str() + "' function"; @@ -63,8 +78,23 @@ public: DOTGraphTraitsPrinter(StringRef GraphName, char &ID) : FunctionPass(ID), Name(GraphName) {} + /// @brief Return true if this function should be processed. + /// + /// An implementation of this class my override this function to indicate that + /// only certain functions should be printed. + /// + /// @param Analysis The current analysis result for this function. + virtual bool processFunction(Function &F, AnalysisT &Analysis) { + return true; + } + bool runOnFunction(Function &F) override { - GraphT Graph = AnalysisGraphTraitsT::getGraph(&getAnalysis<AnalysisT>()); + auto &Analysis = getAnalysis<AnalysisT>(); + + if (!processFunction(F, Analysis)) + return false; + + GraphT Graph = AnalysisGraphTraitsT::getGraph(&Analysis); std::string Filename = Name + "." + F.getName().str() + ".dot"; std::error_code EC; diff --git a/include/llvm/Analysis/DemandedBits.h b/include/llvm/Analysis/DemandedBits.h new file mode 100644 index 000000000000..42932bfd3491 --- /dev/null +++ b/include/llvm/Analysis/DemandedBits.h @@ -0,0 +1,75 @@ +//===-- llvm/Analysis/DemandedBits.h - Determine demanded bits --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass implements a demanded bits analysis. A demanded bit is one that +// contributes to a result; bits that are not demanded can be either zero or +// one without affecting control or data flow. For example in this sequence: +// +// %1 = add i32 %x, %y +// %2 = trunc i32 %1 to i16 +// +// Only the lowest 16 bits of %1 are demanded; the rest are removed by the +// trunc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_DEMANDED_BITS_H +#define LLVM_ANALYSIS_DEMANDED_BITS_H + +#include "llvm/Pass.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" + +namespace llvm { + +class FunctionPass; +class Function; +class Instruction; +class DominatorTree; +class AssumptionCache; + +struct DemandedBits : public FunctionPass { + static char ID; // Pass identification, replacement for typeid + DemandedBits(); + + bool runOnFunction(Function& F) override; + void getAnalysisUsage(AnalysisUsage& AU) const override; + void print(raw_ostream &OS, const Module *M) const override; + + /// Return the bits demanded from instruction I. + APInt getDemandedBits(Instruction *I); + + /// Return true if, during analysis, I could not be reached. + bool isInstructionDead(Instruction *I); + +private: + void performAnalysis(); + void determineLiveOperandBits(const Instruction *UserI, + const Instruction *I, unsigned OperandNo, + const APInt &AOut, APInt &AB, + APInt &KnownZero, APInt &KnownOne, + APInt &KnownZero2, APInt &KnownOne2); + + AssumptionCache *AC; + DominatorTree *DT; + Function *F; + bool Analyzed; + + // The set of visited instructions (non-integer-typed only). + SmallPtrSet<Instruction*, 128> Visited; + DenseMap<Instruction *, APInt> AliveBits; +}; + +/// Create a demanded bits analysis pass. +FunctionPass *createDemandedBitsPass(); + +} // End llvm namespace + +#endif diff --git a/include/llvm/Analysis/DependenceAnalysis.h b/include/llvm/Analysis/DependenceAnalysis.h index a08ce574ea56..5290552b41dc 100644 --- a/include/llvm/Analysis/DependenceAnalysis.h +++ b/include/llvm/Analysis/DependenceAnalysis.h @@ -42,11 +42,11 @@ #include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/Instructions.h" #include "llvm/Pass.h" namespace llvm { - class AliasAnalysis; class Loop; class LoopInfo; class ScalarEvolution; @@ -69,6 +69,15 @@ namespace llvm { /// as singly-linked lists, with the "next" fields stored in the dependence /// itelf. class Dependence { + protected: + Dependence(const Dependence &) = default; + + // FIXME: When we move to MSVC 2015 as the base compiler for Visual Studio + // support, uncomment this line to allow a defaulted move constructor for + // Dependence. Currently, FullDependence relies on the copy constructor, but + // that is acceptable given the triviality of the class. + // Dependence(Dependence &&) = default; + public: Dependence(Instruction *Source, Instruction *Destination) : @@ -176,38 +185,30 @@ namespace llvm { /// getNextPredecessor - Returns the value of the NextPredecessor /// field. - const Dependence *getNextPredecessor() const { - return NextPredecessor; - } - + const Dependence *getNextPredecessor() const { return NextPredecessor; } + /// getNextSuccessor - Returns the value of the NextSuccessor /// field. - const Dependence *getNextSuccessor() const { - return NextSuccessor; - } - + const Dependence *getNextSuccessor() const { return NextSuccessor; } + /// setNextPredecessor - Sets the value of the NextPredecessor /// field. - void setNextPredecessor(const Dependence *pred) { - NextPredecessor = pred; - } - + void setNextPredecessor(const Dependence *pred) { NextPredecessor = pred; } + /// setNextSuccessor - Sets the value of the NextSuccessor /// field. - void setNextSuccessor(const Dependence *succ) { - NextSuccessor = succ; - } - + void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; } + /// dump - For debugging purposes, dumps a dependence to OS. /// void dump(raw_ostream &OS) const; + private: Instruction *Src, *Dst; const Dependence *NextPredecessor, *NextSuccessor; friend class DependenceAnalysis; }; - /// FullDependence - This class represents a dependence between two memory /// references in a function. It contains detailed information about the /// dependence (direction vectors, etc.) and is used when the compiler is @@ -216,11 +217,15 @@ namespace llvm { /// (for output, flow, and anti dependences), the dependence implies an /// ordering, where the source must precede the destination; in contrast, /// input dependences are unordered. - class FullDependence : public Dependence { + class FullDependence final : public Dependence { public: FullDependence(Instruction *Src, Instruction *Dst, bool LoopIndependent, unsigned Levels); - ~FullDependence() override { delete[] DV; } + + FullDependence(FullDependence &&RHS) + : Dependence(std::move(RHS)), Levels(RHS.Levels), + LoopIndependent(RHS.LoopIndependent), Consistent(RHS.Consistent), + DV(std::move(RHS.DV)) {} /// isLoopIndependent - Returns true if this is a loop-independent /// dependence. @@ -268,16 +273,16 @@ namespace llvm { unsigned short Levels; bool LoopIndependent; bool Consistent; // Init to true, then refine. - DVEntry *DV; + std::unique_ptr<DVEntry[]> DV; friend class DependenceAnalysis; }; - /// DependenceAnalysis - This class is the main dependence-analysis driver. /// class DependenceAnalysis : public FunctionPass { void operator=(const DependenceAnalysis &) = delete; DependenceAnalysis(const DependenceAnalysis &) = delete; + public: /// depends - Tests for a dependence between the Src and Dst instructions. /// Returns NULL if no dependence; otherwise, returns a Dependence (or a @@ -387,6 +392,7 @@ namespace llvm { const SCEV *B; const SCEV *C; const Loop *AssociatedLoop; + public: /// isEmpty - Return true if the constraint is of kind Empty. bool isEmpty() const { return Kind == Empty; } @@ -453,7 +459,6 @@ namespace llvm { void dump(raw_ostream &OS) const; }; - /// establishNestingLevels - Examines the loop nesting of the Src and Dst /// instructions and establishes their shared loops. Sets the variables /// CommonLevels, SrcLevels, and MaxLevels. @@ -521,10 +526,10 @@ namespace llvm { /// in LoopNest. bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const; - /// Makes sure all subscript pairs share the same integer type by + /// Makes sure all subscript pairs share the same integer type by /// sign-extending as necessary. /// Sign-extending a subscript is safe because getelementptr assumes the - /// array subscripts are signed. + /// array subscripts are signed. void unifySubscriptType(ArrayRef<Subscript *> Pairs); /// removeMatchingExtensions - Examines a subscript pair. @@ -806,7 +811,6 @@ namespace llvm { const SCEV *Delta) const; /// testBounds - Returns true iff the current bounds are plausible. - /// bool testBounds(unsigned char DirKind, unsigned Level, BoundInfo *Bound, @@ -913,9 +917,8 @@ namespace llvm { void updateDirection(Dependence::DVEntry &Level, const Constraint &CurConstraint) const; - bool tryDelinearize(const SCEV *SrcSCEV, const SCEV *DstSCEV, - SmallVectorImpl<Subscript> &Pair, - const SCEV *ElementSize); + bool tryDelinearize(Instruction *Src, Instruction *Dst, + SmallVectorImpl<Subscript> &Pair); public: static char ID; // Class identification, replacement for typeinfo diff --git a/include/llvm/Analysis/DivergenceAnalysis.h b/include/llvm/Analysis/DivergenceAnalysis.h new file mode 100644 index 000000000000..aa2de571ba1b --- /dev/null +++ b/include/llvm/Analysis/DivergenceAnalysis.h @@ -0,0 +1,48 @@ +//===- llvm/Analysis/DivergenceAnalysis.h - Divergence Analysis -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// The divergence analysis is an LLVM pass which can be used to find out +// if a branch instruction in a GPU program is divergent or not. It can help +// branch optimizations such as jump threading and loop unswitching to make +// better decisions. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/DenseSet.h" +#include "llvm/IR/Function.h" +#include "llvm/Pass.h" + +namespace llvm { +class Value; +class DivergenceAnalysis : public FunctionPass { +public: + static char ID; + + DivergenceAnalysis() : FunctionPass(ID) { + initializeDivergenceAnalysisPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnFunction(Function &F) override; + + // Print all divergent branches in the function. + void print(raw_ostream &OS, const Module *) const override; + + // Returns true if V is divergent. + bool isDivergent(const Value *V) const { return DivergentValues.count(V); } + + // Returns true if V is uniform/non-divergent. + bool isUniform(const Value *V) const { return !isDivergent(V); } + +private: + // Stores all divergent values. + DenseSet<const Value *> DivergentValues; +}; +} // End llvm namespace diff --git a/include/llvm/Analysis/EHPersonalities.h b/include/llvm/Analysis/EHPersonalities.h new file mode 100644 index 000000000000..59e9672b88e5 --- /dev/null +++ b/include/llvm/Analysis/EHPersonalities.h @@ -0,0 +1,94 @@ +//===- EHPersonalities.h - Compute EH-related information -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_EHPERSONALITIES_H +#define LLVM_ANALYSIS_EHPERSONALITIES_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/TinyPtrVector.h" +#include "llvm/Support/ErrorHandling.h" + +namespace llvm { +class BasicBlock; +class Function; +class Value; + +enum class EHPersonality { + Unknown, + GNU_Ada, + GNU_C, + GNU_CXX, + GNU_ObjC, + MSVC_X86SEH, + MSVC_Win64SEH, + MSVC_CXX, + CoreCLR +}; + +/// \brief See if the given exception handling personality function is one +/// that we understand. If so, return a description of it; otherwise return +/// Unknown. +EHPersonality classifyEHPersonality(const Value *Pers); + +/// \brief Returns true if this personality function catches asynchronous +/// exceptions. +inline bool isAsynchronousEHPersonality(EHPersonality Pers) { + // The two SEH personality functions can catch asynch exceptions. We assume + // unknown personalities don't catch asynch exceptions. + switch (Pers) { + case EHPersonality::MSVC_X86SEH: + case EHPersonality::MSVC_Win64SEH: + return true; + default: + return false; + } + llvm_unreachable("invalid enum"); +} + +/// \brief Returns true if this is a personality function that invokes +/// handler funclets (which must return to it). +inline bool isFuncletEHPersonality(EHPersonality Pers) { + switch (Pers) { + case EHPersonality::MSVC_CXX: + case EHPersonality::MSVC_X86SEH: + case EHPersonality::MSVC_Win64SEH: + case EHPersonality::CoreCLR: + return true; + default: + return false; + } + llvm_unreachable("invalid enum"); +} + +/// \brief Return true if this personality may be safely removed if there +/// are no invoke instructions remaining in the current function. +inline bool isNoOpWithoutInvoke(EHPersonality Pers) { + switch (Pers) { + case EHPersonality::Unknown: + return false; + // All known personalities currently have this behavior + default: + return true; + } + llvm_unreachable("invalid enum"); +} + +bool canSimplifyInvokeNoUnwind(const Function *F); + +typedef TinyPtrVector<BasicBlock *> ColorVector; + +/// \brief If an EH funclet personality is in use (see isFuncletEHPersonality), +/// this will recompute which blocks are in which funclet. It is possible that +/// some blocks are in multiple funclets. Consider this analysis to be +/// expensive. +DenseMap<BasicBlock *, ColorVector> colorEHFunclets(Function &F); + +} // end namespace llvm + +#endif diff --git a/include/llvm/Analysis/GlobalsModRef.h b/include/llvm/Analysis/GlobalsModRef.h new file mode 100644 index 000000000000..bcd102e7ded2 --- /dev/null +++ b/include/llvm/Analysis/GlobalsModRef.h @@ -0,0 +1,160 @@ +//===- GlobalsModRef.h - Simple Mod/Ref AA for Globals ----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This is the interface for a simple mod/ref and alias analysis over globals. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_GLOBALSMODREF_H +#define LLVM_ANALYSIS_GLOBALSMODREF_H + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Pass.h" +#include <list> + +namespace llvm { + +/// An alias analysis result set for globals. +/// +/// This focuses on handling aliasing properties of globals and interprocedural +/// function call mod/ref information. +class GlobalsAAResult : public AAResultBase<GlobalsAAResult> { + friend AAResultBase<GlobalsAAResult>; + + class FunctionInfo; + + const DataLayout &DL; + + /// The globals that do not have their addresses taken. + SmallPtrSet<const GlobalValue *, 8> NonAddressTakenGlobals; + + /// IndirectGlobals - The memory pointed to by this global is known to be + /// 'owned' by the global. + SmallPtrSet<const GlobalValue *, 8> IndirectGlobals; + + /// AllocsForIndirectGlobals - If an instruction allocates memory for an + /// indirect global, this map indicates which one. + DenseMap<const Value *, const GlobalValue *> AllocsForIndirectGlobals; + + /// For each function, keep track of what globals are modified or read. + DenseMap<const Function *, FunctionInfo> FunctionInfos; + + /// A map of functions to SCC. The SCCs are described by a simple integer + /// ID that is only useful for comparing for equality (are two functions + /// in the same SCC or not?) + DenseMap<const Function *, unsigned> FunctionToSCCMap; + + /// Handle to clear this analysis on deletion of values. + struct DeletionCallbackHandle final : CallbackVH { + GlobalsAAResult *GAR; + std::list<DeletionCallbackHandle>::iterator I; + + DeletionCallbackHandle(GlobalsAAResult &GAR, Value *V) + : CallbackVH(V), GAR(&GAR) {} + + void deleted() override; + }; + + /// List of callbacks for globals being tracked by this analysis. Note that + /// these objects are quite large, but we only anticipate having one per + /// global tracked by this analysis. There are numerous optimizations we + /// could perform to the memory utilization here if this becomes a problem. + std::list<DeletionCallbackHandle> Handles; + + explicit GlobalsAAResult(const DataLayout &DL, const TargetLibraryInfo &TLI); + +public: + GlobalsAAResult(GlobalsAAResult &&Arg); + + static GlobalsAAResult analyzeModule(Module &M, const TargetLibraryInfo &TLI, + CallGraph &CG); + + //------------------------------------------------ + // Implement the AliasAnalysis API + // + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); + + using AAResultBase::getModRefInfo; + ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + + /// getModRefBehavior - Return the behavior of the specified function if + /// called from the specified call site. The call site may be null in which + /// case the most generic behavior of this function should be returned. + FunctionModRefBehavior getModRefBehavior(const Function *F); + + /// getModRefBehavior - Return the behavior of the specified function if + /// called from the specified call site. The call site may be null in which + /// case the most generic behavior of this function should be returned. + FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS); + +private: + FunctionInfo *getFunctionInfo(const Function *F); + + void AnalyzeGlobals(Module &M); + void AnalyzeCallGraph(CallGraph &CG, Module &M); + bool AnalyzeUsesOfPointer(Value *V, + SmallPtrSetImpl<Function *> *Readers = nullptr, + SmallPtrSetImpl<Function *> *Writers = nullptr, + GlobalValue *OkayStoreDest = nullptr); + bool AnalyzeIndirectGlobalMemory(GlobalVariable *GV); + void CollectSCCMembership(CallGraph &CG); + + bool isNonEscapingGlobalNoAlias(const GlobalValue *GV, const Value *V); + ModRefInfo getModRefInfoForArgument(ImmutableCallSite CS, + const GlobalValue *GV); +}; + +/// Analysis pass providing a never-invalidated alias analysis result. +class GlobalsAA { +public: + typedef GlobalsAAResult Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + GlobalsAAResult run(Module &M, AnalysisManager<Module> *AM); + + /// \brief Provide access to a name for this pass for debugging purposes. + static StringRef name() { return "GlobalsAA"; } + +private: + static char PassID; +}; + +/// Legacy wrapper pass to provide the GlobalsAAResult object. +class GlobalsAAWrapperPass : public ModulePass { + std::unique_ptr<GlobalsAAResult> Result; + +public: + static char ID; + + GlobalsAAWrapperPass(); + + GlobalsAAResult &getResult() { return *Result; } + const GlobalsAAResult &getResult() const { return *Result; } + + bool runOnModule(Module &M) override; + bool doFinalization(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +//===--------------------------------------------------------------------===// +// +// createGlobalsAAWrapperPass - This pass provides alias and mod/ref info for +// global values that do not have their addresses taken. +// +ModulePass *createGlobalsAAWrapperPass(); +} + +#endif diff --git a/include/llvm/Analysis/IVUsers.h b/include/llvm/Analysis/IVUsers.h index 00dbcbdd7806..37d01490dac6 100644 --- a/include/llvm/Analysis/IVUsers.h +++ b/include/llvm/Analysis/IVUsers.h @@ -34,7 +34,7 @@ class DataLayout; /// The Expr member keeps track of the expression, User is the actual user /// instruction of the operand, and 'OperandValToReplace' is the operand of /// the User that is the use. -class IVStrideUse : public CallbackVH, public ilist_node<IVStrideUse> { +class IVStrideUse final : public CallbackVH, public ilist_node<IVStrideUse> { friend class IVUsers; public: IVStrideUse(IVUsers *P, Instruction* U, Value *O) diff --git a/include/llvm/Analysis/InlineCost.h b/include/llvm/Analysis/InlineCost.h index 79ed74d82411..35f991cb3f67 100644 --- a/include/llvm/Analysis/InlineCost.h +++ b/include/llvm/Analysis/InlineCost.h @@ -23,7 +23,7 @@ class AssumptionCacheTracker; class CallSite; class DataLayout; class Function; -class TargetTransformInfoWrapperPass; +class TargetTransformInfo; namespace InlineConstants { // Various magic constants used to adjust heuristics. @@ -98,46 +98,31 @@ public: int getCostDelta() const { return Threshold - getCost(); } }; -/// \brief Cost analyzer used by inliner. -class InlineCostAnalysis : public CallGraphSCCPass { - TargetTransformInfoWrapperPass *TTIWP; - AssumptionCacheTracker *ACT; - -public: - static char ID; - - InlineCostAnalysis(); - ~InlineCostAnalysis() override; - - // Pass interface implementation. - void getAnalysisUsage(AnalysisUsage &AU) const override; - bool runOnSCC(CallGraphSCC &SCC) override; - - /// \brief Get an InlineCost object representing the cost of inlining this - /// callsite. - /// - /// Note that threshold is passed into this function. Only costs below the - /// threshold are computed with any accuracy. The threshold can be used to - /// bound the computation necessary to determine whether the cost is - /// sufficiently low to warrant inlining. - /// - /// Also note that calling this function *dynamically* computes the cost of - /// inlining the callsite. It is an expensive, heavyweight call. - InlineCost getInlineCost(CallSite CS, int Threshold); - - /// \brief Get an InlineCost with the callee explicitly specified. - /// This allows you to calculate the cost of inlining a function via a - /// pointer. This behaves exactly as the version with no explicit callee - /// parameter in all other respects. - // - // Note: This is used by out-of-tree passes, please do not remove without - // adding a replacement API. - InlineCost getInlineCost(CallSite CS, Function *Callee, int Threshold); - - /// \brief Minimal filter to detect invalid constructs for inlining. - bool isInlineViable(Function &Callee); -}; +/// \brief Get an InlineCost object representing the cost of inlining this +/// callsite. +/// +/// Note that threshold is passed into this function. Only costs below the +/// threshold are computed with any accuracy. The threshold can be used to +/// bound the computation necessary to determine whether the cost is +/// sufficiently low to warrant inlining. +/// +/// Also note that calling this function *dynamically* computes the cost of +/// inlining the callsite. It is an expensive, heavyweight call. +InlineCost getInlineCost(CallSite CS, int Threshold, + TargetTransformInfo &CalleeTTI, + AssumptionCacheTracker *ACT); + +/// \brief Get an InlineCost with the callee explicitly specified. +/// This allows you to calculate the cost of inlining a function via a +/// pointer. This behaves exactly as the version with no explicit callee +/// parameter in all other respects. +// +InlineCost getInlineCost(CallSite CS, Function *Callee, int Threshold, + TargetTransformInfo &CalleeTTI, + AssumptionCacheTracker *ACT); +/// \brief Minimal filter to detect invalid constructs for inlining. +bool isInlineViable(Function &Callee); } #endif diff --git a/include/llvm/Analysis/InstructionSimplify.h b/include/llvm/Analysis/InstructionSimplify.h index d44c5ff4078d..ed313dae9ab1 100644 --- a/include/llvm/Analysis/InstructionSimplify.h +++ b/include/llvm/Analysis/InstructionSimplify.h @@ -207,7 +207,7 @@ namespace llvm { const TargetLibraryInfo *TLI = nullptr, const DominatorTree *DT = nullptr, AssumptionCache *AC = nullptr, - Instruction *CxtI = nullptr); + const Instruction *CxtI = nullptr); /// SimplifyFCmpInst - Given operands for an FCmpInst, see if we can /// fold the result. If not, this returns null. diff --git a/include/llvm/Analysis/IteratedDominanceFrontier.h b/include/llvm/Analysis/IteratedDominanceFrontier.h index 5a339f10f50f..a1ded2554d44 100644 --- a/include/llvm/Analysis/IteratedDominanceFrontier.h +++ b/include/llvm/Analysis/IteratedDominanceFrontier.h @@ -34,7 +34,7 @@ namespace llvm { class BasicBlock; template <class T> class DomTreeNodeBase; typedef DomTreeNodeBase<BasicBlock> DomTreeNode; -class DominatorTree; +template <class T> class DominatorTreeBase; /// \brief Determine the iterated dominance frontier, given a set of defining /// blocks, and optionally, a set of live-in blocks. @@ -47,7 +47,7 @@ class DominatorTree; class IDFCalculator { public: - IDFCalculator(DominatorTree &DT) : DT(DT), useLiveIn(false) {} + IDFCalculator(DominatorTreeBase<BasicBlock> &DT) : DT(DT), useLiveIn(false) {} /// \brief Give the IDF calculator the set of blocks in which the value is /// defined. This is equivalent to the set of starting blocks it should be @@ -85,7 +85,7 @@ public: void calculate(SmallVectorImpl<BasicBlock *> &IDFBlocks); private: - DominatorTree &DT; + DominatorTreeBase<BasicBlock> &DT; bool useLiveIn; DenseMap<DomTreeNode *, unsigned> DomLevels; const SmallPtrSetImpl<BasicBlock *> *LiveInBlocks; diff --git a/include/llvm/Analysis/LazyCallGraph.h b/include/llvm/Analysis/LazyCallGraph.h index b0b9068de34b..ef3d5e8fe3df 100644 --- a/include/llvm/Analysis/LazyCallGraph.h +++ b/include/llvm/Analysis/LazyCallGraph.h @@ -54,7 +54,7 @@ namespace llvm { class PreservedAnalyses; class raw_ostream; -/// \brief A lazily constructed view of the call graph of a module. +/// A lazily constructed view of the call graph of a module. /// /// With the edges of this graph, the motivating constraint that we are /// attempting to maintain is that function-local optimization, CGSCC-local @@ -107,7 +107,7 @@ public: typedef SmallVector<PointerUnion<Function *, Node *>, 4> NodeVectorT; typedef SmallVectorImpl<PointerUnion<Function *, Node *>> NodeVectorImplT; - /// \brief A lazy iterator used for both the entry nodes and child nodes. + /// A lazy iterator used for both the entry nodes and child nodes. /// /// When this iterator is dereferenced, if not yet available, a function will /// be scanned for "calls" or uses of functions and its child information @@ -152,7 +152,7 @@ public: } }; - /// \brief A node in the call graph. + /// A node in the call graph. /// /// This represents a single node. It's primary roles are to cache the list of /// callees, de-duplicate and provide fast testing of whether a function is @@ -172,25 +172,23 @@ public: mutable NodeVectorT Callees; DenseMap<Function *, size_t> CalleeIndexMap; - /// \brief Basic constructor implements the scanning of F into Callees and + /// Basic constructor implements the scanning of F into Callees and /// CalleeIndexMap. Node(LazyCallGraph &G, Function &F); - /// \brief Internal helper to insert a callee. + /// Internal helper to insert a callee. void insertEdgeInternal(Function &Callee); - /// \brief Internal helper to insert a callee. + /// Internal helper to insert a callee. void insertEdgeInternal(Node &CalleeN); - /// \brief Internal helper to remove a callee from this node. + /// Internal helper to remove a callee from this node. void removeEdgeInternal(Function &Callee); public: typedef LazyCallGraph::iterator iterator; - Function &getFunction() const { - return F; - }; + Function &getFunction() const { return F; } iterator begin() const { return iterator(*G, Callees.begin(), Callees.end()); @@ -202,7 +200,7 @@ public: bool operator!=(const Node &N) const { return !operator==(N); } }; - /// \brief An SCC of the call graph. + /// An SCC of the call graph. /// /// This represents a Strongly Connected Component of the call graph as /// a collection of call graph nodes. While the order of nodes in the SCC is @@ -226,7 +224,8 @@ public: public: typedef SmallVectorImpl<Node *>::const_iterator iterator; - typedef pointee_iterator<SmallPtrSet<SCC *, 1>::const_iterator> parent_iterator; + typedef pointee_iterator<SmallPtrSet<SCC *, 1>::const_iterator> + parent_iterator; iterator begin() const { return Nodes.begin(); } iterator end() const { return Nodes.end(); } @@ -235,24 +234,24 @@ public: parent_iterator parent_end() const { return ParentSCCs.end(); } iterator_range<parent_iterator> parents() const { - return iterator_range<parent_iterator>(parent_begin(), parent_end()); + return make_range(parent_begin(), parent_end()); } - /// \brief Test if this SCC is a parent of \a C. + /// Test if this SCC is a parent of \a C. bool isParentOf(const SCC &C) const { return C.isChildOf(*this); } - /// \brief Test if this SCC is an ancestor of \a C. + /// Test if this SCC is an ancestor of \a C. bool isAncestorOf(const SCC &C) const { return C.isDescendantOf(*this); } - /// \brief Test if this SCC is a child of \a C. + /// Test if this SCC is a child of \a C. bool isChildOf(const SCC &C) const { return ParentSCCs.count(const_cast<SCC *>(&C)); } - /// \brief Test if this SCC is a descendant of \a C. + /// Test if this SCC is a descendant of \a C. bool isDescendantOf(const SCC &C) const; - /// \brief Short name useful for debugging or logging. + /// Short name useful for debugging or logging. /// /// We use the name of the first function in the SCC to name the SCC for /// the purposes of debugging and logging. @@ -267,22 +266,21 @@ public: /// Note that these methods sometimes have complex runtimes, so be careful /// how you call them. - /// \brief Insert an edge from one node in this SCC to another in this SCC. + /// Insert an edge from one node in this SCC to another in this SCC. /// /// By the definition of an SCC, this does not change the nature or make-up /// of any SCCs. void insertIntraSCCEdge(Node &CallerN, Node &CalleeN); - /// \brief Insert an edge whose tail is in this SCC and head is in some - /// child SCC. + /// Insert an edge whose tail is in this SCC and head is in some child SCC. /// /// There must be an existing path from the caller to the callee. This /// operation is inexpensive and does not change the set of SCCs in the /// graph. void insertOutgoingEdge(Node &CallerN, Node &CalleeN); - /// \brief Insert an edge whose tail is in a descendant SCC and head is in - /// this SCC. + /// Insert an edge whose tail is in a descendant SCC and head is in this + /// SCC. /// /// There must be an existing path from the callee to the caller in this /// case. NB! This is has the potential to be a very expensive function. It @@ -297,7 +295,7 @@ public: /// implementation for details, but that use case might impact users. SmallVector<SCC *, 1> insertIncomingEdge(Node &CallerN, Node &CalleeN); - /// \brief Remove an edge whose source is in this SCC and target is *not*. + /// Remove an edge whose source is in this SCC and target is *not*. /// /// This removes an inter-SCC edge. All inter-SCC edges originating from /// this SCC have been fully explored by any in-flight DFS SCC formation, @@ -309,7 +307,7 @@ public: /// them. void removeInterSCCEdge(Node &CallerN, Node &CalleeN); - /// \brief Remove an edge which is entirely within this SCC. + /// Remove an edge which is entirely within this SCC. /// /// Both the \a Caller and the \a Callee must be within this SCC. Removing /// such an edge make break cycles that form this SCC and thus this @@ -346,7 +344,7 @@ public: ///@} }; - /// \brief A post-order depth-first SCC iterator over the call graph. + /// A post-order depth-first SCC iterator over the call graph. /// /// This iterator triggers the Tarjan DFS-based formation of the SCC DAG for /// the call graph, walking it lazily in depth-first post-order. That is, it @@ -358,7 +356,7 @@ public: friend class LazyCallGraph; friend class LazyCallGraph::Node; - /// \brief Nonce type to select the constructor for the end iterator. + /// Nonce type to select the constructor for the end iterator. struct IsAtEndT {}; LazyCallGraph *G; @@ -387,7 +385,7 @@ public: } }; - /// \brief Construct a graph for the given module. + /// Construct a graph for the given module. /// /// This sets up the graph and computes all of the entry points of the graph. /// No function definitions are scanned until their nodes in the graph are @@ -410,22 +408,20 @@ public: } iterator_range<postorder_scc_iterator> postorder_sccs() { - return iterator_range<postorder_scc_iterator>(postorder_scc_begin(), - postorder_scc_end()); + return make_range(postorder_scc_begin(), postorder_scc_end()); } - /// \brief Lookup a function in the graph which has already been scanned and - /// added. + /// Lookup a function in the graph which has already been scanned and added. Node *lookup(const Function &F) const { return NodeMap.lookup(&F); } - /// \brief Lookup a function's SCC in the graph. + /// Lookup a function's SCC in the graph. /// /// \returns null if the function hasn't been assigned an SCC via the SCC /// iterator walk. SCC *lookupSCC(Node &N) const { return SCCMap.lookup(&N); } - /// \brief Get a graph node for a given function, scanning it to populate the - /// graph data as necessary. + /// Get a graph node for a given function, scanning it to populate the graph + /// data as necessary. Node &get(Function &F) { Node *&N = NodeMap[&F]; if (N) @@ -444,18 +440,18 @@ public: /// Once you begin manipulating a call graph's SCCs, you must perform all /// mutation of the graph via the SCC methods. - /// \brief Update the call graph after inserting a new edge. + /// Update the call graph after inserting a new edge. void insertEdge(Node &Caller, Function &Callee); - /// \brief Update the call graph after inserting a new edge. + /// Update the call graph after inserting a new edge. void insertEdge(Function &Caller, Function &Callee) { return insertEdge(get(Caller), Callee); } - /// \brief Update the call graph after deleting an edge. + /// Update the call graph after deleting an edge. void removeEdge(Node &Caller, Function &Callee); - /// \brief Update the call graph after deleting an edge. + /// Update the call graph after deleting an edge. void removeEdge(Function &Caller, Function &Callee) { return removeEdge(get(Caller), Callee); } @@ -463,57 +459,56 @@ public: ///@} private: - /// \brief Allocator that holds all the call graph nodes. + /// Allocator that holds all the call graph nodes. SpecificBumpPtrAllocator<Node> BPA; - /// \brief Maps function->node for fast lookup. + /// Maps function->node for fast lookup. DenseMap<const Function *, Node *> NodeMap; - /// \brief The entry nodes to the graph. + /// The entry nodes to the graph. /// /// These nodes are reachable through "external" means. Put another way, they /// escape at the module scope. NodeVectorT EntryNodes; - /// \brief Map of the entry nodes in the graph to their indices in - /// \c EntryNodes. + /// Map of the entry nodes in the graph to their indices in \c EntryNodes. DenseMap<Function *, size_t> EntryIndexMap; - /// \brief Allocator that holds all the call graph SCCs. + /// Allocator that holds all the call graph SCCs. SpecificBumpPtrAllocator<SCC> SCCBPA; - /// \brief Maps Function -> SCC for fast lookup. + /// Maps Function -> SCC for fast lookup. DenseMap<Node *, SCC *> SCCMap; - /// \brief The leaf SCCs of the graph. + /// The leaf SCCs of the graph. /// /// These are all of the SCCs which have no children. SmallVector<SCC *, 4> LeafSCCs; - /// \brief Stack of nodes in the DFS walk. + /// Stack of nodes in the DFS walk. SmallVector<std::pair<Node *, iterator>, 4> DFSStack; - /// \brief Set of entry nodes not-yet-processed into SCCs. + /// Set of entry nodes not-yet-processed into SCCs. SmallVector<Function *, 4> SCCEntryNodes; - /// \brief Stack of nodes the DFS has walked but not yet put into a SCC. + /// Stack of nodes the DFS has walked but not yet put into a SCC. SmallVector<Node *, 4> PendingSCCStack; - /// \brief Counter for the next DFS number to assign. + /// Counter for the next DFS number to assign. int NextDFSNumber; - /// \brief Helper to insert a new function, with an already looked-up entry in + /// Helper to insert a new function, with an already looked-up entry in /// the NodeMap. Node &insertInto(Function &F, Node *&MappedN); - /// \brief Helper to update pointers back to the graph object during moves. + /// Helper to update pointers back to the graph object during moves. void updateGraphPtrs(); - /// \brief Helper to form a new SCC out of the top of a DFSStack-like + /// Helper to form a new SCC out of the top of a DFSStack-like /// structure. SCC *formSCC(Node *RootN, SmallVectorImpl<Node *> &NodeStack); - /// \brief Retrieve the next node in the post-order SCC walk of the call graph. + /// Retrieve the next node in the post-order SCC walk of the call graph. SCC *getNextSCCInPostOrder(); }; @@ -535,17 +530,17 @@ template <> struct GraphTraits<LazyCallGraph *> { static ChildIteratorType child_end(NodeType *N) { return N->end(); } }; -/// \brief An analysis pass which computes the call graph for a module. +/// An analysis pass which computes the call graph for a module. class LazyCallGraphAnalysis { public: - /// \brief Inform generic clients of the result type. + /// Inform generic clients of the result type. typedef LazyCallGraph Result; static void *ID() { return (void *)&PassID; } static StringRef name() { return "Lazy CallGraph Analysis"; } - /// \brief Compute the \c LazyCallGraph for the module \c M. + /// Compute the \c LazyCallGraph for the module \c M. /// /// This just builds the set of entry points to the call graph. The rest is /// built lazily as it is walked. @@ -555,7 +550,7 @@ private: static char PassID; }; -/// \brief A pass which prints the call graph to a \c raw_ostream. +/// A pass which prints the call graph to a \c raw_ostream. /// /// This is primarily useful for testing the analysis. class LazyCallGraphPrinterPass { diff --git a/include/llvm/Analysis/LazyValueInfo.h b/include/llvm/Analysis/LazyValueInfo.h index 1051cff5efb7..42002062dca2 100644 --- a/include/llvm/Analysis/LazyValueInfo.h +++ b/include/llvm/Analysis/LazyValueInfo.h @@ -25,7 +25,7 @@ namespace llvm { class Instruction; class TargetLibraryInfo; class Value; - + /// This pass computes, caches, and vends lazy value constraint information. class LazyValueInfo : public FunctionPass { AssumptionCache *AC; @@ -45,23 +45,22 @@ public: enum Tristate { Unknown = -1, False = 0, True = 1 }; - - + // Public query interface. - + /// Determine whether the specified value comparison with a constant is known /// to be true or false on the specified CFG edge. /// Pred is a CmpInst predicate. Tristate getPredicateOnEdge(unsigned Pred, Value *V, Constant *C, BasicBlock *FromBB, BasicBlock *ToBB, Instruction *CxtI = nullptr); - + /// Determine whether the specified value comparison with a constant is known /// to be true or false at the specified instruction /// (from an assume intrinsic). Pred is a CmpInst predicate. Tristate getPredicateAt(unsigned Pred, Value *V, Constant *C, Instruction *CxtI); - + /// Determine whether the specified value is known to be a /// constant at the end of the specified block. Return null if not. Constant *getConstant(Value *V, BasicBlock *BB, Instruction *CxtI = nullptr); @@ -70,14 +69,14 @@ public: /// constant on the specified edge. Return null if not. Constant *getConstantOnEdge(Value *V, BasicBlock *FromBB, BasicBlock *ToBB, Instruction *CxtI = nullptr); - + /// Inform the analysis cache that we have threaded an edge from /// PredBB to OldSucc to be from PredBB to NewSucc instead. void threadEdge(BasicBlock *PredBB, BasicBlock *OldSucc, BasicBlock *NewSucc); - + /// Inform the analysis cache that we have erased a block. void eraseBlock(BasicBlock *BB); - + // Implementation boilerplate. void getAnalysisUsage(AnalysisUsage &AU) const override; diff --git a/include/llvm/Analysis/LibCallAliasAnalysis.h b/include/llvm/Analysis/LibCallAliasAnalysis.h deleted file mode 100644 index 6589ac13c746..000000000000 --- a/include/llvm/Analysis/LibCallAliasAnalysis.h +++ /dev/null @@ -1,71 +0,0 @@ -//===- LibCallAliasAnalysis.h - Implement AliasAnalysis for libcalls ------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the LibCallAliasAnalysis class. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_ANALYSIS_LIBCALLALIASANALYSIS_H -#define LLVM_ANALYSIS_LIBCALLALIASANALYSIS_H - -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/IR/Module.h" -#include "llvm/Pass.h" - -namespace llvm { - class LibCallInfo; - struct LibCallFunctionInfo; - - /// LibCallAliasAnalysis - Alias analysis driven from LibCallInfo. - struct LibCallAliasAnalysis : public FunctionPass, public AliasAnalysis { - static char ID; // Class identification - - LibCallInfo *LCI; - - explicit LibCallAliasAnalysis(LibCallInfo *LC = nullptr) - : FunctionPass(ID), LCI(LC) { - initializeLibCallAliasAnalysisPass(*PassRegistry::getPassRegistry()); - } - explicit LibCallAliasAnalysis(char &ID, LibCallInfo *LC) - : FunctionPass(ID), LCI(LC) { - initializeLibCallAliasAnalysisPass(*PassRegistry::getPassRegistry()); - } - ~LibCallAliasAnalysis() override; - - ModRefResult getModRefInfo(ImmutableCallSite CS, - const MemoryLocation &Loc) override; - - ModRefResult getModRefInfo(ImmutableCallSite CS1, - ImmutableCallSite CS2) override { - // TODO: Could compare two direct calls against each other if we cared to. - return AliasAnalysis::getModRefInfo(CS1, CS2); - } - - void getAnalysisUsage(AnalysisUsage &AU) const override; - - bool runOnFunction(Function &F) override; - - /// getAdjustedAnalysisPointer - This method is used when a pass implements - /// an analysis interface through multiple inheritance. If needed, it - /// should override this to adjust the this pointer as needed for the - /// specified pass info. - void *getAdjustedAnalysisPointer(const void *PI) override { - if (PI == &AliasAnalysis::ID) - return (AliasAnalysis*)this; - return this; - } - - private: - ModRefResult AnalyzeLibCallDetails(const LibCallFunctionInfo *FI, - ImmutableCallSite CS, - const MemoryLocation &Loc); - }; -} // End of llvm namespace - -#endif diff --git a/include/llvm/Analysis/LibCallSemantics.h b/include/llvm/Analysis/LibCallSemantics.h deleted file mode 100644 index b4bef310e590..000000000000 --- a/include/llvm/Analysis/LibCallSemantics.h +++ /dev/null @@ -1,225 +0,0 @@ -//===- LibCallSemantics.h - Describe library semantics --------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines interfaces that can be used to describe language specific -// runtime library interfaces (e.g. libc, libm, etc) to LLVM optimizers. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_ANALYSIS_LIBCALLSEMANTICS_H -#define LLVM_ANALYSIS_LIBCALLSEMANTICS_H - -#include "llvm/Analysis/AliasAnalysis.h" - -namespace llvm { -class InvokeInst; - - /// LibCallLocationInfo - This struct describes a set of memory locations that - /// are accessed by libcalls. Identification of a location is doing with a - /// simple callback function. - /// - /// For example, the LibCallInfo may be set up to model the behavior of - /// standard libm functions. The location that they may be interested in is - /// an abstract location that represents errno for the current target. In - /// this case, a location for errno is anything such that the predicate - /// returns true. On Mac OS X, this predicate would return true if the - /// pointer is the result of a call to "__error()". - /// - /// Locations can also be defined in a constant-sensitive way. For example, - /// it is possible to define a location that returns true iff it is passed - /// into the call as a specific argument. This is useful for modeling things - /// like "printf", which can store to memory, but only through pointers passed - /// with a '%n' constraint. - /// - struct LibCallLocationInfo { - // TODO: Flags: isContextSensitive etc. - - /// isLocation - Return a LocResult if the specified pointer refers to this - /// location for the specified call site. This returns "Yes" if we can tell - /// that the pointer *does definitely* refer to the location, "No" if we can - /// tell that the location *definitely does not* refer to the location, and - /// returns "Unknown" if we cannot tell for certain. - enum LocResult { - Yes, No, Unknown - }; - LocResult (*isLocation)(ImmutableCallSite CS, const MemoryLocation &Loc); - }; - - /// LibCallFunctionInfo - Each record in the array of FunctionInfo structs - /// records the behavior of one libcall that is known by the optimizer. This - /// captures things like the side effects of the call. Side effects are - /// modeled both universally (in the readnone/readonly) sense, but also - /// potentially against a set of abstract locations defined by the optimizer. - /// This allows an optimizer to define that some libcall (e.g. sqrt) is - /// side-effect free except that it might modify errno (thus, the call is - /// *not* universally readonly). Or it might say that the side effects - /// are unknown other than to say that errno is not modified. - /// - struct LibCallFunctionInfo { - /// Name - This is the name of the libcall this describes. - const char *Name; - - /// TODO: Constant folding function: Constant* vector -> Constant*. - - /// UniversalBehavior - This captures the absolute mod/ref behavior without - /// any specific context knowledge. For example, if the function is known - /// to be readonly, this would be set to 'ref'. If known to be readnone, - /// this is set to NoModRef. - AliasAnalysis::ModRefResult UniversalBehavior; - - /// LocationMRInfo - This pair captures info about whether a specific - /// location is modified or referenced by a libcall. - struct LocationMRInfo { - /// LocationID - ID # of the accessed location or ~0U for array end. - unsigned LocationID; - /// MRInfo - Mod/Ref info for this location. - AliasAnalysis::ModRefResult MRInfo; - }; - - /// DetailsType - Indicate the sense of the LocationDetails array. This - /// controls how the LocationDetails array is interpreted. - enum { - /// DoesOnly - If DetailsType is set to DoesOnly, then we know that the - /// *only* mod/ref behavior of this function is captured by the - /// LocationDetails array. If we are trying to say that 'sqrt' can only - /// modify errno, we'd have the {errnoloc,mod} in the LocationDetails - /// array and have DetailsType set to DoesOnly. - DoesOnly, - - /// DoesNot - If DetailsType is set to DoesNot, then the sense of the - /// LocationDetails array is completely inverted. This means that we *do - /// not* know everything about the side effects of this libcall, but we do - /// know things that the libcall cannot do. This is useful for complex - /// functions like 'ctime' which have crazy mod/ref behavior, but are - /// known to never read or write errno. In this case, we'd have - /// {errnoloc,modref} in the LocationDetails array and DetailsType would - /// be set to DoesNot, indicating that ctime does not read or write the - /// errno location. - DoesNot - } DetailsType; - - /// LocationDetails - This is a pointer to an array of LocationMRInfo - /// structs which indicates the behavior of the libcall w.r.t. specific - /// locations. For example, if this libcall is known to only modify - /// 'errno', it would have a LocationDetails array with the errno ID and - /// 'mod' in it. See the DetailsType field for how this is interpreted. - /// - /// In the "DoesOnly" case, this information is 'may' information for: there - /// is no guarantee that the specified side effect actually does happen, - /// just that it could. In the "DoesNot" case, this is 'must not' info. - /// - /// If this pointer is null, no details are known. - /// - const LocationMRInfo *LocationDetails; - }; - - - /// LibCallInfo - Abstract interface to query about library call information. - /// Instances of this class return known information about some set of - /// libcalls. - /// - class LibCallInfo { - // Implementation details of this object, private. - mutable void *Impl; - mutable const LibCallLocationInfo *Locations; - mutable unsigned NumLocations; - public: - LibCallInfo() : Impl(nullptr), Locations(nullptr), NumLocations(0) {} - virtual ~LibCallInfo(); - - //===------------------------------------------------------------------===// - // Accessor Methods: Efficient access to contained data. - //===------------------------------------------------------------------===// - - /// getLocationInfo - Return information about the specified LocationID. - const LibCallLocationInfo &getLocationInfo(unsigned LocID) const; - - - /// getFunctionInfo - Return the LibCallFunctionInfo object corresponding to - /// the specified function if we have it. If not, return null. - const LibCallFunctionInfo *getFunctionInfo(const Function *F) const; - - - //===------------------------------------------------------------------===// - // Implementation Methods: Subclasses should implement these. - //===------------------------------------------------------------------===// - - /// getLocationInfo - Return descriptors for the locations referenced by - /// this set of libcalls. - virtual unsigned getLocationInfo(const LibCallLocationInfo *&Array) const { - return 0; - } - - /// getFunctionInfoArray - Return an array of descriptors that describe the - /// set of libcalls represented by this LibCallInfo object. This array is - /// terminated by an entry with a NULL name. - virtual const LibCallFunctionInfo *getFunctionInfoArray() const = 0; - }; - - enum class EHPersonality { - Unknown, - GNU_Ada, - GNU_C, - GNU_CXX, - GNU_ObjC, - MSVC_X86SEH, - MSVC_Win64SEH, - MSVC_CXX, - }; - - /// \brief See if the given exception handling personality function is one - /// that we understand. If so, return a description of it; otherwise return - /// Unknown. - EHPersonality classifyEHPersonality(const Value *Pers); - - /// \brief Returns true if this personality function catches asynchronous - /// exceptions. - inline bool isAsynchronousEHPersonality(EHPersonality Pers) { - // The two SEH personality functions can catch asynch exceptions. We assume - // unknown personalities don't catch asynch exceptions. - switch (Pers) { - case EHPersonality::MSVC_X86SEH: - case EHPersonality::MSVC_Win64SEH: - return true; - default: return false; - } - llvm_unreachable("invalid enum"); - } - - /// \brief Returns true if this is an MSVC personality function. - inline bool isMSVCEHPersonality(EHPersonality Pers) { - // The two SEH personality functions can catch asynch exceptions. We assume - // unknown personalities don't catch asynch exceptions. - switch (Pers) { - case EHPersonality::MSVC_CXX: - case EHPersonality::MSVC_X86SEH: - case EHPersonality::MSVC_Win64SEH: - return true; - default: return false; - } - llvm_unreachable("invalid enum"); - } - - /// \brief Return true if this personality may be safely removed if there - /// are no invoke instructions remaining in the current function. - inline bool isNoOpWithoutInvoke(EHPersonality Pers) { - switch (Pers) { - case EHPersonality::Unknown: - return false; - // All known personalities currently have this behavior - default: return true; - } - llvm_unreachable("invalid enum"); - } - - bool canSimplifyInvokeNoUnwind(const Function *F); - -} // end namespace llvm - -#endif diff --git a/include/llvm/Analysis/Loads.h b/include/llvm/Analysis/Loads.h index 42667d2af14a..939663b0def1 100644 --- a/include/llvm/Analysis/Loads.h +++ b/include/llvm/Analysis/Loads.h @@ -14,11 +14,12 @@ #ifndef LLVM_ANALYSIS_LOADS_H #define LLVM_ANALYSIS_LOADS_H +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/Support/CommandLine.h" namespace llvm { -class AliasAnalysis; class DataLayout; class MDNode; @@ -29,15 +30,19 @@ class MDNode; bool isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom, unsigned Align); +/// DefMaxInstsToScan - the default number of maximum instructions +/// to scan in the block, used by FindAvailableLoadedValue(). +extern cl::opt<unsigned> DefMaxInstsToScan; + /// FindAvailableLoadedValue - Scan the ScanBB block backwards (starting at /// the instruction before ScanFrom) checking to see if we have the value at /// the memory address *Ptr locally available within a small number of /// instructions. If the value is available, return it. /// -/// If not, return the iterator for the last validated instruction that the +/// If not, return the iterator for the last validated instruction that the /// value would be live through. If we scanned the entire block and didn't /// find something that invalidates *Ptr or provides it, ScanFrom would be -/// left at begin() and this returns null. ScanFrom could also be left +/// left at begin() and this returns null. ScanFrom could also be left /// /// MaxInstsToScan specifies the maximum instructions to scan in the block. /// If it is set to 0, it will scan the whole block. You can also optionally @@ -48,7 +53,7 @@ bool isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom, /// is found, it is left unmodified. Value *FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, - unsigned MaxInstsToScan = 6, + unsigned MaxInstsToScan = DefMaxInstsToScan, AliasAnalysis *AA = nullptr, AAMDNodes *AATags = nullptr); diff --git a/include/llvm/Analysis/LoopAccessAnalysis.h b/include/llvm/Analysis/LoopAccessAnalysis.h index 476e4b6686bb..871d35e99b74 100644 --- a/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/include/llvm/Analysis/LoopAccessAnalysis.h @@ -29,10 +29,11 @@ namespace llvm { class Value; class DataLayout; -class AliasAnalysis; class ScalarEvolution; class Loop; class SCEV; +class SCEVUnionPredicate; +class LoopAccessInfo; /// Optimization analysis message produced during vectorization. Messages inform /// the user why vectorization did not occur. @@ -136,6 +137,14 @@ public: // We couldn't determine the direction or the distance. Unknown, // Lexically forward. + // + // FIXME: If we only have loop-independent forward dependences (e.g. a + // read and write of A[i]), LAA will locally deem the dependence "safe" + // without querying the MemoryDepChecker. Therefore we can miss + // enumerating loop-independent forward dependences in + // getDependences. Note that as soon as there are different + // indices used to access the same array, the MemoryDepChecker *is* + // queried and the dependence list is complete. Forward, // Forward, but if vectorized, is likely to prevent store-to-load // forwarding. @@ -162,13 +171,20 @@ public: Dependence(unsigned Source, unsigned Destination, DepType Type) : Source(Source), Destination(Destination), Type(Type) {} + /// \brief Return the source instruction of the dependence. + Instruction *getSource(const LoopAccessInfo &LAI) const; + /// \brief Return the destination instruction of the dependence. + Instruction *getDestination(const LoopAccessInfo &LAI) const; + /// \brief Dependence types that don't prevent vectorization. static bool isSafeForVectorization(DepType Type); - /// \brief Dependence types that can be queried from the analysis. - static bool isInterestingDependence(DepType Type); + /// \brief Lexically forward dependence. + bool isForward() const; + /// \brief Lexically backward dependence. + bool isBackward() const; - /// \brief Lexically backward dependence types. + /// \brief May be a lexically backward dependence type (includes Unknown). bool isPossiblyBackward() const; /// \brief Print the dependence. \p Instr is used to map the instruction @@ -177,10 +193,10 @@ public: const SmallVectorImpl<Instruction *> &Instrs) const; }; - MemoryDepChecker(ScalarEvolution *Se, const Loop *L) - : SE(Se), InnermostLoop(L), AccessIdx(0), + MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L) + : PSE(PSE), InnermostLoop(L), AccessIdx(0), ShouldRetryWithRuntimeCheck(false), SafeForVectorization(true), - RecordInterestingDependences(true) {} + RecordDependences(true) {} /// \brief Register the location (instructions are given increasing numbers) /// of a write access. @@ -218,14 +234,14 @@ public: /// vectorize the loop with a dynamic array access check. bool shouldRetryWithRuntimeCheck() { return ShouldRetryWithRuntimeCheck; } - /// \brief Returns the interesting dependences. If null is returned we - /// exceeded the MaxInterestingDependence threshold and this information is - /// not available. - const SmallVectorImpl<Dependence> *getInterestingDependences() const { - return RecordInterestingDependences ? &InterestingDependences : nullptr; + /// \brief Returns the memory dependences. If null is returned we exceeded + /// the MaxDependences threshold and this information is not + /// available. + const SmallVectorImpl<Dependence> *getDependences() const { + return RecordDependences ? &Dependences : nullptr; } - void clearInterestingDependences() { InterestingDependences.clear(); } + void clearDependences() { Dependences.clear(); } /// \brief The vector of memory access instructions. The indices are used as /// instruction identifiers in the Dependence class. @@ -233,12 +249,29 @@ public: return InstMap; } + /// \brief Generate a mapping between the memory instructions and their + /// indices according to program order. + DenseMap<Instruction *, unsigned> generateInstructionOrderMap() const { + DenseMap<Instruction *, unsigned> OrderMap; + + for (unsigned I = 0; I < InstMap.size(); ++I) + OrderMap[InstMap[I]] = I; + + return OrderMap; + } + /// \brief Find the set of instructions that read or write via \p Ptr. SmallVector<Instruction *, 4> getInstructionsForAccess(Value *Ptr, bool isWrite) const; private: - ScalarEvolution *SE; + /// A wrapper around ScalarEvolution, used to add runtime SCEV checks, and + /// applies dynamic knowledge to simplify SCEV expressions and convert them + /// to a more usable form. We need this in case assumptions about SCEV + /// expressions need to be made in order to avoid unknown dependences. For + /// example we might assume a unit stride for a pointer in order to prove + /// that a memory access is strided and doesn't wrap. + PredicatedScalarEvolution &PSE; const Loop *InnermostLoop; /// \brief Maps access locations (ptr, read/write) to program order. @@ -261,15 +294,14 @@ private: /// vectorization. bool SafeForVectorization; - //// \brief True if InterestingDependences reflects the dependences in the - //// loop. If false we exceeded MaxInterestingDependence and - //// InterestingDependences is invalid. - bool RecordInterestingDependences; + //// \brief True if Dependences reflects the dependences in the + //// loop. If false we exceeded MaxDependences and + //// Dependences is invalid. + bool RecordDependences; - /// \brief Interesting memory dependences collected during the analysis as - /// defined by isInterestingDependence. Only valid if - /// RecordInterestingDependences is true. - SmallVector<Dependence, 8> InterestingDependences; + /// \brief Memory dependences collected during the analysis. Only valid if + /// RecordDependences is true. + SmallVector<Dependence, 8> Dependences; /// \brief Check whether there is a plausible dependence between the two /// accesses. @@ -327,11 +359,17 @@ public: void reset() { Need = false; Pointers.clear(); + Checks.clear(); } /// Insert a pointer and calculate the start and end SCEVs. + /// \p We need Preds in order to compute the SCEV expression of the pointer + /// according to the assumptions that we've made during the analysis. + /// The method might also version the pointer stride according to \p Strides, + /// and change \p Preds. void insert(Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId, - unsigned ASId, const ValueToValueMap &Strides); + unsigned ASId, const ValueToValueMap &Strides, + PredicatedScalarEvolution &PSE); /// \brief No run-time memory checking is necessary. bool empty() const { return Pointers.empty(); } @@ -368,33 +406,38 @@ public: SmallVector<unsigned, 2> Members; }; - /// \brief Groups pointers such that a single memcheck is required - /// between two different groups. This will clear the CheckingGroups vector - /// and re-compute it. We will only group dependecies if \p UseDependencies - /// is true, otherwise we will create a separate group for each pointer. - void groupChecks(MemoryDepChecker::DepCandidates &DepCands, - bool UseDependencies); + /// \brief A memcheck which made up of a pair of grouped pointers. + /// + /// These *have* to be const for now, since checks are generated from + /// CheckingPtrGroups in LAI::addRuntimeChecks which is a const member + /// function. FIXME: once check-generation is moved inside this class (after + /// the PtrPartition hack is removed), we could drop const. + typedef std::pair<const CheckingPtrGroup *, const CheckingPtrGroup *> + PointerCheck; + + /// \brief Generate the checks and store it. This also performs the grouping + /// of pointers to reduce the number of memchecks necessary. + void generateChecks(MemoryDepChecker::DepCandidates &DepCands, + bool UseDependencies); + + /// \brief Returns the checks that generateChecks created. + const SmallVector<PointerCheck, 4> &getChecks() const { return Checks; } /// \brief Decide if we need to add a check between two groups of pointers, /// according to needsChecking. - bool needsChecking(const CheckingPtrGroup &M, const CheckingPtrGroup &N, - const SmallVectorImpl<int> *PtrPartition) const; - - /// \brief Return true if any pointer requires run-time checking according - /// to needsChecking. - bool needsAnyChecking(const SmallVectorImpl<int> *PtrPartition) const; + bool needsChecking(const CheckingPtrGroup &M, + const CheckingPtrGroup &N) const; /// \brief Returns the number of run-time checks required according to /// needsChecking. - unsigned getNumberOfChecks(const SmallVectorImpl<int> *PtrPartition) const; + unsigned getNumberOfChecks() const { return Checks.size(); } /// \brief Print the list run-time memory checks necessary. - /// - /// If \p PtrPartition is set, it contains the partition number for - /// pointers (-1 if the pointer belongs to multiple partitions). In this - /// case omit checks between pointers belonging to the same partition. - void print(raw_ostream &OS, unsigned Depth = 0, - const SmallVectorImpl<int> *PtrPartition = nullptr) const; + void print(raw_ostream &OS, unsigned Depth = 0) const; + + /// Print \p Checks. + void printChecks(raw_ostream &OS, const SmallVectorImpl<PointerCheck> &Checks, + unsigned Depth = 0) const; /// This flag indicates if we need to add the runtime check. bool Need; @@ -405,18 +448,41 @@ public: /// Holds a partitioning of pointers into "check groups". SmallVector<CheckingPtrGroup, 2> CheckingGroups; -private: + /// \brief Check if pointers are in the same partition + /// + /// \p PtrToPartition contains the partition number for pointers (-1 if the + /// pointer belongs to multiple partitions). + static bool + arePointersInSamePartition(const SmallVectorImpl<int> &PtrToPartition, + unsigned PtrIdx1, unsigned PtrIdx2); + /// \brief Decide whether we need to issue a run-time check for pointer at /// index \p I and \p J to prove their independence. - /// - /// If \p PtrPartition is set, it contains the partition number for - /// pointers (-1 if the pointer belongs to multiple partitions). In this - /// case omit checks between pointers belonging to the same partition. - bool needsChecking(unsigned I, unsigned J, - const SmallVectorImpl<int> *PtrPartition) const; + bool needsChecking(unsigned I, unsigned J) const; + + /// \brief Return PointerInfo for pointer at index \p PtrIdx. + const PointerInfo &getPointerInfo(unsigned PtrIdx) const { + return Pointers[PtrIdx]; + } + +private: + /// \brief Groups pointers such that a single memcheck is required + /// between two different groups. This will clear the CheckingGroups vector + /// and re-compute it. We will only group dependecies if \p UseDependencies + /// is true, otherwise we will create a separate group for each pointer. + void groupChecks(MemoryDepChecker::DepCandidates &DepCands, + bool UseDependencies); + + /// Generate the checks and return them. + SmallVector<PointerCheck, 4> + generateChecks() const; /// Holds a pointer to the ScalarEvolution analysis. ScalarEvolution *SE; + + /// \brief Set of run-time checks required to establish independence of + /// otherwise may-aliasing pointers in the loop. + SmallVector<PointerCheck, 4> Checks; }; /// \brief Drive the analysis of memory accesses in the loop @@ -433,6 +499,13 @@ private: /// generates run-time checks to prove independence. This is done by /// AccessAnalysis::canCheckPtrAtRT and the checks are maintained by the /// RuntimePointerCheck class. +/// +/// If pointers can wrap or can't be expressed as affine AddRec expressions by +/// ScalarEvolution, we will generate run-time checks by emitting a +/// SCEVUnionPredicate. +/// +/// Checks for both memory dependences and the SCEV predicates contained in the +/// PSE must be emitted in order for the results of this analysis to be valid. class LoopAccessInfo { public: LoopAccessInfo(Loop *L, ScalarEvolution *SE, const DataLayout &DL, @@ -450,9 +523,8 @@ public: /// \brief Number of memchecks required to prove independence of otherwise /// may-alias pointers. - unsigned getNumRuntimePointerChecks( - const SmallVectorImpl<int> *PtrPartition = nullptr) const { - return PtrRtChecking.getNumberOfChecks(PtrPartition); + unsigned getNumRuntimePointerChecks() const { + return PtrRtChecking.getNumberOfChecks(); } /// Return true if the block BB needs to be predicated in order for the loop @@ -472,13 +544,18 @@ public: /// Returns a pair of instructions where the first element is the first /// instruction generated in possibly a sequence of instructions and the /// second value is the final comparator value or NULL if no check is needed. + std::pair<Instruction *, Instruction *> + addRuntimeChecks(Instruction *Loc) const; + + /// \brief Generete the instructions for the checks in \p PointerChecks. /// - /// If \p PtrPartition is set, it contains the partition number for pointers - /// (-1 if the pointer belongs to multiple partitions). In this case omit - /// checks between pointers belonging to the same partition. + /// Returns a pair of instructions where the first element is the first + /// instruction generated in possibly a sequence of instructions and the + /// second value is the final comparator value or NULL if no check is needed. std::pair<Instruction *, Instruction *> - addRuntimeCheck(Instruction *Loc, - const SmallVectorImpl<int> *PtrPartition = nullptr) const; + addRuntimeChecks(Instruction *Loc, + const SmallVectorImpl<RuntimePointerChecking::PointerCheck> + &PointerChecks) const; /// \brief The diagnostics report generated for the analysis. E.g. why we /// couldn't analyze the loop. @@ -510,6 +587,13 @@ public: return StoreToLoopInvariantAddress; } + /// Used to add runtime SCEV checks. Simplifies SCEV expressions and converts + /// them to a more usable form. All SCEV expressions during the analysis + /// should be re-written (and therefore simplified) according to PSE. + /// A user of LoopAccessAnalysis will need to emit the runtime checks + /// associated with this predicate. + PredicatedScalarEvolution PSE; + private: /// \brief Analyze the loop. Substitute symbolic strides using Strides. void analyzeLoop(const ValueToValueMap &Strides); @@ -529,7 +613,6 @@ private: MemoryDepChecker DepChecker; Loop *TheLoop; - ScalarEvolution *SE; const DataLayout &DL; const TargetLibraryInfo *TLI; AliasAnalysis *AA; @@ -556,18 +639,24 @@ private: Value *stripIntegerCast(Value *V); ///\brief Return the SCEV corresponding to a pointer with the symbolic stride -///replaced with constant one. +/// replaced with constant one, assuming \p Preds is true. +/// +/// If necessary this method will version the stride of the pointer according +/// to \p PtrToStride and therefore add a new predicate to \p Preds. /// /// If \p OrigPtr is not null, use it to look up the stride value instead of \p /// Ptr. \p PtrToStride provides the mapping between the pointer value and its /// stride as collected by LoopVectorizationLegality::collectStridedAccess. -const SCEV *replaceSymbolicStrideSCEV(ScalarEvolution *SE, +const SCEV *replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, const ValueToValueMap &PtrToStride, Value *Ptr, Value *OrigPtr = nullptr); /// \brief Check the stride of the pointer and ensure that it does not wrap in -/// the address space. -int isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp, +/// the address space, assuming \p Preds is true. +/// +/// If necessary this method will version the stride of the pointer according +/// to \p PtrToStride and therefore add a new predicate to \p Preds. +int isStridedPtr(PredicatedScalarEvolution &PSE, Value *Ptr, const Loop *Lp, const ValueToValueMap &StridesMap); /// \brief This analysis provides dependence information for the memory accesses @@ -616,6 +705,17 @@ private: DominatorTree *DT; LoopInfo *LI; }; + +inline Instruction *MemoryDepChecker::Dependence::getSource( + const LoopAccessInfo &LAI) const { + return LAI.getDepChecker().getMemoryInstructions()[Source]; +} + +inline Instruction *MemoryDepChecker::Dependence::getDestination( + const LoopAccessInfo &LAI) const { + return LAI.getDepChecker().getMemoryInstructions()[Destination]; +} + } // End llvm namespace #endif diff --git a/include/llvm/Analysis/LoopInfo.h b/include/llvm/Analysis/LoopInfo.h index 3ec83f2c21fd..c219bd85a48a 100644 --- a/include/llvm/Analysis/LoopInfo.h +++ b/include/llvm/Analysis/LoopInfo.h @@ -37,6 +37,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/Pass.h" #include <algorithm> @@ -72,6 +73,10 @@ class LoopBase { SmallPtrSet<const BlockT*, 8> DenseBlockSet; + /// Indicator that this loops has been "unlooped", so there's no loop here + /// anymore. + bool IsUnloop = false; + LoopBase(const LoopBase<BlockT, LoopT> &) = delete; const LoopBase<BlockT, LoopT>& operator=(const LoopBase<BlockT, LoopT> &) = delete; @@ -140,12 +145,22 @@ public: typedef typename std::vector<BlockT*>::const_iterator block_iterator; block_iterator block_begin() const { return Blocks.begin(); } block_iterator block_end() const { return Blocks.end(); } + inline iterator_range<block_iterator> blocks() const { + return make_range(block_begin(), block_end()); + } /// getNumBlocks - Get the number of blocks in this loop in constant time. unsigned getNumBlocks() const { return Blocks.size(); } + /// Mark this loop as having been unlooped - the last backedge was removed and + /// we no longer have a loop. + void markUnlooped() { IsUnloop = true; } + + /// Return true if this no longer represents a loop. + bool isUnloop() const { return IsUnloop; } + /// isLoopExiting - True if terminator in the block can branch to another /// block that is outside of the current loop. /// @@ -398,6 +413,9 @@ public: /// isLCSSAForm - Return true if the Loop is in LCSSA form bool isLCSSAForm(DominatorTree &DT) const; + /// \brief Return true if this Loop and all inner subloops are in LCSSA form. + bool isRecursivelyLCSSAForm(DominatorTree &DT) const; + /// isLoopSimplifyForm - Return true if the Loop is in the form that /// the LoopSimplify form transforms loops to, which is sometimes called /// normal form. @@ -622,7 +640,7 @@ public: } /// Create the loop forest using a stable algorithm. - void Analyze(DominatorTreeBase<BlockT> &DomTree); + void analyze(const DominatorTreeBase<BlockT> &DomTree); // Debugging void print(raw_ostream &OS) const; @@ -642,6 +660,7 @@ class LoopInfo : public LoopInfoBase<BasicBlock, Loop> { LoopInfo(const LoopInfo &) = delete; public: LoopInfo() {} + explicit LoopInfo(const DominatorTreeBase<BasicBlock> &DomTree); LoopInfo(LoopInfo &&Arg) : BaseT(std::move(static_cast<BaseT &>(Arg))) {} LoopInfo &operator=(LoopInfo &&RHS) { @@ -653,8 +672,9 @@ public: /// updateUnloop - Update LoopInfo after removing the last backedge from a /// loop--now the "unloop". This updates the loop forest and parent loops for - /// each block so that Unloop is no longer referenced, but the caller must - /// actually delete the Unloop object. + /// each block so that Unloop is no longer referenced, but does not actually + /// delete the Unloop object. Generally, the loop pass manager should manage + /// deleting the Unloop. void updateUnloop(Loop *Unloop); /// replacementPreservesLCSSAForm - Returns true if replacing From with To @@ -677,6 +697,78 @@ public: // it as a replacement will not break LCSSA form. return ToLoop->contains(getLoopFor(From->getParent())); } + + /// \brief Checks if moving a specific instruction can break LCSSA in any + /// loop. + /// + /// Return true if moving \p Inst to before \p NewLoc will break LCSSA, + /// assuming that the function containing \p Inst and \p NewLoc is currently + /// in LCSSA form. + bool movementPreservesLCSSAForm(Instruction *Inst, Instruction *NewLoc) { + assert(Inst->getFunction() == NewLoc->getFunction() && + "Can't reason about IPO!"); + + auto *OldBB = Inst->getParent(); + auto *NewBB = NewLoc->getParent(); + + // Movement within the same loop does not break LCSSA (the equality check is + // to avoid doing a hashtable lookup in case of intra-block movement). + if (OldBB == NewBB) + return true; + + auto *OldLoop = getLoopFor(OldBB); + auto *NewLoop = getLoopFor(NewBB); + + if (OldLoop == NewLoop) + return true; + + // Check if Outer contains Inner; with the null loop counting as the + // "outermost" loop. + auto Contains = [](const Loop *Outer, const Loop *Inner) { + return !Outer || Outer->contains(Inner); + }; + + // To check that the movement of Inst to before NewLoc does not break LCSSA, + // we need to check two sets of uses for possible LCSSA violations at + // NewLoc: the users of NewInst, and the operands of NewInst. + + // If we know we're hoisting Inst out of an inner loop to an outer loop, + // then the uses *of* Inst don't need to be checked. + + if (!Contains(NewLoop, OldLoop)) { + for (Use &U : Inst->uses()) { + auto *UI = cast<Instruction>(U.getUser()); + auto *UBB = isa<PHINode>(UI) ? cast<PHINode>(UI)->getIncomingBlock(U) + : UI->getParent(); + if (UBB != NewBB && getLoopFor(UBB) != NewLoop) + return false; + } + } + + // If we know we're sinking Inst from an outer loop into an inner loop, then + // the *operands* of Inst don't need to be checked. + + if (!Contains(OldLoop, NewLoop)) { + // See below on why we can't handle phi nodes here. + if (isa<PHINode>(Inst)) + return false; + + for (Use &U : Inst->operands()) { + auto *DefI = dyn_cast<Instruction>(U.get()); + if (!DefI) + return false; + + // This would need adjustment if we allow Inst to be a phi node -- the + // new use block won't simply be NewBB. + + auto *DefBlock = DefI->getParent(); + if (DefBlock != NewBB && getLoopFor(DefBlock) != NewLoop) + return false; + } + } + + return true; + } }; // Allow clients to walk the list of nested loops... @@ -759,6 +851,19 @@ public: void getAnalysisUsage(AnalysisUsage &AU) const override; }; +/// \brief Pass for printing a loop's contents as LLVM's text IR assembly. +class PrintLoopPass { + raw_ostream &OS; + std::string Banner; + +public: + PrintLoopPass(); + PrintLoopPass(raw_ostream &OS, const std::string &Banner = ""); + + PreservedAnalyses run(Loop &L); + static StringRef name() { return "PrintLoopPass"; } +}; + } // End llvm namespace #endif diff --git a/include/llvm/Analysis/LoopInfoImpl.h b/include/llvm/Analysis/LoopInfoImpl.h index f5cc856f6247..824fc7e8f155 100644 --- a/include/llvm/Analysis/LoopInfoImpl.h +++ b/include/llvm/Analysis/LoopInfoImpl.h @@ -269,7 +269,7 @@ void LoopBase<BlockT, LoopT>::verifyLoop() const { // A non-header loop shouldn't be reachable from outside the loop, // though it is permitted if the predecessor is not itself actually // reachable. - BlockT *EntryBB = BB->getParent()->begin(); + BlockT *EntryBB = &BB->getParent()->front(); for (BlockT *CB : depth_first(EntryBB)) for (unsigned i = 0, e = OutsideLoopPreds.size(); i != e; ++i) assert(CB != OutsideLoopPreds[i] && @@ -345,7 +345,7 @@ void LoopBase<BlockT, LoopT>::print(raw_ostream &OS, unsigned Depth) const { template<class BlockT, class LoopT> static void discoverAndMapSubloop(LoopT *L, ArrayRef<BlockT*> Backedges, LoopInfoBase<BlockT, LoopT> *LI, - DominatorTreeBase<BlockT> &DomTree) { + const DominatorTreeBase<BlockT> &DomTree) { typedef GraphTraits<Inverse<BlockT*> > InvBlockTraits; unsigned NumBlocks = 0; @@ -468,10 +468,10 @@ void PopulateLoopsDFS<BlockT, LoopT>::insertIntoLoop(BlockT *Block) { /// insertions per block. template<class BlockT, class LoopT> void LoopInfoBase<BlockT, LoopT>:: -Analyze(DominatorTreeBase<BlockT> &DomTree) { +analyze(const DominatorTreeBase<BlockT> &DomTree) { // Postorder traversal of the dominator tree. - DomTreeNodeBase<BlockT>* DomRoot = DomTree.getRootNode(); + const DomTreeNodeBase<BlockT> *DomRoot = DomTree.getRootNode(); for (auto DomNode : post_order(DomRoot)) { BlockT *Header = DomNode->getBlock(); diff --git a/include/llvm/Analysis/LoopPass.h b/include/llvm/Analysis/LoopPass.h index 8650000fcfb6..2cf734e53bb4 100644 --- a/include/llvm/Analysis/LoopPass.h +++ b/include/llvm/Analysis/LoopPass.h @@ -127,20 +127,9 @@ public: } public: - // Delete loop from the loop queue and loop nest (LoopInfo). - void deleteLoopFromQueue(Loop *L); - - // Insert loop into the loop queue and add it as a child of the - // given parent. - void insertLoop(Loop *L, Loop *ParentLoop); - - // Insert a loop into the loop queue. - void insertLoopIntoQueue(Loop *L); - - // Reoptimize this loop. LPPassManager will re-insert this loop into the - // queue. This allows LoopPass to change loop nest for the loop. This - // utility may send LPPassManager into infinite loops so use caution. - void redoLoop(Loop *L); + // Add a new loop into the loop queue as a child of the given parent, or at + // the top level if \c ParentLoop is null. + Loop &addLoop(Loop *ParentLoop); //===--------------------------------------------------------------------===// /// SimpleAnalysis - Provides simple interface to update analysis info @@ -163,8 +152,6 @@ public: private: std::deque<Loop *> LQ; - bool skipThisLoop; - bool redoThisLoop; LoopInfo *LI; Loop *CurrentLoop; }; diff --git a/include/llvm/Analysis/MemoryBuiltins.h b/include/llvm/Analysis/MemoryBuiltins.h index 805a43dfb070..87fb3efaf50e 100644 --- a/include/llvm/Analysis/MemoryBuiltins.h +++ b/include/llvm/Analysis/MemoryBuiltins.h @@ -60,11 +60,6 @@ bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast = false); /// \brief Tests if a value is a call or invoke to a library function that -/// reallocates memory (such as realloc). -bool isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); - -/// \brief Tests if a value is a call or invoke to a library function that /// allocates memory and never returns null (such as operator new). bool isOperatorNewLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast = false); diff --git a/include/llvm/Analysis/MemoryDependenceAnalysis.h b/include/llvm/Analysis/MemoryDependenceAnalysis.h index 511898071c22..daa1ba91c071 100644 --- a/include/llvm/Analysis/MemoryDependenceAnalysis.h +++ b/include/llvm/Analysis/MemoryDependenceAnalysis.h @@ -28,7 +28,6 @@ namespace llvm { class FunctionPass; class Instruction; class CallSite; - class AliasAnalysis; class AssumptionCache; class MemoryDependenceAnalysis; class PredIteratorCache; @@ -97,6 +96,7 @@ namespace llvm { typedef PointerIntPair<Instruction*, 2, DepType> PairTy; PairTy Value; explicit MemDepResult(PairTy V) : Value(V) {} + public: MemDepResult() : Value(nullptr, Invalid) {} @@ -164,6 +164,7 @@ namespace llvm { bool operator!=(const MemDepResult &M) const { return Value != M.Value; } bool operator<(const MemDepResult &M) const { return Value < M.Value; } bool operator>(const MemDepResult &M) const { return Value > M.Value; } + private: friend class MemoryDependenceAnalysis; /// Dirty - Entries with this marker occur in a LocalDeps map or @@ -190,6 +191,7 @@ namespace llvm { class NonLocalDepEntry { BasicBlock *BB; MemDepResult Result; + public: NonLocalDepEntry(BasicBlock *bb, MemDepResult result) : BB(bb), Result(result) {} @@ -215,6 +217,7 @@ namespace llvm { class NonLocalDepResult { NonLocalDepEntry Entry; Value *Address; + public: NonLocalDepResult(BasicBlock *bb, MemDepResult result, Value *address) : Entry(bb, result), Address(address) {} @@ -261,6 +264,7 @@ namespace llvm { public: typedef std::vector<NonLocalDepEntry> NonLocalDepInfo; + private: /// ValueIsLoadPair - This is a pair<Value*, bool> where the bool is true if /// the dependence is a read only dependence, false if read/write. @@ -302,7 +306,6 @@ namespace llvm { SmallPtrSet<ValueIsLoadPair, 4> > ReverseNonLocalPtrDepTy; ReverseNonLocalPtrDepTy ReverseNonLocalPtrDeps; - /// PerInstNLInfo - This is the instruction we keep for each cached access /// that we have for an instruction. The pointer is an owning pointer and /// the bool indicates whether we have any dirty bits in the set. @@ -326,6 +329,7 @@ namespace llvm { AliasAnalysis *AA; DominatorTree *DT; AssumptionCache *AC; + const TargetLibraryInfo *TLI; PredIteratorCache PredCache; public: @@ -363,14 +367,13 @@ namespace llvm { /// that. const NonLocalDepInfo &getNonLocalCallDependency(CallSite QueryCS); - /// getNonLocalPointerDependency - Perform a full dependency query for an /// access to the QueryInst's specified memory location, returning the set /// of instructions that either define or clobber the value. /// /// Warning: For a volatile query instruction, the dependencies will be /// accurate, and thus usable for reordering, but it is never legal to - /// remove the query instruction. + /// remove the query instruction. /// /// This method assumes the pointer has a "NonLocal" dependency within /// QueryInst's parent basic block. @@ -394,12 +397,12 @@ namespace llvm { /// critical edges. void invalidateCachedPredecessors(); - /// getPointerDependencyFrom - Return the instruction on which a memory - /// location depends. If isLoad is true, this routine ignores may-aliases - /// with read-only operations. If isLoad is false, this routine ignores - /// may-aliases with reads from read-only locations. If possible, pass - /// the query instruction as well; this function may take advantage of - /// the metadata annotated to the query instruction to refine the result. + /// \brief Return the instruction on which a memory location depends. + /// If isLoad is true, this routine ignores may-aliases with read-only + /// operations. If isLoad is false, this routine ignores may-aliases + /// with reads from read-only locations. If possible, pass the query + /// instruction as well; this function may take advantage of the metadata + /// annotated to the query instruction to refine the result. /// /// Note that this is an uncached query, and thus may be inefficient. /// @@ -409,6 +412,21 @@ namespace llvm { BasicBlock *BB, Instruction *QueryInst = nullptr); + MemDepResult getSimplePointerDependencyFrom(const MemoryLocation &MemLoc, + bool isLoad, + BasicBlock::iterator ScanIt, + BasicBlock *BB, + Instruction *QueryInst); + + /// This analysis looks for other loads and stores with invariant.group + /// metadata and the same pointer operand. Returns Unknown if it does not + /// find anything, and Def if it can be assumed that 2 instructions load or + /// store the same value. + /// FIXME: This analysis works only on single block because of restrictions + /// at the call site. + MemDepResult getInvariantGroupPointerDependency(LoadInst *LI, + BasicBlock *BB); + /// getLoadLoadClobberFullWidthSize - This is a little bit of analysis that /// looks at a memory location for a load (specified by MemLocBase, Offs, /// and Size) and compares it against a load. If the specified load could @@ -442,7 +460,6 @@ namespace llvm { /// verifyRemoved - Verify that the specified instruction does not occur /// in our internal data structures. void verifyRemoved(Instruction *Inst) const; - }; } // End llvm namespace diff --git a/include/llvm/Analysis/ObjCARCAliasAnalysis.h b/include/llvm/Analysis/ObjCARCAliasAnalysis.h new file mode 100644 index 000000000000..ac01154bac6c --- /dev/null +++ b/include/llvm/Analysis/ObjCARCAliasAnalysis.h @@ -0,0 +1,102 @@ +//===- ObjCARCAliasAnalysis.h - ObjC ARC Alias Analysis ---------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file declares a simple ARC-aware AliasAnalysis using special knowledge +/// of Objective C to enhance other optimization passes which rely on the Alias +/// Analysis infrastructure. +/// +/// WARNING: This file knows about certain library functions. It recognizes them +/// by name, and hardwires knowledge of their semantics. +/// +/// WARNING: This file knows about how certain Objective-C library functions are +/// used. Naive LLVM IR transformations which would otherwise be +/// behavior-preserving may break these assumptions. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_OBJCARCALIASANALYSIS_H +#define LLVM_ANALYSIS_OBJCARCALIASANALYSIS_H + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Pass.h" + +namespace llvm { +namespace objcarc { + +/// \brief This is a simple alias analysis implementation that uses knowledge +/// of ARC constructs to answer queries. +/// +/// TODO: This class could be generalized to know about other ObjC-specific +/// tricks. Such as knowing that ivars in the non-fragile ABI are non-aliasing +/// even though their offsets are dynamic. +class ObjCARCAAResult : public AAResultBase<ObjCARCAAResult> { + friend AAResultBase<ObjCARCAAResult>; + + const DataLayout &DL; + +public: + explicit ObjCARCAAResult(const DataLayout &DL, const TargetLibraryInfo &TLI) + : AAResultBase(TLI), DL(DL) {} + ObjCARCAAResult(ObjCARCAAResult &&Arg) + : AAResultBase(std::move(Arg)), DL(Arg.DL) {} + + /// Handle invalidation events from the new pass manager. + /// + /// By definition, this result is stateless and so remains valid. + bool invalidate(Function &, const PreservedAnalyses &) { return false; } + + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); + bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal); + + using AAResultBase::getModRefBehavior; + FunctionModRefBehavior getModRefBehavior(const Function *F); + + using AAResultBase::getModRefInfo; + ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); +}; + +/// Analysis pass providing a never-invalidated alias analysis result. +class ObjCARCAA { +public: + typedef ObjCARCAAResult Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + ObjCARCAAResult run(Function &F, AnalysisManager<Function> *AM); + + /// \brief Provide access to a name for this pass for debugging purposes. + static StringRef name() { return "ObjCARCAA"; } + +private: + static char PassID; +}; + +/// Legacy wrapper pass to provide the ObjCARCAAResult object. +class ObjCARCAAWrapperPass : public ImmutablePass { + std::unique_ptr<ObjCARCAAResult> Result; + +public: + static char ID; + + ObjCARCAAWrapperPass(); + + ObjCARCAAResult &getResult() { return *Result; } + const ObjCARCAAResult &getResult() const { return *Result; } + + bool doInitialization(Module &M) override; + bool doFinalization(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +} // namespace objcarc +} // namespace llvm + +#endif diff --git a/include/llvm/Analysis/ObjCARCAnalysisUtils.h b/include/llvm/Analysis/ObjCARCAnalysisUtils.h new file mode 100644 index 000000000000..29d99c9d316d --- /dev/null +++ b/include/llvm/Analysis/ObjCARCAnalysisUtils.h @@ -0,0 +1,287 @@ +//===- ObjCARCAnalysisUtils.h - ObjC ARC Analysis Utilities -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file defines common analysis utilities used by the ObjC ARC Optimizer. +/// ARC stands for Automatic Reference Counting and is a system for managing +/// reference counts for objects in Objective C. +/// +/// WARNING: This file knows about certain library functions. It recognizes them +/// by name, and hardwires knowledge of their semantics. +/// +/// WARNING: This file knows about how certain Objective-C library functions are +/// used. Naive LLVM IR transformations which would otherwise be +/// behavior-preserving may break these assumptions. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_ANALYSIS_OBJCARCANALYSISUTILS_H +#define LLVM_LIB_ANALYSIS_OBJCARCANALYSISUTILS_H + +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Optional.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/ObjCARCInstKind.h" +#include "llvm/Analysis/Passes.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +namespace llvm { +class raw_ostream; +} + +namespace llvm { +namespace objcarc { + +/// \brief A handy option to enable/disable all ARC Optimizations. +extern bool EnableARCOpts; + +/// \brief Test if the given module looks interesting to run ARC optimization +/// on. +inline bool ModuleHasARC(const Module &M) { + return + M.getNamedValue("objc_retain") || + M.getNamedValue("objc_release") || + M.getNamedValue("objc_autorelease") || + M.getNamedValue("objc_retainAutoreleasedReturnValue") || + M.getNamedValue("objc_retainBlock") || + M.getNamedValue("objc_autoreleaseReturnValue") || + M.getNamedValue("objc_autoreleasePoolPush") || + M.getNamedValue("objc_loadWeakRetained") || + M.getNamedValue("objc_loadWeak") || + M.getNamedValue("objc_destroyWeak") || + M.getNamedValue("objc_storeWeak") || + M.getNamedValue("objc_initWeak") || + M.getNamedValue("objc_moveWeak") || + M.getNamedValue("objc_copyWeak") || + M.getNamedValue("objc_retainedObject") || + M.getNamedValue("objc_unretainedObject") || + M.getNamedValue("objc_unretainedPointer") || + M.getNamedValue("clang.arc.use"); +} + +/// \brief This is a wrapper around getUnderlyingObject which also knows how to +/// look through objc_retain and objc_autorelease calls, which we know to return +/// their argument verbatim. +inline const Value *GetUnderlyingObjCPtr(const Value *V, + const DataLayout &DL) { + for (;;) { + V = GetUnderlyingObject(V, DL); + if (!IsForwarding(GetBasicARCInstKind(V))) + break; + V = cast<CallInst>(V)->getArgOperand(0); + } + + return V; +} + +/// The RCIdentity root of a value \p V is a dominating value U for which +/// retaining or releasing U is equivalent to retaining or releasing V. In other +/// words, ARC operations on \p V are equivalent to ARC operations on \p U. +/// +/// We use this in the ARC optimizer to make it easier to match up ARC +/// operations by always mapping ARC operations to RCIdentityRoots instead of +/// pointers themselves. +/// +/// The two ways that we see RCIdentical values in ObjC are via: +/// +/// 1. PointerCasts +/// 2. Forwarding Calls that return their argument verbatim. +/// +/// Thus this function strips off pointer casts and forwarding calls. *NOTE* +/// This implies that two RCIdentical values must alias. +inline const Value *GetRCIdentityRoot(const Value *V) { + for (;;) { + V = V->stripPointerCasts(); + if (!IsForwarding(GetBasicARCInstKind(V))) + break; + V = cast<CallInst>(V)->getArgOperand(0); + } + return V; +} + +/// Helper which calls const Value *GetRCIdentityRoot(const Value *V) and just +/// casts away the const of the result. For documentation about what an +/// RCIdentityRoot (and by extension GetRCIdentityRoot is) look at that +/// function. +inline Value *GetRCIdentityRoot(Value *V) { + return const_cast<Value *>(GetRCIdentityRoot((const Value *)V)); +} + +/// \brief Assuming the given instruction is one of the special calls such as +/// objc_retain or objc_release, return the RCIdentity root of the argument of +/// the call. +inline Value *GetArgRCIdentityRoot(Value *Inst) { + return GetRCIdentityRoot(cast<CallInst>(Inst)->getArgOperand(0)); +} + +inline bool IsNullOrUndef(const Value *V) { + return isa<ConstantPointerNull>(V) || isa<UndefValue>(V); +} + +inline bool IsNoopInstruction(const Instruction *I) { + return isa<BitCastInst>(I) || + (isa<GetElementPtrInst>(I) && + cast<GetElementPtrInst>(I)->hasAllZeroIndices()); +} + +/// \brief Test whether the given value is possible a retainable object pointer. +inline bool IsPotentialRetainableObjPtr(const Value *Op) { + // Pointers to static or stack storage are not valid retainable object + // pointers. + if (isa<Constant>(Op) || isa<AllocaInst>(Op)) + return false; + // Special arguments can not be a valid retainable object pointer. + if (const Argument *Arg = dyn_cast<Argument>(Op)) + if (Arg->hasByValAttr() || + Arg->hasInAllocaAttr() || + Arg->hasNestAttr() || + Arg->hasStructRetAttr()) + return false; + // Only consider values with pointer types. + // + // It seemes intuitive to exclude function pointer types as well, since + // functions are never retainable object pointers, however clang occasionally + // bitcasts retainable object pointers to function-pointer type temporarily. + PointerType *Ty = dyn_cast<PointerType>(Op->getType()); + if (!Ty) + return false; + // Conservatively assume anything else is a potential retainable object + // pointer. + return true; +} + +inline bool IsPotentialRetainableObjPtr(const Value *Op, + AliasAnalysis &AA) { + // First make the rudimentary check. + if (!IsPotentialRetainableObjPtr(Op)) + return false; + + // Objects in constant memory are not reference-counted. + if (AA.pointsToConstantMemory(Op)) + return false; + + // Pointers in constant memory are not pointing to reference-counted objects. + if (const LoadInst *LI = dyn_cast<LoadInst>(Op)) + if (AA.pointsToConstantMemory(LI->getPointerOperand())) + return false; + + // Otherwise assume the worst. + return true; +} + +/// \brief Helper for GetARCInstKind. Determines what kind of construct CS +/// is. +inline ARCInstKind GetCallSiteClass(ImmutableCallSite CS) { + for (ImmutableCallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end(); + I != E; ++I) + if (IsPotentialRetainableObjPtr(*I)) + return CS.onlyReadsMemory() ? ARCInstKind::User : ARCInstKind::CallOrUser; + + return CS.onlyReadsMemory() ? ARCInstKind::None : ARCInstKind::Call; +} + +/// \brief Return true if this value refers to a distinct and identifiable +/// object. +/// +/// This is similar to AliasAnalysis's isIdentifiedObject, except that it uses +/// special knowledge of ObjC conventions. +inline bool IsObjCIdentifiedObject(const Value *V) { + // Assume that call results and arguments have their own "provenance". + // Constants (including GlobalVariables) and Allocas are never + // reference-counted. + if (isa<CallInst>(V) || isa<InvokeInst>(V) || + isa<Argument>(V) || isa<Constant>(V) || + isa<AllocaInst>(V)) + return true; + + if (const LoadInst *LI = dyn_cast<LoadInst>(V)) { + const Value *Pointer = + GetRCIdentityRoot(LI->getPointerOperand()); + if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Pointer)) { + // A constant pointer can't be pointing to an object on the heap. It may + // be reference-counted, but it won't be deleted. + if (GV->isConstant()) + return true; + StringRef Name = GV->getName(); + // These special variables are known to hold values which are not + // reference-counted pointers. + if (Name.startswith("\01l_objc_msgSend_fixup_")) + return true; + + StringRef Section = GV->getSection(); + if (Section.find("__message_refs") != StringRef::npos || + Section.find("__objc_classrefs") != StringRef::npos || + Section.find("__objc_superrefs") != StringRef::npos || + Section.find("__objc_methname") != StringRef::npos || + Section.find("__cstring") != StringRef::npos) + return true; + } + } + + return false; +} + +enum class ARCMDKindID { + ImpreciseRelease, + CopyOnEscape, + NoObjCARCExceptions, +}; + +/// A cache of MDKinds used by various ARC optimizations. +class ARCMDKindCache { + Module *M; + + /// The Metadata Kind for clang.imprecise_release metadata. + llvm::Optional<unsigned> ImpreciseReleaseMDKind; + + /// The Metadata Kind for clang.arc.copy_on_escape metadata. + llvm::Optional<unsigned> CopyOnEscapeMDKind; + + /// The Metadata Kind for clang.arc.no_objc_arc_exceptions metadata. + llvm::Optional<unsigned> NoObjCARCExceptionsMDKind; + +public: + void init(Module *Mod) { + M = Mod; + ImpreciseReleaseMDKind = NoneType::None; + CopyOnEscapeMDKind = NoneType::None; + NoObjCARCExceptionsMDKind = NoneType::None; + } + + unsigned get(ARCMDKindID ID) { + switch (ID) { + case ARCMDKindID::ImpreciseRelease: + if (!ImpreciseReleaseMDKind) + ImpreciseReleaseMDKind = + M->getContext().getMDKindID("clang.imprecise_release"); + return *ImpreciseReleaseMDKind; + case ARCMDKindID::CopyOnEscape: + if (!CopyOnEscapeMDKind) + CopyOnEscapeMDKind = + M->getContext().getMDKindID("clang.arc.copy_on_escape"); + return *CopyOnEscapeMDKind; + case ARCMDKindID::NoObjCARCExceptions: + if (!NoObjCARCExceptionsMDKind) + NoObjCARCExceptionsMDKind = + M->getContext().getMDKindID("clang.arc.no_objc_arc_exceptions"); + return *NoObjCARCExceptionsMDKind; + } + llvm_unreachable("Covered switch isn't covered?!"); + } +}; + +} // end namespace objcarc +} // end namespace llvm + +#endif diff --git a/include/llvm/Analysis/ObjCARCInstKind.h b/include/llvm/Analysis/ObjCARCInstKind.h new file mode 100644 index 000000000000..13efb4b160be --- /dev/null +++ b/include/llvm/Analysis/ObjCARCInstKind.h @@ -0,0 +1,123 @@ +//===- ObjCARCInstKind.h - ARC instruction equivalence classes --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_OBJCARCINSTKIND_H +#define LLVM_ANALYSIS_OBJCARCINSTKIND_H + +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Function.h" + +namespace llvm { +namespace objcarc { + +/// \enum ARCInstKind +/// +/// \brief Equivalence classes of instructions in the ARC Model. +/// +/// Since we do not have "instructions" to represent ARC concepts in LLVM IR, +/// we instead operate on equivalence classes of instructions. +/// +/// TODO: This should be split into two enums: a runtime entry point enum +/// (possibly united with the ARCRuntimeEntrypoint class) and an enum that deals +/// with effects of instructions in the ARC model (which would handle the notion +/// of a User or CallOrUser). +enum class ARCInstKind { + Retain, ///< objc_retain + RetainRV, ///< objc_retainAutoreleasedReturnValue + RetainBlock, ///< objc_retainBlock + Release, ///< objc_release + Autorelease, ///< objc_autorelease + AutoreleaseRV, ///< objc_autoreleaseReturnValue + AutoreleasepoolPush, ///< objc_autoreleasePoolPush + AutoreleasepoolPop, ///< objc_autoreleasePoolPop + NoopCast, ///< objc_retainedObject, etc. + FusedRetainAutorelease, ///< objc_retainAutorelease + FusedRetainAutoreleaseRV, ///< objc_retainAutoreleaseReturnValue + LoadWeakRetained, ///< objc_loadWeakRetained (primitive) + StoreWeak, ///< objc_storeWeak (primitive) + InitWeak, ///< objc_initWeak (derived) + LoadWeak, ///< objc_loadWeak (derived) + MoveWeak, ///< objc_moveWeak (derived) + CopyWeak, ///< objc_copyWeak (derived) + DestroyWeak, ///< objc_destroyWeak (derived) + StoreStrong, ///< objc_storeStrong (derived) + IntrinsicUser, ///< clang.arc.use + CallOrUser, ///< could call objc_release and/or "use" pointers + Call, ///< could call objc_release + User, ///< could "use" a pointer + None ///< anything that is inert from an ARC perspective. +}; + +raw_ostream &operator<<(raw_ostream &OS, const ARCInstKind Class); + +/// \brief Test if the given class is a kind of user. +bool IsUser(ARCInstKind Class); + +/// \brief Test if the given class is objc_retain or equivalent. +bool IsRetain(ARCInstKind Class); + +/// \brief Test if the given class is objc_autorelease or equivalent. +bool IsAutorelease(ARCInstKind Class); + +/// \brief Test if the given class represents instructions which return their +/// argument verbatim. +bool IsForwarding(ARCInstKind Class); + +/// \brief Test if the given class represents instructions which do nothing if +/// passed a null pointer. +bool IsNoopOnNull(ARCInstKind Class); + +/// \brief Test if the given class represents instructions which are always safe +/// to mark with the "tail" keyword. +bool IsAlwaysTail(ARCInstKind Class); + +/// \brief Test if the given class represents instructions which are never safe +/// to mark with the "tail" keyword. +bool IsNeverTail(ARCInstKind Class); + +/// \brief Test if the given class represents instructions which are always safe +/// to mark with the nounwind attribute. +bool IsNoThrow(ARCInstKind Class); + +/// Test whether the given instruction can autorelease any pointer or cause an +/// autoreleasepool pop. +bool CanInterruptRV(ARCInstKind Class); + +/// \brief Determine if F is one of the special known Functions. If it isn't, +/// return ARCInstKind::CallOrUser. +ARCInstKind GetFunctionClass(const Function *F); + +/// \brief Determine which objc runtime call instruction class V belongs to. +/// +/// This is similar to GetARCInstKind except that it only detects objc +/// runtime calls. This allows it to be faster. +/// +inline ARCInstKind GetBasicARCInstKind(const Value *V) { + if (const CallInst *CI = dyn_cast<CallInst>(V)) { + if (const Function *F = CI->getCalledFunction()) + return GetFunctionClass(F); + // Otherwise, be conservative. + return ARCInstKind::CallOrUser; + } + + // Otherwise, be conservative. + return isa<InvokeInst>(V) ? ARCInstKind::CallOrUser : ARCInstKind::User; +} + +/// Map V to its ARCInstKind equivalence class. +ARCInstKind GetARCInstKind(const Value *V); + +/// Returns false if conservatively we can prove that any instruction mapped to +/// this kind can not decrement ref counts. Returns true otherwise. +bool CanDecrementRefCount(ARCInstKind Kind); + +} // end namespace objcarc +} // end namespace llvm + +#endif diff --git a/include/llvm/Analysis/OrderedBasicBlock.h b/include/llvm/Analysis/OrderedBasicBlock.h new file mode 100644 index 000000000000..5aa813eb4832 --- /dev/null +++ b/include/llvm/Analysis/OrderedBasicBlock.h @@ -0,0 +1,66 @@ +//===- llvm/Analysis/OrderedBasicBlock.h --------------------- -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the OrderedBasicBlock class. OrderedBasicBlock maintains +// an interface where clients can query if one instruction comes before another +// in a BasicBlock. Since BasicBlock currently lacks a reliable way to query +// relative position between instructions one can use OrderedBasicBlock to do +// such queries. OrderedBasicBlock is lazily built on a source BasicBlock and +// maintains an internal Instruction -> Position map. A OrderedBasicBlock +// instance should be discarded whenever the source BasicBlock changes. +// +// It's currently used by the CaptureTracker in order to find relative +// positions of a pair of instructions inside a BasicBlock. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_ORDEREDBASICBLOCK_H +#define LLVM_ANALYSIS_ORDEREDBASICBLOCK_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/IR/BasicBlock.h" + +namespace llvm { + +class Instruction; +class BasicBlock; + +class OrderedBasicBlock { +private: + /// \brief Map a instruction to its position in a BasicBlock. + SmallDenseMap<const Instruction *, unsigned, 32> NumberedInsts; + + /// \brief Keep track of last instruction inserted into \p NumberedInsts. + /// It speeds up queries for uncached instructions by providing a start point + /// for new queries in OrderedBasicBlock::comesBefore. + BasicBlock::const_iterator LastInstFound; + + /// \brief The position/number to tag the next instruction to be found. + unsigned NextInstPos; + + /// \brief The source BasicBlock to map. + const BasicBlock *BB; + + /// \brief Given no cached results, find if \p A comes before \p B in \p BB. + /// Cache and number out instruction while walking \p BB. + bool comesBefore(const Instruction *A, const Instruction *B); + +public: + OrderedBasicBlock(const BasicBlock *BasicB); + + /// \brief Find out whether \p A dominates \p B, meaning whether \p A + /// comes before \p B in \p BB. This is a simplification that considers + /// cached instruction positions and ignores other basic blocks, being + /// only relevant to compare relative instructions positions inside \p BB. + bool dominates(const Instruction *A, const Instruction *B); +}; + +} // End llvm namespace + +#endif diff --git a/include/llvm/Analysis/PHITransAddr.h b/include/llvm/Analysis/PHITransAddr.h index cbdbb88f7407..f0f34f3a51f5 100644 --- a/include/llvm/Analysis/PHITransAddr.h +++ b/include/llvm/Analysis/PHITransAddr.h @@ -48,6 +48,7 @@ class PHITransAddr { /// InstInputs - The inputs for our symbolic address. SmallVector<Instruction*, 4> InstInputs; + public: PHITransAddr(Value *addr, const DataLayout &DL, AssumptionCache *AC) : Addr(addr), DL(DL), TLI(nullptr), AC(AC) { @@ -55,9 +56,9 @@ public: if (Instruction *I = dyn_cast<Instruction>(Addr)) InstInputs.push_back(I); } - + Value *getAddr() const { return Addr; } - + /// NeedsPHITranslationFromBlock - Return true if moving from the specified /// BasicBlock to its predecessors requires PHI translation. bool NeedsPHITranslationFromBlock(BasicBlock *BB) const { @@ -68,12 +69,12 @@ public: return true; return false; } - + /// IsPotentiallyPHITranslatable - If this needs PHI translation, return true /// if we have some hope of doing it. This should be used as a filter to /// avoid calling PHITranslateValue in hopeless situations. bool IsPotentiallyPHITranslatable() const; - + /// PHITranslateValue - PHI translate the current address up the CFG from /// CurBB to Pred, updating our state to reflect any needed changes. If /// 'MustDominate' is true, the translated value must dominate @@ -90,18 +91,19 @@ public: /// Value *PHITranslateWithInsertion(BasicBlock *CurBB, BasicBlock *PredBB, const DominatorTree &DT, - SmallVectorImpl<Instruction*> &NewInsts); - + SmallVectorImpl<Instruction *> &NewInsts); + void dump() const; - + /// Verify - Check internal consistency of this data structure. If the /// structure is valid, it returns true. If invalid, it prints errors and /// returns false. bool Verify() const; + private: Value *PHITranslateSubExpr(Value *V, BasicBlock *CurBB, BasicBlock *PredBB, const DominatorTree *DT); - + /// InsertPHITranslatedSubExpr - Insert a computation of the PHI translated /// version of 'V' for the edge PredBB->CurBB into the end of the PredBB /// block. All newly created instructions are added to the NewInsts list. @@ -109,8 +111,8 @@ private: /// Value *InsertPHITranslatedSubExpr(Value *InVal, BasicBlock *CurBB, BasicBlock *PredBB, const DominatorTree &DT, - SmallVectorImpl<Instruction*> &NewInsts); - + SmallVectorImpl<Instruction *> &NewInsts); + /// AddAsInput - If the specified value is an instruction, add it as an input. Value *AddAsInput(Value *V) { // If V is an instruction, it is now an input. @@ -118,7 +120,6 @@ private: InstInputs.push_back(VI); return V; } - }; } // end namespace llvm diff --git a/include/llvm/Analysis/Passes.h b/include/llvm/Analysis/Passes.h index d112ab1823b4..da17457d3446 100644 --- a/include/llvm/Analysis/Passes.h +++ b/include/llvm/Analysis/Passes.h @@ -22,27 +22,6 @@ namespace llvm { class ModulePass; class Pass; class PassInfo; - class LibCallInfo; - - //===--------------------------------------------------------------------===// - // - // createGlobalsModRefPass - This pass provides alias and mod/ref info for - // global values that do not have their addresses taken. - // - Pass *createGlobalsModRefPass(); - - //===--------------------------------------------------------------------===// - // - // createAliasDebugger - This pass helps debug clients of AA - // - Pass *createAliasDebugger(); - - //===--------------------------------------------------------------------===// - // - // createAliasAnalysisCounterPass - This pass counts alias queries and how the - // alias analysis implementation responds. - // - ModulePass *createAliasAnalysisCounterPass(); //===--------------------------------------------------------------------===// // @@ -53,59 +32,10 @@ namespace llvm { //===--------------------------------------------------------------------===// // - // createNoAAPass - This pass implements a "I don't know" alias analysis. - // - ImmutablePass *createNoAAPass(); - - //===--------------------------------------------------------------------===// - // - // createBasicAliasAnalysisPass - This pass implements the stateless alias - // analysis. - // - ImmutablePass *createBasicAliasAnalysisPass(); - - //===--------------------------------------------------------------------===// - // - // createCFLAliasAnalysisPass - This pass implements a set-based approach to - // alias analysis. - // - ImmutablePass *createCFLAliasAnalysisPass(); - - //===--------------------------------------------------------------------===// - // - /// createLibCallAliasAnalysisPass - Create an alias analysis pass that knows - /// about the semantics of a set of libcalls specified by LCI. The newly - /// constructed pass takes ownership of the pointer that is provided. - /// - FunctionPass *createLibCallAliasAnalysisPass(LibCallInfo *LCI); - - //===--------------------------------------------------------------------===// - // - // createScalarEvolutionAliasAnalysisPass - This pass implements a simple - // alias analysis using ScalarEvolution queries. - // - FunctionPass *createScalarEvolutionAliasAnalysisPass(); - - //===--------------------------------------------------------------------===// - // - // createTypeBasedAliasAnalysisPass - This pass implements metadata-based - // type-based alias analysis. - // - ImmutablePass *createTypeBasedAliasAnalysisPass(); - - //===--------------------------------------------------------------------===// - // - // createScopedNoAliasAAPass - This pass implements metadata-based - // scoped noalias analysis. - // - ImmutablePass *createScopedNoAliasAAPass(); - - //===--------------------------------------------------------------------===// - // - // createObjCARCAliasAnalysisPass - This pass implements ObjC-ARC-based + // createObjCARCAAWrapperPass - This pass implements ObjC-ARC-based // alias analysis. // - ImmutablePass *createObjCARCAliasAnalysisPass(); + ImmutablePass *createObjCARCAAWrapperPass(); FunctionPass *createPAEvalPass(); diff --git a/include/llvm/Analysis/RegionInfo.h b/include/llvm/Analysis/RegionInfo.h index 8560f1f67160..4988386fdc82 100644 --- a/include/llvm/Analysis/RegionInfo.h +++ b/include/llvm/Analysis/RegionInfo.h @@ -47,7 +47,7 @@ namespace llvm { -// RegionTraits - Class to be specialized for different users of RegionInfo +// Class to be specialized for different users of RegionInfo // (i.e. BasicBlocks or MachineBasicBlocks). This is only to avoid needing to // pass around an unreasonable number of template parameters. template <class FuncT_> @@ -282,17 +282,16 @@ class RegionBase : public RegionNodeBase<Tr> { // Save the BasicBlock RegionNodes that are element of this Region. mutable BBNodeMapT BBNodeMap; - /// verifyBBInRegion - Check if a BB is in this Region. This check also works + /// Check if a BB is in this Region. This check also works /// if the region is incorrectly built. (EXPENSIVE!) void verifyBBInRegion(BlockT *BB) const; - /// verifyWalk - Walk over all the BBs of the region starting from BB and + /// Walk over all the BBs of the region starting from BB and /// verify that all reachable basic blocks are elements of the region. /// (EXPENSIVE!) void verifyWalk(BlockT *BB, std::set<BlockT *> *visitedBB) const; - /// verifyRegionNest - Verify if the region and its children are valid - /// regions (EXPENSIVE!) + /// Verify if the region and its children are valid regions (EXPENSIVE!) void verifyRegionNest() const; public: @@ -688,45 +687,50 @@ private: /// Map every BB to the smallest region, that contains BB. BBtoRegionMap BBtoRegion; - // isCommonDomFrontier - Returns true if BB is in the dominance frontier of + // Check whether the entries of BBtoRegion for the BBs of region + // SR are correct. Triggers an assertion if not. Calls itself recursively for + // subregions. + void verifyBBMap(const RegionT *SR) const; + + // Returns true if BB is in the dominance frontier of // entry, because it was inherited from exit. In the other case there is an // edge going from entry to BB without passing exit. bool isCommonDomFrontier(BlockT *BB, BlockT *entry, BlockT *exit) const; - // isRegion - Check if entry and exit surround a valid region, based on + // Check if entry and exit surround a valid region, based on // dominance tree and dominance frontier. bool isRegion(BlockT *entry, BlockT *exit) const; - // insertShortCut - Saves a shortcut pointing from entry to exit. + // Saves a shortcut pointing from entry to exit. // This function may extend this shortcut if possible. void insertShortCut(BlockT *entry, BlockT *exit, BBtoBBMap *ShortCut) const; - // getNextPostDom - Returns the next BB that postdominates N, while skipping + // Returns the next BB that postdominates N, while skipping // all post dominators that cannot finish a canonical region. DomTreeNodeT *getNextPostDom(DomTreeNodeT *N, BBtoBBMap *ShortCut) const; - // isTrivialRegion - A region is trivial, if it contains only one BB. + // A region is trivial, if it contains only one BB. bool isTrivialRegion(BlockT *entry, BlockT *exit) const; - // createRegion - Creates a single entry single exit region. + // Creates a single entry single exit region. RegionT *createRegion(BlockT *entry, BlockT *exit); - // findRegionsWithEntry - Detect all regions starting with bb 'entry'. + // Detect all regions starting with bb 'entry'. void findRegionsWithEntry(BlockT *entry, BBtoBBMap *ShortCut); - // scanForRegions - Detects regions in F. + // Detects regions in F. void scanForRegions(FuncT &F, BBtoBBMap *ShortCut); - // getTopMostParent - Get the top most parent with the same entry block. + // Get the top most parent with the same entry block. RegionT *getTopMostParent(RegionT *region); - // buildRegionsTree - build the region hierarchy after all region detected. + // Build the region hierarchy after all region detected. void buildRegionsTree(DomTreeNodeT *N, RegionT *region); - // updateStatistics - Update statistic about created regions. + // Update statistic about created regions. virtual void updateStatistics(RegionT *R) = 0; - // calculate - detect all regions in function and build the region tree. + // Detect all regions in function and build the region tree. void calculate(FuncT &F); public: @@ -796,12 +800,6 @@ public: RegionT *getTopLevelRegion() const { return TopLevelRegion; } - /// @brief Update RegionInfo after a basic block was split. - /// - /// @param NewBB The basic block that was created before OldBB. - /// @param OldBB The old basic block. - void splitBlock(BlockT *NewBB, BlockT *OldBB); - /// @brief Clear the Node Cache for all Regions. /// /// @see Region::clearNodeCache() @@ -847,6 +845,19 @@ public: void recalculate(Function &F, DominatorTree *DT, PostDominatorTree *PDT, DominanceFrontier *DF); + +#ifndef NDEBUG + /// @brief Opens a viewer to show the GraphViz visualization of the regions. + /// + /// Useful during debugging as an alternative to dump(). + void view(); + + /// @brief Opens a viewer to show the GraphViz visualization of this region + /// without instructions in the BasicBlocks. + /// + /// Useful during debugging as an alternative to dump(). + void viewOnly(); +#endif }; class RegionInfoPass : public FunctionPass { diff --git a/include/llvm/Analysis/RegionInfoImpl.h b/include/llvm/Analysis/RegionInfoImpl.h index b31eefc15f78..134cd8f96fbe 100644 --- a/include/llvm/Analysis/RegionInfoImpl.h +++ b/include/llvm/Analysis/RegionInfoImpl.h @@ -236,7 +236,7 @@ std::string RegionBase<Tr>::getNameStr() const { template <class Tr> void RegionBase<Tr>::verifyBBInRegion(BlockT *BB) const { if (!contains(BB)) - llvm_unreachable("Broken region found!"); + llvm_unreachable("Broken region found: enumerated BB not in region!"); BlockT *entry = getEntry(), *exit = getExit(); @@ -244,7 +244,8 @@ void RegionBase<Tr>::verifyBBInRegion(BlockT *BB) const { SE = BlockTraits::child_end(BB); SI != SE; ++SI) { if (!contains(*SI) && exit != *SI) - llvm_unreachable("Broken region found!"); + llvm_unreachable("Broken region found: edges leaving the region must go " + "to the exit node!"); } if (entry != BB) { @@ -252,7 +253,8 @@ void RegionBase<Tr>::verifyBBInRegion(BlockT *BB) const { SE = InvBlockTraits::child_end(BB); SI != SE; ++SI) { if (!contains(*SI)) - llvm_unreachable("Broken region found!"); + llvm_unreachable("Broken region found: edges entering the region must " + "go to the entry node!"); } } } @@ -442,16 +444,14 @@ typename Tr::RegionT *RegionBase<Tr>::getExpandedRegion() const { if (NumSuccessors == 0) return nullptr; - for (PredIterTy PI = InvBlockTraits::child_begin(getExit()), - PE = InvBlockTraits::child_end(getExit()); - PI != PE; ++PI) { - if (!DT->dominates(getEntry(), *PI)) - return nullptr; - } - RegionT *R = RI->getRegionFor(exit); if (R->getEntry() != exit) { + for (PredIterTy PI = InvBlockTraits::child_begin(getExit()), + PE = InvBlockTraits::child_end(getExit()); + PI != PE; ++PI) + if (!contains(*PI)) + return nullptr; if (Tr::getNumSuccessors(exit) == 1) return new RegionT(getEntry(), *BlockTraits::child_begin(exit), RI, DT); return nullptr; @@ -460,13 +460,11 @@ typename Tr::RegionT *RegionBase<Tr>::getExpandedRegion() const { while (R->getParent() && R->getParent()->getEntry() == exit) R = R->getParent(); - if (!DT->dominates(getEntry(), R->getExit())) { - for (PredIterTy PI = InvBlockTraits::child_begin(getExit()), - PE = InvBlockTraits::child_end(getExit()); - PI != PE; ++PI) { - if (!DT->dominates(R->getExit(), *PI)) - return nullptr; - } + for (PredIterTy PI = InvBlockTraits::child_begin(getExit()), + PE = InvBlockTraits::child_end(getExit()); + PI != PE; ++PI) { + if (!(contains(*PI) || R->contains(*PI))) + return nullptr; } return new RegionT(getEntry(), R->getExit(), RI, DT); @@ -542,6 +540,21 @@ RegionInfoBase<Tr>::~RegionInfoBase() { } template <class Tr> +void RegionInfoBase<Tr>::verifyBBMap(const RegionT *R) const { + assert(R && "Re must be non-null"); + for (auto I = R->element_begin(), E = R->element_end(); I != E; ++I) { + if (I->isSubRegion()) { + const RegionT *SR = I->template getNodeAs<RegionT>(); + verifyBBMap(SR); + } else { + BlockT *BB = I->template getNodeAs<BlockT>(); + if (getRegionFor(BB) != R) + llvm_unreachable("BB map does not match region nesting"); + } + } +} + +template <class Tr> bool RegionInfoBase<Tr>::isCommonDomFrontier(BlockT *BB, BlockT *entry, BlockT *exit) const { for (PredIterTy PI = InvBlockTraits::child_begin(BB), @@ -786,7 +799,14 @@ void RegionInfoBase<Tr>::releaseMemory() { template <class Tr> void RegionInfoBase<Tr>::verifyAnalysis() const { + // Do only verify regions if explicitely activated using XDEBUG or + // -verify-region-info + if (!RegionInfoBase<Tr>::VerifyRegionInfo) + return; + TopLevelRegion->verifyRegionNest(); + + verifyBBMap(TopLevelRegion); } // Region pass manager support. @@ -887,20 +907,6 @@ RegionInfoBase<Tr>::getCommonRegion(SmallVectorImpl<BlockT *> &BBs) const { } template <class Tr> -void RegionInfoBase<Tr>::splitBlock(BlockT *NewBB, BlockT *OldBB) { - RegionT *R = getRegionFor(OldBB); - - setRegionFor(NewBB, R); - - while (R->getEntry() == OldBB && !R->isTopLevelRegion()) { - R->replaceEntry(NewBB); - R = R->getParent(); - } - - setRegionFor(OldBB, R); -} - -template <class Tr> void RegionInfoBase<Tr>::calculate(FuncT &F) { typedef typename std::add_pointer<FuncT>::type FuncPtrT; diff --git a/include/llvm/Analysis/RegionPrinter.h b/include/llvm/Analysis/RegionPrinter.h index 758748aad9e6..8f0035cfd8e6 100644 --- a/include/llvm/Analysis/RegionPrinter.h +++ b/include/llvm/Analysis/RegionPrinter.h @@ -17,10 +17,55 @@ namespace llvm { class FunctionPass; + class Function; + class RegionInfo; + FunctionPass *createRegionViewerPass(); FunctionPass *createRegionOnlyViewerPass(); FunctionPass *createRegionPrinterPass(); FunctionPass *createRegionOnlyPrinterPass(); + +#ifndef NDEBUG + /// @brief Open a viewer to display the GraphViz vizualization of the analysis + /// result. + /// + /// Practical to call in the debugger. + /// Includes the instructions in each BasicBlock. + /// + /// @param RI The analysis to display. + void viewRegion(llvm::RegionInfo *RI); + + /// @brief Analyze the regions of a function and open its GraphViz + /// visualization in a viewer. + /// + /// Useful to call in the debugger. + /// Includes the instructions in each BasicBlock. + /// The result of a new analysis may differ from the RegionInfo the pass + /// manager currently holds. + /// + /// @param F Function to analyze. + void viewRegion(const llvm::Function *F); + + /// @brief Open a viewer to display the GraphViz vizualization of the analysis + /// result. + /// + /// Useful to call in the debugger. + /// Shows only the BasicBlock names without their instructions. + /// + /// @param RI The analysis to display. + void viewRegionOnly(llvm::RegionInfo *RI); + + /// @brief Analyze the regions of a function and open its GraphViz + /// visualization in a viewer. + /// + /// Useful to call in the debugger. + /// Shows only the BasicBlock names without their instructions. + /// The result of a new analysis may differ from the RegionInfo the pass + /// manager currently holds. + /// + /// @param F Function to analyze. + void viewRegionOnly(const llvm::Function *F); +#endif } // End llvm namespace #endif diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index d47cab829ced..c08335de3e7d 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -23,10 +23,12 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/FoldingSet.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" #include "llvm/Support/Allocator.h" @@ -44,30 +46,33 @@ namespace llvm { class DataLayout; class TargetLibraryInfo; class LLVMContext; - class Loop; - class LoopInfo; class Operator; - class SCEVUnknown; class SCEV; - template<> struct FoldingSetTrait<SCEV>; + class SCEVAddRecExpr; + class SCEVConstant; + class SCEVExpander; + class SCEVPredicate; + class SCEVUnknown; - /// SCEV - This class represents an analyzed expression in the program. These - /// are opaque objects that the client is not allowed to do much with - /// directly. + template <> struct FoldingSetTrait<SCEV>; + template <> struct FoldingSetTrait<SCEVPredicate>; + + /// This class represents an analyzed expression in the program. These are + /// opaque objects that the client is not allowed to do much with directly. /// class SCEV : public FoldingSetNode { friend struct FoldingSetTrait<SCEV>; - /// FastID - A reference to an Interned FoldingSetNodeID for this node. - /// The ScalarEvolution's BumpPtrAllocator holds the data. + /// A reference to an Interned FoldingSetNodeID for this node. The + /// ScalarEvolution's BumpPtrAllocator holds the data. FoldingSetNodeIDRef FastID; // The SCEV baseclass this node corresponds to const unsigned short SCEVType; protected: - /// SubclassData - This field is initialized to zero and may be used in - /// subclasses to store miscellaneous information. + /// This field is initialized to zero and may be used in subclasses to store + /// miscellaneous information. unsigned short SubclassData; private: @@ -104,37 +109,32 @@ namespace llvm { unsigned getSCEVType() const { return SCEVType; } - /// getType - Return the LLVM type of this SCEV expression. + /// Return the LLVM type of this SCEV expression. /// Type *getType() const; - /// isZero - Return true if the expression is a constant zero. + /// Return true if the expression is a constant zero. /// bool isZero() const; - /// isOne - Return true if the expression is a constant one. + /// Return true if the expression is a constant one. /// bool isOne() const; - /// isAllOnesValue - Return true if the expression is a constant - /// all-ones value. + /// Return true if the expression is a constant all-ones value. /// bool isAllOnesValue() const; - /// isNonConstantNegative - Return true if the specified scev is negated, - /// but not a constant. + /// Return true if the specified scev is negated, but not a constant. bool isNonConstantNegative() const; - /// print - Print out the internal representation of this scalar to the - /// specified stream. This should really only be used for debugging - /// purposes. + /// Print out the internal representation of this scalar to the specified + /// stream. This should really only be used for debugging purposes. void print(raw_ostream &OS) const; -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - /// dump - This method is used for debugging. + /// This method is used for debugging. /// void dump() const; -#endif }; // Specialize FoldingSetTrait for SCEV to avoid needing to compute @@ -157,11 +157,10 @@ namespace llvm { return OS; } - /// SCEVCouldNotCompute - An object of this class is returned by queries that - /// could not be answered. For example, if you ask for the number of - /// iterations of a linked-list traversal loop, you will get one of these. - /// None of the standard SCEV operations are valid on this class, it is just a - /// marker. + /// An object of this class is returned by queries that could not be answered. + /// For example, if you ask for the number of iterations of a linked-list + /// traversal loop, you will get one of these. None of the standard SCEV + /// operations are valid on this class, it is just a marker. struct SCEVCouldNotCompute : public SCEV { SCEVCouldNotCompute(); @@ -169,22 +168,162 @@ namespace llvm { static bool classof(const SCEV *S); }; - /// ScalarEvolution - This class is the main scalar evolution driver. Because - /// client code (intentionally) can't do much with the SCEV objects directly, - /// they must ask this class for services. - /// - class ScalarEvolution : public FunctionPass { + /// SCEVPredicate - This class represents an assumption made using SCEV + /// expressions which can be checked at run-time. + class SCEVPredicate : public FoldingSetNode { + friend struct FoldingSetTrait<SCEVPredicate>; + + /// A reference to an Interned FoldingSetNodeID for this node. The + /// ScalarEvolution's BumpPtrAllocator holds the data. + FoldingSetNodeIDRef FastID; + + public: + enum SCEVPredicateKind { P_Union, P_Equal }; + + protected: + SCEVPredicateKind Kind; + ~SCEVPredicate() = default; + SCEVPredicate(const SCEVPredicate&) = default; + SCEVPredicate &operator=(const SCEVPredicate&) = default; + + public: + SCEVPredicate(const FoldingSetNodeIDRef ID, SCEVPredicateKind Kind); + + SCEVPredicateKind getKind() const { return Kind; } + + /// \brief Returns the estimated complexity of this predicate. + /// This is roughly measured in the number of run-time checks required. + virtual unsigned getComplexity() const { return 1; } + + /// \brief Returns true if the predicate is always true. This means that no + /// assumptions were made and nothing needs to be checked at run-time. + virtual bool isAlwaysTrue() const = 0; + + /// \brief Returns true if this predicate implies \p N. + virtual bool implies(const SCEVPredicate *N) const = 0; + + /// \brief Prints a textual representation of this predicate with an + /// indentation of \p Depth. + virtual void print(raw_ostream &OS, unsigned Depth = 0) const = 0; + + /// \brief Returns the SCEV to which this predicate applies, or nullptr + /// if this is a SCEVUnionPredicate. + virtual const SCEV *getExpr() const = 0; + }; + + inline raw_ostream &operator<<(raw_ostream &OS, const SCEVPredicate &P) { + P.print(OS); + return OS; + } + + // Specialize FoldingSetTrait for SCEVPredicate to avoid needing to compute + // temporary FoldingSetNodeID values. + template <> + struct FoldingSetTrait<SCEVPredicate> + : DefaultFoldingSetTrait<SCEVPredicate> { + + static void Profile(const SCEVPredicate &X, FoldingSetNodeID &ID) { + ID = X.FastID; + } + + static bool Equals(const SCEVPredicate &X, const FoldingSetNodeID &ID, + unsigned IDHash, FoldingSetNodeID &TempID) { + return ID == X.FastID; + } + static unsigned ComputeHash(const SCEVPredicate &X, + FoldingSetNodeID &TempID) { + return X.FastID.ComputeHash(); + } + }; + + /// SCEVEqualPredicate - This class represents an assumption that two SCEV + /// expressions are equal, and this can be checked at run-time. We assume + /// that the left hand side is a SCEVUnknown and the right hand side a + /// constant. + class SCEVEqualPredicate final : public SCEVPredicate { + /// We assume that LHS == RHS, where LHS is a SCEVUnknown and RHS a + /// constant. + const SCEVUnknown *LHS; + const SCEVConstant *RHS; + public: - /// LoopDisposition - An enum describing the relationship between a - /// SCEV and a loop. + SCEVEqualPredicate(const FoldingSetNodeIDRef ID, const SCEVUnknown *LHS, + const SCEVConstant *RHS); + + /// Implementation of the SCEVPredicate interface + bool implies(const SCEVPredicate *N) const override; + void print(raw_ostream &OS, unsigned Depth = 0) const override; + bool isAlwaysTrue() const override; + const SCEV *getExpr() const override; + + /// \brief Returns the left hand side of the equality. + const SCEVUnknown *getLHS() const { return LHS; } + + /// \brief Returns the right hand side of the equality. + const SCEVConstant *getRHS() const { return RHS; } + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const SCEVPredicate *P) { + return P->getKind() == P_Equal; + } + }; + + /// SCEVUnionPredicate - This class represents a composition of other + /// SCEV predicates, and is the class that most clients will interact with. + /// This is equivalent to a logical "AND" of all the predicates in the union. + class SCEVUnionPredicate final : public SCEVPredicate { + private: + typedef DenseMap<const SCEV *, SmallVector<const SCEVPredicate *, 4>> + PredicateMap; + + /// Vector with references to all predicates in this union. + SmallVector<const SCEVPredicate *, 16> Preds; + /// Maps SCEVs to predicates for quick look-ups. + PredicateMap SCEVToPreds; + + public: + SCEVUnionPredicate(); + + const SmallVectorImpl<const SCEVPredicate *> &getPredicates() const { + return Preds; + } + + /// \brief Adds a predicate to this union. + void add(const SCEVPredicate *N); + + /// \brief Returns a reference to a vector containing all predicates + /// which apply to \p Expr. + ArrayRef<const SCEVPredicate *> getPredicatesForExpr(const SCEV *Expr); + + /// Implementation of the SCEVPredicate interface + bool isAlwaysTrue() const override; + bool implies(const SCEVPredicate *N) const override; + void print(raw_ostream &OS, unsigned Depth) const override; + const SCEV *getExpr() const override; + + /// \brief We estimate the complexity of a union predicate as the size + /// number of predicates in the union. + unsigned getComplexity() const override { return Preds.size(); } + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const SCEVPredicate *P) { + return P->getKind() == P_Union; + } + }; + + /// The main scalar evolution driver. Because client code (intentionally) + /// can't do much with the SCEV objects directly, they must ask this class + /// for services. + class ScalarEvolution { + public: + /// An enum describing the relationship between a SCEV and a loop. enum LoopDisposition { LoopVariant, ///< The SCEV is loop-variant (unknown). LoopInvariant, ///< The SCEV is loop-invariant. LoopComputable ///< The SCEV varies predictably with the loop. }; - /// BlockDisposition - An enum describing the relationship between a - /// SCEV and a basic block. + /// An enum describing the relationship between a SCEV and a basic block. enum BlockDisposition { DoesNotDominateBlock, ///< The SCEV does not dominate the block. DominatesBlock, ///< The SCEV dominates the block. @@ -207,9 +346,9 @@ namespace llvm { } private: - /// SCEVCallbackVH - A CallbackVH to arrange for ScalarEvolution to be - /// notified whenever a Value is deleted. - class SCEVCallbackVH : public CallbackVH { + /// A CallbackVH to arrange for ScalarEvolution to be notified whenever a + /// Value is deleted. + class SCEVCallbackVH final : public CallbackVH { ScalarEvolution *SE; void deleted() override; void allUsesReplacedWith(Value *New) override; @@ -221,35 +360,34 @@ namespace llvm { friend class SCEVExpander; friend class SCEVUnknown; - /// F - The function we are analyzing. + /// The function we are analyzing. /// - Function *F; + Function &F; - /// The tracker for @llvm.assume intrinsics in this function. - AssumptionCache *AC; - - /// LI - The loop information for the function we are currently analyzing. + /// The target library information for the target we are targeting. /// - LoopInfo *LI; + TargetLibraryInfo &TLI; + + /// The tracker for @llvm.assume intrinsics in this function. + AssumptionCache &AC; - /// TLI - The target library information for the target we are targeting. + /// The dominator tree. /// - TargetLibraryInfo *TLI; + DominatorTree &DT; - /// DT - The dominator tree. + /// The loop information for the function we are currently analyzing. /// - DominatorTree *DT; + LoopInfo &LI; - /// CouldNotCompute - This SCEV is used to represent unknown trip - /// counts and things. - SCEVCouldNotCompute CouldNotCompute; + /// This SCEV is used to represent unknown trip counts and things. + std::unique_ptr<SCEVCouldNotCompute> CouldNotCompute; - /// ValueExprMapType - The typedef for ValueExprMap. + /// The typedef for ValueExprMap. /// typedef DenseMap<SCEVCallbackVH, const SCEV *, DenseMapInfo<Value *> > ValueExprMapType; - /// ValueExprMap - This is a cache of the values we have analyzed so far. + /// This is a cache of the values we have analyzed so far. /// ValueExprMapType ValueExprMap; @@ -260,10 +398,14 @@ namespace llvm { /// conditions dominating the backedge of a loop. bool WalkingBEDominatingConds; - /// ExitLimit - Information about the number of loop iterations for which a - /// loop exit's branch condition evaluates to the not-taken path. This is a - /// temporary pair of exact and max expressions that are eventually - /// summarized in ExitNotTakenInfo and BackedgeTakenInfo. + /// Set to true by isKnownPredicateViaSplitting when we're trying to prove a + /// predicate by splitting it into a set of independent predicates. + bool ProvingSplitPredicate; + + /// Information about the number of loop iterations for which a loop exit's + /// branch condition evaluates to the not-taken path. This is a temporary + /// pair of exact and max expressions that are eventually summarized in + /// ExitNotTakenInfo and BackedgeTakenInfo. struct ExitLimit { const SCEV *Exact; const SCEV *Max; @@ -272,16 +414,16 @@ namespace llvm { ExitLimit(const SCEV *E, const SCEV *M) : Exact(E), Max(M) {} - /// hasAnyInfo - Test whether this ExitLimit contains any computed - /// information, or whether it's all SCEVCouldNotCompute values. + /// Test whether this ExitLimit contains any computed information, or + /// whether it's all SCEVCouldNotCompute values. bool hasAnyInfo() const { return !isa<SCEVCouldNotCompute>(Exact) || !isa<SCEVCouldNotCompute>(Max); } }; - /// ExitNotTakenInfo - Information about the number of times a particular - /// loop exit may be reached before exiting the loop. + /// Information about the number of times a particular loop exit may be + /// reached before exiting the loop. struct ExitNotTakenInfo { AssertingVH<BasicBlock> ExitingBlock; const SCEV *ExactNotTaken; @@ -289,14 +431,14 @@ namespace llvm { ExitNotTakenInfo() : ExitingBlock(nullptr), ExactNotTaken(nullptr) {} - /// isCompleteList - Return true if all loop exits are computable. + /// Return true if all loop exits are computable. bool isCompleteList() const { return NextExit.getInt() == 0; } void setIncomplete() { NextExit.setInt(1); } - /// getNextExit - Return a pointer to the next exit's not-taken info. + /// Return a pointer to the next exit's not-taken info. ExitNotTakenInfo *getNextExit() const { return NextExit.getPointer(); } @@ -304,16 +446,16 @@ namespace llvm { void setNextExit(ExitNotTakenInfo *ENT) { NextExit.setPointer(ENT); } }; - /// BackedgeTakenInfo - Information about the backedge-taken count - /// of a loop. This currently includes an exact count and a maximum count. + /// Information about the backedge-taken count of a loop. This currently + /// includes an exact count and a maximum count. /// class BackedgeTakenInfo { - /// ExitNotTaken - A list of computable exits and their not-taken counts. - /// Loops almost never have more than one computable exit. + /// A list of computable exits and their not-taken counts. Loops almost + /// never have more than one computable exit. ExitNotTakenInfo ExitNotTaken; - /// Max - An expression indicating the least maximum backedge-taken - /// count of the loop that is known, or a SCEVCouldNotCompute. + /// An expression indicating the least maximum backedge-taken count of the + /// loop that is known, or a SCEVCouldNotCompute. const SCEV *Max; public: @@ -324,80 +466,78 @@ namespace llvm { SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts, bool Complete, const SCEV *MaxCount); - /// hasAnyInfo - Test whether this BackedgeTakenInfo contains any - /// computed information, or whether it's all SCEVCouldNotCompute - /// values. + /// Test whether this BackedgeTakenInfo contains any computed information, + /// or whether it's all SCEVCouldNotCompute values. bool hasAnyInfo() const { return ExitNotTaken.ExitingBlock || !isa<SCEVCouldNotCompute>(Max); } - /// getExact - Return an expression indicating the exact backedge-taken - /// count of the loop if it is known, or SCEVCouldNotCompute - /// otherwise. This is the number of times the loop header can be - /// guaranteed to execute, minus one. + /// Return an expression indicating the exact backedge-taken count of the + /// loop if it is known, or SCEVCouldNotCompute otherwise. This is the + /// number of times the loop header can be guaranteed to execute, minus + /// one. const SCEV *getExact(ScalarEvolution *SE) const; - /// getExact - Return the number of times this loop exit may fall through - /// to the back edge, or SCEVCouldNotCompute. The loop is guaranteed not - /// to exit via this block before this number of iterations, but may exit - /// via another block. + /// Return the number of times this loop exit may fall through to the back + /// edge, or SCEVCouldNotCompute. The loop is guaranteed not to exit via + /// this block before this number of iterations, but may exit via another + /// block. const SCEV *getExact(BasicBlock *ExitingBlock, ScalarEvolution *SE) const; - /// getMax - Get the max backedge taken count for the loop. + /// Get the max backedge taken count for the loop. const SCEV *getMax(ScalarEvolution *SE) const; /// Return true if any backedge taken count expressions refer to the given /// subexpression. bool hasOperand(const SCEV *S, ScalarEvolution *SE) const; - /// clear - Invalidate this result and free associated memory. + /// Invalidate this result and free associated memory. void clear(); }; - /// BackedgeTakenCounts - Cache the backedge-taken count of the loops for - /// this function as they are computed. + /// Cache the backedge-taken count of the loops for this function as they + /// are computed. DenseMap<const Loop*, BackedgeTakenInfo> BackedgeTakenCounts; - /// ConstantEvolutionLoopExitValue - This map contains entries for all of - /// the PHI instructions that we attempt to compute constant evolutions for. - /// This allows us to avoid potentially expensive recomputation of these - /// properties. An instruction maps to null if we are unable to compute its - /// exit value. + /// This map contains entries for all of the PHI instructions that we + /// attempt to compute constant evolutions for. This allows us to avoid + /// potentially expensive recomputation of these properties. An instruction + /// maps to null if we are unable to compute its exit value. DenseMap<PHINode*, Constant*> ConstantEvolutionLoopExitValue; - /// ValuesAtScopes - This map contains entries for all the expressions - /// that we attempt to compute getSCEVAtScope information for, which can - /// be expensive in extreme cases. + /// This map contains entries for all the expressions that we attempt to + /// compute getSCEVAtScope information for, which can be expensive in + /// extreme cases. DenseMap<const SCEV *, SmallVector<std::pair<const Loop *, const SCEV *>, 2> > ValuesAtScopes; - /// LoopDispositions - Memoized computeLoopDisposition results. + /// Memoized computeLoopDisposition results. DenseMap<const SCEV *, SmallVector<PointerIntPair<const Loop *, 2, LoopDisposition>, 2>> LoopDispositions; - /// computeLoopDisposition - Compute a LoopDisposition value. + /// Compute a LoopDisposition value. LoopDisposition computeLoopDisposition(const SCEV *S, const Loop *L); - /// BlockDispositions - Memoized computeBlockDisposition results. + /// Memoized computeBlockDisposition results. DenseMap< const SCEV *, SmallVector<PointerIntPair<const BasicBlock *, 2, BlockDisposition>, 2>> BlockDispositions; - /// computeBlockDisposition - Compute a BlockDisposition value. + /// Compute a BlockDisposition value. BlockDisposition computeBlockDisposition(const SCEV *S, const BasicBlock *BB); - /// UnsignedRanges - Memoized results from getRange + /// Memoized results from getRange DenseMap<const SCEV *, ConstantRange> UnsignedRanges; - /// SignedRanges - Memoized results from getRange + /// Memoized results from getRange DenseMap<const SCEV *, ConstantRange> SignedRanges; - /// RangeSignHint - Used to parameterize getRange + /// Used to parameterize getRange enum RangeSignHint { HINT_RANGE_UNSIGNED, HINT_RANGE_SIGNED }; - /// setRange - Set the memoized range for the given SCEV. + /// Set the memoized range for the given SCEV. const ConstantRange &setRange(const SCEV *S, RangeSignHint Hint, const ConstantRange &CR) { DenseMap<const SCEV *, ConstantRange> &Cache = @@ -410,198 +550,275 @@ namespace llvm { return Pair.first->second; } - /// getRange - Determine the range for a particular SCEV. + /// Determine the range for a particular SCEV. ConstantRange getRange(const SCEV *S, RangeSignHint Hint); - /// createSCEV - We know that there is no SCEV for the specified value. - /// Analyze the expression. + /// We know that there is no SCEV for the specified value. Analyze the + /// expression. const SCEV *createSCEV(Value *V); - /// createNodeForPHI - Provide the special handling we need to analyze PHI - /// SCEVs. + /// Provide the special handling we need to analyze PHI SCEVs. const SCEV *createNodeForPHI(PHINode *PN); - /// createNodeForGEP - Provide the special handling we need to analyze GEP - /// SCEVs. + /// Helper function called from createNodeForPHI. + const SCEV *createAddRecFromPHI(PHINode *PN); + + /// Helper function called from createNodeForPHI. + const SCEV *createNodeFromSelectLikePHI(PHINode *PN); + + /// Provide special handling for a select-like instruction (currently this + /// is either a select instruction or a phi node). \p I is the instruction + /// being processed, and it is assumed equivalent to "Cond ? TrueVal : + /// FalseVal". + const SCEV *createNodeForSelectOrPHI(Instruction *I, Value *Cond, + Value *TrueVal, Value *FalseVal); + + /// Provide the special handling we need to analyze GEP SCEVs. const SCEV *createNodeForGEP(GEPOperator *GEP); - /// computeSCEVAtScope - Implementation code for getSCEVAtScope; called - /// at most once for each SCEV+Loop pair. + /// Implementation code for getSCEVAtScope; called at most once for each + /// SCEV+Loop pair. /// const SCEV *computeSCEVAtScope(const SCEV *S, const Loop *L); - /// ForgetSymbolicValue - This looks up computed SCEV values for all - /// instructions that depend on the given instruction and removes them from - /// the ValueExprMap map if they reference SymName. This is used during PHI - /// resolution. + /// This looks up computed SCEV values for all instructions that depend on + /// the given instruction and removes them from the ValueExprMap map if they + /// reference SymName. This is used during PHI resolution. void ForgetSymbolicName(Instruction *I, const SCEV *SymName); - /// getBackedgeTakenInfo - Return the BackedgeTakenInfo for the given - /// loop, lazily computing new values if the loop hasn't been analyzed - /// yet. + /// Return the BackedgeTakenInfo for the given loop, lazily computing new + /// values if the loop hasn't been analyzed yet. const BackedgeTakenInfo &getBackedgeTakenInfo(const Loop *L); - /// ComputeBackedgeTakenCount - Compute the number of times the specified - /// loop will iterate. - BackedgeTakenInfo ComputeBackedgeTakenCount(const Loop *L); + /// Compute the number of times the specified loop will iterate. + BackedgeTakenInfo computeBackedgeTakenCount(const Loop *L); - /// ComputeExitLimit - Compute the number of times the backedge of the - /// specified loop will execute if it exits via the specified block. - ExitLimit ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock); + /// Compute the number of times the backedge of the specified loop will + /// execute if it exits via the specified block. + ExitLimit computeExitLimit(const Loop *L, BasicBlock *ExitingBlock); - /// ComputeExitLimitFromCond - Compute the number of times the backedge of - /// the specified loop will execute if its exit condition were a conditional - /// branch of ExitCond, TBB, and FBB. - ExitLimit ComputeExitLimitFromCond(const Loop *L, + /// Compute the number of times the backedge of the specified loop will + /// execute if its exit condition were a conditional branch of ExitCond, + /// TBB, and FBB. + ExitLimit computeExitLimitFromCond(const Loop *L, Value *ExitCond, BasicBlock *TBB, BasicBlock *FBB, bool IsSubExpr); - /// ComputeExitLimitFromICmp - Compute the number of times the backedge of - /// the specified loop will execute if its exit condition were a conditional - /// branch of the ICmpInst ExitCond, TBB, and FBB. - ExitLimit ComputeExitLimitFromICmp(const Loop *L, + /// Compute the number of times the backedge of the specified loop will + /// execute if its exit condition were a conditional branch of the ICmpInst + /// ExitCond, TBB, and FBB. + ExitLimit computeExitLimitFromICmp(const Loop *L, ICmpInst *ExitCond, BasicBlock *TBB, BasicBlock *FBB, bool IsSubExpr); - /// ComputeExitLimitFromSingleExitSwitch - Compute the number of times the - /// backedge of the specified loop will execute if its exit condition were a - /// switch with a single exiting case to ExitingBB. + /// Compute the number of times the backedge of the specified loop will + /// execute if its exit condition were a switch with a single exiting case + /// to ExitingBB. ExitLimit - ComputeExitLimitFromSingleExitSwitch(const Loop *L, SwitchInst *Switch, + computeExitLimitFromSingleExitSwitch(const Loop *L, SwitchInst *Switch, BasicBlock *ExitingBB, bool IsSubExpr); - /// ComputeLoadConstantCompareExitLimit - Given an exit condition - /// of 'icmp op load X, cst', try to see if we can compute the - /// backedge-taken count. - ExitLimit ComputeLoadConstantCompareExitLimit(LoadInst *LI, + /// Given an exit condition of 'icmp op load X, cst', try to see if we can + /// compute the backedge-taken count. + ExitLimit computeLoadConstantCompareExitLimit(LoadInst *LI, Constant *RHS, const Loop *L, ICmpInst::Predicate p); - /// ComputeExitCountExhaustively - If the loop is known to execute a - /// constant number of times (the condition evolves only from constants), - /// try to evaluate a few iterations of the loop until we get the exit - /// condition gets a value of ExitWhen (true or false). If we cannot - /// evaluate the exit count of the loop, return CouldNotCompute. - const SCEV *ComputeExitCountExhaustively(const Loop *L, + /// Compute the exit limit of a loop that is controlled by a + /// "(IV >> 1) != 0" type comparison. We cannot compute the exact trip + /// count in these cases (since SCEV has no way of expressing them), but we + /// can still sometimes compute an upper bound. + /// + /// Return an ExitLimit for a loop whose backedge is guarded by `LHS Pred + /// RHS`. + ExitLimit computeShiftCompareExitLimit(Value *LHS, Value *RHS, + const Loop *L, + ICmpInst::Predicate Pred); + + /// If the loop is known to execute a constant number of times (the + /// condition evolves only from constants), try to evaluate a few iterations + /// of the loop until we get the exit condition gets a value of ExitWhen + /// (true or false). If we cannot evaluate the exit count of the loop, + /// return CouldNotCompute. + const SCEV *computeExitCountExhaustively(const Loop *L, Value *Cond, bool ExitWhen); - /// HowFarToZero - Return the number of times an exit condition comparing - /// the specified value to zero will execute. If not computable, return - /// CouldNotCompute. + /// Return the number of times an exit condition comparing the specified + /// value to zero will execute. If not computable, return CouldNotCompute. ExitLimit HowFarToZero(const SCEV *V, const Loop *L, bool IsSubExpr); - /// HowFarToNonZero - Return the number of times an exit condition checking - /// the specified value for nonzero will execute. If not computable, return + /// Return the number of times an exit condition checking the specified + /// value for nonzero will execute. If not computable, return /// CouldNotCompute. ExitLimit HowFarToNonZero(const SCEV *V, const Loop *L); - /// HowManyLessThans - Return the number of times an exit condition - /// containing the specified less-than comparison will execute. If not - /// computable, return CouldNotCompute. isSigned specifies whether the - /// less-than is signed. + /// Return the number of times an exit condition containing the specified + /// less-than comparison will execute. If not computable, return + /// CouldNotCompute. isSigned specifies whether the less-than is signed. ExitLimit HowManyLessThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, bool isSigned, bool IsSubExpr); ExitLimit HowManyGreaterThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, bool isSigned, bool IsSubExpr); - /// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB - /// (which may not be an immediate predecessor) which has exactly one - /// successor from which BB is reachable, or null if no such block is - /// found. + /// Return a predecessor of BB (which may not be an immediate predecessor) + /// which has exactly one successor from which BB is reachable, or null if + /// no such block is found. std::pair<BasicBlock *, BasicBlock *> getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB); - /// isImpliedCond - Test whether the condition described by Pred, LHS, and - /// RHS is true whenever the given FoundCondValue value evaluates to true. + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the given FoundCondValue value evaluates to true. bool isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, Value *FoundCondValue, bool Inverse); - /// isImpliedCondOperands - Test whether the condition described by Pred, - /// LHS, and RHS is true whenever the condition described by Pred, FoundLHS, - /// and FoundRHS is true. + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by FoundPred, FoundLHS, FoundRHS is + /// true. + bool isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS, ICmpInst::Predicate FoundPred, + const SCEV *FoundLHS, const SCEV *FoundRHS); + + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. bool isImpliedCondOperands(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS, const SCEV *FoundRHS); - /// isImpliedCondOperandsHelper - Test whether the condition described by - /// Pred, LHS, and RHS is true whenever the condition described by Pred, - /// FoundLHS, and FoundRHS is true. + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. bool isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS, const SCEV *FoundRHS); - /// isImpliedCondOperandsViaRanges - Test whether the condition described by - /// Pred, LHS, and RHS is true whenever the condition described by Pred, - /// FoundLHS, and FoundRHS is true. Utility function used by - /// isImpliedCondOperands. + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. Utility function used by isImpliedCondOperands. bool isImpliedCondOperandsViaRanges(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS, const SCEV *FoundRHS); - /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is - /// in the header of its containing loop, we know the loop executes a - /// constant number of times, and the PHI node is just a recurrence - /// involving constants, fold it. + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. + /// + /// This routine tries to rule out certain kinds of integer overflow, and + /// then tries to reason about arithmetic properties of the predicates. + bool isImpliedCondOperandsViaNoOverflow(ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS, + const SCEV *FoundLHS, + const SCEV *FoundRHS); + + /// If we know that the specified Phi is in the header of its containing + /// loop, we know the loop executes a constant number of times, and the PHI + /// node is just a recurrence involving constants, fold it. Constant *getConstantEvolutionLoopExitValue(PHINode *PN, const APInt& BEs, const Loop *L); - /// isKnownPredicateWithRanges - Test if the given expression is known to - /// satisfy the condition described by Pred and the known constant ranges - /// of LHS and RHS. + /// Test if the given expression is known to satisfy the condition described + /// by Pred and the known constant ranges of LHS and RHS. /// bool isKnownPredicateWithRanges(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS); - /// forgetMemoizedResults - Drop memoized information computed for S. + /// Try to prove the condition described by "LHS Pred RHS" by ruling out + /// integer overflow. + /// + /// For instance, this will return true for "A s< (A + C)<nsw>" if C is + /// positive. + bool isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS); + + /// Try to split Pred LHS RHS into logical conjunctions (and's) and try to + /// prove them individually. + bool isKnownPredicateViaSplitting(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS); + + /// Try to match the Expr as "(L + R)<Flags>". + bool splitBinaryAdd(const SCEV *Expr, const SCEV *&L, const SCEV *&R, + SCEV::NoWrapFlags &Flags); + + /// Return true if More == (Less + C), where C is a constant. This is + /// intended to be used as a cheaper substitute for full SCEV subtraction. + bool computeConstantDifference(const SCEV *Less, const SCEV *More, + APInt &C); + + /// Drop memoized information computed for S. void forgetMemoizedResults(const SCEV *S); + /// Return an existing SCEV for V if there is one, otherwise return nullptr. + const SCEV *getExistingSCEV(Value *V); + /// Return false iff given SCEV contains a SCEVUnknown with NULL value- /// pointer. bool checkValidity(const SCEV *S) const; - // Return true if `ExtendOpTy`({`Start`,+,`Step`}) can be proved to be equal - // to {`ExtendOpTy`(`Start`),+,`ExtendOpTy`(`Step`)}. This is equivalent to - // proving no signed (resp. unsigned) wrap in {`Start`,+,`Step`} if - // `ExtendOpTy` is `SCEVSignExtendExpr` (resp. `SCEVZeroExtendExpr`). - // + /// Return true if `ExtendOpTy`({`Start`,+,`Step`}) can be proved to be + /// equal to {`ExtendOpTy`(`Start`),+,`ExtendOpTy`(`Step`)}. This is + /// equivalent to proving no signed (resp. unsigned) wrap in + /// {`Start`,+,`Step`} if `ExtendOpTy` is `SCEVSignExtendExpr` + /// (resp. `SCEVZeroExtendExpr`). + /// template<typename ExtendOpTy> bool proveNoWrapByVaryingStart(const SCEV *Start, const SCEV *Step, const Loop *L); + bool isMonotonicPredicateImpl(const SCEVAddRecExpr *LHS, + ICmpInst::Predicate Pred, bool &Increasing); + + /// Return true if, for all loop invariant X, the predicate "LHS `Pred` X" + /// is monotonically increasing or decreasing. In the former case set + /// `Increasing` to true and in the latter case set `Increasing` to false. + /// + /// A predicate is said to be monotonically increasing if may go from being + /// false to being true as the loop iterates, but never the other way + /// around. A predicate is said to be monotonically decreasing if may go + /// from being true to being false as the loop iterates, but never the other + /// way around. + bool isMonotonicPredicate(const SCEVAddRecExpr *LHS, + ICmpInst::Predicate Pred, bool &Increasing); + + // Return SCEV no-wrap flags that can be proven based on reasoning + // about how poison produced from no-wrap flags on this value + // (e.g. a nuw add) would trigger undefined behavior on overflow. + SCEV::NoWrapFlags getNoWrapFlagsFromUB(const Value *V); + public: - static char ID; // Pass identification, replacement for typeid - ScalarEvolution(); + ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, + DominatorTree &DT, LoopInfo &LI); + ~ScalarEvolution(); + ScalarEvolution(ScalarEvolution &&Arg); - LLVMContext &getContext() const { return F->getContext(); } + LLVMContext &getContext() const { return F.getContext(); } - /// isSCEVable - Test if values of the given type are analyzable within - /// the SCEV framework. This primarily includes integer types, and it - /// can optionally include pointer types if the ScalarEvolution class - /// has access to target-specific information. + /// Test if values of the given type are analyzable within the SCEV + /// framework. This primarily includes integer types, and it can optionally + /// include pointer types if the ScalarEvolution class has access to + /// target-specific information. bool isSCEVable(Type *Ty) const; - /// getTypeSizeInBits - Return the size in bits of the specified type, - /// for which isSCEVable must return true. + /// Return the size in bits of the specified type, for which isSCEVable must + /// return true. uint64_t getTypeSizeInBits(Type *Ty) const; - /// getEffectiveSCEVType - Return a type with the same bitwidth as - /// the given type and which represents how SCEV will treat the given - /// type, for which isSCEVable must return true. For pointer types, - /// this is the pointer-sized integer type. + /// Return a type with the same bitwidth as the given type and which + /// represents how SCEV will treat the given type, for which isSCEVable must + /// return true. For pointer types, this is the pointer-sized integer type. Type *getEffectiveSCEVType(Type *Ty) const; - /// getSCEV - Return a SCEV expression for the full generality of the - /// specified expression. + /// Return a SCEV expression for the full generality of the specified + /// expression. const SCEV *getSCEV(Value *V); const SCEV *getConstant(ConstantInt *V); @@ -615,35 +832,24 @@ namespace llvm { SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap); const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap) { - SmallVector<const SCEV *, 2> Ops; - Ops.push_back(LHS); - Ops.push_back(RHS); + SmallVector<const SCEV *, 2> Ops = {LHS, RHS}; return getAddExpr(Ops, Flags); } const SCEV *getAddExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap) { - SmallVector<const SCEV *, 3> Ops; - Ops.push_back(Op0); - Ops.push_back(Op1); - Ops.push_back(Op2); + SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2}; return getAddExpr(Ops, Flags); } const SCEV *getMulExpr(SmallVectorImpl<const SCEV *> &Ops, SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap); const SCEV *getMulExpr(const SCEV *LHS, const SCEV *RHS, - SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap) - { - SmallVector<const SCEV *, 2> Ops; - Ops.push_back(LHS); - Ops.push_back(RHS); + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap) { + SmallVector<const SCEV *, 2> Ops = {LHS, RHS}; return getMulExpr(Ops, Flags); } const SCEV *getMulExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2, SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap) { - SmallVector<const SCEV *, 3> Ops; - Ops.push_back(Op0); - Ops.push_back(Op1); - Ops.push_back(Op2); + SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2}; return getMulExpr(Ops, Flags); } const SCEV *getUDivExpr(const SCEV *LHS, const SCEV *RHS); @@ -675,81 +881,80 @@ namespace llvm { const SCEV *getUnknown(Value *V); const SCEV *getCouldNotCompute(); - /// getSizeOfExpr - Return an expression for sizeof AllocTy that is type - /// IntTy + /// \brief Return a SCEV for the constant 0 of a specific type. + const SCEV *getZero(Type *Ty) { return getConstant(Ty, 0); } + + /// \brief Return a SCEV for the constant 1 of a specific type. + const SCEV *getOne(Type *Ty) { return getConstant(Ty, 1); } + + /// Return an expression for sizeof AllocTy that is type IntTy /// const SCEV *getSizeOfExpr(Type *IntTy, Type *AllocTy); - /// getOffsetOfExpr - Return an expression for offsetof on the given field - /// with type IntTy + /// Return an expression for offsetof on the given field with type IntTy /// const SCEV *getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo); - /// getNegativeSCEV - Return the SCEV object corresponding to -V. + /// Return the SCEV object corresponding to -V. /// - const SCEV *getNegativeSCEV(const SCEV *V); + const SCEV *getNegativeSCEV(const SCEV *V, + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap); - /// getNotSCEV - Return the SCEV object corresponding to ~V. + /// Return the SCEV object corresponding to ~V. /// const SCEV *getNotSCEV(const SCEV *V); - /// getMinusSCEV - Return LHS-RHS. Minus is represented in SCEV as A+B*-1. + /// Return LHS-RHS. Minus is represented in SCEV as A+B*-1. const SCEV *getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap); - /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion - /// of the input value to the specified type. If the type must be - /// extended, it is zero extended. + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is zero extended. const SCEV *getTruncateOrZeroExtend(const SCEV *V, Type *Ty); - /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion - /// of the input value to the specified type. If the type must be - /// extended, it is sign extended. + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is sign extended. const SCEV *getTruncateOrSignExtend(const SCEV *V, Type *Ty); - /// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of - /// the input value to the specified type. If the type must be extended, - /// it is zero extended. The conversion must not be narrowing. + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is zero extended. The + /// conversion must not be narrowing. const SCEV *getNoopOrZeroExtend(const SCEV *V, Type *Ty); - /// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of - /// the input value to the specified type. If the type must be extended, - /// it is sign extended. The conversion must not be narrowing. + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is sign extended. The + /// conversion must not be narrowing. const SCEV *getNoopOrSignExtend(const SCEV *V, Type *Ty); - /// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of - /// the input value to the specified type. If the type must be extended, - /// it is extended with unspecified bits. The conversion must not be - /// narrowing. + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. If the type must be extended, it is extended with + /// unspecified bits. The conversion must not be narrowing. const SCEV *getNoopOrAnyExtend(const SCEV *V, Type *Ty); - /// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the - /// input value to the specified type. The conversion must not be - /// widening. + /// Return a SCEV corresponding to a conversion of the input value to the + /// specified type. The conversion must not be widening. const SCEV *getTruncateOrNoop(const SCEV *V, Type *Ty); - /// getUMaxFromMismatchedTypes - Promote the operands to the wider of - /// the types using zero-extension, and then perform a umax operation - /// with them. + /// Promote the operands to the wider of the types using zero-extension, and + /// then perform a umax operation with them. const SCEV *getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); - /// getUMinFromMismatchedTypes - Promote the operands to the wider of - /// the types using zero-extension, and then perform a umin operation - /// with them. + /// Promote the operands to the wider of the types using zero-extension, and + /// then perform a umin operation with them. const SCEV *getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); - /// getPointerBase - Transitively follow the chain of pointer-type operands - /// until reaching a SCEV that does not have a single pointer operand. This - /// returns a SCEVUnknown pointer for well-formed pointer-type expressions, - /// but corner cases do exist. + /// Transitively follow the chain of pointer-type operands until reaching a + /// SCEV that does not have a single pointer operand. This returns a + /// SCEVUnknown pointer for well-formed pointer-type expressions, but corner + /// cases do exist. const SCEV *getPointerBase(const SCEV *V); - /// getSCEVAtScope - Return a SCEV expression for the specified value - /// at the specified scope in the program. The L value specifies a loop - /// nest to evaluate the expression at, where null is the top-level or a - /// specified loop is immediately inside of the loop. + /// Return a SCEV expression for the specified value at the specified scope + /// in the program. The L value specifies a loop nest to evaluate the + /// expression at, where null is the top-level or a specified loop is + /// immediately inside of the loop. /// /// This method can be used to compute the exit value for a variable defined /// in a loop by querying what the value will hold in the parent loop. @@ -758,19 +963,17 @@ namespace llvm { /// original value V is returned. const SCEV *getSCEVAtScope(const SCEV *S, const Loop *L); - /// getSCEVAtScope - This is a convenience function which does - /// getSCEVAtScope(getSCEV(V), L). + /// This is a convenience function which does getSCEVAtScope(getSCEV(V), L). const SCEV *getSCEVAtScope(Value *V, const Loop *L); - /// isLoopEntryGuardedByCond - Test whether entry to the loop is protected - /// by a conditional between LHS and RHS. This is used to help avoid max - /// expressions in loop trip counts, and to eliminate casts. + /// Test whether entry to the loop is protected by a conditional between LHS + /// and RHS. This is used to help avoid max expressions in loop trip + /// counts, and to eliminate casts. bool isLoopEntryGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS); - /// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is - /// protected by a conditional between LHS and RHS. This is used to - /// to eliminate casts. + /// Test whether the backedge of the loop is protected by a conditional + /// between LHS and RHS. This is used to to eliminate casts. bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS); @@ -781,13 +984,13 @@ namespace llvm { /// the single exiting block passed to it. See that routine for details. unsigned getSmallConstantTripCount(Loop *L); - /// getSmallConstantTripCount - Returns the maximum trip count of this loop - /// as a normal unsigned value. Returns 0 if the trip count is unknown or - /// not constant. This "trip count" assumes that control exits via - /// ExitingBlock. More precisely, it is the number of times that control may - /// reach ExitingBlock before taking the branch. For loops with multiple - /// exits, it may not be the number times that the loop header executes if - /// the loop exits prematurely via another branch. + /// Returns the maximum trip count of this loop as a normal unsigned + /// value. Returns 0 if the trip count is unknown or not constant. This + /// "trip count" assumes that control exits via ExitingBlock. More + /// precisely, it is the number of times that control may reach ExitingBlock + /// before taking the branch. For loops with multiple exits, it may not be + /// the number times that the loop header executes if the loop exits + /// prematurely via another branch. unsigned getSmallConstantTripCount(Loop *L, BasicBlock *ExitingBlock); /// \brief Returns the largest constant divisor of the trip count of the @@ -798,25 +1001,25 @@ namespace llvm { /// the single exiting block passed to it. See that routine for details. unsigned getSmallConstantTripMultiple(Loop *L); - /// getSmallConstantTripMultiple - Returns the largest constant divisor of - /// the trip count of this loop as a normal unsigned value, if - /// possible. This means that the actual trip count is always a multiple of - /// the returned value (don't forget the trip count could very well be zero - /// as well!). As explained in the comments for getSmallConstantTripCount, - /// this assumes that control exits the loop via ExitingBlock. + /// Returns the largest constant divisor of the trip count of this loop as a + /// normal unsigned value, if possible. This means that the actual trip + /// count is always a multiple of the returned value (don't forget the trip + /// count could very well be zero as well!). As explained in the comments + /// for getSmallConstantTripCount, this assumes that control exits the loop + /// via ExitingBlock. unsigned getSmallConstantTripMultiple(Loop *L, BasicBlock *ExitingBlock); - // getExitCount - Get the expression for the number of loop iterations for - // which this loop is guaranteed not to exit via ExitingBlock. Otherwise - // return SCEVCouldNotCompute. + /// Get the expression for the number of loop iterations for which this loop + /// is guaranteed not to exit via ExitingBlock. Otherwise return + /// SCEVCouldNotCompute. const SCEV *getExitCount(Loop *L, BasicBlock *ExitingBlock); - /// getBackedgeTakenCount - If the specified loop has a predictable - /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute - /// object. The backedge-taken count is the number of times the loop header - /// will be branched to from within the loop. This is one less than the - /// trip count of the loop, since it doesn't count the first iteration, - /// when the header is branched to from outside the loop. + /// If the specified loop has a predictable backedge-taken count, return it, + /// otherwise return a SCEVCouldNotCompute object. The backedge-taken count + /// is the number of times the loop header will be branched to from within + /// the loop. This is one less than the trip count of the loop, since it + /// doesn't count the first iteration, when the header is branched to from + /// outside the loop. /// /// Note that it is not valid to call this method on a loop without a /// loop-invariant backedge-taken count (see @@ -824,24 +1027,23 @@ namespace llvm { /// const SCEV *getBackedgeTakenCount(const Loop *L); - /// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except - /// return the least SCEV value that is known never to be less than the - /// actual backedge taken count. + /// Similar to getBackedgeTakenCount, except return the least SCEV value + /// that is known never to be less than the actual backedge taken count. const SCEV *getMaxBackedgeTakenCount(const Loop *L); - /// hasLoopInvariantBackedgeTakenCount - Return true if the specified loop - /// has an analyzable loop-invariant backedge-taken count. + /// Return true if the specified loop has an analyzable loop-invariant + /// backedge-taken count. bool hasLoopInvariantBackedgeTakenCount(const Loop *L); - /// forgetLoop - This method should be called by the client when it has - /// changed a loop in a way that may effect ScalarEvolution's ability to - /// compute a trip count, or if the loop is deleted. This call is - /// potentially expensive for large loop bodies. + /// This method should be called by the client when it has changed a loop in + /// a way that may effect ScalarEvolution's ability to compute a trip count, + /// or if the loop is deleted. This call is potentially expensive for large + /// loop bodies. void forgetLoop(const Loop *L); - /// forgetValue - This method should be called by the client when it has - /// changed a value in a way that may effect its value, or which may - /// disconnect it from a def-use chain linking it to a loop. + /// This method should be called by the client when it has changed a value + /// in a way that may effect its value, or which may disconnect it from a + /// def-use chain linking it to a loop. void forgetValue(Value *V); /// \brief Called when the client has changed the disposition of values in @@ -851,92 +1053,97 @@ namespace llvm { /// recompute is simpler. void forgetLoopDispositions(const Loop *L) { LoopDispositions.clear(); } - /// GetMinTrailingZeros - Determine the minimum number of zero bits that S - /// is guaranteed to end in (at every loop iteration). It is, at the same - /// time, the minimum number of times S is divisible by 2. For example, - /// given {4,+,8} it returns 2. If S is guaranteed to be 0, it returns the - /// bitwidth of S. + /// Determine the minimum number of zero bits that S is guaranteed to end in + /// (at every loop iteration). It is, at the same time, the minimum number + /// of times S is divisible by 2. For example, given {4,+,8} it returns 2. + /// If S is guaranteed to be 0, it returns the bitwidth of S. uint32_t GetMinTrailingZeros(const SCEV *S); - /// getUnsignedRange - Determine the unsigned range for a particular SCEV. + /// Determine the unsigned range for a particular SCEV. /// ConstantRange getUnsignedRange(const SCEV *S) { return getRange(S, HINT_RANGE_UNSIGNED); } - /// getSignedRange - Determine the signed range for a particular SCEV. + /// Determine the signed range for a particular SCEV. /// ConstantRange getSignedRange(const SCEV *S) { return getRange(S, HINT_RANGE_SIGNED); } - /// isKnownNegative - Test if the given expression is known to be negative. + /// Test if the given expression is known to be negative. /// bool isKnownNegative(const SCEV *S); - /// isKnownPositive - Test if the given expression is known to be positive. + /// Test if the given expression is known to be positive. /// bool isKnownPositive(const SCEV *S); - /// isKnownNonNegative - Test if the given expression is known to be - /// non-negative. + /// Test if the given expression is known to be non-negative. /// bool isKnownNonNegative(const SCEV *S); - /// isKnownNonPositive - Test if the given expression is known to be - /// non-positive. + /// Test if the given expression is known to be non-positive. /// bool isKnownNonPositive(const SCEV *S); - /// isKnownNonZero - Test if the given expression is known to be - /// non-zero. + /// Test if the given expression is known to be non-zero. /// bool isKnownNonZero(const SCEV *S); - /// isKnownPredicate - Test if the given expression is known to satisfy - /// the condition described by Pred, LHS, and RHS. + /// Test if the given expression is known to satisfy the condition described + /// by Pred, LHS, and RHS. /// bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS); - /// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with - /// predicate Pred. Return true iff any changes were made. If the - /// operands are provably equal or unequal, LHS and RHS are set to - /// the same value and Pred is set to either ICMP_EQ or ICMP_NE. + /// Return true if the result of the predicate LHS `Pred` RHS is loop + /// invariant with respect to L. Set InvariantPred, InvariantLHS and + /// InvariantLHS so that InvariantLHS `InvariantPred` InvariantRHS is the + /// loop invariant form of LHS `Pred` RHS. + bool isLoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS, const Loop *L, + ICmpInst::Predicate &InvariantPred, + const SCEV *&InvariantLHS, + const SCEV *&InvariantRHS); + + /// Simplify LHS and RHS in a comparison with predicate Pred. Return true + /// iff any changes were made. If the operands are provably equal or + /// unequal, LHS and RHS are set to the same value and Pred is set to either + /// ICMP_EQ or ICMP_NE. /// bool SimplifyICmpOperands(ICmpInst::Predicate &Pred, const SCEV *&LHS, const SCEV *&RHS, unsigned Depth = 0); - /// getLoopDisposition - Return the "disposition" of the given SCEV with - /// respect to the given loop. + /// Return the "disposition" of the given SCEV with respect to the given + /// loop. LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L); - /// isLoopInvariant - Return true if the value of the given SCEV is - /// unchanging in the specified loop. + /// Return true if the value of the given SCEV is unchanging in the + /// specified loop. bool isLoopInvariant(const SCEV *S, const Loop *L); - /// hasComputableLoopEvolution - Return true if the given SCEV changes value - /// in a known way in the specified loop. This property being true implies - /// that the value is variant in the loop AND that we can emit an expression - /// to compute the value of the expression at any particular loop iteration. + /// Return true if the given SCEV changes value in a known way in the + /// specified loop. This property being true implies that the value is + /// variant in the loop AND that we can emit an expression to compute the + /// value of the expression at any particular loop iteration. bool hasComputableLoopEvolution(const SCEV *S, const Loop *L); - /// getLoopDisposition - Return the "disposition" of the given SCEV with - /// respect to the given block. + /// Return the "disposition" of the given SCEV with respect to the given + /// block. BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB); - /// dominates - Return true if elements that makes up the given SCEV - /// dominate the specified basic block. + /// Return true if elements that makes up the given SCEV dominate the + /// specified basic block. bool dominates(const SCEV *S, const BasicBlock *BB); - /// properlyDominates - Return true if elements that makes up the given SCEV - /// properly dominate the specified basic block. + /// Return true if elements that makes up the given SCEV properly dominate + /// the specified basic block. bool properlyDominates(const SCEV *S, const BasicBlock *BB); - /// hasOperand - Test whether the given SCEV has Op as a direct or - /// indirect operand. + /// Test whether the given SCEV has Op as a direct or indirect operand. bool hasOperand(const SCEV *S, const SCEV *Op) const; /// Return the size of an element read or written by Inst. @@ -948,11 +1155,8 @@ namespace llvm { SmallVectorImpl<const SCEV *> &Sizes, const SCEV *ElementSize) const; - bool runOnFunction(Function &F) override; - void releaseMemory() override; - void getAnalysisUsage(AnalysisUsage &AU) const override; - void print(raw_ostream &OS, const Module* = nullptr) const override; - void verifyAnalysis() const override; + void print(raw_ostream &OS) const; + void verify() const; /// Collect parametric terms occurring in step expressions. void collectParametricTerms(const SCEV *Expr, @@ -1034,6 +1238,18 @@ namespace llvm { SmallVectorImpl<const SCEV *> &Sizes, const SCEV *ElementSize); + /// Return the DataLayout associated with the module this SCEV instance is + /// operating on. + const DataLayout &getDataLayout() const { + return F.getParent()->getDataLayout(); + } + + const SCEVPredicate *getEqualPredicate(const SCEVUnknown *LHS, + const SCEVConstant *RHS); + + /// Re-writes the SCEV according to the Predicates in \p Preds. + const SCEV *rewriteUsingPredicate(const SCEV *Scev, SCEVUnionPredicate &A); + private: /// Compute the backedge taken count knowing the interval difference, the /// stride and presence of the equality in the comparison. @@ -1054,13 +1270,112 @@ namespace llvm { private: FoldingSet<SCEV> UniqueSCEVs; + FoldingSet<SCEVPredicate> UniquePreds; BumpPtrAllocator SCEVAllocator; - /// FirstUnknown - The head of a linked list of all SCEVUnknown - /// values that have been allocated. This is used by releaseMemory - /// to locate them all and call their destructors. + /// The head of a linked list of all SCEVUnknown values that have been + /// allocated. This is used by releaseMemory to locate them all and call + /// their destructors. SCEVUnknown *FirstUnknown; }; + + /// \brief Analysis pass that exposes the \c ScalarEvolution for a function. + class ScalarEvolutionAnalysis { + static char PassID; + + public: + typedef ScalarEvolution Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + /// \brief Provide a name for the analysis for debugging and logging. + static StringRef name() { return "ScalarEvolutionAnalysis"; } + + ScalarEvolution run(Function &F, AnalysisManager<Function> *AM); + }; + + /// \brief Printer pass for the \c ScalarEvolutionAnalysis results. + class ScalarEvolutionPrinterPass { + raw_ostream &OS; + + public: + explicit ScalarEvolutionPrinterPass(raw_ostream &OS) : OS(OS) {} + PreservedAnalyses run(Function &F, AnalysisManager<Function> *AM); + + static StringRef name() { return "ScalarEvolutionPrinterPass"; } + }; + + class ScalarEvolutionWrapperPass : public FunctionPass { + std::unique_ptr<ScalarEvolution> SE; + + public: + static char ID; + + ScalarEvolutionWrapperPass(); + + ScalarEvolution &getSE() { return *SE; } + const ScalarEvolution &getSE() const { return *SE; } + + bool runOnFunction(Function &F) override; + void releaseMemory() override; + void getAnalysisUsage(AnalysisUsage &AU) const override; + void print(raw_ostream &OS, const Module * = nullptr) const override; + void verifyAnalysis() const override; + }; + + /// An interface layer with SCEV used to manage how we see SCEV expressions + /// for values in the context of existing predicates. We can add new + /// predicates, but we cannot remove them. + /// + /// This layer has multiple purposes: + /// - provides a simple interface for SCEV versioning. + /// - guarantees that the order of transformations applied on a SCEV + /// expression for a single Value is consistent across two different + /// getSCEV calls. This means that, for example, once we've obtained + /// an AddRec expression for a certain value through expression + /// rewriting, we will continue to get an AddRec expression for that + /// Value. + /// - lowers the number of expression rewrites. + class PredicatedScalarEvolution { + public: + PredicatedScalarEvolution(ScalarEvolution &SE); + const SCEVUnionPredicate &getUnionPredicate() const; + /// \brief Returns the SCEV expression of V, in the context of the current + /// SCEV predicate. + /// The order of transformations applied on the expression of V returned + /// by ScalarEvolution is guaranteed to be preserved, even when adding new + /// predicates. + const SCEV *getSCEV(Value *V); + /// \brief Adds a new predicate. + void addPredicate(const SCEVPredicate &Pred); + /// \brief Returns the ScalarEvolution analysis used. + ScalarEvolution *getSE() const { return &SE; } + + private: + /// \brief Increments the version number of the predicate. + /// This needs to be called every time the SCEV predicate changes. + void updateGeneration(); + /// Holds a SCEV and the version number of the SCEV predicate used to + /// perform the rewrite of the expression. + typedef std::pair<unsigned, const SCEV *> RewriteEntry; + /// Maps a SCEV to the rewrite result of that SCEV at a certain version + /// number. If this number doesn't match the current Generation, we will + /// need to do a rewrite. To preserve the transformation order of previous + /// rewrites, we will rewrite the previous result instead of the original + /// SCEV. + DenseMap<const SCEV *, RewriteEntry> RewriteMap; + /// The ScalarEvolution analysis. + ScalarEvolution &SE; + /// The SCEVPredicate that forms our context. We will rewrite all + /// expressions assuming that this predicate true. + SCEVUnionPredicate Preds; + /// Marks the version of the SCEV predicate used. When rewriting a SCEV + /// expression we mark it with the version of the predicate. We use this to + /// figure out if the predicate has changed from the last rewrite of the + /// SCEV. If so, we need to perform a new rewrite. + unsigned Generation; + }; } #endif diff --git a/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h b/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h new file mode 100644 index 000000000000..7bbbf5562047 --- /dev/null +++ b/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h @@ -0,0 +1,79 @@ +//===- ScalarEvolutionAliasAnalysis.h - SCEV-based AA -----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This is the interface for a SCEV-based alias analysis. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_SCALAREVOLUTIONALIASANALYSIS_H +#define LLVM_ANALYSIS_SCALAREVOLUTIONALIASANALYSIS_H + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +namespace llvm { + +/// A simple alias analysis implementation that uses ScalarEvolution to answer +/// queries. +class SCEVAAResult : public AAResultBase<SCEVAAResult> { + ScalarEvolution &SE; + +public: + explicit SCEVAAResult(const TargetLibraryInfo &TLI, ScalarEvolution &SE) + : AAResultBase(TLI), SE(SE) {} + SCEVAAResult(SCEVAAResult &&Arg) : AAResultBase(std::move(Arg)), SE(Arg.SE) {} + + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); + +private: + Value *GetBaseValue(const SCEV *S); +}; + +/// Analysis pass providing a never-invalidated alias analysis result. +class SCEVAA { +public: + typedef SCEVAAResult Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + SCEVAAResult run(Function &F, AnalysisManager<Function> *AM); + + /// \brief Provide access to a name for this pass for debugging purposes. + static StringRef name() { return "SCEVAA"; } + +private: + static char PassID; +}; + +/// Legacy wrapper pass to provide the SCEVAAResult object. +class SCEVAAWrapperPass : public FunctionPass { + std::unique_ptr<SCEVAAResult> Result; + +public: + static char ID; + + SCEVAAWrapperPass(); + + SCEVAAResult &getResult() { return *Result; } + const SCEVAAResult &getResult() const { return *Result; } + + bool runOnFunction(Function &F) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +/// Creates an instance of \c SCEVAAWrapperPass. +FunctionPass *createSCEVAAWrapperPass(); + +} + +#endif diff --git a/include/llvm/Analysis/ScalarEvolutionExpander.h b/include/llvm/Analysis/ScalarEvolutionExpander.h index 8ec2078258d1..b9939168a99d 100644 --- a/include/llvm/Analysis/ScalarEvolutionExpander.h +++ b/include/llvm/Analysis/ScalarEvolutionExpander.h @@ -117,9 +117,14 @@ namespace llvm { /// \brief Return true for expressions that may incur non-trivial cost to /// evaluate at runtime. - bool isHighCostExpansion(const SCEV *Expr, Loop *L) { + /// + /// At is an optional parameter which specifies point in code where user is + /// going to expand this expression. Sometimes this knowledge can lead to a + /// more accurate cost estimation. + bool isHighCostExpansion(const SCEV *Expr, Loop *L, + const Instruction *At = nullptr) { SmallPtrSet<const SCEV *, 8> Processed; - return isHighCostExpansionHelper(Expr, L, Processed); + return isHighCostExpansionHelper(Expr, L, At, Processed); } /// \brief This method returns the canonical induction variable of the @@ -146,6 +151,22 @@ namespace llvm { /// block. Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I); + /// \brief Generates a code sequence that evaluates this predicate. + /// The inserted instructions will be at position \p Loc. + /// The result will be of type i1 and will have a value of 0 when the + /// predicate is false and 1 otherwise. + Value *expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc); + + /// \brief A specialized variant of expandCodeForPredicate, handling the + /// case when we are expanding code for a SCEVEqualPredicate. + Value *expandEqualPredicate(const SCEVEqualPredicate *Pred, + Instruction *Loc); + + /// \brief A specialized variant of expandCodeForPredicate, handling the + /// case when we are expanding code for a SCEVUnionPredicate. + Value *expandUnionPredicate(const SCEVUnionPredicate *Pred, + Instruction *Loc); + /// \brief Set the current IV increment loop and position. void setIVIncInsertPos(const Loop *L, Instruction *Pos) { assert(!CanonicalMode && @@ -193,11 +214,22 @@ namespace llvm { void setChainedPhi(PHINode *PN) { ChainedPhis.insert(PN); } + /// \brief Try to find LLVM IR value for S available at the point At. + /// + /// L is a hint which tells in which loop to look for the suitable value. + /// On success return value which is equivalent to the expanded S at point + /// At. Return nullptr if value was not found. + /// + /// Note that this function does not perform an exhaustive search. I.e if it + /// didn't find any value it does not mean that there is no such value. + Value *findExistingExpansion(const SCEV *S, const Instruction *At, Loop *L); + private: LLVMContext &getContext() const { return SE.getContext(); } /// \brief Recursive helper function for isHighCostExpansion. bool isHighCostExpansionHelper(const SCEV *S, Loop *L, + const Instruction *At, SmallPtrSetImpl<const SCEV *> &Processed); /// \brief Insert the specified binary operator, doing a small amount diff --git a/include/llvm/Analysis/ScalarEvolutionExpressions.h b/include/llvm/Analysis/ScalarEvolutionExpressions.h index da24de281d47..16992680577c 100644 --- a/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -43,6 +43,7 @@ namespace llvm { SCEV(ID, scConstant), V(v) {} public: ConstantInt *getValue() const { return V; } + const APInt &getAPInt() const { return getValue()->getValue(); } Type *getType() const { return V->getType(); } @@ -404,7 +405,7 @@ namespace llvm { /// value, and only represent it as its LLVM Value. This is the "bottom" /// value for the analysis. /// - class SCEVUnknown : public SCEV, private CallbackVH { + class SCEVUnknown final : public SCEV, private CallbackVH { friend class ScalarEvolution; // Implement CallbackVH. @@ -553,64 +554,56 @@ namespace llvm { T.visitAll(Root); } - typedef DenseMap<const Value*, Value*> ValueToValueMap; - - /// The SCEVParameterRewriter takes a scalar evolution expression and updates - /// the SCEVUnknown components following the Map (Value -> Value). - struct SCEVParameterRewriter - : public SCEVVisitor<SCEVParameterRewriter, const SCEV*> { + /// Recursively visits a SCEV expression and re-writes it. + template<typename SC> + class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> { + protected: + ScalarEvolution &SE; public: - static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE, - ValueToValueMap &Map, - bool InterpretConsts = false) { - SCEVParameterRewriter Rewriter(SE, Map, InterpretConsts); - return Rewriter.visit(Scev); - } - - SCEVParameterRewriter(ScalarEvolution &S, ValueToValueMap &M, bool C) - : SE(S), Map(M), InterpretConsts(C) {} + SCEVRewriteVisitor(ScalarEvolution &SE) : SE(SE) {} const SCEV *visitConstant(const SCEVConstant *Constant) { return Constant; } const SCEV *visitTruncateExpr(const SCEVTruncateExpr *Expr) { - const SCEV *Operand = visit(Expr->getOperand()); + const SCEV *Operand = ((SC*)this)->visit(Expr->getOperand()); return SE.getTruncateExpr(Operand, Expr->getType()); } const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { - const SCEV *Operand = visit(Expr->getOperand()); + const SCEV *Operand = ((SC*)this)->visit(Expr->getOperand()); return SE.getZeroExtendExpr(Operand, Expr->getType()); } const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { - const SCEV *Operand = visit(Expr->getOperand()); + const SCEV *Operand = ((SC*)this)->visit(Expr->getOperand()); return SE.getSignExtendExpr(Operand, Expr->getType()); } const SCEV *visitAddExpr(const SCEVAddExpr *Expr) { SmallVector<const SCEV *, 2> Operands; for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); + Operands.push_back(((SC*)this)->visit(Expr->getOperand(i))); return SE.getAddExpr(Operands); } const SCEV *visitMulExpr(const SCEVMulExpr *Expr) { SmallVector<const SCEV *, 2> Operands; for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); + Operands.push_back(((SC*)this)->visit(Expr->getOperand(i))); return SE.getMulExpr(Operands); } const SCEV *visitUDivExpr(const SCEVUDivExpr *Expr) { - return SE.getUDivExpr(visit(Expr->getLHS()), visit(Expr->getRHS())); + return SE.getUDivExpr(((SC*)this)->visit(Expr->getLHS()), + ((SC*)this)->visit(Expr->getRHS())); } const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { SmallVector<const SCEV *, 2> Operands; for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); + Operands.push_back(((SC*)this)->visit(Expr->getOperand(i))); return SE.getAddRecExpr(Operands, Expr->getLoop(), Expr->getNoWrapFlags()); } @@ -618,18 +611,43 @@ namespace llvm { const SCEV *visitSMaxExpr(const SCEVSMaxExpr *Expr) { SmallVector<const SCEV *, 2> Operands; for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); + Operands.push_back(((SC*)this)->visit(Expr->getOperand(i))); return SE.getSMaxExpr(Operands); } const SCEV *visitUMaxExpr(const SCEVUMaxExpr *Expr) { SmallVector<const SCEV *, 2> Operands; for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); + Operands.push_back(((SC*)this)->visit(Expr->getOperand(i))); return SE.getUMaxExpr(Operands); } const SCEV *visitUnknown(const SCEVUnknown *Expr) { + return Expr; + } + + const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { + return Expr; + } + }; + + typedef DenseMap<const Value*, Value*> ValueToValueMap; + + /// The SCEVParameterRewriter takes a scalar evolution expression and updates + /// the SCEVUnknown components following the Map (Value -> Value). + class SCEVParameterRewriter : public SCEVRewriteVisitor<SCEVParameterRewriter> { + public: + static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE, + ValueToValueMap &Map, + bool InterpretConsts = false) { + SCEVParameterRewriter Rewriter(SE, Map, InterpretConsts); + return Rewriter.visit(Scev); + } + + SCEVParameterRewriter(ScalarEvolution &SE, ValueToValueMap &M, bool C) + : SCEVRewriteVisitor(SE), Map(M), InterpretConsts(C) {} + + const SCEV *visitUnknown(const SCEVUnknown *Expr) { Value *V = Expr->getValue(); if (Map.count(V)) { Value *NV = Map[V]; @@ -640,68 +658,26 @@ namespace llvm { return Expr; } - const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { - return Expr; - } - private: - ScalarEvolution &SE; ValueToValueMap ⤅ bool InterpretConsts; }; typedef DenseMap<const Loop*, const SCEV*> LoopToScevMapT; - /// The SCEVApplyRewriter takes a scalar evolution expression and applies + /// The SCEVLoopAddRecRewriter takes a scalar evolution expression and applies /// the Map (Loop -> SCEV) to all AddRecExprs. - struct SCEVApplyRewriter - : public SCEVVisitor<SCEVApplyRewriter, const SCEV*> { + class SCEVLoopAddRecRewriter + : public SCEVRewriteVisitor<SCEVLoopAddRecRewriter> { public: static const SCEV *rewrite(const SCEV *Scev, LoopToScevMapT &Map, ScalarEvolution &SE) { - SCEVApplyRewriter Rewriter(SE, Map); + SCEVLoopAddRecRewriter Rewriter(SE, Map); return Rewriter.visit(Scev); } - SCEVApplyRewriter(ScalarEvolution &S, LoopToScevMapT &M) - : SE(S), Map(M) {} - - const SCEV *visitConstant(const SCEVConstant *Constant) { - return Constant; - } - - const SCEV *visitTruncateExpr(const SCEVTruncateExpr *Expr) { - const SCEV *Operand = visit(Expr->getOperand()); - return SE.getTruncateExpr(Operand, Expr->getType()); - } - - const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { - const SCEV *Operand = visit(Expr->getOperand()); - return SE.getZeroExtendExpr(Operand, Expr->getType()); - } - - const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { - const SCEV *Operand = visit(Expr->getOperand()); - return SE.getSignExtendExpr(Operand, Expr->getType()); - } - - const SCEV *visitAddExpr(const SCEVAddExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); - return SE.getAddExpr(Operands); - } - - const SCEV *visitMulExpr(const SCEVMulExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); - return SE.getMulExpr(Operands); - } - - const SCEV *visitUDivExpr(const SCEVUDivExpr *Expr) { - return SE.getUDivExpr(visit(Expr->getLHS()), visit(Expr->getRHS())); - } + SCEVLoopAddRecRewriter(ScalarEvolution &SE, LoopToScevMapT &M) + : SCEVRewriteVisitor(SE), Map(M) {} const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { SmallVector<const SCEV *, 2> Operands; @@ -714,41 +690,18 @@ namespace llvm { if (0 == Map.count(L)) return Res; - const SCEVAddRecExpr *Rec = (const SCEVAddRecExpr *) Res; + const SCEVAddRecExpr *Rec = cast<SCEVAddRecExpr>(Res); return Rec->evaluateAtIteration(Map[L], SE); } - const SCEV *visitSMaxExpr(const SCEVSMaxExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); - return SE.getSMaxExpr(Operands); - } - - const SCEV *visitUMaxExpr(const SCEVUMaxExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) - Operands.push_back(visit(Expr->getOperand(i))); - return SE.getUMaxExpr(Operands); - } - - const SCEV *visitUnknown(const SCEVUnknown *Expr) { - return Expr; - } - - const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { - return Expr; - } - private: - ScalarEvolution &SE; LoopToScevMapT ⤅ }; /// Applies the Map (Loop -> SCEV) to the given Scev. static inline const SCEV *apply(const SCEV *Scev, LoopToScevMapT &Map, ScalarEvolution &SE) { - return SCEVApplyRewriter::rewrite(Scev, Map, SE); + return SCEVLoopAddRecRewriter::rewrite(Scev, Map, SE); } } diff --git a/include/llvm/Analysis/ScopedNoAliasAA.h b/include/llvm/Analysis/ScopedNoAliasAA.h new file mode 100644 index 000000000000..175561687157 --- /dev/null +++ b/include/llvm/Analysis/ScopedNoAliasAA.h @@ -0,0 +1,92 @@ +//===- ScopedNoAliasAA.h - Scoped No-Alias Alias Analysis -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This is the interface for a metadata-based scoped no-alias analysis. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_SCOPEDNOALIASAA_H +#define LLVM_ANALYSIS_SCOPEDNOALIASAA_H + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +namespace llvm { + +/// A simple AA result which uses scoped-noalias metadata to answer queries. +class ScopedNoAliasAAResult : public AAResultBase<ScopedNoAliasAAResult> { + friend AAResultBase<ScopedNoAliasAAResult>; + +public: + explicit ScopedNoAliasAAResult(const TargetLibraryInfo &TLI) + : AAResultBase(TLI) {} + ScopedNoAliasAAResult(ScopedNoAliasAAResult &&Arg) + : AAResultBase(std::move(Arg)) {} + + /// Handle invalidation events from the new pass manager. + /// + /// By definition, this result is stateless and so remains valid. + bool invalidate(Function &, const PreservedAnalyses &) { return false; } + + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); + ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); + +private: + bool mayAliasInScopes(const MDNode *Scopes, const MDNode *NoAlias) const; + void collectMDInDomain(const MDNode *List, const MDNode *Domain, + SmallPtrSetImpl<const MDNode *> &Nodes) const; +}; + +/// Analysis pass providing a never-invalidated alias analysis result. +class ScopedNoAliasAA { +public: + typedef ScopedNoAliasAAResult Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + ScopedNoAliasAAResult run(Function &F, AnalysisManager<Function> *AM); + + /// \brief Provide access to a name for this pass for debugging purposes. + static StringRef name() { return "ScopedNoAliasAA"; } + +private: + static char PassID; +}; + +/// Legacy wrapper pass to provide the ScopedNoAliasAAResult object. +class ScopedNoAliasAAWrapperPass : public ImmutablePass { + std::unique_ptr<ScopedNoAliasAAResult> Result; + +public: + static char ID; + + ScopedNoAliasAAWrapperPass(); + + ScopedNoAliasAAResult &getResult() { return *Result; } + const ScopedNoAliasAAResult &getResult() const { return *Result; } + + bool doInitialization(Module &M) override; + bool doFinalization(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +//===--------------------------------------------------------------------===// +// +// createScopedNoAliasAAWrapperPass - This pass implements metadata-based +// scoped noalias analysis. +// +ImmutablePass *createScopedNoAliasAAWrapperPass(); +} + +#endif diff --git a/include/llvm/Analysis/SparsePropagation.h b/include/llvm/Analysis/SparsePropagation.h index 9ccae5ff89b7..2c7f5dd73547 100644 --- a/include/llvm/Analysis/SparsePropagation.h +++ b/include/llvm/Analysis/SparsePropagation.h @@ -21,19 +21,19 @@ #include <vector> namespace llvm { - class Value; - class Constant; - class Argument; - class Instruction; - class PHINode; - class TerminatorInst; - class BasicBlock; - class Function; - class SparseSolver; - class raw_ostream; - - template<typename T> class SmallVectorImpl; - +class Value; +class Constant; +class Argument; +class Instruction; +class PHINode; +class TerminatorInst; +class BasicBlock; +class Function; +class SparseSolver; +class raw_ostream; + +template <typename T> class SmallVectorImpl; + /// AbstractLatticeFunction - This class is implemented by the dataflow instance /// to specify what the lattice values are and how they handle merges etc. /// This gives the client the power to compute lattice values from instructions, @@ -44,8 +44,10 @@ namespace llvm { class AbstractLatticeFunction { public: typedef void *LatticeVal; + private: LatticeVal UndefVal, OverdefinedVal, UntrackedVal; + public: AbstractLatticeFunction(LatticeVal undefVal, LatticeVal overdefinedVal, LatticeVal untrackedVal) { @@ -54,18 +56,16 @@ public: UntrackedVal = untrackedVal; } virtual ~AbstractLatticeFunction(); - + LatticeVal getUndefVal() const { return UndefVal; } LatticeVal getOverdefinedVal() const { return OverdefinedVal; } LatticeVal getUntrackedVal() const { return UntrackedVal; } - + /// IsUntrackedValue - If the specified Value is something that is obviously /// uninteresting to the analysis (and would always return UntrackedVal), /// this function can return true to avoid pointless work. - virtual bool IsUntrackedValue(Value *V) { - return false; - } - + virtual bool IsUntrackedValue(Value *V) { return false; } + /// ComputeConstant - Given a constant value, compute and return a lattice /// value corresponding to the specified constant. virtual LatticeVal ComputeConstant(Constant *C) { @@ -74,10 +74,8 @@ public: /// IsSpecialCasedPHI - Given a PHI node, determine whether this PHI node is /// one that the we want to handle through ComputeInstructionState. - virtual bool IsSpecialCasedPHI(PHINode *PN) { - return false; - } - + virtual bool IsSpecialCasedPHI(PHINode *PN) { return false; } + /// GetConstant - If the specified lattice value is representable as an LLVM /// constant value, return it. Otherwise return null. The returned value /// must be in the same LLVM type as Val. @@ -90,42 +88,41 @@ public: virtual LatticeVal ComputeArgument(Argument *I) { return getOverdefinedVal(); // always safe } - + /// MergeValues - Compute and return the merge of the two specified lattice /// values. Merging should only move one direction down the lattice to /// guarantee convergence (toward overdefined). virtual LatticeVal MergeValues(LatticeVal X, LatticeVal Y) { return getOverdefinedVal(); // always safe, never useful. } - + /// ComputeInstructionState - Given an instruction and a vector of its operand /// values, compute the result value of the instruction. virtual LatticeVal ComputeInstructionState(Instruction &I, SparseSolver &SS) { return getOverdefinedVal(); // always safe, never useful. } - + /// PrintValue - Render the specified lattice value to the specified stream. virtual void PrintValue(LatticeVal V, raw_ostream &OS); }; - /// SparseSolver - This class is a general purpose solver for Sparse Conditional /// Propagation with a programmable lattice function. /// class SparseSolver { typedef AbstractLatticeFunction::LatticeVal LatticeVal; - + /// LatticeFunc - This is the object that knows the lattice and how to do /// compute transfer functions. AbstractLatticeFunction *LatticeFunc; - - DenseMap<Value*, LatticeVal> ValueState; // The state each value is in. - SmallPtrSet<BasicBlock*, 16> BBExecutable; // The bbs that are executable. - - std::vector<Instruction*> InstWorkList; // Worklist of insts to process. - - std::vector<BasicBlock*> BBWorkList; // The BasicBlock work list - + + DenseMap<Value *, LatticeVal> ValueState; // The state each value is in. + SmallPtrSet<BasicBlock *, 16> BBExecutable; // The bbs that are executable. + + std::vector<Instruction *> InstWorkList; // Worklist of insts to process. + + std::vector<BasicBlock *> BBWorkList; // The BasicBlock work list + /// KnownFeasibleEdges - Entries in this set are edges which have already had /// PHI nodes retriggered. typedef std::pair<BasicBlock*,BasicBlock*> Edge; @@ -133,17 +130,16 @@ class SparseSolver { SparseSolver(const SparseSolver&) = delete; void operator=(const SparseSolver&) = delete; + public: explicit SparseSolver(AbstractLatticeFunction *Lattice) - : LatticeFunc(Lattice) {} - ~SparseSolver() { - delete LatticeFunc; - } - + : LatticeFunc(Lattice) {} + ~SparseSolver() { delete LatticeFunc; } + /// Solve - Solve for constants and executable blocks. /// void Solve(Function &F); - + void Print(Function &F, raw_ostream &OS) const; /// getLatticeState - Return the LatticeVal object that corresponds to the @@ -153,7 +149,7 @@ public: DenseMap<Value*, LatticeVal>::const_iterator I = ValueState.find(V); return I != ValueState.end() ? I->second : LatticeFunc->getUntrackedVal(); } - + /// getOrInitValueState - Return the LatticeVal object that corresponds to the /// value, initializing the value's state if it hasn't been entered into the /// map yet. This function is necessary because not all values should start @@ -161,7 +157,7 @@ public: /// constants should be marked as constants. /// LatticeVal getOrInitValueState(Value *V); - + /// isEdgeFeasible - Return true if the control flow edge from the 'From' /// basic block to the 'To' basic block is currently feasible. If /// AggressiveUndef is true, then this treats values with unknown lattice @@ -176,29 +172,28 @@ public: bool isBlockExecutable(BasicBlock *BB) const { return BBExecutable.count(BB); } - + private: /// UpdateState - When the state for some instruction is potentially updated, /// this function notices and adds I to the worklist if needed. void UpdateState(Instruction &Inst, LatticeVal V); - + /// MarkBlockExecutable - This method can be used by clients to mark all of /// the blocks that are known to be intrinsically live in the processed unit. void MarkBlockExecutable(BasicBlock *BB); - + /// markEdgeExecutable - Mark a basic block as executable, adding it to the BB /// work list if it is not already executable. void markEdgeExecutable(BasicBlock *Source, BasicBlock *Dest); - + /// getFeasibleSuccessors - Return a vector of booleans to indicate which /// successors are reachable from a given terminator instruction. void getFeasibleSuccessors(TerminatorInst &TI, SmallVectorImpl<bool> &Succs, bool AggressiveUndef); - + void visitInst(Instruction &I); void visitPHINode(PHINode &I); void visitTerminatorInst(TerminatorInst &TI); - }; } // end namespace llvm diff --git a/include/llvm/Analysis/TargetLibraryInfo.def b/include/llvm/Analysis/TargetLibraryInfo.def index 1c1fdfef980d..7798e3c88248 100644 --- a/include/llvm/Analysis/TargetLibraryInfo.def +++ b/include/llvm/Analysis/TargetLibraryInfo.def @@ -27,6 +27,86 @@ #define TLI_DEFINE_STRING_INTERNAL(string_repr) string_repr, #endif +/// void *new(unsigned int); +TLI_DEFINE_ENUM_INTERNAL(msvc_new_int) +TLI_DEFINE_STRING_INTERNAL("??2@YAPAXI@Z") + +/// void *new(unsigned int, nothrow); +TLI_DEFINE_ENUM_INTERNAL(msvc_new_int_nothrow) +TLI_DEFINE_STRING_INTERNAL("??2@YAPAXIABUnothrow_t@std@@@Z") + +/// void *new(unsigned long long); +TLI_DEFINE_ENUM_INTERNAL(msvc_new_longlong) +TLI_DEFINE_STRING_INTERNAL("??2@YAPEAX_K@Z") + +/// void *new(unsigned long long, nothrow); +TLI_DEFINE_ENUM_INTERNAL(msvc_new_longlong_nothrow) +TLI_DEFINE_STRING_INTERNAL("??2@YAPEAX_KAEBUnothrow_t@std@@@Z") + +/// void operator delete(void*); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_ptr32) +TLI_DEFINE_STRING_INTERNAL("??3@YAXPAX@Z") + +/// void operator delete(void*, nothrow); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_ptr32_nothrow) +TLI_DEFINE_STRING_INTERNAL("??3@YAXPAXABUnothrow_t@std@@@Z") + +/// void operator delete(void*, unsigned int); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_ptr32_int) +TLI_DEFINE_STRING_INTERNAL("??3@YAXPAXI@Z") + +/// void operator delete(void*); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_ptr64) +TLI_DEFINE_STRING_INTERNAL("??3@YAXPEAX@Z") + +/// void operator delete(void*, nothrow); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_ptr64_nothrow) +TLI_DEFINE_STRING_INTERNAL("??3@YAXPEAXAEBUnothrow_t@std@@@Z") + +/// void operator delete(void*, unsigned long long); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_ptr64_longlong) +TLI_DEFINE_STRING_INTERNAL("??3@YAXPEAX_K@Z") + +/// void *new[](unsigned int); +TLI_DEFINE_ENUM_INTERNAL(msvc_new_array_int) +TLI_DEFINE_STRING_INTERNAL("??_U@YAPAXI@Z") + +/// void *new[](unsigned int, nothrow); +TLI_DEFINE_ENUM_INTERNAL(msvc_new_array_int_nothrow) +TLI_DEFINE_STRING_INTERNAL("??_U@YAPAXIABUnothrow_t@std@@@Z") + +/// void *new[](unsigned long long); +TLI_DEFINE_ENUM_INTERNAL(msvc_new_array_longlong) +TLI_DEFINE_STRING_INTERNAL("??_U@YAPEAX_K@Z") + +/// void *new[](unsigned long long, nothrow); +TLI_DEFINE_ENUM_INTERNAL(msvc_new_array_longlong_nothrow) +TLI_DEFINE_STRING_INTERNAL("??_U@YAPEAX_KAEBUnothrow_t@std@@@Z") + +/// void operator delete[](void*); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_array_ptr32) +TLI_DEFINE_STRING_INTERNAL("??_V@YAXPAX@Z") + +/// void operator delete[](void*, nothrow); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_array_ptr32_nothrow) +TLI_DEFINE_STRING_INTERNAL("??_V@YAXPAXABUnothrow_t@std@@@Z") + +/// void operator delete[](void*, unsigned int); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_array_ptr32_int) +TLI_DEFINE_STRING_INTERNAL("??_V@YAXPAXI@Z") + +/// void operator delete[](void*); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_array_ptr64) +TLI_DEFINE_STRING_INTERNAL("??_V@YAXPEAX@Z") + +/// void operator delete[](void*, nothrow); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_array_ptr64_nothrow) +TLI_DEFINE_STRING_INTERNAL("??_V@YAXPEAXAEBUnothrow_t@std@@@Z") + +/// void operator delete[](void*, unsigned long long); +TLI_DEFINE_ENUM_INTERNAL(msvc_delete_array_ptr64_longlong) +TLI_DEFINE_STRING_INTERNAL("??_V@YAXPEAX_K@Z") + /// int _IO_getc(_IO_FILE * __fp); TLI_DEFINE_ENUM_INTERNAL(under_IO_getc) TLI_DEFINE_STRING_INTERNAL("_IO_getc") @@ -406,6 +486,15 @@ TLI_DEFINE_STRING_INTERNAL("floorf") /// long double floorl(long double x); TLI_DEFINE_ENUM_INTERNAL(floorl) TLI_DEFINE_STRING_INTERNAL("floorl") +/// int fls(int i); +TLI_DEFINE_ENUM_INTERNAL(fls) +TLI_DEFINE_STRING_INTERNAL("fls") +/// int flsl(long int i); +TLI_DEFINE_ENUM_INTERNAL(flsl) +TLI_DEFINE_STRING_INTERNAL("flsl") +/// int flsll(long long int i); +TLI_DEFINE_ENUM_INTERNAL(flsll) +TLI_DEFINE_STRING_INTERNAL("flsll") /// double fmax(double x, double y); TLI_DEFINE_ENUM_INTERNAL(fmax) TLI_DEFINE_STRING_INTERNAL("fmax") @@ -664,6 +753,7 @@ TLI_DEFINE_STRING_INTERNAL("modff") /// long double modfl(long double value, long double *iptr); TLI_DEFINE_ENUM_INTERNAL(modfl) TLI_DEFINE_STRING_INTERNAL("modfl") + /// double nearbyint(double x); TLI_DEFINE_ENUM_INTERNAL(nearbyint) TLI_DEFINE_STRING_INTERNAL("nearbyint") diff --git a/include/llvm/Analysis/TargetLibraryInfo.h b/include/llvm/Analysis/TargetLibraryInfo.h index e0a1ee378274..7becdf033dd2 100644 --- a/include/llvm/Analysis/TargetLibraryInfo.h +++ b/include/llvm/Analysis/TargetLibraryInfo.h @@ -42,7 +42,7 @@ class PreservedAnalyses; /// /// This class constructs tables that hold the target library information and /// make it available. However, it is somewhat expensive to compute and only -/// depends on the triple. So users typicaly interact with the \c +/// depends on the triple. So users typically interact with the \c /// TargetLibraryInfo wrapper below. class TargetLibraryInfoImpl { friend class TargetLibraryInfo; @@ -201,13 +201,13 @@ public: } bool isFunctionVectorizable(StringRef F, unsigned VF) const { return Impl->isFunctionVectorizable(F, VF); - }; + } bool isFunctionVectorizable(StringRef F) const { return Impl->isFunctionVectorizable(F); - }; + } StringRef getVectorizedFunction(StringRef F, unsigned VF) const { return Impl->getVectorizedFunction(F, VF); - }; + } /// \brief Tests if the function is both available and a candidate for /// optimized code generation. diff --git a/include/llvm/Analysis/TargetTransformInfo.h b/include/llvm/Analysis/TargetTransformInfo.h index 01f00896410e..3913cc3f107c 100644 --- a/include/llvm/Analysis/TargetTransformInfo.h +++ b/include/llvm/Analysis/TargetTransformInfo.h @@ -42,11 +42,13 @@ class Value; /// \brief Information about a load/store intrinsic defined by the target. struct MemIntrinsicInfo { MemIntrinsicInfo() - : ReadMem(false), WriteMem(false), Vol(false), MatchingId(0), + : ReadMem(false), WriteMem(false), IsSimple(false), MatchingId(0), NumMemRefs(0), PtrVal(nullptr) {} bool ReadMem; bool WriteMem; - bool Vol; + /// True only if this memory operation is non-volatile, non-atomic, and + /// unordered. (See LoadInst/StoreInst for details on each) + bool IsSimple; // Same Id is set by the target for corresponding load/store intrinsics. unsigned short MatchingId; int NumMemRefs; @@ -97,11 +99,14 @@ public: /// /// Many APIs in this interface return a cost. This enum defines the /// fundamental values that should be used to interpret (and produce) those - /// costs. The costs are returned as an unsigned rather than a member of this + /// costs. The costs are returned as an int rather than a member of this /// enumeration because it is expected that the cost of one IR instruction /// may have a multiplicative factor to it or otherwise won't fit directly /// into the enum. Moreover, it is common to sum or average costs which works /// better as simple integral values. Thus this enum only provides constants. + /// Also note that the returned costs are signed integers to make it natural + /// to add, subtract, and test with zero (a common boundary condition). It is + /// not expected that 2^32 is a realistic cost to be modeling at any point. /// /// Note that these costs should usually reflect the intersection of code-size /// cost and execution cost. A free instruction is typically one that folds @@ -128,15 +133,15 @@ public: /// /// The returned cost is defined in terms of \c TargetCostConstants, see its /// comments for a detailed explanation of the cost values. - unsigned getOperationCost(unsigned Opcode, Type *Ty, - Type *OpTy = nullptr) const; + int getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy = nullptr) const; /// \brief Estimate the cost of a GEP operation when lowered. /// /// The contract for this function is the same as \c getOperationCost except /// that it supports an interface that provides extra information specific to /// the GEP operation. - unsigned getGEPCost(const Value *Ptr, ArrayRef<const Value *> Operands) const; + int getGEPCost(Type *PointeeType, const Value *Ptr, + ArrayRef<const Value *> Operands) const; /// \brief Estimate the cost of a function call when lowered. /// @@ -147,31 +152,30 @@ public: /// This is the most basic query for estimating call cost: it only knows the /// function type and (potentially) the number of arguments at the call site. /// The latter is only interesting for varargs function types. - unsigned getCallCost(FunctionType *FTy, int NumArgs = -1) const; + int getCallCost(FunctionType *FTy, int NumArgs = -1) const; /// \brief Estimate the cost of calling a specific function when lowered. /// /// This overload adds the ability to reason about the particular function /// being called in the event it is a library call with special lowering. - unsigned getCallCost(const Function *F, int NumArgs = -1) const; + int getCallCost(const Function *F, int NumArgs = -1) const; /// \brief Estimate the cost of calling a specific function when lowered. /// /// This overload allows specifying a set of candidate argument values. - unsigned getCallCost(const Function *F, - ArrayRef<const Value *> Arguments) const; + int getCallCost(const Function *F, ArrayRef<const Value *> Arguments) const; /// \brief Estimate the cost of an intrinsic when lowered. /// /// Mirrors the \c getCallCost method but uses an intrinsic identifier. - unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef<Type *> ParamTys) const; + int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Type *> ParamTys) const; /// \brief Estimate the cost of an intrinsic when lowered. /// /// Mirrors the \c getCallCost method but uses an intrinsic identifier. - unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef<const Value *> Arguments) const; + int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<const Value *> Arguments) const; /// \brief Estimate the cost of a given IR user when lowered. /// @@ -188,7 +192,7 @@ public: /// /// The returned cost is defined in terms of \c TargetCostConstants, see its /// comments for a detailed explanation of the cost values. - unsigned getUserCost(const User *U) const; + int getUserCost(const User *U) const; /// \brief Return true if branch divergence exists. /// @@ -308,12 +312,17 @@ public: bool HasBaseReg, int64_t Scale, unsigned AddrSpace = 0) const; - /// \brief Return true if the target works with masked instruction - /// AVX2 allows masks for consecutive load and store for i32 and i64 elements. - /// AVX-512 architecture will also allow masks for non-consecutive memory - /// accesses. - bool isLegalMaskedStore(Type *DataType, int Consecutive) const; - bool isLegalMaskedLoad(Type *DataType, int Consecutive) const; + /// \brief Return true if the target supports masked load/store + /// AVX2 and AVX-512 targets allow masks for consecutive load and store for + /// 32 and 64 bit elements. + bool isLegalMaskedStore(Type *DataType) const; + bool isLegalMaskedLoad(Type *DataType) const; + + /// \brief Return true if the target supports masked gather/scatter + /// AVX-512 fully supports gather and scatter for vectors with 32 and 64 + /// bits scalar type. + bool isLegalMaskedScatter(Type *DataType) const; + bool isLegalMaskedGather(Type *DataType) const; /// \brief Return the cost of the scaling factor used in the addressing /// mode represented by AM for this target, for a load/store @@ -350,6 +359,9 @@ public: /// \brief Don't restrict interleaved unrolling to small loops. bool enableAggressiveInterleaving(bool LoopHasReductions) const; + /// \brief Enable matching of interleaved access groups. + bool enableInterleavedAccessVectorization() const; + /// \brief Return hardware support for population count. PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) const; @@ -358,19 +370,19 @@ public: /// \brief Return the expected cost of supporting the floating point operation /// of the specified type. - unsigned getFPOpCost(Type *Ty) const; + int getFPOpCost(Type *Ty) const; /// \brief Return the expected cost of materializing for the given integer /// immediate of the specified type. - unsigned getIntImmCost(const APInt &Imm, Type *Ty) const; + int getIntImmCost(const APInt &Imm, Type *Ty) const; /// \brief Return the expected cost of materialization for the given integer /// immediate of the specified type for a given instruction. The cost can be /// zero if the immediate can be folded into the specified instruction. - unsigned getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, - Type *Ty) const; - unsigned getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, - Type *Ty) const; + int getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, + Type *Ty) const; + int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, + Type *Ty) const; /// @} /// \name Vector Target Information @@ -410,43 +422,51 @@ public: unsigned getMaxInterleaveFactor(unsigned VF) const; /// \return The expected cost of arithmetic ops, such as mul, xor, fsub, etc. - unsigned - getArithmeticInstrCost(unsigned Opcode, Type *Ty, - OperandValueKind Opd1Info = OK_AnyValue, - OperandValueKind Opd2Info = OK_AnyValue, - OperandValueProperties Opd1PropInfo = OP_None, - OperandValueProperties Opd2PropInfo = OP_None) const; + int getArithmeticInstrCost( + unsigned Opcode, Type *Ty, OperandValueKind Opd1Info = OK_AnyValue, + OperandValueKind Opd2Info = OK_AnyValue, + OperandValueProperties Opd1PropInfo = OP_None, + OperandValueProperties Opd2PropInfo = OP_None) const; /// \return The cost of a shuffle instruction of kind Kind and of type Tp. /// The index and subtype parameters are used by the subvector insertion and /// extraction shuffle kinds. - unsigned getShuffleCost(ShuffleKind Kind, Type *Tp, int Index = 0, - Type *SubTp = nullptr) const; + int getShuffleCost(ShuffleKind Kind, Type *Tp, int Index = 0, + Type *SubTp = nullptr) const; /// \return The expected cost of cast instructions, such as bitcast, trunc, /// zext, etc. - unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) const; + int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) const; /// \return The expected cost of control-flow related instructions such as /// Phi, Ret, Br. - unsigned getCFInstrCost(unsigned Opcode) const; + int getCFInstrCost(unsigned Opcode) const; /// \returns The expected cost of compare and select instructions. - unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, - Type *CondTy = nullptr) const; + int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, + Type *CondTy = nullptr) const; /// \return The expected cost of vector Insert and Extract. /// Use -1 to indicate that there is no information on the index value. - unsigned getVectorInstrCost(unsigned Opcode, Type *Val, - unsigned Index = -1) const; + int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index = -1) const; /// \return The cost of Load and Store instructions. - unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, - unsigned AddressSpace) const; + int getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + unsigned AddressSpace) const; /// \return The cost of masked Load and Store instructions. - unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, - unsigned AddressSpace) const; + int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + unsigned AddressSpace) const; + + /// \return The cost of Gather or Scatter operation + /// \p Opcode - is a type of memory access Load or Store + /// \p DataTy - a vector type of the data to be loaded or stored + /// \p Ptr - pointer [or vector of pointers] - address[es] in memory + /// \p VariableMask - true when the memory access is predicated with a mask + /// that is not a compile-time constant + /// \p Alignment - alignment of single element + int getGatherScatterOpCost(unsigned Opcode, Type *DataTy, Value *Ptr, + bool VariableMask, unsigned Alignment) const; /// \return The cost of the interleaved memory operation. /// \p Opcode is the memory operation code @@ -456,11 +476,9 @@ public: /// load allows gaps) /// \p Alignment is the alignment of the memory operation /// \p AddressSpace is address space of the pointer. - unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, - unsigned Factor, - ArrayRef<unsigned> Indices, - unsigned Alignment, - unsigned AddressSpace) const; + int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, + ArrayRef<unsigned> Indices, unsigned Alignment, + unsigned AddressSpace) const; /// \brief Calculate the cost of performing a vector reduction. /// @@ -475,16 +493,18 @@ public: /// Split: /// (v0, v1, v2, v3) /// ((v0+v2), (v1+v3), undef, undef) - unsigned getReductionCost(unsigned Opcode, Type *Ty, - bool IsPairwiseForm) const; + int getReductionCost(unsigned Opcode, Type *Ty, bool IsPairwiseForm) const; + + /// \returns The cost of Intrinsic instructions. Types analysis only. + int getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<Type *> Tys) const; - /// \returns The cost of Intrinsic instructions. - unsigned getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, - ArrayRef<Type *> Tys) const; + /// \returns The cost of Intrinsic instructions. Analyses the real arguments. + int getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<Value *> Args) const; /// \returns The cost of Call instructions. - unsigned getCallInstrCost(Function *F, Type *RetTy, - ArrayRef<Type *> Tys) const; + int getCallInstrCost(Function *F, Type *RetTy, ArrayRef<Type *> Tys) const; /// \returns The number of pieces into which the provided type must be /// split during legalization. Zero is returned when the answer is unknown. @@ -497,7 +517,7 @@ public: /// The 'IsComplex' parameter is a hint that the address computation is likely /// to involve multiple instructions and as such unlikely to be merged into /// the address indexing mode. - unsigned getAddressComputationCost(Type *Ty, bool IsComplex = false) const; + int getAddressComputationCost(Type *Ty, bool IsComplex = false) const; /// \returns The cost, if any, of keeping values of the given types alive /// over a callsite. @@ -521,8 +541,8 @@ public: /// \returns True if the two functions have compatible attributes for inlining /// purposes. - bool hasCompatibleFunctionAttributes(const Function *Caller, - const Function *Callee) const; + bool areInlineCompatible(const Function *Caller, + const Function *Callee) const; /// @} @@ -542,18 +562,18 @@ class TargetTransformInfo::Concept { public: virtual ~Concept() = 0; virtual const DataLayout &getDataLayout() const = 0; - virtual unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) = 0; - virtual unsigned getGEPCost(const Value *Ptr, - ArrayRef<const Value *> Operands) = 0; - virtual unsigned getCallCost(FunctionType *FTy, int NumArgs) = 0; - virtual unsigned getCallCost(const Function *F, int NumArgs) = 0; - virtual unsigned getCallCost(const Function *F, + virtual int getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) = 0; + virtual int getGEPCost(Type *PointeeType, const Value *Ptr, + ArrayRef<const Value *> Operands) = 0; + virtual int getCallCost(FunctionType *FTy, int NumArgs) = 0; + virtual int getCallCost(const Function *F, int NumArgs) = 0; + virtual int getCallCost(const Function *F, + ArrayRef<const Value *> Arguments) = 0; + virtual int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Type *> ParamTys) = 0; + virtual int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, ArrayRef<const Value *> Arguments) = 0; - virtual unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef<Type *> ParamTys) = 0; - virtual unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef<const Value *> Arguments) = 0; - virtual unsigned getUserCost(const User *U) = 0; + virtual int getUserCost(const User *U) = 0; virtual bool hasBranchDivergence() = 0; virtual bool isSourceOfDivergence(const Value *V) = 0; virtual bool isLoweredToCall(const Function *F) = 0; @@ -564,8 +584,10 @@ public: int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) = 0; - virtual bool isLegalMaskedStore(Type *DataType, int Consecutive) = 0; - virtual bool isLegalMaskedLoad(Type *DataType, int Consecutive) = 0; + virtual bool isLegalMaskedStore(Type *DataType) = 0; + virtual bool isLegalMaskedLoad(Type *DataType) = 0; + virtual bool isLegalMaskedScatter(Type *DataType) = 0; + virtual bool isLegalMaskedGather(Type *DataType) = 0; virtual int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) = 0; @@ -576,14 +598,15 @@ public: virtual unsigned getJumpBufSize() = 0; virtual bool shouldBuildLookupTables() = 0; virtual bool enableAggressiveInterleaving(bool LoopHasReductions) = 0; + virtual bool enableInterleavedAccessVectorization() = 0; virtual PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) = 0; virtual bool haveFastSqrt(Type *Ty) = 0; - virtual unsigned getFPOpCost(Type *Ty) = 0; - virtual unsigned getIntImmCost(const APInt &Imm, Type *Ty) = 0; - virtual unsigned getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, - Type *Ty) = 0; - virtual unsigned getIntImmCost(Intrinsic::ID IID, unsigned Idx, - const APInt &Imm, Type *Ty) = 0; + virtual int getFPOpCost(Type *Ty) = 0; + virtual int getIntImmCost(const APInt &Imm, Type *Ty) = 0; + virtual int getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, + Type *Ty) = 0; + virtual int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, + Type *Ty) = 0; virtual unsigned getNumberOfRegisters(bool Vector) = 0; virtual unsigned getRegisterBitWidth(bool Vector) = 0; virtual unsigned getMaxInterleaveFactor(unsigned VF) = 0; @@ -592,40 +615,44 @@ public: OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo, OperandValueProperties Opd2PropInfo) = 0; - virtual unsigned getShuffleCost(ShuffleKind Kind, Type *Tp, int Index, - Type *SubTp) = 0; - virtual unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) = 0; - virtual unsigned getCFInstrCost(unsigned Opcode) = 0; - virtual unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, - Type *CondTy) = 0; - virtual unsigned getVectorInstrCost(unsigned Opcode, Type *Val, - unsigned Index) = 0; - virtual unsigned getMemoryOpCost(unsigned Opcode, Type *Src, - unsigned Alignment, - unsigned AddressSpace) = 0; - virtual unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src, + virtual int getShuffleCost(ShuffleKind Kind, Type *Tp, int Index, + Type *SubTp) = 0; + virtual int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) = 0; + virtual int getCFInstrCost(unsigned Opcode) = 0; + virtual int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, + Type *CondTy) = 0; + virtual int getVectorInstrCost(unsigned Opcode, Type *Val, + unsigned Index) = 0; + virtual int getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + unsigned AddressSpace) = 0; + virtual int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, + unsigned Alignment, + unsigned AddressSpace) = 0; + virtual int getGatherScatterOpCost(unsigned Opcode, Type *DataTy, + Value *Ptr, bool VariableMask, + unsigned Alignment) = 0; + virtual int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, + unsigned Factor, + ArrayRef<unsigned> Indices, unsigned Alignment, unsigned AddressSpace) = 0; - virtual unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, - unsigned Factor, - ArrayRef<unsigned> Indices, - unsigned Alignment, - unsigned AddressSpace) = 0; - virtual unsigned getReductionCost(unsigned Opcode, Type *Ty, - bool IsPairwiseForm) = 0; - virtual unsigned getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, - ArrayRef<Type *> Tys) = 0; - virtual unsigned getCallInstrCost(Function *F, Type *RetTy, + virtual int getReductionCost(unsigned Opcode, Type *Ty, + bool IsPairwiseForm) = 0; + virtual int getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, ArrayRef<Type *> Tys) = 0; + virtual int getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<Value *> Args) = 0; + virtual int getCallInstrCost(Function *F, Type *RetTy, + ArrayRef<Type *> Tys) = 0; virtual unsigned getNumberOfParts(Type *Tp) = 0; - virtual unsigned getAddressComputationCost(Type *Ty, bool IsComplex) = 0; + virtual int getAddressComputationCost(Type *Ty, bool IsComplex) = 0; virtual unsigned getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) = 0; virtual bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) = 0; virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType) = 0; - virtual bool hasCompatibleFunctionAttributes(const Function *Caller, - const Function *Callee) const = 0; + virtual bool areInlineCompatible(const Function *Caller, + const Function *Callee) const = 0; }; template <typename T> @@ -640,32 +667,32 @@ public: return Impl.getDataLayout(); } - unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) override { + int getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) override { return Impl.getOperationCost(Opcode, Ty, OpTy); } - unsigned getGEPCost(const Value *Ptr, - ArrayRef<const Value *> Operands) override { - return Impl.getGEPCost(Ptr, Operands); + int getGEPCost(Type *PointeeType, const Value *Ptr, + ArrayRef<const Value *> Operands) override { + return Impl.getGEPCost(PointeeType, Ptr, Operands); } - unsigned getCallCost(FunctionType *FTy, int NumArgs) override { + int getCallCost(FunctionType *FTy, int NumArgs) override { return Impl.getCallCost(FTy, NumArgs); } - unsigned getCallCost(const Function *F, int NumArgs) override { + int getCallCost(const Function *F, int NumArgs) override { return Impl.getCallCost(F, NumArgs); } - unsigned getCallCost(const Function *F, - ArrayRef<const Value *> Arguments) override { + int getCallCost(const Function *F, + ArrayRef<const Value *> Arguments) override { return Impl.getCallCost(F, Arguments); } - unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef<Type *> ParamTys) override { + int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Type *> ParamTys) override { return Impl.getIntrinsicCost(IID, RetTy, ParamTys); } - unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef<const Value *> Arguments) override { + int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<const Value *> Arguments) override { return Impl.getIntrinsicCost(IID, RetTy, Arguments); } - unsigned getUserCost(const User *U) override { return Impl.getUserCost(U); } + int getUserCost(const User *U) override { return Impl.getUserCost(U); } bool hasBranchDivergence() override { return Impl.hasBranchDivergence(); } bool isSourceOfDivergence(const Value *V) override { return Impl.isSourceOfDivergence(V); @@ -688,11 +715,17 @@ public: return Impl.isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace); } - bool isLegalMaskedStore(Type *DataType, int Consecutive) override { - return Impl.isLegalMaskedStore(DataType, Consecutive); + bool isLegalMaskedStore(Type *DataType) override { + return Impl.isLegalMaskedStore(DataType); + } + bool isLegalMaskedLoad(Type *DataType) override { + return Impl.isLegalMaskedLoad(DataType); + } + bool isLegalMaskedScatter(Type *DataType) override { + return Impl.isLegalMaskedScatter(DataType); } - bool isLegalMaskedLoad(Type *DataType, int Consecutive) override { - return Impl.isLegalMaskedLoad(DataType, Consecutive); + bool isLegalMaskedGather(Type *DataType) override { + return Impl.isLegalMaskedGather(DataType); } int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, @@ -715,24 +748,25 @@ public: bool enableAggressiveInterleaving(bool LoopHasReductions) override { return Impl.enableAggressiveInterleaving(LoopHasReductions); } + bool enableInterleavedAccessVectorization() override { + return Impl.enableInterleavedAccessVectorization(); + } PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) override { return Impl.getPopcntSupport(IntTyWidthInBit); } bool haveFastSqrt(Type *Ty) override { return Impl.haveFastSqrt(Ty); } - unsigned getFPOpCost(Type *Ty) override { - return Impl.getFPOpCost(Ty); - } + int getFPOpCost(Type *Ty) override { return Impl.getFPOpCost(Ty); } - unsigned getIntImmCost(const APInt &Imm, Type *Ty) override { + int getIntImmCost(const APInt &Imm, Type *Ty) override { return Impl.getIntImmCost(Imm, Ty); } - unsigned getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, - Type *Ty) override { + int getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, + Type *Ty) override { return Impl.getIntImmCost(Opc, Idx, Imm, Ty); } - unsigned getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, - Type *Ty) override { + int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, + Type *Ty) override { return Impl.getIntImmCost(IID, Idx, Imm, Ty); } unsigned getNumberOfRegisters(bool Vector) override { @@ -752,56 +786,62 @@ public: return Impl.getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info, Opd1PropInfo, Opd2PropInfo); } - unsigned getShuffleCost(ShuffleKind Kind, Type *Tp, int Index, - Type *SubTp) override { + int getShuffleCost(ShuffleKind Kind, Type *Tp, int Index, + Type *SubTp) override { return Impl.getShuffleCost(Kind, Tp, Index, SubTp); } - unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) override { + int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) override { return Impl.getCastInstrCost(Opcode, Dst, Src); } - unsigned getCFInstrCost(unsigned Opcode) override { + int getCFInstrCost(unsigned Opcode) override { return Impl.getCFInstrCost(Opcode); } - unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, - Type *CondTy) override { + int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy) override { return Impl.getCmpSelInstrCost(Opcode, ValTy, CondTy); } - unsigned getVectorInstrCost(unsigned Opcode, Type *Val, - unsigned Index) override { + int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) override { return Impl.getVectorInstrCost(Opcode, Val, Index); } - unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, - unsigned AddressSpace) override { + int getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + unsigned AddressSpace) override { return Impl.getMemoryOpCost(Opcode, Src, Alignment, AddressSpace); } - unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, - unsigned AddressSpace) override { + int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + unsigned AddressSpace) override { return Impl.getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace); } - unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, - unsigned Factor, - ArrayRef<unsigned> Indices, - unsigned Alignment, - unsigned AddressSpace) override { + int getGatherScatterOpCost(unsigned Opcode, Type *DataTy, + Value *Ptr, bool VariableMask, + unsigned Alignment) override { + return Impl.getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, + Alignment); + } + int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, + ArrayRef<unsigned> Indices, unsigned Alignment, + unsigned AddressSpace) override { return Impl.getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, Alignment, AddressSpace); } - unsigned getReductionCost(unsigned Opcode, Type *Ty, - bool IsPairwiseForm) override { + int getReductionCost(unsigned Opcode, Type *Ty, + bool IsPairwiseForm) override { return Impl.getReductionCost(Opcode, Ty, IsPairwiseForm); } - unsigned getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, - ArrayRef<Type *> Tys) override { + int getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<Type *> Tys) override { return Impl.getIntrinsicInstrCost(ID, RetTy, Tys); } - unsigned getCallInstrCost(Function *F, Type *RetTy, - ArrayRef<Type *> Tys) override { + int getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<Value *> Args) override { + return Impl.getIntrinsicInstrCost(ID, RetTy, Args); + } + int getCallInstrCost(Function *F, Type *RetTy, + ArrayRef<Type *> Tys) override { return Impl.getCallInstrCost(F, RetTy, Tys); } unsigned getNumberOfParts(Type *Tp) override { return Impl.getNumberOfParts(Tp); } - unsigned getAddressComputationCost(Type *Ty, bool IsComplex) override { + int getAddressComputationCost(Type *Ty, bool IsComplex) override { return Impl.getAddressComputationCost(Ty, IsComplex); } unsigned getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) override { @@ -815,9 +855,9 @@ public: Type *ExpectedType) override { return Impl.getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); } - bool hasCompatibleFunctionAttributes(const Function *Caller, - const Function *Callee) const override { - return Impl.hasCompatibleFunctionAttributes(Caller, Callee); + bool areInlineCompatible(const Function *Caller, + const Function *Callee) const override { + return Impl.areInlineCompatible(Caller, Callee); } }; @@ -856,7 +896,7 @@ public: /// /// The callback will be called with a particular function for which the TTI /// is needed and must return a TTI object for that function. - TargetIRAnalysis(std::function<Result(Function &)> TTICallback); + TargetIRAnalysis(std::function<Result(const Function &)> TTICallback); // Value semantics. We spell out the constructors for MSVC. TargetIRAnalysis(const TargetIRAnalysis &Arg) @@ -872,7 +912,7 @@ public: return *this; } - Result run(Function &F); + Result run(const Function &F); private: static char PassID; @@ -887,10 +927,10 @@ private: /// the analysis and thus use a function_ref which would be lighter weight. /// This may also be less error prone as the callback is likely to reference /// the external TargetMachine, and that reference needs to never dangle. - std::function<Result(Function &)> TTICallback; + std::function<Result(const Function &)> TTICallback; /// \brief Helper function used as the callback in the default constructor. - static Result getDefaultTTI(Function &F); + static Result getDefaultTTI(const Function &F); }; /// \brief Wrapper pass for TargetTransformInfo. @@ -914,7 +954,7 @@ public: explicit TargetTransformInfoWrapperPass(TargetIRAnalysis TIRA); - TargetTransformInfo &getTTI(Function &F); + TargetTransformInfo &getTTI(const Function &F); }; /// \brief Create an analysis pass wrapper around a TTI object. diff --git a/include/llvm/Analysis/TargetTransformInfoImpl.h b/include/llvm/Analysis/TargetTransformInfoImpl.h index 035cb04870a1..43815234051e 100644 --- a/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -19,8 +19,10 @@ #include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" +#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/Operator.h" #include "llvm/IR/Type.h" +#include "llvm/Analysis/VectorUtils.h" namespace llvm { @@ -60,6 +62,14 @@ public: // Otherwise, the default basic cost is used. return TTI::TCC_Basic; + case Instruction::FDiv: + case Instruction::FRem: + case Instruction::SDiv: + case Instruction::SRem: + case Instruction::UDiv: + case Instruction::URem: + return TTI::TCC_Expensive; + case Instruction::IntToPtr: { // An inttoptr cast is free so long as the input is a legal integer type // which doesn't contain values outside the range of a pointer. @@ -92,7 +102,8 @@ public: } } - unsigned getGEPCost(const Value *Ptr, ArrayRef<const Value *> Operands) { + unsigned getGEPCost(Type *PointeeType, const Value *Ptr, + ArrayRef<const Value *> Operands) { // In the basic model, we just assume that all-constant GEPs will be folded // into their uses via addressing modes. for (unsigned Idx = 0, Size = Operands.size(); Idx != Size; ++Idx) @@ -137,9 +148,6 @@ public: case Intrinsic::objectsize: case Intrinsic::ptr_annotation: case Intrinsic::var_annotation: - case Intrinsic::experimental_gc_result_int: - case Intrinsic::experimental_gc_result_float: - case Intrinsic::experimental_gc_result_ptr: case Intrinsic::experimental_gc_result: case Intrinsic::experimental_gc_relocate: // These intrinsics don't actually represent code after lowering. @@ -199,9 +207,13 @@ public: return !BaseGV && BaseOffset == 0 && (Scale == 0 || Scale == 1); } - bool isLegalMaskedStore(Type *DataType, int Consecutive) { return false; } + bool isLegalMaskedStore(Type *DataType) { return false; } + + bool isLegalMaskedLoad(Type *DataType) { return false; } + + bool isLegalMaskedScatter(Type *DataType) { return false; } - bool isLegalMaskedLoad(Type *DataType, int Consecutive) { return false; } + bool isLegalMaskedGather(Type *DataType) { return false; } int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) { @@ -226,6 +238,8 @@ public: bool enableAggressiveInterleaving(bool LoopHasReductions) { return false; } + bool enableInterleavedAccessVectorization() { return false; } + TTI::PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) { return TTI::PSK_Software; } @@ -287,6 +301,12 @@ public: return 1; } + unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy, Value *Ptr, + bool VariableMask, + unsigned Alignment) { + return 1; + } + unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, @@ -299,6 +319,10 @@ public: ArrayRef<Type *> Tys) { return 1; } + unsigned getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<Value *> Args) { + return 1; + } unsigned getCallInstrCost(Function *F, Type *RetTy, ArrayRef<Type *> Tys) { return 1; @@ -321,8 +345,8 @@ public: return nullptr; } - bool hasCompatibleFunctionAttributes(const Function *Caller, - const Function *Callee) const { + bool areInlineCompatible(const Function *Caller, + const Function *Callee) const { return (Caller->getFnAttribute("target-cpu") == Callee->getFnAttribute("target-cpu")) && (Caller->getFnAttribute("target-features") == @@ -378,6 +402,61 @@ public: return static_cast<T *>(this)->getCallCost(F, Arguments.size()); } + using BaseT::getGEPCost; + + unsigned getGEPCost(Type *PointeeType, const Value *Ptr, + ArrayRef<const Value *> Operands) { + const GlobalValue *BaseGV = nullptr; + if (Ptr != nullptr) { + // TODO: will remove this when pointers have an opaque type. + assert(Ptr->getType()->getScalarType()->getPointerElementType() == + PointeeType && + "explicit pointee type doesn't match operand's pointee type"); + BaseGV = dyn_cast<GlobalValue>(Ptr->stripPointerCasts()); + } + bool HasBaseReg = (BaseGV == nullptr); + int64_t BaseOffset = 0; + int64_t Scale = 0; + + // Assumes the address space is 0 when Ptr is nullptr. + unsigned AS = + (Ptr == nullptr ? 0 : Ptr->getType()->getPointerAddressSpace()); + auto GTI = gep_type_begin(PointerType::get(PointeeType, AS), Operands); + for (auto I = Operands.begin(); I != Operands.end(); ++I, ++GTI) { + // We assume that the cost of Scalar GEP with constant index and the + // cost of Vector GEP with splat constant index are the same. + const ConstantInt *ConstIdx = dyn_cast<ConstantInt>(*I); + if (!ConstIdx) + if (auto Splat = getSplatValue(*I)) + ConstIdx = dyn_cast<ConstantInt>(Splat); + if (isa<SequentialType>(*GTI)) { + int64_t ElementSize = DL.getTypeAllocSize(GTI.getIndexedType()); + if (ConstIdx) + BaseOffset += ConstIdx->getSExtValue() * ElementSize; + else { + // Needs scale register. + if (Scale != 0) + // No addressing mode takes two scale registers. + return TTI::TCC_Basic; + Scale = ElementSize; + } + } else { + StructType *STy = cast<StructType>(*GTI); + // For structures the index is always splat or scalar constant + assert(ConstIdx && "Unexpected GEP index"); + uint64_t Field = ConstIdx->getZExtValue(); + BaseOffset += DL.getStructLayout(STy)->getElementOffset(Field); + } + } + + if (static_cast<T *>(this)->isLegalAddressingMode( + PointerType::get(*GTI, AS), const_cast<GlobalValue *>(BaseGV), + BaseOffset, HasBaseReg, Scale, AS)) { + return TTI::TCC_Free; + } + return TTI::TCC_Basic; + } + using BaseT::getIntrinsicCost; unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, @@ -397,9 +476,9 @@ public: return TTI::TCC_Free; // Model all PHI nodes as free. if (const GEPOperator *GEP = dyn_cast<GEPOperator>(U)) { - SmallVector<const Value *, 4> Indices(GEP->idx_begin(), GEP->idx_end()); - return static_cast<T *>(this) - ->getGEPCost(GEP->getPointerOperand(), Indices); + SmallVector<Value *, 4> Indices(GEP->idx_begin(), GEP->idx_end()); + return static_cast<T *>(this)->getGEPCost( + GEP->getSourceElementType(), GEP->getPointerOperand(), Indices); } if (auto CS = ImmutableCallSite(U)) { diff --git a/include/llvm/Analysis/TypeBasedAliasAnalysis.h b/include/llvm/Analysis/TypeBasedAliasAnalysis.h new file mode 100644 index 000000000000..7b44ac73f1fa --- /dev/null +++ b/include/llvm/Analysis/TypeBasedAliasAnalysis.h @@ -0,0 +1,93 @@ +//===- TypeBasedAliasAnalysis.h - Type-Based Alias Analysis -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This is the interface for a metadata-based TBAA. See the source file for +/// details on the algorithm. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_TYPEBASEDALIASANALYSIS_H +#define LLVM_ANALYSIS_TYPEBASEDALIASANALYSIS_H + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Metadata.h" +#include "llvm/Pass.h" + +namespace llvm { + +/// A simple AA result that uses TBAA metadata to answer queries. +class TypeBasedAAResult : public AAResultBase<TypeBasedAAResult> { + friend AAResultBase<TypeBasedAAResult>; + +public: + explicit TypeBasedAAResult(const TargetLibraryInfo &TLI) + : AAResultBase(TLI) {} + TypeBasedAAResult(TypeBasedAAResult &&Arg) : AAResultBase(std::move(Arg)) {} + + /// Handle invalidation events from the new pass manager. + /// + /// By definition, this result is stateless and so remains valid. + bool invalidate(Function &, const PreservedAnalyses &) { return false; } + + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); + bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal); + FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS); + FunctionModRefBehavior getModRefBehavior(const Function *F); + ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2); + +private: + bool Aliases(const MDNode *A, const MDNode *B) const; + bool PathAliases(const MDNode *A, const MDNode *B) const; +}; + +/// Analysis pass providing a never-invalidated alias analysis result. +class TypeBasedAA { +public: + typedef TypeBasedAAResult Result; + + /// \brief Opaque, unique identifier for this analysis pass. + static void *ID() { return (void *)&PassID; } + + TypeBasedAAResult run(Function &F, AnalysisManager<Function> *AM); + + /// \brief Provide access to a name for this pass for debugging purposes. + static StringRef name() { return "TypeBasedAA"; } + +private: + static char PassID; +}; + +/// Legacy wrapper pass to provide the TypeBasedAAResult object. +class TypeBasedAAWrapperPass : public ImmutablePass { + std::unique_ptr<TypeBasedAAResult> Result; + +public: + static char ID; + + TypeBasedAAWrapperPass(); + + TypeBasedAAResult &getResult() { return *Result; } + const TypeBasedAAResult &getResult() const { return *Result; } + + bool doInitialization(Module &M) override; + bool doFinalization(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +//===--------------------------------------------------------------------===// +// +// createTypeBasedAAWrapperPass - This pass implements metadata-based +// type-based alias analysis. +// +ImmutablePass *createTypeBasedAAWrapperPass(); +} + +#endif diff --git a/include/llvm/Analysis/ValueTracking.h b/include/llvm/Analysis/ValueTracking.h index 653821d02271..8e0291068472 100644 --- a/include/llvm/Analysis/ValueTracking.h +++ b/include/llvm/Analysis/ValueTracking.h @@ -16,20 +16,23 @@ #define LLVM_ANALYSIS_VALUETRACKING_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/IR/ConstantRange.h" #include "llvm/IR/Instruction.h" #include "llvm/Support/DataTypes.h" namespace llvm { - class Value; - class Instruction; class APInt; - class DataLayout; - class StringRef; - class MDNode; + class AddOperator; class AssumptionCache; + class DataLayout; class DominatorTree; - class TargetLibraryInfo; + class Instruction; + class Loop; class LoopInfo; + class MDNode; + class StringRef; + class TargetLibraryInfo; + class Value; /// Determine which bits of V are known to be either zero or one and return /// them in the KnownZero/KnownOne bit sets. @@ -46,9 +49,10 @@ namespace llvm { const DominatorTree *DT = nullptr); /// Compute known bits from the range metadata. /// \p KnownZero the set of bits that are known to be zero + /// \p KnownOne the set of bits that are known to be one void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, - APInt &KnownZero); - /// Returns true if LHS and RHS have no common bits set. + APInt &KnownZero, APInt &KnownOne); + /// Return true if LHS and RHS have no common bits set. bool haveNoCommonBitsSet(Value *LHS, Value *RHS, const DataLayout &DL, AssumptionCache *AC = nullptr, const Instruction *CxtI = nullptr, @@ -66,7 +70,7 @@ namespace llvm { /// exactly one bit set when defined. For vectors return true if every /// element is known to be a power of two when defined. Supports values with /// integer or pointer type and vectors of integers. If 'OrZero' is set then - /// returns true if the given value is either a power of two or zero. + /// return true if the given value is either a power of two or zero. bool isKnownToBeAPowerOfTwo(Value *V, const DataLayout &DL, bool OrZero = false, unsigned Depth = 0, AssumptionCache *AC = nullptr, @@ -82,6 +86,19 @@ namespace llvm { const Instruction *CxtI = nullptr, const DominatorTree *DT = nullptr); + /// Returns true if the give value is known to be non-negative. + bool isKnownNonNegative(Value *V, const DataLayout &DL, unsigned Depth = 0, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); + + /// isKnownNonEqual - Return true if the given values are known to be + /// non-equal when defined. Supports scalar integer types only. + bool isKnownNonEqual(Value *V1, Value *V2, const DataLayout &DL, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); + /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use /// this predicate to simplify operations downstream. Mask is known to be /// zero for bits that V cannot have. @@ -118,12 +135,12 @@ namespace llvm { bool LookThroughSExt = false, unsigned Depth = 0); - /// CannotBeNegativeZero - Return true if we can prove that the specified FP + /// CannotBeNegativeZero - Return true if we can prove that the specified FP /// value is never equal to -0.0. /// bool CannotBeNegativeZero(const Value *V, unsigned Depth = 0); - /// CannotBeOrderedLessThanZero - Return true if we can prove that the + /// CannotBeOrderedLessThanZero - Return true if we can prove that the /// specified FP value is either a NaN or never less than 0.0. /// bool CannotBeOrderedLessThanZero(const Value *V, unsigned Depth = 0); @@ -134,7 +151,7 @@ namespace llvm { /// i16 0xF0F0, double 0.0 etc. If the value can't be handled with a repeated /// byte store (e.g. i16 0x1234), return null. Value *isBytewiseValue(Value *V); - + /// FindInsertedValue - Given an aggregrate and an sequence of indices, see if /// the scalar value indexed is already around as a register, for example if /// it were inserted directly into the aggregrate. @@ -156,7 +173,7 @@ namespace llvm { return GetPointerBaseWithConstantOffset(const_cast<Value *>(Ptr), Offset, DL); } - + /// getConstantStringInfo - This function computes the length of a /// null-terminated C string pointed to by V. If successful, it returns true /// and returns the string in Str. If unsuccessful, it returns false. This @@ -227,7 +244,17 @@ namespace llvm { const Instruction *CtxI = nullptr, const DominatorTree *DT = nullptr, const TargetLibraryInfo *TLI = nullptr); - + + /// Returns true if V is always a dereferenceable pointer with alignment + /// greater or equal than requested. If the context instruction is specified + /// performs context-sensitive analysis and returns true if the pointer is + /// dereferenceable at the specified instruction. + bool isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, + const DataLayout &DL, + const Instruction *CtxI = nullptr, + const DominatorTree *DT = nullptr, + const TargetLibraryInfo *TLI = nullptr); + /// isSafeToSpeculativelyExecute - Return true if the instruction does not /// have any effects besides calculating the result and does not have /// undefined behavior. @@ -257,6 +284,16 @@ namespace llvm { const DominatorTree *DT = nullptr, const TargetLibraryInfo *TLI = nullptr); + /// Returns true if the result or effects of the given instructions \p I + /// depend on or influence global memory. + /// Memory dependence arises for example if the instruction reads from + /// memory or may produce effects or undefined behaviour. Memory dependent + /// instructions generally cannot be reorderd with respect to other memory + /// dependent instructions or moved into non-dominated basic blocks. + /// Instructions which just compute a value based on the values of their + /// operands are not memory dependent. + bool mayBeMemoryDependent(const Instruction &I); + /// isKnownNonNull - Return true if this pointer couldn't possibly be null by /// its definition. This returns true for allocas, non-extern-weak globals /// and byval arguments. @@ -288,16 +325,98 @@ namespace llvm { AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT); - + OverflowResult computeOverflowForSignedAdd(Value *LHS, Value *RHS, + const DataLayout &DL, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); + /// This version also leverages the sign bit of Add if known. + OverflowResult computeOverflowForSignedAdd(AddOperator *Add, + const DataLayout &DL, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); + + /// Return true if this function can prove that the instruction I will + /// always transfer execution to one of its successors (including the next + /// instruction that follows within a basic block). E.g. this is not + /// guaranteed for function calls that could loop infinitely. + /// + /// In other words, this function returns false for instructions that may + /// transfer execution or fail to transfer execution in a way that is not + /// captured in the CFG nor in the sequence of instructions within a basic + /// block. + /// + /// Undefined behavior is assumed not to happen, so e.g. division is + /// guaranteed to transfer execution to the following instruction even + /// though division by zero might cause undefined behavior. + bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I); + + /// Return true if this function can prove that the instruction I + /// is executed for every iteration of the loop L. + /// + /// Note that this currently only considers the loop header. + bool isGuaranteedToExecuteForEveryIteration(const Instruction *I, + const Loop *L); + + /// Return true if this function can prove that I is guaranteed to yield + /// full-poison (all bits poison) if at least one of its operands are + /// full-poison (all bits poison). + /// + /// The exact rules for how poison propagates through instructions have + /// not been settled as of 2015-07-10, so this function is conservative + /// and only considers poison to be propagated in uncontroversial + /// cases. There is no attempt to track values that may be only partially + /// poison. + bool propagatesFullPoison(const Instruction *I); + + /// Return either nullptr or an operand of I such that I will trigger + /// undefined behavior if I is executed and that operand has a full-poison + /// value (all bits poison). + const Value *getGuaranteedNonFullPoisonOp(const Instruction *I); + + /// Return true if this function can prove that if PoisonI is executed + /// and yields a full-poison value (all bits poison), then that will + /// trigger undefined behavior. + /// + /// Note that this currently only considers the basic block that is + /// the parent of I. + bool isKnownNotFullPoison(const Instruction *PoisonI); + /// \brief Specific patterns of select instructions we can match. enum SelectPatternFlavor { SPF_UNKNOWN = 0, - SPF_SMIN, // Signed minimum - SPF_UMIN, // Unsigned minimum - SPF_SMAX, // Signed maximum - SPF_UMAX, // Unsigned maximum - SPF_ABS, // Absolute value - SPF_NABS // Negated absolute value + SPF_SMIN, /// Signed minimum + SPF_UMIN, /// Unsigned minimum + SPF_SMAX, /// Signed maximum + SPF_UMAX, /// Unsigned maximum + SPF_FMINNUM, /// Floating point minnum + SPF_FMAXNUM, /// Floating point maxnum + SPF_ABS, /// Absolute value + SPF_NABS /// Negated absolute value + }; + /// \brief Behavior when a floating point min/max is given one NaN and one + /// non-NaN as input. + enum SelectPatternNaNBehavior { + SPNB_NA = 0, /// NaN behavior not applicable. + SPNB_RETURNS_NAN, /// Given one NaN input, returns the NaN. + SPNB_RETURNS_OTHER, /// Given one NaN input, returns the non-NaN. + SPNB_RETURNS_ANY /// Given one NaN input, can return either (or + /// it has been determined that no operands can + /// be NaN). + }; + struct SelectPatternResult { + SelectPatternFlavor Flavor; + SelectPatternNaNBehavior NaNBehavior; /// Only applicable if Flavor is + /// SPF_FMINNUM or SPF_FMAXNUM. + bool Ordered; /// When implementing this min/max pattern as + /// fcmp; select, does the fcmp have to be + /// ordered? + + /// \brief Return true if \p SPF is a min or a max pattern. + static bool isMinOrMax(SelectPatternFlavor SPF) { + return !(SPF == SPF_UNKNOWN || SPF == SPF_ABS || SPF == SPF_NABS); + } }; /// Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind /// and providing the out parameter results if we successfully match. @@ -314,9 +433,26 @@ namespace llvm { /// /// -> LHS = %a, RHS = i32 4, *CastOp = Instruction::SExt /// - SelectPatternFlavor matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, + SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp = nullptr); + /// Parse out a conservative ConstantRange from !range metadata. + /// + /// E.g. if RangeMD is !{i32 0, i32 10, i32 15, i32 20} then return [0, 20). + ConstantRange getConstantRangeFromMetadata(MDNode &RangeMD); + + /// Return true if RHS is known to be implied by LHS. A & B must be i1 + /// (boolean) values or a vector of such values. Note that the truth table for + /// implication is the same as <=u on i1 values (but not <=s!). The truth + /// table for both is: + /// | T | F (B) + /// T | T | F + /// F | T | T + /// (A) + bool isImpliedCondition(Value *LHS, Value *RHS, const DataLayout &DL, + unsigned Depth = 0, AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); } // end namespace llvm #endif diff --git a/include/llvm/Analysis/VectorUtils.h b/include/llvm/Analysis/VectorUtils.h index d8e9ca42e623..531803adf5e4 100644 --- a/include/llvm/Analysis/VectorUtils.h +++ b/include/llvm/Analysis/VectorUtils.h @@ -14,15 +14,19 @@ #ifndef LLVM_TRANSFORMS_UTILS_VECTORUTILS_H #define LLVM_TRANSFORMS_UTILS_VECTORUTILS_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/MapVector.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" namespace llvm { +struct DemandedBits; class GetElementPtrInst; class Loop; class ScalarEvolution; +class TargetTransformInfo; class Type; class Value; @@ -62,8 +66,8 @@ Intrinsic::ID getIntrinsicIDForCall(CallInst *CI, const TargetLibraryInfo *TLI); /// pointer. unsigned getGEPInductionOperand(const GetElementPtrInst *Gep); -/// \brief If the argument is a GEP, then returns the operand identified by -/// getGEPInductionOperand. However, if there is some other non-loop-invariant +/// \brief If the argument is a GEP, then returns the operand identified by +/// getGEPInductionOperand. However, if there is some other non-loop-invariant /// operand, it returns that instead. Value *stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp); @@ -79,6 +83,50 @@ Value *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp); /// from the vector. Value *findScalarElement(Value *V, unsigned EltNo); +/// \brief Get splat value if the input is a splat vector or return nullptr. +/// The value may be extracted from a splat constants vector or from +/// a sequence of instructions that broadcast a single value into a vector. +const Value *getSplatValue(const Value *V); + +/// \brief Compute a map of integer instructions to their minimum legal type +/// size. +/// +/// C semantics force sub-int-sized values (e.g. i8, i16) to be promoted to int +/// type (e.g. i32) whenever arithmetic is performed on them. +/// +/// For targets with native i8 or i16 operations, usually InstCombine can shrink +/// the arithmetic type down again. However InstCombine refuses to create +/// illegal types, so for targets without i8 or i16 registers, the lengthening +/// and shrinking remains. +/// +/// Most SIMD ISAs (e.g. NEON) however support vectors of i8 or i16 even when +/// their scalar equivalents do not, so during vectorization it is important to +/// remove these lengthens and truncates when deciding the profitability of +/// vectorization. +/// +/// This function analyzes the given range of instructions and determines the +/// minimum type size each can be converted to. It attempts to remove or +/// minimize type size changes across each def-use chain, so for example in the +/// following code: +/// +/// %1 = load i8, i8* +/// %2 = add i8 %1, 2 +/// %3 = load i16, i16* +/// %4 = zext i8 %2 to i32 +/// %5 = zext i16 %3 to i32 +/// %6 = add i32 %4, %5 +/// %7 = trunc i32 %6 to i16 +/// +/// Instruction %6 must be done at least in i16, so computeMinimumValueSizes +/// will return: {%1: 16, %2: 16, %3: 16, %4: 16, %5: 16, %6: 16, %7: 16}. +/// +/// If the optional TargetTransformInfo is provided, this function tries harder +/// to do less work by only looking at illegal types. +MapVector<Instruction*, uint64_t> +computeMinimumValueSizes(ArrayRef<BasicBlock*> Blocks, + DemandedBits &DB, + const TargetTransformInfo *TTI=nullptr); + } // llvm namespace #endif diff --git a/include/llvm/AsmParser/Parser.h b/include/llvm/AsmParser/Parser.h index 52151409f946..96a15c1ec45c 100644 --- a/include/llvm/AsmParser/Parser.h +++ b/include/llvm/AsmParser/Parser.h @@ -18,6 +18,7 @@ namespace llvm { +class Constant; class LLVMContext; class Module; struct SlotMapping; @@ -79,6 +80,17 @@ std::unique_ptr<Module> parseAssembly(MemoryBufferRef F, SMDiagnostic &Err, bool parseAssemblyInto(MemoryBufferRef F, Module &M, SMDiagnostic &Err, SlotMapping *Slots = nullptr); +/// Parse a type and a constant value in the given string. +/// +/// The constant value can be any LLVM constant, including a constant +/// expression. +/// +/// \param Slots The optional slot mapping that will restore the parsing state +/// of the module. +/// \return null on error. +Constant *parseConstantValue(StringRef Asm, SMDiagnostic &Err, const Module &M, + const SlotMapping *Slots = nullptr); + } // End llvm namespace #endif diff --git a/include/llvm/AsmParser/SlotMapping.h b/include/llvm/AsmParser/SlotMapping.h index c5f61d25c3a8..bd7e8fcad8bc 100644 --- a/include/llvm/AsmParser/SlotMapping.h +++ b/include/llvm/AsmParser/SlotMapping.h @@ -14,6 +14,7 @@ #ifndef LLVM_ASMPARSER_SLOTMAPPING_H #define LLVM_ASMPARSER_SLOTMAPPING_H +#include "llvm/ADT/StringMap.h" #include "llvm/IR/TrackingMDRef.h" #include <map> #include <vector> @@ -21,12 +22,19 @@ namespace llvm { class GlobalValue; +class Type; -/// This struct contains the mapping from the slot numbers to unnamed metadata -/// nodes and global values. +/// This struct contains the mappings from the slot numbers to unnamed metadata +/// nodes, global values and types. It also contains the mapping for the named +/// types. +/// It can be used to save the parsing state of an LLVM IR module so that the +/// textual references to the values in the module can be parsed outside of the +/// module's source. struct SlotMapping { std::vector<GlobalValue *> GlobalValues; std::map<unsigned, TrackingMDNodeRef> MetadataNodes; + StringMap<Type *> NamedTypes; + std::map<unsigned, Type *> Types; }; } // end namespace llvm diff --git a/include/llvm/Bitcode/BitcodeWriterPass.h b/include/llvm/Bitcode/BitcodeWriterPass.h index ae915c688ba0..a1272cf156e5 100644 --- a/include/llvm/Bitcode/BitcodeWriterPass.h +++ b/include/llvm/Bitcode/BitcodeWriterPass.h @@ -29,8 +29,12 @@ class PreservedAnalyses; /// /// If \c ShouldPreserveUseListOrder, encode use-list order so it can be /// reproduced when deserialized. +/// +/// If \c EmitFunctionSummary, emit the function summary index (currently +/// for use in ThinLTO optimization). ModulePass *createBitcodeWriterPass(raw_ostream &Str, - bool ShouldPreserveUseListOrder = false); + bool ShouldPreserveUseListOrder = false, + bool EmitFunctionSummary = false); /// \brief Pass for writing a module of IR out to a bitcode file. /// @@ -39,15 +43,21 @@ ModulePass *createBitcodeWriterPass(raw_ostream &Str, class BitcodeWriterPass { raw_ostream &OS; bool ShouldPreserveUseListOrder; + bool EmitFunctionSummary; public: /// \brief Construct a bitcode writer pass around a particular output stream. /// /// If \c ShouldPreserveUseListOrder, encode use-list order so it can be /// reproduced when deserialized. + /// + /// If \c EmitFunctionSummary, emit the function summary index (currently + /// for use in ThinLTO optimization). explicit BitcodeWriterPass(raw_ostream &OS, - bool ShouldPreserveUseListOrder = false) - : OS(OS), ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {} + bool ShouldPreserveUseListOrder = false, + bool EmitFunctionSummary = false) + : OS(OS), ShouldPreserveUseListOrder(ShouldPreserveUseListOrder), + EmitFunctionSummary(EmitFunctionSummary) {} /// \brief Run the bitcode writer pass, and output the module to the selected /// output stream. diff --git a/include/llvm/Bitcode/BitstreamReader.h b/include/llvm/Bitcode/BitstreamReader.h index 4c040a7f3e22..c0cf6cde887f 100644 --- a/include/llvm/Bitcode/BitstreamReader.h +++ b/include/llvm/Bitcode/BitstreamReader.h @@ -325,6 +325,8 @@ public: // If we run out of data, stop at the end of the stream. if (BytesRead == 0) { + CurWord = 0; + BitsInCurWord = 0; Size = NextChar; return; } diff --git a/include/llvm/Bitcode/BitstreamWriter.h b/include/llvm/Bitcode/BitstreamWriter.h index 9f23023a1419..438f4a6fb69b 100644 --- a/include/llvm/Bitcode/BitstreamWriter.h +++ b/include/llvm/Bitcode/BitstreamWriter.h @@ -15,6 +15,8 @@ #ifndef LLVM_BITCODE_BITSTREAMWRITER_H #define LLVM_BITCODE_BITSTREAMWRITER_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Bitcode/BitCodes.h" @@ -45,9 +47,9 @@ class BitstreamWriter { struct Block { unsigned PrevCodeSize; - unsigned StartSizeWord; + size_t StartSizeWord; std::vector<IntrusiveRefCntPtr<BitCodeAbbrev>> PrevAbbrevs; - Block(unsigned PCS, unsigned SSW) : PrevCodeSize(PCS), StartSizeWord(SSW) {} + Block(unsigned PCS, size_t SSW) : PrevCodeSize(PCS), StartSizeWord(SSW) {} }; /// BlockScope - This tracks the current blocks that we have entered. @@ -61,12 +63,6 @@ class BitstreamWriter { }; std::vector<BlockInfo> BlockInfoRecords; - // BackpatchWord - Backpatch a 32-bit word in the output with the specified - // value. - void BackpatchWord(unsigned ByteNo, unsigned NewWord) { - support::endian::write32le(&Out[ByteNo], NewWord); - } - void WriteByte(unsigned char Value) { Out.push_back(Value); } @@ -77,12 +73,10 @@ class BitstreamWriter { reinterpret_cast<const char *>(&Value + 1)); } - unsigned GetBufferOffset() const { - return Out.size(); - } + size_t GetBufferOffset() const { return Out.size(); } - unsigned GetWordIndex() const { - unsigned Offset = GetBufferOffset(); + size_t GetWordIndex() const { + size_t Offset = GetBufferOffset(); assert((Offset & 3) == 0 && "Not 32-bit aligned"); return Offset / 4; } @@ -99,10 +93,25 @@ public: /// \brief Retrieve the current position in the stream, in bits. uint64_t GetCurrentBitNo() const { return GetBufferOffset() * 8 + CurBit; } + /// \brief Retrieve the number of bits currently used to encode an abbrev ID. + unsigned GetAbbrevIDWidth() const { return CurCodeSize; } + //===--------------------------------------------------------------------===// // Basic Primitives for emitting bits to the stream. //===--------------------------------------------------------------------===// + /// Backpatch a 32-bit word in the output at the given bit offset + /// with the specified value. + void BackpatchWord(uint64_t BitNo, unsigned NewWord) { + using namespace llvm::support; + unsigned ByteNo = BitNo / 8; + assert((!endian::readAtBitAlignment<uint32_t, little, unaligned>( + &Out[ByteNo], BitNo & 7)) && + "Expected to be patching over 0-value placeholders"); + endian::writeAtBitAlignment<uint32_t, little, unaligned>( + &Out[ByteNo], NewWord, BitNo & 7); + } + void Emit(uint32_t Val, unsigned NumBits) { assert(NumBits && NumBits <= 32 && "Invalid value size!"); assert((Val & ~(~0U >> (32-NumBits))) == 0 && "High bits set!"); @@ -200,7 +209,7 @@ public: EmitVBR(CodeLen, bitc::CodeLenWidth); FlushToWord(); - unsigned BlockSizeWordIndex = GetWordIndex(); + size_t BlockSizeWordIndex = GetWordIndex(); unsigned OldCodeSize = CurCodeSize; // Emit a placeholder, which will be replaced when the block is popped. @@ -231,11 +240,11 @@ public: FlushToWord(); // Compute the size of the block, in words, not counting the size field. - unsigned SizeInWords = GetWordIndex() - B.StartSizeWord - 1; - unsigned ByteNo = B.StartSizeWord*4; + size_t SizeInWords = GetWordIndex() - B.StartSizeWord - 1; + uint64_t BitNo = uint64_t(B.StartSizeWord) * 32; // Update the block size field in the header of this sub-block. - BackpatchWord(ByteNo, SizeInWords); + BackpatchWord(BitNo, SizeInWords); // Restore the inner block's code size and abbrev table. CurCodeSize = B.PrevCodeSize; @@ -285,10 +294,12 @@ private: /// EmitRecordWithAbbrevImpl - This is the core implementation of the record /// emission code. If BlobData is non-null, then it specifies an array of /// data that should be emitted as part of the Blob or Array operand that is - /// known to exist at the end of the record. - template<typename uintty> - void EmitRecordWithAbbrevImpl(unsigned Abbrev, SmallVectorImpl<uintty> &Vals, - StringRef Blob) { + /// known to exist at the end of the record. If Code is specified, then + /// it is the record code to emit before the Vals, which must not contain + /// the code. + template <typename uintty> + void EmitRecordWithAbbrevImpl(unsigned Abbrev, ArrayRef<uintty> Vals, + StringRef Blob, Optional<unsigned> Code) { const char *BlobData = Blob.data(); unsigned BlobLen = (unsigned) Blob.size(); unsigned AbbrevNo = Abbrev-bitc::FIRST_APPLICATION_ABBREV; @@ -297,9 +308,23 @@ private: EmitCode(Abbrev); + unsigned i = 0, e = static_cast<unsigned>(Abbv->getNumOperandInfos()); + if (Code) { + assert(e && "Expected non-empty abbreviation"); + const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i++); + + if (Op.isLiteral()) + EmitAbbreviatedLiteral(Op, Code.getValue()); + else { + assert(Op.getEncoding() != BitCodeAbbrevOp::Array && + Op.getEncoding() != BitCodeAbbrevOp::Blob && + "Expected literal or scalar"); + EmitAbbreviatedField(Op, Code.getValue()); + } + } + unsigned RecordIdx = 0; - for (unsigned i = 0, e = static_cast<unsigned>(Abbv->getNumOperandInfos()); - i != e; ++i) { + for (; i != e; ++i) { const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); if (Op.isLiteral()) { assert(RecordIdx < Vals.size() && "Invalid abbrev/record"); @@ -307,7 +332,7 @@ private: ++RecordIdx; } else if (Op.getEncoding() == BitCodeAbbrevOp::Array) { // Array case. - assert(i+2 == e && "array op not second to last?"); + assert(i + 2 == e && "array op not second to last?"); const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i); // If this record has blob data, emit it, otherwise we must have record @@ -381,32 +406,29 @@ public: /// EmitRecord - Emit the specified record to the stream, using an abbrev if /// we have one to compress the output. - template<typename uintty> - void EmitRecord(unsigned Code, SmallVectorImpl<uintty> &Vals, - unsigned Abbrev = 0) { + template <typename Container> + void EmitRecord(unsigned Code, const Container &Vals, unsigned Abbrev = 0) { if (!Abbrev) { // If we don't have an abbrev to use, emit this in its fully unabbreviated // form. + auto Count = static_cast<uint32_t>(makeArrayRef(Vals).size()); EmitCode(bitc::UNABBREV_RECORD); EmitVBR(Code, 6); - EmitVBR(static_cast<uint32_t>(Vals.size()), 6); - for (unsigned i = 0, e = static_cast<unsigned>(Vals.size()); i != e; ++i) + EmitVBR(Count, 6); + for (unsigned i = 0, e = Count; i != e; ++i) EmitVBR64(Vals[i], 6); return; } - // Insert the code into Vals to treat it uniformly. - Vals.insert(Vals.begin(), Code); - - EmitRecordWithAbbrev(Abbrev, Vals); + EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), StringRef(), Code); } /// EmitRecordWithAbbrev - Emit a record with the specified abbreviation. /// Unlike EmitRecord, the code for the record should be included in Vals as /// the first entry. - template<typename uintty> - void EmitRecordWithAbbrev(unsigned Abbrev, SmallVectorImpl<uintty> &Vals) { - EmitRecordWithAbbrevImpl(Abbrev, Vals, StringRef()); + template <typename Container> + void EmitRecordWithAbbrev(unsigned Abbrev, const Container &Vals) { + EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), StringRef(), None); } /// EmitRecordWithBlob - Emit the specified record to the stream, using an @@ -414,29 +436,30 @@ public: /// specified by the pointer and length specified at the end. In contrast to /// EmitRecord, this routine expects that the first entry in Vals is the code /// of the record. - template<typename uintty> - void EmitRecordWithBlob(unsigned Abbrev, SmallVectorImpl<uintty> &Vals, + template <typename Container> + void EmitRecordWithBlob(unsigned Abbrev, const Container &Vals, StringRef Blob) { - EmitRecordWithAbbrevImpl(Abbrev, Vals, Blob); + EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), Blob, None); } - template<typename uintty> - void EmitRecordWithBlob(unsigned Abbrev, SmallVectorImpl<uintty> &Vals, + template <typename Container> + void EmitRecordWithBlob(unsigned Abbrev, const Container &Vals, const char *BlobData, unsigned BlobLen) { - return EmitRecordWithAbbrevImpl(Abbrev, Vals, StringRef(BlobData, BlobLen)); + return EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), + StringRef(BlobData, BlobLen), None); } /// EmitRecordWithArray - Just like EmitRecordWithBlob, works with records /// that end with an array. - template<typename uintty> - void EmitRecordWithArray(unsigned Abbrev, SmallVectorImpl<uintty> &Vals, - StringRef Array) { - EmitRecordWithAbbrevImpl(Abbrev, Vals, Array); + template <typename Container> + void EmitRecordWithArray(unsigned Abbrev, const Container &Vals, + StringRef Array) { + EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), Array, None); } - template<typename uintty> - void EmitRecordWithArray(unsigned Abbrev, SmallVectorImpl<uintty> &Vals, - const char *ArrayData, unsigned ArrayLen) { - return EmitRecordWithAbbrevImpl(Abbrev, Vals, StringRef(ArrayData, - ArrayLen)); + template <typename Container> + void EmitRecordWithArray(unsigned Abbrev, const Container &Vals, + const char *ArrayData, unsigned ArrayLen) { + return EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), + StringRef(ArrayData, ArrayLen), None); } //===--------------------------------------------------------------------===// diff --git a/include/llvm/Bitcode/LLVMBitCodes.h b/include/llvm/Bitcode/LLVMBitCodes.h index 7130ee755237..bcc84bedbed0 100644 --- a/include/llvm/Bitcode/LLVMBitCodes.h +++ b/include/llvm/Bitcode/LLVMBitCodes.h @@ -23,28 +23,52 @@ namespace llvm { namespace bitc { // The only top-level block type defined is for a module. - enum BlockIDs { - // Blocks - MODULE_BLOCK_ID = FIRST_APPLICATION_BLOCKID, +enum BlockIDs { + // Blocks + MODULE_BLOCK_ID = FIRST_APPLICATION_BLOCKID, - // Module sub-block id's. - PARAMATTR_BLOCK_ID, - PARAMATTR_GROUP_BLOCK_ID, + // Module sub-block id's. + PARAMATTR_BLOCK_ID, + PARAMATTR_GROUP_BLOCK_ID, - CONSTANTS_BLOCK_ID, - FUNCTION_BLOCK_ID, + CONSTANTS_BLOCK_ID, + FUNCTION_BLOCK_ID, - UNUSED_ID1, + // Block intended to contains information on the bitcode versioning. + // Can be used to provide better error messages when we fail to parse a + // bitcode file. + IDENTIFICATION_BLOCK_ID, - VALUE_SYMTAB_BLOCK_ID, - METADATA_BLOCK_ID, - METADATA_ATTACHMENT_ID, + VALUE_SYMTAB_BLOCK_ID, + METADATA_BLOCK_ID, + METADATA_ATTACHMENT_ID, - TYPE_BLOCK_ID_NEW, + TYPE_BLOCK_ID_NEW, - USELIST_BLOCK_ID - }; + USELIST_BLOCK_ID, + + MODULE_STRTAB_BLOCK_ID, + FUNCTION_SUMMARY_BLOCK_ID, + + OPERAND_BUNDLE_TAGS_BLOCK_ID, + + METADATA_KIND_BLOCK_ID +}; + +/// Identification block contains a string that describes the producer details, +/// and an epoch that defines the auto-upgrade capability. +enum IdentificationCodes { + IDENTIFICATION_CODE_STRING = 1, // IDENTIFICATION: [strchr x N] + IDENTIFICATION_CODE_EPOCH = 2, // EPOCH: [epoch#] +}; +/// The epoch that defines the auto-upgrade compatibility for the bitcode. +/// +/// LLVM guarantees in a major release that a minor release can read bitcode +/// generated by previous minor releases. We translate this by making the reader +/// accepting only bitcode with the same epoch, except for the X.0 release which +/// also accepts N-1. +enum { BITCODE_CURRENT_EPOCH = 0 }; /// MODULE blocks have a number of optional fields and subblocks. enum ModuleCodes { @@ -66,13 +90,21 @@ namespace bitc { MODULE_CODE_FUNCTION = 8, // ALIAS: [alias type, aliasee val#, linkage, visibility] - MODULE_CODE_ALIAS = 9, + MODULE_CODE_ALIAS_OLD = 9, // MODULE_CODE_PURGEVALS: [numvals] MODULE_CODE_PURGEVALS = 10, MODULE_CODE_GCNAME = 11, // GCNAME: [strchr x N] MODULE_CODE_COMDAT = 12, // COMDAT: [selection_kind, name] + + MODULE_CODE_VSTOFFSET = 13, // VSTOFFSET: [offset] + + // ALIAS: [alias value type, addrspace, aliasee val#, linkage, visibility] + MODULE_CODE_ALIAS = 14, + + // METADATA_VALUES: [numvals] + MODULE_CODE_METADATA_VALUES = 15, }; /// PARAMATTR blocks have code for defining a parameter attribute set. @@ -121,7 +153,13 @@ namespace bitc { TYPE_CODE_STRUCT_NAME = 19, // STRUCT_NAME: [strchr x N] TYPE_CODE_STRUCT_NAMED = 20,// STRUCT_NAMED: [ispacked, eltty x N] - TYPE_CODE_FUNCTION = 21 // FUNCTION: [vararg, retty, paramty x N] + TYPE_CODE_FUNCTION = 21, // FUNCTION: [vararg, retty, paramty x N] + + TYPE_CODE_TOKEN = 22 // TOKEN + }; + + enum OperandBundleTagCode { + OPERAND_BUNDLE_TAG = 1, // TAG: [strchr x N] }; // The type symbol table only has one code (TST_ENTRY_CODE). @@ -129,10 +167,25 @@ namespace bitc { TST_CODE_ENTRY = 1 // TST_ENTRY: [typeid, namechar x N] }; - // The value symbol table only has one code (VST_ENTRY_CODE). + // Value symbol table codes. enum ValueSymtabCodes { - VST_CODE_ENTRY = 1, // VST_ENTRY: [valid, namechar x N] - VST_CODE_BBENTRY = 2 // VST_BBENTRY: [bbid, namechar x N] + VST_CODE_ENTRY = 1, // VST_ENTRY: [valueid, namechar x N] + VST_CODE_BBENTRY = 2, // VST_BBENTRY: [bbid, namechar x N] + VST_CODE_FNENTRY = 3, // VST_FNENTRY: [valueid, offset, namechar x N] + // VST_COMBINED_FNENTRY: [offset, namechar x N] + VST_CODE_COMBINED_FNENTRY = 4 + }; + + // The module path symbol table only has one code (MST_CODE_ENTRY). + enum ModulePathSymtabCodes { + MST_CODE_ENTRY = 1, // MST_ENTRY: [modid, namechar x N] + }; + + // The function summary section uses different codes in the per-module + // and combined index cases. + enum FunctionSummarySymtabCodes { + FS_CODE_PERMODULE_ENTRY = 1, // FS_ENTRY: [valueid, islocal, instcount] + FS_CODE_COMBINED_ENTRY = 2, // FS_ENTRY: [modid, instcount] }; enum MetadataCodes { @@ -167,7 +220,9 @@ namespace bitc { METADATA_EXPRESSION = 29, // [distinct, n x element] METADATA_OBJC_PROPERTY = 30, // [distinct, name, file, line, ...] METADATA_IMPORTED_ENTITY=31, // [distinct, tag, scope, entity, line, name] - METADATA_MODULE=32, // [distinct, scope, name, ...] + METADATA_MODULE = 32, // [distinct, scope, name, ...] + METADATA_MACRO = 33, // [distinct, macinfo, line, name, value] + METADATA_MACRO_FILE = 34, // [distinct, macinfo, line, file, ...] }; // The constants block (CONSTANTS_BLOCK_ID) describes emission for each @@ -287,6 +342,16 @@ namespace bitc { SYNCHSCOPE_CROSSTHREAD = 1 }; + /// Markers and flags for call instruction. + enum CallMarkersFlags { + CALL_TAIL = 0, + CALL_CCONV = 1, + CALL_MUSTTAIL = 14, + CALL_EXPLICIT_TYPE = 15, + CALL_NOTAIL = 16, + CALL_FMF = 17 // Call has optional fast-math-flags. + }; + // The function body block (FUNCTION_BLOCK_ID) describes function bodies. It // can contain a constant block (CONSTANTS_BLOCK_ID). enum FunctionCodes { @@ -354,6 +419,14 @@ namespace bitc { FUNC_CODE_INST_CMPXCHG = 46, // CMPXCHG: [ptrty,ptr,valty,cmp,new, align, // vol,ordering,synchscope] FUNC_CODE_INST_LANDINGPAD = 47, // LANDINGPAD: [ty,val,num,id0,val0...] + FUNC_CODE_INST_CLEANUPRET = 48, // CLEANUPRET: [val] or [val,bb#] + FUNC_CODE_INST_CATCHRET = 49, // CATCHRET: [val,bb#] + FUNC_CODE_INST_CATCHPAD = 50, // CATCHPAD: [bb#,bb#,num,args...] + FUNC_CODE_INST_CLEANUPPAD = 51, // CLEANUPPAD: [num,args...] + FUNC_CODE_INST_CATCHSWITCH = 52, // CATCHSWITCH: [num,args...] or [num,args...,bb] + // 53 is unused. + // 54 is unused. + FUNC_CODE_OPERAND_BUNDLE = 55, // OPERAND_BUNDLE: [tag#, value...] }; enum UseListCodes { @@ -407,7 +480,12 @@ namespace bitc { ATTR_KIND_DEREFERENCEABLE_OR_NULL = 42, ATTR_KIND_CONVERGENT = 43, ATTR_KIND_SAFESTACK = 44, - ATTR_KIND_ARGMEMONLY = 45 + ATTR_KIND_ARGMEMONLY = 45, + ATTR_KIND_SWIFT_SELF = 46, + ATTR_KIND_SWIFT_ERROR = 47, + ATTR_KIND_NO_RECURSE = 48, + ATTR_KIND_INACCESSIBLEMEM_ONLY = 49, + ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY = 50 }; enum ComdatSelectionKindCodes { diff --git a/include/llvm/Bitcode/ReaderWriter.h b/include/llvm/Bitcode/ReaderWriter.h index 452ec3bd0187..60d865fd2355 100644 --- a/include/llvm/Bitcode/ReaderWriter.h +++ b/include/llvm/Bitcode/ReaderWriter.h @@ -15,6 +15,7 @@ #define LLVM_BITCODE_READERWRITER_H #include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/FunctionInfo.h" #include "llvm/Support/Endian.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/MemoryBuffer.h" @@ -36,27 +37,54 @@ namespace llvm { ErrorOr<std::unique_ptr<Module>> getLazyBitcodeModule(std::unique_ptr<MemoryBuffer> &&Buffer, LLVMContext &Context, - DiagnosticHandlerFunction DiagnosticHandler = nullptr, bool ShouldLazyLoadMetadata = false); /// Read the header of the specified stream and prepare for lazy /// deserialization and streaming of function bodies. - ErrorOr<std::unique_ptr<Module>> getStreamedBitcodeModule( - StringRef Name, std::unique_ptr<DataStreamer> Streamer, - LLVMContext &Context, - DiagnosticHandlerFunction DiagnosticHandler = nullptr); + ErrorOr<std::unique_ptr<Module>> + getStreamedBitcodeModule(StringRef Name, + std::unique_ptr<DataStreamer> Streamer, + LLVMContext &Context); /// Read the header of the specified bitcode buffer and extract just the /// triple information. If successful, this returns a string. On error, this /// returns "". - std::string - getBitcodeTargetTriple(MemoryBufferRef Buffer, LLVMContext &Context, - DiagnosticHandlerFunction DiagnosticHandler = nullptr); + std::string getBitcodeTargetTriple(MemoryBufferRef Buffer, + LLVMContext &Context); + + /// Read the header of the specified bitcode buffer and extract just the + /// producer string information. If successful, this returns a string. On + /// error, this returns "". + std::string getBitcodeProducerString(MemoryBufferRef Buffer, + LLVMContext &Context); /// Read the specified bitcode file, returning the module. - ErrorOr<std::unique_ptr<Module>> - parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context, - DiagnosticHandlerFunction DiagnosticHandler = nullptr); + ErrorOr<std::unique_ptr<Module>> parseBitcodeFile(MemoryBufferRef Buffer, + LLVMContext &Context); + + /// Check if the given bitcode buffer contains a function summary block. + bool hasFunctionSummary(MemoryBufferRef Buffer, + DiagnosticHandlerFunction DiagnosticHandler); + + /// Parse the specified bitcode buffer, returning the function info index. + /// If IsLazy is true, parse the entire function summary into + /// the index. Otherwise skip the function summary section, and only create + /// an index object with a map from function name to function summary offset. + /// The index is used to perform lazy function summary reading later. + ErrorOr<std::unique_ptr<FunctionInfoIndex>> + getFunctionInfoIndex(MemoryBufferRef Buffer, + DiagnosticHandlerFunction DiagnosticHandler, + bool IsLazy = false); + + /// This method supports lazy reading of function summary data from the + /// combined index during function importing. When reading the combined index + /// file, getFunctionInfoIndex is first invoked with IsLazy=true. + /// Then this method is called for each function considered for importing, + /// to parse the summary information for the given function name into + /// the index. + std::error_code readFunctionSummary( + MemoryBufferRef Buffer, DiagnosticHandlerFunction DiagnosticHandler, + StringRef FunctionName, std::unique_ptr<FunctionInfoIndex> Index); /// \brief Write the specified module to the specified raw output stream. /// @@ -66,8 +94,18 @@ namespace llvm { /// If \c ShouldPreserveUseListOrder, encode the use-list order for each \a /// Value in \c M. These will be reconstructed exactly when \a M is /// deserialized. + /// + /// If \c EmitFunctionSummary, emit the function summary index (currently + /// for use in ThinLTO optimization). void WriteBitcodeToFile(const Module *M, raw_ostream &Out, - bool ShouldPreserveUseListOrder = false); + bool ShouldPreserveUseListOrder = false, + bool EmitFunctionSummary = false); + + /// Write the specified function summary index to the given raw output stream, + /// where it will be written in a new bitcode block. This is used when + /// writing the combined index file for ThinLTO. + void WriteFunctionSummaryToFile(const FunctionInfoIndex &Index, + raw_ostream &Out); /// isBitcodeWrapper - Return true if the given bytes are the magic bytes /// for an LLVM IR bitcode wrapper. @@ -159,7 +197,7 @@ namespace llvm { BitcodeDiagnosticInfo(std::error_code EC, DiagnosticSeverity Severity, const Twine &Msg); void print(DiagnosticPrinter &DP) const override; - std::error_code getError() const { return EC; }; + std::error_code getError() const { return EC; } static bool classof(const DiagnosticInfo *DI) { return DI->getKind() == DK_Bitcode; diff --git a/include/llvm/CodeGen/Analysis.h b/include/llvm/CodeGen/Analysis.h index 82d1e8ada17d..38e64ad3be29 100644 --- a/include/llvm/CodeGen/Analysis.h +++ b/include/llvm/CodeGen/Analysis.h @@ -15,6 +15,7 @@ #define LLVM_CODEGEN_ANALYSIS_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/IR/CallSite.h" @@ -23,6 +24,8 @@ namespace llvm { class GlobalValue; +class MachineBasicBlock; +class MachineFunction; class TargetLoweringBase; class TargetLowering; class TargetMachine; @@ -37,7 +40,7 @@ struct EVT; /// Given an LLVM IR aggregate type and a sequence of insertvalue or /// extractvalue indices that identify a member, return the linearized index of /// the start of the member, i.e the number of element in memory before the -/// seeked one. This is disconnected from the number of bytes. +/// sought one. This is disconnected from the number of bytes. /// /// \param Ty is the type indexed by \p Indices. /// \param Indices is an optional pointer in the indices list to the current @@ -115,6 +118,9 @@ bool returnTypeIsEligibleForTailCall(const Function *F, // or we are in LTO. bool canBeOmittedFromSymbolTable(const GlobalValue *GV); +DenseMap<const MachineBasicBlock *, int> +getFuncletMembership(const MachineFunction &MF); + } // End llvm namespace #endif diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h index fe7efae325c4..f5e778b2f262 100644 --- a/include/llvm/CodeGen/AsmPrinter.h +++ b/include/llvm/CodeGen/AsmPrinter.h @@ -165,6 +165,9 @@ public: /// Return information about data layout. const DataLayout &getDataLayout() const; + /// Return the pointer size from the TargetMachine + unsigned getPointerSize() const; + /// Return information about subtarget. const MCSubtargetInfo &getSubtargetInfo() const; @@ -233,7 +236,12 @@ public: /// Print assembly representations of the jump tables used by the current /// function to the current output stream. /// - void EmitJumpTableInfo(); + virtual void EmitJumpTableInfo(); + + /// Emit the control variable for an emulated TLS variable. + virtual void EmitEmulatedTLSControlVariable(const GlobalVariable *GV, + MCSymbol *EmittedSym, + bool AllZeroInitValue); /// Emit the specified global variable to the .s file. virtual void EmitGlobalVariable(const GlobalVariable *GV); @@ -254,7 +262,7 @@ public: const MCExpr *lowerConstant(const Constant *CV); /// \brief Print a general LLVM constant to the .s file. - void EmitGlobalConstant(const Constant *CV); + void EmitGlobalConstant(const DataLayout &DL, const Constant *CV); /// \brief Unnamed constant global variables solely contaning a pointer to /// another globals variable act like a global variable "proxy", or GOT @@ -317,7 +325,9 @@ public: /// Targets can override this to change how global constants that are part of /// a C++ static/global constructor list are emitted. - virtual void EmitXXStructor(const Constant *CV) { EmitGlobalConstant(CV); } + virtual void EmitXXStructor(const DataLayout &DL, const Constant *CV) { + EmitGlobalConstant(DL, CV); + } /// Return true if the basic block has exactly one predecessor and the control /// transfer mechanism between the predecessor and this block is a @@ -404,9 +414,6 @@ public: void EmitULEB128(uint64_t Value, const char *Desc = nullptr, unsigned PadTo = 0) const; - /// Emit a .byte 42 directive for a DW_CFA_xxx value. - void EmitCFAByte(unsigned Val) const; - /// Emit a .byte 42 directive that corresponds to an encoding. If verbose /// assembly output is enabled, we output comments describing the encoding. /// Desc is a string saying what the encoding is specifying (e.g. "LSDA"). @@ -446,7 +453,16 @@ public: void emitCFIInstruction(const MCCFIInstruction &Inst) const; /// \brief Emit Dwarf abbreviation table. - void emitDwarfAbbrevs(const std::vector<DIEAbbrev *>& Abbrevs) const; + template <typename T> void emitDwarfAbbrevs(const T &Abbrevs) const { + // For each abbreviation. + for (const auto &Abbrev : Abbrevs) + emitDwarfAbbrev(*Abbrev); + + // Mark end of abbreviations. + EmitULEB128(0, "EOM(3)"); + } + + void emitDwarfAbbrev(const DIEAbbrev &Abbrev) const; /// \brief Recursively emit Dwarf DIE tree. void emitDwarfDIE(const DIE &Die) const; @@ -532,7 +548,8 @@ private: void EmitLLVMUsedList(const ConstantArray *InitList); /// Emit llvm.ident metadata in an '.ident' directive. void EmitModuleIdents(Module &M); - void EmitXXStructorList(const Constant *List, bool isCtor); + void EmitXXStructorList(const DataLayout &DL, const Constant *List, + bool isCtor); GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy &C); }; } diff --git a/include/llvm/CodeGen/AtomicExpandUtils.h b/include/llvm/CodeGen/AtomicExpandUtils.h new file mode 100644 index 000000000000..ac18eac8a1ce --- /dev/null +++ b/include/llvm/CodeGen/AtomicExpandUtils.h @@ -0,0 +1,57 @@ +//===-- AtomicExpandUtils.h - Utilities for expanding atomic instructions -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/STLExtras.h" +#include "llvm/IR/IRBuilder.h" + +namespace llvm { +class Value; +class AtomicRMWInst; + + +/// Parameters (see the expansion example below): +/// (the builder, %addr, %loaded, %new_val, ordering, +/// /* OUT */ %success, /* OUT */ %new_loaded) +typedef function_ref<void(IRBuilder<> &, Value *, Value *, Value *, + AtomicOrdering, Value *&, Value *&)> CreateCmpXchgInstFun; + +/// \brief Expand an atomic RMW instruction into a loop utilizing +/// cmpxchg. You'll want to make sure your target machine likes cmpxchg +/// instructions in the first place and that there isn't another, better, +/// transformation available (for example AArch32/AArch64 have linked loads). +/// +/// This is useful in passes which can't rewrite the more exotic RMW +/// instructions directly into a platform specific intrinsics (because, say, +/// those intrinsics don't exist). If such a pass is able to expand cmpxchg +/// instructions directly however, then, with this function, it could avoid two +/// extra module passes (avoiding passes by `-atomic-expand` and itself). A +/// specific example would be PNaCl's `RewriteAtomics` pass. +/// +/// Given: atomicrmw some_op iN* %addr, iN %incr ordering +/// +/// The standard expansion we produce is: +/// [...] +/// %init_loaded = load atomic iN* %addr +/// br label %loop +/// loop: +/// %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ] +/// %new = some_op iN %loaded, %incr +/// ; This is what -atomic-expand will produce using this function on i686 targets: +/// %pair = cmpxchg iN* %addr, iN %loaded, iN %new_val +/// %new_loaded = extractvalue { iN, i1 } %pair, 0 +/// %success = extractvalue { iN, i1 } %pair, 1 +/// ; End callback produced IR +/// br i1 %success, label %atomicrmw.end, label %loop +/// atomicrmw.end: +/// [...] +/// +/// Returns true if the containing function was modified. +bool +expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, CreateCmpXchgInstFun Factory); +} diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h index 9ba25169fda6..d99054eb6f36 100644 --- a/include/llvm/CodeGen/BasicTTIImpl.h +++ b/include/llvm/CodeGen/BasicTTIImpl.h @@ -166,7 +166,7 @@ public: } if (IID == Intrinsic::ctlz) { - if (getTLI()->isCheapToSpeculateCtlz()) + if (getTLI()->isCheapToSpeculateCtlz()) return TargetTransformInfo::TCC_Basic; return TargetTransformInfo::TCC_Expensive; } @@ -256,7 +256,7 @@ public: for (BasicBlock::iterator J = BB->begin(), JE = BB->end(); J != JE; ++J) if (isa<CallInst>(J) || isa<InvokeInst>(J)) { - ImmutableCallSite CS(J); + ImmutableCallSite CS(&*J); if (const Function *F = CS.getCalledFunction()) { if (!static_cast<T *>(this)->isLoweredToCall(F)) continue; @@ -302,12 +302,8 @@ public: if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { // The operation is legal. Assume it costs 1. - // If the type is split to multiple registers, assume that there is some - // overhead to this. // TODO: Once we have extract/insert subvector cost we need to use them. - if (LT.first > 1) - return LT.first * 2 * OpCost; - return LT.first * 1 * OpCost; + return LT.first * OpCost; } if (!TLI->isOperationExpand(ISD, LT.second)) { @@ -496,13 +492,11 @@ public: // itself. Unless the corresponding extending load or truncating store is // legal, then this will scalarize. TargetLowering::LegalizeAction LA = TargetLowering::Expand; - EVT MemVT = getTLI()->getValueType(DL, Src, true); - if (MemVT.isSimple() && MemVT != MVT::Other) { - if (Opcode == Instruction::Store) - LA = getTLI()->getTruncStoreAction(LT.second, MemVT.getSimpleVT()); - else - LA = getTLI()->getLoadExtAction(ISD::EXTLOAD, LT.second, MemVT); - } + EVT MemVT = getTLI()->getValueType(DL, Src); + if (Opcode == Instruction::Store) + LA = getTLI()->getTruncStoreAction(LT.second, MemVT); + else + LA = getTLI()->getLoadExtAction(ISD::EXTLOAD, LT.second, MemVT); if (LA != TargetLowering::Legal && LA != TargetLowering::Custom) { // This is a vector load/store for some illegal type that is scalarized. @@ -530,7 +524,8 @@ public: VectorType *SubVT = VectorType::get(VT->getElementType(), NumSubElts); // Firstly, the cost of load/store operation. - unsigned Cost = getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace); + unsigned Cost = static_cast<T *>(this)->getMemoryOpCost( + Opcode, VecTy, Alignment, AddressSpace); // Then plus the cost of interleave operation. if (Opcode == Instruction::Load) { @@ -545,18 +540,20 @@ public: assert(Indices.size() <= Factor && "Interleaved memory op has too many members"); + for (unsigned Index : Indices) { assert(Index < Factor && "Invalid index for interleaved memory op"); // Extract elements from loaded vector for each sub vector. for (unsigned i = 0; i < NumSubElts; i++) - Cost += getVectorInstrCost(Instruction::ExtractElement, VT, - Index + i * Factor); + Cost += static_cast<T *>(this)->getVectorInstrCost( + Instruction::ExtractElement, VT, Index + i * Factor); } unsigned InsSubCost = 0; for (unsigned i = 0; i < NumSubElts; i++) - InsSubCost += getVectorInstrCost(Instruction::InsertElement, SubVT, i); + InsSubCost += static_cast<T *>(this)->getVectorInstrCost( + Instruction::InsertElement, SubVT, i); Cost += Indices.size() * InsSubCost; } else { @@ -571,17 +568,51 @@ public: unsigned ExtSubCost = 0; for (unsigned i = 0; i < NumSubElts; i++) - ExtSubCost += getVectorInstrCost(Instruction::ExtractElement, SubVT, i); - - Cost += Factor * ExtSubCost; + ExtSubCost += static_cast<T *>(this)->getVectorInstrCost( + Instruction::ExtractElement, SubVT, i); + Cost += ExtSubCost * Factor; for (unsigned i = 0; i < NumElts; i++) - Cost += getVectorInstrCost(Instruction::InsertElement, VT, i); + Cost += static_cast<T *>(this) + ->getVectorInstrCost(Instruction::InsertElement, VT, i); } return Cost; } + /// Get intrinsic cost based on arguments + unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Value *> Args) { + switch (IID) { + default: { + SmallVector<Type *, 4> Types; + for (Value *Op : Args) + Types.push_back(Op->getType()); + return getIntrinsicInstrCost(IID, RetTy, Types); + } + case Intrinsic::masked_scatter: { + Value *Mask = Args[3]; + bool VarMask = !isa<Constant>(Mask); + unsigned Alignment = cast<ConstantInt>(Args[2])->getZExtValue(); + return + static_cast<T *>(this)->getGatherScatterOpCost(Instruction::Store, + Args[0]->getType(), + Args[1], VarMask, + Alignment); + } + case Intrinsic::masked_gather: { + Value *Mask = Args[2]; + bool VarMask = !isa<Constant>(Mask); + unsigned Alignment = cast<ConstantInt>(Args[1])->getZExtValue(); + return + static_cast<T *>(this)->getGatherScatterOpCost(Instruction::Load, + RetTy, Args[0], VarMask, + Alignment); + } + } + } + + /// Get intrinsic cost based on argument types unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, ArrayRef<Type *> Tys) { unsigned ISD = 0; @@ -800,7 +831,7 @@ class BasicTTIImpl : public BasicTTIImplBase<BasicTTIImpl> { const TargetLoweringBase *getTLI() const { return TLI; } public: - explicit BasicTTIImpl(const TargetMachine *ST, Function &F); + explicit BasicTTIImpl(const TargetMachine *ST, const Function &F); // Provide value semantics. MSVC requires that we spell all of these out. BasicTTIImpl(const BasicTTIImpl &Arg) diff --git a/include/llvm/CodeGen/CalcSpillWeights.h b/include/llvm/CodeGen/CalcSpillWeights.h index 91fb0a9d7e77..17c9415a81cb 100644 --- a/include/llvm/CodeGen/CalcSpillWeights.h +++ b/include/llvm/CodeGen/CalcSpillWeights.h @@ -20,6 +20,7 @@ namespace llvm { class LiveIntervals; class MachineBlockFrequencyInfo; class MachineLoopInfo; + class VirtRegMap; /// \brief Normalize the spill weight of a live interval /// @@ -51,6 +52,7 @@ namespace llvm { private: MachineFunction &MF; LiveIntervals &LIS; + VirtRegMap *VRM; const MachineLoopInfo &Loops; const MachineBlockFrequencyInfo &MBFI; DenseMap<unsigned, float> Hint; @@ -58,10 +60,10 @@ namespace llvm { public: VirtRegAuxInfo(MachineFunction &mf, LiveIntervals &lis, - const MachineLoopInfo &loops, + VirtRegMap *vrm, const MachineLoopInfo &loops, const MachineBlockFrequencyInfo &mbfi, NormalizingFn norm = normalizeSpillWeight) - : MF(mf), LIS(lis), Loops(loops), MBFI(mbfi), normalize(norm) {} + : MF(mf), LIS(lis), VRM(vrm), Loops(loops), MBFI(mbfi), normalize(norm) {} /// \brief (re)compute li's spill weight and allocation hint. void calculateSpillWeightAndHint(LiveInterval &li); @@ -70,6 +72,7 @@ namespace llvm { /// \brief Compute spill weights and allocation hints for all virtual register /// live intervals. void calculateSpillWeightsAndHints(LiveIntervals &LIS, MachineFunction &MF, + VirtRegMap *VRM, const MachineLoopInfo &MLI, const MachineBlockFrequencyInfo &MBFI, VirtRegAuxInfo::NormalizingFn norm = diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h index 1fd4eeb46b38..415abb90da57 100644 --- a/include/llvm/CodeGen/CallingConvLower.h +++ b/include/llvm/CodeGen/CallingConvLower.h @@ -201,6 +201,7 @@ private: LLVMContext &Context; unsigned StackOffset; + unsigned MaxStackArgAlign; SmallVector<uint32_t, 16> UsedRegs; SmallVector<CCValAssign, 4> PendingLocs; @@ -270,7 +271,18 @@ public: CallingConv::ID getCallingConv() const { return CallingConv; } bool isVarArg() const { return IsVarArg; } - unsigned getNextStackOffset() const { return StackOffset; } + /// getNextStackOffset - Return the next stack offset such that all stack + /// slots satisfy their alignment requirements. + unsigned getNextStackOffset() const { + return StackOffset; + } + + /// getAlignedCallFrameSize - Return the size of the call frame needed to + /// be able to store all arguments and such that the alignment requirement + /// of each of the arguments is satisfied. + unsigned getAlignedCallFrameSize() const { + return RoundUpToAlignment(StackOffset, MaxStackArgAlign); + } /// isAllocated - Return true if the specified register (or an alias) is /// allocated. @@ -357,7 +369,7 @@ public: /// AllocateRegBlock - Attempt to allocate a block of RegsRequired consecutive /// registers. If this is not possible, return zero. Otherwise, return the first /// register of the block that were allocated, marking the entire block as allocated. - unsigned AllocateRegBlock(ArrayRef<uint16_t> Regs, unsigned RegsRequired) { + unsigned AllocateRegBlock(ArrayRef<MCPhysReg> Regs, unsigned RegsRequired) { if (RegsRequired > Regs.size()) return 0; @@ -400,9 +412,10 @@ public: /// and alignment. unsigned AllocateStack(unsigned Size, unsigned Align) { assert(Align && ((Align - 1) & Align) == 0); // Align is power of 2. - StackOffset = ((StackOffset + Align - 1) & ~(Align - 1)); + StackOffset = RoundUpToAlignment(StackOffset, Align); unsigned Result = StackOffset; StackOffset += Size; + MaxStackArgAlign = std::max(Align, MaxStackArgAlign); MF.getFrameInfo()->ensureMaxAlignment(Align); return Result; } diff --git a/include/llvm/CodeGen/CommandFlags.h b/include/llvm/CodeGen/CommandFlags.h index bedb7d5549eb..0d37dc00422f 100644 --- a/include/llvm/CodeGen/CommandFlags.h +++ b/include/llvm/CodeGen/CommandFlags.h @@ -182,6 +182,11 @@ OverrideStackAlignment("stack-alignment", cl::desc("Override default stack alignment"), cl::init(0)); +cl::opt<bool> +StackRealign("stackrealign", + cl::desc("Force align the stack to the minimum alignment"), + cl::init(false)); + cl::opt<std::string> TrapFuncName("trap-func", cl::Hidden, cl::desc("Emit a call to trap function rather than a trap instruction"), @@ -219,6 +224,10 @@ FunctionSections("function-sections", cl::desc("Emit functions into separate sections"), cl::init(false)); +cl::opt<bool> EmulatedTLS("emulated-tls", + cl::desc("Use emulated TLS model"), + cl::init(false)); + cl::opt<bool> UniqueSectionNames("unique-section-names", cl::desc("Give unique names to every section"), cl::init(true)); @@ -238,6 +247,26 @@ JTableType("jump-table-type", "Create one table per unique function type."), clEnumValEnd)); +cl::opt<llvm::EABI> EABIVersion( + "meabi", cl::desc("Set EABI type (default depends on triple):"), + cl::init(EABI::Default), + cl::values(clEnumValN(EABI::Default, "default", + "Triple default EABI version"), + clEnumValN(EABI::EABI4, "4", "EABI version 4"), + clEnumValN(EABI::EABI5, "5", "EABI version 5"), + clEnumValN(EABI::GNU, "gnu", "EABI GNU"), clEnumValEnd)); + +cl::opt<DebuggerKind> +DebuggerTuningOpt("debugger-tune", + cl::desc("Tune debug info for a particular debugger"), + cl::init(DebuggerKind::Default), + cl::values( + clEnumValN(DebuggerKind::GDB, "gdb", "gdb"), + clEnumValN(DebuggerKind::LLDB, "lldb", "lldb"), + clEnumValN(DebuggerKind::SCE, "sce", + "SCE targets (e.g. PS4)"), + clEnumValEnd)); + // Common utility function tightly tied to the options listed here. Initializes // a TargetOptions object with CodeGen flags and returns it. static inline TargetOptions InitTargetOptionsFromCodeGenFlags() { @@ -260,11 +289,14 @@ static inline TargetOptions InitTargetOptionsFromCodeGenFlags() { Options.DataSections = DataSections; Options.FunctionSections = FunctionSections; Options.UniqueSectionNames = UniqueSectionNames; + Options.EmulatedTLS = EmulatedTLS; Options.MCOptions = InitMCTargetOptionsFromFlags(); Options.JTType = JTableType; Options.ThreadModel = TMModel; + Options.EABIVersion = EABIVersion; + Options.DebuggerTuning = DebuggerTuningOpt; return Options; } @@ -325,6 +357,10 @@ static inline void setFunctionAttributes(StringRef CPU, StringRef Features, "disable-tail-calls", toStringRef(DisableTailCalls)); + if (StackRealign) + NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex, + "stackrealign"); + if (TrapFuncName.getNumOccurrences() > 0) for (auto &B : F) for (auto &I : B) diff --git a/include/llvm/CodeGen/DFAPacketizer.h b/include/llvm/CodeGen/DFAPacketizer.h index c44a7e0b6736..40ec201107e8 100644 --- a/include/llvm/CodeGen/DFAPacketizer.h +++ b/include/llvm/CodeGen/DFAPacketizer.h @@ -40,22 +40,51 @@ class InstrItineraryData; class DefaultVLIWScheduler; class SUnit; +// -------------------------------------------------------------------- +// Definitions shared between DFAPacketizer.cpp and DFAPacketizerEmitter.cpp + +// DFA_MAX_RESTERMS * DFA_MAX_RESOURCES must fit within sizeof DFAInput. +// This is verified in DFAPacketizer.cpp:DFAPacketizer::DFAPacketizer. +// +// e.g. terms x resource bit combinations that fit in uint32_t: +// 4 terms x 8 bits = 32 bits +// 3 terms x 10 bits = 30 bits +// 2 terms x 16 bits = 32 bits +// +// e.g. terms x resource bit combinations that fit in uint64_t: +// 8 terms x 8 bits = 64 bits +// 7 terms x 9 bits = 63 bits +// 6 terms x 10 bits = 60 bits +// 5 terms x 12 bits = 60 bits +// 4 terms x 16 bits = 64 bits <--- current +// 3 terms x 21 bits = 63 bits +// 2 terms x 32 bits = 64 bits +// +#define DFA_MAX_RESTERMS 4 // The max # of AND'ed resource terms. +#define DFA_MAX_RESOURCES 16 // The max # of resource bits in one term. + +typedef uint64_t DFAInput; +typedef int64_t DFAStateInput; +#define DFA_TBLTYPE "int64_t" // For generating DFAStateInputTable. +// -------------------------------------------------------------------- + class DFAPacketizer { private: - typedef std::pair<unsigned, unsigned> UnsignPair; + typedef std::pair<unsigned, DFAInput> UnsignPair; + const InstrItineraryData *InstrItins; int CurrentState; - const int (*DFAStateInputTable)[2]; + const DFAStateInput (*DFAStateInputTable)[2]; const unsigned *DFAStateEntryTable; // CachedTable is a map from <FromState, Input> to ToState. DenseMap<UnsignPair, unsigned> CachedTable; // ReadTable - Read the DFA transition table and update CachedTable. - void ReadTable(unsigned int state); + void ReadTable(unsigned state); public: - DFAPacketizer(const InstrItineraryData *I, const int (*SIT)[2], + DFAPacketizer(const InstrItineraryData *I, const DFAStateInput (*SIT)[2], const unsigned *SET); // Reset the current state to make all resources available. @@ -63,6 +92,12 @@ public: CurrentState = 0; } + // getInsnInput - Return the DFAInput for an instruction class. + DFAInput getInsnInput(unsigned InsnClass); + + // getInsnInput - Return the DFAInput for an instruction class input vector. + static DFAInput getInsnInput(const std::vector<unsigned> &InsnClass); + // canReserveResources - Check if the resources occupied by a MCInstrDesc // are available in the current state. bool canReserveResources(const llvm::MCInstrDesc *MID); @@ -93,6 +128,7 @@ class VLIWPacketizerList { protected: MachineFunction &MF; const TargetInstrInfo *TII; + AliasAnalysis *AA; // The VLIW Scheduler. DefaultVLIWScheduler *VLIWScheduler; @@ -106,7 +142,9 @@ protected: std::map<MachineInstr*, SUnit*> MIToSUnit; public: - VLIWPacketizerList(MachineFunction &MF, MachineLoopInfo &MLI, bool IsPostRA); + // The AliasAnalysis parameter can be nullptr. + VLIWPacketizerList(MachineFunction &MF, MachineLoopInfo &MLI, + AliasAnalysis *AA); virtual ~VLIWPacketizerList(); @@ -126,8 +164,10 @@ public: return MII; } - // endPacket - End the current packet. - void endPacket(MachineBasicBlock *MBB, MachineInstr *MI); + // End the current packet and reset the state of the packetizer. + // Overriding this function allows the target-specific packetizer + // to perform custom finalization. + virtual void endPacket(MachineBasicBlock *MBB, MachineInstr *MI); // initPacketizerState - perform initialization before packetizing // an instruction. This function is supposed to be overrided by @@ -135,14 +175,24 @@ public: virtual void initPacketizerState() { return; } // ignorePseudoInstruction - Ignore bundling of pseudo instructions. - virtual bool ignorePseudoInstruction(MachineInstr *I, - MachineBasicBlock *MBB) { + virtual bool ignorePseudoInstruction(const MachineInstr *I, + const MachineBasicBlock *MBB) { return false; } // isSoloInstruction - return true if instruction MI can not be packetized // with any other instruction, which means that MI itself is a packet. - virtual bool isSoloInstruction(MachineInstr *MI) { + virtual bool isSoloInstruction(const MachineInstr *MI) { + return true; + } + + // Check if the packetizer should try to add the given instruction to + // the current packet. One reasons for which it may not be desirable + // to include an instruction in the current packet could be that it + // would cause a stall. + // If this function returns "false", the current packet will be ended, + // and the instruction will be added to the next packet. + virtual bool shouldAddToPacket(const MachineInstr *MI) { return true; } diff --git a/include/llvm/CodeGen/DIE.h b/include/llvm/CodeGen/DIE.h index f07712a676da..fa612d981dec 100644 --- a/include/llvm/CodeGen/DIE.h +++ b/include/llvm/CodeGen/DIE.h @@ -100,10 +100,8 @@ public: /// void Emit(const AsmPrinter *AP) const; -#ifndef NDEBUG void print(raw_ostream &O); void dump(); -#endif }; //===--------------------------------------------------------------------===// @@ -143,9 +141,7 @@ public: void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// @@ -164,9 +160,7 @@ public: void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// @@ -185,9 +179,7 @@ public: void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// @@ -203,9 +195,7 @@ public: void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// @@ -223,9 +213,7 @@ public: void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// @@ -252,9 +240,7 @@ public: : sizeof(int32_t); } -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// @@ -273,9 +259,7 @@ public: return 8; } -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// @@ -295,9 +279,7 @@ public: void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// @@ -444,10 +426,8 @@ public: /// unsigned SizeOf(const AsmPrinter *AP) const; -#ifndef NDEBUG void print(raw_ostream &O) const; void dump() const; -#endif }; struct IntrusiveBackListNode { @@ -566,64 +546,70 @@ class DIEValueList { ListTy List; public: - bool empty() const { return List.empty(); } - - class const_iterator; - class iterator - : public iterator_adaptor_base<iterator, ListTy::iterator, + class const_value_iterator; + class value_iterator + : public iterator_adaptor_base<value_iterator, ListTy::iterator, std::forward_iterator_tag, DIEValue> { - friend class const_iterator; - typedef iterator_adaptor_base<iterator, ListTy::iterator, + friend class const_value_iterator; + typedef iterator_adaptor_base<value_iterator, ListTy::iterator, std::forward_iterator_tag, DIEValue> iterator_adaptor; public: - iterator() = default; - explicit iterator(ListTy::iterator X) : iterator_adaptor(X) {} + value_iterator() = default; + explicit value_iterator(ListTy::iterator X) : iterator_adaptor(X) {} explicit operator bool() const { return bool(wrapped()); } DIEValue &operator*() const { return wrapped()->V; } }; - class const_iterator - : public iterator_adaptor_base<const_iterator, ListTy::const_iterator, - std::forward_iterator_tag, - const DIEValue> { - typedef iterator_adaptor_base<const_iterator, ListTy::const_iterator, + class const_value_iterator : public iterator_adaptor_base< + const_value_iterator, ListTy::const_iterator, + std::forward_iterator_tag, const DIEValue> { + typedef iterator_adaptor_base<const_value_iterator, ListTy::const_iterator, std::forward_iterator_tag, const DIEValue> iterator_adaptor; public: - const_iterator() = default; - const_iterator(DIEValueList::iterator X) : iterator_adaptor(X.wrapped()) {} - explicit const_iterator(ListTy::const_iterator X) : iterator_adaptor(X) {} + const_value_iterator() = default; + const_value_iterator(DIEValueList::value_iterator X) + : iterator_adaptor(X.wrapped()) {} + explicit const_value_iterator(ListTy::const_iterator X) + : iterator_adaptor(X) {} explicit operator bool() const { return bool(wrapped()); } const DIEValue &operator*() const { return wrapped()->V; } }; - iterator insert(BumpPtrAllocator &Alloc, DIEValue V) { + typedef iterator_range<value_iterator> value_range; + typedef iterator_range<const_value_iterator> const_value_range; + + value_iterator addValue(BumpPtrAllocator &Alloc, DIEValue V) { List.push_back(*new (Alloc) Node(V)); - return iterator(ListTy::toIterator(List.back())); + return value_iterator(ListTy::toIterator(List.back())); } - template <class... Ts> - iterator emplace(BumpPtrAllocator &Alloc, Ts &&... Args) { - return insert(Alloc, DIEValue(std::forward<Ts>(Args)...)); + template <class T> + value_iterator addValue(BumpPtrAllocator &Alloc, dwarf::Attribute Attribute, + dwarf::Form Form, T &&Value) { + return addValue(Alloc, DIEValue(Attribute, Form, std::forward<T>(Value))); } - iterator begin() { return iterator(List.begin()); } - iterator end() { return iterator(List.end()); } - const_iterator begin() const { return const_iterator(List.begin()); } - const_iterator end() const { return const_iterator(List.end()); } + value_range values() { + return llvm::make_range(value_iterator(List.begin()), + value_iterator(List.end())); + } + const_value_range values() const { + return llvm::make_range(const_value_iterator(List.begin()), + const_value_iterator(List.end())); + } }; //===--------------------------------------------------------------------===// /// DIE - A structured debug information entry. Has an abbreviation which /// describes its organization. -class DIE : IntrusiveBackListNode { +class DIE : IntrusiveBackListNode, public DIEValueList { friend class IntrusiveBackList<DIE>; -protected: /// Offset - Offset in debug info section. /// unsigned Offset; @@ -643,14 +629,7 @@ protected: DIE *Parent = nullptr; - /// Attribute values. - /// - DIEValueList Values; - -protected: - DIE() : Offset(0), Size(0) {} - -private: + DIE() = delete; explicit DIE(dwarf::Tag Tag) : Offset(0), Size(0), Tag(Tag) {} public: @@ -677,20 +656,6 @@ public: return llvm::make_range(Children.begin(), Children.end()); } - typedef DIEValueList::iterator value_iterator; - typedef iterator_range<value_iterator> value_range; - - value_range values() { - return llvm::make_range(Values.begin(), Values.end()); - } - - typedef DIEValueList::const_iterator const_value_iterator; - typedef iterator_range<const_value_iterator> const_value_range; - - const_value_range values() const { - return llvm::make_range(Values.begin(), Values.end()); - } - DIE *getParent() const { return Parent; } /// Generate the abbreviation for this DIE. @@ -711,17 +676,6 @@ public: void setOffset(unsigned O) { Offset = O; } void setSize(unsigned S) { Size = S; } - /// addValue - Add a value and attributes to a DIE. - /// - value_iterator addValue(BumpPtrAllocator &Alloc, DIEValue Value) { - return Values.insert(Alloc, Value); - } - template <class T> - value_iterator addValue(BumpPtrAllocator &Alloc, dwarf::Attribute Attribute, - dwarf::Form Form, T &&Value) { - return Values.emplace(Alloc, Attribute, Form, std::forward<T>(Value)); - } - /// Add a child to the DIE. DIE &addChild(DIE *Child) { assert(!Child->getParent() && "Child should be orphaned"); @@ -736,16 +690,14 @@ public: /// gives \a DIEValue::isNone) if no such attribute exists. DIEValue findAttribute(dwarf::Attribute Attribute) const; -#ifndef NDEBUG void print(raw_ostream &O, unsigned IndentCount = 0) const; void dump(); -#endif }; //===--------------------------------------------------------------------===// /// DIELoc - Represents an expression location. // -class DIELoc : public DIE { +class DIELoc : public DIEValueList { mutable unsigned Size; // Size in bytes excluding size header. public: @@ -773,15 +725,13 @@ public: void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; //===--------------------------------------------------------------------===// /// DIEBlock - Represents a block of values. // -class DIEBlock : public DIE { +class DIEBlock : public DIEValueList { mutable unsigned Size; // Size in bytes excluding size header. public: @@ -806,9 +756,7 @@ public: void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; -#ifndef NDEBUG void print(raw_ostream &O) const; -#endif }; } // end llvm namespace diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h index f04a7cd69664..cc4e37059bb8 100644 --- a/include/llvm/CodeGen/FastISel.h +++ b/include/llvm/CodeGen/FastISel.h @@ -419,11 +419,11 @@ protected: const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2); - /// \brief Emit a MachineInstr with two register operands and a result + /// \brief Emit a MachineInstr with a floating point immediate, and a result /// register in the given register class. - unsigned fastEmitInst_rf(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, unsigned Op0, - bool Op0IsKill, const ConstantFP *FPImm); + unsigned fastEmitInst_f(unsigned MachineInstOpcode, + const TargetRegisterClass *RC, + const ConstantFP *FPImm); /// \brief Emit a MachineInstr with two register operands, an immediate, and a /// result register in the given register class. @@ -432,23 +432,11 @@ protected: bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm); - /// \brief Emit a MachineInstr with two register operands, two immediates - /// operands, and a result register in the given register class. - unsigned fastEmitInst_rrii(unsigned MachineInstOpcode, - const TargetRegisterClass *RC, unsigned Op0, - bool Op0IsKill, unsigned Op1, bool Op1IsKill, - uint64_t Imm1, uint64_t Imm2); - /// \brief Emit a MachineInstr with a single immediate operand, and a result /// register in the given register class. unsigned fastEmitInst_i(unsigned MachineInstrOpcode, const TargetRegisterClass *RC, uint64_t Imm); - /// \brief Emit a MachineInstr with a two immediate operands. - unsigned fastEmitInst_ii(unsigned MachineInstrOpcode, - const TargetRegisterClass *RC, uint64_t Imm1, - uint64_t Imm2); - /// \brief Emit a MachineInstr for an extract_subreg from a specified index of /// a superregister to a specified type. unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, @@ -462,6 +450,11 @@ protected: /// immediate (fall-through) successor, and update the CFG. void fastEmitBranch(MachineBasicBlock *MBB, DebugLoc DL); + /// Emit an unconditional branch to \p FalseMBB, obtains the branch weight + /// and adds TrueMBB and FalseMBB to the successor list. + void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, + MachineBasicBlock *FalseMBB); + /// \brief Update the value map to include the new mapping for this /// instruction, or insert an extra copy to get the result in a previous /// determined register. @@ -566,6 +559,9 @@ private: /// across heavy instructions like calls. void flushLocalValueMap(); + /// \brief Removes dead local value instructions after SavedLastLocalvalue. + void removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue); + /// \brief Insertion point before trying to select the current instruction. MachineBasicBlock::iterator SavedInsertPt; diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h index 82c762ed850f..09a9991912da 100644 --- a/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -62,6 +62,9 @@ public: /// registers. bool CanLowerReturn; + /// True if part of the CSRs will be handled via explicit copies. + bool SplitCSR; + /// DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg /// allocated to hold a pointer to the hidden sret parameter. unsigned DemoteRegister; @@ -72,7 +75,10 @@ public: /// ValueMap - Since we emit code for the function a basic block at a time, /// we must remember which virtual registers hold the values for /// cross-basic-block values. - DenseMap<const Value*, unsigned> ValueMap; + DenseMap<const Value *, unsigned> ValueMap; + + /// Track virtual registers created for exception pointers. + DenseMap<const Value *, unsigned> CatchPadExceptionPointers; // Keep track of frame indices allocated for statepoints as they could be used // across basic block boundaries. @@ -99,7 +105,7 @@ public: /// RegFixups - Registers which need to be replaced after isel is done. DenseMap<unsigned, unsigned> RegFixups; - /// StatepointStackSlots - A list of temporary stack slots (frame indices) + /// StatepointStackSlots - A list of temporary stack slots (frame indices) /// used to spill values at a statepoint. We store them here to enable /// reuse of the same stack slots across different statepoints in different /// basic blocks. @@ -111,11 +117,6 @@ public: /// MBB - The current insert position inside the current block. MachineBasicBlock::iterator InsertPt; -#ifndef NDEBUG - SmallPtrSet<const Instruction *, 8> CatchInfoLost; - SmallPtrSet<const Instruction *, 8> CatchInfoFound; -#endif - struct LiveOutInfo { unsigned NumSignBits : 31; bool IsValid : 1; @@ -161,10 +162,13 @@ public: } unsigned CreateReg(MVT VT); - + unsigned CreateRegs(Type *Ty); - + unsigned InitializeRegForValue(const Value *V) { + // Tokens never live in vregs. + if (V->getType()->isTokenTy()) + return 0; unsigned &R = ValueMap[V]; assert(R == 0 && "Already initialized this value register!"); return R = CreateRegs(V->getType()); @@ -231,6 +235,9 @@ public: /// getArgumentFrameIndex - Get frame index for the byval argument. int getArgumentFrameIndex(const Argument *A); + unsigned getCatchPadExceptionPointerVReg(const Value *CPI, + const TargetRegisterClass *RC); + private: void addSEHHandlersForLPads(ArrayRef<const LandingPadInst *> LPads); diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h index e883bd196ea3..163117b0781c 100644 --- a/include/llvm/CodeGen/GCMetadata.h +++ b/include/llvm/CodeGen/GCMetadata.h @@ -160,9 +160,9 @@ class GCModuleInfo : public ImmutablePass { public: /// Lookup the GCStrategy object associated with the given gc name. /// Objects are owned internally; No caller should attempt to delete the - /// returned objects. + /// returned objects. GCStrategy *getGCStrategy(const StringRef Name); - + /// List of per function info objects. In theory, Each of these /// may be associated with a different GC. typedef std::vector<std::unique_ptr<GCFunctionInfo>> FuncInfoVec; diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h index a1b8e895898f..3088a86a3260 100644 --- a/include/llvm/CodeGen/GCStrategy.h +++ b/include/llvm/CodeGen/GCStrategy.h @@ -117,11 +117,11 @@ public: /** @name Statepoint Specific Properties */ ///@{ - /// If the value specified can be reliably distinguished, returns true for + /// If the type specified can be reliably distinguished, returns true for /// pointers to GC managed locations and false for pointers to non-GC /// managed locations. Note a GCStrategy can always return 'None' (i.e. an /// empty optional indicating it can't reliably distinguish. - virtual Optional<bool> isGCManagedPointer(const Value *V) const { + virtual Optional<bool> isGCManagedPointer(const Type *Ty) const { return None; } ///@} diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h index fa44301a2d4a..158ff3cd36a8 100644 --- a/include/llvm/CodeGen/ISDOpcodes.h +++ b/include/llvm/CodeGen/ISDOpcodes.h @@ -108,6 +108,10 @@ namespace ISD { /// and returns an outchain. EH_SJLJ_LONGJMP, + /// OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) + /// The target initializes the dispatch table here. + EH_SJLJ_SETUP_DISPATCH, + /// TargetConstant* - Like Constant*, but the DAG does not do any folding, /// simplification, or lowering of the constant. They are used for constants /// which are known to fit in the immediate fields of their users, or for @@ -332,7 +336,7 @@ namespace ISD { SHL, SRA, SRL, ROTL, ROTR, /// Byte Swap and Counting operators. - BSWAP, CTTZ, CTLZ, CTPOP, + BSWAP, CTTZ, CTLZ, CTPOP, BITREVERSE, /// Bit counting operators with an undefined result for zero inputs. CTTZ_ZERO_UNDEF, CTLZ_ZERO_UNDEF, @@ -364,9 +368,14 @@ namespace ISD { /// then the result type must also be a vector type. SETCC, + /// Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but + /// op #2 is a *carry value*. This operator checks the result of + /// "LHS - RHS - Carry", and can be used to compare two wide integers: + /// (setcce lhshi rhshi (subc lhslo rhslo) cc). Only valid for integers. + SETCCE, + /// SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded - /// integer shift operations, just like ADD/SUB_PARTS. The operation - /// ordering is: + /// integer shift operations. The operation ordering is: /// [Lo,Hi] = op [LoLHS,HiLHS], Amt SHL_PARTS, SRA_PARTS, SRL_PARTS, @@ -506,7 +515,15 @@ namespace ISD { FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR, + /// FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two + /// values. + /// In the case where a single input is NaN, the non-NaN input is returned. + /// + /// The return value of (FMINNUM 0.0, -0.0) could be either 0.0 or -0.0. FMINNUM, FMAXNUM, + /// FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that + /// when a single input is NaN, NaN is returned. + FMINNAN, FMAXNAN, /// FSINCOS - Compute both fsin and fcos as a single operation. FSINCOS, @@ -575,6 +592,18 @@ namespace ISD { /// take a chain as input and return a chain. EH_LABEL, + /// CATCHPAD - Represents a catchpad instruction. + CATCHPAD, + + /// CATCHRET - Represents a return from a catch block funclet. Used for + /// MSVC compatible exception handling. Takes a chain operand and a + /// destination basic block operand. + CATCHRET, + + /// CLEANUPRET - Represents a return from a cleanup block funclet. Used for + /// MSVC compatible exception handling. Takes only a chain operand. + CLEANUPRET, + /// STACKSAVE - STACKSAVE has one operand, an input chain. It produces a /// value, the same type as the pointer type for the system, and an output /// chain. @@ -618,9 +647,11 @@ namespace ISD { PCMARKER, /// READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic. - /// The only operand is a chain and a value and a chain are produced. The - /// value is the contents of the architecture specific cycle counter like - /// register (or other high accuracy low latency clock source) + /// It produces a chain and one i64 value. The only operand is a chain. + /// If i64 is not legal, the result will be expanded into smaller values. + /// Still, it returns an i64, so targets should set legality for i64. + /// The result is the content of the architecture-specific cycle + /// counter-like register (or other high accuracy low latency clock source). READCYCLECOUNTER, /// HANDLENODE node - Used as a handle for various purposes. @@ -719,6 +750,12 @@ namespace ISD { GC_TRANSITION_START, GC_TRANSITION_END, + /// GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of + /// the most recent dynamic alloca. For most targets that would be 0, but + /// for some others (e.g. PowerPC, PowerPC64) that would be compile-time + /// known nonzero constant. The only operand here is the chain. + GET_DYNAMIC_AREA_OFFSET, + /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END diff --git a/include/llvm/CodeGen/IntrinsicLowering.h b/include/llvm/CodeGen/IntrinsicLowering.h index 9e6ab7d45977..a404b9b70d3a 100644 --- a/include/llvm/CodeGen/IntrinsicLowering.h +++ b/include/llvm/CodeGen/IntrinsicLowering.h @@ -19,41 +19,40 @@ #include "llvm/IR/Intrinsics.h" namespace llvm { - class CallInst; - class Module; - class DataLayout; - - class IntrinsicLowering { - const DataLayout& DL; - - - bool Warned; - public: - explicit IntrinsicLowering(const DataLayout &DL) : - DL(DL), Warned(false) {} - - /// AddPrototypes - This method, if called, causes all of the prototypes - /// that might be needed by an intrinsic lowering implementation to be - /// inserted into the module specified. - void AddPrototypes(Module &M); - - /// LowerIntrinsicCall - This method replaces a call with the LLVM function - /// which should be used to implement the specified intrinsic function call. - /// If an intrinsic function must be implemented by the code generator - /// (such as va_start), this function should print a message and abort. - /// - /// Otherwise, if an intrinsic function call can be lowered, the code to - /// implement it (often a call to a non-intrinsic function) is inserted - /// _after_ the call instruction and the call is deleted. The caller must - /// be capable of handling this kind of change. - /// - void LowerIntrinsicCall(CallInst *CI); - - /// LowerToByteSwap - Replace a call instruction into a call to bswap - /// intrinsic. Return false if it has determined the call is not a - /// simple integer bswap. - static bool LowerToByteSwap(CallInst *CI); - }; +class CallInst; +class Module; +class DataLayout; + +class IntrinsicLowering { + const DataLayout &DL; + + bool Warned; + +public: + explicit IntrinsicLowering(const DataLayout &DL) : DL(DL), Warned(false) {} + + /// AddPrototypes - This method, if called, causes all of the prototypes + /// that might be needed by an intrinsic lowering implementation to be + /// inserted into the module specified. + void AddPrototypes(Module &M); + + /// LowerIntrinsicCall - This method replaces a call with the LLVM function + /// which should be used to implement the specified intrinsic function call. + /// If an intrinsic function must be implemented by the code generator + /// (such as va_start), this function should print a message and abort. + /// + /// Otherwise, if an intrinsic function call can be lowered, the code to + /// implement it (often a call to a non-intrinsic function) is inserted + /// _after_ the call instruction and the call is deleted. The caller must + /// be capable of handling this kind of change. + /// + void LowerIntrinsicCall(CallInst *CI); + + /// LowerToByteSwap - Replace a call instruction into a call to bswap + /// intrinsic. Return false if it has determined the call is not a + /// simple integer bswap. + static bool LowerToByteSwap(CallInst *CI); +}; } #endif diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h index 9b8b91c9b80e..0157bf9117e5 100644 --- a/include/llvm/CodeGen/LiveInterval.h +++ b/include/llvm/CodeGen/LiveInterval.h @@ -25,6 +25,7 @@ #include "llvm/CodeGen/SlotIndexes.h" #include "llvm/Support/AlignOf.h" #include "llvm/Support/Allocator.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <cassert> #include <climits> #include <set> @@ -595,15 +596,15 @@ namespace llvm { class SubRange : public LiveRange { public: SubRange *Next; - unsigned LaneMask; + LaneBitmask LaneMask; /// Constructs a new SubRange object. - SubRange(unsigned LaneMask) + SubRange(LaneBitmask LaneMask) : Next(nullptr), LaneMask(LaneMask) { } /// Constructs a new SubRange object by copying liveness from @p Other. - SubRange(unsigned LaneMask, const LiveRange &Other, + SubRange(LaneBitmask LaneMask, const LiveRange &Other, BumpPtrAllocator &Allocator) : LiveRange(Other, Allocator), Next(nullptr), LaneMask(LaneMask) { } @@ -677,7 +678,8 @@ namespace llvm { /// Creates a new empty subregister live range. The range is added at the /// beginning of the subrange list; subrange iterators stay valid. - SubRange *createSubRange(BumpPtrAllocator &Allocator, unsigned LaneMask) { + SubRange *createSubRange(BumpPtrAllocator &Allocator, + LaneBitmask LaneMask) { SubRange *Range = new (Allocator) SubRange(LaneMask); appendSubRange(Range); return Range; @@ -685,7 +687,8 @@ namespace llvm { /// Like createSubRange() but the new range is filled with a copy of the /// liveness information in @p CopyFrom. - SubRange *createSubRangeFrom(BumpPtrAllocator &Allocator, unsigned LaneMask, + SubRange *createSubRangeFrom(BumpPtrAllocator &Allocator, + LaneBitmask LaneMask, const LiveRange &CopyFrom) { SubRange *Range = new (Allocator) SubRange(LaneMask, CopyFrom, Allocator); appendSubRange(Range); @@ -842,11 +845,6 @@ namespace llvm { LiveIntervals &LIS; IntEqClasses EqClass; - // Note that values a and b are connected. - void Connect(unsigned a, unsigned b); - - unsigned Renumber(); - public: explicit ConnectedVNInfoEqClasses(LiveIntervals &lis) : LIS(lis) {} @@ -858,12 +856,12 @@ namespace llvm { /// the equivalence class assigned the VNI. unsigned getEqClass(const VNInfo *VNI) const { return EqClass[VNI->id]; } - /// Distribute - Distribute values in LIV[0] into a separate LiveInterval - /// for each connected component. LIV must have a LiveInterval for each - /// connected component. The LiveIntervals in Liv[1..] must be empty. - /// Instructions using LIV[0] are rewritten. - void Distribute(LiveInterval *LIV[], MachineRegisterInfo &MRI); - + /// Distribute values in \p LI into a separate LiveIntervals + /// for each connected component. LIV must have an empty LiveInterval for + /// each additional connected component. The first connected component is + /// left in \p LI. + void Distribute(LiveInterval &LI, LiveInterval *LIV[], + MachineRegisterInfo &MRI); }; } diff --git a/include/llvm/CodeGen/LiveIntervalAnalysis.h b/include/llvm/CodeGen/LiveIntervalAnalysis.h index 9673f80e0856..87421e2f83b4 100644 --- a/include/llvm/CodeGen/LiveIntervalAnalysis.h +++ b/include/llvm/CodeGen/LiveIntervalAnalysis.h @@ -22,6 +22,7 @@ #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -36,7 +37,6 @@ namespace llvm { extern cl::opt<bool> UseSegmentSetForPhysRegs; - class AliasAnalysis; class BitVector; class BlockFrequency; class LiveRangeCalc; @@ -147,13 +147,12 @@ extern cl::opt<bool> UseSegmentSetForPhysRegs; LiveInterval::Segment addSegmentToEndOfBlock(unsigned reg, MachineInstr* startInst); - /// shrinkToUses - After removing some uses of a register, shrink its live - /// range to just the remaining uses. This method does not compute reaching - /// defs for new uses, and it doesn't remove dead defs. - /// Dead PHIDef values are marked as unused. - /// New dead machine instructions are added to the dead vector. - /// Return true if the interval may have been separated into multiple - /// connected components. + /// After removing some uses of a register, shrink its live range to just + /// the remaining uses. This method does not compute reaching defs for new + /// uses, and it doesn't remove dead defs. + /// Dead PHIDef values are marked as unused. New dead machine instructions + /// are added to the dead vector. Returns true if the interval may have been + /// separated into multiple connected components. bool shrinkToUses(LiveInterval *li, SmallVectorImpl<MachineInstr*> *dead = nullptr); @@ -161,6 +160,8 @@ extern cl::opt<bool> UseSegmentSetForPhysRegs; /// shrinkToUses(LiveInterval *li, SmallVectorImpl<MachineInstr*> *dead) /// that works on a subregister live range and only looks at uses matching /// the lane mask of the subregister range. + /// This may leave the subrange empty which needs to be cleaned up with + /// LiveInterval::removeEmptySubranges() afterwards. void shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg); /// extendToIndices - Extend the live range of LI to reach all points in @@ -257,11 +258,6 @@ extern cl::opt<bool> UseSegmentSetForPhysRegs; Indexes->replaceMachineInstrInMaps(MI, NewMI); } - bool findLiveInMBBs(SlotIndex Start, SlotIndex End, - SmallVectorImpl<MachineBasicBlock*> &MBBs) const { - return Indexes->findLiveInMBBs(Start, End, MBBs); - } - VNInfo::Allocator& getVNInfoAllocator() { return VNInfoAllocator; } void getAnalysisUsage(AnalysisUsage &AU) const override; @@ -406,6 +402,10 @@ extern cl::opt<bool> UseSegmentSetForPhysRegs; /// that start at position @p Pos. void removeVRegDefAt(LiveInterval &LI, SlotIndex Pos); + /// Split separate components in LiveInterval \p LI into separate intervals. + void splitSeparateComponents(LiveInterval &LI, + SmallVectorImpl<LiveInterval*> &SplitLIs); + private: /// Compute live intervals for all virtual registers. void computeVirtRegs(); @@ -440,7 +440,7 @@ extern cl::opt<bool> UseSegmentSetForPhysRegs; void repairOldRegInRange(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, const SlotIndex endIdx, LiveRange &LR, - unsigned Reg, unsigned LaneMask = ~0u); + unsigned Reg, LaneBitmask LaneMask = ~0u); class HMEditor; }; diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h index 6475e7b4af37..3bdf5ae8d013 100644 --- a/include/llvm/CodeGen/LivePhysRegs.h +++ b/include/llvm/CodeGen/LivePhysRegs.h @@ -109,7 +109,7 @@ public: /// \brief Simulates liveness when stepping forward over an /// instruction(bundle): Remove killed-uses, add defs. This is the not /// recommended way, because it depends on accurate kill flags. If possible - /// use stepBackwards() instead of this function. + /// use stepBackward() instead of this function. /// The clobbers set will be the list of registers either defined or clobbered /// by a regmask. The operand will identify whether this is a regmask or /// register operand. @@ -122,9 +122,9 @@ public: void addLiveIns(const MachineBasicBlock *MBB, bool AddPristines = false); /// \brief Adds all live-out registers of basic block @p MBB; After prologue/ - /// epilogue insertion \p AddPristines should be set to true to insert the - /// pristine registers. - void addLiveOuts(const MachineBasicBlock *MBB, bool AddPristines = false); + /// epilogue insertion \p AddPristinesAndCSRs should be set to true. + void addLiveOuts(const MachineBasicBlock *MBB, + bool AddPristinesAndCSRs = false); typedef SparseSet<unsigned>::const_iterator const_iterator; const_iterator begin() const { return LiveRegs.begin(); } diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h index c97c636abbb4..2271e3352aa2 100644 --- a/include/llvm/CodeGen/LiveRangeEdit.h +++ b/include/llvm/CodeGen/LiveRangeEdit.h @@ -21,6 +21,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Target/TargetMachine.h" @@ -28,7 +29,6 @@ namespace llvm { -class AliasAnalysis; class LiveIntervals; class MachineBlockFrequencyInfo; class MachineLoopInfo; diff --git a/include/llvm/CodeGen/LiveRegMatrix.h b/include/llvm/CodeGen/LiveRegMatrix.h index 86a0c7bd626f..e169058ca563 100644 --- a/include/llvm/CodeGen/LiveRegMatrix.h +++ b/include/llvm/CodeGen/LiveRegMatrix.h @@ -32,13 +32,11 @@ namespace llvm { class LiveInterval; class LiveIntervalAnalysis; -class MachineRegisterInfo; class TargetRegisterInfo; class VirtRegMap; class LiveRegMatrix : public MachineFunctionPass { const TargetRegisterInfo *TRI; - MachineRegisterInfo *MRI; LiveIntervals *LIS; VirtRegMap *VRM; diff --git a/include/llvm/CodeGen/LiveStackAnalysis.h b/include/llvm/CodeGen/LiveStackAnalysis.h index f495507c66ec..3ffbe3d775b4 100644 --- a/include/llvm/CodeGen/LiveStackAnalysis.h +++ b/include/llvm/CodeGen/LiveStackAnalysis.h @@ -25,76 +25,74 @@ namespace llvm { - class LiveStacks : public MachineFunctionPass { - const TargetRegisterInfo *TRI; - - /// Special pool allocator for VNInfo's (LiveInterval val#). - /// - VNInfo::Allocator VNInfoAllocator; - - /// S2IMap - Stack slot indices to live interval mapping. - /// - typedef std::unordered_map<int, LiveInterval> SS2IntervalMap; - SS2IntervalMap S2IMap; - - /// S2RCMap - Stack slot indices to register class mapping. - std::map<int, const TargetRegisterClass*> S2RCMap; - - public: - static char ID; // Pass identification, replacement for typeid - LiveStacks() : MachineFunctionPass(ID) { - initializeLiveStacksPass(*PassRegistry::getPassRegistry()); - } - - typedef SS2IntervalMap::iterator iterator; - typedef SS2IntervalMap::const_iterator const_iterator; - const_iterator begin() const { return S2IMap.begin(); } - const_iterator end() const { return S2IMap.end(); } - iterator begin() { return S2IMap.begin(); } - iterator end() { return S2IMap.end(); } - - unsigned getNumIntervals() const { return (unsigned)S2IMap.size(); } - - LiveInterval &getOrCreateInterval(int Slot, const TargetRegisterClass *RC); - - LiveInterval &getInterval(int Slot) { - assert(Slot >= 0 && "Spill slot indice must be >= 0"); - SS2IntervalMap::iterator I = S2IMap.find(Slot); - assert(I != S2IMap.end() && "Interval does not exist for stack slot"); - return I->second; - } - - const LiveInterval &getInterval(int Slot) const { - assert(Slot >= 0 && "Spill slot indice must be >= 0"); - SS2IntervalMap::const_iterator I = S2IMap.find(Slot); - assert(I != S2IMap.end() && "Interval does not exist for stack slot"); - return I->second; - } - - bool hasInterval(int Slot) const { - return S2IMap.count(Slot); - } - - const TargetRegisterClass *getIntervalRegClass(int Slot) const { - assert(Slot >= 0 && "Spill slot indice must be >= 0"); - std::map<int, const TargetRegisterClass*>::const_iterator - I = S2RCMap.find(Slot); - assert(I != S2RCMap.end() && - "Register class info does not exist for stack slot"); - return I->second; - } - - VNInfo::Allocator& getVNInfoAllocator() { return VNInfoAllocator; } - - void getAnalysisUsage(AnalysisUsage &AU) const override; - void releaseMemory() override; - - /// runOnMachineFunction - pass entry point - bool runOnMachineFunction(MachineFunction&) override; - - /// print - Implement the dump method. - void print(raw_ostream &O, const Module* = nullptr) const override; - }; +class LiveStacks : public MachineFunctionPass { + const TargetRegisterInfo *TRI; + + /// Special pool allocator for VNInfo's (LiveInterval val#). + /// + VNInfo::Allocator VNInfoAllocator; + + /// S2IMap - Stack slot indices to live interval mapping. + /// + typedef std::unordered_map<int, LiveInterval> SS2IntervalMap; + SS2IntervalMap S2IMap; + + /// S2RCMap - Stack slot indices to register class mapping. + std::map<int, const TargetRegisterClass *> S2RCMap; + +public: + static char ID; // Pass identification, replacement for typeid + LiveStacks() : MachineFunctionPass(ID) { + initializeLiveStacksPass(*PassRegistry::getPassRegistry()); + } + + typedef SS2IntervalMap::iterator iterator; + typedef SS2IntervalMap::const_iterator const_iterator; + const_iterator begin() const { return S2IMap.begin(); } + const_iterator end() const { return S2IMap.end(); } + iterator begin() { return S2IMap.begin(); } + iterator end() { return S2IMap.end(); } + + unsigned getNumIntervals() const { return (unsigned)S2IMap.size(); } + + LiveInterval &getOrCreateInterval(int Slot, const TargetRegisterClass *RC); + + LiveInterval &getInterval(int Slot) { + assert(Slot >= 0 && "Spill slot indice must be >= 0"); + SS2IntervalMap::iterator I = S2IMap.find(Slot); + assert(I != S2IMap.end() && "Interval does not exist for stack slot"); + return I->second; + } + + const LiveInterval &getInterval(int Slot) const { + assert(Slot >= 0 && "Spill slot indice must be >= 0"); + SS2IntervalMap::const_iterator I = S2IMap.find(Slot); + assert(I != S2IMap.end() && "Interval does not exist for stack slot"); + return I->second; + } + + bool hasInterval(int Slot) const { return S2IMap.count(Slot); } + + const TargetRegisterClass *getIntervalRegClass(int Slot) const { + assert(Slot >= 0 && "Spill slot indice must be >= 0"); + std::map<int, const TargetRegisterClass *>::const_iterator I = + S2RCMap.find(Slot); + assert(I != S2RCMap.end() && + "Register class info does not exist for stack slot"); + return I->second; + } + + VNInfo::Allocator &getVNInfoAllocator() { return VNInfoAllocator; } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + void releaseMemory() override; + + /// runOnMachineFunction - pass entry point + bool runOnMachineFunction(MachineFunction &) override; + + /// print - Implement the dump method. + void print(raw_ostream &O, const Module * = nullptr) const override; +}; } #endif /* LLVM_CODEGEN_LIVESTACK_ANALYSIS_H */ diff --git a/include/llvm/CodeGen/MIRParser/MIRParser.h b/include/llvm/CodeGen/MIRParser/MIRParser.h index 67b756d5e886..a569d5ec1f5e 100644 --- a/include/llvm/CodeGen/MIRParser/MIRParser.h +++ b/include/llvm/CodeGen/MIRParser/MIRParser.h @@ -1,4 +1,4 @@ -//===- MIRParser.h - MIR serialization format parser ----------------------===// +//===- MIRParser.h - MIR serialization format parser ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -37,7 +37,7 @@ class MIRParser : public MachineFunctionInitializer { public: MIRParser(std::unique_ptr<MIRParserImpl> Impl); MIRParser(const MIRParser &) = delete; - ~MIRParser(); + ~MIRParser() override; /// Parse the optional LLVM IR module that's embedded in the MIR file. /// @@ -78,4 +78,4 @@ createMIRParser(std::unique_ptr<MemoryBuffer> Contents, LLVMContext &Context); } // end namespace llvm -#endif +#endif // LLVM_CODEGEN_MIRPARSER_MIRPARSER_H diff --git a/include/llvm/CodeGen/MIRYamlMapping.h b/include/llvm/CodeGen/MIRYamlMapping.h index 9798e5cef645..14d3744741c5 100644 --- a/include/llvm/CodeGen/MIRYamlMapping.h +++ b/include/llvm/CodeGen/MIRYamlMapping.h @@ -19,6 +19,7 @@ #define LLVM_LIB_CODEGEN_MIRYAMLMAPPING_H #include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/Support/YAMLTraits.h" #include <vector> @@ -72,54 +73,109 @@ template <> struct ScalarTraits<FlowStringValue> { static bool mustQuote(StringRef Scalar) { return needsQuotes(Scalar); } }; +struct BlockStringValue { + StringValue Value; +}; + +template <> struct BlockScalarTraits<BlockStringValue> { + static void output(const BlockStringValue &S, void *Ctx, raw_ostream &OS) { + return ScalarTraits<StringValue>::output(S.Value, Ctx, OS); + } + + static StringRef input(StringRef Scalar, void *Ctx, BlockStringValue &S) { + return ScalarTraits<StringValue>::input(Scalar, Ctx, S.Value); + } +}; + +/// A wrapper around unsigned which contains a source range that's being set +/// during parsing. +struct UnsignedValue { + unsigned Value; + SMRange SourceRange; + + UnsignedValue() : Value(0) {} + UnsignedValue(unsigned Value) : Value(Value) {} + + bool operator==(const UnsignedValue &Other) const { + return Value == Other.Value; + } +}; + +template <> struct ScalarTraits<UnsignedValue> { + static void output(const UnsignedValue &Value, void *Ctx, raw_ostream &OS) { + return ScalarTraits<unsigned>::output(Value.Value, Ctx, OS); + } + + static StringRef input(StringRef Scalar, void *Ctx, UnsignedValue &Value) { + if (const auto *Node = + reinterpret_cast<yaml::Input *>(Ctx)->getCurrentNode()) + Value.SourceRange = Node->getSourceRange(); + return ScalarTraits<unsigned>::input(Scalar, Ctx, Value.Value); + } + + static bool mustQuote(StringRef Scalar) { + return ScalarTraits<unsigned>::mustQuote(Scalar); + } +}; + +template <> struct ScalarEnumerationTraits<MachineJumpTableInfo::JTEntryKind> { + static void enumeration(yaml::IO &IO, + MachineJumpTableInfo::JTEntryKind &EntryKind) { + IO.enumCase(EntryKind, "block-address", + MachineJumpTableInfo::EK_BlockAddress); + IO.enumCase(EntryKind, "gp-rel64-block-address", + MachineJumpTableInfo::EK_GPRel64BlockAddress); + IO.enumCase(EntryKind, "gp-rel32-block-address", + MachineJumpTableInfo::EK_GPRel32BlockAddress); + IO.enumCase(EntryKind, "label-difference32", + MachineJumpTableInfo::EK_LabelDifference32); + IO.enumCase(EntryKind, "inline", MachineJumpTableInfo::EK_Inline); + IO.enumCase(EntryKind, "custom32", MachineJumpTableInfo::EK_Custom32); + } +}; + } // end namespace yaml } // end namespace llvm LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::StringValue) LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::FlowStringValue) +LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::UnsignedValue) namespace llvm { namespace yaml { struct VirtualRegisterDefinition { - unsigned ID; + UnsignedValue ID; StringValue Class; - // TODO: Serialize the virtual register hints. + StringValue PreferredRegister; + // TODO: Serialize the target specific register hints. }; template <> struct MappingTraits<VirtualRegisterDefinition> { static void mapping(IO &YamlIO, VirtualRegisterDefinition &Reg) { YamlIO.mapRequired("id", Reg.ID); YamlIO.mapRequired("class", Reg.Class); + YamlIO.mapOptional("preferred-register", Reg.PreferredRegister, + StringValue()); // Don't print out when it's empty. } static const bool flow = true; }; -struct MachineBasicBlock { - unsigned ID; - StringValue Name; - unsigned Alignment = 0; - bool IsLandingPad = false; - bool AddressTaken = false; - // TODO: Serialize the successor weights. - std::vector<FlowStringValue> Successors; - std::vector<FlowStringValue> LiveIns; - std::vector<StringValue> Instructions; +struct MachineFunctionLiveIn { + StringValue Register; + StringValue VirtualRegister; }; -template <> struct MappingTraits<MachineBasicBlock> { - static void mapping(IO &YamlIO, MachineBasicBlock &MBB) { - YamlIO.mapRequired("id", MBB.ID); - YamlIO.mapOptional("name", MBB.Name, - StringValue()); // Don't print out an empty name. - YamlIO.mapOptional("alignment", MBB.Alignment); - YamlIO.mapOptional("isLandingPad", MBB.IsLandingPad); - YamlIO.mapOptional("addressTaken", MBB.AddressTaken); - YamlIO.mapOptional("successors", MBB.Successors); - YamlIO.mapOptional("liveins", MBB.LiveIns); - YamlIO.mapOptional("instructions", MBB.Instructions); +template <> struct MappingTraits<MachineFunctionLiveIn> { + static void mapping(IO &YamlIO, MachineFunctionLiveIn &LiveIn) { + YamlIO.mapRequired("reg", LiveIn.Register); + YamlIO.mapOptional( + "virtual-reg", LiveIn.VirtualRegister, + StringValue()); // Don't print the virtual register when it's empty. } + + static const bool flow = true; }; /// Serializable representation of stack object from the MachineFrameInfo class. @@ -128,16 +184,21 @@ template <> struct MappingTraits<MachineBasicBlock> { /// determined by the object's type and frame information flags. /// Dead stack objects aren't serialized. /// -/// TODO: Determine isPreallocated flag by mapping between objects and local -/// objects (Serialize local objects). +/// The 'isPreallocated' flag is determined by the local offset. struct MachineStackObject { enum ObjectType { DefaultType, SpillSlot, VariableSized }; - // TODO: Serialize LLVM alloca reference. - unsigned ID; + UnsignedValue ID; + StringValue Name; + // TODO: Serialize unnamed LLVM alloca reference. ObjectType Type = DefaultType; int64_t Offset = 0; uint64_t Size = 0; unsigned Alignment = 0; + StringValue CalleeSavedRegister; + Optional<int64_t> LocalOffset; + StringValue DebugVar; + StringValue DebugExpr; + StringValue DebugLoc; }; template <> struct ScalarEnumerationTraits<MachineStackObject::ObjectType> { @@ -151,6 +212,8 @@ template <> struct ScalarEnumerationTraits<MachineStackObject::ObjectType> { template <> struct MappingTraits<MachineStackObject> { static void mapping(yaml::IO &YamlIO, MachineStackObject &Object) { YamlIO.mapRequired("id", Object.ID); + YamlIO.mapOptional("name", Object.Name, + StringValue()); // Don't print out an empty name. YamlIO.mapOptional( "type", Object.Type, MachineStackObject::DefaultType); // Don't print the default type. @@ -158,6 +221,15 @@ template <> struct MappingTraits<MachineStackObject> { if (Object.Type != MachineStackObject::VariableSized) YamlIO.mapRequired("size", Object.Size); YamlIO.mapOptional("alignment", Object.Alignment); + YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister, + StringValue()); // Don't print it out when it's empty. + YamlIO.mapOptional("local-offset", Object.LocalOffset); + YamlIO.mapOptional("di-variable", Object.DebugVar, + StringValue()); // Don't print it out when it's empty. + YamlIO.mapOptional("di-expression", Object.DebugExpr, + StringValue()); // Don't print it out when it's empty. + YamlIO.mapOptional("di-location", Object.DebugLoc, + StringValue()); // Don't print it out when it's empty. } static const bool flow = true; @@ -167,13 +239,14 @@ template <> struct MappingTraits<MachineStackObject> { /// MachineFrameInfo class. struct FixedMachineStackObject { enum ObjectType { DefaultType, SpillSlot }; - unsigned ID; + UnsignedValue ID; ObjectType Type = DefaultType; int64_t Offset = 0; uint64_t Size = 0; unsigned Alignment = 0; bool IsImmutable = false; bool IsAliased = false; + StringValue CalleeSavedRegister; }; template <> @@ -198,22 +271,64 @@ template <> struct MappingTraits<FixedMachineStackObject> { YamlIO.mapOptional("isImmutable", Object.IsImmutable); YamlIO.mapOptional("isAliased", Object.IsAliased); } + YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister, + StringValue()); // Don't print it out when it's empty. } static const bool flow = true; }; +struct MachineConstantPoolValue { + UnsignedValue ID; + StringValue Value; + unsigned Alignment = 0; +}; + +template <> struct MappingTraits<MachineConstantPoolValue> { + static void mapping(IO &YamlIO, MachineConstantPoolValue &Constant) { + YamlIO.mapRequired("id", Constant.ID); + YamlIO.mapOptional("value", Constant.Value); + YamlIO.mapOptional("alignment", Constant.Alignment); + } +}; + +struct MachineJumpTable { + struct Entry { + UnsignedValue ID; + std::vector<FlowStringValue> Blocks; + }; + + MachineJumpTableInfo::JTEntryKind Kind = MachineJumpTableInfo::EK_Custom32; + std::vector<Entry> Entries; +}; + +template <> struct MappingTraits<MachineJumpTable::Entry> { + static void mapping(IO &YamlIO, MachineJumpTable::Entry &Entry) { + YamlIO.mapRequired("id", Entry.ID); + YamlIO.mapOptional("blocks", Entry.Blocks); + } +}; + } // end namespace yaml } // end namespace llvm +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineFunctionLiveIn) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::VirtualRegisterDefinition) -LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineBasicBlock) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineStackObject) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::FixedMachineStackObject) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineConstantPoolValue) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineJumpTable::Entry) namespace llvm { namespace yaml { +template <> struct MappingTraits<MachineJumpTable> { + static void mapping(IO &YamlIO, MachineJumpTable &JT) { + YamlIO.mapRequired("kind", JT.Kind); + YamlIO.mapOptional("entries", JT.Entries); + } +}; + /// Serializable representation of MachineFrameInfo. /// /// Doesn't serialize attributes like 'StackAlignment', 'IsStackRealignable' and @@ -231,14 +346,14 @@ struct MachineFrameInfo { unsigned MaxAlignment = 0; bool AdjustsStack = false; bool HasCalls = false; - // TODO: Serialize StackProtectorIdx and FunctionContextIdx + StringValue StackProtector; + // TODO: Serialize FunctionContextIdx unsigned MaxCallFrameSize = 0; - // TODO: Serialize callee saved info. - // TODO: Serialize local frame objects. bool HasOpaqueSPAdjustment = false; bool HasVAStart = false; bool HasMustTailInVarArgFunc = false; - // TODO: Serialize save and restore MBB references. + StringValue SavePoint; + StringValue RestorePoint; }; template <> struct MappingTraits<MachineFrameInfo> { @@ -252,10 +367,16 @@ template <> struct MappingTraits<MachineFrameInfo> { YamlIO.mapOptional("maxAlignment", MFI.MaxAlignment); YamlIO.mapOptional("adjustsStack", MFI.AdjustsStack); YamlIO.mapOptional("hasCalls", MFI.HasCalls); + YamlIO.mapOptional("stackProtector", MFI.StackProtector, + StringValue()); // Don't print it out when it's empty. YamlIO.mapOptional("maxCallFrameSize", MFI.MaxCallFrameSize); YamlIO.mapOptional("hasOpaqueSPAdjustment", MFI.HasOpaqueSPAdjustment); YamlIO.mapOptional("hasVAStart", MFI.HasVAStart); YamlIO.mapOptional("hasMustTailInVarArgFunc", MFI.HasMustTailInVarArgFunc); + YamlIO.mapOptional("savePoint", MFI.SavePoint, + StringValue()); // Don't print it out when it's empty. + YamlIO.mapOptional("restorePoint", MFI.RestorePoint, + StringValue()); // Don't print it out when it's empty. } }; @@ -269,14 +390,16 @@ struct MachineFunction { bool TracksRegLiveness = false; bool TracksSubRegLiveness = false; std::vector<VirtualRegisterDefinition> VirtualRegisters; + std::vector<MachineFunctionLiveIn> LiveIns; + Optional<std::vector<FlowStringValue>> CalleeSavedRegisters; // TODO: Serialize the various register masks. - // TODO: Serialize live in registers. // Frame information MachineFrameInfo FrameInfo; std::vector<FixedMachineStackObject> FixedStackObjects; std::vector<MachineStackObject> StackObjects; - - std::vector<MachineBasicBlock> BasicBlocks; + std::vector<MachineConstantPoolValue> Constants; /// Constant pool. + MachineJumpTable JumpTableInfo; + BlockStringValue Body; }; template <> struct MappingTraits<MachineFunction> { @@ -289,10 +412,15 @@ template <> struct MappingTraits<MachineFunction> { YamlIO.mapOptional("tracksRegLiveness", MF.TracksRegLiveness); YamlIO.mapOptional("tracksSubRegLiveness", MF.TracksSubRegLiveness); YamlIO.mapOptional("registers", MF.VirtualRegisters); + YamlIO.mapOptional("liveins", MF.LiveIns); + YamlIO.mapOptional("calleeSavedRegisters", MF.CalleeSavedRegisters); YamlIO.mapOptional("frameInfo", MF.FrameInfo); YamlIO.mapOptional("fixedStack", MF.FixedStackObjects); YamlIO.mapOptional("stack", MF.StackObjects); - YamlIO.mapOptional("body", MF.BasicBlocks); + YamlIO.mapOptional("constants", MF.Constants); + if (!YamlIO.outputting() || !MF.JumpTableInfo.Entries.empty()) + YamlIO.mapOptional("jumpTable", MF.JumpTableInfo); + YamlIO.mapOptional("body", MF.Body); } }; diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h index 5e5f45cae8fb..3d58c499823e 100644 --- a/include/llvm/CodeGen/MachineBasicBlock.h +++ b/include/llvm/CodeGen/MachineBasicBlock.h @@ -16,6 +16,8 @@ #include "llvm/ADT/GraphTraits.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/Support/BranchProbability.h" +#include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/DataTypes.h" #include <functional> @@ -25,11 +27,15 @@ class Pass; class BasicBlock; class MachineFunction; class MCSymbol; +class MIPrinter; class SlotIndexes; class StringRef; class raw_ostream; class MachineBranchProbabilityInfo; +// Forward declaration to avoid circular include problem with TargetRegisterInfo +typedef unsigned LaneBitmask; + template <> struct ilist_traits<MachineInstr> : public ilist_default_traits<MachineInstr> { private: @@ -52,57 +58,76 @@ public: void addNodeToList(MachineInstr* N); void removeNodeFromList(MachineInstr* N); void transferNodesFromList(ilist_traits &SrcTraits, - ilist_iterator<MachineInstr> first, - ilist_iterator<MachineInstr> last); + ilist_iterator<MachineInstr> First, + ilist_iterator<MachineInstr> Last); void deleteNode(MachineInstr *N); private: void createNode(const MachineInstr &); }; -class MachineBasicBlock : public ilist_node<MachineBasicBlock> { +class MachineBasicBlock + : public ilist_node_with_parent<MachineBasicBlock, MachineFunction> { +public: + /// Pair of physical register and lane mask. + /// This is not simply a std::pair typedef because the members should be named + /// clearly as they both have an integer type. + struct RegisterMaskPair { + public: + MCPhysReg PhysReg; + LaneBitmask LaneMask; + + RegisterMaskPair(MCPhysReg PhysReg, LaneBitmask LaneMask) + : PhysReg(PhysReg), LaneMask(LaneMask) {} + }; + +private: typedef ilist<MachineInstr> Instructions; Instructions Insts; const BasicBlock *BB; int Number; MachineFunction *xParent; - /// Predecessors/Successors - Keep track of the predecessor / successor - /// basicblocks. + /// Keep track of the predecessor / successor basic blocks. std::vector<MachineBasicBlock *> Predecessors; std::vector<MachineBasicBlock *> Successors; - /// Weights - Keep track of the weights to the successors. This vector - /// has the same order as Successors, or it is empty if we don't use it - /// (disable optimization). - std::vector<uint32_t> Weights; - typedef std::vector<uint32_t>::iterator weight_iterator; - typedef std::vector<uint32_t>::const_iterator const_weight_iterator; + /// Keep track of the probabilities to the successors. This vector has the + /// same order as Successors, or it is empty if we don't use it (disable + /// optimization). + std::vector<BranchProbability> Probs; + typedef std::vector<BranchProbability>::iterator probability_iterator; + typedef std::vector<BranchProbability>::const_iterator + const_probability_iterator; + + /// Keep track of the physical registers that are livein of the basicblock. + typedef std::vector<RegisterMaskPair> LiveInVector; + LiveInVector LiveIns; + + /// Alignment of the basic block. Zero if the basic block does not need to be + /// aligned. The alignment is specified as log2(bytes). + unsigned Alignment = 0; - /// LiveIns - Keep track of the physical registers that are livein of - /// the basicblock. - std::vector<unsigned> LiveIns; + /// Indicate that this basic block is entered via an exception handler. + bool IsEHPad = false; - /// Alignment - Alignment of the basic block. Zero if the basic block does - /// not need to be aligned. - /// The alignment is specified as log2(bytes). - unsigned Alignment; + /// Indicate that this basic block is potentially the target of an indirect + /// branch. + bool AddressTaken = false; - /// IsLandingPad - Indicate that this basic block is entered via an - /// exception handler. - bool IsLandingPad; + /// Indicate that this basic block is the entry block of an EH funclet. + bool IsEHFuncletEntry = false; - /// AddressTaken - Indicate that this basic block is potentially the - /// target of an indirect branch. - bool AddressTaken; + /// Indicate that this basic block is the entry block of a cleanup funclet. + bool IsCleanupFuncletEntry = false; /// \brief since getSymbol is a relatively heavy-weight operation, the symbol /// is only computed once and is cached. - mutable MCSymbol *CachedMCSymbol; + mutable MCSymbol *CachedMCSymbol = nullptr; // Intrusive list support MachineBasicBlock() {} - explicit MachineBasicBlock(MachineFunction &mf, const BasicBlock *bb); + explicit MachineBasicBlock(MachineFunction &MF, const BasicBlock *BB); ~MachineBasicBlock(); @@ -110,50 +135,44 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> { friend class MachineFunction; public: - /// getBasicBlock - Return the LLVM basic block that this instance - /// corresponded to originally. Note that this may be NULL if this instance - /// does not correspond directly to an LLVM basic block. - /// + /// Return the LLVM basic block that this instance corresponded to originally. + /// Note that this may be NULL if this instance does not correspond directly + /// to an LLVM basic block. const BasicBlock *getBasicBlock() const { return BB; } - /// getName - Return the name of the corresponding LLVM basic block, or - /// "(null)". + /// Return the name of the corresponding LLVM basic block, or "(null)". StringRef getName() const; - /// getFullName - Return a formatted string to identify this block and its - /// parent function. + /// Return a formatted string to identify this block and its parent function. std::string getFullName() const; - /// hasAddressTaken - Test whether this block is potentially the target - /// of an indirect branch. + /// Test whether this block is potentially the target of an indirect branch. bool hasAddressTaken() const { return AddressTaken; } - /// setHasAddressTaken - Set this block to reflect that it potentially - /// is the target of an indirect branch. + /// Set this block to reflect that it potentially is the target of an indirect + /// branch. void setHasAddressTaken() { AddressTaken = true; } - /// getParent - Return the MachineFunction containing this basic block. - /// + /// Return the MachineFunction containing this basic block. const MachineFunction *getParent() const { return xParent; } MachineFunction *getParent() { return xParent; } - - /// bundle_iterator - MachineBasicBlock iterator that automatically skips over - /// MIs that are inside bundles (i.e. walk top level MIs only). + /// MachineBasicBlock iterator that automatically skips over MIs that are + /// inside bundles (i.e. walk top level MIs only). template<typename Ty, typename IterTy> class bundle_iterator : public std::iterator<std::bidirectional_iterator_tag, Ty, ptrdiff_t> { IterTy MII; public: - bundle_iterator(IterTy mii) : MII(mii) {} + bundle_iterator(IterTy MI) : MII(MI) {} - bundle_iterator(Ty &mi) : MII(mi) { - assert(!mi.isBundledWithPred() && + bundle_iterator(Ty &MI) : MII(MI) { + assert(!MI.isBundledWithPred() && "It's not legal to initialize bundle_iterator with a bundled MI"); } - bundle_iterator(Ty *mi) : MII(mi) { - assert((!mi || !mi->isBundledWithPred()) && + bundle_iterator(Ty *MI) : MII(MI) { + assert((!MI || !MI->isBundledWithPred()) && "It's not legal to initialize bundle_iterator with a bundled MI"); } // Template allows conversion from const to nonconst. @@ -165,13 +184,13 @@ public: Ty &operator*() const { return *MII; } Ty *operator->() const { return &operator*(); } - operator Ty*() const { return MII; } + operator Ty *() const { return MII.getNodePtrUnchecked(); } - bool operator==(const bundle_iterator &x) const { - return MII == x.MII; + bool operator==(const bundle_iterator &X) const { + return MII == X.MII; } - bool operator!=(const bundle_iterator &x) const { - return !operator==(x); + bool operator!=(const bundle_iterator &X) const { + return !operator==(X); } // Increment and decrement operators... @@ -247,11 +266,16 @@ public: reverse_iterator rend () { return instr_rend(); } const_reverse_iterator rend () const { return instr_rend(); } + /// Support for MachineInstr::getNextNode(). + static Instructions MachineBasicBlock::*getSublistAccess(MachineInstr *) { + return &MachineBasicBlock::Insts; + } + inline iterator_range<iterator> terminators() { - return iterator_range<iterator>(getFirstTerminator(), end()); + return make_range(getFirstTerminator(), end()); } inline iterator_range<const_iterator> terminators() const { - return iterator_range<const_iterator>(getFirstTerminator(), end()); + return make_range(getFirstTerminator(), end()); } // Machine-CFG iterators @@ -301,16 +325,16 @@ public: bool succ_empty() const { return Successors.empty(); } inline iterator_range<pred_iterator> predecessors() { - return iterator_range<pred_iterator>(pred_begin(), pred_end()); + return make_range(pred_begin(), pred_end()); } inline iterator_range<const_pred_iterator> predecessors() const { - return iterator_range<const_pred_iterator>(pred_begin(), pred_end()); + return make_range(pred_begin(), pred_end()); } inline iterator_range<succ_iterator> successors() { - return iterator_range<succ_iterator>(succ_begin(), succ_end()); + return make_range(succ_begin(), succ_end()); } inline iterator_range<const_succ_iterator> successors() const { - return iterator_range<const_succ_iterator>(succ_begin(), succ_end()); + return make_range(succ_begin(), succ_end()); } // LiveIn management methods. @@ -318,131 +342,177 @@ public: /// Adds the specified register as a live in. Note that it is an error to add /// the same register to the same set more than once unless the intention is /// to call sortUniqueLiveIns after all registers are added. - void addLiveIn(unsigned Reg) { LiveIns.push_back(Reg); } + void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask = ~0u) { + LiveIns.push_back(RegisterMaskPair(PhysReg, LaneMask)); + } + void addLiveIn(const RegisterMaskPair &RegMaskPair) { + LiveIns.push_back(RegMaskPair); + } /// Sorts and uniques the LiveIns vector. It can be significantly faster to do /// this than repeatedly calling isLiveIn before calling addLiveIn for every /// LiveIn insertion. - void sortUniqueLiveIns() { - std::sort(LiveIns.begin(), LiveIns.end()); - LiveIns.erase(std::unique(LiveIns.begin(), LiveIns.end()), LiveIns.end()); - } + void sortUniqueLiveIns(); /// Add PhysReg as live in to this block, and ensure that there is a copy of /// PhysReg to a virtual register of class RC. Return the virtual register /// that is a copy of the live in PhysReg. - unsigned addLiveIn(unsigned PhysReg, const TargetRegisterClass *RC); + unsigned addLiveIn(MCPhysReg PhysReg, const TargetRegisterClass *RC); - /// removeLiveIn - Remove the specified register from the live in set. - /// - void removeLiveIn(unsigned Reg); + /// Remove the specified register from the live in set. + void removeLiveIn(MCPhysReg Reg, LaneBitmask LaneMask = ~0u); - /// isLiveIn - Return true if the specified register is in the live in set. - /// - bool isLiveIn(unsigned Reg) const; + /// Return true if the specified register is in the live in set. + bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask = ~0u) const; // Iteration support for live in sets. These sets are kept in sorted // order by their register number. - typedef std::vector<unsigned>::const_iterator livein_iterator; + typedef LiveInVector::const_iterator livein_iterator; livein_iterator livein_begin() const { return LiveIns.begin(); } livein_iterator livein_end() const { return LiveIns.end(); } bool livein_empty() const { return LiveIns.empty(); } + iterator_range<livein_iterator> liveins() const { + return make_range(livein_begin(), livein_end()); + } - /// getAlignment - Return alignment of the basic block. - /// The alignment is specified as log2(bytes). - /// + /// Get the clobber mask for the start of this basic block. Funclets use this + /// to prevent register allocation across funclet transitions. + const uint32_t *getBeginClobberMask(const TargetRegisterInfo *TRI) const; + + /// Get the clobber mask for the end of the basic block. + /// \see getBeginClobberMask() + const uint32_t *getEndClobberMask(const TargetRegisterInfo *TRI) const; + + /// Return alignment of the basic block. The alignment is specified as + /// log2(bytes). unsigned getAlignment() const { return Alignment; } - /// setAlignment - Set alignment of the basic block. - /// The alignment is specified as log2(bytes). - /// + /// Set alignment of the basic block. The alignment is specified as + /// log2(bytes). void setAlignment(unsigned Align) { Alignment = Align; } - /// isLandingPad - Returns true if the block is a landing pad. That is - /// this basic block is entered via an exception handler. - bool isLandingPad() const { return IsLandingPad; } + /// Returns true if the block is a landing pad. That is this basic block is + /// entered via an exception handler. + bool isEHPad() const { return IsEHPad; } - /// setIsLandingPad - Indicates the block is a landing pad. That is - /// this basic block is entered via an exception handler. - void setIsLandingPad(bool V = true) { IsLandingPad = V; } + /// Indicates the block is a landing pad. That is this basic block is entered + /// via an exception handler. + void setIsEHPad(bool V = true) { IsEHPad = V; } - /// getLandingPadSuccessor - If this block has a successor that is a landing - /// pad, return it. Otherwise return NULL. + /// If this block has a successor that is a landing pad, return it. Otherwise + /// return NULL. const MachineBasicBlock *getLandingPadSuccessor() const; + bool hasEHPadSuccessor() const; + + /// Returns true if this is the entry block of an EH funclet. + bool isEHFuncletEntry() const { return IsEHFuncletEntry; } + + /// Indicates if this is the entry block of an EH funclet. + void setIsEHFuncletEntry(bool V = true) { IsEHFuncletEntry = V; } + + /// Returns true if this is the entry block of a cleanup funclet. + bool isCleanupFuncletEntry() const { return IsCleanupFuncletEntry; } + + /// Indicates if this is the entry block of a cleanup funclet. + void setIsCleanupFuncletEntry(bool V = true) { IsCleanupFuncletEntry = V; } + // Code Layout methods. - /// moveBefore/moveAfter - move 'this' block before or after the specified - /// block. This only moves the block, it does not modify the CFG or adjust - /// potential fall-throughs at the end of the block. + /// Move 'this' block before or after the specified block. This only moves + /// the block, it does not modify the CFG or adjust potential fall-throughs at + /// the end of the block. void moveBefore(MachineBasicBlock *NewAfter); void moveAfter(MachineBasicBlock *NewBefore); - /// updateTerminator - Update the terminator instructions in block to account - /// for changes to the layout. If the block previously used a fallthrough, - /// it may now need a branch, and if it previously used branching it may now - /// be able to use a fallthrough. + /// Update the terminator instructions in block to account for changes to the + /// layout. If the block previously used a fallthrough, it may now need a + /// branch, and if it previously used branching it may now be able to use a + /// fallthrough. void updateTerminator(); // Machine-CFG mutators - /// addSuccessor - Add succ as a successor of this MachineBasicBlock. - /// The Predecessors list of succ is automatically updated. WEIGHT - /// parameter is stored in Weights list and it may be used by - /// MachineBranchProbabilityInfo analysis to calculate branch probability. + /// Add Succ as a successor of this MachineBasicBlock. The Predecessors list + /// of Succ is automatically updated. PROB parameter is stored in + /// Probabilities list. The default probability is set as unknown. Mixing + /// known and unknown probabilities in successor list is not allowed. When all + /// successors have unknown probabilities, 1 / N is returned as the + /// probability for each successor, where N is the number of successors. /// /// Note that duplicate Machine CFG edges are not allowed. - /// - void addSuccessor(MachineBasicBlock *succ, uint32_t weight = 0); - - /// Set successor weight of a given iterator. - void setSuccWeight(succ_iterator I, uint32_t weight); - - /// removeSuccessor - Remove successor from the successors list of this - /// MachineBasicBlock. The Predecessors list of succ is automatically updated. - /// - void removeSuccessor(MachineBasicBlock *succ); - - /// removeSuccessor - Remove specified successor from the successors list of - /// this MachineBasicBlock. The Predecessors list of succ is automatically - /// updated. Return the iterator to the element after the one removed. - /// - succ_iterator removeSuccessor(succ_iterator I); - - /// replaceSuccessor - Replace successor OLD with NEW and update weight info. - /// + void addSuccessor(MachineBasicBlock *Succ, + BranchProbability Prob = BranchProbability::getUnknown()); + + /// Add Succ as a successor of this MachineBasicBlock. The Predecessors list + /// of Succ is automatically updated. The probability is not provided because + /// BPI is not available (e.g. -O0 is used), in which case edge probabilities + /// won't be used. Using this interface can save some space. + void addSuccessorWithoutProb(MachineBasicBlock *Succ); + + /// Set successor probability of a given iterator. + void setSuccProbability(succ_iterator I, BranchProbability Prob); + + /// Normalize probabilities of all successors so that the sum of them becomes + /// one. This is usually done when the current update on this MBB is done, and + /// the sum of its successors' probabilities is not guaranteed to be one. The + /// user is responsible for the correct use of this function. + /// MBB::removeSuccessor() has an option to do this automatically. + void normalizeSuccProbs() { + BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); + } + + /// Validate successors' probabilities and check if the sum of them is + /// approximate one. This only works in DEBUG mode. + void validateSuccProbs() const; + + /// Remove successor from the successors list of this MachineBasicBlock. The + /// Predecessors list of Succ is automatically updated. + /// If NormalizeSuccProbs is true, then normalize successors' probabilities + /// after the successor is removed. + void removeSuccessor(MachineBasicBlock *Succ, + bool NormalizeSuccProbs = false); + + /// Remove specified successor from the successors list of this + /// MachineBasicBlock. The Predecessors list of Succ is automatically updated. + /// If NormalizeSuccProbs is true, then normalize successors' probabilities + /// after the successor is removed. + /// Return the iterator to the element after the one removed. + succ_iterator removeSuccessor(succ_iterator I, + bool NormalizeSuccProbs = false); + + /// Replace successor OLD with NEW and update probability info. void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New); + /// Transfers all the successors from MBB to this machine basic block (i.e., + /// copies all the successors FromMBB and remove all the successors from + /// FromMBB). + void transferSuccessors(MachineBasicBlock *FromMBB); - /// transferSuccessors - Transfers all the successors from MBB to this - /// machine basic block (i.e., copies all the successors fromMBB and - /// remove all the successors from fromMBB). - void transferSuccessors(MachineBasicBlock *fromMBB); + /// Transfers all the successors, as in transferSuccessors, and update PHI + /// operands in the successor blocks which refer to FromMBB to refer to this. + void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB); - /// transferSuccessorsAndUpdatePHIs - Transfers all the successors, as - /// in transferSuccessors, and update PHI operands in the successor blocks - /// which refer to fromMBB to refer to this. - void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *fromMBB); + /// Return true if any of the successors have probabilities attached to them. + bool hasSuccessorProbabilities() const { return !Probs.empty(); } - /// isPredecessor - Return true if the specified MBB is a predecessor of this - /// block. + /// Return true if the specified MBB is a predecessor of this block. bool isPredecessor(const MachineBasicBlock *MBB) const; - /// isSuccessor - Return true if the specified MBB is a successor of this - /// block. + /// Return true if the specified MBB is a successor of this block. bool isSuccessor(const MachineBasicBlock *MBB) const; - /// isLayoutSuccessor - Return true if the specified MBB will be emitted - /// immediately after this block, such that if this block exits by - /// falling through, control will transfer to the specified MBB. Note - /// that MBB need not be a successor at all, for example if this block - /// ends with an unconditional branch to some other block. + /// Return true if the specified MBB will be emitted immediately after this + /// block, such that if this block exits by falling through, control will + /// transfer to the specified MBB. Note that MBB need not be a successor at + /// all, for example if this block ends with an unconditional branch to some + /// other block. bool isLayoutSuccessor(const MachineBasicBlock *MBB) const; - /// canFallThrough - Return true if the block can implicitly transfer - /// control to the block after it by falling off the end of it. This should - /// return false if it can reach the block after it, but it uses an explicit - /// branch to do so (e.g., a table jump). True is a conservative answer. + /// Return true if the block can implicitly transfer control to the block + /// after it by falling off the end of it. This should return false if it can + /// reach the block after it, but it uses an explicit branch to do so (e.g., a + /// table jump). True is a conservative answer. bool canFallThrough(); /// Returns a pointer to the first instruction in this block that is not a @@ -452,40 +522,44 @@ public: /// Returns end() is there's no non-PHI instruction. iterator getFirstNonPHI(); - /// SkipPHIsAndLabels - Return the first instruction in MBB after I that is - /// not a PHI or a label. This is the correct point to insert copies at the - /// beginning of a basic block. + /// Return the first instruction in MBB after I that is not a PHI or a label. + /// This is the correct point to insert copies at the beginning of a basic + /// block. iterator SkipPHIsAndLabels(iterator I); - /// getFirstTerminator - returns an iterator to the first terminator - /// instruction of this basic block. If a terminator does not exist, - /// it returns end() + /// Returns an iterator to the first terminator instruction of this basic + /// block. If a terminator does not exist, it returns end(). iterator getFirstTerminator(); const_iterator getFirstTerminator() const { return const_cast<MachineBasicBlock *>(this)->getFirstTerminator(); } - /// getFirstInstrTerminator - Same getFirstTerminator but it ignores bundles - /// and return an instr_iterator instead. + /// Same getFirstTerminator but it ignores bundles and return an + /// instr_iterator instead. instr_iterator getFirstInstrTerminator(); - /// getFirstNonDebugInstr - returns an iterator to the first non-debug - /// instruction in the basic block, or end() + /// Returns an iterator to the first non-debug instruction in the basic block, + /// or end(). iterator getFirstNonDebugInstr(); const_iterator getFirstNonDebugInstr() const { return const_cast<MachineBasicBlock *>(this)->getFirstNonDebugInstr(); } - /// getLastNonDebugInstr - returns an iterator to the last non-debug - /// instruction in the basic block, or end() + /// Returns an iterator to the last non-debug instruction in the basic block, + /// or end(). iterator getLastNonDebugInstr(); const_iterator getLastNonDebugInstr() const { return const_cast<MachineBasicBlock *>(this)->getLastNonDebugInstr(); } - /// SplitCriticalEdge - Split the critical edge from this block to the - /// given successor block, and return the newly created block, or null - /// if splitting is not possible. + /// Convenience function that returns true if the block ends in a return + /// instruction. + bool isReturnBlock() const { + return !empty() && back().isReturn(); + } + + /// Split the critical edge from this block to the given successor block, and + /// return the newly created block, or null if splitting is not possible. /// /// This function updates LiveVariables, MachineDominatorTree, and /// MachineLoopInfo, as applicable. @@ -570,7 +644,7 @@ public: /// remove_instr to remove individual instructions from a bundle. MachineInstr *remove(MachineInstr *I) { assert(!I->isBundled() && "Cannot remove bundled instructions"); - return Insts.remove(I); + return Insts.remove(instr_iterator(I)); } /// Remove the possibly bundled instruction from the instruction list @@ -605,30 +679,29 @@ public: From.getInstrIterator(), To.getInstrIterator()); } - /// removeFromParent - This method unlinks 'this' from the containing - /// function, and returns it, but does not delete it. + /// This method unlinks 'this' from the containing function, and returns it, + /// but does not delete it. MachineBasicBlock *removeFromParent(); - /// eraseFromParent - This method unlinks 'this' from the containing - /// function and deletes it. + /// This method unlinks 'this' from the containing function and deletes it. void eraseFromParent(); - /// ReplaceUsesOfBlockWith - Given a machine basic block that branched to - /// 'Old', change the code and CFG so that it branches to 'New' instead. + /// Given a machine basic block that branched to 'Old', change the code and + /// CFG so that it branches to 'New' instead. void ReplaceUsesOfBlockWith(MachineBasicBlock *Old, MachineBasicBlock *New); - /// CorrectExtraCFGEdges - Various pieces of code can cause excess edges in - /// the CFG to be inserted. If we have proven that MBB can only branch to - /// DestA and DestB, remove any other MBB successors from the CFG. DestA and - /// DestB can be null. Besides DestA and DestB, retain other edges leading - /// to LandingPads (currently there can be only one; we don't check or require - /// that here). Note it is possible that DestA and/or DestB are LandingPads. + /// Various pieces of code can cause excess edges in the CFG to be inserted. + /// If we have proven that MBB can only branch to DestA and DestB, remove any + /// other MBB successors from the CFG. DestA and DestB can be null. Besides + /// DestA and DestB, retain other edges leading to LandingPads (currently + /// there can be only one; we don't check or require that here). Note it is + /// possible that DestA and/or DestB are LandingPads. bool CorrectExtraCFGEdges(MachineBasicBlock *DestA, MachineBasicBlock *DestB, - bool isCond); + bool IsCond); - /// findDebugLoc - find the next valid DebugLoc starting at MBBI, skipping - /// any DBG_VALUE instructions. Return UnknownLoc if there is none. + /// Find the next valid DebugLoc starting at MBBI, skipping any DBG_VALUE + /// instructions. Return UnknownLoc if there is none. DebugLoc findDebugLoc(instr_iterator MBBI); DebugLoc findDebugLoc(iterator MBBI) { return findDebugLoc(MBBI.getInstrIterator()); @@ -636,12 +709,9 @@ public: /// Possible outcome of a register liveness query to computeRegisterLiveness() enum LivenessQueryResult { - LQR_Live, ///< Register is known to be live. - LQR_OverlappingLive, ///< Register itself is not live, but some overlapping - ///< register is. - LQR_Dead, ///< Register is known to be dead. - LQR_Unknown ///< Register liveness not decidable from local - ///< neighborhood. + LQR_Live, ///< Register is known to be (at least partially) live. + LQR_Dead, ///< Register is known to be fully dead. + LQR_Unknown ///< Register liveness not decidable from local neighborhood. }; /// Return whether (physical) register \p Reg has been <def>ined and not @@ -666,49 +736,43 @@ public: // Printing method used by LoopInfo. void printAsOperand(raw_ostream &OS, bool PrintType = true) const; - /// getNumber - MachineBasicBlocks are uniquely numbered at the function - /// level, unless they're not in a MachineFunction yet, in which case this - /// will return -1. - /// + /// MachineBasicBlocks are uniquely numbered at the function level, unless + /// they're not in a MachineFunction yet, in which case this will return -1. int getNumber() const { return Number; } void setNumber(int N) { Number = N; } - /// getSymbol - Return the MCSymbol for this basic block. - /// + /// Return the MCSymbol for this basic block. MCSymbol *getSymbol() const; private: - /// getWeightIterator - Return weight iterator corresponding to the I - /// successor iterator. - weight_iterator getWeightIterator(succ_iterator I); - const_weight_iterator getWeightIterator(const_succ_iterator I) const; + /// Return probability iterator corresponding to the I successor iterator. + probability_iterator getProbabilityIterator(succ_iterator I); + const_probability_iterator + getProbabilityIterator(const_succ_iterator I) const; friend class MachineBranchProbabilityInfo; + friend class MIPrinter; - /// getSuccWeight - Return weight of the edge from this block to MBB. This - /// method should NOT be called directly, but by using getEdgeWeight method - /// from MachineBranchProbabilityInfo class. - uint32_t getSuccWeight(const_succ_iterator Succ) const; - + /// Return probability of the edge from this block to MBB. This method should + /// NOT be called directly, but by using getEdgeProbability method from + /// MachineBranchProbabilityInfo class. + BranchProbability getSuccProbability(const_succ_iterator Succ) const; // Methods used to maintain doubly linked list of blocks... friend struct ilist_traits<MachineBasicBlock>; // Machine-CFG mutators - /// addPredecessor - Remove pred as a predecessor of this MachineBasicBlock. - /// Don't do this unless you know what you're doing, because it doesn't - /// update pred's successors list. Use pred->addSuccessor instead. - /// - void addPredecessor(MachineBasicBlock *pred); + /// Remove Pred as a predecessor of this MachineBasicBlock. Don't do this + /// unless you know what you're doing, because it doesn't update Pred's + /// successors list. Use Pred->addSuccessor instead. + void addPredecessor(MachineBasicBlock *Pred); - /// removePredecessor - Remove pred as a predecessor of this - /// MachineBasicBlock. Don't do this unless you know what you're - /// doing, because it doesn't update pred's successors list. Use - /// pred->removeSuccessor instead. - /// - void removePredecessor(MachineBasicBlock *pred); + /// Remove Pred as a predecessor of this MachineBasicBlock. Don't do this + /// unless you know what you're doing, because it doesn't update Pred's + /// successors list. Use Pred->removeSuccessor instead. + void removePredecessor(MachineBasicBlock *Pred); }; raw_ostream& operator<<(raw_ostream &OS, const MachineBasicBlock &MBB); @@ -726,7 +790,7 @@ struct MBB2NumberFunctor : //===--------------------------------------------------------------------===// // Provide specializations of GraphTraits to be able to treat a -// MachineFunction as a graph of MachineBasicBlocks... +// MachineFunction as a graph of MachineBasicBlocks. // template <> struct GraphTraits<MachineBasicBlock *> { @@ -756,7 +820,7 @@ template <> struct GraphTraits<const MachineBasicBlock *> { }; // Provide specializations of GraphTraits to be able to treat a -// MachineFunction as a graph of MachineBasicBlocks... and to walk it +// MachineFunction as a graph of MachineBasicBlocks and to walk it // in inverse order. Inverse order for a function is considered // to be when traversing the predecessor edges of a MBB // instead of the successor edges. diff --git a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h index 7ba749559c0f..81b0524cf0a4 100644 --- a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h +++ b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h @@ -18,6 +18,7 @@ #include "llvm/Pass.h" #include "llvm/Support/BranchProbability.h" #include <climits> +#include <numeric> namespace llvm { @@ -44,20 +45,15 @@ public: AU.setPreservesAll(); } - // Return edge weight. If we don't have any informations about it - return - // DEFAULT_WEIGHT. - uint32_t getEdgeWeight(const MachineBasicBlock *Src, - const MachineBasicBlock *Dst) const; - - // Same thing, but using a const_succ_iterator from Src. This is faster when - // the iterator is already available. - uint32_t getEdgeWeight(const MachineBasicBlock *Src, - MachineBasicBlock::const_succ_iterator Dst) const; + // Return edge probability. + BranchProbability getEdgeProbability(const MachineBasicBlock *Src, + const MachineBasicBlock *Dst) const; - // Get sum of the block successors' weights, potentially scaling them to fit - // within 32-bits. If scaling is required, sets Scale based on the necessary - // adjustment. Any edge weights used with the sum should be divided by Scale. - uint32_t getSumForBlock(const MachineBasicBlock *MBB, uint32_t &Scale) const; + // Same as above, but using a const_succ_iterator from Src. This is faster + // when the iterator is already available. + BranchProbability + getEdgeProbability(const MachineBasicBlock *Src, + MachineBasicBlock::const_succ_iterator Dst) const; // A 'Hot' edge is an edge which probability is >= 80%. bool isEdgeHot(const MachineBasicBlock *Src, @@ -67,15 +63,6 @@ public: // NB: This routine's complexity is linear on the number of successors. MachineBasicBlock *getHotSucc(MachineBasicBlock *MBB) const; - // Return a probability as a fraction between 0 (0% probability) and - // 1 (100% probability), however the value is never equal to 0, and can be 1 - // only iff SRC block has only one successor. - // NB: This routine's complexity is linear on the number of successors of - // Src. Querying sequentially for each successor's probability is a quadratic - // query pattern. - BranchProbability getEdgeProbability(const MachineBasicBlock *Src, - const MachineBasicBlock *Dst) const; - // Print value between 0 (0% probability) and 1 (100% probability), // however the value is never equal to 0, and can be 1 only iff SRC block // has only one successor. diff --git a/include/llvm/CodeGen/MachineCombinerPattern.h b/include/llvm/CodeGen/MachineCombinerPattern.h index 176af14dc317..f3891227746f 100644 --- a/include/llvm/CodeGen/MachineCombinerPattern.h +++ b/include/llvm/CodeGen/MachineCombinerPattern.h @@ -17,13 +17,30 @@ namespace llvm { -/// Enumeration of instruction pattern supported by machine combiner -/// -/// -namespace MachineCombinerPattern { -// Forward declaration -enum MC_PATTERN : int; -} // end namespace MachineCombinerPattern +/// These are instruction patterns matched by the machine combiner pass. +enum class MachineCombinerPattern { + // These are commutative variants for reassociating a computation chain. See + // the comments before getMachineCombinerPatterns() in TargetInstrInfo.cpp. + REASSOC_AX_BY, + REASSOC_AX_YB, + REASSOC_XA_BY, + REASSOC_XA_YB, + + // These are multiply-add patterns matched by the AArch64 machine combiner. + MULADDW_OP1, + MULADDW_OP2, + MULSUBW_OP1, + MULSUBW_OP2, + MULADDWI_OP1, + MULSUBWI_OP1, + MULADDX_OP1, + MULADDX_OP2, + MULSUBX_OP1, + MULSUBX_OP2, + MULADDXI_OP1, + MULSUBXI_OP1 +}; + } // end namespace llvm #endif diff --git a/include/llvm/CodeGen/MachineConstantPool.h b/include/llvm/CodeGen/MachineConstantPool.h index 628400322f60..d2036c4a29a5 100644 --- a/include/llvm/CodeGen/MachineConstantPool.h +++ b/include/llvm/CodeGen/MachineConstantPool.h @@ -46,13 +46,6 @@ public: /// Type *getType() const { return Ty; } - - /// getRelocationInfo - This method classifies the entry according to - /// whether or not it may generate a relocation entry. This must be - /// conservative, so if it might codegen to a relocatable entry, it should say - /// so. The return values are the same as Constant::getRelocationInfo(). - virtual unsigned getRelocationInfo() const = 0; - virtual int getExistingMachineCPValue(MachineConstantPool *CP, unsigned Alignment) = 0; @@ -67,7 +60,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, V.print(OS); return OS; } - /// This class is a data container for one entry in a MachineConstantPool. /// It contains a pointer to the value and an offset from the start of @@ -90,9 +82,9 @@ public: Val.ConstVal = V; } MachineConstantPoolEntry(MachineConstantPoolValue *V, unsigned A) - : Alignment(A) { - Val.MachineCPVal = V; - Alignment |= 1U << (sizeof(unsigned)*CHAR_BIT-1); + : Alignment(A) { + Val.MachineCPVal = V; + Alignment |= 1U << (sizeof(unsigned) * CHAR_BIT - 1); } /// isMachineConstantPoolEntry - Return true if the MachineConstantPoolEntry @@ -102,28 +94,20 @@ public: return (int)Alignment < 0; } - int getAlignment() const { - return Alignment & ~(1 << (sizeof(unsigned)*CHAR_BIT-1)); + int getAlignment() const { + return Alignment & ~(1 << (sizeof(unsigned) * CHAR_BIT - 1)); } Type *getType() const; - - /// getRelocationInfo - This method classifies the entry according to - /// whether or not it may generate a relocation entry. This must be - /// conservative, so if it might codegen to a relocatable entry, it should say - /// so. The return values are: - /// - /// 0: This constant pool entry is guaranteed to never have a relocation - /// applied to it (because it holds a simple constant like '4'). - /// 1: This entry has relocations, but the entries are guaranteed to be - /// resolvable by the static linker, so the dynamic linker will never see - /// them. - /// 2: This entry may have arbitrary relocations. - unsigned getRelocationInfo() const; + + /// This method classifies the entry according to whether or not it may + /// generate a relocation entry. This must be conservative, so if it might + /// codegen to a relocatable entry, it should say so. + bool needsRelocation() const; SectionKind getSectionKind(const DataLayout *DL) const; }; - + /// The MachineConstantPool class keeps track of constants referenced by a /// function which must be spilled to memory. This is used for constants which /// are unable to be used directly as operands to instructions, which typically @@ -148,17 +132,18 @@ public: explicit MachineConstantPool(const DataLayout &DL) : PoolAlignment(1), DL(DL) {} ~MachineConstantPool(); - + /// getConstantPoolAlignment - Return the alignment required by /// the whole constant pool, of which the first element must be aligned. unsigned getConstantPoolAlignment() const { return PoolAlignment; } - + /// getConstantPoolIndex - Create a new entry in the constant pool or return /// an existing one. User must specify the minimum required alignment for /// the object. unsigned getConstantPoolIndex(const Constant *C, unsigned Alignment); - unsigned getConstantPoolIndex(MachineConstantPoolValue *V,unsigned Alignment); - + unsigned getConstantPoolIndex(MachineConstantPoolValue *V, + unsigned Alignment); + /// isEmpty - Return true if this constant pool contains no constants. bool isEmpty() const { return Constants.empty(); } diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h index 735dd069cf7f..a69936f6e267 100644 --- a/include/llvm/CodeGen/MachineDominators.h +++ b/include/llvm/CodeGen/MachineDominators.h @@ -246,21 +246,29 @@ public: /// iterable by generic graph iterators. /// -template<class T> struct GraphTraits; +template <class Node, class ChildIterator> +struct MachineDomTreeGraphTraitsBase { + typedef Node NodeType; + typedef ChildIterator ChildIteratorType; -template <> struct GraphTraits<MachineDomTreeNode *> { - typedef MachineDomTreeNode NodeType; - typedef NodeType::iterator ChildIteratorType; - - static NodeType *getEntryNode(NodeType *N) { - return N; - } - static inline ChildIteratorType child_begin(NodeType* N) { + static NodeType *getEntryNode(NodeType *N) { return N; } + static inline ChildIteratorType child_begin(NodeType *N) { return N->begin(); } - static inline ChildIteratorType child_end(NodeType* N) { - return N->end(); - } + static inline ChildIteratorType child_end(NodeType *N) { return N->end(); } +}; + +template <class T> struct GraphTraits; + +template <> +struct GraphTraits<MachineDomTreeNode *> + : public MachineDomTreeGraphTraitsBase<MachineDomTreeNode, + MachineDomTreeNode::iterator> {}; + +template <> +struct GraphTraits<const MachineDomTreeNode *> + : public MachineDomTreeGraphTraitsBase<const MachineDomTreeNode, + MachineDomTreeNode::const_iterator> { }; template <> struct GraphTraits<MachineDominatorTree*> diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h index cbc4e66ccc46..48e8ca75052e 100644 --- a/include/llvm/CodeGen/MachineFrameInfo.h +++ b/include/llvm/CodeGen/MachineFrameInfo.h @@ -101,6 +101,13 @@ class MachineFrameInfo { // cannot alias any other memory objects. bool isSpillSlot; + /// If true, this stack slot is used to spill a value (could be deopt + /// and/or GC related) over a statepoint. We know that the address of the + /// slot can't alias any LLVM IR value. This is very similiar to a Spill + /// Slot, but is created by statepoint lowering is SelectionDAG, not the + /// register allocator. + bool isStatepointSpillSlot; + /// If this stack object is originated from an Alloca instruction /// this value saves the original IR allocation. Can be NULL. const AllocaInst *Alloca; @@ -118,13 +125,24 @@ class MachineFrameInfo { StackObject(uint64_t Sz, unsigned Al, int64_t SP, bool IM, bool isSS, const AllocaInst *Val, bool A) : SPOffset(SP), Size(Sz), Alignment(Al), isImmutable(IM), - isSpillSlot(isSS), Alloca(Val), PreAllocated(false), isAliased(A) {} + isSpillSlot(isSS), isStatepointSpillSlot(false), Alloca(Val), + PreAllocated(false), isAliased(A) {} }; /// The alignment of the stack. unsigned StackAlignment; /// Can the stack be realigned. + /// Targets that set this to false don't have the ability to overalign + /// their stack frame, and thus, overaligned allocas are all treated + /// as dynamic allocations and the target must handle them as part + /// of DYNAMIC_STACKALLOC lowering. + /// FIXME: There is room for improvement in this case, in terms of + /// grouping overaligned allocas into a "secondary stack frame" and + /// then only use a single alloca to allocate this frame and only a + /// single virtual register to access it. Currently, without such an + /// optimization, each such alloca gets it's own dynamic + /// realignment. bool StackRealignable; /// The list of stack objects allocated. @@ -168,7 +186,7 @@ class MachineFrameInfo { /// SP then OffsetAdjustment is zero; if FP is used, OffsetAdjustment is set /// to the distance between the initial SP and the value in FP. For many /// targets, this value is only used when generating debug info (via - /// TargetRegisterInfo::getFrameIndexOffset); when generating code, the + /// TargetRegisterInfo::getFrameIndexReference); when generating code, the /// corresponding adjustments are performed directly. int OffsetAdjustment; @@ -198,7 +216,7 @@ class MachineFrameInfo { /// This contains the size of the largest call frame if the target uses frame /// setup/destroy pseudo instructions (as defined in the TargetFrameInfo /// class). This information is important for frame pointer elimination. - /// If is only valid during and after prolog/epilog code insertion. + /// It is only valid during and after prolog/epilog code insertion. unsigned MaxCallFrameSize; /// The prolog/epilog code inserter fills in this vector with each @@ -288,6 +306,7 @@ public: /// Return the index for the stack protector object. int getStackProtectorIndex() const { return StackProtectorIdx; } void setStackProtectorIndex(int I) { StackProtectorIdx = I; } + bool hasStackProtectorIndex() const { return StackProtectorIdx != -1; } /// Return the index for the function context object. /// This object is used for SjLj exceptions. @@ -337,14 +356,14 @@ public: } /// Get the local offset mapping for a for an object. - std::pair<int, int64_t> getLocalFrameObjectMap(int i) { + std::pair<int, int64_t> getLocalFrameObjectMap(int i) const { assert (i >= 0 && (unsigned)i < LocalFrameObjects.size() && "Invalid local object reference!"); return LocalFrameObjects[i]; } /// Return the number of objects allocated into the local object block. - int64_t getLocalFrameObjectCount() { return LocalFrameObjects.size(); } + int64_t getLocalFrameObjectCount() const { return LocalFrameObjects.size(); } /// Set the size of the local object blob. void setLocalFrameSize(int64_t sz) { LocalFrameSize = sz; } @@ -361,7 +380,9 @@ public: /// Get whether the local allocation blob should be allocated together or /// let PEI allocate the locals in it directly. - bool getUseLocalStackAllocationBlock() {return UseLocalStackAllocationBlock;} + bool getUseLocalStackAllocationBlock() const { + return UseLocalStackAllocationBlock; + } /// setUseLocalStackAllocationBlock - Set whether the local allocation blob /// should be allocated together or let PEI allocate the locals in it @@ -534,6 +555,12 @@ public: return Objects[ObjectIdx+NumFixedObjects].isSpillSlot; } + bool isStatepointSpillSlotObjectIndex(int ObjectIdx) const { + assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && + "Invalid Object Idx!"); + return Objects[ObjectIdx+NumFixedObjects].isStatepointSpillSlot; + } + /// Returns true if the specified index corresponds to a dead object. bool isDeadObjectIndex(int ObjectIdx) const { assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && @@ -549,6 +576,13 @@ public: return Objects[ObjectIdx + NumFixedObjects].Size == 0; } + void markAsStatepointSpillSlotObjectIndex(int ObjectIdx) { + assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && + "Invalid Object Idx!"); + Objects[ObjectIdx+NumFixedObjects].isStatepointSpillSlot = true; + assert(isStatepointSpillSlotObjectIndex(ObjectIdx) && "inconsistent"); + } + /// Create a new statically sized stack object, returning /// a nonnegative identifier to represent it. int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS, diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h index c15ee1c006cd..82c30d39afd6 100644 --- a/include/llvm/CodeGen/MachineFunction.h +++ b/include/llvm/CodeGen/MachineFunction.h @@ -38,10 +38,12 @@ class MachineJumpTableInfo; class MachineModuleInfo; class MCContext; class Pass; +class PseudoSourceValueManager; class TargetMachine; class TargetSubtargetInfo; class TargetRegisterClass; struct MachinePointerInfo; +struct WinEHFuncInfo; template <> struct ilist_traits<MachineBasicBlock> @@ -102,10 +104,14 @@ class MachineFunction { // Keep track of constants which are spilled to memory MachineConstantPool *ConstantPool; - + // Keep track of jump tables for switch instructions MachineJumpTableInfo *JumpTableInfo; + // Keeps track of Windows exception handling related data. This will be null + // for functions that aren't using a funclet-based EH personality. + WinEHFuncInfo *WinEHInfo = nullptr; + // Function-level unique numbering for MachineBasicBlocks. When a // MachineBasicBlock is inserted into a MachineFunction is it automatically // numbered and this vector keeps track of the mapping from ID's to MBB's. @@ -131,7 +137,7 @@ class MachineFunction { /// this translation unit. /// unsigned FunctionNumber; - + /// Alignment - The alignment of the function. unsigned Alignment; @@ -145,6 +151,9 @@ class MachineFunction { /// True if the function includes any inline assembly. bool HasInlineAsm; + // Allocation management for pseudo source values. + std::unique_ptr<PseudoSourceValueManager> PSVManager; + MachineFunction(const MachineFunction &) = delete; void operator=(const MachineFunction&) = delete; public: @@ -155,6 +164,8 @@ public: MachineModuleInfo &getMMI() const { return MMI; } MCContext &getContext() const { return Ctx; } + PseudoSourceValueManager &getPSVManager() const { return *PSVManager; } + /// Return the DataLayout attached to the Module associated to this MF. const DataLayout &getDataLayout() const; @@ -198,7 +209,7 @@ public: MachineFrameInfo *getFrameInfo() { return FrameInfo; } const MachineFrameInfo *getFrameInfo() const { return FrameInfo; } - /// getJumpTableInfo - Return the jump table info object for the current + /// getJumpTableInfo - Return the jump table info object for the current /// function. This object contains information about jump tables in the /// current function. If the current function has no jump tables, this will /// return null. @@ -209,13 +220,18 @@ public: /// does already exist, allocate one. MachineJumpTableInfo *getOrCreateJumpTableInfo(unsigned JTEntryKind); - /// getConstantPool - Return the constant pool object for the current /// function. /// MachineConstantPool *getConstantPool() { return ConstantPool; } const MachineConstantPool *getConstantPool() const { return ConstantPool; } + /// getWinEHFuncInfo - Return information about how the current function uses + /// Windows exception handling. Returns null for functions that don't use + /// funclets for exception handling. + const WinEHFuncInfo *getWinEHFuncInfo() const { return WinEHInfo; } + WinEHFuncInfo *getWinEHFuncInfo() { return WinEHInfo; } + /// getAlignment - Return the alignment (log2, not bytes) of the function. /// unsigned getAlignment() const { return Alignment; } @@ -284,14 +300,14 @@ public: /// getNumBlockIDs - Return the number of MBB ID's allocated. /// unsigned getNumBlockIDs() const { return (unsigned)MBBNumbering.size(); } - + /// RenumberBlocks - This discards all of the MachineBasicBlock numbers and /// recomputes them. This guarantees that the MBB numbers are sequential, /// dense, and match the ordering of the blocks within the function. If a /// specific MachineBasicBlock is specified, only that block and those after /// it are renumbered. void RenumberBlocks(MachineBasicBlock *MBBFrom = nullptr); - + /// print - Print out the MachineFunction in a format suitable for debugging /// to the specified stream. /// @@ -326,6 +342,12 @@ public: typedef std::reverse_iterator<const_iterator> const_reverse_iterator; typedef std::reverse_iterator<iterator> reverse_iterator; + /// Support for MachineBasicBlock::getNextNode(). + static BasicBlockListType MachineFunction::* + getSublistAccess(MachineBasicBlock *) { + return &MachineFunction::BasicBlocks; + } + /// addLiveIn - Add the specified physical register as a live-in value and /// create a corresponding virtual register for it. unsigned addLiveIn(unsigned PReg, const TargetRegisterClass *RC); @@ -358,15 +380,21 @@ public: void splice(iterator InsertPt, iterator MBBI) { BasicBlocks.splice(InsertPt, BasicBlocks, MBBI); } + void splice(iterator InsertPt, MachineBasicBlock *MBB) { + BasicBlocks.splice(InsertPt, BasicBlocks, MBB); + } void splice(iterator InsertPt, iterator MBBI, iterator MBBE) { BasicBlocks.splice(InsertPt, BasicBlocks, MBBI, MBBE); } - void remove(iterator MBBI) { - BasicBlocks.remove(MBBI); - } - void erase(iterator MBBI) { - BasicBlocks.erase(MBBI); + void remove(iterator MBBI) { BasicBlocks.remove(MBBI); } + void remove(MachineBasicBlock *MBBI) { BasicBlocks.remove(MBBI); } + void erase(iterator MBBI) { BasicBlocks.erase(MBBI); } + void erase(MachineBasicBlock *MBBI) { BasicBlocks.erase(MBBI); } + + template <typename Comp> + void sort(Comp comp) { + BasicBlocks.sort(comp); } //===--------------------------------------------------------------------===// @@ -425,7 +453,7 @@ public: unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr); - + /// getMachineMemOperand - Allocate a new MachineMemOperand by copying /// an existing one, adjusting by an offset and using the given size. /// MachineMemOperands are owned by the MachineFunction and need not be @@ -475,16 +503,19 @@ public: extractStoreMemRefs(MachineInstr::mmo_iterator Begin, MachineInstr::mmo_iterator End); + /// Allocate a string and populate it with the given external symbol name. + const char *createExternalSymbolName(StringRef Name); + //===--------------------------------------------------------------------===// // Label Manipulation. // - + /// getJTISymbol - Return the MCSymbol for the specified non-empty jump table. /// If isLinkerPrivate is specified, an 'l' label is returned, otherwise a /// normal 'L' label is returned. - MCSymbol *getJTISymbol(unsigned JTI, MCContext &Ctx, + MCSymbol *getJTISymbol(unsigned JTI, MCContext &Ctx, bool isLinkerPrivate = false) const; - + /// getPICBaseSymbol - Return a function-local symbol to represent the PIC /// base. MCSymbol *getPICBaseSymbol() const; diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index de7e0a29ea0d..978864e96ca5 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -23,6 +23,7 @@ #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugLoc.h" @@ -34,7 +35,6 @@ namespace llvm { template <typename T> class SmallVectorImpl; -class AliasAnalysis; class TargetInstrInfo; class TargetRegisterClass; class TargetRegisterInfo; @@ -48,7 +48,8 @@ class MachineMemOperand; /// MachineFunction is deleted, all the contained MachineInstrs are deallocated /// without having their destructor called. /// -class MachineInstr : public ilist_node<MachineInstr> { +class MachineInstr + : public ilist_node_with_parent<MachineInstr, MachineBasicBlock> { public: typedef MachineMemOperand **mmo_iterator; @@ -64,8 +65,10 @@ public: NoFlags = 0, FrameSetup = 1 << 0, // Instruction is used as a part of // function frame setup code. - BundledPred = 1 << 1, // Instruction has bundled predecessors. - BundledSucc = 1 << 2 // Instruction has bundled successors. + FrameDestroy = 1 << 1, // Instruction is used as a part of + // function frame destruction code. + BundledPred = 1 << 2, // Instruction has bundled predecessors. + BundledSucc = 1 << 3 // Instruction has bundled successors. }; private: const MCInstrDesc *MCID; // Instruction descriptor. @@ -89,6 +92,12 @@ private: // information to AsmPrinter. uint8_t NumMemRefs; // Information on memory references. + // Note that MemRefs == nullptr, means 'don't know', not 'no memory access'. + // Calling code must treat missing information conservatively. If the number + // of memory operands required to be precise exceeds the maximum value of + // NumMemRefs - currently 256 - we remove the operands entirely. Note also + // that this is a non-owning reference to a shared copy on write buffer owned + // by the MachineFunction and created via MF.allocateMemRefsArray. mmo_iterator MemRefs; DebugLoc debugLoc; // Source line information. @@ -293,42 +302,46 @@ public: const_mop_iterator operands_end() const { return Operands + NumOperands; } iterator_range<mop_iterator> operands() { - return iterator_range<mop_iterator>(operands_begin(), operands_end()); + return make_range(operands_begin(), operands_end()); } iterator_range<const_mop_iterator> operands() const { - return iterator_range<const_mop_iterator>(operands_begin(), operands_end()); + return make_range(operands_begin(), operands_end()); } iterator_range<mop_iterator> explicit_operands() { - return iterator_range<mop_iterator>( - operands_begin(), operands_begin() + getNumExplicitOperands()); + return make_range(operands_begin(), + operands_begin() + getNumExplicitOperands()); } iterator_range<const_mop_iterator> explicit_operands() const { - return iterator_range<const_mop_iterator>( - operands_begin(), operands_begin() + getNumExplicitOperands()); + return make_range(operands_begin(), + operands_begin() + getNumExplicitOperands()); } iterator_range<mop_iterator> implicit_operands() { - return iterator_range<mop_iterator>(explicit_operands().end(), - operands_end()); + return make_range(explicit_operands().end(), operands_end()); } iterator_range<const_mop_iterator> implicit_operands() const { - return iterator_range<const_mop_iterator>(explicit_operands().end(), - operands_end()); + return make_range(explicit_operands().end(), operands_end()); } + /// Returns a range over all explicit operands that are register definitions. + /// Implicit definition are not included! iterator_range<mop_iterator> defs() { - return iterator_range<mop_iterator>( - operands_begin(), operands_begin() + getDesc().getNumDefs()); + return make_range(operands_begin(), + operands_begin() + getDesc().getNumDefs()); } + /// \copydoc defs() iterator_range<const_mop_iterator> defs() const { - return iterator_range<const_mop_iterator>( - operands_begin(), operands_begin() + getDesc().getNumDefs()); + return make_range(operands_begin(), + operands_begin() + getDesc().getNumDefs()); } + /// Returns a range that includes all operands that are register uses. + /// This may include unrelated operands which are not register uses. iterator_range<mop_iterator> uses() { - return iterator_range<mop_iterator>( - operands_begin() + getDesc().getNumDefs(), operands_end()); + return make_range(operands_begin() + getDesc().getNumDefs(), + operands_end()); } + /// \copydoc uses() iterator_range<const_mop_iterator> uses() const { - return iterator_range<const_mop_iterator>( - operands_begin() + getDesc().getNumDefs(), operands_end()); + return make_range(operands_begin() + getDesc().getNumDefs(), + operands_end()); } /// Returns the number of the operand iterator \p I points to. @@ -339,13 +352,16 @@ public: /// Access to memory operands of the instruction mmo_iterator memoperands_begin() const { return MemRefs; } mmo_iterator memoperands_end() const { return MemRefs + NumMemRefs; } + /// Return true if we don't have any memory operands which described the the + /// memory access done by this instruction. If this is true, calling code + /// must be conservative. bool memoperands_empty() const { return NumMemRefs == 0; } iterator_range<mmo_iterator> memoperands() { - return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end()); + return make_range(memoperands_begin(), memoperands_end()); } iterator_range<mmo_iterator> memoperands() const { - return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end()); + return make_range(memoperands_begin(), memoperands_end()); } /// Return true if this instruction has exactly one MachineMemOperand. @@ -489,8 +505,8 @@ public: } /// Return true if this instruction is convergent. - /// Convergent instructions can only be moved to locations that are - /// control-equivalent to their initial position. + /// Convergent instructions can not be made control-dependent on any + /// additional values. bool isConvergent(QueryType Type = AnyInBundle) const { return hasProperty(MCID::Convergent, Type); } @@ -897,6 +913,13 @@ public: return (Idx == -1) ? nullptr : &getOperand(Idx); } + const MachineOperand *findRegisterUseOperand( + unsigned Reg, bool isKill = false, + const TargetRegisterInfo *TRI = nullptr) const { + return const_cast<MachineInstr *>(this)-> + findRegisterUseOperand(Reg, isKill, TRI); + } + /// Returns the operand index that is a def of the specified register or /// -1 if it is not found. If isDead is true, defs that are not dead are /// skipped. If Overlap is true, then it also looks for defs that merely @@ -1048,7 +1071,7 @@ public: /// Mark all subregister defs of register @p Reg with the undef flag. /// This function is used when we determined to have a subregister def in an /// otherwise undefined super register. - void addRegisterDefReadUndef(unsigned Reg); + void setRegisterDefReadUndef(unsigned Reg, bool IsUndef = true); /// We have determined MI defines a register. Make sure there is an operand /// defining Reg. @@ -1094,6 +1117,9 @@ public: /// bool hasUnmodeledSideEffects() const; + /// Returns true if it is illegal to fold a load across this instruction. + bool isLoadFoldBarrier() const; + /// Return true if all the defs of this instruction are dead. bool allDefsAreDead() const; @@ -1159,8 +1185,11 @@ public: assert(NumMemRefs == NewMemRefsEnd - NewMemRefs && "Too many memrefs"); } - /// Clear this MachineInstr's memory reference descriptor list. - void clearMemRefs() { + /// Clear this MachineInstr's memory reference descriptor list. This resets + /// the memrefs to their most conservative state. This should be used only + /// as a last resort since it greatly pessimizes our knowledge of the memory + /// access performed by the instruction. + void dropMemRefs() { MemRefs = nullptr; NumMemRefs = 0; } @@ -1174,6 +1203,8 @@ public: } } + /// Add all implicit def and use operands to this instruction. + void addImplicitDefUseOperands(MachineFunction &MF); private: /// If this instruction is embedded into a MachineFunction, return the @@ -1181,9 +1212,6 @@ private: /// return null. MachineRegisterInfo *getRegInfo(); - /// Add all implicit def and use operands to this instruction. - void addImplicitDefUseOperands(MachineFunction &MF); - /// Unlink all of the register operands in this instruction from their /// respective use lists. This requires that the operands already be on their /// use lists. diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h index 4f68f38b7bbf..aa5f4b24df61 100644 --- a/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/include/llvm/CodeGen/MachineInstrBuilder.h @@ -49,11 +49,10 @@ public: MachineInstrBuilder() : MF(nullptr), MI(nullptr) {} /// Create a MachineInstrBuilder for manipulating an existing instruction. - /// F must be the machine function that was used to allocate I. + /// F must be the machine function that was used to allocate I. MachineInstrBuilder(MachineFunction &F, MachineInstr *I) : MF(&F), MI(I) {} /// Allow automatic conversion to the machine instruction we are working on. - /// operator MachineInstr*() const { return MI; } MachineInstr *operator->() const { return MI; } operator MachineBasicBlock::iterator() const { return MI; } @@ -62,11 +61,9 @@ public: /// explicitly. MachineInstr *getInstr() const { return MI; } - /// addReg - Add a new virtual register operand... - /// - const - MachineInstrBuilder &addReg(unsigned RegNo, unsigned flags = 0, - unsigned SubReg = 0) const { + /// Add a new virtual register operand. + const MachineInstrBuilder &addReg(unsigned RegNo, unsigned flags = 0, + unsigned SubReg = 0) const { assert((flags & 0x1) == 0 && "Passing in 'true' to addReg is forbidden! Use enums instead."); MI->addOperand(*MF, MachineOperand::CreateReg(RegNo, @@ -82,8 +79,7 @@ public: return *this; } - /// addImm - Add a new immediate operand. - /// + /// Add a new immediate operand. const MachineInstrBuilder &addImm(int64_t Val) const { MI->addOperand(*MF, MachineOperand::CreateImm(Val)); return *this; @@ -204,44 +200,44 @@ public: // Add a displacement from an existing MachineOperand with an added offset. const MachineInstrBuilder &addDisp(const MachineOperand &Disp, int64_t off, unsigned char TargetFlags = 0) const { + // If caller specifies new TargetFlags then use it, otherwise the + // default behavior is to copy the target flags from the existing + // MachineOperand. This means if the caller wants to clear the + // target flags it needs to do so explicitly. + if (0 == TargetFlags) + TargetFlags = Disp.getTargetFlags(); + switch (Disp.getType()) { default: llvm_unreachable("Unhandled operand type in addDisp()"); case MachineOperand::MO_Immediate: return addImm(Disp.getImm() + off); - case MachineOperand::MO_GlobalAddress: { - // If caller specifies new TargetFlags then use it, otherwise the - // default behavior is to copy the target flags from the existing - // MachineOperand. This means if the caller wants to clear the - // target flags it needs to do so explicitly. - if (TargetFlags) - return addGlobalAddress(Disp.getGlobal(), Disp.getOffset() + off, - TargetFlags); + case MachineOperand::MO_ConstantPoolIndex: + return addConstantPoolIndex(Disp.getIndex(), Disp.getOffset() + off, + TargetFlags); + case MachineOperand::MO_GlobalAddress: return addGlobalAddress(Disp.getGlobal(), Disp.getOffset() + off, - Disp.getTargetFlags()); - } + TargetFlags); } } /// Copy all the implicit operands from OtherMI onto this one. - const MachineInstrBuilder ©ImplicitOps(const MachineInstr *OtherMI) { + const MachineInstrBuilder & + copyImplicitOps(const MachineInstr *OtherMI) const { MI->copyImplicitOps(*MF, OtherMI); return *this; } }; -/// BuildMI - Builder interface. Specify how to create the initial instruction -/// itself. -/// +/// Builder interface. Specify how to create the initial instruction itself. inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID) { return MachineInstrBuilder(MF, MF.CreateMachineInstr(MCID, DL)); } -/// BuildMI - This version of the builder sets up the first operand as a +/// This version of the builder sets up the first operand as a /// destination virtual register. -/// inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID, @@ -250,10 +246,9 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, .addReg(DestReg, RegState::Define); } -/// BuildMI - This version of the builder inserts the newly-built -/// instruction before the given position in the given MachineBasicBlock, and -/// sets up the first operand as a destination virtual register. -/// +/// This version of the builder inserts the newly-built instruction before +/// the given position in the given MachineBasicBlock, and sets up the first +/// operand as a destination virtual register. inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineBasicBlock::iterator I, DebugLoc DL, @@ -282,7 +277,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, const MCInstrDesc &MCID, unsigned DestReg) { if (I->isInsideBundle()) { - MachineBasicBlock::instr_iterator MII = I; + MachineBasicBlock::instr_iterator MII(I); return BuildMI(BB, MII, DL, MCID, DestReg); } @@ -290,10 +285,9 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, return BuildMI(BB, MII, DL, MCID, DestReg); } -/// BuildMI - This version of the builder inserts the newly-built -/// instruction before the given position in the given MachineBasicBlock, and -/// does NOT take a destination register. -/// +/// This version of the builder inserts the newly-built instruction before the +/// given position in the given MachineBasicBlock, and does NOT take a +/// destination register. inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineBasicBlock::iterator I, DebugLoc DL, @@ -319,7 +313,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, DebugLoc DL, const MCInstrDesc &MCID) { if (I->isInsideBundle()) { - MachineBasicBlock::instr_iterator MII = I; + MachineBasicBlock::instr_iterator MII(I); return BuildMI(BB, MII, DL, MCID); } @@ -327,20 +321,17 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, return BuildMI(BB, MII, DL, MCID); } -/// BuildMI - This version of the builder inserts the newly-built -/// instruction at the end of the given MachineBasicBlock, and does NOT take a -/// destination register. -/// +/// This version of the builder inserts the newly-built instruction at the end +/// of the given MachineBasicBlock, and does NOT take a destination register. inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB, DebugLoc DL, const MCInstrDesc &MCID) { return BuildMI(*BB, BB->end(), DL, MCID); } -/// BuildMI - This version of the builder inserts the newly-built -/// instruction at the end of the given MachineBasicBlock, and sets up the first -/// operand as a destination virtual register. -/// +/// This version of the builder inserts the newly-built instruction at the +/// end of the given MachineBasicBlock, and sets up the first operand as a +/// destination virtual register. inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB, DebugLoc DL, const MCInstrDesc &MCID, @@ -348,11 +339,10 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock *BB, return BuildMI(*BB, BB->end(), DL, MCID, DestReg); } -/// BuildMI - This version of the builder builds a DBG_VALUE intrinsic +/// This version of the builder builds a DBG_VALUE intrinsic /// for either a value in a register or a register-indirect+offset /// address. The convention is that a DBG_VALUE is indirect iff the /// second operand is an immediate. -/// inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID, bool IsIndirect, unsigned Reg, unsigned Offset, @@ -377,10 +367,9 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, } } -/// BuildMI - This version of the builder builds a DBG_VALUE intrinsic +/// This version of the builder builds a DBG_VALUE intrinsic /// for either a value in a register or a register-indirect+offset /// address and inserts it at position I. -/// inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineBasicBlock::iterator I, DebugLoc DL, const MCInstrDesc &MCID, bool IsIndirect, @@ -476,7 +465,7 @@ public: if (I == Begin) { if (!empty()) MI->bundleWithSucc(); - Begin = MI; + Begin = MI->getIterator(); return *this; } if (I == End) { diff --git a/include/llvm/CodeGen/MachineInstrBundle.h b/include/llvm/CodeGen/MachineInstrBundle.h index 122022486345..4fbe206fceb9 100644 --- a/include/llvm/CodeGen/MachineInstrBundle.h +++ b/include/llvm/CodeGen/MachineInstrBundle.h @@ -28,7 +28,7 @@ namespace llvm { void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI); - + /// finalizeBundle - Same functionality as the previous finalizeBundle except /// the last instruction in the bundle is not provided as an input. This is /// used in cases where bundles are pre-determined by marking instructions @@ -44,23 +44,23 @@ bool finalizeBundles(MachineFunction &MF); /// getBundleStart - Returns the first instruction in the bundle containing MI. /// inline MachineInstr *getBundleStart(MachineInstr *MI) { - MachineBasicBlock::instr_iterator I = MI; + MachineBasicBlock::instr_iterator I(MI); while (I->isBundledWithPred()) --I; - return I; + return &*I; } inline const MachineInstr *getBundleStart(const MachineInstr *MI) { - MachineBasicBlock::const_instr_iterator I = MI; + MachineBasicBlock::const_instr_iterator I(MI); while (I->isBundledWithPred()) --I; - return I; + return &*I; } /// Return an iterator pointing beyond the bundle containing MI. inline MachineBasicBlock::instr_iterator getBundleEnd(MachineInstr *MI) { - MachineBasicBlock::instr_iterator I = MI; + MachineBasicBlock::instr_iterator I(MI); while (I->isBundledWithSucc()) ++I; return ++I; @@ -69,7 +69,7 @@ getBundleEnd(MachineInstr *MI) { /// Return an iterator pointing beyond the bundle containing MI. inline MachineBasicBlock::const_instr_iterator getBundleEnd(const MachineInstr *MI) { - MachineBasicBlock::const_instr_iterator I = MI; + MachineBasicBlock::const_instr_iterator I(MI); while (I->isBundledWithSucc()) ++I; return ++I; @@ -116,10 +116,10 @@ protected: /// explicit MachineOperandIteratorBase(MachineInstr *MI, bool WholeBundle) { if (WholeBundle) { - InstrI = getBundleStart(MI); + InstrI = getBundleStart(MI)->getIterator(); InstrE = MI->getParent()->instr_end(); } else { - InstrI = InstrE = MI; + InstrI = InstrE = MI->getIterator(); ++InstrE; } OpI = InstrI->operands_begin(); @@ -164,27 +164,32 @@ public: bool Tied; }; - /// PhysRegInfo - Information about a physical register used by a set of + /// Information about how a physical register Reg is used by a set of /// operands. struct PhysRegInfo { - /// Clobbers - Reg or an overlapping register is defined, or a regmask - /// clobbers Reg. - bool Clobbers; - - /// Defines - Reg or a super-register is defined. - bool Defines; - - /// Reads - Read or a super-register is read. - bool Reads; - - /// ReadsOverlap - Reg or an overlapping register is read. - bool ReadsOverlap; - - /// DefinesDead - All defs of a Reg or a super-register are dead. - bool DefinesDead; - - /// There is a kill of Reg or a super-register. - bool Kills; + /// There is a regmask operand indicating Reg is clobbered. + /// \see MachineOperand::CreateRegMask(). + bool Clobbered; + + /// Reg or one of its aliases is defined. The definition may only cover + /// parts of the register. + bool Defined; + /// Reg or a super-register is defined. The definition covers the full + /// register. + bool FullyDefined; + + /// Reg or ont of its aliases is read. The register may only be read + /// partially. + bool Read; + /// Reg or a super-register is read. The full register is read. + bool FullyRead; + + /// Reg is FullyDefined and all defs of reg or an overlapping register are + /// dead. + bool DeadDef; + + /// There is a use operand of reg or a super-register with kill flag set. + bool Killed; }; /// analyzeVirtReg - Analyze how the current instruction or bundle uses a diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h index a73b92f9a252..1ca0d90465a4 100644 --- a/include/llvm/CodeGen/MachineMemOperand.h +++ b/include/llvm/CodeGen/MachineMemOperand.h @@ -27,6 +27,7 @@ namespace llvm { class FoldingSetNodeID; class MDNode; class raw_ostream; +class MachineFunction; class ModuleSlotTracker; /// MachinePointerInfo - This class contains a discriminated union of @@ -62,22 +63,23 @@ struct MachinePointerInfo { /// getConstantPool - Return a MachinePointerInfo record that refers to the /// constant pool. - static MachinePointerInfo getConstantPool(); + static MachinePointerInfo getConstantPool(MachineFunction &MF); /// getFixedStack - Return a MachinePointerInfo record that refers to the /// the specified FrameIndex. - static MachinePointerInfo getFixedStack(int FI, int64_t offset = 0); + static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, + int64_t Offset = 0); /// getJumpTable - Return a MachinePointerInfo record that refers to a /// jump table entry. - static MachinePointerInfo getJumpTable(); + static MachinePointerInfo getJumpTable(MachineFunction &MF); /// getGOT - Return a MachinePointerInfo record that refers to a /// GOT entry. - static MachinePointerInfo getGOT(); + static MachinePointerInfo getGOT(MachineFunction &MF); /// getStack - stack pointer relative access. - static MachinePointerInfo getStack(int64_t Offset); + static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset); }; diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h index 4cdfe2463c99..77571124a1b8 100644 --- a/include/llvm/CodeGen/MachineModuleInfo.h +++ b/include/llvm/CodeGen/MachineModuleInfo.h @@ -35,11 +35,12 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Analysis/LibCallSemantics.h" +#include "llvm/Analysis/EHPersonalities.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/ValueHandle.h" #include "llvm/MC/MCContext.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/MC/MachineLocation.h" #include "llvm/Pass.h" #include "llvm/Support/DataTypes.h" @@ -59,7 +60,6 @@ class MachineFunction; class Module; class PointerType; class StructType; -struct WinEHFuncInfo; struct SEHHandler { // Filter or finally function. Null indicates a catch-all. @@ -79,13 +79,10 @@ struct LandingPadInfo { SmallVector<MCSymbol *, 1> EndLabels; // Labels after invoke. SmallVector<SEHHandler, 1> SEHHandlers; // SEH handlers active at this lpad. MCSymbol *LandingPadLabel; // Label at beginning of landing pad. - const Function *Personality; // Personality function. std::vector<int> TypeIds; // List of type ids (filters negative). - int WinEHState; // WinEH specific state number. explicit LandingPadInfo(MachineBasicBlock *MBB) - : LandingPadBlock(MBB), LandingPadLabel(nullptr), Personality(nullptr), - WinEHState(-1) {} + : LandingPadBlock(MBB), LandingPadLabel(nullptr) {} }; //===----------------------------------------------------------------------===// @@ -163,6 +160,13 @@ class MachineModuleInfo : public ImmutablePass { bool CallsEHReturn; bool CallsUnwindInit; + bool HasEHFunclets; + + // TODO: Ideally, what we'd like is to have a switch that allows emitting + // synchronous (precise at call-sites only) CFA into .eh_frame. However, + // even under this switch, we'd like .debug_frame to be precise when using. + // -g. At this moment, there's no way to specify that some CFI directives + // go into .eh_frame only, while others go into .debug_frame only. /// DbgInfoAvailable - True if debugging information is available /// in this module. @@ -182,8 +186,6 @@ class MachineModuleInfo : public ImmutablePass { EHPersonality PersonalityTypeCache; - DenseMap<const Function *, std::unique_ptr<WinEHFuncInfo>> FuncInfoMap; - public: static char ID; // Pass identification, replacement for typeid @@ -220,12 +222,6 @@ public: void setModule(const Module *M) { TheModule = M; } const Module *getModule() const { return TheModule; } - const Function *getWinEHParent(const Function *F) const; - WinEHFuncInfo &getWinEHFuncInfo(const Function *F); - bool hasWinEHFuncInfo(const Function *F) const { - return FuncInfoMap.count(getWinEHParent(F)) > 0; - } - /// getInfo - Keep track of various per-function pieces of information for /// backends that would like to do so. /// @@ -252,6 +248,9 @@ public: bool callsUnwindInit() const { return CallsUnwindInit; } void setCallsUnwindInit(bool b) { CallsUnwindInit = b; } + bool hasEHFunclets() const { return HasEHFunclets; } + void setHasEHFunclets(bool V) { HasEHFunclets = V; } + bool usesVAFloatArgument() const { return UsesVAFloatArgument; } @@ -318,16 +317,8 @@ public: /// addPersonality - Provide the personality function for the exception /// information. - void addPersonality(MachineBasicBlock *LandingPad, - const Function *Personality); void addPersonality(const Function *Personality); - void addWinEHState(MachineBasicBlock *LandingPad, int State); - - /// getPersonalityIndex - Get index of the current personality function inside - /// Personalitites array - unsigned getPersonalityIndex() const; - /// getPersonalities - Return array of personality functions ever seen. const std::vector<const Function *>& getPersonalities() const { return Personalities; @@ -426,13 +417,6 @@ public: return FilterIds; } - /// getPersonality - Return a personality function if available. The presence - /// of one is required to emit exception handling info. - const Function *getPersonality() const; - - /// Classify the personality function amongst known EH styles. - EHPersonality getPersonalityType(); - /// setVariableDbgInfo - Collect information used to emit debugging /// information of a variable. void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, diff --git a/include/llvm/CodeGen/MachineModuleInfoImpls.h b/include/llvm/CodeGen/MachineModuleInfoImpls.h index a67f9b5666b1..e7472145e71f 100644 --- a/include/llvm/CodeGen/MachineModuleInfoImpls.h +++ b/include/llvm/CodeGen/MachineModuleInfoImpls.h @@ -18,79 +18,71 @@ #include "llvm/CodeGen/MachineModuleInfo.h" namespace llvm { - class MCSymbol; - - /// MachineModuleInfoMachO - This is a MachineModuleInfoImpl implementation - /// for MachO targets. - class MachineModuleInfoMachO : public MachineModuleInfoImpl { - /// FnStubs - Darwin '$stub' stubs. The key is something like "Lfoo$stub", - /// the value is something like "_foo". - DenseMap<MCSymbol*, StubValueTy> FnStubs; - - /// GVStubs - Darwin '$non_lazy_ptr' stubs. The key is something like - /// "Lfoo$non_lazy_ptr", the value is something like "_foo". The extra bit - /// is true if this GV is external. - DenseMap<MCSymbol*, StubValueTy> GVStubs; - - /// HiddenGVStubs - Darwin '$non_lazy_ptr' stubs. The key is something like - /// "Lfoo$non_lazy_ptr", the value is something like "_foo". Unlike GVStubs - /// these are for things with hidden visibility. The extra bit is true if - /// this GV is external. - DenseMap<MCSymbol*, StubValueTy> HiddenGVStubs; - - virtual void anchor(); // Out of line virtual method. - public: - MachineModuleInfoMachO(const MachineModuleInfo &) {} - - StubValueTy &getFnStubEntry(MCSymbol *Sym) { - assert(Sym && "Key cannot be null"); - return FnStubs[Sym]; - } - - StubValueTy &getGVStubEntry(MCSymbol *Sym) { - assert(Sym && "Key cannot be null"); - return GVStubs[Sym]; - } - - StubValueTy &getHiddenGVStubEntry(MCSymbol *Sym) { - assert(Sym && "Key cannot be null"); - return HiddenGVStubs[Sym]; - } - - /// Accessor methods to return the set of stubs in sorted order. - SymbolListTy GetFnStubList() { - return getSortedStubs(FnStubs); - } - SymbolListTy GetGVStubList() { - return getSortedStubs(GVStubs); - } - SymbolListTy GetHiddenGVStubList() { - return getSortedStubs(HiddenGVStubs); - } - }; - - /// MachineModuleInfoELF - This is a MachineModuleInfoImpl implementation - /// for ELF targets. - class MachineModuleInfoELF : public MachineModuleInfoImpl { - /// GVStubs - These stubs are used to materialize global addresses in PIC - /// mode. - DenseMap<MCSymbol*, StubValueTy> GVStubs; - - virtual void anchor(); // Out of line virtual method. - public: - MachineModuleInfoELF(const MachineModuleInfo &) {} - - StubValueTy &getGVStubEntry(MCSymbol *Sym) { - assert(Sym && "Key cannot be null"); - return GVStubs[Sym]; - } - - /// Accessor methods to return the set of stubs in sorted order. - - SymbolListTy GetGVStubList() { - return getSortedStubs(GVStubs); - } - }; +class MCSymbol; + +/// MachineModuleInfoMachO - This is a MachineModuleInfoImpl implementation +/// for MachO targets. +class MachineModuleInfoMachO : public MachineModuleInfoImpl { + /// FnStubs - Darwin '$stub' stubs. The key is something like "Lfoo$stub", + /// the value is something like "_foo". + DenseMap<MCSymbol *, StubValueTy> FnStubs; + + /// GVStubs - Darwin '$non_lazy_ptr' stubs. The key is something like + /// "Lfoo$non_lazy_ptr", the value is something like "_foo". The extra bit + /// is true if this GV is external. + DenseMap<MCSymbol *, StubValueTy> GVStubs; + + /// HiddenGVStubs - Darwin '$non_lazy_ptr' stubs. The key is something like + /// "Lfoo$non_lazy_ptr", the value is something like "_foo". Unlike GVStubs + /// these are for things with hidden visibility. The extra bit is true if + /// this GV is external. + DenseMap<MCSymbol *, StubValueTy> HiddenGVStubs; + + virtual void anchor(); // Out of line virtual method. +public: + MachineModuleInfoMachO(const MachineModuleInfo &) {} + + StubValueTy &getFnStubEntry(MCSymbol *Sym) { + assert(Sym && "Key cannot be null"); + return FnStubs[Sym]; + } + + StubValueTy &getGVStubEntry(MCSymbol *Sym) { + assert(Sym && "Key cannot be null"); + return GVStubs[Sym]; + } + + StubValueTy &getHiddenGVStubEntry(MCSymbol *Sym) { + assert(Sym && "Key cannot be null"); + return HiddenGVStubs[Sym]; + } + + /// Accessor methods to return the set of stubs in sorted order. + SymbolListTy GetFnStubList() { return getSortedStubs(FnStubs); } + SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); } + SymbolListTy GetHiddenGVStubList() { return getSortedStubs(HiddenGVStubs); } +}; + +/// MachineModuleInfoELF - This is a MachineModuleInfoImpl implementation +/// for ELF targets. +class MachineModuleInfoELF : public MachineModuleInfoImpl { + /// GVStubs - These stubs are used to materialize global addresses in PIC + /// mode. + DenseMap<MCSymbol *, StubValueTy> GVStubs; + + virtual void anchor(); // Out of line virtual method. +public: + MachineModuleInfoELF(const MachineModuleInfo &) {} + + StubValueTy &getGVStubEntry(MCSymbol *Sym) { + assert(Sym && "Key cannot be null"); + return GVStubs[Sym]; + } + + /// Accessor methods to return the set of stubs in sorted order. + + SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); } +}; } // end namespace llvm diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h index 5e607cdae48e..04191bc1b74f 100644 --- a/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/include/llvm/CodeGen/MachineRegisterInfo.h @@ -73,7 +73,7 @@ private: /// PhysRegUseDefLists - This is an array of the head of the use/def list for /// physical registers. - std::vector<MachineOperand *> PhysRegUseDefLists; + std::unique_ptr<MachineOperand *[]> PhysRegUseDefLists; /// getRegUseDefListHead - Return the head pointer for the register use/def /// list for the specified virtual or physical register. @@ -95,20 +95,8 @@ private: return MO->Contents.Reg.Next; } - /// UsedRegUnits - This is a bit vector that is computed and set by the - /// register allocator, and must be kept up to date by passes that run after - /// register allocation (though most don't modify this). This is used - /// so that the code generator knows which callee save registers to save and - /// for other target specific uses. - /// This vector has bits set for register units that are modified in the - /// current function. It doesn't include registers clobbered by function - /// calls with register mask operands. - BitVector UsedRegUnits; - /// UsedPhysRegMask - Additional used physregs including aliases. /// This bit vector represents all the registers clobbered by function calls. - /// It can model things that UsedRegUnits can't, such as function calls that - /// clobber ymm7 but preserve the low half in xmm7. BitVector UsedPhysRegMask; /// ReservedRegs - This is a bit vector of reserved registers. The target @@ -246,7 +234,7 @@ public: static reg_iterator reg_end() { return reg_iterator(nullptr); } inline iterator_range<reg_iterator> reg_operands(unsigned Reg) const { - return iterator_range<reg_iterator>(reg_begin(Reg), reg_end()); + return make_range(reg_begin(Reg), reg_end()); } /// reg_instr_iterator/reg_instr_begin/reg_instr_end - Walk all defs and uses @@ -262,8 +250,7 @@ public: inline iterator_range<reg_instr_iterator> reg_instructions(unsigned Reg) const { - return iterator_range<reg_instr_iterator>(reg_instr_begin(Reg), - reg_instr_end()); + return make_range(reg_instr_begin(Reg), reg_instr_end()); } /// reg_bundle_iterator/reg_bundle_begin/reg_bundle_end - Walk all defs and uses @@ -278,8 +265,7 @@ public: } inline iterator_range<reg_bundle_iterator> reg_bundles(unsigned Reg) const { - return iterator_range<reg_bundle_iterator>(reg_bundle_begin(Reg), - reg_bundle_end()); + return make_range(reg_bundle_begin(Reg), reg_bundle_end()); } /// reg_empty - Return true if there are no instructions using or defining the @@ -299,8 +285,7 @@ public: inline iterator_range<reg_nodbg_iterator> reg_nodbg_operands(unsigned Reg) const { - return iterator_range<reg_nodbg_iterator>(reg_nodbg_begin(Reg), - reg_nodbg_end()); + return make_range(reg_nodbg_begin(Reg), reg_nodbg_end()); } /// reg_instr_nodbg_iterator/reg_instr_nodbg_begin/reg_instr_nodbg_end - Walk @@ -317,8 +302,7 @@ public: inline iterator_range<reg_instr_nodbg_iterator> reg_nodbg_instructions(unsigned Reg) const { - return iterator_range<reg_instr_nodbg_iterator>(reg_instr_nodbg_begin(Reg), - reg_instr_nodbg_end()); + return make_range(reg_instr_nodbg_begin(Reg), reg_instr_nodbg_end()); } /// reg_bundle_nodbg_iterator/reg_bundle_nodbg_begin/reg_bundle_nodbg_end - Walk @@ -333,10 +317,9 @@ public: return reg_bundle_nodbg_iterator(nullptr); } - inline iterator_range<reg_bundle_nodbg_iterator> + inline iterator_range<reg_bundle_nodbg_iterator> reg_nodbg_bundles(unsigned Reg) const { - return iterator_range<reg_bundle_nodbg_iterator>(reg_bundle_nodbg_begin(Reg), - reg_bundle_nodbg_end()); + return make_range(reg_bundle_nodbg_begin(Reg), reg_bundle_nodbg_end()); } /// reg_nodbg_empty - Return true if the only instructions using or defining @@ -354,7 +337,7 @@ public: static def_iterator def_end() { return def_iterator(nullptr); } inline iterator_range<def_iterator> def_operands(unsigned Reg) const { - return iterator_range<def_iterator>(def_begin(Reg), def_end()); + return make_range(def_begin(Reg), def_end()); } /// def_instr_iterator/def_instr_begin/def_instr_end - Walk all defs of the @@ -370,8 +353,7 @@ public: inline iterator_range<def_instr_iterator> def_instructions(unsigned Reg) const { - return iterator_range<def_instr_iterator>(def_instr_begin(Reg), - def_instr_end()); + return make_range(def_instr_begin(Reg), def_instr_end()); } /// def_bundle_iterator/def_bundle_begin/def_bundle_end - Walk all defs of the @@ -386,8 +368,7 @@ public: } inline iterator_range<def_bundle_iterator> def_bundles(unsigned Reg) const { - return iterator_range<def_bundle_iterator>(def_bundle_begin(Reg), - def_bundle_end()); + return make_range(def_bundle_begin(Reg), def_bundle_end()); } /// def_empty - Return true if there are no instructions defining the @@ -412,7 +393,7 @@ public: static use_iterator use_end() { return use_iterator(nullptr); } inline iterator_range<use_iterator> use_operands(unsigned Reg) const { - return iterator_range<use_iterator>(use_begin(Reg), use_end()); + return make_range(use_begin(Reg), use_end()); } /// use_instr_iterator/use_instr_begin/use_instr_end - Walk all uses of the @@ -428,8 +409,7 @@ public: inline iterator_range<use_instr_iterator> use_instructions(unsigned Reg) const { - return iterator_range<use_instr_iterator>(use_instr_begin(Reg), - use_instr_end()); + return make_range(use_instr_begin(Reg), use_instr_end()); } /// use_bundle_iterator/use_bundle_begin/use_bundle_end - Walk all uses of the @@ -444,8 +424,7 @@ public: } inline iterator_range<use_bundle_iterator> use_bundles(unsigned Reg) const { - return iterator_range<use_bundle_iterator>(use_bundle_begin(Reg), - use_bundle_end()); + return make_range(use_bundle_begin(Reg), use_bundle_end()); } /// use_empty - Return true if there are no instructions using the specified @@ -474,8 +453,7 @@ public: inline iterator_range<use_nodbg_iterator> use_nodbg_operands(unsigned Reg) const { - return iterator_range<use_nodbg_iterator>(use_nodbg_begin(Reg), - use_nodbg_end()); + return make_range(use_nodbg_begin(Reg), use_nodbg_end()); } /// use_instr_nodbg_iterator/use_instr_nodbg_begin/use_instr_nodbg_end - Walk @@ -492,8 +470,7 @@ public: inline iterator_range<use_instr_nodbg_iterator> use_nodbg_instructions(unsigned Reg) const { - return iterator_range<use_instr_nodbg_iterator>(use_instr_nodbg_begin(Reg), - use_instr_nodbg_end()); + return make_range(use_instr_nodbg_begin(Reg), use_instr_nodbg_end()); } /// use_bundle_nodbg_iterator/use_bundle_nodbg_begin/use_bundle_nodbg_end - Walk @@ -510,8 +487,7 @@ public: inline iterator_range<use_bundle_nodbg_iterator> use_nodbg_bundles(unsigned Reg) const { - return iterator_range<use_bundle_nodbg_iterator>(use_bundle_nodbg_begin(Reg), - use_bundle_nodbg_end()); + return make_range(use_bundle_nodbg_begin(Reg), use_bundle_nodbg_end()); } /// use_nodbg_empty - Return true if there are no non-Debug instructions @@ -540,7 +516,7 @@ public: /// apply sub registers to ToReg in order to obtain a final/proper physical /// register. void replaceRegWith(unsigned FromReg, unsigned ToReg); - + /// getVRegDef - Return the machine instr that defines the specified virtual /// register or null if none is found. This assumes that the code is in SSA /// form, so there should only be one definition. @@ -626,6 +602,12 @@ public: RegAllocHints[VReg].second = PrefReg; } + /// Specify the preferred register allocation hint for the specified virtual + /// register. + void setSimpleHint(unsigned VReg, unsigned PrefReg) { + setRegAllocationHint(VReg, /*Type=*/0, PrefReg); + } + /// getRegAllocationHint - Return the register allocation hint for the /// specified virtual register. std::pair<unsigned, unsigned> @@ -650,41 +632,15 @@ public: /// Return true if the specified register is modified in this function. /// This checks that no defining machine operands exist for the register or /// any of its aliases. Definitions found on functions marked noreturn are - /// ignored. + /// ignored. The register is also considered modified when it is set in the + /// UsedPhysRegMask. bool isPhysRegModified(unsigned PhysReg) const; - //===--------------------------------------------------------------------===// - // Physical Register Use Info - //===--------------------------------------------------------------------===// - - /// isPhysRegUsed - Return true if the specified register is used in this - /// function. Also check for clobbered aliases and registers clobbered by - /// function calls with register mask operands. - /// - /// This only works after register allocation. - bool isPhysRegUsed(unsigned Reg) const { - if (UsedPhysRegMask.test(Reg)) - return true; - for (MCRegUnitIterator Units(Reg, getTargetRegisterInfo()); - Units.isValid(); ++Units) - if (UsedRegUnits.test(*Units)) - return true; - return false; - } - - /// Mark the specified register unit as used in this function. - /// This should only be called during and after register allocation. - void setRegUnitUsed(unsigned RegUnit) { - UsedRegUnits.set(RegUnit); - } - - /// setPhysRegUsed - Mark the specified register used in this function. - /// This should only be called during and after register allocation. - void setPhysRegUsed(unsigned Reg) { - for (MCRegUnitIterator Units(Reg, getTargetRegisterInfo()); - Units.isValid(); ++Units) - UsedRegUnits.set(*Units); - } + /// Return true if the specified register is modified or read in this + /// function. This checks that no machine operands exist for the register or + /// any of its aliases. The register is also considered used when it is set + /// in the UsedPhysRegMask. + bool isPhysRegUsed(unsigned PhysReg) const; /// addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used. /// This corresponds to the bit mask attached to register mask operands. @@ -692,15 +648,9 @@ public: UsedPhysRegMask.setBitsNotInMask(RegMask); } - /// setPhysRegUnused - Mark the specified register unused in this function. - /// This should only be called during and after register allocation. - void setPhysRegUnused(unsigned Reg) { - UsedPhysRegMask.reset(Reg); - for (MCRegUnitIterator Units(Reg, getTargetRegisterInfo()); - Units.isValid(); ++Units) - UsedRegUnits.reset(*Units); - } + const BitVector &getUsedPhysRegsMask() const { return UsedPhysRegMask; } + void setUsedPhysRegMask(BitVector &Mask) { UsedPhysRegMask = Mask; } //===--------------------------------------------------------------------===// // Reserved Register Info @@ -797,7 +747,7 @@ public: /// Returns a mask covering all bits that can appear in lane masks of /// subregisters of the virtual register @p Reg. - unsigned getMaxLaneMaskForVReg(unsigned Reg) const; + LaneBitmask getMaxLaneMaskForVReg(unsigned Reg) const; /// defusechain_iterator - This class provides iterator support for machine /// operands in the function that use or define a specific register. If diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index e80e14e5ccf7..358fd5a3732a 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -77,6 +77,7 @@ #ifndef LLVM_CODEGEN_MACHINESCHEDULER_H #define LLVM_CODEGEN_MACHINESCHEDULER_H +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachinePassRegistry.h" #include "llvm/CodeGen/RegisterPressure.h" #include "llvm/CodeGen/ScheduleDAGInstrs.h" @@ -87,7 +88,6 @@ namespace llvm { extern cl::opt<bool> ForceTopDown; extern cl::opt<bool> ForceBottomUp; -class AliasAnalysis; class LiveIntervals; class MachineDominatorTree; class MachineLoopInfo; @@ -156,8 +156,12 @@ struct MachineSchedPolicy { bool OnlyTopDown; bool OnlyBottomUp; + // Disable heuristic that tries to fetch nodes from long dependency chains + // first. + bool DisableLatencyHeuristic; + MachineSchedPolicy(): ShouldTrackPressure(false), OnlyTopDown(false), - OnlyBottomUp(false) {} + OnlyBottomUp(false), DisableLatencyHeuristic(false) {} }; /// MachineSchedStrategy - Interface to the scheduling algorithm used by @@ -175,6 +179,8 @@ public: MachineBasicBlock::iterator End, unsigned NumRegionInstrs) {} + virtual void dumpPolicy() {} + /// Check if pressure tracking is needed before building the DAG and /// initializing this strategy. Called after initPolicy. virtual bool shouldTrackPressure() const { return true; } @@ -222,6 +228,7 @@ public: class ScheduleDAGMI : public ScheduleDAGInstrs { protected: AliasAnalysis *AA; + LiveIntervals *LIS; std::unique_ptr<MachineSchedStrategy> SchedImpl; /// Topo - A topological ordering for SUnits which permits fast IsReachable @@ -248,11 +255,11 @@ protected: #endif public: ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S, - bool IsPostRA) - : ScheduleDAGInstrs(*C->MF, C->MLI, IsPostRA, - /*RemoveKillFlags=*/IsPostRA, C->LIS), - AA(C->AA), SchedImpl(std::move(S)), Topo(SUnits, &ExitSU), CurrentTop(), - CurrentBottom(), NextClusterPred(nullptr), NextClusterSucc(nullptr) { + bool RemoveKillFlags) + : ScheduleDAGInstrs(*C->MF, C->MLI, RemoveKillFlags), AA(C->AA), + LIS(C->LIS), SchedImpl(std::move(S)), Topo(SUnits, &ExitSU), + CurrentTop(), CurrentBottom(), NextClusterPred(nullptr), + NextClusterSucc(nullptr) { #ifndef NDEBUG NumInstrsScheduled = 0; #endif @@ -261,6 +268,9 @@ public: // Provide a vtable anchor ~ScheduleDAGMI() override; + // Returns LiveIntervals instance for use in DAG mutators and such. + LiveIntervals *getLIS() const { return LIS; } + /// Return true if this DAG supports VReg liveness and RegPressure. virtual bool hasVRegLiveness() const { return false; } @@ -380,7 +390,7 @@ protected: public: ScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S) - : ScheduleDAGMI(C, std::move(S), /*IsPostRA=*/false), + : ScheduleDAGMI(C, std::move(S), /*RemoveKillFlags=*/false), RegClassInfo(C->RegClassInfo), DFSResult(nullptr), ShouldTrackPressure(false), RPTracker(RegPressure), TopRPTracker(TopPressure), BotRPTracker(BotPressure) {} @@ -858,6 +868,8 @@ public: MachineBasicBlock::iterator End, unsigned NumRegionInstrs) override; + void dumpPolicy() override; + bool shouldTrackPressure() const override { return RegionPolicy.ShouldTrackPressure; } @@ -915,7 +927,7 @@ public: MachineBasicBlock::iterator End, unsigned NumRegionInstrs) override { /* no configurable policy */ - }; + } /// PostRA scheduling does not track pressure. bool shouldTrackPressure() const override { return false; } diff --git a/include/llvm/CodeGen/MachineValueType.h b/include/llvm/CodeGen/MachineValueType.h index a728df354677..04d6ee3be531 100644 --- a/include/llvm/CodeGen/MachineValueType.h +++ b/include/llvm/CodeGen/MachineValueType.h @@ -56,53 +56,66 @@ class MVT { FIRST_FP_VALUETYPE = f16, LAST_FP_VALUETYPE = ppcf128, - v2i1 = 13, // 2 x i1 - v4i1 = 14, // 4 x i1 - v8i1 = 15, // 8 x i1 - v16i1 = 16, // 16 x i1 - v32i1 = 17, // 32 x i1 - v64i1 = 18, // 64 x i1 - - v1i8 = 19, // 1 x i8 - v2i8 = 20, // 2 x i8 - v4i8 = 21, // 4 x i8 - v8i8 = 22, // 8 x i8 - v16i8 = 23, // 16 x i8 - v32i8 = 24, // 32 x i8 - v64i8 = 25, // 64 x i8 - v1i16 = 26, // 1 x i16 - v2i16 = 27, // 2 x i16 - v4i16 = 28, // 4 x i16 - v8i16 = 29, // 8 x i16 - v16i16 = 30, // 16 x i16 - v32i16 = 31, // 32 x i16 - v1i32 = 32, // 1 x i32 - v2i32 = 33, // 2 x i32 - v4i32 = 34, // 4 x i32 - v8i32 = 35, // 8 x i32 - v16i32 = 36, // 16 x i32 - v1i64 = 37, // 1 x i64 - v2i64 = 38, // 2 x i64 - v4i64 = 39, // 4 x i64 - v8i64 = 40, // 8 x i64 - v16i64 = 41, // 16 x i64 - v1i128 = 42, // 1 x i128 - + v2i1 = 13, // 2 x i1 + v4i1 = 14, // 4 x i1 + v8i1 = 15, // 8 x i1 + v16i1 = 16, // 16 x i1 + v32i1 = 17, // 32 x i1 + v64i1 = 18, // 64 x i1 + v512i1 = 19, // 512 x i1 + v1024i1 = 20, // 1024 x i1 + + v1i8 = 21, // 1 x i8 + v2i8 = 22, // 2 x i8 + v4i8 = 23, // 4 x i8 + v8i8 = 24, // 8 x i8 + v16i8 = 25, // 16 x i8 + v32i8 = 26, // 32 x i8 + v64i8 = 27, // 64 x i8 + v128i8 = 28, //128 x i8 + v256i8 = 29, //256 x i8 + + v1i16 = 30, // 1 x i16 + v2i16 = 31, // 2 x i16 + v4i16 = 32, // 4 x i16 + v8i16 = 33, // 8 x i16 + v16i16 = 34, // 16 x i16 + v32i16 = 35, // 32 x i16 + v64i16 = 36, // 64 x i16 + v128i16 = 37, //128 x i16 + + v1i32 = 38, // 1 x i32 + v2i32 = 39, // 2 x i32 + v4i32 = 40, // 4 x i32 + v8i32 = 41, // 8 x i32 + v16i32 = 42, // 16 x i32 + v32i32 = 43, // 32 x i32 + v64i32 = 44, // 64 x i32 + + v1i64 = 45, // 1 x i64 + v2i64 = 46, // 2 x i64 + v4i64 = 47, // 4 x i64 + v8i64 = 48, // 8 x i64 + v16i64 = 49, // 16 x i64 + v32i64 = 50, // 32 x i64 + + v1i128 = 51, // 1 x i128 + FIRST_INTEGER_VECTOR_VALUETYPE = v2i1, LAST_INTEGER_VECTOR_VALUETYPE = v1i128, - v2f16 = 43, // 2 x f16 - v4f16 = 44, // 4 x f16 - v8f16 = 45, // 8 x f16 - v1f32 = 46, // 1 x f32 - v2f32 = 47, // 2 x f32 - v4f32 = 48, // 4 x f32 - v8f32 = 49, // 8 x f32 - v16f32 = 50, // 16 x f32 - v1f64 = 51, // 1 x f64 - v2f64 = 52, // 2 x f64 - v4f64 = 53, // 4 x f64 - v8f64 = 54, // 8 x f64 + v2f16 = 52, // 2 x f16 + v4f16 = 53, // 4 x f16 + v8f16 = 54, // 8 x f16 + v1f32 = 55, // 1 x f32 + v2f32 = 56, // 2 x f32 + v4f32 = 57, // 4 x f32 + v8f32 = 58, // 8 x f32 + v16f32 = 59, // 16 x f32 + v1f64 = 60, // 1 x f64 + v2f64 = 61, // 2 x f64 + v4f64 = 62, // 4 x f64 + v8f64 = 63, // 8 x f64 FIRST_FP_VECTOR_VALUETYPE = v2f16, LAST_FP_VECTOR_VALUETYPE = v8f64, @@ -110,23 +123,26 @@ class MVT { FIRST_VECTOR_VALUETYPE = v2i1, LAST_VECTOR_VALUETYPE = v8f64, - x86mmx = 55, // This is an X86 MMX value + x86mmx = 64, // This is an X86 MMX value - Glue = 56, // This glues nodes together during pre-RA sched + Glue = 65, // This glues nodes together during pre-RA sched - isVoid = 57, // This has no value + isVoid = 66, // This has no value - Untyped = 58, // This value takes a register, but has + Untyped = 67, // This value takes a register, but has // unspecified type. The register class // will be determined by the opcode. FIRST_VALUETYPE = 0, // This is always the beginning of the list. - LAST_VALUETYPE = 59, // This always remains at the end of the list. + LAST_VALUETYPE = 68, // This always remains at the end of the list. // This is the current maximum for LAST_VALUETYPE. // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors // This value must be a multiple of 32. - MAX_ALLOWED_VALUETYPE = 64, + MAX_ALLOWED_VALUETYPE = 96, + + // Token - A value of type llvm::TokenTy + token = 249, // Metadata - This is MDNode or MDString. Metadata = 250, @@ -238,14 +254,23 @@ class MVT { /// is512BitVector - Return true if this is a 512-bit vector type. bool is512BitVector() const { - return (SimpleTy == MVT::v8f64 || SimpleTy == MVT::v16f32 || - SimpleTy == MVT::v64i8 || SimpleTy == MVT::v32i16 || - SimpleTy == MVT::v8i64 || SimpleTy == MVT::v16i32); + return (SimpleTy == MVT::v16f32 || SimpleTy == MVT::v8f64 || + SimpleTy == MVT::v512i1 || SimpleTy == MVT::v64i8 || + SimpleTy == MVT::v32i16 || SimpleTy == MVT::v16i32 || + SimpleTy == MVT::v8i64); } /// is1024BitVector - Return true if this is a 1024-bit vector type. bool is1024BitVector() const { - return (SimpleTy == MVT::v16i64); + return (SimpleTy == MVT::v1024i1 || SimpleTy == MVT::v128i8 || + SimpleTy == MVT::v64i16 || SimpleTy == MVT::v32i32 || + SimpleTy == MVT::v16i64); + } + + /// is2048BitVector - Return true if this is a 1024-bit vector type. + bool is2048BitVector() const { + return (SimpleTy == MVT::v256i8 || SimpleTy == MVT::v128i16 || + SimpleTy == MVT::v64i32 || SimpleTy == MVT::v32i64); } /// isOverloaded - Return true if this is an overloaded type for TableGen. @@ -282,35 +307,44 @@ class MVT { switch (SimpleTy) { default: llvm_unreachable("Not a vector MVT!"); - case v2i1 : - case v4i1 : - case v8i1 : - case v16i1 : - case v32i1 : - case v64i1: return i1; - case v1i8 : - case v2i8 : - case v4i8 : - case v8i8 : + case v2i1: + case v4i1: + case v8i1: + case v16i1: + case v32i1: + case v64i1: + case v512i1: + case v1024i1: return i1; + case v1i8: + case v2i8: + case v4i8: + case v8i8: case v16i8: case v32i8: - case v64i8: return i8; + case v64i8: + case v128i8: + case v256i8: return i8; case v1i16: case v2i16: case v4i16: case v8i16: case v16i16: - case v32i16: return i16; + case v32i16: + case v64i16: + case v128i16: return i16; case v1i32: case v2i32: case v4i32: case v8i32: - case v16i32: return i32; + case v16i32: + case v32i32: + case v64i32: return i32; case v1i64: case v2i64: case v4i64: case v8i64: - case v16i64: return i64; + case v16i64: + case v32i64: return i64; case v1i128: return i128; case v2f16: case v4f16: @@ -331,19 +365,28 @@ class MVT { switch (SimpleTy) { default: llvm_unreachable("Not a vector MVT!"); + case v1024i1: return 1024; + case v512i1: return 512; + case v256i8: return 256; + case v128i8: + case v128i16: return 128; + case v64i1: + case v64i8: + case v64i16: + case v64i32: return 64; case v32i1: case v32i8: - case v32i16: return 32; - case v64i1: - case v64i8: return 64; + case v32i16: + case v32i32: + case v32i64: return 32; case v16i1: case v16i8: case v16i16: case v16i32: case v16i64: case v16f32: return 16; - case v8i1 : - case v8i8 : + case v8i1: + case v8i8: case v8i16: case v8i32: case v8i64: @@ -390,6 +433,9 @@ class MVT { case vAny: case Any: llvm_unreachable("Value type is overloaded."); + case token: + llvm_unreachable("Token type is a sentinel that cannot be used " + "in codegen and has no size"); case Metadata: llvm_unreachable("Value type is metadata."); case i1 : return 1; @@ -440,13 +486,22 @@ class MVT { case v4i64: case v8f32: case v4f64: return 256; + case v512i1: case v64i8: case v32i16: case v16i32: case v8i64: case v16f32: case v8f64: return 512; - case v16i64:return 1024; + case v1024i1: + case v128i8: + case v64i16: + case v32i32: + case v16i64: return 1024; + case v256i8: + case v128i16: + case v64i32: + case v32i64: return 2048; } } @@ -528,29 +583,35 @@ class MVT { default: break; case MVT::i1: - if (NumElements == 2) return MVT::v2i1; - if (NumElements == 4) return MVT::v4i1; - if (NumElements == 8) return MVT::v8i1; - if (NumElements == 16) return MVT::v16i1; - if (NumElements == 32) return MVT::v32i1; - if (NumElements == 64) return MVT::v64i1; + if (NumElements == 2) return MVT::v2i1; + if (NumElements == 4) return MVT::v4i1; + if (NumElements == 8) return MVT::v8i1; + if (NumElements == 16) return MVT::v16i1; + if (NumElements == 32) return MVT::v32i1; + if (NumElements == 64) return MVT::v64i1; + if (NumElements == 512) return MVT::v512i1; + if (NumElements == 1024) return MVT::v1024i1; break; case MVT::i8: - if (NumElements == 1) return MVT::v1i8; - if (NumElements == 2) return MVT::v2i8; - if (NumElements == 4) return MVT::v4i8; - if (NumElements == 8) return MVT::v8i8; - if (NumElements == 16) return MVT::v16i8; - if (NumElements == 32) return MVT::v32i8; - if (NumElements == 64) return MVT::v64i8; + if (NumElements == 1) return MVT::v1i8; + if (NumElements == 2) return MVT::v2i8; + if (NumElements == 4) return MVT::v4i8; + if (NumElements == 8) return MVT::v8i8; + if (NumElements == 16) return MVT::v16i8; + if (NumElements == 32) return MVT::v32i8; + if (NumElements == 64) return MVT::v64i8; + if (NumElements == 128) return MVT::v128i8; + if (NumElements == 256) return MVT::v256i8; break; case MVT::i16: - if (NumElements == 1) return MVT::v1i16; - if (NumElements == 2) return MVT::v2i16; - if (NumElements == 4) return MVT::v4i16; - if (NumElements == 8) return MVT::v8i16; - if (NumElements == 16) return MVT::v16i16; - if (NumElements == 32) return MVT::v32i16; + if (NumElements == 1) return MVT::v1i16; + if (NumElements == 2) return MVT::v2i16; + if (NumElements == 4) return MVT::v4i16; + if (NumElements == 8) return MVT::v8i16; + if (NumElements == 16) return MVT::v16i16; + if (NumElements == 32) return MVT::v32i16; + if (NumElements == 64) return MVT::v64i16; + if (NumElements == 128) return MVT::v128i16; break; case MVT::i32: if (NumElements == 1) return MVT::v1i32; @@ -558,6 +619,8 @@ class MVT { if (NumElements == 4) return MVT::v4i32; if (NumElements == 8) return MVT::v8i32; if (NumElements == 16) return MVT::v16i32; + if (NumElements == 32) return MVT::v32i32; + if (NumElements == 64) return MVT::v64i32; break; case MVT::i64: if (NumElements == 1) return MVT::v1i64; @@ -565,6 +628,7 @@ class MVT { if (NumElements == 4) return MVT::v4i64; if (NumElements == 8) return MVT::v8i64; if (NumElements == 16) return MVT::v16i64; + if (NumElements == 32) return MVT::v32i64; break; case MVT::i128: if (NumElements == 1) return MVT::v1i128; diff --git a/include/llvm/CodeGen/ParallelCG.h b/include/llvm/CodeGen/ParallelCG.h new file mode 100644 index 000000000000..fa7002fa21fb --- /dev/null +++ b/include/llvm/CodeGen/ParallelCG.h @@ -0,0 +1,43 @@ +//===-- llvm/CodeGen/ParallelCG.h - Parallel code generation ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This header declares functions that can be used for parallel code generation. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_PARALLELCG_H +#define LLVM_CODEGEN_PARALLELCG_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/CodeGen.h" +#include "llvm/Target/TargetMachine.h" + +namespace llvm { + +class Module; +class TargetOptions; +class raw_pwrite_stream; + +/// Split M into OSs.size() partitions, and generate code for each. Writes +/// OSs.size() output files to the output streams in OSs. The resulting output +/// files if linked together are intended to be equivalent to the single output +/// file that would have been code generated from M. +/// +/// \returns M if OSs.size() == 1, otherwise returns std::unique_ptr<Module>(). +std::unique_ptr<Module> +splitCodeGen(std::unique_ptr<Module> M, ArrayRef<raw_pwrite_stream *> OSs, + StringRef CPU, StringRef Features, const TargetOptions &Options, + Reloc::Model RM = Reloc::Default, + CodeModel::Model CM = CodeModel::Default, + CodeGenOpt::Level OL = CodeGenOpt::Default, + TargetMachine::CodeGenFileType FT = TargetMachine::CGFT_ObjectFile); + +} // namespace llvm + +#endif diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index 5d8292174476..f45f0ed57d6b 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -120,9 +120,6 @@ protected: /// Default setting for -enable-tail-merge on this target. bool EnableTailMerge; - /// Default setting for -enable-shrink-wrap on this target. - bool EnableShrinkWrap; - public: TargetPassConfig(TargetMachine *tm, PassManagerBase &pm); // Dummy constructor. @@ -173,7 +170,8 @@ public: void substitutePass(AnalysisID StandardID, IdentifyingPassPtr TargetID); /// Insert InsertedPassID pass after TargetPassID pass. - void insertPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID); + void insertPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID, + bool VerifyAfter = true, bool PrintAfter = true); /// Allow the target to enable a specific standard pass by default. void enablePass(AnalysisID PassID) { substitutePass(PassID, PassID); } @@ -228,7 +226,7 @@ public: /// /// This can also be used to plug a new MachineSchedStrategy into an instance /// of the standard ScheduleDAGMI: - /// return new ScheduleDAGMI(C, make_unique<MyStrategy>(C), /* IsPostRA= */false) + /// return new ScheduleDAGMI(C, make_unique<MyStrategy>(C), /*RemoveKillFlags=*/false) /// /// Return NULL to select the default (generic) machine scheduler. virtual ScheduleDAGInstrs * @@ -585,6 +583,9 @@ namespace llvm { /// StackSlotColoring - This pass performs stack slot coloring. extern char &StackSlotColoringID; + /// \brief This pass lays out funclets contiguously. + extern char &FuncletLayoutID; + /// createStackProtectorPass - This pass adds stack protectors to functions. /// FunctionPass *createStackProtectorPass(const TargetMachine *TM); @@ -639,6 +640,9 @@ namespace llvm { /// the intrinsic for later emission to the StackMap. extern char &StackMapLivenessID; + /// LiveDebugValues pass + extern char &LiveDebugValuesID; + /// createJumpInstrTables - This pass creates jump-instruction tables. ModulePass *createJumpInstrTablesPass(); diff --git a/include/llvm/CodeGen/PseudoSourceValue.h b/include/llvm/CodeGen/PseudoSourceValue.h index a518b6233250..f67552030db4 100644 --- a/include/llvm/CodeGen/PseudoSourceValue.h +++ b/include/llvm/CodeGen/PseudoSourceValue.h @@ -14,97 +14,170 @@ #ifndef LLVM_CODEGEN_PSEUDOSOURCEVALUE_H #define LLVM_CODEGEN_PSEUDOSOURCEVALUE_H +#include "llvm/ADT/StringMap.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/Value.h" +#include "llvm/IR/ValueMap.h" +#include <map> namespace llvm { - class MachineFrameInfo; - class MachineMemOperand; - class raw_ostream; - - raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MMO); - - /// PseudoSourceValue - Special value supplied for machine level alias - /// analysis. It indicates that a memory access references the functions - /// stack frame (e.g., a spill slot), below the stack frame (e.g., argument - /// space), or constant pool. - class PseudoSourceValue { - private: - friend class MachineMemOperand; // For printCustom(). - - /// printCustom - Implement printing for PseudoSourceValue. This is called - /// from Value::print or Value's operator<<. - /// - virtual void printCustom(raw_ostream &O) const; - - public: - /// isFixed - Whether this is a FixedStackPseudoSourceValue. - bool isFixed; - - explicit PseudoSourceValue(bool isFixed = false); - - virtual ~PseudoSourceValue(); - - /// isConstant - Test whether the memory pointed to by this - /// PseudoSourceValue has a constant value. - /// - virtual bool isConstant(const MachineFrameInfo *) const; - - /// isAliased - Test whether the memory pointed to by this - /// PseudoSourceValue may also be pointed to by an LLVM IR Value. - virtual bool isAliased(const MachineFrameInfo *) const; - - /// mayAlias - Return true if the memory pointed to by this - /// PseudoSourceValue can ever alias an LLVM IR Value. - virtual bool mayAlias(const MachineFrameInfo *) const; - - /// A pseudo source value referencing a fixed stack frame entry, - /// e.g., a spill slot. - static const PseudoSourceValue *getFixedStack(int FI); - - /// A pseudo source value referencing the area below the stack frame of - /// a function, e.g., the argument space. - static const PseudoSourceValue *getStack(); - - /// A pseudo source value referencing the global offset table - /// (or something the like). - static const PseudoSourceValue *getGOT(); - - /// A pseudo source value referencing the constant pool. Since constant - /// pools are constant, this doesn't need to identify a specific constant - /// pool entry. - static const PseudoSourceValue *getConstantPool(); - - /// A pseudo source value referencing a jump table. Since jump tables are - /// constant, this doesn't need to identify a specific jump table. - static const PseudoSourceValue *getJumpTable(); + +class MachineFrameInfo; +class MachineMemOperand; +class raw_ostream; + +raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MMO); + +/// Special value supplied for machine level alias analysis. It indicates that +/// a memory access references the functions stack frame (e.g., a spill slot), +/// below the stack frame (e.g., argument space), or constant pool. +class PseudoSourceValue { +public: + enum PSVKind { + Stack, + GOT, + JumpTable, + ConstantPool, + FixedStack, + GlobalValueCallEntry, + ExternalSymbolCallEntry }; - /// FixedStackPseudoSourceValue - A specialized PseudoSourceValue - /// for holding FixedStack values, which must include a frame - /// index. - class FixedStackPseudoSourceValue : public PseudoSourceValue { - const int FI; - public: - explicit FixedStackPseudoSourceValue(int fi) : - PseudoSourceValue(true), FI(fi) {} +private: + PSVKind Kind; - /// classof - Methods for support type inquiry through isa, cast, and - /// dyn_cast: - /// - static inline bool classof(const PseudoSourceValue *V) { - return V->isFixed == true; - } + friend class MachineMemOperand; // For printCustom(). - bool isConstant(const MachineFrameInfo *MFI) const override; + /// Implement printing for PseudoSourceValue. This is called from + /// Value::print or Value's operator<<. + virtual void printCustom(raw_ostream &O) const; - bool isAliased(const MachineFrameInfo *MFI) const override; +public: + explicit PseudoSourceValue(PSVKind Kind); - bool mayAlias(const MachineFrameInfo *) const override; + virtual ~PseudoSourceValue(); - void printCustom(raw_ostream &OS) const override; + PSVKind kind() const { return Kind; } - int getFrameIndex() const { return FI; } - }; -} // End llvm namespace + bool isStack() const { return Kind == Stack; } + bool isGOT() const { return Kind == GOT; } + bool isConstantPool() const { return Kind == ConstantPool; } + bool isJumpTable() const { return Kind == JumpTable; } + + /// Test whether the memory pointed to by this PseudoSourceValue has a + /// constant value. + virtual bool isConstant(const MachineFrameInfo *) const; + + /// Test whether the memory pointed to by this PseudoSourceValue may also be + /// pointed to by an LLVM IR Value. + virtual bool isAliased(const MachineFrameInfo *) const; + + /// Return true if the memory pointed to by this PseudoSourceValue can ever + /// alias an LLVM IR Value. + virtual bool mayAlias(const MachineFrameInfo *) const; +}; + +/// A specialized PseudoSourceValue for holding FixedStack values, which must +/// include a frame index. +class FixedStackPseudoSourceValue : public PseudoSourceValue { + const int FI; + +public: + explicit FixedStackPseudoSourceValue(int FI) + : PseudoSourceValue(FixedStack), FI(FI) {} + + static inline bool classof(const PseudoSourceValue *V) { + return V->kind() == FixedStack; + } + + bool isConstant(const MachineFrameInfo *MFI) const override; + + bool isAliased(const MachineFrameInfo *MFI) const override; + + bool mayAlias(const MachineFrameInfo *) const override; + + void printCustom(raw_ostream &OS) const override; + + int getFrameIndex() const { return FI; } +}; + +class CallEntryPseudoSourceValue : public PseudoSourceValue { +protected: + CallEntryPseudoSourceValue(PSVKind Kind); + +public: + bool isConstant(const MachineFrameInfo *) const override; + bool isAliased(const MachineFrameInfo *) const override; + bool mayAlias(const MachineFrameInfo *) const override; +}; + +/// A specialized pseudo soruce value for holding GlobalValue values. +class GlobalValuePseudoSourceValue : public CallEntryPseudoSourceValue { + const GlobalValue *GV; + +public: + GlobalValuePseudoSourceValue(const GlobalValue *GV); + + static inline bool classof(const PseudoSourceValue *V) { + return V->kind() == GlobalValueCallEntry; + } + + const GlobalValue *getValue() const { return GV; } +}; + +/// A specialized pseudo source value for holding external symbol values. +class ExternalSymbolPseudoSourceValue : public CallEntryPseudoSourceValue { + const char *ES; + +public: + ExternalSymbolPseudoSourceValue(const char *ES); + + static inline bool classof(const PseudoSourceValue *V) { + return V->kind() == ExternalSymbolCallEntry; + } + + const char *getSymbol() const { return ES; } +}; + +/// Manages creation of pseudo source values. +class PseudoSourceValueManager { + const PseudoSourceValue StackPSV, GOTPSV, JumpTablePSV, ConstantPoolPSV; + std::map<int, std::unique_ptr<FixedStackPseudoSourceValue>> FSValues; + StringMap<std::unique_ptr<const ExternalSymbolPseudoSourceValue>> + ExternalCallEntries; + ValueMap<const GlobalValue *, + std::unique_ptr<const GlobalValuePseudoSourceValue>> + GlobalCallEntries; + +public: + PseudoSourceValueManager(); + + /// Return a pseudo source value referencing the area below the stack frame of + /// a function, e.g., the argument space. + const PseudoSourceValue *getStack(); + + /// Return a pseudo source value referencing the global offset table + /// (or something the like). + const PseudoSourceValue *getGOT(); + + /// Return a pseudo source value referencing the constant pool. Since constant + /// pools are constant, this doesn't need to identify a specific constant + /// pool entry. + const PseudoSourceValue *getConstantPool(); + + /// Return a pseudo source value referencing a jump table. Since jump tables + /// are constant, this doesn't need to identify a specific jump table. + const PseudoSourceValue *getJumpTable(); + + /// Return a pseudo source value referencing a fixed stack frame entry, + /// e.g., a spill slot. + const PseudoSourceValue *getFixedStack(int FI); + + const PseudoSourceValue *getGlobalValueCallEntry(const GlobalValue *GV); + + const PseudoSourceValue *getExternalSymbolCallEntry(const char *ES); +}; + +} // end namespace llvm #endif diff --git a/include/llvm/CodeGen/RegAllocPBQP.h b/include/llvm/CodeGen/RegAllocPBQP.h index 6046e46547b2..4122811a9e5c 100644 --- a/include/llvm/CodeGen/RegAllocPBQP.h +++ b/include/llvm/CodeGen/RegAllocPBQP.h @@ -134,7 +134,7 @@ inline hash_code hash_value(const AllowedRegVector &OptRegs) { hash_combine_range(OStart, OEnd)); } -/// \brief Holds graph-level metadata relevent to PBQP RA problems. +/// \brief Holds graph-level metadata relevant to PBQP RA problems. class GraphMetadata { private: typedef ValuePool<AllowedRegVector> AllowedRegVecPool; diff --git a/include/llvm/CodeGen/RegAllocRegistry.h b/include/llvm/CodeGen/RegAllocRegistry.h index ca495778446f..5c7e9999cc9a 100644 --- a/include/llvm/CodeGen/RegAllocRegistry.h +++ b/include/llvm/CodeGen/RegAllocRegistry.h @@ -33,12 +33,10 @@ public: static MachinePassRegistry Registry; RegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C) - : MachinePassRegistryNode(N, D, (MachinePassCtor)C) - { - Registry.Add(this); + : MachinePassRegistryNode(N, D, (MachinePassCtor)C) { + Registry.Add(this); } ~RegisterRegAlloc() { Registry.Remove(this); } - // Accessors. // @@ -57,7 +55,6 @@ public: static void setListener(MachinePassRegistryListener *L) { Registry.setListener(L); } - }; } // end namespace llvm diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h index 9d8843d1d74a..987634fb36c3 100644 --- a/include/llvm/CodeGen/RegisterPressure.h +++ b/include/llvm/CodeGen/RegisterPressure.h @@ -125,11 +125,13 @@ class PressureDiff { enum { MaxPSets = 16 }; PressureChange PressureChanges[MaxPSets]; -public: + typedef PressureChange* iterator; + iterator nonconst_begin() { return &PressureChanges[0]; } + iterator nonconst_end() { return &PressureChanges[MaxPSets]; } + +public: typedef const PressureChange* const_iterator; - iterator begin() { return &PressureChanges[0]; } - iterator end() { return &PressureChanges[MaxPSets]; } const_iterator begin() const { return &PressureChanges[0]; } const_iterator end() const { return &PressureChanges[MaxPSets]; } @@ -191,30 +193,56 @@ struct RegPressureDelta { } }; -/// \brief A set of live virtual registers and physical register units. +/// A set of live virtual registers and physical register units. /// -/// Virtual and physical register numbers require separate sparse sets, but most -/// of the RegisterPressureTracker handles them uniformly. -struct LiveRegSet { - SparseSet<unsigned> PhysRegs; - SparseSet<unsigned, VirtReg2IndexFunctor> VirtRegs; +/// This is a wrapper around a SparseSet which deals with mapping register unit +/// and virtual register indexes to an index usable by the sparse set. +class LiveRegSet { +private: + SparseSet<unsigned> Regs; + unsigned NumRegUnits; + + unsigned getSparseIndexFromReg(unsigned Reg) const { + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return TargetRegisterInfo::virtReg2Index(Reg) + NumRegUnits; + assert(Reg < NumRegUnits); + return Reg; + } + unsigned getRegFromSparseIndex(unsigned SparseIndex) const { + if (SparseIndex >= NumRegUnits) + return TargetRegisterInfo::index2VirtReg(SparseIndex-NumRegUnits); + return SparseIndex; + } + +public: + void clear(); + void init(const MachineRegisterInfo &MRI); bool contains(unsigned Reg) const { - if (TargetRegisterInfo::isVirtualRegister(Reg)) - return VirtRegs.count(Reg); - return PhysRegs.count(Reg); + unsigned SparseIndex = getSparseIndexFromReg(Reg); + return Regs.count(SparseIndex); } bool insert(unsigned Reg) { - if (TargetRegisterInfo::isVirtualRegister(Reg)) - return VirtRegs.insert(Reg).second; - return PhysRegs.insert(Reg).second; + unsigned SparseIndex = getSparseIndexFromReg(Reg); + return Regs.insert(SparseIndex).second; } bool erase(unsigned Reg) { - if (TargetRegisterInfo::isVirtualRegister(Reg)) - return VirtRegs.erase(Reg); - return PhysRegs.erase(Reg); + unsigned SparseIndex = getSparseIndexFromReg(Reg); + return Regs.erase(SparseIndex); + } + + size_t size() const { + return Regs.size(); + } + + template<typename ContainerT> + void appendTo(ContainerT &To) const { + for (unsigned I : Regs) { + unsigned Reg = getRegFromSparseIndex(I); + To.push_back(Reg); + } } }; @@ -300,16 +328,12 @@ public: // position changes while pressure does not. void setPos(MachineBasicBlock::const_iterator Pos) { CurrPos = Pos; } - /// \brief Get the SlotIndex for the first nondebug instruction including or - /// after the current position. - SlotIndex getCurrSlot() const; - /// Recede across the previous instruction. - bool recede(SmallVectorImpl<unsigned> *LiveUses = nullptr, + void recede(SmallVectorImpl<unsigned> *LiveUses = nullptr, PressureDiff *PDiff = nullptr); /// Advance across the current instruction. - bool advance(); + void advance(); /// Finalize the region boundaries and recored live ins and live outs. void closeRegion(); @@ -326,17 +350,15 @@ public: ArrayRef<unsigned> getLiveThru() const { return LiveThruPressure; } /// Get the resulting register pressure over the traversed region. - /// This result is complete if either advance() or recede() has returned true, - /// or if closeRegion() was explicitly invoked. + /// This result is complete if closeRegion() was explicitly invoked. RegisterPressure &getPressure() { return P; } const RegisterPressure &getPressure() const { return P; } /// Get the register set pressure at the current position, which may be less /// than the pressure across the traversed region. - std::vector<unsigned> &getRegSetPressureAtPos() { return CurrSetPressure; } - - void discoverLiveOut(unsigned Reg); - void discoverLiveIn(unsigned Reg); + const std::vector<unsigned> &getRegSetPressureAtPos() const { + return CurrSetPressure; + } bool isTopClosed() const; bool isBottomClosed() const; @@ -412,7 +434,12 @@ public: void dump() const; protected: - const LiveRange *getLiveRange(unsigned Reg) const; + void discoverLiveOut(unsigned Reg); + void discoverLiveIn(unsigned Reg); + + /// \brief Get the SlotIndex for the first nondebug instruction including or + /// after the current position. + SlotIndex getCurrSlot() const; void increaseRegPressure(ArrayRef<unsigned> Regs); void decreaseRegPressure(ArrayRef<unsigned> Regs); diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h index df3fd34e0af6..122c78534253 100644 --- a/include/llvm/CodeGen/RegisterScavenging.h +++ b/include/llvm/CodeGen/RegisterScavenging.h @@ -74,10 +74,6 @@ public: /// Start tracking liveness from the begin of the specific basic block. void enterBasicBlock(MachineBasicBlock *mbb); - /// Allow resetting register state info for multiple - /// passes over/within the same function. - void initRegState(); - /// Move the internal MBB iterator and update register states. void forward(); @@ -104,10 +100,8 @@ public: MBBI = I; } - MachineBasicBlock::iterator getCurrentPosition() const { - return MBBI; - } - + MachineBasicBlock::iterator getCurrentPosition() const { return MBBI; } + /// Return if a specific register is currently used. bool isRegUsed(unsigned Reg, bool includeReserved = true) const; @@ -152,7 +146,7 @@ public: } /// Tell the scavenger a register is used. - void setRegUsed(unsigned Reg); + void setRegUsed(unsigned Reg, LaneBitmask LaneMask = ~0u); private: /// Returns true if a register is reserved. It is never "unused". bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); } @@ -169,10 +163,10 @@ private: /// Processes the current instruction and fill the KillRegUnits and /// DefRegUnits bit vectors. void determineKillsAndDefs(); - + /// Add all Reg Units that Reg contains to BV. void addRegUnits(BitVector &BV, unsigned Reg); - + /// Return the candidate register that is unused for the longest after /// StartMI. UseMI is set to the instruction where the search stopped. /// @@ -182,6 +176,9 @@ private: unsigned InstrLimit, MachineBasicBlock::iterator &UseMI); + /// Allow resetting register state info for multiple + /// passes over/within the same function. + void initRegState(); }; } // End llvm namespace diff --git a/include/llvm/CodeGen/RuntimeLibcalls.h b/include/llvm/CodeGen/RuntimeLibcalls.h index 2be5de640e29..7db03459f9bf 100644 --- a/include/llvm/CodeGen/RuntimeLibcalls.h +++ b/include/llvm/CodeGen/RuntimeLibcalls.h @@ -231,13 +231,9 @@ namespace RTLIB { FPROUND_F80_F64, FPROUND_F128_F64, FPROUND_PPCF128_F64, - FPTOSINT_F32_I8, - FPTOSINT_F32_I16, FPTOSINT_F32_I32, FPTOSINT_F32_I64, FPTOSINT_F32_I128, - FPTOSINT_F64_I8, - FPTOSINT_F64_I16, FPTOSINT_F64_I32, FPTOSINT_F64_I64, FPTOSINT_F64_I128, @@ -250,13 +246,9 @@ namespace RTLIB { FPTOSINT_PPCF128_I32, FPTOSINT_PPCF128_I64, FPTOSINT_PPCF128_I128, - FPTOUINT_F32_I8, - FPTOUINT_F32_I16, FPTOUINT_F32_I32, FPTOUINT_F32_I64, FPTOUINT_F32_I128, - FPTOUINT_F64_I8, - FPTOUINT_F64_I16, FPTOUINT_F64_I32, FPTOUINT_F64_I64, FPTOUINT_F64_I128, diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h index 839131416560..bda9dbd51fff 100644 --- a/include/llvm/CodeGen/ScheduleDAG.h +++ b/include/llvm/CodeGen/ScheduleDAG.h @@ -20,11 +20,11 @@ #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/Target/TargetLowering.h" namespace llvm { - class AliasAnalysis; class SUnit; class MachineConstantPool; class MachineFunction; @@ -122,18 +122,7 @@ namespace llvm { } /// Return true if the specified SDep is equivalent except for latency. - bool overlaps(const SDep &Other) const { - if (Dep != Other.Dep) return false; - switch (Dep.getInt()) { - case Data: - case Anti: - case Output: - return Contents.Reg == Other.Contents.Reg; - case Order: - return Contents.OrdKind == Other.Contents.OrdKind; - } - llvm_unreachable("Invalid dependency kind!"); - } + bool overlaps(const SDep &Other) const; bool operator==(const SDep &Other) const { return overlaps(Other) && Latency == Other.Latency; @@ -157,19 +146,13 @@ namespace llvm { } //// getSUnit - Return the SUnit to which this edge points. - SUnit *getSUnit() const { - return Dep.getPointer(); - } + SUnit *getSUnit() const; //// setSUnit - Assign the SUnit to which this edge points. - void setSUnit(SUnit *SU) { - Dep.setPointer(SU); - } + void setSUnit(SUnit *SU); /// getKind - Return an enum value representing the kind of the dependence. - Kind getKind() const { - return Dep.getInt(); - } + Kind getKind() const; /// isCtrl - Shorthand for getKind() != SDep::Data. bool isCtrl() const { @@ -374,7 +357,7 @@ namespace llvm { /// correspond to schedulable entities (e.g. instructions) and do not have a /// valid ID. Consequently, always check for boundary nodes before accessing /// an assoicative data structure keyed on node ID. - bool isBoundaryNode() const { return NodeNum == BoundaryID; }; + bool isBoundaryNode() const { return NodeNum == BoundaryID; } /// setNode - Assign the representative SDNode for this SUnit. /// This may be used during pre-regalloc scheduling. @@ -490,6 +473,30 @@ namespace llvm { void ComputeHeight(); }; + /// Return true if the specified SDep is equivalent except for latency. + inline bool SDep::overlaps(const SDep &Other) const { + if (Dep != Other.Dep) + return false; + switch (Dep.getInt()) { + case Data: + case Anti: + case Output: + return Contents.Reg == Other.Contents.Reg; + case Order: + return Contents.OrdKind == Other.Contents.OrdKind; + } + llvm_unreachable("Invalid dependency kind!"); + } + + //// getSUnit - Return the SUnit to which this edge points. + inline SUnit *SDep::getSUnit() const { return Dep.getPointer(); } + + //// setSUnit - Assign the SUnit to which this edge points. + inline void SDep::setSUnit(SUnit *SU) { Dep.setPointer(SU); } + + /// getKind - Return an enum value representing the kind of the dependence. + inline SDep::Kind SDep::getKind() const { return Dep.getInt(); } + //===--------------------------------------------------------------------===// /// SchedulingPriorityQueue - This interface is used to plug different /// priorities computation algorithms into the list scheduler. It implements diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h index b56d5ec8ce63..c574df094911 100644 --- a/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -26,22 +26,32 @@ namespace llvm { class MachineFrameInfo; class MachineLoopInfo; class MachineDominatorTree; - class LiveIntervals; class RegPressureTracker; class PressureDiffs; /// An individual mapping from virtual register number to SUnit. struct VReg2SUnit { unsigned VirtReg; + LaneBitmask LaneMask; SUnit *SU; - VReg2SUnit(unsigned reg, SUnit *su): VirtReg(reg), SU(su) {} + VReg2SUnit(unsigned VReg, LaneBitmask LaneMask, SUnit *SU) + : VirtReg(VReg), LaneMask(LaneMask), SU(SU) {} unsigned getSparseSetIndex() const { return TargetRegisterInfo::virtReg2Index(VirtReg); } }; + /// Mapping from virtual register to SUnit including an operand index. + struct VReg2SUnitOperIdx : public VReg2SUnit { + unsigned OperandIndex; + + VReg2SUnitOperIdx(unsigned VReg, LaneBitmask LaneMask, + unsigned OperandIndex, SUnit *SU) + : VReg2SUnit(VReg, LaneMask, SU), OperandIndex(OperandIndex) {} + }; + /// Record a physical register access. /// For non-data-dependent uses, OpIdx == -1. struct PhysRegSUOper { @@ -69,7 +79,10 @@ namespace llvm { /// Track local uses of virtual registers. These uses are gathered by the DAG /// builder and may be consulted by the scheduler to avoid iterating an entire /// vreg use list. - typedef SparseMultiSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2UseMap; + typedef SparseMultiSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2SUnitMultiMap; + + typedef SparseMultiSet<VReg2SUnitOperIdx, VirtReg2IndexFunctor> + VReg2SUnitOperIdxMultiMap; /// ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of /// MachineInstrs. @@ -78,15 +91,9 @@ namespace llvm { const MachineLoopInfo *MLI; const MachineFrameInfo *MFI; - /// Live Intervals provides reaching defs in preRA scheduling. - LiveIntervals *LIS; - /// TargetSchedModel provides an interface to the machine model. TargetSchedModel SchedModel; - /// isPostRA flag indicates vregs cannot be present. - bool IsPostRA; - /// True if the DAG builder should remove kill flags (in preparation for /// rescheduling). bool RemoveKillFlags; @@ -98,6 +105,9 @@ namespace llvm { /// it has taken responsibility for scheduling the terminator correctly. bool CanHandleTerminators; + /// Whether lane masks should get tracked. + bool TrackLaneMasks; + /// State specific to the current scheduling region. /// ------------------------------------------------ @@ -120,7 +130,7 @@ namespace llvm { /// After calling BuildSchedGraph, each vreg used in the scheduling region /// is mapped to a set of SUnits. These include all local vreg uses, not /// just the uses for a singly defined vreg. - VReg2UseMap VRegUses; + VReg2SUnitMultiMap VRegUses; /// State internal to DAG building. /// ------------------------------- @@ -132,8 +142,12 @@ namespace llvm { Reg2SUnitsMap Defs; Reg2SUnitsMap Uses; - /// Track the last instruction in this region defining each virtual register. - VReg2SUnitMap VRegDefs; + /// Tracks the last instruction(s) in this region defining each virtual + /// register. There may be multiple current definitions for a register with + /// disjunct lanemasks. + VReg2SUnitMultiMap CurrentVRegDefs; + /// Tracks the last instructions in this region using each virtual register. + VReg2SUnitOperIdxMultiMap CurrentVRegUses; /// PendingLoads - Remember where unknown loads are after the most recent /// unknown store, as we iterate. As with Defs and Uses, this is here @@ -154,17 +168,10 @@ namespace llvm { public: explicit ScheduleDAGInstrs(MachineFunction &mf, const MachineLoopInfo *mli, - bool IsPostRAFlag, - bool RemoveKillFlags = false, - LiveIntervals *LIS = nullptr); + bool RemoveKillFlags = false); ~ScheduleDAGInstrs() override {} - bool isPostRA() const { return IsPostRA; } - - /// \brief Expose LiveIntervals for use in DAG mutators and such. - LiveIntervals *getLIS() const { return LIS; } - /// \brief Get the machine model for instruction scheduling. const TargetSchedModel *getSchedModel() const { return &SchedModel; } @@ -206,7 +213,8 @@ namespace llvm { /// input. void buildSchedGraph(AliasAnalysis *AA, RegPressureTracker *RPTracker = nullptr, - PressureDiffs *PDiffs = nullptr); + PressureDiffs *PDiffs = nullptr, + bool TrackLaneMasks = false); /// addSchedBarrierDeps - Add dependencies from instructions in the current /// list of instructions being scheduled to scheduling barrier. We want to @@ -253,6 +261,12 @@ namespace llvm { /// Other adjustments may be made to the instruction if necessary. Return /// true if the operand has been deleted, false if not. bool toggleKillFlag(MachineInstr *MI, MachineOperand &MO); + + /// Returns a mask for which lanes get read/written by the given (register) + /// machine operand. + LaneBitmask getLaneMaskForMO(const MachineOperand &MO) const; + + void collectVRegUses(SUnit *SU); }; /// newSUnit - Creates a new SUnit and return a ptr to it. diff --git a/include/llvm/CodeGen/SchedulerRegistry.h b/include/llvm/CodeGen/SchedulerRegistry.h index 51ac7f28527f..a7a6227664de 100644 --- a/include/llvm/CodeGen/SchedulerRegistry.h +++ b/include/llvm/CodeGen/SchedulerRegistry.h @@ -52,12 +52,6 @@ public: static RegisterScheduler *getList() { return (RegisterScheduler *)Registry.getList(); } - static FunctionPassCtor getDefault() { - return (FunctionPassCtor)Registry.getDefault(); - } - static void setDefault(FunctionPassCtor C) { - Registry.setDefault((MachinePassCtor)C); - } static void setListener(MachinePassRegistryListener *L) { Registry.setListener(L); } diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 1ee92380e690..a21e9ae881a7 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -19,6 +19,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/ilist.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/SelectionDAGNodes.h" @@ -31,7 +32,6 @@ namespace llvm { -class AliasAnalysis; class MachineConstantPoolValue; class MachineFunction; class MDNode; @@ -215,6 +215,8 @@ class SelectionDAG { /// Tracks dbg_value information through SDISel. SDDbgInfo *DbgInfo; + uint16_t NextPersistentId = 0; + public: /// Clients of various APIs that cause global effects on /// the DAG can optionally implement this interface. This allows the clients @@ -324,11 +326,10 @@ public: } iterator_range<allnodes_iterator> allnodes() { - return iterator_range<allnodes_iterator>(allnodes_begin(), allnodes_end()); + return make_range(allnodes_begin(), allnodes_end()); } iterator_range<allnodes_const_iterator> allnodes() const { - return iterator_range<allnodes_const_iterator>(allnodes_begin(), - allnodes_end()); + return make_range(allnodes_begin(), allnodes_end()); } /// Return the root tag of the SelectionDAG. @@ -532,7 +533,7 @@ public: SDVTList VTs = getVTList(MVT::Other, MVT::Glue); SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue }; return getNode(ISD::CopyToReg, dl, VTs, - ArrayRef<SDValue>(Ops, Glue.getNode() ? 4 : 3)); + makeArrayRef(Ops, Glue.getNode() ? 4 : 3)); } // Similar to last getCopyToReg() except parameter Reg is a SDValue @@ -541,7 +542,7 @@ public: SDVTList VTs = getVTList(MVT::Other, MVT::Glue); SDValue Ops[] = { Chain, Reg, N, Glue }; return getNode(ISD::CopyToReg, dl, VTs, - ArrayRef<SDValue>(Ops, Glue.getNode() ? 4 : 3)); + makeArrayRef(Ops, Glue.getNode() ? 4 : 3)); } SDValue getCopyFromReg(SDValue Chain, SDLoc dl, unsigned Reg, EVT VT) { @@ -558,7 +559,7 @@ public: SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue); SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue }; return getNode(ISD::CopyFromReg, dl, VTs, - ArrayRef<SDValue>(Ops, Glue.getNode() ? 3 : 2)); + makeArrayRef(Ops, Glue.getNode() ? 3 : 2)); } SDValue getCondCode(ISD::CondCode Cond); @@ -670,7 +671,7 @@ public: SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef<SDUse> Ops); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, - ArrayRef<SDValue> Ops); + ArrayRef<SDValue> Ops, const SDNodeFlags *Flags = nullptr); SDValue getNode(unsigned Opcode, SDLoc DL, ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops); SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, @@ -687,7 +688,7 @@ public: SDValue N3, SDValue N4); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, SDValue N3, SDValue N4, SDValue N5); - + // Specialize again based on number of operands for nodes with a VTList // rather than a single VT. SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs); @@ -901,6 +902,12 @@ public: /// the target's desired shift amount type. SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op); + /// Expand the specified \c ISD::VAARG node as the Legalize pass would. + SDValue expandVAArg(SDNode *Node); + + /// Expand the specified \c ISD::VACOPY node as the Legalize pass would. + SDValue expandVACopy(SDNode *Node); + /// *Mutate* the specified node in-place to have the /// specified operands. If the resultant node already exists in the DAG, /// this does not modify the specified node, instead it returns the node that @@ -1072,6 +1079,10 @@ public: // target info. switch (Opcode) { case ISD::ADD: + case ISD::SMIN: + case ISD::SMAX: + case ISD::UMIN: + case ISD::UMAX: case ISD::MUL: case ISD::MULHU: case ISD::MULHS: @@ -1088,6 +1099,8 @@ public: case ISD::ADDE: case ISD::FMINNUM: case ISD::FMAXNUM: + case ISD::FMINNAN: + case ISD::FMAXNAN: return true; default: return false; } @@ -1150,6 +1163,10 @@ public: const ConstantSDNode *Cst1, const ConstantSDNode *Cst2); + SDValue FoldConstantVectorArithmetic(unsigned Opcode, SDLoc DL, + EVT VT, ArrayRef<SDValue> Ops, + const SDNodeFlags *Flags = nullptr); + /// Constant fold a setcc to true or false. SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, SDLoc dl); @@ -1199,6 +1216,10 @@ public: /// other positive zero. bool isEqualTo(SDValue A, SDValue B) const; + /// Return true if A and B have no common bits set. As an example, this can + /// allow an 'add' to be transformed into an 'or'. + bool haveNoCommonBitsSet(SDValue A, SDValue B) const; + /// Utility function used by legalize and lowering to /// "unroll" a vector operation by splitting out the scalars and operating /// on each element individually. If the ResNE is 0, fully unroll the vector diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 4821d1aae9e5..23816bde07c0 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -44,6 +44,7 @@ class GlobalValue; class MachineBasicBlock; class MachineConstantPoolValue; class SDNode; +class BinaryWithFlagsSDNode; class Value; class MCSymbol; template <typename T> struct DenseMapInfo; @@ -81,11 +82,6 @@ namespace ISD { /// all ConstantFPSDNode or undef. bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); - /// Return true if the specified node is a - /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low - /// element is not an undef. - bool isScalarToVector(const SDNode *N); - /// Return true if the node has at least one operand /// and all operands of the specified node are ISD::UNDEF. bool allOperandsUndef(const SDNode *N); @@ -139,7 +135,7 @@ public: return SDValue(Node, R); } - // Return true if this node is an operand of N. + /// Return true if this node is an operand of N. bool isOperandOf(const SDNode *N) const; /// Return the ValueType of the referenced return value. @@ -167,6 +163,7 @@ public: inline bool isTargetMemoryOpcode() const; inline bool isTargetOpcode() const; inline bool isMachineOpcode() const; + inline bool isUndef() const; inline unsigned getMachineOpcode() const; inline const DebugLoc &getDebugLoc() const; inline void dump() const; @@ -318,6 +315,61 @@ template<> struct simplify_type<SDUse> { } }; +/// These are IR-level optimization flags that may be propagated to SDNodes. +/// TODO: This data structure should be shared by the IR optimizer and the +/// the backend. +struct SDNodeFlags { +private: + bool NoUnsignedWrap : 1; + bool NoSignedWrap : 1; + bool Exact : 1; + bool UnsafeAlgebra : 1; + bool NoNaNs : 1; + bool NoInfs : 1; + bool NoSignedZeros : 1; + bool AllowReciprocal : 1; + +public: + /// Default constructor turns off all optimization flags. + SDNodeFlags() { + NoUnsignedWrap = false; + NoSignedWrap = false; + Exact = false; + UnsafeAlgebra = false; + NoNaNs = false; + NoInfs = false; + NoSignedZeros = false; + AllowReciprocal = false; + } + + // These are mutators for each flag. + void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } + void setNoSignedWrap(bool b) { NoSignedWrap = b; } + void setExact(bool b) { Exact = b; } + void setUnsafeAlgebra(bool b) { UnsafeAlgebra = b; } + void setNoNaNs(bool b) { NoNaNs = b; } + void setNoInfs(bool b) { NoInfs = b; } + void setNoSignedZeros(bool b) { NoSignedZeros = b; } + void setAllowReciprocal(bool b) { AllowReciprocal = b; } + + // These are accessors for each flag. + bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } + bool hasNoSignedWrap() const { return NoSignedWrap; } + bool hasExact() const { return Exact; } + bool hasUnsafeAlgebra() const { return UnsafeAlgebra; } + bool hasNoNaNs() const { return NoNaNs; } + bool hasNoInfs() const { return NoInfs; } + bool hasNoSignedZeros() const { return NoSignedZeros; } + bool hasAllowReciprocal() const { return AllowReciprocal; } + + /// Return a raw encoding of the flags. + /// This function should only be used to add data to the NodeID value. + unsigned getRawFlags() const { + return (NoUnsignedWrap << 0) | (NoSignedWrap << 1) | (Exact << 2) | + (UnsafeAlgebra << 3) | (NoNaNs << 4) | (NoInfs << 5) | + (NoSignedZeros << 6) | (AllowReciprocal << 7); + } +}; /// Represents one node in the SelectionDAG. /// @@ -374,6 +426,10 @@ private: friend struct ilist_traits<SDNode>; public: + /// Unique and persistent id per SDNode in the DAG. + /// Used for debug printing. + uint16_t PersistentId; + //===--------------------------------------------------------------------===// // Accessors // @@ -395,6 +451,9 @@ public: return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; } + /// Return true if the type of the node type undefined. + bool isUndef() const { return NodeType == ISD::UNDEF; } + /// Test if this node is a memory intrinsic (with valid pointer information). /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for /// non-memory intrinsics (with chains) that are not really instances of @@ -517,10 +576,10 @@ public: static use_iterator use_end() { return use_iterator(nullptr); } inline iterator_range<use_iterator> uses() { - return iterator_range<use_iterator>(use_begin(), use_end()); + return make_range(use_begin(), use_end()); } inline iterator_range<use_iterator> uses() const { - return iterator_range<use_iterator>(use_begin(), use_end()); + return make_range(use_begin(), use_end()); } /// Return true if there are exactly NUSES uses of the indicated value. @@ -592,8 +651,8 @@ public: }; iterator_range<value_op_iterator> op_values() const { - return iterator_range<value_op_iterator>(value_op_iterator(op_begin()), - value_op_iterator(op_end())); + return make_range(value_op_iterator(op_begin()), + value_op_iterator(op_end())); } SDVTList getVTList() const { @@ -605,27 +664,11 @@ public: /// to which the glue operand points. Otherwise return NULL. SDNode *getGluedNode() const { if (getNumOperands() != 0 && - getOperand(getNumOperands()-1).getValueType() == MVT::Glue) + getOperand(getNumOperands()-1).getValueType() == MVT::Glue) return getOperand(getNumOperands()-1).getNode(); return nullptr; } - // If this is a pseudo op, like copyfromreg, look to see if there is a - // real target node glued to it. If so, return the target node. - const SDNode *getGluedMachineNode() const { - const SDNode *FoundNode = this; - - // Climb up glue edges until a machine-opcode node is found, or the - // end of the chain is reached. - while (!FoundNode->isMachineOpcode()) { - const SDNode *N = FoundNode->getGluedNode(); - if (!N) break; - FoundNode = N; - } - - return FoundNode; - } - /// If this node has a glue value with a user, return /// the user (there is at most one). Otherwise return NULL. SDNode *getGluedUser() const { @@ -635,6 +678,10 @@ public: return nullptr; } + /// This could be defined as a virtual function and implemented more simply + /// and directly, but it is not to avoid creating a vtable for this class. + const SDNodeFlags *getFlags() const; + /// Return the number of values defined/returned by this operator. unsigned getNumValues() const { return NumValues; } @@ -909,6 +956,9 @@ inline bool SDValue::isMachineOpcode() const { inline unsigned SDValue::getMachineOpcode() const { return Node->getMachineOpcode(); } +inline bool SDValue::isUndef() const { + return Node->isUndef(); +} inline bool SDValue::use_empty() const { return !Node->hasAnyUseOfValue(ResNo); } @@ -943,62 +993,6 @@ inline void SDUse::setNode(SDNode *N) { if (N) N->addUse(*this); } -/// These are IR-level optimization flags that may be propagated to SDNodes. -/// TODO: This data structure should be shared by the IR optimizer and the -/// the backend. -struct SDNodeFlags { -private: - bool NoUnsignedWrap : 1; - bool NoSignedWrap : 1; - bool Exact : 1; - bool UnsafeAlgebra : 1; - bool NoNaNs : 1; - bool NoInfs : 1; - bool NoSignedZeros : 1; - bool AllowReciprocal : 1; - -public: - /// Default constructor turns off all optimization flags. - SDNodeFlags() { - NoUnsignedWrap = false; - NoSignedWrap = false; - Exact = false; - UnsafeAlgebra = false; - NoNaNs = false; - NoInfs = false; - NoSignedZeros = false; - AllowReciprocal = false; - } - - // These are mutators for each flag. - void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } - void setNoSignedWrap(bool b) { NoSignedWrap = b; } - void setExact(bool b) { Exact = b; } - void setUnsafeAlgebra(bool b) { UnsafeAlgebra = b; } - void setNoNaNs(bool b) { NoNaNs = b; } - void setNoInfs(bool b) { NoInfs = b; } - void setNoSignedZeros(bool b) { NoSignedZeros = b; } - void setAllowReciprocal(bool b) { AllowReciprocal = b; } - - // These are accessors for each flag. - bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } - bool hasNoSignedWrap() const { return NoSignedWrap; } - bool hasExact() const { return Exact; } - bool hasUnsafeAlgebra() const { return UnsafeAlgebra; } - bool hasNoNaNs() const { return NoNaNs; } - bool hasNoInfs() const { return NoInfs; } - bool hasNoSignedZeros() const { return NoSignedZeros; } - bool hasAllowReciprocal() const { return AllowReciprocal; } - - /// Return a raw encoding of the flags. - /// This function should only be used to add data to the NodeID value. - unsigned getRawFlags() const { - return (NoUnsignedWrap << 0) | (NoSignedWrap << 1) | (Exact << 2) | - (UnsafeAlgebra << 3) | (NoNaNs << 4) | (NoInfs << 5) | - (NoSignedZeros << 6) | (AllowReciprocal << 7); - } -}; - /// This class is used for single-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. class UnarySDNode : public SDNode { @@ -1080,6 +1074,9 @@ class HandleSDNode : public SDNode { public: explicit HandleSDNode(SDValue X) : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { + // HandleSDNodes are never inserted into the DAG, so they won't be + // auto-numbered. Use ID 65535 as a sentinel. + PersistentId = 0xffff; InitOperands(&Op, X); } ~HandleSDNode(); @@ -1497,6 +1494,15 @@ public: } }; +/// Returns true if \p V is a constant integer zero. +bool isNullConstant(SDValue V); +/// Returns true if \p V is an FP constant with a value of positive zero. +bool isNullFPConstant(SDValue V); +/// Returns true if \p V is an integer constant with all bits set. +bool isAllOnesConstant(SDValue V); +/// Returns true if \p V is a constant integer one. +bool isOneConstant(SDValue V); + class GlobalAddressSDNode : public SDNode { const GlobalValue *TheGlobal; int64_t Offset; @@ -1697,6 +1703,14 @@ public: ConstantFPSDNode * getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; + /// \brief If this is a constant FP splat and the splatted constant FP is an + /// exact power or 2, return the log base 2 integer value. Otherwise, + /// return -1. + /// + /// The BitWidth specifies the necessary bit precision. + int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, + uint32_t BitWidth) const; + bool isConstant() const; static inline bool classof(const SDNode *N) { @@ -2003,9 +2017,9 @@ class MaskedLoadStoreSDNode : public MemSDNode { public: friend class SelectionDAG; MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, - SDValue *Operands, unsigned numOperands, - SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) - : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + SDValue *Operands, unsigned numOperands, SDVTList VTs, + EVT MemVT, MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { InitOperands(Ops, Operands, numOperands); } @@ -2036,7 +2050,7 @@ public: ISD::LoadExtType getExtensionType() const { return ISD::LoadExtType(SubclassData & 3); - } + } const SDValue &getSrc0() const { return getOperand(3); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD; @@ -2103,17 +2117,18 @@ public: class MaskedGatherSDNode : public MaskedGatherScatterSDNode { public: friend class SelectionDAG; - MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands, + MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT, MMO) { assert(getValue().getValueType() == getValueType(0) && - "Incompatible type of the PathThru value in MaskedGatherSDNode"); - assert(getMask().getValueType().getVectorNumElements() == - getValueType(0).getVectorNumElements() && - "Vector width mismatch between mask and data"); - assert(getMask().getValueType().getScalarType() == MVT::i1 && + "Incompatible type of the PassThru value in MaskedGatherSDNode"); + assert(getMask().getValueType().getVectorNumElements() == + getValueType(0).getVectorNumElements() && "Vector width mismatch between mask and data"); + assert(getIndex().getValueType().getVectorNumElements() == + getValueType(0).getVectorNumElements() && + "Vector width mismatch between index and data"); } static bool classof(const SDNode *N) { @@ -2131,11 +2146,12 @@ public: SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT, MMO) { - assert(getMask().getValueType().getVectorNumElements() == - getValue().getValueType().getVectorNumElements() && - "Vector width mismatch between mask and data"); - assert(getMask().getValueType().getScalarType() == MVT::i1 && + assert(getMask().getValueType().getVectorNumElements() == + getValue().getValueType().getVectorNumElements() && "Vector width mismatch between mask and data"); + assert(getIndex().getValueType().getVectorNumElements() == + getValue().getValueType().getVectorNumElements() && + "Vector width mismatch between index and data"); } static bool classof(const SDNode *N) { diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h index 9d6d6f5b1be0..7b621bee259f 100644 --- a/include/llvm/CodeGen/SlotIndexes.h +++ b/include/llvm/CodeGen/SlotIndexes.h @@ -155,7 +155,7 @@ namespace llvm { "Attempt to construct index with 0 pointer."); } - /// Returns true if this is a valid index. Invalid indicies do + /// Returns true if this is a valid index. Invalid indices do /// not point into an index table, and cannot be compared. bool isValid() const { return lie.getPointer(); @@ -272,7 +272,7 @@ namespace llvm { SlotIndex getNextSlot() const { Slot s = getSlot(); if (s == Slot_Dead) { - return SlotIndex(listEntry()->getNextNode(), Slot_Block); + return SlotIndex(&*++listEntry()->getIterator(), Slot_Block); } return SlotIndex(listEntry(), s + 1); } @@ -280,7 +280,7 @@ namespace llvm { /// Returns the next index. This is the index corresponding to the this /// index's slot, but for the next instruction. SlotIndex getNextIndex() const { - return SlotIndex(listEntry()->getNextNode(), getSlot()); + return SlotIndex(&*++listEntry()->getIterator(), getSlot()); } /// Returns the previous slot in the index list. This could be either the @@ -292,7 +292,7 @@ namespace llvm { SlotIndex getPrevSlot() const { Slot s = getSlot(); if (s == Slot_Block) { - return SlotIndex(listEntry()->getPrevNode(), Slot_Dead); + return SlotIndex(&*--listEntry()->getIterator(), Slot_Dead); } return SlotIndex(listEntry(), s - 1); } @@ -300,7 +300,7 @@ namespace llvm { /// Returns the previous index. This is the index corresponding to this /// index's slot, but for the previous instruction. SlotIndex getPrevIndex() const { - return SlotIndex(listEntry()->getPrevNode(), getSlot()); + return SlotIndex(&*--listEntry()->getIterator(), getSlot()); } }; @@ -333,6 +333,8 @@ namespace llvm { /// This pass assigns indexes to each instruction. class SlotIndexes : public MachineFunctionPass { private: + // IndexListEntry allocator. + BumpPtrAllocator ileAllocator; typedef ilist<IndexListEntry> IndexList; IndexList indexList; @@ -353,9 +355,6 @@ namespace llvm { /// and MBB id. SmallVector<IdxMBBPair, 8> idx2MBBMap; - // IndexListEntry allocator. - BumpPtrAllocator ileAllocator; - IndexListEntry* createEntry(MachineInstr *mi, unsigned index) { IndexListEntry *entry = static_cast<IndexListEntry*>( @@ -377,6 +376,11 @@ namespace llvm { initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); } + ~SlotIndexes() { + // The indexList's nodes are all allocated in the BumpPtrAllocator. + indexList.clearAndLeakNodesUnsafely(); + } + void getAnalysisUsage(AnalysisUsage &au) const override; void releaseMemory() override; @@ -427,11 +431,11 @@ namespace llvm { /// Returns the next non-null index, if one exists. /// Otherwise returns getLastIndex(). SlotIndex getNextNonNullIndex(SlotIndex Index) { - IndexList::iterator I = Index.listEntry(); + IndexList::iterator I = Index.listEntry()->getIterator(); IndexList::iterator E = indexList.end(); while (++I != E) if (I->getInstr()) - return SlotIndex(I, Index.getSlot()); + return SlotIndex(&*I, Index.getSlot()); // We reached the end of the function. return getLastIndex(); } @@ -502,49 +506,52 @@ namespace llvm { return getMBBRange(mbb).second; } + /// Iterator over the idx2MBBMap (sorted pairs of slot index of basic block + /// begin and basic block) + typedef SmallVectorImpl<IdxMBBPair>::const_iterator MBBIndexIterator; + /// Move iterator to the next IdxMBBPair where the SlotIndex is greater or + /// equal to \p To. + MBBIndexIterator advanceMBBIndex(MBBIndexIterator I, SlotIndex To) const { + return std::lower_bound(I, idx2MBBMap.end(), To); + } + /// Get an iterator pointing to the IdxMBBPair with the biggest SlotIndex + /// that is greater or equal to \p Idx. + MBBIndexIterator findMBBIndex(SlotIndex Idx) const { + return advanceMBBIndex(idx2MBBMap.begin(), Idx); + } + /// Returns an iterator for the begin of the idx2MBBMap. + MBBIndexIterator MBBIndexBegin() const { + return idx2MBBMap.begin(); + } + /// Return an iterator for the end of the idx2MBBMap. + MBBIndexIterator MBBIndexEnd() const { + return idx2MBBMap.end(); + } + /// Returns the basic block which the given index falls in. MachineBasicBlock* getMBBFromIndex(SlotIndex index) const { if (MachineInstr *MI = getInstructionFromIndex(index)) return MI->getParent(); - SmallVectorImpl<IdxMBBPair>::const_iterator I = - std::lower_bound(idx2MBBMap.begin(), idx2MBBMap.end(), index); + + MBBIndexIterator I = findMBBIndex(index); // Take the pair containing the index - SmallVectorImpl<IdxMBBPair>::const_iterator J = - ((I != idx2MBBMap.end() && I->first > index) || - (I == idx2MBBMap.end() && idx2MBBMap.size()>0)) ? (I-1): I; + MBBIndexIterator J = + ((I != MBBIndexEnd() && I->first > index) || + (I == MBBIndexEnd() && !idx2MBBMap.empty())) ? std::prev(I) : I; - assert(J != idx2MBBMap.end() && J->first <= index && + assert(J != MBBIndexEnd() && J->first <= index && index < getMBBEndIdx(J->second) && "index does not correspond to an MBB"); return J->second; } - bool findLiveInMBBs(SlotIndex start, SlotIndex end, - SmallVectorImpl<MachineBasicBlock*> &mbbs) const { - SmallVectorImpl<IdxMBBPair>::const_iterator itr = - std::lower_bound(idx2MBBMap.begin(), idx2MBBMap.end(), start); - bool resVal = false; - - while (itr != idx2MBBMap.end()) { - if (itr->first >= end) - break; - mbbs.push_back(itr->second); - resVal = true; - ++itr; - } - return resVal; - } - /// Returns the MBB covering the given range, or null if the range covers /// more than one basic block. MachineBasicBlock* getMBBCoveringRange(SlotIndex start, SlotIndex end) const { assert(start < end && "Backwards ranges not allowed."); - - SmallVectorImpl<IdxMBBPair>::const_iterator itr = - std::lower_bound(idx2MBBMap.begin(), idx2MBBMap.end(), start); - - if (itr == idx2MBBMap.end()) { + MBBIndexIterator itr = findMBBIndex(start); + if (itr == MBBIndexEnd()) { itr = std::prev(itr); return itr->second; } @@ -580,11 +587,11 @@ namespace llvm { IndexList::iterator prevItr, nextItr; if (Late) { // Insert mi's index immediately before the following instruction. - nextItr = getIndexAfter(mi).listEntry(); + nextItr = getIndexAfter(mi).listEntry()->getIterator(); prevItr = std::prev(nextItr); } else { // Insert mi's index immediately after the preceding instruction. - prevItr = getIndexBefore(mi).listEntry(); + prevItr = getIndexBefore(mi).listEntry()->getIterator(); nextItr = std::next(prevItr); } @@ -646,11 +653,11 @@ namespace llvm { if (nextMBB == mbb->getParent()->end()) { startEntry = &indexList.back(); endEntry = createEntry(nullptr, 0); - newItr = indexList.insertAfter(startEntry, endEntry); + newItr = indexList.insertAfter(startEntry->getIterator(), endEntry); } else { startEntry = createEntry(nullptr, 0); - endEntry = getMBBStartIdx(nextMBB).listEntry(); - newItr = indexList.insert(endEntry, startEntry); + endEntry = getMBBStartIdx(&*nextMBB).listEntry(); + newItr = indexList.insert(endEntry->getIterator(), startEntry); } SlotIndex startIdx(startEntry, SlotIndex::Slot_Block); diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h index fdc1a9143ed2..972a616ad779 100644 --- a/include/llvm/CodeGen/StackMaps.h +++ b/include/llvm/CodeGen/StackMaps.h @@ -13,6 +13,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/Support/Debug.h" #include <map> #include <vector> diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index 10c099d2c2f5..2f1379131cbd 100644 --- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -41,12 +41,12 @@ public: ~TargetLoweringObjectFileELF() override {} - void emitPersonalityValue(MCStreamer &Streamer, const TargetMachine &TM, + void emitPersonalityValue(MCStreamer &Streamer, const DataLayout &TM, const MCSymbol *Sym) const override; /// Given a constant with the SectionKind, return a section that it should be /// placed in. - MCSection *getSectionForConstant(SectionKind Kind, + MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind, const Constant *C) const override; MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, @@ -103,7 +103,7 @@ public: Mangler &Mang, const TargetMachine &TM) const override; - MCSection *getSectionForConstant(SectionKind Kind, + MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind, const Constant *C) const override; /// The mach-o version of this method defaults to returning a stub reference. @@ -123,6 +123,9 @@ public: const MCValue &MV, int64_t Offset, MachineModuleInfo *MMI, MCStreamer &Streamer) const override; + + void getNameWithPrefix(SmallVectorImpl<char> &OutName, const GlobalValue *GV, + Mangler &Mang, const TargetMachine &TM) const override; }; @@ -140,8 +143,7 @@ public: const TargetMachine &TM) const override; void getNameWithPrefix(SmallVectorImpl<char> &OutName, const GlobalValue *GV, - bool CannotUsePrivateLabel, Mangler &Mang, - const TargetMachine &TM) const override; + Mangler &Mang, const TargetMachine &TM) const override; MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang, const TargetMachine &TM) const override; diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h index 751fac411ce6..81054aba066f 100644 --- a/include/llvm/CodeGen/TargetSchedule.h +++ b/include/llvm/CodeGen/TargetSchedule.h @@ -81,6 +81,12 @@ public: return nullptr; } + /// \brief Return true if this machine model includes an instruction-level + /// scheduling model or cycle-to-cycle itinerary data. + bool hasInstrSchedModelOrItineraries() const { + return hasInstrSchedModel() || hasInstrItineraries(); + } + /// \brief Identify the processor corresponding to the current subtarget. unsigned getProcessorID() const { return SchedModel.getProcessorID(); } diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h index e1a9fd38290b..929eb88a0393 100644 --- a/include/llvm/CodeGen/ValueTypes.h +++ b/include/llvm/CodeGen/ValueTypes.h @@ -89,6 +89,19 @@ namespace llvm { return VecTy; } + /// Return the type converted to an equivalently sized integer or vector + /// with integer element type. Similar to changeVectorElementTypeToInteger, + /// but also handles scalars. + EVT changeTypeToInteger() { + if (isVector()) + return changeVectorElementTypeToInteger(); + + if (isSimple()) + return MVT::getIntegerVT(getSizeInBits()); + + return changeExtendedTypeToInteger(); + } + /// isSimple - Test if the given EVT is simple (as opposed to being /// extended). bool isSimple() const { @@ -151,6 +164,11 @@ namespace llvm { return isSimple() ? V.is1024BitVector() : isExtended1024BitVector(); } + /// is2048BitVector - Return true if this is a 2048-bit vector type. + bool is2048BitVector() const { + return isSimple() ? V.is2048BitVector() : isExtended2048BitVector(); + } + /// isOverloaded - Return true if this is an overloaded type for TableGen. bool isOverloaded() const { return (V==MVT::iAny || V==MVT::fAny || V==MVT::vAny || V==MVT::iPTRAny); @@ -342,6 +360,7 @@ namespace llvm { // Methods for handling the Extended-type case in functions above. // These are all out-of-line to prevent users of this header file // from having a dependency on Type.h. + EVT changeExtendedTypeToInteger() const; EVT changeExtendedVectorElementTypeToInteger() const; static EVT getExtendedIntegerVT(LLVMContext &C, unsigned BitWidth); static EVT getExtendedVectorVT(LLVMContext &C, EVT VT, @@ -356,6 +375,7 @@ namespace llvm { bool isExtended256BitVector() const LLVM_READONLY; bool isExtended512BitVector() const LLVM_READONLY; bool isExtended1024BitVector() const LLVM_READONLY; + bool isExtended2048BitVector() const LLVM_READONLY; EVT getExtendedVectorElementType() const; unsigned getExtendedVectorNumElements() const LLVM_READONLY; unsigned getExtendedSizeInBits() const; diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td index 2b30f14f902c..f29ec42714e8 100644 --- a/include/llvm/CodeGen/ValueTypes.td +++ b/include/llvm/CodeGen/ValueTypes.td @@ -33,55 +33,70 @@ def f80 : ValueType<80 , 10>; // 80-bit floating point value def f128 : ValueType<128, 11>; // 128-bit floating point value def ppcf128: ValueType<128, 12>; // PPC 128-bit floating point value -def v2i1 : ValueType<2 , 13>; // 2 x i1 vector value -def v4i1 : ValueType<4 , 14>; // 4 x i1 vector value -def v8i1 : ValueType<8 , 15>; // 8 x i1 vector value -def v16i1 : ValueType<16, 16>; // 16 x i1 vector value -def v32i1 : ValueType<32 , 17>; // 32 x i1 vector value -def v64i1 : ValueType<64 , 18>; // 64 x i1 vector value -def v1i8 : ValueType<16, 19>; // 1 x i8 vector value -def v2i8 : ValueType<16 , 20>; // 2 x i8 vector value -def v4i8 : ValueType<32 , 21>; // 4 x i8 vector value -def v8i8 : ValueType<64 , 22>; // 8 x i8 vector value -def v16i8 : ValueType<128, 23>; // 16 x i8 vector value -def v32i8 : ValueType<256, 24>; // 32 x i8 vector value -def v64i8 : ValueType<512, 25>; // 64 x i8 vector value -def v1i16 : ValueType<16 , 26>; // 1 x i16 vector value -def v2i16 : ValueType<32 , 27>; // 2 x i16 vector value -def v4i16 : ValueType<64 , 28>; // 4 x i16 vector value -def v8i16 : ValueType<128, 29>; // 8 x i16 vector value -def v16i16 : ValueType<256, 30>; // 16 x i16 vector value -def v32i16 : ValueType<512, 31>; // 32 x i16 vector value -def v1i32 : ValueType<32 , 32>; // 1 x i32 vector value -def v2i32 : ValueType<64 , 33>; // 2 x i32 vector value -def v4i32 : ValueType<128, 34>; // 4 x i32 vector value -def v8i32 : ValueType<256, 35>; // 8 x i32 vector value -def v16i32 : ValueType<512, 36>; // 16 x i32 vector value -def v1i64 : ValueType<64 , 37>; // 1 x i64 vector value -def v2i64 : ValueType<128, 38>; // 2 x i64 vector value -def v4i64 : ValueType<256, 39>; // 4 x i64 vector value -def v8i64 : ValueType<512, 40>; // 8 x i64 vector value -def v16i64 : ValueType<1024,41>; // 16 x i64 vector value -def v1i128 : ValueType<128, 42>; // 1 x i128 vector value - -def v2f16 : ValueType<32 , 43>; // 2 x f16 vector value -def v4f16 : ValueType<64 , 44>; // 4 x f16 vector value -def v8f16 : ValueType<128, 45>; // 8 x f16 vector value -def v1f32 : ValueType<32 , 46>; // 1 x f32 vector value -def v2f32 : ValueType<64 , 47>; // 2 x f32 vector value -def v4f32 : ValueType<128, 48>; // 4 x f32 vector value -def v8f32 : ValueType<256, 49>; // 8 x f32 vector value -def v16f32 : ValueType<512, 50>; // 16 x f32 vector value -def v1f64 : ValueType<64, 51>; // 1 x f64 vector value -def v2f64 : ValueType<128, 52>; // 2 x f64 vector value -def v4f64 : ValueType<256, 53>; // 4 x f64 vector value -def v8f64 : ValueType<512, 54>; // 8 x f64 vector value - - -def x86mmx : ValueType<64 , 55>; // X86 MMX value -def FlagVT : ValueType<0 , 56>; // Pre-RA sched glue -def isVoid : ValueType<0 , 57>; // Produces no value -def untyped: ValueType<8 , 58>; // Produces an untyped value +def v2i1 : ValueType<2 , 13>; // 2 x i1 vector value +def v4i1 : ValueType<4 , 14>; // 4 x i1 vector value +def v8i1 : ValueType<8 , 15>; // 8 x i1 vector value +def v16i1 : ValueType<16, 16>; // 16 x i1 vector value +def v32i1 : ValueType<32 , 17>; // 32 x i1 vector value +def v64i1 : ValueType<64 , 18>; // 64 x i1 vector value +def v512i1 : ValueType<512, 19>; // 512 x i8 vector value +def v1024i1: ValueType<1024,20>; //1024 x i8 vector value + +def v1i8 : ValueType<16, 21>; // 1 x i8 vector value +def v2i8 : ValueType<16 , 22>; // 2 x i8 vector value +def v4i8 : ValueType<32 , 23>; // 4 x i8 vector value +def v8i8 : ValueType<64 , 24>; // 8 x i8 vector value +def v16i8 : ValueType<128, 25>; // 16 x i8 vector value +def v32i8 : ValueType<256, 26>; // 32 x i8 vector value +def v64i8 : ValueType<512, 27>; // 64 x i8 vector value +def v128i8 : ValueType<1024,28>; //128 x i8 vector value +def v256i8 : ValueType<2048,29>; //256 x i8 vector value + +def v1i16 : ValueType<16 , 30>; // 1 x i16 vector value +def v2i16 : ValueType<32 , 31>; // 2 x i16 vector value +def v4i16 : ValueType<64 , 32>; // 4 x i16 vector value +def v8i16 : ValueType<128, 33>; // 8 x i16 vector value +def v16i16 : ValueType<256, 34>; // 16 x i16 vector value +def v32i16 : ValueType<512, 35>; // 32 x i16 vector value +def v64i16 : ValueType<1024,36>; // 64 x i16 vector value +def v128i16: ValueType<2048,37>; //128 x i16 vector value + +def v1i32 : ValueType<32 , 38>; // 1 x i32 vector value +def v2i32 : ValueType<64 , 39>; // 2 x i32 vector value +def v4i32 : ValueType<128, 40>; // 4 x i32 vector value +def v8i32 : ValueType<256, 41>; // 8 x i32 vector value +def v16i32 : ValueType<512, 42>; // 16 x i32 vector value +def v32i32 : ValueType<1024,43>; // 32 x i32 vector value +def v64i32 : ValueType<2048,44>; // 32 x i32 vector value + +def v1i64 : ValueType<64 , 45>; // 1 x i64 vector value +def v2i64 : ValueType<128, 46>; // 2 x i64 vector value +def v4i64 : ValueType<256, 47>; // 4 x i64 vector value +def v8i64 : ValueType<512, 48>; // 8 x i64 vector value +def v16i64 : ValueType<1024,49>; // 16 x i64 vector value +def v32i64 : ValueType<2048,50>; // 32 x i64 vector value + +def v1i128 : ValueType<128, 51>; // 1 x i128 vector value + +def v2f16 : ValueType<32 , 52>; // 2 x f16 vector value +def v4f16 : ValueType<64 , 53>; // 4 x f16 vector value +def v8f16 : ValueType<128, 54>; // 8 x f16 vector value +def v1f32 : ValueType<32 , 55>; // 1 x f32 vector value +def v2f32 : ValueType<64 , 56>; // 2 x f32 vector value +def v4f32 : ValueType<128, 57>; // 4 x f32 vector value +def v8f32 : ValueType<256, 58>; // 8 x f32 vector value +def v16f32 : ValueType<512, 59>; // 16 x f32 vector value +def v1f64 : ValueType<64, 60>; // 1 x f64 vector value +def v2f64 : ValueType<128, 61>; // 2 x f64 vector value +def v4f64 : ValueType<256, 62>; // 4 x f64 vector value +def v8f64 : ValueType<512, 63>; // 8 x f64 vector value + + +def x86mmx : ValueType<64 , 64>; // X86 MMX value +def FlagVT : ValueType<0 , 65>; // Pre-RA sched glue +def isVoid : ValueType<0 , 66>; // Produces no value +def untyped: ValueType<8 , 67>; // Produces an untyped value +def token : ValueType<0 , 249>; // TokenTy def MetadataVT: ValueType<0, 250>; // Metadata // Pseudo valuetype mapped to the current pointer size to any address space. diff --git a/include/llvm/CodeGen/WinEHFuncInfo.h b/include/llvm/CodeGen/WinEHFuncInfo.h index 75638a058a30..70d558f5cfbd 100644 --- a/include/llvm/CodeGen/WinEHFuncInfo.h +++ b/include/llvm/CodeGen/WinEHFuncInfo.h @@ -14,145 +14,103 @@ #ifndef LLVM_CODEGEN_WINEHFUNCINFO_H #define LLVM_CODEGEN_WINEHFUNCINFO_H +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/TinyPtrVector.h" -#include "llvm/ADT/DenseMap.h" namespace llvm { +class AllocaInst; class BasicBlock; +class CatchReturnInst; class Constant; class Function; class GlobalVariable; class InvokeInst; class IntrinsicInst; class LandingPadInst; +class MCExpr; class MCSymbol; +class MachineBasicBlock; class Value; -enum ActionType { Catch, Cleanup }; - -class ActionHandler { -public: - ActionHandler(BasicBlock *BB, ActionType Type) - : StartBB(BB), Type(Type), EHState(-1), HandlerBlockOrFunc(nullptr) {} - - ActionType getType() const { return Type; } - BasicBlock *getStartBlock() const { return StartBB; } - - bool hasBeenProcessed() { return HandlerBlockOrFunc != nullptr; } +// The following structs respresent the .xdata tables for various +// Windows-related EH personalities. - void setHandlerBlockOrFunc(Constant *F) { HandlerBlockOrFunc = F; } - Constant *getHandlerBlockOrFunc() { return HandlerBlockOrFunc; } +typedef PointerUnion<const BasicBlock *, MachineBasicBlock *> MBBOrBasicBlock; - void setEHState(int State) { EHState = State; } - int getEHState() const { return EHState; } - -private: - BasicBlock *StartBB; - ActionType Type; - int EHState; - - // Can be either a BlockAddress or a Function depending on the EH personality. - Constant *HandlerBlockOrFunc; -}; - -class CatchHandler : public ActionHandler { -public: - CatchHandler(BasicBlock *BB, Constant *Selector, BasicBlock *NextBB) - : ActionHandler(BB, ActionType::Catch), Selector(Selector), - NextBB(NextBB), ExceptionObjectVar(nullptr), - ExceptionObjectIndex(-1) {} - - // Method for support type inquiry through isa, cast, and dyn_cast: - static inline bool classof(const ActionHandler *H) { - return H->getType() == ActionType::Catch; - } - - Constant *getSelector() const { return Selector; } - BasicBlock *getNextBB() const { return NextBB; } - - const Value *getExceptionVar() { return ExceptionObjectVar; } - TinyPtrVector<BasicBlock *> &getReturnTargets() { return ReturnTargets; } - - void setExceptionVar(const Value *Val) { ExceptionObjectVar = Val; } - void setExceptionVarIndex(int Index) { ExceptionObjectIndex = Index; } - int getExceptionVarIndex() const { return ExceptionObjectIndex; } - void setReturnTargets(TinyPtrVector<BasicBlock *> &Targets) { - ReturnTargets = Targets; - } - -private: - Constant *Selector; - BasicBlock *NextBB; - // While catch handlers are being outlined the ExceptionObjectVar field will - // be populated with the instruction in the parent frame that corresponds - // to the exception object (or nullptr if the catch does not use an - // exception object) and the ExceptionObjectIndex field will be -1. - // When the parseEHActions function is called to populate a vector of - // instances of this class, the ExceptionObjectVar field will be nullptr - // and the ExceptionObjectIndex will be the index of the exception object in - // the parent function's localescape block. - const Value *ExceptionObjectVar; - int ExceptionObjectIndex; - TinyPtrVector<BasicBlock *> ReturnTargets; +struct CxxUnwindMapEntry { + int ToState; + MBBOrBasicBlock Cleanup; }; -class CleanupHandler : public ActionHandler { -public: - CleanupHandler(BasicBlock *BB) : ActionHandler(BB, ActionType::Cleanup) {} +/// Similar to CxxUnwindMapEntry, but supports SEH filters. +struct SEHUnwindMapEntry { + /// If unwinding continues through this handler, transition to the handler at + /// this state. This indexes into SEHUnwindMap. + int ToState = -1; - // Method for support type inquiry through isa, cast, and dyn_cast: - static inline bool classof(const ActionHandler *H) { - return H->getType() == ActionType::Cleanup; - } -}; - -void parseEHActions(const IntrinsicInst *II, - SmallVectorImpl<std::unique_ptr<ActionHandler>> &Actions); + bool IsFinally = false; -// The following structs respresent the .xdata for functions using C++ -// exceptions on Windows. + /// Holds the filter expression function. + const Function *Filter = nullptr; -struct WinEHUnwindMapEntry { - int ToState; - Function *Cleanup; + /// Holds the __except or __finally basic block. + MBBOrBasicBlock Handler; }; struct WinEHHandlerType { int Adjectives; + /// The CatchObj starts out life as an LLVM alloca and is eventually turned + /// frame index. + union { + const AllocaInst *Alloca; + int FrameIndex; + } CatchObj = {}; GlobalVariable *TypeDescriptor; - int CatchObjRecoverIdx; - Function *Handler; + MBBOrBasicBlock Handler; }; struct WinEHTryBlockMapEntry { - int TryLow; - int TryHigh; + int TryLow = -1; + int TryHigh = -1; + int CatchHigh = -1; SmallVector<WinEHHandlerType, 1> HandlerArray; }; +enum class ClrHandlerType { Catch, Finally, Fault, Filter }; + +struct ClrEHUnwindMapEntry { + MBBOrBasicBlock Handler; + uint32_t TypeToken; + int Parent; + ClrHandlerType HandlerType; +}; + struct WinEHFuncInfo { - DenseMap<const Function *, const LandingPadInst *> RootLPad; - DenseMap<const Function *, const InvokeInst *> LastInvoke; - DenseMap<const Function *, int> HandlerEnclosedState; - DenseMap<const Function *, bool> LastInvokeVisited; - DenseMap<const LandingPadInst *, int> LandingPadStateMap; - DenseMap<const Function *, int> CatchHandlerParentFrameObjIdx; - DenseMap<const Function *, int> CatchHandlerParentFrameObjOffset; - DenseMap<const Function *, int> CatchHandlerMaxState; - DenseMap<const Function *, int> HandlerBaseState; - SmallVector<WinEHUnwindMapEntry, 4> UnwindMap; + DenseMap<const Instruction *, int> EHPadStateMap; + DenseMap<const FuncletPadInst *, int> FuncletBaseStateMap; + DenseMap<const InvokeInst *, int> InvokeStateMap; + DenseMap<const CatchReturnInst *, const BasicBlock *> + CatchRetSuccessorColorMap; + DenseMap<MCSymbol *, std::pair<int, MCSymbol *>> LabelToStateMap; + SmallVector<CxxUnwindMapEntry, 4> CxxUnwindMap; SmallVector<WinEHTryBlockMapEntry, 4> TryBlockMap; - SmallVector<std::pair<MCSymbol *, int>, 4> IPToStateList; + SmallVector<SEHUnwindMapEntry, 4> SEHUnwindMap; + SmallVector<ClrEHUnwindMapEntry, 4> ClrEHUnwindMap; int UnwindHelpFrameIdx = INT_MAX; - int UnwindHelpFrameOffset = -1; - unsigned NumIPToStateFuncsVisited = 0; + int PSPSymFrameIdx = INT_MAX; + + int getLastStateNumber() const { return CxxUnwindMap.size() - 1; } + + void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, + MCSymbol *InvokeEnd); - /// localescape index of the 32-bit EH registration node. Set by - /// WinEHStatePass and used indirectly by SEH filter functions of the parent. - int EHRegNodeEscapeIndex = INT_MAX; + int EHRegNodeFrameIndex = INT_MAX; + int EHRegNodeEndOffset = INT_MAX; + int SEHSetFrameOffset = INT_MAX; - WinEHFuncInfo() {} + WinEHFuncInfo(); }; /// Analyze the IR in ParentFn and it's handlers to build WinEHFuncInfo, which @@ -161,5 +119,12 @@ struct WinEHFuncInfo { void calculateWinCXXEHStateNumbers(const Function *ParentFn, WinEHFuncInfo &FuncInfo); +void calculateSEHStateNumbers(const Function *ParentFn, + WinEHFuncInfo &FuncInfo); + +void calculateClrEHStateNumbers(const Function *Fn, WinEHFuncInfo &FuncInfo); + +void calculateCatchReturnSuccessorColors(const Function *Fn, + WinEHFuncInfo &FuncInfo); } #endif // LLVM_CODEGEN_WINEHFUNCINFO_H diff --git a/include/llvm/Config/config.h.cmake b/include/llvm/Config/config.h.cmake index b9fd4504ad76..6a5ac889e6f0 100644 --- a/include/llvm/Config/config.h.cmake +++ b/include/llvm/Config/config.h.cmake @@ -324,9 +324,6 @@ /* Define to 1 if you have the <sys/uio.h> header file. */ #cmakedefine HAVE_SYS_UIO_H ${HAVE_SYS_UIO_H} -/* Define to 1 if you have <sys/wait.h> that is POSIX.1 compatible. */ -#cmakedefine HAVE_SYS_WAIT_H ${HAVE_SYS_WAIT_H} - /* Define if the setupterm() function is supported this platform. */ #cmakedefine HAVE_TERMINFO ${HAVE_TERMINFO} @@ -423,8 +420,10 @@ /* Installation directory for data files */ #cmakedefine LLVM_DATADIR "${LLVM_DATADIR}" -/* Target triple LLVM will generate code for by default */ -#cmakedefine LLVM_DEFAULT_TARGET_TRIPLE "${LLVM_DEFAULT_TARGET_TRIPLE}" +/* Target triple LLVM will generate code for by default + * Doesn't use `cmakedefine` because it is allowed to be empty. + */ +#define LLVM_DEFAULT_TARGET_TRIPLE "${LLVM_DEFAULT_TARGET_TRIPLE}" /* Installation directory for documentation */ #cmakedefine LLVM_DOCSDIR "${LLVM_DOCSDIR}" diff --git a/include/llvm/Config/config.h.in b/include/llvm/Config/config.h.in index 09706499ea30..498aa9ee6b5d 100644 --- a/include/llvm/Config/config.h.in +++ b/include/llvm/Config/config.h.in @@ -152,6 +152,9 @@ /* Define to 1 if you have the `shell32' library (-lshell32). */ #undef HAVE_LIBSHELL32 +/* Define to 1 if you have the `uuid' library (-luuid). */ +#undef HAVE_LIBUUID + /* Define to 1 if you have the `z' library (-lz). */ #undef HAVE_LIBZ @@ -229,9 +232,6 @@ /* Have pthread_rwlock_init */ #undef HAVE_PTHREAD_RWLOCK_INIT -/* Define to 1 if srand48/lrand48/drand48 exist in <stdlib.h> */ -#undef HAVE_RAND48 - /* Define to 1 if you have the `realpath' function. */ #undef HAVE_REALPATH @@ -250,15 +250,9 @@ /* Define to 1 if you have the `setrlimit' function. */ #undef HAVE_SETRLIMIT -/* Define to 1 if you have the `siglongjmp' function. */ -#undef HAVE_SIGLONGJMP - /* Define to 1 if you have the <signal.h> header file. */ #undef HAVE_SIGNAL_H -/* Define to 1 if you have the `sigsetjmp' function. */ -#undef HAVE_SIGSETJMP - /* Define to 1 if you have the <stdint.h> header file. */ #undef HAVE_STDINT_H @@ -318,9 +312,6 @@ /* Define to 1 if you have the <sys/uio.h> header file. */ #undef HAVE_SYS_UIO_H -/* Define to 1 if you have <sys/wait.h> that is POSIX.1 compatible. */ -#undef HAVE_SYS_WAIT_H - /* Define if the setupterm() function is supported this platform. */ #undef HAVE_TERMINFO @@ -333,9 +324,6 @@ /* Define to 1 if you have the <unistd.h> header file. */ #undef HAVE_UNISTD_H -/* Define to 1 if you have the <utime.h> header file. */ -#undef HAVE_UTIME_H - /* Define to 1 if the system has the type `u_int64_t'. */ #undef HAVE_U_INT64_T diff --git a/include/llvm/DebugInfo/CodeView/CodeView.h b/include/llvm/DebugInfo/CodeView/CodeView.h new file mode 100644 index 000000000000..7728120d68de --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/CodeView.h @@ -0,0 +1,367 @@ +//===- CodeView.h -----------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_CODEVIEW_H +#define LLVM_DEBUGINFO_CODEVIEW_CODEVIEW_H + +#include <cinttypes> + +namespace llvm { +namespace codeview { + +enum class CallingConvention : uint8_t { + NearC = 0x00, // near right to left push, caller pops stack + FarC = 0x01, // far right to left push, caller pops stack + NearPascal = 0x02, // near left to right push, callee pops stack + FarPascal = 0x03, // far left to right push, callee pops stack + NearFast = 0x04, // near left to right push with regs, callee pops stack + FarFast = 0x05, // far left to right push with regs, callee pops stack + NearStdCall = 0x07, // near standard call + FarStdCall = 0x08, // far standard call + NearSysCall = 0x09, // near sys call + FarSysCall = 0x0a, // far sys call + ThisCall = 0x0b, // this call (this passed in register) + MipsCall = 0x0c, // Mips call + Generic = 0x0d, // Generic call sequence + AlphaCall = 0x0e, // Alpha call + PpcCall = 0x0f, // PPC call + SHCall = 0x10, // Hitachi SuperH call + ArmCall = 0x11, // ARM call + AM33Call = 0x12, // AM33 call + TriCall = 0x13, // TriCore Call + SH5Call = 0x14, // Hitachi SuperH-5 call + M32RCall = 0x15, // M32R Call + ClrCall = 0x16, // clr call + Inline = + 0x17, // Marker for routines always inlined and thus lacking a convention + NearVector = 0x18 // near left to right push with regs, callee pops stack +}; + +enum class ClassOptions : uint16_t { + None = 0x0000, + Packed = 0x0001, + HasConstructorOrDestructor = 0x0002, + HasOverloadedOperator = 0x0004, + Nested = 0x0008, + ContainsNestedClass = 0x0010, + HasOverloadedAssignmentOperator = 0x0020, + HasConversionOperator = 0x0040, + ForwardReference = 0x0080, + Scoped = 0x0100, + HasUniqueName = 0x0200, + Sealed = 0x0400, + Intrinsic = 0x2000 +}; + +inline ClassOptions operator|(ClassOptions a, ClassOptions b) { + return static_cast<ClassOptions>(static_cast<uint16_t>(a) | + static_cast<uint16_t>(b)); +} + +inline ClassOptions operator&(ClassOptions a, ClassOptions b) { + return static_cast<ClassOptions>(static_cast<uint16_t>(a) & + static_cast<uint16_t>(b)); +} + +inline ClassOptions operator~(ClassOptions a) { + return static_cast<ClassOptions>(~static_cast<uint16_t>(a)); +} + +enum class FrameProcedureOptions : uint32_t { + None = 0x00000000, + HasAlloca = 0x00000001, + HasSetJmp = 0x00000002, + HasLongJmp = 0x00000004, + HasInlineAssembly = 0x00000008, + HasExceptionHandling = 0x00000010, + MarkedInline = 0x00000020, + HasStructuredExceptionHandling = 0x00000040, + Naked = 0x00000080, + SecurityChecks = 0x00000100, + AsynchronousExceptionHandling = 0x00000200, + NoStackOrderingForSecurityChecks = 0x00000400, + Inlined = 0x00000800, + StrictSecurityChecks = 0x00001000, + SafeBuffers = 0x00002000, + ProfileGuidedOptimization = 0x00040000, + ValidProfileCounts = 0x00080000, + OptimizedForSpeed = 0x00100000, + GuardCfg = 0x00200000, + GuardCfw = 0x00400000 +}; + +inline FrameProcedureOptions operator|(FrameProcedureOptions a, + FrameProcedureOptions b) { + return static_cast<FrameProcedureOptions>(static_cast<uint32_t>(a) | + static_cast<uint32_t>(b)); +} + +inline FrameProcedureOptions operator&(FrameProcedureOptions a, + FrameProcedureOptions b) { + return static_cast<FrameProcedureOptions>(static_cast<uint32_t>(a) & + static_cast<uint32_t>(b)); +} + +inline FrameProcedureOptions operator~(FrameProcedureOptions a) { + return static_cast<FrameProcedureOptions>(~static_cast<uint32_t>(a)); +} + +enum class FunctionOptions : uint8_t { + None = 0x00, + CxxReturnUdt = 0x01, + Constructor = 0x02, + ConstructorWithVirtualBases = 0x04 +}; + +inline FunctionOptions operator|(FunctionOptions a, FunctionOptions b) { + return static_cast<FunctionOptions>(static_cast<uint8_t>(a) | + static_cast<uint8_t>(b)); +} + +inline FunctionOptions operator&(FunctionOptions a, FunctionOptions b) { + return static_cast<FunctionOptions>(static_cast<uint8_t>(a) & + static_cast<uint8_t>(b)); +} + +inline FunctionOptions operator~(FunctionOptions a) { + return static_cast<FunctionOptions>(~static_cast<uint8_t>(a)); +} + +enum class HfaKind : uint8_t { + None = 0x00, + Float = 0x01, + Double = 0x02, + Other = 0x03 +}; + +enum class MemberAccess : uint8_t { + None = 0, + Private = 1, + Protected = 2, + Public = 3 +}; + +enum class MethodKind : uint8_t { + Vanilla = 0x00, + Virtual = 0x01, + Static = 0x02, + Friend = 0x03, + IntroducingVirtual = 0x04, + PureVirtual = 0x05, + PureIntroducingVirtual = 0x06 +}; + +enum class MethodOptions : uint16_t { + None = 0x0000, + Pseudo = 0x0020, + CompilerGenerated = 0x0100, + Sealed = 0x0200 +}; + +inline MethodOptions operator|(MethodOptions a, MethodOptions b) { + return static_cast<MethodOptions>(static_cast<uint16_t>(a) | + static_cast<uint16_t>(b)); +} + +inline MethodOptions operator&(MethodOptions a, MethodOptions b) { + return static_cast<MethodOptions>(static_cast<uint16_t>(a) & + static_cast<uint16_t>(b)); +} + +inline MethodOptions operator~(MethodOptions a) { + return static_cast<MethodOptions>(~static_cast<uint16_t>(a)); +} + +enum class ModifierOptions : uint16_t { + None = 0x0000, + Const = 0x0001, + Volatile = 0x0002, + Unaligned = 0x0004 +}; + +inline ModifierOptions operator|(ModifierOptions a, ModifierOptions b) { + return static_cast<ModifierOptions>(static_cast<uint16_t>(a) | + static_cast<uint16_t>(b)); +} + +inline ModifierOptions operator&(ModifierOptions a, ModifierOptions b) { + return static_cast<ModifierOptions>(static_cast<uint16_t>(a) & + static_cast<uint16_t>(b)); +} + +inline ModifierOptions operator~(ModifierOptions a) { + return static_cast<ModifierOptions>(~static_cast<uint16_t>(a)); +} + +enum class ModuleSubstreamKind : uint32_t { + Symbols = 0xf1, + Lines = 0xf2, + StringTable = 0xf3, + FileChecksums = 0xf4, + FrameData = 0xf5, + InlineeLines = 0xf6, + CrossScopeImports = 0xf7, + CrossScopeExports = 0xf8 +}; + +enum class PointerKind : uint8_t { + Near16 = 0x00, // 16 bit pointer + Far16 = 0x01, // 16:16 far pointer + Huge16 = 0x02, // 16:16 huge pointer + BasedOnSegment = 0x03, // based on segment + BasedOnValue = 0x04, // based on value of base + BasedOnSegmentValue = 0x05, // based on segment value of base + BasedOnAddress = 0x06, // based on address of base + BasedOnSegmentAddress = 0x07, // based on segment address of base + BasedOnType = 0x08, // based on type + BasedOnSelf = 0x09, // based on self + Near32 = 0x0a, // 32 bit pointer + Far32 = 0x0b, // 16:32 pointer + Near64 = 0x0c // 64 bit pointer +}; + +enum class PointerMode : uint8_t { + Pointer = 0x00, // "normal" pointer + LValueReference = 0x01, // "old" reference + PointerToDataMember = 0x02, // pointer to data member + PointerToMemberFunction = 0x03, // pointer to member function + RValueReference = 0x04 // r-value reference +}; + +enum class PointerOptions : uint32_t { + None = 0x00000000, + Flat32 = 0x00000100, + Volatile = 0x00000200, + Const = 0x00000400, + Unaligned = 0x00000800, + Restrict = 0x00001000, + WinRTSmartPointer = 0x00080000 +}; + +inline PointerOptions operator|(PointerOptions a, PointerOptions b) { + return static_cast<PointerOptions>(static_cast<uint16_t>(a) | + static_cast<uint16_t>(b)); +} + +inline PointerOptions operator&(PointerOptions a, PointerOptions b) { + return static_cast<PointerOptions>(static_cast<uint16_t>(a) & + static_cast<uint16_t>(b)); +} + +inline PointerOptions operator~(PointerOptions a) { + return static_cast<PointerOptions>(~static_cast<uint16_t>(a)); +} + +enum class PointerToMemberRepresentation : uint16_t { + Unknown = 0x00, // not specified (pre VC8) + SingleInheritanceData = 0x01, // member data, single inheritance + MultipleInheritanceData = 0x02, // member data, multiple inheritance + VirtualInheritanceData = 0x03, // member data, virtual inheritance + GeneralData = 0x04, // member data, most general + SingleInheritanceFunction = 0x05, // member function, single inheritance + MultipleInheritanceFunction = 0x06, // member function, multiple inheritance + VirtualInheritanceFunction = 0x07, // member function, virtual inheritance + GeneralFunction = 0x08 // member function, most general +}; + +enum class TypeRecordKind : uint16_t { + None = 0, + + VirtualTableShape = 0x000a, + Label = 0x000e, + EndPrecompiledHeader = 0x0014, + + Modifier = 0x1001, + Pointer = 0x1002, + Procedure = 0x1008, + MemberFunction = 0x1009, + + Oem = 0x100f, + Oem2 = 0x1011, + + ArgumentList = 0x1201, + FieldList = 0x1203, + BitField = 0x1205, + MethodList = 0x1206, + + BaseClass = 0x1400, + VirtualBaseClass = 0x1401, + IndirectVirtualBaseClass = 0x1402, + Index = 0x1404, + VirtualFunctionTablePointer = 0x1409, + + Enumerate = 0x1502, + Array = 0x1503, + Class = 0x1504, + Structure = 0x1505, + Union = 0x1506, + Enum = 0x1507, + Alias = 0x150a, + Member = 0x150d, + StaticMember = 0x150e, + Method = 0x150f, + NestedType = 0x1510, + OneMethod = 0x1511, + VirtualFunctionTable = 0x151d, + + FunctionId = 0x1601, + MemberFunctionId = 0x1602, + BuildInfo = 0x1603, + SubstringList = 0x1604, + StringId = 0x1605, + UdtSourceLine = 0x1606, + + SByte = 0x8000, + Int16 = 0x8001, + UInt16 = 0x8002, + Int32 = 0x8003, + UInt32 = 0x8004, + Single = 0x8005, + Double = 0x8006, + Float80 = 0x8007, + Float128 = 0x8008, + Int64 = 0x8009, + UInt64 = 0x800a, + Float48 = 0x800b, + Complex32 = 0x800c, + Complex64 = 0x800d, + Complex80 = 0x800e, + Complex128 = 0x800f, + VarString = 0x8010, + + Int128 = 0x8017, + UInt128 = 0x8018, + + Decimal = 0x8019, + Date = 0x801a, + Utf8String = 0x801b, + + Float16 = 0x801c +}; + +enum class VirtualTableSlotKind : uint8_t { + Near16 = 0x00, + Far16 = 0x01, + This = 0x02, + Outer = 0x03, + Meta = 0x04, + Near = 0x05, + Far = 0x06 +}; + +enum class WindowsRTClassKind : uint8_t { + None = 0x00, + RefClass = 0x01, + ValueClass = 0x02, + Interface = 0x03 +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/CodeViewOStream.h b/include/llvm/DebugInfo/CodeView/CodeViewOStream.h new file mode 100644 index 000000000000..14d057a249a5 --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/CodeViewOStream.h @@ -0,0 +1,39 @@ +//===- CodeViewOStream.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_CODEVIEWOSTREAM_H +#define LLVM_DEBUGINFO_CODEVIEW_CODEVIEWOSTREAM_H + +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" + +namespace llvm { +namespace codeview { + +template <typename Writer> class CodeViewOStream { +private: + CodeViewOStream(const CodeViewOStream &) = delete; + CodeViewOStream &operator=(const CodeViewOStream &) = delete; + +public: + typedef typename Writer::LabelType LabelType; + +public: + explicit CodeViewOStream(Writer &W); + +private: + uint64_t size() const { return W.tell(); } + +private: + Writer &W; +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/FieldListRecordBuilder.h b/include/llvm/DebugInfo/CodeView/FieldListRecordBuilder.h new file mode 100644 index 000000000000..1ed62487aecc --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/FieldListRecordBuilder.h @@ -0,0 +1,78 @@ +//===- FieldListRecordBuilder.h ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_FIELDLISTRECORDBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_FIELDLISTRECORDBUILDER_H + +#include "llvm/DebugInfo/CodeView/ListRecordBuilder.h" + +namespace llvm { +namespace codeview { + +class MethodInfo { +public: + MethodInfo() : Access(), Kind(), Options(), Type(), VTableSlotOffset(-1) {} + + MethodInfo(MemberAccess Access, MethodKind Kind, MethodOptions Options, + TypeIndex Type, int32_t VTableSlotOffset) + : Access(Access), Kind(Kind), Options(Options), Type(Type), + VTableSlotOffset(VTableSlotOffset) {} + + MemberAccess getAccess() const { return Access; } + MethodKind getKind() const { return Kind; } + MethodOptions getOptions() const { return Options; } + TypeIndex getType() const { return Type; } + int32_t getVTableSlotOffset() const { return VTableSlotOffset; } + +private: + MemberAccess Access; + MethodKind Kind; + MethodOptions Options; + TypeIndex Type; + int32_t VTableSlotOffset; +}; + +class FieldListRecordBuilder : public ListRecordBuilder { +private: + FieldListRecordBuilder(const FieldListRecordBuilder &) = delete; + void operator=(const FieldListRecordBuilder &) = delete; + +public: + FieldListRecordBuilder(); + + void writeBaseClass(MemberAccess Access, TypeIndex Type, uint64_t Offset); + void writeEnumerate(MemberAccess Access, uint64_t Value, StringRef Name); + void writeIndirectVirtualBaseClass(MemberAccess Access, TypeIndex Type, + TypeIndex VirtualBasePointerType, + int64_t VirtualBasePointerOffset, + uint64_t SlotIndex); + void writeMember(MemberAccess Access, TypeIndex Type, uint64_t Offset, + StringRef Name); + void writeOneMethod(MemberAccess Access, MethodKind Kind, + MethodOptions Options, TypeIndex Type, + int32_t VTableSlotOffset, StringRef Name); + void writeOneMethod(const MethodInfo &Method, StringRef Name); + void writeMethod(uint16_t OverloadCount, TypeIndex MethodList, + StringRef Name); + void writeNestedType(TypeIndex Type, StringRef Name); + void writeStaticMember(MemberAccess Access, TypeIndex Type, StringRef Name); + void writeVirtualBaseClass(MemberAccess Access, TypeIndex Type, + TypeIndex VirtualBasePointerType, + int64_t VirtualBasePointerOffset, + uint64_t SlotIndex); + void writeVirtualBaseClass(TypeRecordKind Kind, MemberAccess Access, + TypeIndex Type, TypeIndex VirtualBasePointerType, + int64_t VirtualBasePointerOffset, + uint64_t SlotIndex); + void writeVirtualFunctionTablePointer(TypeIndex Type); +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/FunctionId.h b/include/llvm/DebugInfo/CodeView/FunctionId.h new file mode 100644 index 000000000000..1af3da810b5a --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/FunctionId.h @@ -0,0 +1,56 @@ +//===- FunctionId.h ---------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_FUNCTIONID_H +#define LLVM_DEBUGINFO_CODEVIEW_FUNCTIONID_H + +#include <cinttypes> + +namespace llvm { +namespace codeview { + +class FunctionId { +public: + FunctionId() : Index(0) {} + + explicit FunctionId(uint32_t Index) : Index(Index) {} + + uint32_t getIndex() const { return Index; } + +private: + uint32_t Index; +}; + +inline bool operator==(const FunctionId &A, const FunctionId &B) { + return A.getIndex() == B.getIndex(); +} + +inline bool operator!=(const FunctionId &A, const FunctionId &B) { + return A.getIndex() != B.getIndex(); +} + +inline bool operator<(const FunctionId &A, const FunctionId &B) { + return A.getIndex() < B.getIndex(); +} + +inline bool operator<=(const FunctionId &A, const FunctionId &B) { + return A.getIndex() <= B.getIndex(); +} + +inline bool operator>(const FunctionId &A, const FunctionId &B) { + return A.getIndex() > B.getIndex(); +} + +inline bool operator>=(const FunctionId &A, const FunctionId &B) { + return A.getIndex() >= B.getIndex(); +} +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/Line.h b/include/llvm/DebugInfo/CodeView/Line.h new file mode 100644 index 000000000000..a7cdbdaac32f --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/Line.h @@ -0,0 +1,124 @@ +//===- Line.h ---------------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_LINE_H +#define LLVM_DEBUGINFO_CODEVIEW_LINE_H + +#include <cinttypes> + +namespace llvm { +namespace codeview { + +class LineInfo { +public: + static const uint32_t AlwaysStepIntoLineNumber = 0xfeefee; + static const uint32_t NeverStepIntoLineNumber = 0xf00f00; + +private: + static const uint32_t StartLineMask = 0x00ffffff; + static const uint32_t EndLineDeltaMask = 0x7f000000; + static const int EndLineDeltaShift = 24; + static const uint32_t StatementFlag = 0x80000000u; + +public: + LineInfo(uint32_t StartLine, uint32_t EndLine, bool IsStatement); + + uint32_t getStartLine() const { return LineData & StartLineMask; } + + uint32_t getLineDelta() const { + return (LineData & EndLineDeltaMask) >> EndLineDeltaShift; + } + + uint32_t getEndLine() const { return getStartLine() + getLineDelta(); } + + bool isStatement() const { return (LineData & StatementFlag) != 0; } + + uint32_t getRawData() const { return LineData; } + + bool isAlwaysStepInto() const { + return getStartLine() == AlwaysStepIntoLineNumber; + } + + bool isNeverStepInto() const { + return getStartLine() == NeverStepIntoLineNumber; + } + +private: + uint32_t LineData; +}; + +class ColumnInfo { +private: + static const uint32_t StartColumnMask = 0x0000ffffu; + static const uint32_t EndColumnMask = 0xffff0000u; + static const int EndColumnShift = 16; + +public: + ColumnInfo(uint16_t StartColumn, uint16_t EndColumn) { + ColumnData = + (static_cast<uint32_t>(StartColumn) & StartColumnMask) | + ((static_cast<uint32_t>(EndColumn) << EndColumnShift) & EndColumnMask); + } + + uint16_t getStartColumn() const { + return static_cast<uint16_t>(ColumnData & StartColumnMask); + } + + uint16_t getEndColumn() const { + return static_cast<uint16_t>((ColumnData & EndColumnMask) >> + EndColumnShift); + } + + uint32_t getRawData() const { return ColumnData; } + +private: + uint32_t ColumnData; +}; + +class Line { +private: + int32_t CodeOffset; + LineInfo LineInf; + ColumnInfo ColumnInf; + +public: + Line(int32_t CodeOffset, uint32_t StartLine, uint32_t EndLine, + uint16_t StartColumn, uint16_t EndColumn, bool IsStatement) + : CodeOffset(CodeOffset), LineInf(StartLine, EndLine, IsStatement), + ColumnInf(StartColumn, EndColumn) {} + + Line(int32_t CodeOffset, LineInfo LineInf, ColumnInfo ColumnInf) + : CodeOffset(CodeOffset), LineInf(LineInf), ColumnInf(ColumnInf) {} + + LineInfo getLineInfo() const { return LineInf; } + + ColumnInfo getColumnInfo() const { return ColumnInf; } + + int32_t getCodeOffset() const { return CodeOffset; } + + uint32_t getStartLine() const { return LineInf.getStartLine(); } + + uint32_t getLineDelta() const { return LineInf.getLineDelta(); } + + uint32_t getEndLine() const { return LineInf.getEndLine(); } + + uint16_t getStartColumn() const { return ColumnInf.getStartColumn(); } + + uint16_t getEndColumn() const { return ColumnInf.getEndColumn(); } + + bool isStatement() const { return LineInf.isStatement(); } + + bool isAlwaysStepInto() const { return LineInf.isAlwaysStepInto(); } + + bool isNeverStepInto() const { return LineInf.isNeverStepInto(); } +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/ListRecordBuilder.h b/include/llvm/DebugInfo/CodeView/ListRecordBuilder.h new file mode 100644 index 000000000000..df0a2e08a418 --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/ListRecordBuilder.h @@ -0,0 +1,43 @@ +//===- ListRecordBuilder.h --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_LISTRECORDBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_LISTRECORDBUILDER_H + +#include "llvm/DebugInfo/CodeView/TypeRecordBuilder.h" + +namespace llvm { +namespace codeview { + +class ListRecordBuilder { +private: + ListRecordBuilder(const ListRecordBuilder &) = delete; + ListRecordBuilder &operator=(const ListRecordBuilder &) = delete; + +protected: + const int MethodKindShift = 2; + + explicit ListRecordBuilder(TypeRecordKind Kind); + +public: + llvm::StringRef str() { return Builder.str(); } + +protected: + void finishSubRecord(); + + TypeRecordBuilder &getBuilder() { return Builder; } + +private: + TypeRecordBuilder Builder; + SmallVector<size_t, 4> ContinuationOffsets; +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h b/include/llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h new file mode 100644 index 000000000000..5bfe2a068672 --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h @@ -0,0 +1,68 @@ +//===- MemoryTypeTableBuilder.h ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_MEMORYTYPETABLEBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_MEMORYTYPETABLEBUILDER_H + +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/TypeTableBuilder.h" +#include <functional> +#include <memory> +#include <unordered_map> +#include <vector> + +namespace llvm { +namespace codeview { + +class MemoryTypeTableBuilder : public TypeTableBuilder { +public: + class Record { + public: + explicit Record(llvm::StringRef RData); + + const char *data() const { return Data.get(); } + uint16_t size() const { return Size; } + + private: + uint16_t Size; + std::unique_ptr<char[]> Data; + }; + +private: + class RecordHash : std::unary_function<llvm::StringRef, size_t> { + public: + size_t operator()(llvm::StringRef Val) const { + return static_cast<size_t>(llvm::hash_value(Val)); + } + }; + +public: + MemoryTypeTableBuilder() {} + + template <typename TFunc> void ForEachRecord(TFunc Func) { + uint32_t Index = TypeIndex::FirstNonSimpleIndex; + + for (const std::unique_ptr<Record> &R : Records) { + Func(TypeIndex(Index), R.get()); + ++Index; + } + } + +private: + virtual TypeIndex writeRecord(llvm::StringRef Data) override; + +private: + std::vector<std::unique_ptr<Record>> Records; + std::unordered_map<llvm::StringRef, TypeIndex, RecordHash> HashedRecords; +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/MethodListRecordBuilder.h b/include/llvm/DebugInfo/CodeView/MethodListRecordBuilder.h new file mode 100644 index 000000000000..faa404d41b1f --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/MethodListRecordBuilder.h @@ -0,0 +1,35 @@ +//===- MethodListRecordBuilder.h --------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_METHODLISTRECORDBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_METHODLISTRECORDBUILDER_H + +#include "llvm/DebugInfo/CodeView/ListRecordBuilder.h" + +namespace llvm { +namespace codeview { + +class MethodInfo; + +class MethodListRecordBuilder : public ListRecordBuilder { +private: + MethodListRecordBuilder(const MethodListRecordBuilder &) = delete; + MethodListRecordBuilder &operator=(const MethodListRecordBuilder &) = delete; + +public: + MethodListRecordBuilder(); + + void writeMethod(MemberAccess Access, MethodKind Kind, MethodOptions Options, + TypeIndex Type, int32_t VTableSlotOffset); + void writeMethod(const MethodInfo &Method); +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/TypeIndex.h b/include/llvm/DebugInfo/CodeView/TypeIndex.h new file mode 100644 index 000000000000..d3a541be4c62 --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/TypeIndex.h @@ -0,0 +1,176 @@ +//===- TypeIndex.h ----------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPEINDEX_H +#define LLVM_DEBUGINFO_CODEVIEW_TYPEINDEX_H + +#include <cassert> +#include <cinttypes> + +namespace llvm { +namespace codeview { + +enum class SimpleTypeKind : uint32_t { + None = 0x0000, // uncharacterized type (no type) + Void = 0x0003, // void + NotTranslated = 0x0007, // type not translated by cvpack + HResult = 0x0008, // OLE/COM HRESULT + + SignedCharacter = 0x0010, // 8 bit signed + UnsignedCharacter = 0x0020, // 8 bit unsigned + NarrowCharacter = 0x0070, // really a char + WideCharacter = 0x0071, // wide char + + SByte = 0x0068, // 8 bit signed int + Byte = 0x0069, // 8 bit unsigned int + Int16Short = 0x0011, // 16 bit signed + UInt16Short = 0x0021, // 16 bit unsigned + Int16 = 0x0072, // 16 bit signed int + UInt16 = 0x0073, // 16 bit unsigned int + Int32Long = 0x0012, // 32 bit signed + UInt32Long = 0x0022, // 32 bit unsigned + Int32 = 0x0074, // 32 bit signed int + UInt32 = 0x0075, // 32 bit unsigned int + Int64Quad = 0x0013, // 64 bit signed + UInt64Quad = 0x0023, // 64 bit unsigned + Int64 = 0x0076, // 64 bit signed int + UInt64 = 0x0077, // 64 bit unsigned int + Int128 = 0x0078, // 128 bit signed int + UInt128 = 0x0079, // 128 bit unsigned int + + Float16 = 0x0046, // 16 bit real + Float32 = 0x0040, // 32 bit real + Float32PartialPrecision = 0x0045, // 32 bit PP real + Float48 = 0x0044, // 48 bit real + Float64 = 0x0041, // 64 bit real + Float80 = 0x0042, // 80 bit real + Float128 = 0x0043, // 128 bit real + + Complex32 = 0x0050, // 32 bit complex + Complex64 = 0x0051, // 64 bit complex + Complex80 = 0x0052, // 80 bit complex + Complex128 = 0x0053, // 128 bit complex + + Boolean8 = 0x0030, // 8 bit boolean + Boolean16 = 0x0031, // 16 bit boolean + Boolean32 = 0x0032, // 32 bit boolean + Boolean64 = 0x0033 // 64 bit boolean +}; + +enum class SimpleTypeMode : uint32_t { + Direct = 0x00000000, // Not a pointer + NearPointer = 0x00000100, // Near pointer + FarPointer = 0x00000200, // Far pointer + HugePointer = 0x00000300, // Huge pointer + NearPointer32 = 0x00000400, // 32 bit near pointer + FarPointer32 = 0x00000500, // 32 bit far pointer + NearPointer64 = 0x00000600, // 64 bit near pointer + NearPointer128 = 0x00000700 // 128 bit near pointer +}; + +class TypeIndex { +public: + static const uint32_t FirstNonSimpleIndex = 0x1000; + static const uint32_t SimpleKindMask = 0x000000ff; + static const uint32_t SimpleModeMask = 0x00000700; + +public: + TypeIndex() : Index(0) {} + explicit TypeIndex(uint32_t Index) : Index(Index) {} + explicit TypeIndex(SimpleTypeKind Kind) + : Index(static_cast<uint32_t>(Kind)) {} + TypeIndex(SimpleTypeKind Kind, SimpleTypeMode Mode) + : Index(static_cast<uint32_t>(Kind) | static_cast<uint32_t>(Mode)) {} + + uint32_t getIndex() const { return Index; } + bool isSimple() const { return Index < FirstNonSimpleIndex; } + + SimpleTypeKind getSimpleKind() const { + assert(isSimple()); + return static_cast<SimpleTypeKind>(Index & SimpleKindMask); + } + + SimpleTypeMode getSimpleMode() const { + assert(isSimple()); + return static_cast<SimpleTypeMode>(Index & SimpleModeMask); + } + + static TypeIndex Void() { return TypeIndex(SimpleTypeKind::Void); } + static TypeIndex VoidPointer32() { + return TypeIndex(SimpleTypeKind::Void, SimpleTypeMode::NearPointer32); + } + static TypeIndex VoidPointer64() { + return TypeIndex(SimpleTypeKind::Void, SimpleTypeMode::NearPointer64); + } + + static TypeIndex SignedCharacter() { + return TypeIndex(SimpleTypeKind::SignedCharacter); + } + static TypeIndex UnsignedCharacter() { + return TypeIndex(SimpleTypeKind::UnsignedCharacter); + } + static TypeIndex NarrowCharacter() { + return TypeIndex(SimpleTypeKind::NarrowCharacter); + } + static TypeIndex WideCharacter() { + return TypeIndex(SimpleTypeKind::WideCharacter); + } + static TypeIndex Int16Short() { + return TypeIndex(SimpleTypeKind::Int16Short); + } + static TypeIndex UInt16Short() { + return TypeIndex(SimpleTypeKind::UInt16Short); + } + static TypeIndex Int32() { return TypeIndex(SimpleTypeKind::Int32); } + static TypeIndex UInt32() { return TypeIndex(SimpleTypeKind::UInt32); } + static TypeIndex Int32Long() { return TypeIndex(SimpleTypeKind::Int32Long); } + static TypeIndex UInt32Long() { + return TypeIndex(SimpleTypeKind::UInt32Long); + } + static TypeIndex Int64() { return TypeIndex(SimpleTypeKind::Int64); } + static TypeIndex UInt64() { return TypeIndex(SimpleTypeKind::UInt64); } + static TypeIndex Int64Quad() { return TypeIndex(SimpleTypeKind::Int64Quad); } + static TypeIndex UInt64Quad() { + return TypeIndex(SimpleTypeKind::UInt64Quad); + } + + static TypeIndex Float32() { return TypeIndex(SimpleTypeKind::Float32); } + static TypeIndex Float64() { return TypeIndex(SimpleTypeKind::Float64); } + +private: + uint32_t Index; +}; + +inline bool operator==(const TypeIndex &A, const TypeIndex &B) { + return A.getIndex() == B.getIndex(); +} + +inline bool operator!=(const TypeIndex &A, const TypeIndex &B) { + return A.getIndex() != B.getIndex(); +} + +inline bool operator<(const TypeIndex &A, const TypeIndex &B) { + return A.getIndex() < B.getIndex(); +} + +inline bool operator<=(const TypeIndex &A, const TypeIndex &B) { + return A.getIndex() <= B.getIndex(); +} + +inline bool operator>(const TypeIndex &A, const TypeIndex &B) { + return A.getIndex() > B.getIndex(); +} + +inline bool operator>=(const TypeIndex &A, const TypeIndex &B) { + return A.getIndex() >= B.getIndex(); +} +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/TypeRecord.h b/include/llvm/DebugInfo/CodeView/TypeRecord.h new file mode 100644 index 000000000000..21755f5d9b09 --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/TypeRecord.h @@ -0,0 +1,270 @@ +//===- TypeRecord.h ---------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPERECORD_H +#define LLVM_DEBUGINFO_CODEVIEW_TYPERECORD_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include <cinttypes> + +namespace llvm { +namespace codeview { + +class TypeRecord { +protected: + explicit TypeRecord(TypeRecordKind Kind) : Kind(Kind) {} + +public: + TypeRecordKind getKind() const { return Kind; } + +private: + TypeRecordKind Kind; +}; + +class ModifierRecord : public TypeRecord { +public: + ModifierRecord(TypeIndex ModifiedType, ModifierOptions Options) + : TypeRecord(TypeRecordKind::Modifier), ModifiedType(ModifiedType), + Options(Options) {} + + TypeIndex getModifiedType() const { return ModifiedType; } + ModifierOptions getOptions() const { return Options; } + +private: + TypeIndex ModifiedType; + ModifierOptions Options; +}; + +class ProcedureRecord : public TypeRecord { +public: + ProcedureRecord(TypeIndex ReturnType, CallingConvention CallConv, + FunctionOptions Options, uint16_t ParameterCount, + TypeIndex ArgumentList) + : TypeRecord(TypeRecordKind::Procedure), ReturnType(ReturnType), + CallConv(CallConv), Options(Options), ParameterCount(ParameterCount), + ArgumentList(ArgumentList) {} + + TypeIndex getReturnType() const { return ReturnType; } + CallingConvention getCallConv() const { return CallConv; } + FunctionOptions getOptions() const { return Options; } + uint16_t getParameterCount() const { return ParameterCount; } + TypeIndex getArgumentList() const { return ArgumentList; } + +private: + TypeIndex ReturnType; + CallingConvention CallConv; + FunctionOptions Options; + uint16_t ParameterCount; + TypeIndex ArgumentList; +}; + +class MemberFunctionRecord : public TypeRecord { +public: + MemberFunctionRecord(TypeIndex ReturnType, TypeIndex ClassType, + TypeIndex ThisType, CallingConvention CallConv, + FunctionOptions Options, uint16_t ParameterCount, + TypeIndex ArgumentList, int32_t ThisPointerAdjustment) + : TypeRecord(TypeRecordKind::MemberFunction), ReturnType(ReturnType), + ClassType(ClassType), ThisType(ThisType), CallConv(CallConv), + Options(Options), ParameterCount(ParameterCount), + ArgumentList(ArgumentList), + ThisPointerAdjustment(ThisPointerAdjustment) {} + + TypeIndex getReturnType() const { return ReturnType; } + TypeIndex getClassType() const { return ClassType; } + TypeIndex getThisType() const { return ThisType; } + CallingConvention getCallConv() const { return CallConv; } + FunctionOptions getOptions() const { return Options; } + uint16_t getParameterCount() const { return ParameterCount; } + TypeIndex getArgumentList() const { return ArgumentList; } + int32_t getThisPointerAdjustment() const { return ThisPointerAdjustment; } + +private: + TypeIndex ReturnType; + TypeIndex ClassType; + TypeIndex ThisType; + CallingConvention CallConv; + FunctionOptions Options; + uint16_t ParameterCount; + TypeIndex ArgumentList; + int32_t ThisPointerAdjustment; +}; + +class ArgumentListRecord : public TypeRecord { +public: + explicit ArgumentListRecord(llvm::ArrayRef<TypeIndex> ArgumentTypes) + : TypeRecord(TypeRecordKind::ArgumentList), ArgumentTypes(ArgumentTypes) { + } + + llvm::ArrayRef<TypeIndex> getArgumentTypes() const { return ArgumentTypes; } + +private: + llvm::ArrayRef<TypeIndex> ArgumentTypes; +}; + +class PointerRecordBase : public TypeRecord { +public: + PointerRecordBase(TypeIndex ReferentType, PointerKind Kind, PointerMode Mode, + PointerOptions Options, uint8_t Size) + : TypeRecord(TypeRecordKind::Pointer), ReferentType(ReferentType), + PtrKind(Kind), Mode(Mode), Options(Options), Size(Size) {} + + TypeIndex getReferentType() const { return ReferentType; } + PointerKind getPointerKind() const { return PtrKind; } + PointerMode getMode() const { return Mode; } + PointerOptions getOptions() const { return Options; } + uint8_t getSize() const { return Size; } + +private: + TypeIndex ReferentType; + PointerKind PtrKind; + PointerMode Mode; + PointerOptions Options; + uint8_t Size; +}; + +class PointerRecord : public PointerRecordBase { +public: + PointerRecord(TypeIndex ReferentType, PointerKind Kind, PointerMode Mode, + PointerOptions Options, uint8_t Size) + : PointerRecordBase(ReferentType, Kind, Mode, Options, Size) {} +}; + +class PointerToMemberRecord : public PointerRecordBase { +public: + PointerToMemberRecord(TypeIndex ReferentType, PointerKind Kind, + PointerMode Mode, PointerOptions Options, uint8_t Size, + TypeIndex ContainingType, + PointerToMemberRepresentation Representation) + : PointerRecordBase(ReferentType, Kind, Mode, Options, Size), + ContainingType(ContainingType), Representation(Representation) {} + + TypeIndex getContainingType() const { return ContainingType; } + PointerToMemberRepresentation getRepresentation() const { + return Representation; + } + +private: + TypeIndex ContainingType; + PointerToMemberRepresentation Representation; +}; + +class ArrayRecord : public TypeRecord { +public: + ArrayRecord(TypeIndex ElementType, TypeIndex IndexType, uint64_t Size, + llvm::StringRef Name) + : TypeRecord(TypeRecordKind::Array), ElementType(ElementType), + IndexType(IndexType), Size(Size), Name(Name) {} + + TypeIndex getElementType() const { return ElementType; } + TypeIndex getIndexType() const { return IndexType; } + uint64_t getSize() const { return Size; } + llvm::StringRef getName() const { return Name; } + +private: + TypeIndex ElementType; + TypeIndex IndexType; + uint64_t Size; + llvm::StringRef Name; +}; + +class TagRecord : public TypeRecord { +protected: + TagRecord(TypeRecordKind Kind, uint16_t MemberCount, ClassOptions Options, + TypeIndex FieldList, StringRef Name, StringRef UniqueName) + : TypeRecord(Kind), MemberCount(MemberCount), Options(Options), + FieldList(FieldList), Name(Name), UniqueName(UniqueName) {} + +public: + uint16_t getMemberCount() const { return MemberCount; } + ClassOptions getOptions() const { return Options; } + TypeIndex getFieldList() const { return FieldList; } + StringRef getName() const { return Name; } + StringRef getUniqueName() const { return UniqueName; } + +private: + uint16_t MemberCount; + ClassOptions Options; + TypeIndex FieldList; + StringRef Name; + StringRef UniqueName; +}; + +class AggregateRecord : public TagRecord { +public: + AggregateRecord(TypeRecordKind Kind, uint16_t MemberCount, + ClassOptions Options, HfaKind Hfa, + WindowsRTClassKind WinRTKind, TypeIndex FieldList, + TypeIndex DerivationList, TypeIndex VTableShape, + uint64_t Size, StringRef Name, StringRef UniqueName) + : TagRecord(Kind, MemberCount, Options, FieldList, Name, UniqueName), + Hfa(Hfa), WinRTKind(WinRTKind), DerivationList(DerivationList), + VTableShape(VTableShape), Size(Size) {} + + HfaKind getHfa() const { return Hfa; } + WindowsRTClassKind getWinRTKind() const { return WinRTKind; } + TypeIndex getDerivationList() const { return DerivationList; } + TypeIndex getVTableShape() const { return VTableShape; } + uint64_t getSize() const { return Size; } + +private: + HfaKind Hfa; + WindowsRTClassKind WinRTKind; + TypeIndex DerivationList; + TypeIndex VTableShape; + uint64_t Size; +}; + +class EnumRecord : public TagRecord { +public: + EnumRecord(uint16_t MemberCount, ClassOptions Options, TypeIndex FieldList, + StringRef Name, StringRef UniqueName, TypeIndex UnderlyingType) + : TagRecord(TypeRecordKind::Enum, MemberCount, Options, FieldList, Name, + UniqueName), + UnderlyingType(UnderlyingType) {} + + TypeIndex getUnderlyingType() const { return UnderlyingType; } + +private: + TypeIndex UnderlyingType; +}; + +class BitFieldRecord : TypeRecord { +public: + BitFieldRecord(TypeIndex Type, uint8_t BitSize, uint8_t BitOffset) + : TypeRecord(TypeRecordKind::BitField), Type(Type), BitSize(BitSize), + BitOffset(BitOffset) {} + + TypeIndex getType() const { return Type; } + uint8_t getBitOffset() const { return BitOffset; } + uint8_t getBitSize() const { return BitSize; } + +private: + TypeIndex Type; + uint8_t BitSize; + uint8_t BitOffset; +}; + +class VirtualTableShapeRecord : TypeRecord { +public: + explicit VirtualTableShapeRecord(ArrayRef<VirtualTableSlotKind> Slots) + : TypeRecord(TypeRecordKind::VirtualTableShape), Slots(Slots) {} + + ArrayRef<VirtualTableSlotKind> getSlots() const { return Slots; } + +private: + ArrayRef<VirtualTableSlotKind> Slots; +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h b/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h new file mode 100644 index 000000000000..1f48cf70666d --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h @@ -0,0 +1,57 @@ +//===- TypeRecordBuilder.h --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPERECORDBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_TYPERECORDBUILDER_H + +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/EndianStream.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace codeview { + +class TypeRecordBuilder { +private: + TypeRecordBuilder(const TypeRecordBuilder &) = delete; + TypeRecordBuilder &operator=(const TypeRecordBuilder &) = delete; + +public: + explicit TypeRecordBuilder(TypeRecordKind Kind); + + void writeUInt8(uint8_t Value); + void writeInt16(int16_t Value); + void writeUInt16(uint16_t Value); + void writeInt32(int32_t Value); + void writeUInt32(uint32_t Value); + void writeInt64(int64_t Value); + void writeUInt64(uint64_t Value); + void writeTypeIndex(TypeIndex TypeInd); + void writeTypeRecordKind(TypeRecordKind Kind); + void writeEncodedInteger(int64_t Value); + void writeEncodedSignedInteger(int64_t Value); + void writeEncodedUnsignedInteger(uint64_t Value); + void writeNullTerminatedString(const char *Value); + void writeNullTerminatedString(StringRef Value); + + llvm::StringRef str(); + + uint64_t size() const { return Stream.tell(); } + +private: + llvm::SmallVector<char, 256> Buffer; + llvm::raw_svector_ostream Stream; + llvm::support::endian::Writer<llvm::support::endianness::little> Writer; +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h b/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h new file mode 100644 index 000000000000..9de110e8236f --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h @@ -0,0 +1,37 @@ +//===- TypeSymbolEmitter.h --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPESYMBOLEMITTER_H +#define LLVM_DEBUGINFO_CODEVIEW_TYPESYMBOLEMITTER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" + +namespace llvm { +namespace codeview { + +class TypeSymbolEmitter { +private: + TypeSymbolEmitter(const TypeSymbolEmitter &) = delete; + TypeSymbolEmitter &operator=(const TypeSymbolEmitter &) = delete; + +protected: + TypeSymbolEmitter() {} + +public: + virtual ~TypeSymbolEmitter() {} + +public: + virtual void writeUserDefinedType(TypeIndex TI, StringRef Name) = 0; +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/CodeView/TypeTableBuilder.h b/include/llvm/DebugInfo/CodeView/TypeTableBuilder.h new file mode 100644 index 000000000000..2c950e8af792 --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/TypeTableBuilder.h @@ -0,0 +1,60 @@ +//===- TypeTableBuilder.h ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPETABLEBUILDER_H +#define LLVM_DEBUGINFO_CODEVIEW_TYPETABLEBUILDER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/Support/Compiler.h" + +namespace llvm { +namespace codeview { + +class FieldListRecordBuilder; +class MethodListRecordBuilder; +class TypeRecordBuilder; + +class TypeTableBuilder { +private: + TypeTableBuilder(const TypeTableBuilder &) = delete; + TypeTableBuilder &operator=(const TypeTableBuilder &) = delete; + +protected: + TypeTableBuilder(); + +public: + virtual ~TypeTableBuilder(); + +public: + TypeIndex writeModifier(const ModifierRecord &Record); + TypeIndex writeProcedure(const ProcedureRecord &Record); + TypeIndex writeMemberFunction(const MemberFunctionRecord &Record); + TypeIndex writeArgumentList(const ArgumentListRecord &Record); + TypeIndex writeRecord(TypeRecordBuilder &builder); + TypeIndex writePointer(const PointerRecord &Record); + TypeIndex writePointerToMember(const PointerToMemberRecord &Record); + TypeIndex writeArray(const ArrayRecord &Record); + TypeIndex writeAggregate(const AggregateRecord &Record); + TypeIndex writeEnum(const EnumRecord &Record); + TypeIndex writeBitField(const BitFieldRecord &Record); + TypeIndex writeVirtualTableShape(const VirtualTableShapeRecord &Record); + + TypeIndex writeFieldList(FieldListRecordBuilder &FieldList); + TypeIndex writeMethodList(MethodListRecordBuilder &MethodList); + +private: + virtual TypeIndex writeRecord(llvm::StringRef record) = 0; +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/DIContext.h b/include/llvm/DebugInfo/DIContext.h index 871e60c56b13..6659a97a042b 100644 --- a/include/llvm/DebugInfo/DIContext.h +++ b/include/llvm/DebugInfo/DIContext.h @@ -57,6 +57,10 @@ class DIInliningInfo { assert(Index < Frames.size()); return Frames[Index]; } + DILineInfo *getMutableFrame(unsigned Index) { + assert(Index < Frames.size()); + return &Frames[Index]; + } uint32_t getNumberOfFrames() const { return Frames.size(); } @@ -65,6 +69,15 @@ class DIInliningInfo { } }; +/// DIGlobal - container for description of a global variable. +struct DIGlobal { + std::string Name; + uint64_t Start; + uint64_t Size; + + DIGlobal() : Name("<invalid>"), Start(0), Size(0) {} +}; + /// A DINameKind is passed to name search methods to specify a /// preference regarding the type of name resolution the caller wants. enum class DINameKind { None, ShortName, LinkageName }; @@ -99,6 +112,7 @@ enum DIDumpType { DIDT_LineDwo, DIDT_Loc, DIDT_LocDwo, + DIDT_Macro, DIDT_Ranges, DIDT_Pubnames, DIDT_Pubtypes, @@ -110,7 +124,9 @@ enum DIDumpType { DIDT_AppleNames, DIDT_AppleTypes, DIDT_AppleNamespaces, - DIDT_AppleObjC + DIDT_AppleObjC, + DIDT_CUIndex, + DIDT_TUIndex, }; class DIContext { @@ -140,17 +156,21 @@ private: /// to be used by the DIContext implementations when applying relocations /// on the fly. class LoadedObjectInfo { +protected: + LoadedObjectInfo(const LoadedObjectInfo &) = default; + LoadedObjectInfo() = default; + public: virtual ~LoadedObjectInfo() = default; - /// Obtain the Load Address of a section by Name. + /// Obtain the Load Address of a section by SectionRef. /// - /// Calculate the address of the section identified by the passed in Name. + /// Calculate the address of the given section. /// The section need not be present in the local address space. The addresses /// need to be consistent with the addresses used to query the DIContext and /// the output of this function should be deterministic, i.e. repeated calls with - /// the same Name should give the same address. - virtual uint64_t getSectionLoadAddress(StringRef Name) const = 0; + /// the same Sec should give the same address. + virtual uint64_t getSectionLoadAddress(const object::SectionRef &Sec) const = 0; /// If conveniently available, return the content of the given Section. /// @@ -162,7 +182,8 @@ public: /// local (unrelocated) object file and applied on the fly. Note that this method /// is used purely for optimzation purposes in the common case of JITting in the /// local address space, so returning false should always be correct. - virtual bool getLoadedSectionContents(StringRef Name, StringRef &Data) const { + virtual bool getLoadedSectionContents(const object::SectionRef &Sec, + StringRef &Data) const { return false; } diff --git a/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h b/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h index 743f9c696e9e..bae3154b3b5f 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h +++ b/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h @@ -18,10 +18,13 @@ class DWARFCompileUnit : public DWARFUnit { public: DWARFCompileUnit(DWARFContext &Context, const DWARFSection &Section, const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, - StringRef SOS, StringRef AOS, bool LE, - const DWARFUnitSectionBase &UnitSection) - : DWARFUnit(Context, Section, DA, RS, SS, SOS, AOS, LE, UnitSection) {} + StringRef SOS, StringRef AOS, StringRef LS, bool LE, + const DWARFUnitSectionBase &UnitSection, + const DWARFUnitIndex::Entry *Entry) + : DWARFUnit(Context, Section, DA, RS, SS, SOS, AOS, LS, LE, UnitSection, + Entry) {} void dump(raw_ostream &OS); + static const DWARFSectionKind Section = DW_SECT_INFO; // VTable anchor. ~DWARFCompileUnit() override; }; diff --git a/include/llvm/DebugInfo/DWARF/DWARFContext.h b/include/llvm/DebugInfo/DWARF/DWARFContext.h index 423c0d32f1b5..c91012bc9a24 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -18,6 +18,7 @@ #include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h" #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugMacro.h" #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h" #include "llvm/DebugInfo/DWARF/DWARFSection.h" #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h" @@ -40,11 +41,14 @@ class DWARFContext : public DIContext { DWARFUnitSection<DWARFCompileUnit> CUs; std::vector<DWARFUnitSection<DWARFTypeUnit>> TUs; + std::unique_ptr<DWARFUnitIndex> CUIndex; + std::unique_ptr<DWARFUnitIndex> TUIndex; std::unique_ptr<DWARFDebugAbbrev> Abbrev; std::unique_ptr<DWARFDebugLoc> Loc; std::unique_ptr<DWARFDebugAranges> Aranges; std::unique_ptr<DWARFDebugLine> Line; std::unique_ptr<DWARFDebugFrame> DebugFrame; + std::unique_ptr<DWARFDebugMacro> Macro; DWARFUnitSection<DWARFCompileUnit> DWOCUs; std::vector<DWARFUnitSection<DWARFTypeUnit>> DWOTUs; @@ -143,6 +147,9 @@ public: return DWOCUs[index].get(); } + const DWARFUnitIndex &getCUIndex(); + const DWARFUnitIndex &getTUIndex(); + /// Get a pointer to the parsed DebugAbbrev object. const DWARFDebugAbbrev *getDebugAbbrev(); @@ -161,6 +168,9 @@ public: /// Get a pointer to the parsed frame information object. const DWARFDebugFrame *getDebugFrame(); + /// Get a pointer to the parsed DebugMacro object. + const DWARFDebugMacro *getDebugMacro(); + /// Get a pointer to a parsed line table corresponding to a compile unit. const DWARFDebugLine::LineTable *getLineTableForUnit(DWARFUnit *cu); @@ -184,6 +194,7 @@ public: virtual const DWARFSection &getLineSection() = 0; virtual StringRef getStringSection() = 0; virtual StringRef getRangeSection() = 0; + virtual StringRef getMacinfoSection() = 0; virtual StringRef getPubNamesSection() = 0; virtual StringRef getPubTypesSection() = 0; virtual StringRef getGnuPubNamesSection() = 0; @@ -203,9 +214,11 @@ public: virtual const DWARFSection& getAppleTypesSection() = 0; virtual const DWARFSection& getAppleNamespacesSection() = 0; virtual const DWARFSection& getAppleObjCSection() = 0; + virtual StringRef getCUIndexSection() = 0; + virtual StringRef getTUIndexSection() = 0; static bool isSupportedVersion(unsigned version) { - return version == 2 || version == 3 || version == 4; + return version == 2 || version == 3 || version == 4 || version == 5; } private: /// Return the compile unit that includes an offset (relative to .debug_info). @@ -232,6 +245,7 @@ class DWARFContextInMemory : public DWARFContext { DWARFSection LineSection; StringRef StringSection; StringRef RangeSection; + StringRef MacinfoSection; StringRef PubNamesSection; StringRef PubTypesSection; StringRef GnuPubNamesSection; @@ -251,6 +265,8 @@ class DWARFContextInMemory : public DWARFContext { DWARFSection AppleTypesSection; DWARFSection AppleNamespacesSection; DWARFSection AppleObjCSection; + StringRef CUIndexSection; + StringRef TUIndexSection; SmallVector<SmallString<32>, 4> UncompressedSections; @@ -268,6 +284,7 @@ public: const DWARFSection &getLineSection() override { return LineSection; } StringRef getStringSection() override { return StringSection; } StringRef getRangeSection() override { return RangeSection; } + StringRef getMacinfoSection() override { return MacinfoSection; } StringRef getPubNamesSection() override { return PubNamesSection; } StringRef getPubTypesSection() override { return PubTypesSection; } StringRef getGnuPubNamesSection() override { return GnuPubNamesSection; } @@ -293,6 +310,8 @@ public: StringRef getAddrSection() override { return AddrSection; } + StringRef getCUIndexSection() override { return CUIndexSection; } + StringRef getTUIndexSection() override { return TUIndexSection; } }; } diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h index 93e7c790ccf9..760950b726b3 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h @@ -196,7 +196,7 @@ public: // Fills the Result argument with the file and line information // corresponding to Address. Returns true on success. - bool getFileLineInfoForAddress(uint64_t Address, const char *CompDir, + bool getFileLineInfoForAddress(uint64_t Address, const char *CompDir, DILineInfoSpecifier::FileLineInfoKind Kind, DILineInfo &Result) const; @@ -247,7 +247,6 @@ private: const RelocAddrMap *RelocMap; LineTableMapTy LineTableMap; }; - } #endif diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h b/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h new file mode 100644 index 000000000000..e17c993d275b --- /dev/null +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h @@ -0,0 +1,59 @@ +//===-- DWARFDebugMacro.h ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_DWARF_DWARFDEBUGMACRO_H +#define LLVM_DEBUGINFO_DWARF_DWARFDEBUGMACRO_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/DataExtractor.h" +#include "llvm/Support/Dwarf.h" + +namespace llvm { + +class raw_ostream; + +class DWARFDebugMacro { + /// A single macro entry within a macro list. + struct Entry { + /// The type of the macro entry. + uint32_t Type; + union { + /// The source line where the macro is defined. + uint64_t Line; + /// Vendor extension constant value. + uint64_t ExtConstant; + }; + + union { + /// The string (name, value) of the macro entry. + const char *MacroStr; + // An unsigned integer indicating the identity of the source file. + uint64_t File; + /// Vendor extension string. + const char *ExtStr; + }; + }; + + typedef SmallVector<Entry, 4> MacroList; + + /// A list of all the macro entries in the debug_macinfo section. + MacroList Macros; + +public: + DWARFDebugMacro() {} + /// Print the macro list found within the debug_macinfo section. + void dump(raw_ostream &OS) const; + /// Parse the debug_macinfo section accessible via the 'data' parameter. + void parse(DataExtractor data); +}; + +} + +#endif diff --git a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h index 7ddcc0d81d59..3c32a3e5b794 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h +++ b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h @@ -84,6 +84,9 @@ public: const DWARFUnit *u) const; static bool skipValue(uint16_t form, DataExtractor debug_info_data, uint32_t *offset_ptr, const DWARFUnit *u); + static bool skipValue(uint16_t form, DataExtractor debug_info_data, + uint32_t *offset_ptr, uint16_t Version, + uint8_t AddrSize); static ArrayRef<uint8_t> getFixedFormSizes(uint8_t AddrSize, uint16_t Version); diff --git a/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h b/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h index f24e27819da2..894a88dce440 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h +++ b/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h @@ -21,13 +21,17 @@ private: public: DWARFTypeUnit(DWARFContext &Context, const DWARFSection &Section, const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, - StringRef SOS, StringRef AOS, bool LE, - const DWARFUnitSectionBase &UnitSection) - : DWARFUnit(Context, Section, DA, RS, SS, SOS, AOS, LE, UnitSection) {} + StringRef SOS, StringRef AOS, StringRef LS, bool LE, + const DWARFUnitSectionBase &UnitSection, + const DWARFUnitIndex::Entry *Entry) + : DWARFUnit(Context, Section, DA, RS, SS, SOS, AOS, LS, LE, UnitSection, + Entry) {} uint32_t getHeaderSize() const override { return DWARFUnit::getHeaderSize() + 12; } void dump(raw_ostream &OS); + static const DWARFSectionKind Section = DW_SECT_TYPES; + protected: bool extractImpl(DataExtractor debug_info, uint32_t *offset_ptr) override; }; diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/include/llvm/DebugInfo/DWARF/DWARFUnit.h index 5604b93f2205..681b2aa19a79 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFUnit.h +++ b/include/llvm/DebugInfo/DWARF/DWARFUnit.h @@ -16,6 +16,7 @@ #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h" #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" #include "llvm/DebugInfo/DWARF/DWARFSection.h" +#include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h" #include <vector> namespace llvm { @@ -39,28 +40,25 @@ public: virtual DWARFUnit *getUnitForOffset(uint32_t Offset) const = 0; void parse(DWARFContext &C, const DWARFSection &Section); - void parseDWO(DWARFContext &C, const DWARFSection &DWOSection); + void parseDWO(DWARFContext &C, const DWARFSection &DWOSection, + DWARFUnitIndex *Index = nullptr); protected: virtual void parseImpl(DWARFContext &Context, const DWARFSection &Section, const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, - StringRef SOS, StringRef AOS, bool isLittleEndian) = 0; + StringRef SOS, StringRef AOS, StringRef LS, + bool isLittleEndian) = 0; ~DWARFUnitSectionBase() = default; }; +const DWARFUnitIndex &getDWARFUnitIndex(DWARFContext &Context, + DWARFSectionKind Kind); + /// Concrete instance of DWARFUnitSection, specialized for one Unit type. template<typename UnitType> class DWARFUnitSection final : public SmallVector<std::unique_ptr<UnitType>, 1>, public DWARFUnitSectionBase { - - struct UnitOffsetComparator { - bool operator()(uint32_t LHS, - const std::unique_ptr<UnitType> &RHS) const { - return LHS < RHS->getNextUnitOffset(); - } - }; - bool Parsed; public: @@ -73,8 +71,11 @@ public: typedef llvm::iterator_range<typename UnitVector::iterator> iterator_range; UnitType *getUnitForOffset(uint32_t Offset) const override { - auto *CU = std::upper_bound(this->begin(), this->end(), Offset, - UnitOffsetComparator()); + auto *CU = std::upper_bound( + this->begin(), this->end(), Offset, + [](uint32_t LHS, const std::unique_ptr<UnitType> &RHS) { + return LHS < RHS->getNextUnitOffset(); + }); if (CU != this->end()) return CU->get(); return nullptr; @@ -83,14 +84,16 @@ public: private: void parseImpl(DWARFContext &Context, const DWARFSection &Section, const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, - StringRef SOS, StringRef AOS, bool LE) override { + StringRef SOS, StringRef AOS, StringRef LS, bool LE) override { if (Parsed) return; + const auto &Index = getDWARFUnitIndex(Context, UnitType::Section); DataExtractor Data(Section.Data, LE, 0); uint32_t Offset = 0; while (Data.isValidOffset(Offset)) { auto U = llvm::make_unique<UnitType>(Context, Section, DA, RS, SS, SOS, - AOS, LE, *this); + AOS, LS, LE, *this, + Index.getFromOffset(Offset)); if (!U->extract(Data, &Offset)) break; this->push_back(std::move(U)); @@ -108,6 +111,7 @@ class DWARFUnit { const DWARFDebugAbbrev *Abbrev; StringRef RangeSection; uint32_t RangeSectionBase; + StringRef LineSection; StringRef StringSection; StringRef StringOffsetSection; StringRef AddrOffsetSection; @@ -134,6 +138,8 @@ class DWARFUnit { }; std::unique_ptr<DWOHolder> DWO; + const DWARFUnitIndex::Entry *IndexEntry; + protected: virtual bool extractImpl(DataExtractor debug_info, uint32_t *offset_ptr); /// Size in bytes of the unit header. @@ -142,13 +148,15 @@ protected: public: DWARFUnit(DWARFContext &Context, const DWARFSection &Section, const DWARFDebugAbbrev *DA, StringRef RS, StringRef SS, - StringRef SOS, StringRef AOS, bool LE, - const DWARFUnitSectionBase &UnitSection); + StringRef SOS, StringRef AOS, StringRef LS, bool LE, + const DWARFUnitSectionBase &UnitSection, + const DWARFUnitIndex::Entry *IndexEntry = nullptr); virtual ~DWARFUnit(); DWARFContext& getContext() const { return Context; } + StringRef getLineSection() const { return LineSection; } StringRef getStringSection() const { return StringSection; } StringRef getStringOffsetSection() const { return StringOffsetSection; } void setAddrOffsetSection(StringRef AOS, uint32_t Base) { @@ -246,12 +254,19 @@ public: assert(!DieArray.empty()); auto it = std::lower_bound( DieArray.begin(), DieArray.end(), Offset, - [=](const DWARFDebugInfoEntryMinimal &LHS, uint32_t Offset) { + [](const DWARFDebugInfoEntryMinimal &LHS, uint32_t Offset) { return LHS.getOffset() < Offset; }); return it == DieArray.end() ? nullptr : &*it; } + uint32_t getLineTableOffset() const { + if (IndexEntry) + if (const auto *Contrib = IndexEntry->getOffset(DW_SECT_LINE)) + return Contrib->Offset; + return 0; + } + private: /// Size in bytes of the .debug_info data associated with this compile unit. size_t getDebugInfoSize() const { return Length + 4 - getHeaderSize(); } diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h new file mode 100644 index 000000000000..a85c2f9f0a23 --- /dev/null +++ b/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h @@ -0,0 +1,81 @@ +//===-- DWARFUnitIndex.h --------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_DEBUGINFO_DWARFUNITINDEX_H +#define LLVM_LIB_DEBUGINFO_DWARFUNITINDEX_H + +#include "llvm/Support/DataExtractor.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/raw_ostream.h" +#include <cstdint> + +namespace llvm { + +enum DWARFSectionKind { + DW_SECT_INFO = 1, + DW_SECT_TYPES, + DW_SECT_ABBREV, + DW_SECT_LINE, + DW_SECT_LOC, + DW_SECT_STR_OFFSETS, + DW_SECT_MACINFO, + DW_SECT_MACRO, +}; + +class DWARFUnitIndex { + struct Header { + uint32_t Version; + uint32_t NumColumns; + uint32_t NumUnits; + uint32_t NumBuckets = 0; + + bool parse(DataExtractor IndexData, uint32_t *OffsetPtr); + void dump(raw_ostream &OS) const; + }; + +public: + class Entry { + public: + struct SectionContribution { + uint32_t Offset; + uint32_t Length; + }; + + private: + const DWARFUnitIndex *Index; + uint64_t Signature; + std::unique_ptr<SectionContribution[]> Contributions; + friend class DWARFUnitIndex; + + public: + const SectionContribution *getOffset(DWARFSectionKind Sec) const; + const SectionContribution *getOffset() const; + }; + +private: + struct Header Header; + + DWARFSectionKind InfoColumnKind; + int InfoColumn = -1; + std::unique_ptr<DWARFSectionKind[]> ColumnKinds; + std::unique_ptr<Entry[]> Rows; + + static StringRef getColumnHeader(DWARFSectionKind DS); + bool parseImpl(DataExtractor IndexData); + +public: + bool parse(DataExtractor IndexData); + DWARFUnitIndex(DWARFSectionKind InfoColumnKind) + : InfoColumnKind(InfoColumnKind) {} + void dump(raw_ostream &OS) const; + const Entry *getFromOffset(uint32_t Offset) const; +}; +} + +#endif diff --git a/include/llvm/DebugInfo/PDB/PDBContext.h b/include/llvm/DebugInfo/PDB/PDBContext.h index 2bb97463f90d..9404a5922449 100644 --- a/include/llvm/DebugInfo/PDB/PDBContext.h +++ b/include/llvm/DebugInfo/PDB/PDBContext.h @@ -32,8 +32,7 @@ class PDBContext : public DIContext { public: PDBContext(const object::COFFObjectFile &Object, - std::unique_ptr<IPDBSession> PDBSession, - bool RelativeAddress); + std::unique_ptr<IPDBSession> PDBSession); static bool classof(const DIContext *DICtx) { return DICtx->getKind() == CK_PDB; diff --git a/include/llvm/DebugInfo/PDB/PDBTypes.h b/include/llvm/DebugInfo/PDB/PDBTypes.h index 2d19e792d3d0..a932a56bb953 100644 --- a/include/llvm/DebugInfo/PDB/PDBTypes.h +++ b/include/llvm/DebugInfo/PDB/PDBTypes.h @@ -11,6 +11,7 @@ #define LLVM_DEBUGINFO_PDB_PDBTYPES_H #include "llvm/Config/llvm-config.h" +#include "llvm/Support/Endian.h" #include <functional> #include <stdint.h> @@ -500,6 +501,35 @@ struct Variant { bool operator!=(const Variant &Other) const { return !(*this == Other); } }; +namespace PDB { +static const char Magic[] = {'M', 'i', 'c', 'r', 'o', 's', 'o', 'f', + 't', ' ', 'C', '/', 'C', '+', '+', ' ', + 'M', 'S', 'F', ' ', '7', '.', '0', '0', + '\r', '\n', '\x1a', 'D', 'S', '\0', '\0', '\0'}; + +// The superblock is overlaid at the beginning of the file (offset 0). +// It starts with a magic header and is followed by information which describes +// the layout of the file system. +struct SuperBlock { + char MagicBytes[sizeof(Magic)]; + // The file system is split into a variable number of fixed size elements. + // These elements are referred to as blocks. The size of a block may vary + // from system to system. + support::ulittle32_t BlockSize; + // This field's purpose is not yet known. + support::ulittle32_t Unknown0; + // This contains the number of blocks resident in the file system. In + // practice, NumBlocks * BlockSize is equivalent to the size of the PDB file. + support::ulittle32_t NumBlocks; + // This contains the number of bytes which make up the directory. + support::ulittle32_t NumDirectoryBytes; + // This field's purpose is not yet known. + support::ulittle32_t Unknown1; + // This contains the block # of the block map. + support::ulittle32_t BlockMapAddr; +}; +} + } // namespace llvm namespace std { @@ -513,4 +543,5 @@ template <> struct hash<llvm::PDB_SymType> { }; } + #endif diff --git a/include/llvm/DebugInfo/Symbolize/DIPrinter.h b/include/llvm/DebugInfo/Symbolize/DIPrinter.h new file mode 100644 index 000000000000..0703fb14da61 --- /dev/null +++ b/include/llvm/DebugInfo/Symbolize/DIPrinter.h @@ -0,0 +1,47 @@ +//===- llvm/DebugInfo/Symbolize/DIPrinter.h ---------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares the DIPrinter class, which is responsible for printing +// structures defined in DebugInfo/DIContext.h +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_SYMBOLIZE_DIPRINTER_H +#define LLVM_DEBUGINFO_SYMBOLIZE_DIPRINTER_H + +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +struct DILineInfo; +class DIInliningInfo; +struct DIGlobal; + +namespace symbolize { + +class DIPrinter { + raw_ostream &OS; + bool PrintFunctionNames; + bool PrintPretty; + void printName(const DILineInfo &Info, bool Inlined); + +public: + DIPrinter(raw_ostream &OS, bool PrintFunctionNames = true, + bool PrintPretty = false) + : OS(OS), PrintFunctionNames(PrintFunctionNames), + PrintPretty(PrintPretty) {} + + DIPrinter &operator<<(const DILineInfo &Info); + DIPrinter &operator<<(const DIInliningInfo &Info); + DIPrinter &operator<<(const DIGlobal &Global); +}; +} +} + +#endif + diff --git a/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h b/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h new file mode 100644 index 000000000000..ff9cc808875d --- /dev/null +++ b/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h @@ -0,0 +1,53 @@ +//===-- SymbolizableModule.h ------------------------------------ C++ -----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares the SymbolizableModule interface. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZABLEMODULE_H +#define LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZABLEMODULE_H + +#include "llvm/DebugInfo/DIContext.h" +#include <memory> +#include <string> + +namespace llvm { +namespace object { +class ObjectFile; +} +} + +namespace llvm { +namespace symbolize { + +using FunctionNameKind = DILineInfoSpecifier::FunctionNameKind; + +class SymbolizableModule { +public: + virtual ~SymbolizableModule() {} + virtual DILineInfo symbolizeCode(uint64_t ModuleOffset, + FunctionNameKind FNKind, + bool UseSymbolTable) const = 0; + virtual DIInliningInfo symbolizeInlinedCode(uint64_t ModuleOffset, + FunctionNameKind FNKind, + bool UseSymbolTable) const = 0; + virtual DIGlobal symbolizeData(uint64_t ModuleOffset) const = 0; + + // Return true if this is a 32-bit x86 PE COFF module. + virtual bool isWin32Module() const = 0; + + // Returns the preferred base of the module, i.e. where the loader would place + // it in memory assuming there were no conflicts. + virtual uint64_t getModulePreferredBase() const = 0; +}; + +} // namespace symbolize +} // namespace llvm + +#endif // LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZABLEMODULE_H diff --git a/include/llvm/DebugInfo/Symbolize/Symbolize.h b/include/llvm/DebugInfo/Symbolize/Symbolize.h new file mode 100644 index 000000000000..ec3ae002659c --- /dev/null +++ b/include/llvm/DebugInfo/Symbolize/Symbolize.h @@ -0,0 +1,105 @@ +//===-- Symbolize.h --------------------------------------------- C++ -----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Header for LLVM symbolization library. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZE_H +#define LLVM_DEBUGINFO_SYMBOLIZE_SYMBOLIZE_H + +#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/ErrorOr.h" +#include <map> +#include <memory> +#include <string> + +namespace llvm { +namespace symbolize { + +using namespace object; +using FunctionNameKind = DILineInfoSpecifier::FunctionNameKind; + +class LLVMSymbolizer { +public: + struct Options { + FunctionNameKind PrintFunctions; + bool UseSymbolTable : 1; + bool Demangle : 1; + bool RelativeAddresses : 1; + std::string DefaultArch; + std::vector<std::string> DsymHints; + Options(FunctionNameKind PrintFunctions = FunctionNameKind::LinkageName, + bool UseSymbolTable = true, bool Demangle = true, + bool RelativeAddresses = false, std::string DefaultArch = "") + : PrintFunctions(PrintFunctions), UseSymbolTable(UseSymbolTable), + Demangle(Demangle), RelativeAddresses(RelativeAddresses), + DefaultArch(DefaultArch) {} + }; + + LLVMSymbolizer(const Options &Opts = Options()) : Opts(Opts) {} + ~LLVMSymbolizer() { + flush(); + } + + ErrorOr<DILineInfo> symbolizeCode(const std::string &ModuleName, + uint64_t ModuleOffset); + ErrorOr<DIInliningInfo> symbolizeInlinedCode(const std::string &ModuleName, + uint64_t ModuleOffset); + ErrorOr<DIGlobal> symbolizeData(const std::string &ModuleName, + uint64_t ModuleOffset); + void flush(); + static std::string DemangleName(const std::string &Name, + const SymbolizableModule *ModInfo); + +private: + // Bundles together object file with code/data and object file with + // corresponding debug info. These objects can be the same. + typedef std::pair<ObjectFile*, ObjectFile*> ObjectPair; + + ErrorOr<SymbolizableModule *> + getOrCreateModuleInfo(const std::string &ModuleName); + ObjectFile *lookUpDsymFile(const std::string &Path, + const MachOObjectFile *ExeObj, + const std::string &ArchName); + ObjectFile *lookUpDebuglinkObject(const std::string &Path, + const ObjectFile *Obj, + const std::string &ArchName); + + /// \brief Returns pair of pointers to object and debug object. + ErrorOr<ObjectPair> getOrCreateObjectPair(const std::string &Path, + const std::string &ArchName); + + /// \brief Return a pointer to object file at specified path, for a specified + /// architecture (e.g. if path refers to a Mach-O universal binary, only one + /// object file from it will be returned). + ErrorOr<ObjectFile *> getOrCreateObject(const std::string &Path, + const std::string &ArchName); + + std::map<std::string, ErrorOr<std::unique_ptr<SymbolizableModule>>> Modules; + + /// \brief Contains cached results of getOrCreateObjectPair(). + std::map<std::pair<std::string, std::string>, ErrorOr<ObjectPair>> + ObjectPairForPathArch; + + /// \brief Contains parsed binary for each path, or parsing error. + std::map<std::string, ErrorOr<OwningBinary<Binary>>> BinaryForPath; + + /// \brief Parsed object file for path/architecture pair, where "path" refers + /// to Mach-O universal binary. + std::map<std::pair<std::string, std::string>, ErrorOr<std::unique_ptr<ObjectFile>>> + ObjectForUBPathAndArch; + + Options Opts; +}; + +} // namespace symbolize +} // namespace llvm + +#endif diff --git a/include/llvm/ExecutionEngine/ExecutionEngine.h b/include/llvm/ExecutionEngine/ExecutionEngine.h index 821c0181ce83..a7302602dcd8 100644 --- a/include/llvm/ExecutionEngine/ExecutionEngine.h +++ b/include/llvm/ExecutionEngine/ExecutionEngine.h @@ -104,7 +104,12 @@ class ExecutionEngine { ExecutionEngineState EEState; /// The target data for the platform for which execution is being performed. - const DataLayout *DL; + /// + /// Note: the DataLayout is LLVMContext specific because it has an + /// internal cache based on type pointers. It makes unsafe to reuse the + /// ExecutionEngine across context, we don't enforce this rule but undefined + /// behavior can occurs if the user tries to do it. + const DataLayout DL; /// Whether lazy JIT compilation is enabled. bool CompilingLazily; @@ -126,8 +131,6 @@ protected: /// optimize for the case where there is only one module. SmallVector<std::unique_ptr<Module>, 1> Modules; - void setDataLayout(const DataLayout *Val) { DL = Val; } - /// getMemoryforGV - Allocate memory for a global variable. virtual char *getMemoryForGV(const GlobalVariable *GV); @@ -194,7 +197,7 @@ public: //===--------------------------------------------------------------------===// - const DataLayout *getDataLayout() const { return DL; } + const DataLayout &getDataLayout() const { return DL; } /// removeModule - Remove a Module from the list of modules. Returns true if /// M is found. @@ -478,7 +481,8 @@ public: } protected: - ExecutionEngine() {} + ExecutionEngine(const DataLayout DL) : DL(std::move(DL)){} + explicit ExecutionEngine(DataLayout DL, std::unique_ptr<Module> M); explicit ExecutionEngine(std::unique_ptr<Module> M); void emitGlobals(); @@ -488,6 +492,9 @@ protected: GenericValue getConstantValue(const Constant *C); void LoadValueFromMemory(GenericValue &Result, GenericValue *Ptr, Type *Ty); + +private: + void Init(std::unique_ptr<Module> M); }; namespace EngineKind { diff --git a/include/llvm/ExecutionEngine/Interpreter.h b/include/llvm/ExecutionEngine/Interpreter.h index f49d0c487fe9..a14707840ad8 100644 --- a/include/llvm/ExecutionEngine/Interpreter.h +++ b/include/llvm/ExecutionEngine/Interpreter.h @@ -16,22 +16,12 @@ #define LLVM_EXECUTIONENGINE_INTERPRETER_H #include "llvm/ExecutionEngine/ExecutionEngine.h" -#include <cstdlib> extern "C" void LLVMLinkInInterpreter(); namespace { struct ForceInterpreterLinking { - ForceInterpreterLinking() { - // We must reference the interpreter in such a way that compilers will not - // delete it all as dead code, even with whole program optimization, - // yet is effectively a NO-OP. As the compiler isn't smart enough - // to know that getenv() never returns -1, this will do the job. - if (std::getenv("bar") != (char*) -1) - return; - - LLVMLinkInInterpreter(); - } + ForceInterpreterLinking() { LLVMLinkInInterpreter(); } } ForceInterpreterLinking; } diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 9694b80d1928..7dab5d1bc67f 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -22,6 +22,7 @@ #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Transforms/Utils/Cloning.h" #include <list> +#include <memory> #include <set> #include "llvm/Support/Debug.h" @@ -36,56 +37,89 @@ namespace orc { /// added to the layer below. When a stub is called it triggers the extraction /// of the function body from the original module. The extracted body is then /// compiled and executed. -template <typename BaseLayerT, typename CompileCallbackMgrT, - typename PartitioningFtor = - std::function<std::set<Function*>(Function&)>> +template <typename BaseLayerT, + typename CompileCallbackMgrT = JITCompileCallbackManager, + typename IndirectStubsMgrT = IndirectStubsManager> class CompileOnDemandLayer { private: - // Utility class for MapValue. Only materializes declarations for global - // variables. - class GlobalDeclMaterializer : public ValueMaterializer { + template <typename MaterializerFtor> + class LambdaMaterializer final : public ValueMaterializer { public: - typedef std::set<const Function*> StubSet; + LambdaMaterializer(MaterializerFtor M) : M(std::move(M)) {} + Value *materializeDeclFor(Value *V) final { return M(V); } - GlobalDeclMaterializer(Module &Dst, const StubSet *StubsToClone = nullptr) - : Dst(Dst), StubsToClone(StubsToClone) {} - - Value* materializeValueFor(Value *V) final { - if (auto *GV = dyn_cast<GlobalVariable>(V)) - return cloneGlobalVariableDecl(Dst, *GV); - else if (auto *F = dyn_cast<Function>(V)) { - auto *ClonedF = cloneFunctionDecl(Dst, *F); - if (StubsToClone && StubsToClone->count(F)) { - GlobalVariable *FnBodyPtr = - createImplPointer(*ClonedF->getType(), *ClonedF->getParent(), - ClonedF->getName() + "$orc_addr", nullptr); - makeStub(*ClonedF, *FnBodyPtr); - ClonedF->setLinkage(GlobalValue::AvailableExternallyLinkage); - ClonedF->addFnAttr(Attribute::AlwaysInline); - } - return ClonedF; - } - // Else. - return nullptr; - } private: - Module &Dst; - const StubSet *StubsToClone; + MaterializerFtor M; }; + template <typename MaterializerFtor> + LambdaMaterializer<MaterializerFtor> + createLambdaMaterializer(MaterializerFtor M) { + return LambdaMaterializer<MaterializerFtor>(std::move(M)); + } + typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT; + class ModuleOwner { + public: + ModuleOwner() = default; + ModuleOwner(const ModuleOwner&) = delete; + ModuleOwner& operator=(const ModuleOwner&) = delete; + virtual ~ModuleOwner() { } + virtual Module& getModule() const = 0; + }; + + template <typename ModulePtrT> + class ModuleOwnerImpl : public ModuleOwner { + public: + ModuleOwnerImpl(ModulePtrT ModulePtr) : ModulePtr(std::move(ModulePtr)) {} + Module& getModule() const override { return *ModulePtr; } + private: + ModulePtrT ModulePtr; + }; + + template <typename ModulePtrT> + std::unique_ptr<ModuleOwner> wrapOwnership(ModulePtrT ModulePtr) { + return llvm::make_unique<ModuleOwnerImpl<ModulePtrT>>(std::move(ModulePtr)); + } + struct LogicalModuleResources { - std::shared_ptr<Module> SourceModule; + std::unique_ptr<ModuleOwner> SourceModuleOwner; std::set<const Function*> StubsToClone; + std::unique_ptr<IndirectStubsMgrT> StubsMgr; + + LogicalModuleResources() = default; + + // Explicit move constructor to make MSVC happy. + LogicalModuleResources(LogicalModuleResources &&Other) + : SourceModuleOwner(std::move(Other.SourceModuleOwner)), + StubsToClone(std::move(Other.StubsToClone)), + StubsMgr(std::move(Other.StubsMgr)) {} + + // Explicit move assignment to make MSVC happy. + LogicalModuleResources& operator=(LogicalModuleResources &&Other) { + SourceModuleOwner = std::move(Other.SourceModuleOwner); + StubsToClone = std::move(Other.StubsToClone); + StubsMgr = std::move(Other.StubsMgr); + } + + JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { + if (Name.endswith("$stub_ptr") && !ExportedSymbolsOnly) { + assert(!ExportedSymbolsOnly && "Stubs are never exported"); + return StubsMgr->findPointer(Name.drop_back(9)); + } + return StubsMgr->findStub(Name, ExportedSymbolsOnly); + } + }; + + struct LogicalDylibResources { typedef std::function<RuntimeDyld::SymbolInfo(const std::string&)> SymbolResolverFtor; SymbolResolverFtor ExternalSymbolResolver; - PartitioningFtor Partitioner; }; typedef LogicalDylib<BaseLayerT, LogicalModuleResources, @@ -95,13 +129,25 @@ private: typedef std::list<CODLogicalDylib> LogicalDylibList; public: + /// @brief Handle to a set of loaded modules. typedef typename LogicalDylibList::iterator ModuleSetHandleT; + /// @brief Module partitioning functor. + typedef std::function<std::set<Function*>(Function&)> PartitioningFtor; + + /// @brief Builder for IndirectStubsManagers. + typedef std::function<std::unique_ptr<IndirectStubsMgrT>()> + IndirectStubsManagerBuilderT; + /// @brief Construct a compile-on-demand layer instance. - CompileOnDemandLayer(BaseLayerT &BaseLayer, CompileCallbackMgrT &CallbackMgr, - bool CloneStubsIntoPartitions) - : BaseLayer(BaseLayer), CompileCallbackMgr(CallbackMgr), + CompileOnDemandLayer(BaseLayerT &BaseLayer, PartitioningFtor Partition, + CompileCallbackMgrT &CallbackMgr, + IndirectStubsManagerBuilderT CreateIndirectStubsManager, + bool CloneStubsIntoPartitions = true) + : BaseLayer(BaseLayer), Partition(Partition), + CompileCallbackMgr(CallbackMgr), + CreateIndirectStubsManager(std::move(CreateIndirectStubsManager)), CloneStubsIntoPartitions(CloneStubsIntoPartitions) {} /// @brief Add a module to the compile-on-demand layer. @@ -122,17 +168,9 @@ public: return Resolver->findSymbol(Name); }; - LDResources.Partitioner = - [](Function &F) { - std::set<Function*> Partition; - Partition.insert(&F); - return Partition; - }; - // Process each of the modules in this module set. for (auto &M : Ms) - addLogicalModule(LogicalDylibs.back(), - std::shared_ptr<Module>(std::move(M))); + addLogicalModule(LogicalDylibs.back(), std::move(M)); return std::prev(LogicalDylibs.end()); } @@ -150,6 +188,10 @@ public: /// @param ExportedSymbolsOnly If true, search only for exported symbols. /// @return A handle for the given named symbol, if it exists. JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) { + for (auto LDI = LogicalDylibs.begin(), LDE = LogicalDylibs.end(); + LDI != LDE; ++LDI) + if (auto Symbol = findSymbolIn(LDI, Name, ExportedSymbolsOnly)) + return Symbol; return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); } @@ -162,85 +204,138 @@ public: private: - void addLogicalModule(CODLogicalDylib &LD, std::shared_ptr<Module> SrcM) { + template <typename ModulePtrT> + void addLogicalModule(CODLogicalDylib &LD, ModulePtrT SrcMPtr) { // Bump the linkage and rename any anonymous/privote members in SrcM to // ensure that everything will resolve properly after we partition SrcM. - makeAllSymbolsExternallyAccessible(*SrcM); + makeAllSymbolsExternallyAccessible(*SrcMPtr); // Create a logical module handle for SrcM within the logical dylib. auto LMH = LD.createLogicalModule(); auto &LMResources = LD.getLogicalModuleResources(LMH); - LMResources.SourceModule = SrcM; - // Create the GVs-and-stubs module. - auto GVsAndStubsM = llvm::make_unique<Module>( - (SrcM->getName() + ".globals_and_stubs").str(), - SrcM->getContext()); - GVsAndStubsM->setDataLayout(SrcM->getDataLayout()); - ValueToValueMapTy VMap; + LMResources.SourceModuleOwner = wrapOwnership(std::move(SrcMPtr)); - // Process module and create stubs. - // We create the stubs before copying the global variables as we know the - // stubs won't refer to any globals (they only refer to their implementation - // pointer) so there's no ordering/value-mapping issues. - for (auto &F : *SrcM) { - - // Skip declarations. - if (F.isDeclaration()) - continue; - - // Record all functions defined by this module. - if (CloneStubsIntoPartitions) - LMResources.StubsToClone.insert(&F); - - // For each definition: create a callback, a stub, and a function body - // pointer. Initialize the function body pointer to point at the callback, - // and set the callback to compile the function body. - auto CCInfo = CompileCallbackMgr.getCompileCallback(SrcM->getContext()); - Function *StubF = cloneFunctionDecl(*GVsAndStubsM, F, &VMap); - GlobalVariable *FnBodyPtr = - createImplPointer(*StubF->getType(), *StubF->getParent(), - StubF->getName() + "$orc_addr", - createIRTypedAddress(*StubF->getFunctionType(), - CCInfo.getAddress())); - makeStub(*StubF, *FnBodyPtr); - CCInfo.setCompileAction( - [this, &LD, LMH, &F]() { + Module &SrcM = LMResources.SourceModuleOwner->getModule(); + + // Create the GlobalValues module. + const DataLayout &DL = SrcM.getDataLayout(); + auto GVsM = llvm::make_unique<Module>((SrcM.getName() + ".globals").str(), + SrcM.getContext()); + GVsM->setDataLayout(DL); + + // Create function stubs. + ValueToValueMapTy VMap; + { + typename IndirectStubsMgrT::StubInitsMap StubInits; + for (auto &F : SrcM) { + // Skip declarations. + if (F.isDeclaration()) + continue; + + // Record all functions defined by this module. + if (CloneStubsIntoPartitions) + LMResources.StubsToClone.insert(&F); + + // Create a callback, associate it with the stub for the function, + // and set the compile action to compile the partition containing the + // function. + auto CCInfo = CompileCallbackMgr.getCompileCallback(); + StubInits[mangle(F.getName(), DL)] = + std::make_pair(CCInfo.getAddress(), + JITSymbolBase::flagsFromGlobalValue(F)); + CCInfo.setCompileAction([this, &LD, LMH, &F]() { return this->extractAndCompile(LD, LMH, F); }); + } + + LMResources.StubsMgr = CreateIndirectStubsManager(); + auto EC = LMResources.StubsMgr->createStubs(StubInits); + (void)EC; + // FIXME: This should be propagated back to the user. Stub creation may + // fail for remote JITs. + assert(!EC && "Error generating stubs"); } - // Now clone the global variable declarations. - GlobalDeclMaterializer GDMat(*GVsAndStubsM); - for (auto &GV : SrcM->globals()) - if (!GV.isDeclaration()) - cloneGlobalVariableDecl(*GVsAndStubsM, GV, &VMap); + // Clone global variable decls. + for (auto &GV : SrcM.globals()) + if (!GV.isDeclaration() && !VMap.count(&GV)) + cloneGlobalVariableDecl(*GVsM, GV, &VMap); + + // And the aliases. + for (auto &A : SrcM.aliases()) + if (!VMap.count(&A)) + cloneGlobalAliasDecl(*GVsM, A, VMap); + + // Now we need to clone the GV and alias initializers. + + // Initializers may refer to functions declared (but not defined) in this + // module. Build a materializer to clone decls on demand. + auto Materializer = createLambdaMaterializer( + [this, &GVsM, &LMResources](Value *V) -> Value* { + if (auto *F = dyn_cast<Function>(V)) { + // Decls in the original module just get cloned. + if (F->isDeclaration()) + return cloneFunctionDecl(*GVsM, *F); + + // Definitions in the original module (which we have emitted stubs + // for at this point) get turned into a constant alias to the stub + // instead. + const DataLayout &DL = GVsM->getDataLayout(); + std::string FName = mangle(F->getName(), DL); + auto StubSym = LMResources.StubsMgr->findStub(FName, false); + unsigned PtrBitWidth = DL.getPointerTypeSizeInBits(F->getType()); + ConstantInt *StubAddr = + ConstantInt::get(GVsM->getContext(), + APInt(PtrBitWidth, StubSym.getAddress())); + Constant *Init = ConstantExpr::getCast(Instruction::IntToPtr, + StubAddr, F->getType()); + return GlobalAlias::create(F->getFunctionType(), + F->getType()->getAddressSpace(), + F->getLinkage(), F->getName(), + Init, GVsM.get()); + } + // else.... + return nullptr; + }); - // Then clone the initializers. - for (auto &GV : SrcM->globals()) + // Clone the global variable initializers. + for (auto &GV : SrcM.globals()) if (!GV.isDeclaration()) - moveGlobalVariableInitializer(GV, VMap, &GDMat); + moveGlobalVariableInitializer(GV, VMap, &Materializer); + + // Clone the global alias initializers. + for (auto &A : SrcM.aliases()) { + auto *NewA = cast<GlobalAlias>(VMap[&A]); + assert(NewA && "Alias not cloned?"); + Value *Init = MapValue(A.getAliasee(), VMap, RF_None, nullptr, + &Materializer); + NewA->setAliasee(cast<Constant>(Init)); + } - // Build a resolver for the stubs module and add it to the base layer. - auto GVsAndStubsResolver = createLambdaResolver( - [&LD](const std::string &Name) { + // Build a resolver for the globals module and add it to the base layer. + auto GVsResolver = createLambdaResolver( + [&LD, LMH](const std::string &Name) { + auto &LMResources = LD.getLogicalModuleResources(LMH); + if (auto Sym = LMResources.StubsMgr->findStub(Name, false)) + return RuntimeDyld::SymbolInfo(Sym.getAddress(), Sym.getFlags()); return LD.getDylibResources().ExternalSymbolResolver(Name); }, [](const std::string &Name) { return RuntimeDyld::SymbolInfo(nullptr); }); - std::vector<std::unique_ptr<Module>> GVsAndStubsMSet; - GVsAndStubsMSet.push_back(std::move(GVsAndStubsM)); - auto GVsAndStubsH = - BaseLayer.addModuleSet(std::move(GVsAndStubsMSet), + std::vector<std::unique_ptr<Module>> GVsMSet; + GVsMSet.push_back(std::move(GVsM)); + auto GVsH = + BaseLayer.addModuleSet(std::move(GVsMSet), llvm::make_unique<SectionMemoryManager>(), - std::move(GVsAndStubsResolver)); - LD.addToLogicalModule(LMH, GVsAndStubsH); + std::move(GVsResolver)); + LD.addToLogicalModule(LMH, GVsH); } - static std::string Mangle(StringRef Name, const DataLayout &DL) { + static std::string mangle(StringRef Name, const DataLayout &DL) { std::string MangledName; { raw_string_ostream MangledNameStream(MangledName); @@ -252,42 +347,35 @@ private: TargetAddress extractAndCompile(CODLogicalDylib &LD, LogicalModuleHandle LMH, Function &F) { - Module &SrcM = *LD.getLogicalModuleResources(LMH).SourceModule; + auto &LMResources = LD.getLogicalModuleResources(LMH); + Module &SrcM = LMResources.SourceModuleOwner->getModule(); // If F is a declaration we must already have compiled it. if (F.isDeclaration()) return 0; // Grab the name of the function being called here. - std::string CalledFnName = Mangle(F.getName(), SrcM.getDataLayout()); + std::string CalledFnName = mangle(F.getName(), SrcM.getDataLayout()); - auto Partition = LD.getDylibResources().Partitioner(F); - auto PartitionH = emitPartition(LD, LMH, Partition); + auto Part = Partition(F); + auto PartH = emitPartition(LD, LMH, Part); TargetAddress CalledAddr = 0; - for (auto *SubF : Partition) { - std::string FName = SubF->getName(); - auto FnBodySym = - BaseLayer.findSymbolIn(PartitionH, Mangle(FName, SrcM.getDataLayout()), - false); - auto FnPtrSym = - BaseLayer.findSymbolIn(*LD.moduleHandlesBegin(LMH), - Mangle(FName + "$orc_addr", - SrcM.getDataLayout()), - false); + for (auto *SubF : Part) { + std::string FnName = mangle(SubF->getName(), SrcM.getDataLayout()); + auto FnBodySym = BaseLayer.findSymbolIn(PartH, FnName, false); assert(FnBodySym && "Couldn't find function body."); - assert(FnPtrSym && "Couldn't find function body pointer."); TargetAddress FnBodyAddr = FnBodySym.getAddress(); - void *FnPtrAddr = reinterpret_cast<void*>( - static_cast<uintptr_t>(FnPtrSym.getAddress())); // If this is the function we're calling record the address so we can // return it from this function. if (SubF == &F) CalledAddr = FnBodyAddr; - memcpy(FnPtrAddr, &FnBodyAddr, sizeof(uintptr_t)); + // Update the function body pointer for the stub. + if (auto EC = LMResources.StubsMgr->updatePointer(FnName, FnBodyAddr)) + return 0; } return CalledAddr; @@ -296,13 +384,13 @@ private: template <typename PartitionT> BaseLayerModuleSetHandleT emitPartition(CODLogicalDylib &LD, LogicalModuleHandle LMH, - const PartitionT &Partition) { + const PartitionT &Part) { auto &LMResources = LD.getLogicalModuleResources(LMH); - Module &SrcM = *LMResources.SourceModule; + Module &SrcM = LMResources.SourceModuleOwner->getModule(); // Create the module. std::string NewName = SrcM.getName(); - for (auto *F : Partition) { + for (auto *F : Part) { NewName += "."; NewName += F->getName(); } @@ -310,15 +398,51 @@ private: auto M = llvm::make_unique<Module>(NewName, SrcM.getContext()); M->setDataLayout(SrcM.getDataLayout()); ValueToValueMapTy VMap; - GlobalDeclMaterializer GDM(*M, &LMResources.StubsToClone); + + auto Materializer = createLambdaMaterializer([this, &LMResources, &M, + &VMap](Value *V) -> Value * { + if (auto *GV = dyn_cast<GlobalVariable>(V)) + return cloneGlobalVariableDecl(*M, *GV); + + if (auto *F = dyn_cast<Function>(V)) { + // Check whether we want to clone an available_externally definition. + if (!LMResources.StubsToClone.count(F)) + return cloneFunctionDecl(*M, *F); + + // Ok - we want an inlinable stub. For that to work we need a decl + // for the stub pointer. + auto *StubPtr = createImplPointer(*F->getType(), *M, + F->getName() + "$stub_ptr", nullptr); + auto *ClonedF = cloneFunctionDecl(*M, *F); + makeStub(*ClonedF, *StubPtr); + ClonedF->setLinkage(GlobalValue::AvailableExternallyLinkage); + ClonedF->addFnAttr(Attribute::AlwaysInline); + return ClonedF; + } + + if (auto *A = dyn_cast<GlobalAlias>(V)) { + auto *Ty = A->getValueType(); + if (Ty->isFunctionTy()) + return Function::Create(cast<FunctionType>(Ty), + GlobalValue::ExternalLinkage, A->getName(), + M.get()); + + return new GlobalVariable(*M, Ty, false, GlobalValue::ExternalLinkage, + nullptr, A->getName(), nullptr, + GlobalValue::NotThreadLocal, + A->getType()->getAddressSpace()); + } + + return nullptr; + }); // Create decls in the new module. - for (auto *F : Partition) + for (auto *F : Part) cloneFunctionDecl(*M, *F, &VMap); // Move the function bodies. - for (auto *F : Partition) - moveFunctionBody(*F, VMap, &GDM); + for (auto *F : Part) + moveFunctionBody(*F, VMap, &Materializer); // Create memory manager and symbol resolver. auto MemMgr = llvm::make_unique<SectionMemoryManager>(); @@ -342,7 +466,10 @@ private: } BaseLayerT &BaseLayer; + PartitioningFtor Partition; CompileCallbackMgrT &CompileCallbackMgr; + IndirectStubsManagerBuilderT CreateIndirectStubsManager; + LogicalDylibList LogicalDylibs; bool CloneStubsIntoPartitions; }; diff --git a/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/include/llvm/ExecutionEngine/Orc/CompileUtils.h index 49a1fbadb295..1e7d211196f5 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileUtils.h +++ b/include/llvm/ExecutionEngine/Orc/CompileUtils.h @@ -40,7 +40,6 @@ public: if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) llvm_unreachable("Target does not support MC emission."); PM.run(M); - ObjStream.flush(); std::unique_ptr<MemoryBuffer> ObjBuffer( new ObjectMemoryBuffer(std::move(ObjBufferSV))); ErrorOr<std::unique_ptr<object::ObjectFile>> Obj = diff --git a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h new file mode 100644 index 000000000000..9fa222c340f8 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h @@ -0,0 +1,108 @@ +//===---- GlobalMappingLayer.h - Run all IR through a functor ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Convenience layer for injecting symbols that will appear in calls to +// findSymbol. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_GLOBALMAPPINGLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_GLOBALMAPPINGLAYER_H + +#include "JITSymbol.h" +#include <map> + +namespace llvm { +namespace orc { + +/// @brief Global mapping layer. +/// +/// This layer overrides the findSymbol method to first search a local symbol +/// table that the client can define. It can be used to inject new symbol +/// mappings into the JIT. Beware, however: symbols within a single IR module or +/// object file will still resolve locally (via RuntimeDyld's symbol table) - +/// such internal references cannot be overriden via this layer. +template <typename BaseLayerT> +class GlobalMappingLayer { +public: + /// @brief Handle to a set of added modules. + typedef typename BaseLayerT::ModuleSetHandleT ModuleSetHandleT; + + /// @brief Construct an GlobalMappingLayer with the given BaseLayer + GlobalMappingLayer(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {} + + /// @brief Add the given module set to the JIT. + /// @return A handle for the added modules. + template <typename ModuleSetT, typename MemoryManagerPtrT, + typename SymbolResolverPtrT> + ModuleSetHandleT addModuleSet(ModuleSetT Ms, + MemoryManagerPtrT MemMgr, + SymbolResolverPtrT Resolver) { + return BaseLayer.addModuleSet(std::move(Ms), std::move(MemMgr), + std::move(Resolver)); + } + + /// @brief Remove the module set associated with the handle H. + void removeModuleSet(ModuleSetHandleT H) { BaseLayer.removeModuleSet(H); } + + /// @brief Manually set the address to return for the given symbol. + void setGlobalMapping(const std::string &Name, TargetAddress Addr) { + SymbolTable[Name] = Addr; + } + + /// @brief Remove the given symbol from the global mapping. + void eraseGlobalMapping(const std::string &Name) { + SymbolTable.erase(Name); + } + + /// @brief Search for the given named symbol. + /// + /// This method will first search the local symbol table, returning + /// any symbol found there. If the symbol is not found in the local + /// table then this call will be passed through to the base layer. + /// + /// @param Name The name of the symbol to search for. + /// @param ExportedSymbolsOnly If true, search only for exported symbols. + /// @return A handle for the given named symbol, if it exists. + JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) { + auto I = SymbolTable.find(Name); + if (I != SymbolTable.end()) + return JITSymbol(I->second, JITSymbolFlags::Exported); + return BaseLayer.findSymbol(Name, ExportedSymbolsOnly); + } + + /// @brief Get the address of the given symbol in the context of the set of + /// modules represented by the handle H. This call is forwarded to the + /// base layer's implementation. + /// @param H The handle for the module set to search in. + /// @param Name The name of the symbol to search for. + /// @param ExportedSymbolsOnly If true, search only for exported symbols. + /// @return A handle for the given named symbol, if it is found in the + /// given module set. + JITSymbol findSymbolIn(ModuleSetHandleT H, const std::string &Name, + bool ExportedSymbolsOnly) { + return BaseLayer.findSymbolIn(H, Name, ExportedSymbolsOnly); + } + + /// @brief Immediately emit and finalize the module set represented by the + /// given handle. + /// @param H Handle for module set to emit/finalize. + void emitAndFinalize(ModuleSetHandleT H) { + BaseLayer.emitAndFinalize(H); + } + +private: + BaseLayerT &BaseLayer; + std::map<std::string, TargetAddress> SymbolTable; +}; + +} // End namespace orc. +} // End namespace llvm. + +#endif // LLVM_EXECUTIONENGINE_ORC_GLOBALMAPPINGLAYER_H diff --git a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h index 637902200786..e4bed95fdabf 100644 --- a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h @@ -85,8 +85,6 @@ public: ModuleSetHandleT H = BaseLayer.addObjectSet(Objects, std::move(MemMgr), std::move(Resolver)); - BaseLayer.takeOwnershipOfBuffers(H, std::move(Buffers)); - return H; } diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index 4b7fc5e84b9c..d6ee3a846b04 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -27,9 +27,8 @@ namespace llvm { namespace orc { -/// @brief Base class for JITLayer independent aspects of -/// JITCompileCallbackManager. -class JITCompileCallbackManagerBase { +/// @brief Target-independent base class for compile callback management. +class JITCompileCallbackManager { public: typedef std::function<TargetAddress()> CompileFtor; @@ -51,18 +50,13 @@ public: CompileFtor &Compile; }; - /// @brief Construct a JITCompileCallbackManagerBase. + /// @brief Construct a JITCompileCallbackManager. /// @param ErrorHandlerAddress The address of an error handler in the target /// process to be used if a compile callback fails. - /// @param NumTrampolinesPerBlock Number of trampolines to emit if there is no - /// available trampoline when getCompileCallback is - /// called. - JITCompileCallbackManagerBase(TargetAddress ErrorHandlerAddress, - unsigned NumTrampolinesPerBlock) - : ErrorHandlerAddress(ErrorHandlerAddress), - NumTrampolinesPerBlock(NumTrampolinesPerBlock) {} + JITCompileCallbackManager(TargetAddress ErrorHandlerAddress) + : ErrorHandlerAddress(ErrorHandlerAddress) {} - virtual ~JITCompileCallbackManagerBase() {} + virtual ~JITCompileCallbackManager() {} /// @brief Execute the callback for the given trampoline id. Called by the JIT /// to compile functions on demand. @@ -90,7 +84,11 @@ public: } /// @brief Reserve a compile callback. - virtual CompileCallbackInfo getCompileCallback(LLVMContext &Context) = 0; + CompileCallbackInfo getCompileCallback() { + TargetAddress TrampolineAddr = getAvailableTrampolineAddr(); + auto &Compile = this->ActiveTrampolines[TrampolineAddr]; + return CompileCallbackInfo(TrampolineAddr, Compile); + } /// @brief Get a CompileCallbackInfo for an existing callback. CompileCallbackInfo getCompileCallbackInfo(TargetAddress TrampolineAddr) { @@ -113,113 +111,229 @@ public: protected: TargetAddress ErrorHandlerAddress; - unsigned NumTrampolinesPerBlock; typedef std::map<TargetAddress, CompileFtor> TrampolineMapT; TrampolineMapT ActiveTrampolines; std::vector<TargetAddress> AvailableTrampolines; + +private: + + TargetAddress getAvailableTrampolineAddr() { + if (this->AvailableTrampolines.empty()) + grow(); + assert(!this->AvailableTrampolines.empty() && + "Failed to grow available trampolines."); + TargetAddress TrampolineAddr = this->AvailableTrampolines.back(); + this->AvailableTrampolines.pop_back(); + return TrampolineAddr; + } + + // Create new trampolines - to be implemented in subclasses. + virtual void grow() = 0; + + virtual void anchor(); }; -/// @brief Manage compile callbacks. -template <typename JITLayerT, typename TargetT> -class JITCompileCallbackManager : public JITCompileCallbackManagerBase { +/// @brief Manage compile callbacks for in-process JITs. +template <typename TargetT> +class LocalJITCompileCallbackManager : public JITCompileCallbackManager { public: - /// @brief Construct a JITCompileCallbackManager. - /// @param JIT JIT layer to emit callback trampolines, etc. into. - /// @param Context LLVMContext to use for trampoline & resolve block modules. + /// @brief Construct a InProcessJITCompileCallbackManager. /// @param ErrorHandlerAddress The address of an error handler in the target /// process to be used if a compile callback fails. - /// @param NumTrampolinesPerBlock Number of trampolines to allocate whenever - /// there is no existing callback trampoline. - /// (Trampolines are allocated in blocks for - /// efficiency.) - JITCompileCallbackManager(JITLayerT &JIT, RuntimeDyld::MemoryManager &MemMgr, - LLVMContext &Context, - TargetAddress ErrorHandlerAddress, - unsigned NumTrampolinesPerBlock) - : JITCompileCallbackManagerBase(ErrorHandlerAddress, - NumTrampolinesPerBlock), - JIT(JIT), MemMgr(MemMgr) { - emitResolverBlock(Context); + LocalJITCompileCallbackManager(TargetAddress ErrorHandlerAddress) + : JITCompileCallbackManager(ErrorHandlerAddress) { + + /// Set up the resolver block. + std::error_code EC; + ResolverBlock = + sys::OwningMemoryBlock( + sys::Memory::allocateMappedMemory(TargetT::ResolverCodeSize, nullptr, + sys::Memory::MF_READ | + sys::Memory::MF_WRITE, EC)); + assert(!EC && "Failed to allocate resolver block"); + + TargetT::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()), + &reenter, this); + + EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), + sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + assert(!EC && "Failed to mprotect resolver block"); } - /// @brief Get/create a compile callback with the given signature. - CompileCallbackInfo getCompileCallback(LLVMContext &Context) final { - TargetAddress TrampolineAddr = getAvailableTrampolineAddr(Context); - auto &Compile = this->ActiveTrampolines[TrampolineAddr]; - return CompileCallbackInfo(TrampolineAddr, Compile); +private: + + static TargetAddress reenter(void *CCMgr, void *TrampolineId) { + JITCompileCallbackManager *Mgr = + static_cast<JITCompileCallbackManager*>(CCMgr); + return Mgr->executeCompileCallback( + static_cast<TargetAddress>( + reinterpret_cast<uintptr_t>(TrampolineId))); + } + + void grow() override { + assert(this->AvailableTrampolines.empty() && "Growing prematurely?"); + + std::error_code EC; + auto TrampolineBlock = + sys::OwningMemoryBlock( + sys::Memory::allocateMappedMemory(TargetT::PageSize, nullptr, + sys::Memory::MF_READ | + sys::Memory::MF_WRITE, EC)); + assert(!EC && "Failed to allocate trampoline block"); + + + unsigned NumTrampolines = + (TargetT::PageSize - TargetT::PointerSize) / TargetT::TrampolineSize; + + uint8_t *TrampolineMem = static_cast<uint8_t*>(TrampolineBlock.base()); + TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(), + NumTrampolines); + + for (unsigned I = 0; I < NumTrampolines; ++I) + this->AvailableTrampolines.push_back( + static_cast<TargetAddress>(reinterpret_cast<uintptr_t>( + TrampolineMem + (I * TargetT::TrampolineSize)))); + + EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(), + sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + assert(!EC && "Failed to mprotect trampoline block"); + + TrampolineBlocks.push_back(std::move(TrampolineBlock)); } + sys::OwningMemoryBlock ResolverBlock; + std::vector<sys::OwningMemoryBlock> TrampolineBlocks; +}; + +/// @brief Base class for managing collections of named indirect stubs. +class IndirectStubsManager { +public: + + /// @brief Map type for initializing the manager. See init. + typedef StringMap<std::pair<TargetAddress, JITSymbolFlags>> StubInitsMap; + + virtual ~IndirectStubsManager() {} + + /// @brief Create a single stub with the given name, target address and flags. + virtual std::error_code createStub(StringRef StubName, TargetAddress StubAddr, + JITSymbolFlags StubFlags) = 0; + + /// @brief Create StubInits.size() stubs with the given names, target + /// addresses, and flags. + virtual std::error_code createStubs(const StubInitsMap &StubInits) = 0; + + /// @brief Find the stub with the given name. If ExportedStubsOnly is true, + /// this will only return a result if the stub's flags indicate that it + /// is exported. + virtual JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0; + + /// @brief Find the implementation-pointer for the stub. + virtual JITSymbol findPointer(StringRef Name) = 0; + + /// @brief Change the value of the implementation pointer for the stub. + virtual std::error_code updatePointer(StringRef Name, TargetAddress NewAddr) = 0; private: + virtual void anchor(); +}; + +/// @brief IndirectStubsManager implementation for a concrete target, e.g. +/// OrcX86_64. (See OrcTargetSupport.h). +template <typename TargetT> +class LocalIndirectStubsManager : public IndirectStubsManager { +public: + + std::error_code createStub(StringRef StubName, TargetAddress StubAddr, + JITSymbolFlags StubFlags) override { + if (auto EC = reserveStubs(1)) + return EC; - std::vector<std::unique_ptr<Module>> - SingletonSet(std::unique_ptr<Module> M) { - std::vector<std::unique_ptr<Module>> Ms; - Ms.push_back(std::move(M)); - return Ms; + createStubInternal(StubName, StubAddr, StubFlags); + + return std::error_code(); } - void emitResolverBlock(LLVMContext &Context) { - std::unique_ptr<Module> M(new Module("resolver_block_module", - Context)); - TargetT::insertResolverBlock(*M, *this); - auto NonResolver = - createLambdaResolver( - [](const std::string &Name) -> RuntimeDyld::SymbolInfo { - llvm_unreachable("External symbols in resolver block?"); - }, - [](const std::string &Name) -> RuntimeDyld::SymbolInfo { - llvm_unreachable("Dylib symbols in resolver block?"); - }); - auto H = JIT.addModuleSet(SingletonSet(std::move(M)), &MemMgr, - std::move(NonResolver)); - JIT.emitAndFinalize(H); - auto ResolverBlockSymbol = - JIT.findSymbolIn(H, TargetT::ResolverBlockName, false); - assert(ResolverBlockSymbol && "Failed to insert resolver block"); - ResolverBlockAddr = ResolverBlockSymbol.getAddress(); + std::error_code createStubs(const StubInitsMap &StubInits) override { + if (auto EC = reserveStubs(StubInits.size())) + return EC; + + for (auto &Entry : StubInits) + createStubInternal(Entry.first(), Entry.second.first, + Entry.second.second); + + return std::error_code(); } - TargetAddress getAvailableTrampolineAddr(LLVMContext &Context) { - if (this->AvailableTrampolines.empty()) - grow(Context); - assert(!this->AvailableTrampolines.empty() && - "Failed to grow available trampolines."); - TargetAddress TrampolineAddr = this->AvailableTrampolines.back(); - this->AvailableTrampolines.pop_back(); - return TrampolineAddr; + JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { + auto I = StubIndexes.find(Name); + if (I == StubIndexes.end()) + return nullptr; + auto Key = I->second.first; + void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second); + assert(StubAddr && "Missing stub address"); + auto StubTargetAddr = + static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(StubAddr)); + auto StubSymbol = JITSymbol(StubTargetAddr, I->second.second); + if (ExportedStubsOnly && !StubSymbol.isExported()) + return nullptr; + return StubSymbol; } - void grow(LLVMContext &Context) { - assert(this->AvailableTrampolines.empty() && "Growing prematurely?"); - std::unique_ptr<Module> M(new Module("trampoline_block", Context)); - auto GetLabelName = - TargetT::insertCompileCallbackTrampolines(*M, ResolverBlockAddr, - this->NumTrampolinesPerBlock, - this->ActiveTrampolines.size()); - auto NonResolver = - createLambdaResolver( - [](const std::string &Name) -> RuntimeDyld::SymbolInfo { - llvm_unreachable("External symbols in trampoline block?"); - }, - [](const std::string &Name) -> RuntimeDyld::SymbolInfo { - llvm_unreachable("Dylib symbols in trampoline block?"); - }); - auto H = JIT.addModuleSet(SingletonSet(std::move(M)), &MemMgr, - std::move(NonResolver)); - JIT.emitAndFinalize(H); - for (unsigned I = 0; I < this->NumTrampolinesPerBlock; ++I) { - std::string Name = GetLabelName(I); - auto TrampolineSymbol = JIT.findSymbolIn(H, Name, false); - assert(TrampolineSymbol && "Failed to emit trampoline."); - this->AvailableTrampolines.push_back(TrampolineSymbol.getAddress()); - } + JITSymbol findPointer(StringRef Name) override { + auto I = StubIndexes.find(Name); + if (I == StubIndexes.end()) + return nullptr; + auto Key = I->second.first; + void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second); + assert(PtrAddr && "Missing pointer address"); + auto PtrTargetAddr = + static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr)); + return JITSymbol(PtrTargetAddr, I->second.second); + } + + std::error_code updatePointer(StringRef Name, TargetAddress NewAddr) override { + auto I = StubIndexes.find(Name); + assert(I != StubIndexes.end() && "No stub pointer for symbol"); + auto Key = I->second.first; + *IndirectStubsInfos[Key.first].getPtr(Key.second) = + reinterpret_cast<void*>(static_cast<uintptr_t>(NewAddr)); + return std::error_code(); + } + +private: + + std::error_code reserveStubs(unsigned NumStubs) { + if (NumStubs <= FreeStubs.size()) + return std::error_code(); + + unsigned NewStubsRequired = NumStubs - FreeStubs.size(); + unsigned NewBlockId = IndirectStubsInfos.size(); + typename TargetT::IndirectStubsInfo ISI; + if (auto EC = TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, + nullptr)) + return EC; + for (unsigned I = 0; I < ISI.getNumStubs(); ++I) + FreeStubs.push_back(std::make_pair(NewBlockId, I)); + IndirectStubsInfos.push_back(std::move(ISI)); + return std::error_code(); + } + + void createStubInternal(StringRef StubName, TargetAddress InitAddr, + JITSymbolFlags StubFlags) { + auto Key = FreeStubs.back(); + FreeStubs.pop_back(); + *IndirectStubsInfos[Key.first].getPtr(Key.second) = + reinterpret_cast<void*>(static_cast<uintptr_t>(InitAddr)); + StubIndexes[StubName] = std::make_pair(Key, StubFlags); } - JITLayerT &JIT; - RuntimeDyld::MemoryManager &MemMgr; - TargetAddress ResolverBlockAddr; + std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos; + typedef std::pair<uint16_t, uint16_t> StubKey; + std::vector<StubKey> FreeStubs; + StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; }; /// @brief Build a function pointer of FunctionType with the given constant @@ -236,7 +350,7 @@ GlobalVariable* createImplPointer(PointerType &PT, Module &M, /// @brief Turn a function declaration into a stub function that makes an /// indirect call using the given function pointer. -void makeStub(Function &F, GlobalVariable &ImplPointer); +void makeStub(Function &F, Value &ImplPointer); /// @brief Raise linkage types and rename as necessary to ensure that all /// symbols are accessible for other modules. @@ -289,6 +403,10 @@ void moveGlobalVariableInitializer(GlobalVariable &OrigGV, ValueMaterializer *Materializer = nullptr, GlobalVariable *NewGV = nullptr); +/// @brief Clone +GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, + ValueToValueMapTy &VMap); + } // End namespace orc. } // End namespace llvm. diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h index 93ba02b38706..a5286ff9adde 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h @@ -67,10 +67,10 @@ private: } else return nullptr; case Emitting: - // Calling "emit" can trigger external symbol lookup (e.g. to check for - // pre-existing definitions of common-symbol), but it will never find in - // this module that it would not have found already, so return null from - // here. + // Calling "emit" can trigger a recursive call to 'find' (e.g. to check + // for pre-existing definitions of common-symbol), but any symbol in + // this module would already have been found internally (in the + // RuntimeDyld that did the lookup), so just return a nullptr here. return nullptr; case Emitted: return B.findSymbolIn(Handle, Name, ExportedSymbolsOnly); diff --git a/include/llvm/ExecutionEngine/Orc/LogicalDylib.h b/include/llvm/ExecutionEngine/Orc/LogicalDylib.h index 28700ef347d6..883fa9eac560 100644 --- a/include/llvm/ExecutionEngine/Orc/LogicalDylib.h +++ b/include/llvm/ExecutionEngine/Orc/LogicalDylib.h @@ -14,6 +14,10 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_LOGICALDYLIB_H #define LLVM_EXECUTIONENGINE_ORC_LOGICALDYLIB_H +#include "llvm/ExecutionEngine/Orc/JITSymbol.h" +#include <string> +#include <vector> + namespace llvm { namespace orc { @@ -28,6 +32,12 @@ private: typedef std::vector<BaseLayerModuleSetHandleT> BaseLayerHandleList; struct LogicalModule { + // Make this move-only to ensure they don't get duplicated across moves of + // LogicalDylib or anything like that. + LogicalModule(LogicalModule &&RHS) + : Resources(std::move(RHS.Resources)), + BaseLayerHandles(std::move(RHS.BaseLayerHandles)) {} + LogicalModule() = default; LogicalModuleResources Resources; BaseLayerHandleList BaseLayerHandles; }; @@ -46,6 +56,13 @@ public: BaseLayer.removeModuleSet(BLH); } + // If possible, remove this and ~LogicalDylib once the work in the dtor is + // moved to members (eg: self-unregistering base layer handles). + LogicalDylib(LogicalDylib &&RHS) + : BaseLayer(std::move(RHS.BaseLayer)), + LogicalModules(std::move(RHS.LogicalModules)), + DylibResources(std::move(RHS.DylibResources)) {} + LogicalModuleHandle createLogicalModule() { LogicalModules.push_back(LogicalModule()); return std::prev(LogicalModules.end()); @@ -69,22 +86,27 @@ public: } JITSymbol findSymbolInLogicalModule(LogicalModuleHandle LMH, - const std::string &Name) { + const std::string &Name, + bool ExportedSymbolsOnly) { + + if (auto StubSym = LMH->Resources.findSymbol(Name, ExportedSymbolsOnly)) + return StubSym; + for (auto BLH : LMH->BaseLayerHandles) - if (auto Symbol = BaseLayer.findSymbolIn(BLH, Name, false)) + if (auto Symbol = BaseLayer.findSymbolIn(BLH, Name, ExportedSymbolsOnly)) return Symbol; return nullptr; } JITSymbol findSymbolInternally(LogicalModuleHandle LMH, const std::string &Name) { - if (auto Symbol = findSymbolInLogicalModule(LMH, Name)) + if (auto Symbol = findSymbolInLogicalModule(LMH, Name, false)) return Symbol; for (auto LMI = LogicalModules.begin(), LME = LogicalModules.end(); LMI != LME; ++LMI) { if (LMI != LMH) - if (auto Symbol = findSymbolInLogicalModule(LMI, Name)) + if (auto Symbol = findSymbolInLogicalModule(LMI, Name, false)) return Symbol; } @@ -92,11 +114,10 @@ public: } JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) { - for (auto &LM : LogicalModules) - for (auto BLH : LM.BaseLayerHandles) - if (auto Symbol = - BaseLayer.findSymbolIn(BLH, Name, ExportedSymbolsOnly)) - return Symbol; + for (auto LMI = LogicalModules.begin(), LME = LogicalModules.end(); + LMI != LME; ++LMI) + if (auto Sym = findSymbolInLogicalModule(LMI, Name, ExportedSymbolsOnly)) + return Sym; return nullptr; } @@ -106,7 +127,6 @@ protected: BaseLayerT BaseLayer; LogicalModuleList LogicalModules; LogicalDylibResources DylibResources; - }; } // End namespace orc. diff --git a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index f3094dafae3c..2acfecfb94dc 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -39,9 +39,12 @@ protected: void operator=(const LinkedObjectSet&) = delete; public: LinkedObjectSet(RuntimeDyld::MemoryManager &MemMgr, - RuntimeDyld::SymbolResolver &Resolver) + RuntimeDyld::SymbolResolver &Resolver, + bool ProcessAllSections) : RTDyld(llvm::make_unique<RuntimeDyld>(MemMgr, Resolver)), - State(Raw) {} + State(Raw) { + RTDyld->setProcessAllSections(ProcessAllSections); + } virtual ~LinkedObjectSet() {} @@ -64,18 +67,9 @@ protected: RTDyld->mapSectionAddress(LocalAddress, TargetAddr); } - void takeOwnershipOfBuffer(std::unique_ptr<MemoryBuffer> B) { - OwnedBuffers.push_back(std::move(B)); - } - protected: std::unique_ptr<RuntimeDyld> RTDyld; enum { Raw, Finalizing, Finalized } State; - - // FIXME: This ownership hack only exists because RuntimeDyldELF still - // wants to be able to inspect the original object when resolving - // relocations. As soon as that can be fixed this should be removed. - std::vector<std::unique_ptr<MemoryBuffer>> OwnedBuffers; }; typedef std::list<std::unique_ptr<LinkedObjectSet>> LinkedObjectSetListT; @@ -83,16 +77,6 @@ protected: public: /// @brief Handle to a set of loaded objects. typedef LinkedObjectSetListT::iterator ObjSetHandleT; - - // Ownership hack. - // FIXME: Remove this as soon as RuntimeDyldELF can apply relocations without - // referencing the original object. - template <typename OwningMBSet> - void takeOwnershipOfBuffers(ObjSetHandleT H, OwningMBSet MBs) { - for (auto &MB : MBs) - (*H)->takeOwnershipOfBuffer(std::move(MB)); - } - }; /// @brief Default (no-op) action to perform when loading objects. @@ -117,16 +101,16 @@ private: class ConcreteLinkedObjectSet : public LinkedObjectSet { public: ConcreteLinkedObjectSet(MemoryManagerPtrT MemMgr, - SymbolResolverPtrT Resolver) - : LinkedObjectSet(*MemMgr, *Resolver), MemMgr(std::move(MemMgr)), - Resolver(std::move(Resolver)) { } + SymbolResolverPtrT Resolver, + bool ProcessAllSections) + : LinkedObjectSet(*MemMgr, *Resolver, ProcessAllSections), + MemMgr(std::move(MemMgr)), Resolver(std::move(Resolver)) { } void Finalize() override { State = Finalizing; RTDyld->resolveRelocations(); RTDyld->registerEHFrames(); MemMgr->finalizeMemory(); - OwnedBuffers.clear(); State = Finalized; } @@ -137,9 +121,11 @@ private: template <typename MemoryManagerPtrT, typename SymbolResolverPtrT> std::unique_ptr<LinkedObjectSet> - createLinkedObjectSet(MemoryManagerPtrT MemMgr, SymbolResolverPtrT Resolver) { + createLinkedObjectSet(MemoryManagerPtrT MemMgr, SymbolResolverPtrT Resolver, + bool ProcessAllSections) { typedef ConcreteLinkedObjectSet<MemoryManagerPtrT, SymbolResolverPtrT> LOS; - return llvm::make_unique<LOS>(std::move(MemMgr), std::move(Resolver)); + return llvm::make_unique<LOS>(std::move(MemMgr), std::move(Resolver), + ProcessAllSections); } public: @@ -158,7 +144,18 @@ public: NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(), NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor()) : NotifyLoaded(std::move(NotifyLoaded)), - NotifyFinalized(std::move(NotifyFinalized)) {} + NotifyFinalized(std::move(NotifyFinalized)), + ProcessAllSections(false) {} + + /// @brief Set the 'ProcessAllSections' flag. + /// + /// If set to true, all sections in each object file will be allocated using + /// the memory manager, rather than just the sections required for execution. + /// + /// This is kludgy, and may be removed in the future. + void setProcessAllSections(bool ProcessAllSections) { + this->ProcessAllSections = ProcessAllSections; + } /// @brief Add a set of objects (or archives) that will be treated as a unit /// for the purposes of symbol lookup and memory management. @@ -180,7 +177,8 @@ public: ObjSetHandleT Handle = LinkedObjSetList.insert( LinkedObjSetList.end(), - createLinkedObjectSet(std::move(MemMgr), std::move(Resolver))); + createLinkedObjectSet(std::move(MemMgr), std::move(Resolver), + ProcessAllSections)); LinkedObjectSet &LOS = **Handle; LoadedObjInfoList LoadedObjInfos; @@ -276,6 +274,7 @@ private: LinkedObjectSetListT LinkedObjSetList; NotifyLoadedFtor NotifyLoaded; NotifyFinalizedFtor NotifyFinalized; + bool ProcessAllSections; }; } // End namespace orc. diff --git a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h index 7af662085474..f96e83ed5a1a 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h @@ -87,14 +87,6 @@ public: BaseLayer.mapSectionAddress(H, LocalAddress, TargetAddr); } - // Ownership hack. - // FIXME: Remove this as soon as RuntimeDyldELF can apply relocations without - // referencing the original object. - template <typename OwningMBSet> - void takeOwnershipOfBuffers(ObjSetHandleT H, OwningMBSet MBs) { - BaseLayer.takeOwnershipOfBuffers(H, std::move(MBs)); - } - /// @brief Access the transform functor directly. TransformFtor &getTransform() { return Transform; } diff --git a/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h b/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h index 309f5a96090e..246d3e0a9fc6 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h +++ b/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h @@ -9,42 +9,92 @@ // // Target specific code for Orc, e.g. callback assembly. // +// Target classes should be part of the JIT *target* process, not the host +// process (except where you're doing hosted JITing and the two are one and the +// same). +// //===----------------------------------------------------------------------===// #ifndef LLVM_EXECUTIONENGINE_ORC_ORCTARGETSUPPORT_H #define LLVM_EXECUTIONENGINE_ORC_ORCTARGETSUPPORT_H #include "IndirectionUtils.h" +#include "llvm/Support/Memory.h" namespace llvm { namespace orc { class OrcX86_64 { public: - static const char *ResolverBlockName; - - /// @brief Insert module-level inline callback asm into module M for the - /// symbols managed by JITResolveCallbackHandler J. - static void insertResolverBlock(Module &M, - JITCompileCallbackManagerBase &JCBM); - - /// @brief Get a label name from the given index. - typedef std::function<std::string(unsigned)> LabelNameFtor; - - /// @brief Insert the requested number of trampolines into the given module. - /// @param M Module to insert the call block into. - /// @param NumCalls Number of calls to create in the call block. - /// @param StartIndex Optional argument specifying the index suffix to start - /// with. - /// @return A functor that provides the symbol name for each entry in the call - /// block. - /// - static LabelNameFtor insertCompileCallbackTrampolines( - Module &M, - TargetAddress TrampolineAddr, - unsigned NumCalls, - unsigned StartIndex = 0); + static const unsigned PageSize = 4096; + static const unsigned PointerSize = 8; + static const unsigned TrampolineSize = 8; + static const unsigned ResolverCodeSize = 0x78; + + typedef TargetAddress (*JITReentryFn)(void *CallbackMgr, + void *TrampolineId); + + /// @brief Write the resolver code into the given memory. The user is be + /// responsible for allocating the memory and setting permissions. + static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, + void *CallbackMgr); + + /// @brief Write the requsted number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. + static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, + unsigned NumTrampolines); + + /// @brief Provide information about stub blocks generated by the + /// makeIndirectStubsBlock function. + class IndirectStubsInfo { + friend class OrcX86_64; + public: + const static unsigned StubSize = 8; + const static unsigned PtrSize = 8; + IndirectStubsInfo() : NumStubs(0) {} + IndirectStubsInfo(IndirectStubsInfo &&Other) + : NumStubs(Other.NumStubs), StubsMem(std::move(Other.StubsMem)) { + Other.NumStubs = 0; + } + IndirectStubsInfo& operator=(IndirectStubsInfo &&Other) { + NumStubs = Other.NumStubs; + Other.NumStubs = 0; + StubsMem = std::move(Other.StubsMem); + return *this; + } + + /// @brief Number of stubs in this block. + unsigned getNumStubs() const { return NumStubs; } + + /// @brief Get a pointer to the stub at the given index, which must be in + /// the range 0 .. getNumStubs() - 1. + void* getStub(unsigned Idx) const { + return static_cast<uint64_t*>(StubsMem.base()) + Idx; + } + + /// @brief Get a pointer to the implementation-pointer at the given index, + /// which must be in the range 0 .. getNumStubs() - 1. + void** getPtr(unsigned Idx) const { + char *PtrsBase = + static_cast<char*>(StubsMem.base()) + NumStubs * StubSize; + return reinterpret_cast<void**>(PtrsBase) + Idx; + } + private: + unsigned NumStubs; + sys::OwningMemoryBlock StubsMem; + }; + + /// @brief Emit at least MinStubs worth of indirect call stubs, rounded out to + /// the nearest page size. + /// + /// E.g. Asking for 4 stubs on x86-64, where stubs are 8-bytes, with 4k + /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 + /// will return a block of 1024 (2-pages worth). + static std::error_code emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, + unsigned MinStubs, + void *InitialPtrVal); }; } // End namespace orc. diff --git a/include/llvm/ExecutionEngine/RuntimeDyld.h b/include/llvm/ExecutionEngine/RuntimeDyld.h index a808d9231167..385b8d0a30b1 100644 --- a/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -17,8 +17,10 @@ #include "JITSymbolFlags.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Object/ObjectFile.h" #include "llvm/Support/Memory.h" #include "llvm/DebugInfo/DIContext.h" +#include <map> #include <memory> namespace llvm { @@ -59,26 +61,33 @@ public: class LoadedObjectInfo : public llvm::LoadedObjectInfo { friend class RuntimeDyldImpl; public: - LoadedObjectInfo(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, - unsigned EndIdx) - : RTDyld(RTDyld), BeginIdx(BeginIdx), EndIdx(EndIdx) { } + typedef std::map<object::SectionRef, unsigned> ObjSectionToIDMap; + + LoadedObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap) + : RTDyld(RTDyld), ObjSecToIDMap(ObjSecToIDMap) { } virtual object::OwningBinary<object::ObjectFile> getObjectForDebug(const object::ObjectFile &Obj) const = 0; - uint64_t getSectionLoadAddress(StringRef Name) const; + uint64_t + getSectionLoadAddress(const object::SectionRef &Sec) const override; protected: virtual void anchor(); RuntimeDyldImpl &RTDyld; - unsigned BeginIdx, EndIdx; + ObjSectionToIDMap ObjSecToIDMap; }; template <typename Derived> struct LoadedObjectInfoHelper : LoadedObjectInfo { - LoadedObjectInfoHelper(RuntimeDyldImpl &RTDyld, unsigned BeginIdx, - unsigned EndIdx) - : LoadedObjectInfo(RTDyld, BeginIdx, EndIdx) {} + protected: + LoadedObjectInfoHelper(const LoadedObjectInfoHelper &) = default; + LoadedObjectInfoHelper() = default; + + public: + LoadedObjectInfoHelper(RuntimeDyldImpl &RTDyld, + LoadedObjectInfo::ObjSectionToIDMap ObjSecToIDMap) + : LoadedObjectInfo(RTDyld, std::move(ObjSecToIDMap)) {} std::unique_ptr<llvm::LoadedObjectInfo> clone() const override { return llvm::make_unique<Derived>(static_cast<const Derived &>(*this)); } @@ -87,7 +96,7 @@ public: /// \brief Memory Management. class MemoryManager { public: - virtual ~MemoryManager() {}; + virtual ~MemoryManager() {} /// Allocate a memory block of (at least) the given size suitable for /// executable code. The SectionID is a unique identifier assigned by the @@ -149,7 +158,7 @@ public: /// \brief Symbol resolution. class SymbolResolver { public: - virtual ~SymbolResolver() {}; + virtual ~SymbolResolver() {} /// This method returns the address of the specified function or variable. /// It is used to resolve symbols during module linking. @@ -244,4 +253,4 @@ private: } // end namespace llvm -#endif +#endif // LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H diff --git a/include/llvm/ExecutionEngine/SectionMemoryManager.h b/include/llvm/ExecutionEngine/SectionMemoryManager.h index 0b0dcb021f14..7bb96eb8b71b 100644 --- a/include/llvm/ExecutionEngine/SectionMemoryManager.h +++ b/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -83,10 +83,28 @@ public: virtual void invalidateInstructionCache(); private: + struct FreeMemBlock { + // The actual block of free memory + sys::MemoryBlock Free; + // If there is a pending allocation from the same reservation right before + // this block, store it's index in PendingMem, to be able to update the + // pending region if part of this block is allocated, rather than having to + // create a new one + unsigned PendingPrefixIndex; + }; + struct MemoryGroup { - SmallVector<sys::MemoryBlock, 16> AllocatedMem; - SmallVector<sys::MemoryBlock, 16> FreeMem; - sys::MemoryBlock Near; + // PendingMem contains all blocks of memory (subblocks of AllocatedMem) + // which have not yet had their permissions applied, but have been given + // out to the user. FreeMem contains all block of memory, which have + // neither had their permissions applied, nor been given out to the user. + SmallVector<sys::MemoryBlock, 16> PendingMem; + SmallVector<FreeMemBlock, 16> FreeMem; + + // All memory blocks that have been requested from the system + SmallVector<sys::MemoryBlock, 16> AllocatedMem; + + sys::MemoryBlock Near; }; uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size, @@ -103,4 +121,3 @@ private: } #endif // LLVM_EXECUTION_ENGINE_SECTION_MEMORY_MANAGER_H - diff --git a/include/llvm/IR/Argument.h b/include/llvm/IR/Argument.h index fc04fe71cbf0..0092f49e49ad 100644 --- a/include/llvm/IR/Argument.h +++ b/include/llvm/IR/Argument.h @@ -21,8 +21,7 @@ namespace llvm { -template<typename ValueSubClass, typename ItemParentClass> - class SymbolTableListTraits; +template <typename NodeTy> class SymbolTableListTraits; /// \brief LLVM Argument representation /// @@ -36,7 +35,7 @@ class Argument : public Value, public ilist_node<Argument> { virtual void anchor(); Function *Parent; - friend class SymbolTableListTraits<Argument, Function>; + friend class SymbolTableListTraits<Argument>; void setParent(Function *parent); public: @@ -64,8 +63,8 @@ public: /// containing function, return the number of bytes known to be /// dereferenceable. Otherwise, zero is returned. uint64_t getDereferenceableBytes() const; - - /// \brief If this argument has the dereferenceable_or_null attribute on + + /// \brief If this argument has the dereferenceable_or_null attribute on /// it in its containing function, return the number of bytes known to be /// dereferenceable. Otherwise, zero is returned. uint64_t getDereferenceableOrNullBytes() const; diff --git a/include/llvm/IR/AssemblyAnnotationWriter.h b/include/llvm/IR/AssemblyAnnotationWriter.h index 19e32a2dcdcc..6e1f5c43e12e 100644 --- a/include/llvm/IR/AssemblyAnnotationWriter.h +++ b/include/llvm/IR/AssemblyAnnotationWriter.h @@ -27,7 +27,6 @@ class formatted_raw_ostream; class AssemblyAnnotationWriter { public: - virtual ~AssemblyAnnotationWriter(); /// emitFunctionAnnot - This may be implemented to emit a string right before @@ -50,7 +49,7 @@ public: /// emitInstructionAnnot - This may be implemented to emit a string right /// before an instruction is emitted. - virtual void emitInstructionAnnot(const Instruction *, + virtual void emitInstructionAnnot(const Instruction *, formatted_raw_ostream &) {} /// printInfoComment - This may be implemented to emit a comment to the diff --git a/include/llvm/IR/Attributes.h b/include/llvm/IR/Attributes.h index 4d6d7da1fa5b..0e3373165407 100644 --- a/include/llvm/IR/Attributes.h +++ b/include/llvm/IR/Attributes.h @@ -33,6 +33,7 @@ class AttributeSetImpl; class AttributeSetNode; class Constant; template<typename T> struct DenseMapInfo; +class Function; class LLVMContext; class Type; @@ -64,65 +65,15 @@ public: enum AttrKind { // IR-Level Attributes None, ///< No attributes have been set - Alignment, ///< Alignment of parameter (5 bits) - ///< stored as log2 of alignment with +1 bias - ///< 0 means unaligned (different from align(1)) - AlwaysInline, ///< inline=always - Builtin, ///< Callee is recognized as a builtin, despite - ///< nobuiltin attribute on its declaration. - ByVal, ///< Pass structure by value - InAlloca, ///< Pass structure in an alloca - Cold, ///< Marks function as being in a cold path. - Convergent, ///< Can only be moved to control-equivalent blocks - InlineHint, ///< Source said inlining was desirable - InReg, ///< Force argument to be passed in register - JumpTable, ///< Build jump-instruction tables and replace refs. - MinSize, ///< Function must be optimized for size first - Naked, ///< Naked function - Nest, ///< Nested function static chain - NoAlias, ///< Considered to not alias after call - NoBuiltin, ///< Callee isn't recognized as a builtin - NoCapture, ///< Function creates no aliases of pointer - NoDuplicate, ///< Call cannot be duplicated - NoImplicitFloat, ///< Disable implicit floating point insts - NoInline, ///< inline=never - NonLazyBind, ///< Function is called early and/or - ///< often, so lazy binding isn't worthwhile - NonNull, ///< Pointer is known to be not null - Dereferenceable, ///< Pointer is known to be dereferenceable - DereferenceableOrNull, ///< Pointer is either null or dereferenceable - NoRedZone, ///< Disable redzone - NoReturn, ///< Mark the function as not returning - NoUnwind, ///< Function doesn't unwind stack - OptimizeForSize, ///< opt_size - OptimizeNone, ///< Function must not be optimized. - ReadNone, ///< Function does not access memory - ReadOnly, ///< Function only reads from memory - ArgMemOnly, ///< Funciton can access memory only using pointers - ///< based on its arguments. - Returned, ///< Return value is always equal to this argument - ReturnsTwice, ///< Function can return twice - SExt, ///< Sign extended before/after call - StackAlignment, ///< Alignment of stack for function (3 bits) - ///< stored as log2 of alignment with +1 bias 0 - ///< means unaligned (different from - ///< alignstack=(1)) - StackProtect, ///< Stack protection. - StackProtectReq, ///< Stack protection required. - StackProtectStrong, ///< Strong Stack protection. - SafeStack, ///< Safe Stack protection. - StructRet, ///< Hidden pointer to structure to return - SanitizeAddress, ///< AddressSanitizer is on. - SanitizeThread, ///< ThreadSanitizer is on. - SanitizeMemory, ///< MemorySanitizer is on. - UWTable, ///< Function must be in a unwind table - ZExt, ///< Zero extended before/after call - + #define GET_ATTR_ENUM + #include "llvm/IR/Attributes.inc" EndAttrKinds ///< Sentinal value useful for loops }; + private: AttributeImpl *pImpl; Attribute(AttributeImpl *A) : pImpl(A) {} + public: Attribute() : pImpl(nullptr) {} @@ -189,11 +140,11 @@ public: unsigned getStackAlignment() const; /// \brief Returns the number of dereferenceable bytes from the - /// dereferenceable attribute (or zero if unknown). + /// dereferenceable attribute. uint64_t getDereferenceableBytes() const; /// \brief Returns the number of dereferenceable_or_null bytes from the - /// dereferenceable_or_null attribute (or zero if unknown). + /// dereferenceable_or_null attribute. uint64_t getDereferenceableOrNullBytes() const; /// \brief The Attribute is converted to a string of equivalent mnemonic. This @@ -226,6 +177,7 @@ public: ReturnIndex = 0U, FunctionIndex = ~0U }; + private: friend class AttrBuilder; friend class AttributeSetImpl; @@ -249,8 +201,8 @@ private: ArrayRef<std::pair<unsigned, AttributeSetNode*> > Attrs); - explicit AttributeSet(AttributeSetImpl *LI) : pImpl(LI) {} + public: AttributeSet() : pImpl(nullptr) {} @@ -276,6 +228,11 @@ public: AttributeSet addAttribute(LLVMContext &C, unsigned Index, StringRef Kind, StringRef Value) const; + /// Add an attribute to the attribute set at the given indices. Because + /// attribute sets are immutable, this returns a new set. + AttributeSet addAttribute(LLVMContext &C, ArrayRef<unsigned> Indices, + Attribute A) const; + /// \brief Add attributes to the attribute set at the given index. Because /// attribute sets are immutable, this returns a new set. AttributeSet addAttributes(LLVMContext &C, unsigned Index, @@ -284,13 +241,13 @@ public: /// \brief Remove the specified attribute at the specified index from this /// attribute list. Because attribute lists are immutable, this returns the /// new list. - AttributeSet removeAttribute(LLVMContext &C, unsigned Index, + AttributeSet removeAttribute(LLVMContext &C, unsigned Index, Attribute::AttrKind Attr) const; /// \brief Remove the specified attributes at the specified index from this /// attribute list. Because attribute lists are immutable, this returns the /// new list. - AttributeSet removeAttributes(LLVMContext &C, unsigned Index, + AttributeSet removeAttributes(LLVMContext &C, unsigned Index, AttributeSet Attrs) const; /// \brief Remove the specified attributes at the specified index from this @@ -439,6 +396,7 @@ class AttrBuilder { uint64_t StackAlignment; uint64_t DerefBytes; uint64_t DerefOrNullBytes; + public: AttrBuilder() : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0), @@ -511,8 +469,8 @@ public: /// \brief Retrieve the stack alignment attribute, if it exists. uint64_t getStackAlignment() const { return StackAlignment; } - /// \brief Retrieve the number of dereferenceable bytes, if the dereferenceable - /// attribute exists (zero is returned otherwise). + /// \brief Retrieve the number of dereferenceable bytes, if the + /// dereferenceable attribute exists (zero is returned otherwise). uint64_t getDereferenceableBytes() const { return DerefBytes; } /// \brief Retrieve the number of dereferenceable_or_null bytes, if the @@ -573,7 +531,14 @@ public: namespace AttributeFuncs { /// \brief Which attributes cannot be applied to a type. -AttrBuilder typeIncompatible(const Type *Ty); +AttrBuilder typeIncompatible(Type *Ty); + +/// \returns Return true if the two functions have compatible target-independent +/// attributes for inlining purposes. +bool areInlineCompatible(const Function &Caller, const Function &Callee); + +/// \brief Merge caller's and callee's attributes. +void mergeAttributesForInlining(Function &Caller, const Function &Callee); } // end AttributeFuncs namespace diff --git a/include/llvm/IR/Attributes.td b/include/llvm/IR/Attributes.td new file mode 100644 index 000000000000..797cd55427b3 --- /dev/null +++ b/include/llvm/IR/Attributes.td @@ -0,0 +1,192 @@ +/// Attribute base class. +class Attr<string S> { + // String representation of this attribute in the IR. + string AttrString = S; +} + +/// Enum attribute. +class EnumAttr<string S> : Attr<S>; + +/// StringBool attribute. +class StrBoolAttr<string S> : Attr<S>; + +/// Target-independent enum attributes. + +/// Alignment of parameter (5 bits) stored as log2 of alignment with +1 bias. +/// 0 means unaligned (different from align(1)). +def Alignment : EnumAttr<"align">; + +/// inline=always. +def AlwaysInline : EnumAttr<"alwaysinline">; + +/// Function can access memory only using pointers based on its arguments. +def ArgMemOnly : EnumAttr<"argmemonly">; + +/// Callee is recognized as a builtin, despite nobuiltin attribute on its +/// declaration. +def Builtin : EnumAttr<"builtin">; + +/// Pass structure by value. +def ByVal : EnumAttr<"byval">; + +/// Marks function as being in a cold path. +def Cold : EnumAttr<"cold">; + +/// Can only be moved to control-equivalent blocks. +def Convergent : EnumAttr<"convergent">; + +/// Pointer is known to be dereferenceable. +def Dereferenceable : EnumAttr<"dereferenceable">; + +/// Pointer is either null or dereferenceable. +def DereferenceableOrNull : EnumAttr<"dereferenceable_or_null">; + +/// Function may only access memory that is inaccessible from IR. +def InaccessibleMemOnly : EnumAttr<"inaccessiblememonly">; + +/// Function may only access memory that is either inaccessible from the IR, +/// or pointed to by its pointer arguments. +def InaccessibleMemOrArgMemOnly : EnumAttr<"inaccessiblemem_or_argmemonly">; + +/// Pass structure in an alloca. +def InAlloca : EnumAttr<"inalloca">; + +/// Source said inlining was desirable. +def InlineHint : EnumAttr<"inlinehint">; + +/// Force argument to be passed in register. +def InReg : EnumAttr<"inreg">; + +/// Build jump-instruction tables and replace refs. +def JumpTable : EnumAttr<"jumptable">; + +/// Function must be optimized for size first. +def MinSize : EnumAttr<"minsize">; + +/// Naked function. +def Naked : EnumAttr<"naked">; + +/// Nested function static chain. +def Nest : EnumAttr<"nest">; + +/// Considered to not alias after call. +def NoAlias : EnumAttr<"noalias">; + +/// Callee isn't recognized as a builtin. +def NoBuiltin : EnumAttr<"nobuiltin">; + +/// Function creates no aliases of pointer. +def NoCapture : EnumAttr<"nocapture">; + +/// Call cannot be duplicated. +def NoDuplicate : EnumAttr<"noduplicate">; + +/// Disable implicit floating point insts. +def NoImplicitFloat : EnumAttr<"noimplicitfloat">; + +/// inline=never. +def NoInline : EnumAttr<"noinline">; + +/// Function is called early and/or often, so lazy binding isn't worthwhile. +def NonLazyBind : EnumAttr<"nonlazybind">; + +/// Pointer is known to be not null. +def NonNull : EnumAttr<"nonnull">; + +/// The function does not recurse. +def NoRecurse : EnumAttr<"norecurse">; + +/// Disable redzone. +def NoRedZone : EnumAttr<"noredzone">; + +/// Mark the function as not returning. +def NoReturn : EnumAttr<"noreturn">; + +/// Function doesn't unwind stack. +def NoUnwind : EnumAttr<"nounwind">; + +/// opt_size. +def OptimizeForSize : EnumAttr<"optsize">; + +/// Function must not be optimized. +def OptimizeNone : EnumAttr<"optnone">; + +/// Function does not access memory. +def ReadNone : EnumAttr<"readnone">; + +/// Function only reads from memory. +def ReadOnly : EnumAttr<"readonly">; + +/// Return value is always equal to this argument. +def Returned : EnumAttr<"returned">; + +/// Function can return twice. +def ReturnsTwice : EnumAttr<"returns_twice">; + +/// Safe Stack protection. +def SafeStack : EnumAttr<"safestack">; + +/// Sign extended before/after call. +def SExt : EnumAttr<"signext">; + +/// Alignment of stack for function (3 bits) stored as log2 of alignment with +/// +1 bias 0 means unaligned (different from alignstack=(1)). +def StackAlignment : EnumAttr<"alignstack">; + +/// Stack protection. +def StackProtect : EnumAttr<"ssp">; + +/// Stack protection required. +def StackProtectReq : EnumAttr<"sspreq">; + +/// Strong Stack protection. +def StackProtectStrong : EnumAttr<"sspstrong">; + +/// Hidden pointer to structure to return. +def StructRet : EnumAttr<"sret">; + +/// AddressSanitizer is on. +def SanitizeAddress : EnumAttr<"sanitize_address">; + +/// ThreadSanitizer is on. +def SanitizeThread : EnumAttr<"sanitize_thread">; + +/// MemorySanitizer is on. +def SanitizeMemory : EnumAttr<"sanitize_memory">; + +/// Function must be in a unwind table. +def UWTable : EnumAttr<"uwtable">; + +/// Zero extended before/after call. +def ZExt : EnumAttr<"zeroext">; + +/// Target-independent string attributes. +def LessPreciseFPMAD : StrBoolAttr<"less-precise-fpmad">; +def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">; +def NoNansFPMath : StrBoolAttr<"no-nans-fp-math">; +def UnsafeFPMath : StrBoolAttr<"unsafe-fp-math">; + +class CompatRule<string F> { + // The name of the function called to check the attribute of the caller and + // callee and decide whether inlining should be allowed. The function's + // signature must match "bool(const Function&, const Function &)", where the + // first parameter is the reference to the caller and the second parameter is + // the reference to the callee. It must return false if the attributes of the + // caller and callee are incompatible, and true otherwise. + string CompatFunc = F; +} + +def : CompatRule<"isEqual<SanitizeAddressAttr>">; +def : CompatRule<"isEqual<SanitizeThreadAttr>">; +def : CompatRule<"isEqual<SanitizeMemoryAttr>">; + +class MergeRule<string F> { + // The name of the function called to merge the attributes of the caller and + // callee. The function's signature must match + // "void(Function&, const Function &)", where the first parameter is the + // reference to the caller and the second parameter is the reference to the + // callee. + string MergeFunc = F; +} + +def : MergeRule<"adjustCallerSSPLevel">; diff --git a/include/llvm/IR/BasicBlock.h b/include/llvm/IR/BasicBlock.h index 66581bfedbe6..c6b54d308ce6 100644 --- a/include/llvm/IR/BasicBlock.h +++ b/include/llvm/IR/BasicBlock.h @@ -30,22 +30,9 @@ class LLVMContext; class BlockAddress; class Function; -// Traits for intrusive list of basic blocks... -template<> struct ilist_traits<BasicBlock> - : public SymbolTableListTraits<BasicBlock, Function> { - - BasicBlock *createSentinel() const; - static void destroySentinel(BasicBlock*) {} - - BasicBlock *provideInitialHead() const { return createSentinel(); } - BasicBlock *ensureHead(BasicBlock*) const { return createSentinel(); } - static void noteHead(BasicBlock*, BasicBlock*) {} - - static ValueSymbolTable *getSymTab(Function *ItemParent); -private: - mutable ilist_half_node<BasicBlock> Sentinel; -}; - +template <> +struct SymbolTableListSentinelTraits<BasicBlock> + : public ilist_half_embedded_sentinel_traits<BasicBlock> {}; /// \brief LLVM Basic Block Representation /// @@ -63,16 +50,17 @@ private: /// modifying a program. However, the verifier will ensure that basic blocks /// are "well formed". class BasicBlock : public Value, // Basic blocks are data objects also - public ilist_node<BasicBlock> { + public ilist_node_with_parent<BasicBlock, Function> { friend class BlockAddress; public: - typedef iplist<Instruction> InstListType; + typedef SymbolTableList<Instruction> InstListType; + private: InstListType InstList; Function *Parent; void setParent(Function *parent); - friend class SymbolTableListTraits<BasicBlock, Function>; + friend class SymbolTableListTraits<BasicBlock>; BasicBlock(const BasicBlock &) = delete; void operator=(const BasicBlock &) = delete; @@ -171,7 +159,7 @@ public: /// \brief Unlink 'this' from the containing function and delete it. /// // \returns an iterator pointing to the element after the erased one. - iplist<BasicBlock>::iterator eraseFromParent(); + SymbolTableList<BasicBlock>::iterator eraseFromParent(); /// \brief Unlink this basic block from its current function and insert it /// into the function that \p MovePos lives in, right before \p MovePos. @@ -253,7 +241,7 @@ public: InstListType &getInstList() { return InstList; } /// \brief Returns a pointer to a member of the instruction list. - static iplist<Instruction> BasicBlock::*getSublistAccess(Instruction*) { + static InstListType BasicBlock::*getSublistAccess(Instruction*) { return &BasicBlock::InstList; } @@ -283,6 +271,8 @@ public: /// should be called while the predecessor still refers to this block. void removePredecessor(BasicBlock *Pred, bool DontDeleteUselessPHIs = false); + bool canSplitPredecessors() const; + /// \brief Split the basic block into two basic blocks at the specified /// instruction. /// @@ -300,6 +290,9 @@ public: /// Also note that this doesn't preserve any passes. To split blocks while /// keeping loop information consistent, use the SplitBlock utility function. BasicBlock *splitBasicBlock(iterator I, const Twine &BBName = ""); + BasicBlock *splitBasicBlock(Instruction *I, const Twine &BBName = "") { + return splitBasicBlock(I->getIterator(), BBName); + } /// \brief Returns true if there are any uses of this basic block other than /// direct branches, switches, etc. to it. @@ -309,6 +302,9 @@ public: /// basic block \p New instead of to it. void replaceSuccessorsPhiUsesWith(BasicBlock *New); + /// \brief Return true if this basic block is an exception handling block. + bool isEHPad() const { return getFirstNonPHI()->isEHPad(); } + /// \brief Return true if this basic block is a landing pad. /// /// Being a ``landing pad'' means that the basic block is the destination of @@ -337,12 +333,6 @@ private: } }; -// createSentinel is used to get hold of the node that marks the end of the -// list... (same trick used here as in ilist_traits<Instruction>) -inline BasicBlock *ilist_traits<BasicBlock>::createSentinel() const { - return static_cast<BasicBlock*>(&Sentinel); -} - // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_SIMPLE_CONVERSION_FUNCTIONS(BasicBlock, LLVMBasicBlockRef) diff --git a/include/llvm/IR/CFG.h b/include/llvm/IR/CFG.h index f78220a52033..e9bf09333a23 100644 --- a/include/llvm/IR/CFG.h +++ b/include/llvm/IR/CFG.h @@ -107,149 +107,13 @@ inline pred_const_range predecessors(const BasicBlock *BB) { } //===----------------------------------------------------------------------===// -// BasicBlock succ_iterator definition +// BasicBlock succ_iterator helpers //===----------------------------------------------------------------------===// -template <class Term_, class BB_> // Successor Iterator -class SuccIterator : public std::iterator<std::random_access_iterator_tag, BB_, - int, BB_ *, BB_ *> { - typedef std::iterator<std::random_access_iterator_tag, BB_, int, BB_ *, BB_ *> - super; - -public: - typedef typename super::pointer pointer; - typedef typename super::reference reference; - -private: - Term_ Term; - unsigned idx; - typedef SuccIterator<Term_, BB_> Self; - - inline bool index_is_valid(int idx) { - return idx >= 0 && (unsigned) idx < Term->getNumSuccessors(); - } - - /// \brief Proxy object to allow write access in operator[] - class SuccessorProxy { - Self it; - - public: - explicit SuccessorProxy(const Self &it) : it(it) {} - - SuccessorProxy(const SuccessorProxy&) = default; - - SuccessorProxy &operator=(SuccessorProxy r) { - *this = reference(r); - return *this; - } - - SuccessorProxy &operator=(reference r) { - it.Term->setSuccessor(it.idx, r); - return *this; - } - - operator reference() const { return *it; } - }; - -public: - explicit inline SuccIterator(Term_ T) : Term(T), idx(0) {// begin iterator - } - inline SuccIterator(Term_ T, bool) // end iterator - : Term(T) { - if (Term) - idx = Term->getNumSuccessors(); - else - // Term == NULL happens, if a basic block is not fully constructed and - // consequently getTerminator() returns NULL. In this case we construct a - // SuccIterator which describes a basic block that has zero successors. - // Defining SuccIterator for incomplete and malformed CFGs is especially - // useful for debugging. - idx = 0; - } - - /// getSuccessorIndex - This is used to interface between code that wants to - /// operate on terminator instructions directly. - unsigned getSuccessorIndex() const { return idx; } - - inline bool operator==(const Self& x) const { return idx == x.idx; } - inline bool operator!=(const Self& x) const { return !operator==(x); } - - inline reference operator*() const { return Term->getSuccessor(idx); } - inline pointer operator->() const { return operator*(); } - - inline Self& operator++() { ++idx; return *this; } // Preincrement - - inline Self operator++(int) { // Postincrement - Self tmp = *this; ++*this; return tmp; - } - - inline Self& operator--() { --idx; return *this; } // Predecrement - inline Self operator--(int) { // Postdecrement - Self tmp = *this; --*this; return tmp; - } - - inline bool operator<(const Self& x) const { - assert(Term == x.Term && "Cannot compare iterators of different blocks!"); - return idx < x.idx; - } - - inline bool operator<=(const Self& x) const { - assert(Term == x.Term && "Cannot compare iterators of different blocks!"); - return idx <= x.idx; - } - inline bool operator>=(const Self& x) const { - assert(Term == x.Term && "Cannot compare iterators of different blocks!"); - return idx >= x.idx; - } - - inline bool operator>(const Self& x) const { - assert(Term == x.Term && "Cannot compare iterators of different blocks!"); - return idx > x.idx; - } - - inline Self& operator+=(int Right) { - unsigned new_idx = idx + Right; - assert(index_is_valid(new_idx) && "Iterator index out of bound"); - idx = new_idx; - return *this; - } - - inline Self operator+(int Right) const { - Self tmp = *this; - tmp += Right; - return tmp; - } - - inline Self& operator-=(int Right) { - return operator+=(-Right); - } - - inline Self operator-(int Right) const { - return operator+(-Right); - } - - inline int operator-(const Self& x) const { - assert(Term == x.Term && "Cannot work on iterators of different blocks!"); - int distance = idx - x.idx; - return distance; - } - - inline SuccessorProxy operator[](int offset) { - Self tmp = *this; - tmp += offset; - return SuccessorProxy(tmp); - } - - /// Get the source BB of this iterator. - inline BB_ *getSource() { - assert(Term && "Source not available, if basic block was malformed"); - return Term->getParent(); - } -}; - -typedef SuccIterator<TerminatorInst*, BasicBlock> succ_iterator; -typedef SuccIterator<const TerminatorInst*, - const BasicBlock> succ_const_iterator; +typedef TerminatorInst::SuccIterator<TerminatorInst *, BasicBlock> + succ_iterator; +typedef TerminatorInst::SuccIterator<const TerminatorInst *, const BasicBlock> + succ_const_iterator; typedef llvm::iterator_range<succ_iterator> succ_range; typedef llvm::iterator_range<succ_const_iterator> succ_const_range; @@ -275,8 +139,8 @@ inline succ_const_range successors(const BasicBlock *BB) { return succ_const_range(succ_begin(BB), succ_end(BB)); } - -template <typename T, typename U> struct isPodLike<SuccIterator<T, U> > { +template <typename T, typename U> +struct isPodLike<TerminatorInst::SuccIterator<T, U>> { static const bool value = isPodLike<T>::value; }; diff --git a/include/llvm/IR/CMakeLists.txt b/include/llvm/IR/CMakeLists.txt index dd8e04f1510c..eade87e05bc9 100644 --- a/include/llvm/IR/CMakeLists.txt +++ b/include/llvm/IR/CMakeLists.txt @@ -1,5 +1,6 @@ -set(LLVM_TARGET_DEFINITIONS Intrinsics.td) +set(LLVM_TARGET_DEFINITIONS Attributes.td) +tablegen(LLVM Attributes.inc -gen-attrs) +set(LLVM_TARGET_DEFINITIONS Intrinsics.td) tablegen(LLVM Intrinsics.gen -gen-intrinsic) - add_public_tablegen_target(intrinsics_gen) diff --git a/include/llvm/IR/CallSite.h b/include/llvm/IR/CallSite.h index 2841781e8a9e..f4b8a8a5a1c9 100644 --- a/include/llvm/IR/CallSite.h +++ b/include/llvm/IR/CallSite.h @@ -41,6 +41,7 @@ template <typename FunTy = const Function, typename BBTy = const BasicBlock, typename ValTy = const Value, typename UserTy = const User, + typename UseTy = const Use, typename InstrTy = const Instruction, typename CallTy = const CallInst, typename InvokeTy = const InvokeInst, @@ -69,6 +70,7 @@ private: } return CallSiteBase(); } + public: /// isCall - true if a CallInst is enclosed. /// Note that !isCall() does not mean it is an InvokeInst enclosed, @@ -116,6 +118,43 @@ public: /// Determine whether this Use is the callee operand's Use. bool isCallee(const Use *U) const { return getCallee() == U; } + /// \brief Determine whether the passed iterator points to an argument + /// operand. + bool isArgOperand(Value::const_user_iterator UI) const { + return isArgOperand(&UI.getUse()); + } + + /// \brief Determine whether the passed use points to an argument operand. + bool isArgOperand(const Use *U) const { + assert(getInstruction() == U->getUser()); + return arg_begin() <= U && U < arg_end(); + } + + /// \brief Determine whether the passed iterator points to a bundle operand. + bool isBundleOperand(Value::const_user_iterator UI) const { + return isBundleOperand(&UI.getUse()); + } + + /// \brief Determine whether the passed use points to a bundle operand. + bool isBundleOperand(const Use *U) const { + assert(getInstruction() == U->getUser()); + if (!hasOperandBundles()) + return false; + unsigned OperandNo = U - (*this)->op_begin(); + return getBundleOperandsStartIndex() <= OperandNo && + OperandNo < getBundleOperandsEndIndex(); + } + + /// \brief Determine whether the passed iterator points to a data operand. + bool isDataOperand(Value::const_user_iterator UI) const { + return isDataOperand(&UI.getUse()); + } + + /// \brief Determine whether the passed use points to a data operand. + bool isDataOperand(const Use *U) const { + return data_operands_begin() <= U && U < data_operands_end(); + } + ValTy *getArgument(unsigned ArgNo) const { assert(arg_begin() + ArgNo < arg_end() && "Argument # out of range!"); return *(arg_begin() + ArgNo); @@ -137,8 +176,7 @@ public: /// it. unsigned getArgumentNo(const Use *U) const { assert(getInstruction() && "Not a call or invoke instruction!"); - assert(arg_begin() <= U && U < arg_end() - && "Argument # out of range!"); + assert(isArgOperand(U) && "Argument # out of range!"); return U - arg_begin(); } @@ -146,21 +184,55 @@ public: /// arguments at this call site. typedef IterTy arg_iterator; - /// arg_begin/arg_end - Return iterators corresponding to the actual argument - /// list for a call site. - IterTy arg_begin() const { - assert(getInstruction() && "Not a call or invoke instruction!"); - // Skip non-arguments - return (*this)->op_begin(); - } - - IterTy arg_end() const { return (*this)->op_end() - getArgumentEndOffset(); } iterator_range<IterTy> args() const { - return iterator_range<IterTy>(arg_begin(), arg_end()); + return make_range(arg_begin(), arg_end()); } bool arg_empty() const { return arg_end() == arg_begin(); } unsigned arg_size() const { return unsigned(arg_end() - arg_begin()); } + /// Given a value use iterator, returns the data operand that corresponds to + /// it. + /// Iterator must actually correspond to a data operand. + unsigned getDataOperandNo(Value::const_user_iterator UI) const { + return getDataOperandNo(&UI.getUse()); + } + + /// Given a use for a data operand, get the data operand number that + /// corresponds to it. + unsigned getDataOperandNo(const Use *U) const { + assert(getInstruction() && "Not a call or invoke instruction!"); + assert(isDataOperand(U) && "Data operand # out of range!"); + return U - data_operands_begin(); + } + + /// Type of iterator to use when looping over data operands at this call site + /// (see below). + typedef IterTy data_operand_iterator; + + /// data_operands_begin/data_operands_end - Return iterators iterating over + /// the call / invoke argument list and bundle operands. For invokes, this is + /// the set of instruction operands except the invoke target and the two + /// successor blocks; and for calls this is the set of instruction operands + /// except the call target. + + IterTy data_operands_begin() const { + assert(getInstruction() && "Not a call or invoke instruction!"); + return (*this)->op_begin(); + } + IterTy data_operands_end() const { + assert(getInstruction() && "Not a call or invoke instruction!"); + return (*this)->op_end() - (isCall() ? 1 : 3); + } + iterator_range<IterTy> data_ops() const { + return make_range(data_operands_begin(), data_operands_end()); + } + bool data_operands_empty() const { + return data_operands_end() == data_operands_begin(); + } + unsigned data_operands_size() const { + return std::distance(data_operands_begin(), data_operands_end()); + } + /// getType - Return the type of the instruction that generated this call site /// Type *getType() const { return (*this)->getType(); } @@ -197,11 +269,11 @@ public: CALLSITE_DELEGATE_GETTER(getNumArgOperands()); } - ValTy *getArgOperand(unsigned i) const { + ValTy *getArgOperand(unsigned i) const { CALLSITE_DELEGATE_GETTER(getArgOperand(i)); } - bool isInlineAsm() const { + bool isInlineAsm() const { if (isCall()) return cast<CallInst>(getInstruction())->isInlineAsm(); return false; @@ -243,6 +315,17 @@ public: CALLSITE_DELEGATE_GETTER(paramHasAttr(i, A)); } + /// \brief Return true if the data operand at index \p i directly or + /// indirectly has the attribute \p A. + /// + /// Normal call or invoke arguments have per operand attributes, as specified + /// in the attribute set attached to this instruction, while operand bundle + /// operands may have some attributes implied by the type of its containing + /// operand bundle. + bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind A) const { + CALLSITE_DELEGATE_GETTER(dataOperandHasImpliedAttr(i, A)); + } + /// @brief Extract the alignment for a call or parameter (0=unknown). uint16_t getParamAlignment(uint16_t i) const { CALLSITE_DELEGATE_GETTER(getParamAlignment(i)); @@ -253,13 +336,20 @@ public: uint64_t getDereferenceableBytes(uint16_t i) const { CALLSITE_DELEGATE_GETTER(getDereferenceableBytes(i)); } - + /// @brief Extract the number of dereferenceable_or_null bytes for a call or /// parameter (0=unknown). uint64_t getDereferenceableOrNullBytes(uint16_t i) const { CALLSITE_DELEGATE_GETTER(getDereferenceableOrNullBytes(i)); } - + + /// @brief Determine if the parameter or return value is marked with NoAlias + /// attribute. + /// @param n The parameter to check. 1 is the first parameter, 0 is the return + bool doesNotAlias(unsigned n) const { + CALLSITE_DELEGATE_GETTER(doesNotAlias(n)); + } + /// \brief Return true if the call should not be treated as a call to a /// builtin. bool isNoBuiltin() const { @@ -315,12 +405,62 @@ public: CALLSITE_DELEGATE_SETTER(setDoesNotThrow()); } + unsigned getNumOperandBundles() const { + CALLSITE_DELEGATE_GETTER(getNumOperandBundles()); + } + + bool hasOperandBundles() const { + CALLSITE_DELEGATE_GETTER(hasOperandBundles()); + } + + unsigned getBundleOperandsStartIndex() const { + CALLSITE_DELEGATE_GETTER(getBundleOperandsStartIndex()); + } + + unsigned getBundleOperandsEndIndex() const { + CALLSITE_DELEGATE_GETTER(getBundleOperandsEndIndex()); + } + + unsigned getNumTotalBundleOperands() const { + CALLSITE_DELEGATE_GETTER(getNumTotalBundleOperands()); + } + + OperandBundleUse getOperandBundleAt(unsigned Index) const { + CALLSITE_DELEGATE_GETTER(getOperandBundleAt(Index)); + } + + Optional<OperandBundleUse> getOperandBundle(StringRef Name) const { + CALLSITE_DELEGATE_GETTER(getOperandBundle(Name)); + } + + Optional<OperandBundleUse> getOperandBundle(uint32_t ID) const { + CALLSITE_DELEGATE_GETTER(getOperandBundle(ID)); + } + + IterTy arg_begin() const { + CALLSITE_DELEGATE_GETTER(arg_begin()); + } + + IterTy arg_end() const { + CALLSITE_DELEGATE_GETTER(arg_end()); + } + #undef CALLSITE_DELEGATE_GETTER #undef CALLSITE_DELEGATE_SETTER - /// @brief Determine whether this argument is not captured. - bool doesNotCapture(unsigned ArgNo) const { - return paramHasAttr(ArgNo + 1, Attribute::NoCapture); + void getOperandBundlesAsDefs(SmallVectorImpl<OperandBundleDef> &Defs) const { + const Instruction *II = getInstruction(); + // Since this is actually a getter that "looks like" a setter, don't use the + // above macros to avoid confusion. + if (isCall()) + cast<CallInst>(II)->getOperandBundlesAsDefs(Defs); + else + cast<InvokeInst>(II)->getOperandBundlesAsDefs(Defs); + } + + /// @brief Determine whether this data operand is not captured. + bool doesNotCapture(unsigned OpNo) const { + return dataOperandHasImpliedAttr(OpNo + 1, Attribute::NoCapture); } /// @brief Determine whether this argument is passed by value. @@ -345,13 +485,13 @@ public: return paramHasAttr(arg_size(), Attribute::InAlloca); } - bool doesNotAccessMemory(unsigned ArgNo) const { - return paramHasAttr(ArgNo + 1, Attribute::ReadNone); + bool doesNotAccessMemory(unsigned OpNo) const { + return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone); } - bool onlyReadsMemory(unsigned ArgNo) const { - return paramHasAttr(ArgNo + 1, Attribute::ReadOnly) || - paramHasAttr(ArgNo + 1, Attribute::ReadNone); + bool onlyReadsMemory(unsigned OpNo) const { + return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadOnly) || + dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone); } /// @brief Return true if the return value is known to be not null. @@ -378,13 +518,6 @@ public: } private: - unsigned getArgumentEndOffset() const { - if (isCall()) - return 1; // Skip Callee - else - return 3; // Skip BB, BB, Callee - } - IterTy getCallee() const { if (isCall()) // Skip Callee return cast<CallInst>(getInstruction())->op_end() - 1; @@ -393,7 +526,7 @@ private: } }; -class CallSite : public CallSiteBase<Function, BasicBlock, Value, User, +class CallSite : public CallSiteBase<Function, BasicBlock, Value, User, Use, Instruction, CallInst, InvokeInst, User::op_iterator> { public: diff --git a/include/llvm/IR/CallingConv.h b/include/llvm/IR/CallingConv.h index 9872e6ec794d..bc050928266e 100644 --- a/include/llvm/IR/CallingConv.h +++ b/include/llvm/IR/CallingConv.h @@ -69,6 +69,12 @@ namespace CallingConv { // (almost) all registers. PreserveAll = 15, + // Swift - Calling convention for Swift. + Swift = 16, + + // CXX_FAST_TLS - Calling convention for access functions. + CXX_FAST_TLS = 17, + // Target - This is the start of the target-specific calling conventions, // e.g. fastcall and thiscall on X86. FirstTargetCC = 64, @@ -144,7 +150,26 @@ namespace CallingConv { /// \brief MSVC calling convention that passes vectors and vector aggregates /// in SSE registers. - X86_VectorCall = 80 + X86_VectorCall = 80, + + /// \brief Calling convention used by HipHop Virtual Machine (HHVM) to + /// perform calls to and from translation cache, and for calling PHP + /// functions. + /// HHVM calling convention supports tail/sibling call elimination. + HHVM = 81, + + /// \brief HHVM calling convention for invoking C/C++ helpers. + HHVM_C = 82, + + /// X86_INTR - x86 hardware interrupt context. Callee may take one or two + /// parameters, where the 1st represents a pointer to hardware context frame + /// and the 2nd represents hardware error code, the presence of the later + /// depends on the interrupt vector taken. Valid for both 32- and 64-bit + /// subtargets. + X86_INTR = 83, + + /// The highest possible calling convention ID. Must be some 2^k - 1. + MaxID = 1023 }; } // End CallingConv namespace diff --git a/include/llvm/IR/Comdat.h b/include/llvm/IR/Comdat.h index 4d4c15fb68cd..fb79e13af3a5 100644 --- a/include/llvm/IR/Comdat.h +++ b/include/llvm/IR/Comdat.h @@ -42,7 +42,7 @@ public: SelectionKind getSelectionKind() const { return SK; } void setSelectionKind(SelectionKind Val) { SK = Val; } StringRef getName() const; - void print(raw_ostream &OS) const; + void print(raw_ostream &OS, bool IsForDebug = false) const; void dump() const; private: diff --git a/include/llvm/IR/Constant.h b/include/llvm/IR/Constant.h index 019b4343a133..bb88905aa57a 100644 --- a/include/llvm/IR/Constant.h +++ b/include/llvm/IR/Constant.h @@ -24,18 +24,18 @@ namespace llvm { /// This is an important base class in LLVM. It provides the common facilities /// of all constant values in an LLVM program. A constant is a value that is /// immutable at runtime. Functions are constants because their address is -/// immutable. Same with global variables. -/// +/// immutable. Same with global variables. +/// /// All constants share the capabilities provided in this class. All constants /// can have a null value. They can have an operand list. Constants can be /// simple (integer and floating point values), complex (arrays and structures), -/// or expression based (computations yielding a constant value composed of +/// or expression based (computations yielding a constant value composed of /// only certain operators and other constant values). -/// -/// Note that Constants are immutable (once created they never change) -/// and are fully shared by structural equivalence. This means that two -/// structurally equivalent constants will always have the same address. -/// Constants are created on demand as needed and never deleted: thus clients +/// +/// Note that Constants are immutable (once created they never change) +/// and are fully shared by structural equivalence. This means that two +/// structurally equivalent constants will always have the same address. +/// Constants are created on demand as needed and never deleted: thus clients /// don't have to worry about the lifetime of the objects. /// @brief LLVM Constant Representation class Constant : public User { @@ -59,7 +59,7 @@ public: /// getAllOnesValue. bool isAllOnesValue() const; - /// isNegativeZeroValue - Return true if the value is what would be returned + /// isNegativeZeroValue - Return true if the value is what would be returned /// by getZeroValueForNegation. bool isNegativeZeroValue() const; @@ -85,29 +85,14 @@ public: /// isConstantUsed - Return true if the constant has users other than constant /// exprs and other dangling things. bool isConstantUsed() const; - - enum PossibleRelocationsTy { - NoRelocation = 0, - LocalRelocation = 1, - GlobalRelocations = 2 - }; - - /// getRelocationInfo - This method classifies the entry according to - /// whether or not it may generate a relocation entry. This must be - /// conservative, so if it might codegen to a relocatable entry, it should say - /// so. The return values are: - /// - /// NoRelocation: This constant pool entry is guaranteed to never have a - /// relocation applied to it (because it holds a simple constant like - /// '4'). - /// LocalRelocation: This entry has relocations, but the entries are - /// guaranteed to be resolvable by the static linker, so the dynamic - /// linker will never see them. - /// GlobalRelocations: This entry may have arbitrary relocations. + + /// This method classifies the entry according to whether or not it may + /// generate a relocation entry. This must be conservative, so if it might + /// codegen to a relocatable entry, it should say so. /// - /// FIXME: This really should not be in VMCore. - PossibleRelocationsTy getRelocationInfo() const; - + /// FIXME: This really should not be in IR. + bool needsRelocation() const; + /// getAggregateElement - For aggregates (struct/array/vector) return the /// constant that corresponds to the specified element if possible, or null if /// not. This can return null if the element index is a ConstantExpr, or if @@ -159,8 +144,8 @@ public: /// getIntegerValue - Return the value for an integer or pointer constant, /// or a vector thereof, with the given scalar value. - static Constant *getIntegerValue(Type* Ty, const APInt &V); - + static Constant *getIntegerValue(Type *Ty, const APInt &V); + /// removeDeadConstantUsers - If there are any dead constant users dangling /// off of this constant, remove them. This method is useful for clients /// that want to check to see if a global is unused, but don't want to deal diff --git a/include/llvm/IR/ConstantRange.h b/include/llvm/IR/ConstantRange.h index 9ded3ca36a70..fb596a3bf16e 100644 --- a/include/llvm/IR/ConstantRange.h +++ b/include/llvm/IR/ConstantRange.h @@ -82,6 +82,17 @@ public: static ConstantRange makeSatisfyingICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other); + /// Return the largest range containing all X such that "X BinOpC C" does not + /// wrap (overflow). + /// + /// Example: + /// typedef OverflowingBinaryOperator OBO; + /// makeNoWrapRegion(Add, i8 1, OBO::NoSignedWrap) == [-128, 127) + /// makeNoWrapRegion(Add, i8 1, OBO::NoUnsignedWrap) == [0, -1) + /// makeNoWrapRegion(Add, i8 0, OBO::NoUnsignedWrap) == Full Set + static ConstantRange makeNoWrapRegion(Instruction::BinaryOps BinOp, + const APInt &C, unsigned NoWrapKind); + /// Return the lower value for this range. /// const APInt &getLower() const { return Lower; } @@ -207,7 +218,7 @@ public: /// Make this range have the bit width given by \p BitWidth. The /// value is zero extended, truncated, or left alone to make it that width. ConstantRange zextOrTrunc(uint32_t BitWidth) const; - + /// Make this range have the bit width given by \p BitWidth. The /// value is sign extended, truncated, or left alone to make it that width. ConstantRange sextOrTrunc(uint32_t BitWidth) const; @@ -258,7 +269,7 @@ public: /// Return a new range that is the logical not of the current set. /// ConstantRange inverse() const; - + /// Print out the bounds to a stream. /// void print(raw_ostream &OS) const; diff --git a/include/llvm/IR/Constants.h b/include/llvm/IR/Constants.h index 0c7a84fc8bfe..a5a20c9c5701 100644 --- a/include/llvm/IR/Constants.h +++ b/include/llvm/IR/Constants.h @@ -590,7 +590,7 @@ public: /// formed with a vector or array of the specified element type. /// ConstantDataArray only works with normal float and int types that are /// stored densely in memory, not with things like i42 or x86_f80. - static bool isElementTypeCompatible(const Type *Ty); + static bool isElementTypeCompatible(Type *Ty); /// getElementAsInteger - If this is a sequential container of integers (of /// any size), return the specified element in the low bits of a uint64_t. @@ -795,7 +795,32 @@ public: } }; +//===----------------------------------------------------------------------===// +/// ConstantTokenNone - a constant token which is empty +/// +class ConstantTokenNone : public Constant { + void *operator new(size_t, unsigned) = delete; + ConstantTokenNone(const ConstantTokenNone &) = delete; + + friend class Constant; + void destroyConstantImpl(); + Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); + +protected: + explicit ConstantTokenNone(LLVMContext &Context) + : Constant(Type::getTokenTy(Context), ConstantTokenNoneVal, nullptr, 0) {} + // allocate space for exactly zero operands + void *operator new(size_t s) { return User::operator new(s, 0); } + +public: + /// Return the ConstantTokenNone. + static ConstantTokenNone *get(LLVMContext &Context); + /// @brief Methods to support type inquiry through isa, cast, and dyn_cast. + static bool classof(const Value *V) { + return V->getValueID() == ConstantTokenNoneVal; + } +}; /// BlockAddress - The address of a basic block. /// @@ -1175,7 +1200,8 @@ public: /// gets constant-folded, the type changes, or the expression is otherwise /// canonicalized. This parameter should almost always be \c false. Constant *getWithOperands(ArrayRef<Constant *> Ops, Type *Ty, - bool OnlyIfReduced = false) const; + bool OnlyIfReduced = false, + Type *SrcTy = nullptr) const; /// getAsInstruction - Returns an Instruction which implements the same /// operation as this ConstantExpr. The instruction is not linked to any basic diff --git a/include/llvm/IR/DIBuilder.h b/include/llvm/IR/DIBuilder.h index aa43c02d5cd8..aeec39541154 100644 --- a/include/llvm/IR/DIBuilder.h +++ b/include/llvm/IR/DIBuilder.h @@ -158,7 +158,9 @@ namespace llvm { /// Create debugging information entry for a c++ /// style reference or rvalue reference type. - DIDerivedType *createReferenceType(unsigned Tag, DIType *RTy); + DIDerivedType *createReferenceType(unsigned Tag, DIType *RTy, + uint64_t SizeInBits = 0, + uint64_t AlignInBits = 0); /// Create debugging information entry for a typedef. /// \param Ty Original type. @@ -375,15 +377,20 @@ namespace llvm { DIType *UnderlyingType, StringRef UniqueIdentifier = ""); /// Create subroutine type. - /// \param File File in which this subroutine is defined. /// \param ParameterTypes An array of subroutine parameter types. This /// includes return type at 0th index. /// \param Flags E.g.: LValueReference. /// These flags are used to emit dwarf attributes. - DISubroutineType *createSubroutineType(DIFile *File, - DITypeRefArray ParameterTypes, + DISubroutineType *createSubroutineType(DITypeRefArray ParameterTypes, unsigned Flags = 0); + /// Create an external type reference. + /// \param Tag Dwarf TAG. + /// \param File File in which the type is defined. + /// \param UniqueIdentifier A unique identifier for the type. + DICompositeType *createExternalTypeRef(unsigned Tag, DIFile *File, + StringRef UniqueIdentifier); + /// Create a new DIType* with "artificial" flag set. DIType *createArtificialType(DIType *Ty); @@ -450,26 +457,36 @@ namespace llvm { unsigned LineNo, DIType *Ty, bool isLocalToUnit, llvm::Constant *Val, MDNode *Decl = nullptr); - /// Create a new descriptor for the specified - /// local variable. - /// \param Tag Dwarf TAG. Usually DW_TAG_auto_variable or - /// DW_TAG_arg_variable. - /// \param Scope Variable scope. - /// \param Name Variable name. - /// \param File File where this variable is defined. - /// \param LineNo Line number. - /// \param Ty Variable Type - /// \param AlwaysPreserve Boolean. Set to true if debug info for this - /// variable should be preserved in optimized build. - /// \param Flags Flags, e.g. artificial variable. - /// \param ArgNo If this variable is an argument then this argument's - /// number. 1 indicates 1st argument. - DILocalVariable *createLocalVariable(unsigned Tag, DIScope *Scope, - StringRef Name, DIFile *File, - unsigned LineNo, DIType *Ty, + /// Create a new descriptor for an auto variable. This is a local variable + /// that is not a subprogram parameter. + /// + /// \c Scope must be a \a DILocalScope, and thus its scope chain eventually + /// leads to a \a DISubprogram. + /// + /// If \c AlwaysPreserve, this variable will be referenced from its + /// containing subprogram, and will survive some optimizations. + DILocalVariable *createAutoVariable(DIScope *Scope, StringRef Name, + DIFile *File, unsigned LineNo, + DIType *Ty, bool AlwaysPreserve = false, - unsigned Flags = 0, - unsigned ArgNo = 0); + unsigned Flags = 0); + + /// Create a new descriptor for a parameter variable. + /// + /// \c Scope must be a \a DILocalScope, and thus its scope chain eventually + /// leads to a \a DISubprogram. + /// + /// \c ArgNo is the index (starting from \c 1) of this variable in the + /// subprogram parameters. \c ArgNo should not conflict with other + /// parameters of the same subprogram. + /// + /// If \c AlwaysPreserve, this variable will be referenced from its + /// containing subprogram, and will survive some optimizations. + DILocalVariable *createParameterVariable(DIScope *Scope, StringRef Name, + unsigned ArgNo, DIFile *File, + unsigned LineNo, DIType *Ty, + bool AlwaysPreserve = false, + unsigned Flags = 0); /// Create a new descriptor for the specified /// variable which has a complex address expression for its address. @@ -499,15 +516,15 @@ namespace llvm { /// \param Flags e.g. is this function prototyped or not. /// These flags are used to emit dwarf attributes. /// \param isOptimized True if optimization is ON. - /// \param Fn llvm::Function pointer. - /// \param TParam Function template parameters. - DISubprogram * - createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName, - DIFile *File, unsigned LineNo, DISubroutineType *Ty, - bool isLocalToUnit, bool isDefinition, unsigned ScopeLine, - unsigned Flags = 0, bool isOptimized = false, - Function *Fn = nullptr, MDNode *TParam = nullptr, - MDNode *Decl = nullptr); + /// \param TParams Function template parameters. + DISubprogram *createFunction(DIScope *Scope, StringRef Name, + StringRef LinkageName, DIFile *File, + unsigned LineNo, DISubroutineType *Ty, + bool isLocalToUnit, bool isDefinition, + unsigned ScopeLine, unsigned Flags = 0, + bool isOptimized = false, + DITemplateParameterArray TParams = nullptr, + DISubprogram *Decl = nullptr); /// Identical to createFunction, /// except that the resulting DbgNode is meant to be RAUWed. @@ -515,18 +532,19 @@ namespace llvm { DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit, bool isDefinition, unsigned ScopeLine, unsigned Flags = 0, - bool isOptimized = false, Function *Fn = nullptr, - MDNode *TParam = nullptr, MDNode *Decl = nullptr); + bool isOptimized = false, DITemplateParameterArray TParams = nullptr, + DISubprogram *Decl = nullptr); /// FIXME: this is added for dragonegg. Once we update dragonegg /// to call resolve function, this will be removed. - DISubprogram * - createFunction(DIScopeRef Scope, StringRef Name, StringRef LinkageName, - DIFile *File, unsigned LineNo, DISubroutineType *Ty, - bool isLocalToUnit, bool isDefinition, unsigned ScopeLine, - unsigned Flags = 0, bool isOptimized = false, - Function *Fn = nullptr, MDNode *TParam = nullptr, - MDNode *Decl = nullptr); + DISubprogram *createFunction(DIScopeRef Scope, StringRef Name, + StringRef LinkageName, DIFile *File, + unsigned LineNo, DISubroutineType *Ty, + bool isLocalToUnit, bool isDefinition, + unsigned ScopeLine, unsigned Flags = 0, + bool isOptimized = false, + DITemplateParameterArray TParams = nullptr, + DISubprogram *Decl = nullptr); /// Create a new descriptor for the specified C++ method. /// See comments in \a DISubprogram* for descriptions of these fields. @@ -545,15 +563,14 @@ namespace llvm { /// \param Flags e.g. is this function prototyped or not. /// This flags are used to emit dwarf attributes. /// \param isOptimized True if optimization is ON. - /// \param Fn llvm::Function pointer. - /// \param TParam Function template parameters. + /// \param TParams Function template parameters. DISubprogram * createMethod(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit, bool isDefinition, unsigned Virtuality = 0, unsigned VTableIndex = 0, DIType *VTableHolder = nullptr, unsigned Flags = 0, bool isOptimized = false, - Function *Fn = nullptr, MDNode *TParam = nullptr); + DITemplateParameterArray TParams = nullptr); /// This creates new descriptor for a namespace with the specified /// parent scope. @@ -685,7 +702,7 @@ namespace llvm { /// has a self-reference -- \a DIBuilder needs to track the array to /// resolve cycles. void replaceArrays(DICompositeType *&T, DINodeArray Elements, - DINodeArray TParems = DINodeArray()); + DINodeArray TParams = DINodeArray()); /// Replace a temporary node. /// diff --git a/include/llvm/IR/DataLayout.h b/include/llvm/IR/DataLayout.h index 892d6c9936c0..19a3a6661feb 100644 --- a/include/llvm/IR/DataLayout.h +++ b/include/llvm/IR/DataLayout.h @@ -475,7 +475,8 @@ inline LLVMTargetDataRef wrap(const DataLayout *P) { class StructLayout { uint64_t StructSize; unsigned StructAlignment; - unsigned NumElements; + bool IsPadded : 1; + unsigned NumElements : 31; uint64_t MemberOffsets[1]; // variable sized array! public: uint64_t getSizeInBytes() const { return StructSize; } @@ -484,6 +485,10 @@ public: unsigned getAlignment() const { return StructAlignment; } + /// Returns whether the struct has padding or not between its fields. + /// NB: Padding in nested element is not taken into account. + bool hasPadding() const { return IsPadded; } + /// \brief Given a valid byte offset into the structure, returns the structure /// index that contains it. unsigned getElementContainingOffset(uint64_t Offset) const; diff --git a/include/llvm/IR/DebugInfo.h b/include/llvm/IR/DebugInfo.h index 5429648ade2c..4caceacbb58e 100644 --- a/include/llvm/IR/DebugInfo.h +++ b/include/llvm/IR/DebugInfo.h @@ -44,9 +44,6 @@ DISubprogram *getDISubprogram(const MDNode *Scope); /// \returns a valid subprogram, if found. Otherwise, return \c nullptr. DISubprogram *getDISubprogram(const Function *F); -/// \brief Find underlying composite type. -DICompositeTypeBase *getDICompositeType(DIType *T); - /// \brief Generate map by visiting all retained types. DITypeIdentifierMap generateDITypeIdentifierMap(const NamedMDNode *CU_Nodes); @@ -108,23 +105,23 @@ public: typedef SmallVectorImpl<DIScope *>::const_iterator scope_iterator; iterator_range<compile_unit_iterator> compile_units() const { - return iterator_range<compile_unit_iterator>(CUs.begin(), CUs.end()); + return make_range(CUs.begin(), CUs.end()); } iterator_range<subprogram_iterator> subprograms() const { - return iterator_range<subprogram_iterator>(SPs.begin(), SPs.end()); + return make_range(SPs.begin(), SPs.end()); } iterator_range<global_variable_iterator> global_variables() const { - return iterator_range<global_variable_iterator>(GVs.begin(), GVs.end()); + return make_range(GVs.begin(), GVs.end()); } iterator_range<type_iterator> types() const { - return iterator_range<type_iterator>(TYs.begin(), TYs.end()); + return make_range(TYs.begin(), TYs.end()); } iterator_range<scope_iterator> scopes() const { - return iterator_range<scope_iterator>(Scopes.begin(), Scopes.end()); + return make_range(Scopes.begin(), Scopes.end()); } unsigned compile_unit_count() const { return CUs.size(); } @@ -146,8 +143,6 @@ private: bool TypeMapInitialized; }; -DenseMap<const Function *, DISubprogram *> makeSubprogramMap(const Module &M); - } // end namespace llvm #endif diff --git a/include/llvm/IR/DebugInfoFlags.def b/include/llvm/IR/DebugInfoFlags.def index d5de8683fd3b..9756c12264b4 100644 --- a/include/llvm/IR/DebugInfoFlags.def +++ b/include/llvm/IR/DebugInfoFlags.def @@ -32,5 +32,6 @@ HANDLE_DI_FLAG((1 << 11), Vector) HANDLE_DI_FLAG((1 << 12), StaticMember) HANDLE_DI_FLAG((1 << 13), LValueReference) HANDLE_DI_FLAG((1 << 14), RValueReference) +HANDLE_DI_FLAG((1 << 15), ExternalTypeRef) #undef HANDLE_DI_FLAG diff --git a/include/llvm/IR/DebugInfoMetadata.h b/include/llvm/IR/DebugInfoMetadata.h index 9c5a95721d79..456313a70e83 100644 --- a/include/llvm/IR/DebugInfoMetadata.h +++ b/include/llvm/IR/DebugInfoMetadata.h @@ -20,15 +20,7 @@ // Helper macros for defining get() overrides. #define DEFINE_MDNODE_GET_UNPACK_IMPL(...) __VA_ARGS__ #define DEFINE_MDNODE_GET_UNPACK(ARGS) DEFINE_MDNODE_GET_UNPACK_IMPL ARGS -#define DEFINE_MDNODE_GET(CLASS, FORMAL, ARGS) \ - static CLASS *get(LLVMContext &Context, DEFINE_MDNODE_GET_UNPACK(FORMAL)) { \ - return getImpl(Context, DEFINE_MDNODE_GET_UNPACK(ARGS), Uniqued); \ - } \ - static CLASS *getIfExists(LLVMContext &Context, \ - DEFINE_MDNODE_GET_UNPACK(FORMAL)) { \ - return getImpl(Context, DEFINE_MDNODE_GET_UNPACK(ARGS), Uniqued, \ - /* ShouldCreate */ false); \ - } \ +#define DEFINE_MDNODE_GET_DISTINCT_TEMPORARY(CLASS, FORMAL, ARGS) \ static CLASS *getDistinct(LLVMContext &Context, \ DEFINE_MDNODE_GET_UNPACK(FORMAL)) { \ return getImpl(Context, DEFINE_MDNODE_GET_UNPACK(ARGS), Distinct); \ @@ -38,6 +30,16 @@ return Temp##CLASS( \ getImpl(Context, DEFINE_MDNODE_GET_UNPACK(ARGS), Temporary)); \ } +#define DEFINE_MDNODE_GET(CLASS, FORMAL, ARGS) \ + static CLASS *get(LLVMContext &Context, DEFINE_MDNODE_GET_UNPACK(FORMAL)) { \ + return getImpl(Context, DEFINE_MDNODE_GET_UNPACK(ARGS), Uniqued); \ + } \ + static CLASS *getIfExists(LLVMContext &Context, \ + DEFINE_MDNODE_GET_UNPACK(FORMAL)) { \ + return getImpl(Context, DEFINE_MDNODE_GET_UNPACK(ARGS), Uniqued, \ + /* ShouldCreate */ false); \ + } \ + DEFINE_MDNODE_GET_DISTINCT_TEMPORARY(CLASS, FORMAL, ARGS) namespace llvm { @@ -67,8 +69,8 @@ public: operator Metadata *() const { return const_cast<Metadata *>(MD); } - bool operator==(const TypedDINodeRef<T> &X) const { return MD == X.MD; }; - bool operator!=(const TypedDINodeRef<T> &X) const { return MD != X.MD; }; + bool operator==(const TypedDINodeRef<T> &X) const { return MD == X.MD; } + bool operator!=(const TypedDINodeRef<T> &X) const { return MD != X.MD; } /// \brief Create a reference. /// @@ -97,6 +99,7 @@ class DITypeRefArray { const MDTuple *N = nullptr; public: + DITypeRefArray() = default; DITypeRefArray(const MDTuple *N) : N(N) {} explicit operator bool() const { return get(); } @@ -574,6 +577,7 @@ public: bool isStaticMember() const { return getFlags() & FlagStaticMember; } bool isLValueReference() const { return getFlags() & FlagLValueReference; } bool isRValueReference() const { return getFlags() & FlagRValueReference; } + bool isExternalTypeRef() const { return getFlags() & FlagExternalTypeRef; } DITypeRef getRef() const { return DITypeRef::get(this); } @@ -646,45 +650,21 @@ public: } }; -/// \brief Base class for DIDerivedType and DICompositeType. -/// -/// TODO: Delete; they're not really related. -class DIDerivedTypeBase : public DIType { -protected: - DIDerivedTypeBase(LLVMContext &C, unsigned ID, StorageType Storage, - unsigned Tag, unsigned Line, uint64_t SizeInBits, - uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, - ArrayRef<Metadata *> Ops) - : DIType(C, ID, Storage, Tag, Line, SizeInBits, AlignInBits, OffsetInBits, - Flags, Ops) {} - ~DIDerivedTypeBase() = default; - -public: - DITypeRef getBaseType() const { return DITypeRef(getRawBaseType()); } - Metadata *getRawBaseType() const { return getOperand(3); } - - static bool classof(const Metadata *MD) { - return MD->getMetadataID() == DIDerivedTypeKind || - MD->getMetadataID() == DICompositeTypeKind || - MD->getMetadataID() == DISubroutineTypeKind; - } -}; - /// \brief Derived types. /// /// This includes qualified types, pointers, references, friends, typedefs, and /// class members. /// /// TODO: Split out members (inheritance, fields, methods, etc.). -class DIDerivedType : public DIDerivedTypeBase { +class DIDerivedType : public DIType { friend class LLVMContextImpl; friend class MDNode; DIDerivedType(LLVMContext &C, StorageType Storage, unsigned Tag, unsigned Line, uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, ArrayRef<Metadata *> Ops) - : DIDerivedTypeBase(C, DIDerivedTypeKind, Storage, Tag, Line, SizeInBits, - AlignInBits, OffsetInBits, Flags, Ops) {} + : DIType(C, DIDerivedTypeKind, Storage, Tag, Line, SizeInBits, + AlignInBits, OffsetInBits, Flags, Ops) {} ~DIDerivedType() = default; static DIDerivedType *getImpl(LLVMContext &Context, unsigned Tag, @@ -732,6 +712,10 @@ public: TempDIDerivedType clone() const { return cloneImpl(); } + //// Get the base type this is derived from. + DITypeRef getBaseType() const { return DITypeRef(getRawBaseType()); } + Metadata *getRawBaseType() const { return getOperand(3); } + /// \brief Get extra data associated with this derived type. /// /// Class type for pointer-to-members, objective-c property node for ivars, @@ -764,88 +748,23 @@ public: } }; -/// \brief Base class for DICompositeType and DISubroutineType. -/// -/// TODO: Delete; they're not really related. -class DICompositeTypeBase : public DIDerivedTypeBase { - unsigned RuntimeLang; - -protected: - DICompositeTypeBase(LLVMContext &C, unsigned ID, StorageType Storage, - unsigned Tag, unsigned Line, unsigned RuntimeLang, - uint64_t SizeInBits, uint64_t AlignInBits, - uint64_t OffsetInBits, unsigned Flags, - ArrayRef<Metadata *> Ops) - : DIDerivedTypeBase(C, ID, Storage, Tag, Line, SizeInBits, AlignInBits, - OffsetInBits, Flags, Ops), - RuntimeLang(RuntimeLang) {} - ~DICompositeTypeBase() = default; - -public: - /// \brief Get the elements of the composite type. - /// - /// \note Calling this is only valid for \a DICompositeType. This assertion - /// can be removed once \a DISubroutineType has been separated from - /// "composite types". - DINodeArray getElements() const { - assert(!isa<DISubroutineType>(this) && "no elements for DISubroutineType"); - return cast_or_null<MDTuple>(getRawElements()); - } - DITypeRef getVTableHolder() const { return DITypeRef(getRawVTableHolder()); } - DITemplateParameterArray getTemplateParams() const { - return cast_or_null<MDTuple>(getRawTemplateParams()); - } - StringRef getIdentifier() const { return getStringOperand(7); } - unsigned getRuntimeLang() const { return RuntimeLang; } - - Metadata *getRawElements() const { return getOperand(4); } - Metadata *getRawVTableHolder() const { return getOperand(5); } - Metadata *getRawTemplateParams() const { return getOperand(6); } - MDString *getRawIdentifier() const { return getOperandAs<MDString>(7); } - - /// \brief Replace operands. - /// - /// If this \a isUniqued() and not \a isResolved(), on a uniquing collision - /// this will be RAUW'ed and deleted. Use a \a TrackingMDRef to keep track - /// of its movement if necessary. - /// @{ - void replaceElements(DINodeArray Elements) { -#ifndef NDEBUG - for (DINode *Op : getElements()) - assert(std::find(Elements->op_begin(), Elements->op_end(), Op) && - "Lost a member during member list replacement"); -#endif - replaceOperandWith(4, Elements.get()); - } - void replaceVTableHolder(DITypeRef VTableHolder) { - replaceOperandWith(5, VTableHolder); - } - void replaceTemplateParams(DITemplateParameterArray TemplateParams) { - replaceOperandWith(6, TemplateParams.get()); - } - /// @} - - static bool classof(const Metadata *MD) { - return MD->getMetadataID() == DICompositeTypeKind || - MD->getMetadataID() == DISubroutineTypeKind; - } -}; - /// \brief Composite types. /// /// TODO: Detach from DerivedTypeBase (split out MDEnumType?). /// TODO: Create a custom, unrelated node for DW_TAG_array_type. -class DICompositeType : public DICompositeTypeBase { +class DICompositeType : public DIType { friend class LLVMContextImpl; friend class MDNode; + unsigned RuntimeLang; + DICompositeType(LLVMContext &C, StorageType Storage, unsigned Tag, unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, ArrayRef<Metadata *> Ops) - : DICompositeTypeBase(C, DICompositeTypeKind, Storage, Tag, Line, - RuntimeLang, SizeInBits, AlignInBits, OffsetInBits, - Flags, Ops) {} + : DIType(C, DICompositeTypeKind, Storage, Tag, Line, SizeInBits, + AlignInBits, OffsetInBits, Flags, Ops), + RuntimeLang(RuntimeLang) {} ~DICompositeType() = default; static DICompositeType * @@ -903,6 +822,45 @@ public: TempDICompositeType clone() const { return cloneImpl(); } + DITypeRef getBaseType() const { return DITypeRef(getRawBaseType()); } + DINodeArray getElements() const { + return cast_or_null<MDTuple>(getRawElements()); + } + DITypeRef getVTableHolder() const { return DITypeRef(getRawVTableHolder()); } + DITemplateParameterArray getTemplateParams() const { + return cast_or_null<MDTuple>(getRawTemplateParams()); + } + StringRef getIdentifier() const { return getStringOperand(7); } + unsigned getRuntimeLang() const { return RuntimeLang; } + + Metadata *getRawBaseType() const { return getOperand(3); } + Metadata *getRawElements() const { return getOperand(4); } + Metadata *getRawVTableHolder() const { return getOperand(5); } + Metadata *getRawTemplateParams() const { return getOperand(6); } + MDString *getRawIdentifier() const { return getOperandAs<MDString>(7); } + + /// \brief Replace operands. + /// + /// If this \a isUniqued() and not \a isResolved(), on a uniquing collision + /// this will be RAUW'ed and deleted. Use a \a TrackingMDRef to keep track + /// of its movement if necessary. + /// @{ + void replaceElements(DINodeArray Elements) { +#ifndef NDEBUG + for (DINode *Op : getElements()) + assert(std::find(Elements->op_begin(), Elements->op_end(), Op) && + "Lost a member during member list replacement"); +#endif + replaceOperandWith(4, Elements.get()); + } + void replaceVTableHolder(DITypeRef VTableHolder) { + replaceOperandWith(5, VTableHolder); + } + void replaceTemplateParams(DITemplateParameterArray TemplateParams) { + replaceOperandWith(6, TemplateParams.get()); + } + /// @} + static bool classof(const Metadata *MD) { return MD->getMetadataID() == DICompositeTypeKind; } @@ -918,17 +876,15 @@ template <class T> TypedDINodeRef<T> TypedDINodeRef<T>::get(const T *N) { /// \brief Type array for a subprogram. /// -/// TODO: Detach from CompositeType, and fold the array of types in directly -/// as operands. -class DISubroutineType : public DICompositeTypeBase { +/// TODO: Fold the array of types in directly as operands. +class DISubroutineType : public DIType { friend class LLVMContextImpl; friend class MDNode; DISubroutineType(LLVMContext &C, StorageType Storage, unsigned Flags, ArrayRef<Metadata *> Ops) - : DICompositeTypeBase(C, DISubroutineTypeKind, Storage, - dwarf::DW_TAG_subroutine_type, 0, 0, 0, 0, 0, Flags, - Ops) {} + : DIType(C, DISubroutineTypeKind, Storage, dwarf::DW_TAG_subroutine_type, + 0, 0, 0, 0, Flags, Ops) {} ~DISubroutineType() = default; static DISubroutineType *getImpl(LLVMContext &Context, unsigned Flags, @@ -957,7 +913,7 @@ public: DITypeRefArray getTypeArray() const { return cast_or_null<MDTuple>(getRawTypeArray()); } - Metadata *getRawTypeArray() const { return getRawElements(); } + Metadata *getRawTypeArray() const { return getOperand(3); } static bool classof(const Metadata *MD) { return MD->getMetadataID() == DISubroutineTypeKind; @@ -981,7 +937,9 @@ class DICompileUnit : public DIScope { : DIScope(C, DICompileUnitKind, Storage, dwarf::DW_TAG_compile_unit, Ops), SourceLanguage(SourceLanguage), IsOptimized(IsOptimized), RuntimeVersion(RuntimeVersion), EmissionKind(EmissionKind), - DWOId(DWOId) {} + DWOId(DWOId) { + assert(Storage != Uniqued); + } ~DICompileUnit() = default; static DICompileUnit * @@ -991,15 +949,16 @@ class DICompileUnit : public DIScope { unsigned EmissionKind, DICompositeTypeArray EnumTypes, DITypeArray RetainedTypes, DISubprogramArray Subprograms, DIGlobalVariableArray GlobalVariables, - DIImportedEntityArray ImportedEntities, uint64_t DWOId, - StorageType Storage, bool ShouldCreate = true) { + DIImportedEntityArray ImportedEntities, DIMacroNodeArray Macros, + uint64_t DWOId, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, SourceLanguage, File, getCanonicalMDString(Context, Producer), IsOptimized, getCanonicalMDString(Context, Flags), RuntimeVersion, getCanonicalMDString(Context, SplitDebugFilename), EmissionKind, EnumTypes.get(), RetainedTypes.get(), Subprograms.get(), GlobalVariables.get(), - ImportedEntities.get(), DWOId, Storage, ShouldCreate); + ImportedEntities.get(), Macros.get(), DWOId, Storage, + ShouldCreate); } static DICompileUnit * getImpl(LLVMContext &Context, unsigned SourceLanguage, Metadata *File, @@ -1007,40 +966,44 @@ class DICompileUnit : public DIScope { unsigned RuntimeVersion, MDString *SplitDebugFilename, unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes, Metadata *Subprograms, Metadata *GlobalVariables, - Metadata *ImportedEntities, uint64_t DWOId, StorageType Storage, - bool ShouldCreate = true); + Metadata *ImportedEntities, Metadata *Macros, uint64_t DWOId, + StorageType Storage, bool ShouldCreate = true); TempDICompileUnit cloneImpl() const { return getTemporary( getContext(), getSourceLanguage(), getFile(), getProducer(), isOptimized(), getFlags(), getRuntimeVersion(), getSplitDebugFilename(), getEmissionKind(), getEnumTypes(), getRetainedTypes(), getSubprograms(), - getGlobalVariables(), getImportedEntities(), DWOId); + getGlobalVariables(), getImportedEntities(), getMacros(), DWOId); } + static void get() = delete; + static void getIfExists() = delete; + public: - DEFINE_MDNODE_GET(DICompileUnit, - (unsigned SourceLanguage, DIFile *File, StringRef Producer, - bool IsOptimized, StringRef Flags, unsigned RuntimeVersion, - StringRef SplitDebugFilename, unsigned EmissionKind, - DICompositeTypeArray EnumTypes, DITypeArray RetainedTypes, - DISubprogramArray Subprograms, - DIGlobalVariableArray GlobalVariables, - DIImportedEntityArray ImportedEntities, uint64_t DWOId), - (SourceLanguage, File, Producer, IsOptimized, Flags, - RuntimeVersion, SplitDebugFilename, EmissionKind, - EnumTypes, RetainedTypes, Subprograms, GlobalVariables, - ImportedEntities, DWOId)) - DEFINE_MDNODE_GET( + DEFINE_MDNODE_GET_DISTINCT_TEMPORARY( + DICompileUnit, + (unsigned SourceLanguage, DIFile *File, StringRef Producer, + bool IsOptimized, StringRef Flags, unsigned RuntimeVersion, + StringRef SplitDebugFilename, unsigned EmissionKind, + DICompositeTypeArray EnumTypes, DITypeArray RetainedTypes, + DISubprogramArray Subprograms, DIGlobalVariableArray GlobalVariables, + DIImportedEntityArray ImportedEntities, DIMacroNodeArray Macros, + uint64_t DWOId), + (SourceLanguage, File, Producer, IsOptimized, Flags, RuntimeVersion, + SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, Subprograms, + GlobalVariables, ImportedEntities, Macros, DWOId)) + DEFINE_MDNODE_GET_DISTINCT_TEMPORARY( DICompileUnit, (unsigned SourceLanguage, Metadata *File, MDString *Producer, bool IsOptimized, MDString *Flags, unsigned RuntimeVersion, MDString *SplitDebugFilename, unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes, Metadata *Subprograms, - Metadata *GlobalVariables, Metadata *ImportedEntities, uint64_t DWOId), + Metadata *GlobalVariables, Metadata *ImportedEntities, Metadata *Macros, + uint64_t DWOId), (SourceLanguage, File, Producer, IsOptimized, Flags, RuntimeVersion, SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, Subprograms, - GlobalVariables, ImportedEntities, DWOId)) + GlobalVariables, ImportedEntities, Macros, DWOId)) TempDICompileUnit clone() const { return cloneImpl(); } @@ -1066,7 +1029,11 @@ public: DIImportedEntityArray getImportedEntities() const { return cast_or_null<MDTuple>(getRawImportedEntities()); } - unsigned getDWOId() const { return DWOId; } + DIMacroNodeArray getMacros() const { + return cast_or_null<MDTuple>(getRawMacros()); + } + uint64_t getDWOId() const { return DWOId; } + void setDWOId(uint64_t DwoId) { DWOId = DwoId; } MDString *getRawProducer() const { return getOperandAs<MDString>(1); } MDString *getRawFlags() const { return getOperandAs<MDString>(2); } @@ -1078,6 +1045,7 @@ public: Metadata *getRawSubprograms() const { return getOperand(6); } Metadata *getRawGlobalVariables() const { return getOperand(7); } Metadata *getRawImportedEntities() const { return getOperand(8); } + Metadata *getRawMacros() const { return getOperand(9); } /// \brief Replace arrays. /// @@ -1100,6 +1068,7 @@ public: void replaceImportedEntities(DIImportedEntityArray N) { replaceOperandWith(8, N.get()); } + void replaceMacros(DIMacroNodeArray N) { replaceOperandWith(9, N.get()); } /// @} static bool classof(const Metadata *MD) { @@ -1157,8 +1126,10 @@ class DILocation : public MDNode { } TempDILocation cloneImpl() const { - return getTemporary(getContext(), getLine(), getColumn(), getScope(), - getInlinedAt()); + // Get the raw scope/inlinedAt since it is possible to invoke this on + // a DILocation containing temporary metadata. + return getTemporary(getContext(), getLine(), getColumn(), getRawScope(), + getRawInlinedAt()); } // Disallow replacing operands. @@ -1276,14 +1247,13 @@ class DISubprogram : public DILocalScope { DISubroutineType *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, DITypeRef ContainingType, unsigned Virtuality, unsigned VirtualIndex, unsigned Flags, bool IsOptimized, - Constant *Function, DITemplateParameterArray TemplateParams, - DISubprogram *Declaration, DILocalVariableArray Variables, - StorageType Storage, bool ShouldCreate = true) { + DITemplateParameterArray TemplateParams, DISubprogram *Declaration, + DILocalVariableArray Variables, StorageType Storage, + bool ShouldCreate = true) { return getImpl(Context, Scope, getCanonicalMDString(Context, Name), getCanonicalMDString(Context, LinkageName), File, Line, Type, IsLocalToUnit, IsDefinition, ScopeLine, ContainingType, Virtuality, VirtualIndex, Flags, IsOptimized, - Function ? ConstantAsMetadata::get(Function) : nullptr, TemplateParams.get(), Declaration, Variables.get(), Storage, ShouldCreate); } @@ -1292,17 +1262,16 @@ class DISubprogram : public DILocalScope { MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, Metadata *ContainingType, unsigned Virtuality, unsigned VirtualIndex, - unsigned Flags, bool IsOptimized, Metadata *Function, - Metadata *TemplateParams, Metadata *Declaration, Metadata *Variables, - StorageType Storage, bool ShouldCreate = true); + unsigned Flags, bool IsOptimized, Metadata *TemplateParams, + Metadata *Declaration, Metadata *Variables, StorageType Storage, + bool ShouldCreate = true); TempDISubprogram cloneImpl() const { - return getTemporary(getContext(), getScope(), getName(), getLinkageName(), - getFile(), getLine(), getType(), isLocalToUnit(), - isDefinition(), getScopeLine(), getContainingType(), - getVirtuality(), getVirtualIndex(), getFlags(), - isOptimized(), getFunctionConstant(), - getTemplateParams(), getDeclaration(), getVariables()); + return getTemporary( + getContext(), getScope(), getName(), getLinkageName(), getFile(), + getLine(), getType(), isLocalToUnit(), isDefinition(), getScopeLine(), + getContainingType(), getVirtuality(), getVirtualIndex(), getFlags(), + isOptimized(), getTemplateParams(), getDeclaration(), getVariables()); } public: @@ -1312,13 +1281,12 @@ public: bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, DITypeRef ContainingType, unsigned Virtuality, unsigned VirtualIndex, unsigned Flags, bool IsOptimized, - Constant *Function = nullptr, DITemplateParameterArray TemplateParams = nullptr, DISubprogram *Declaration = nullptr, DILocalVariableArray Variables = nullptr), (Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit, IsDefinition, ScopeLine, ContainingType, Virtuality, - VirtualIndex, Flags, IsOptimized, Function, TemplateParams, + VirtualIndex, Flags, IsOptimized, TemplateParams, Declaration, Variables)) DEFINE_MDNODE_GET( DISubprogram, @@ -1326,11 +1294,11 @@ public: unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, Metadata *ContainingType, unsigned Virtuality, unsigned VirtualIndex, unsigned Flags, bool IsOptimized, - Metadata *Function = nullptr, Metadata *TemplateParams = nullptr, - Metadata *Declaration = nullptr, Metadata *Variables = nullptr), + Metadata *TemplateParams = nullptr, Metadata *Declaration = nullptr, + Metadata *Variables = nullptr), (Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit, IsDefinition, ScopeLine, ContainingType, Virtuality, VirtualIndex, Flags, IsOptimized, - Function, TemplateParams, Declaration, Variables)) + TemplateParams, Declaration, Variables)) TempDISubprogram clone() const { return cloneImpl(); } @@ -1389,11 +1357,6 @@ public: return DITypeRef(getRawContainingType()); } - Constant *getFunctionConstant() const { - if (auto *C = cast_or_null<ConstantAsMetadata>(getRawFunction())) - return C->getValue(); - return nullptr; - } DITemplateParameterArray getTemplateParams() const { return cast_or_null<MDTuple>(getRawTemplateParams()); } @@ -1407,30 +1370,11 @@ public: Metadata *getRawScope() const { return getOperand(1); } Metadata *getRawType() const { return getOperand(5); } Metadata *getRawContainingType() const { return getOperand(6); } - Metadata *getRawFunction() const { return getOperand(7); } - Metadata *getRawTemplateParams() const { return getOperand(8); } - Metadata *getRawDeclaration() const { return getOperand(9); } - Metadata *getRawVariables() const { return getOperand(10); } - - /// \brief Get a pointer to the function this subprogram describes. - /// - /// This dyn_casts \a getFunctionConstant() to \a Function. - /// - /// FIXME: Should this be looking through bitcasts? - Function *getFunction() const; - - /// \brief Replace the function. - /// - /// If \a isUniqued() and not \a isResolved(), this could node will be - /// RAUW'ed and deleted out from under the caller. Use a \a TrackingMDRef if - /// that's a problem. - /// @{ - void replaceFunction(Function *F); - void replaceFunction(ConstantAsMetadata *MD) { replaceOperandWith(7, MD); } - void replaceFunction(std::nullptr_t) { replaceOperandWith(7, nullptr); } - /// @} + Metadata *getRawTemplateParams() const { return getOperand(7); } + Metadata *getRawDeclaration() const { return getOperand(8); } + Metadata *getRawVariables() const { return getOperand(9); } - /// \brief Check if this subprogram decribes the given function. + /// \brief Check if this subprogram describes the given function. /// /// FIXME: Should this be looking through bitcasts? bool describes(const Function *F) const; @@ -1452,13 +1396,6 @@ public: Metadata *getRawScope() const { return getOperand(1); } - /// \brief Forwarding accessors to LexicalBlock. - /// - /// TODO: Remove these and update code to use \a DILexicalBlock directly. - /// @{ - inline unsigned getLine() const; - inline unsigned getColumn() const; - /// @} static bool classof(const Metadata *MD) { return MD->getMetadataID() == DILexicalBlockKind || MD->getMetadataID() == DILexicalBlockFileKind; @@ -1470,12 +1407,14 @@ class DILexicalBlock : public DILexicalBlockBase { friend class MDNode; unsigned Line; - unsigned Column; + uint16_t Column; DILexicalBlock(LLVMContext &C, StorageType Storage, unsigned Line, unsigned Column, ArrayRef<Metadata *> Ops) : DILexicalBlockBase(C, DILexicalBlockKind, Storage, Ops), Line(Line), - Column(Column) {} + Column(Column) { + assert(Column < (1u << 16) && "Expected 16-bit column"); + } ~DILexicalBlock() = default; static DILexicalBlock *getImpl(LLVMContext &Context, DILocalScope *Scope, @@ -1514,18 +1453,6 @@ public: } }; -unsigned DILexicalBlockBase::getLine() const { - if (auto *N = dyn_cast<DILexicalBlock>(this)) - return N->getLine(); - return 0; -} - -unsigned DILexicalBlockBase::getColumn() const { - if (auto *N = dyn_cast<DILexicalBlock>(this)) - return N->getColumn(); - return 0; -} - class DILexicalBlockFile : public DILexicalBlockBase { friend class LLVMContextImpl; friend class MDNode; @@ -1797,15 +1724,13 @@ public: }; /// \brief Base class for variables. -/// -/// TODO: Hardcode to DW_TAG_variable. class DIVariable : public DINode { unsigned Line; protected: - DIVariable(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag, - unsigned Line, ArrayRef<Metadata *> Ops) - : DINode(C, ID, Storage, Tag, Ops), Line(Line) {} + DIVariable(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Line, + ArrayRef<Metadata *> Ops) + : DINode(C, ID, Storage, dwarf::DW_TAG_variable, Ops), Line(Line) {} ~DIVariable() = default; public: @@ -1850,8 +1775,7 @@ class DIGlobalVariable : public DIVariable { DIGlobalVariable(LLVMContext &C, StorageType Storage, unsigned Line, bool IsLocalToUnit, bool IsDefinition, ArrayRef<Metadata *> Ops) - : DIVariable(C, DIGlobalVariableKind, Storage, dwarf::DW_TAG_variable, - Line, Ops), + : DIVariable(C, DIGlobalVariableKind, Storage, Line, Ops), IsLocalToUnit(IsLocalToUnit), IsDefinition(IsDefinition) {} ~DIGlobalVariable() = default; @@ -1923,8 +1847,6 @@ public: /// \brief Local variable. /// -/// TODO: Split between arguments and otherwise. -/// TODO: Use \c DW_TAG_variable instead of fake tags. /// TODO: Split up flags. class DILocalVariable : public DIVariable { friend class LLVMContextImpl; @@ -1933,42 +1855,42 @@ class DILocalVariable : public DIVariable { unsigned Arg; unsigned Flags; - DILocalVariable(LLVMContext &C, StorageType Storage, unsigned Tag, - unsigned Line, unsigned Arg, unsigned Flags, - ArrayRef<Metadata *> Ops) - : DIVariable(C, DILocalVariableKind, Storage, Tag, Line, Ops), Arg(Arg), + DILocalVariable(LLVMContext &C, StorageType Storage, unsigned Line, + unsigned Arg, unsigned Flags, ArrayRef<Metadata *> Ops) + : DIVariable(C, DILocalVariableKind, Storage, Line, Ops), Arg(Arg), Flags(Flags) {} ~DILocalVariable() = default; - static DILocalVariable *getImpl(LLVMContext &Context, unsigned Tag, - DIScope *Scope, StringRef Name, DIFile *File, - unsigned Line, DITypeRef Type, unsigned Arg, - unsigned Flags, StorageType Storage, + static DILocalVariable *getImpl(LLVMContext &Context, DIScope *Scope, + StringRef Name, DIFile *File, unsigned Line, + DITypeRef Type, unsigned Arg, unsigned Flags, + StorageType Storage, bool ShouldCreate = true) { - return getImpl(Context, Tag, Scope, getCanonicalMDString(Context, Name), - File, Line, Type, Arg, Flags, Storage, ShouldCreate); + return getImpl(Context, Scope, getCanonicalMDString(Context, Name), File, + Line, Type, Arg, Flags, Storage, ShouldCreate); } - static DILocalVariable * - getImpl(LLVMContext &Context, unsigned Tag, Metadata *Scope, MDString *Name, - Metadata *File, unsigned Line, Metadata *Type, unsigned Arg, - unsigned Flags, StorageType Storage, bool ShouldCreate = true); + static DILocalVariable *getImpl(LLVMContext &Context, Metadata *Scope, + MDString *Name, Metadata *File, unsigned Line, + Metadata *Type, unsigned Arg, unsigned Flags, + StorageType Storage, + bool ShouldCreate = true); TempDILocalVariable cloneImpl() const { - return getTemporary(getContext(), getTag(), getScope(), getName(), - getFile(), getLine(), getType(), getArg(), getFlags()); + return getTemporary(getContext(), getScope(), getName(), getFile(), + getLine(), getType(), getArg(), getFlags()); } public: DEFINE_MDNODE_GET(DILocalVariable, - (unsigned Tag, DILocalScope *Scope, StringRef Name, - DIFile *File, unsigned Line, DITypeRef Type, unsigned Arg, + (DILocalScope * Scope, StringRef Name, DIFile *File, + unsigned Line, DITypeRef Type, unsigned Arg, unsigned Flags), - (Tag, Scope, Name, File, Line, Type, Arg, Flags)) + (Scope, Name, File, Line, Type, Arg, Flags)) DEFINE_MDNODE_GET(DILocalVariable, - (unsigned Tag, Metadata *Scope, MDString *Name, - Metadata *File, unsigned Line, Metadata *Type, - unsigned Arg, unsigned Flags), - (Tag, Scope, Name, File, Line, Type, Arg, Flags)) + (Metadata * Scope, MDString *Name, Metadata *File, + unsigned Line, Metadata *Type, unsigned Arg, + unsigned Flags), + (Scope, Name, File, Line, Type, Arg, Flags)) TempDILocalVariable clone() const { return cloneImpl(); } @@ -1979,6 +1901,7 @@ public: return cast<DILocalScope>(DIVariable::getScope()); } + bool isParameter() const { return Arg; } unsigned getArg() const { return Arg; } unsigned getFlags() const { return Flags; } @@ -1988,7 +1911,7 @@ public: /// \brief Check that a location is valid for this variable. /// /// Check that \c DL exists, is in the same subprogram, and has the same - /// inlined-at location as \c this. (Otherwise, it's not a valid attachemnt + /// inlined-at location as \c this. (Otherwise, it's not a valid attachment /// to a \a DbgInfoIntrinsic.) bool isValidLocationForIntrinsic(const DILocation *DL) const { return DL && getScope()->getSubprogram() == DL->getScope()->getSubprogram(); @@ -2284,6 +2207,165 @@ public: } }; +/// \brief Macro Info DWARF-like metadata node. +/// +/// A metadata node with a DWARF macro info (i.e., a constant named +/// \c DW_MACINFO_*, defined in llvm/Support/Dwarf.h). Called \a DIMacroNode +/// because it's potentially used for non-DWARF output. +class DIMacroNode : public MDNode { + friend class LLVMContextImpl; + friend class MDNode; + +protected: + DIMacroNode(LLVMContext &C, unsigned ID, StorageType Storage, unsigned MIType, + ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2 = None) + : MDNode(C, ID, Storage, Ops1, Ops2) { + assert(MIType < 1u << 16); + SubclassData16 = MIType; + } + ~DIMacroNode() = default; + + template <class Ty> Ty *getOperandAs(unsigned I) const { + return cast_or_null<Ty>(getOperand(I)); + } + + StringRef getStringOperand(unsigned I) const { + if (auto *S = getOperandAs<MDString>(I)) + return S->getString(); + return StringRef(); + } + + static MDString *getCanonicalMDString(LLVMContext &Context, StringRef S) { + if (S.empty()) + return nullptr; + return MDString::get(Context, S); + } + +public: + unsigned getMacinfoType() const { return SubclassData16; } + + static bool classof(const Metadata *MD) { + switch (MD->getMetadataID()) { + default: + return false; + case DIMacroKind: + case DIMacroFileKind: + return true; + } + } +}; + +class DIMacro : public DIMacroNode { + friend class LLVMContextImpl; + friend class MDNode; + + unsigned Line; + + DIMacro(LLVMContext &C, StorageType Storage, unsigned MIType, unsigned Line, + ArrayRef<Metadata *> Ops) + : DIMacroNode(C, DIMacroKind, Storage, MIType, Ops), Line(Line) {} + ~DIMacro() = default; + + static DIMacro *getImpl(LLVMContext &Context, unsigned MIType, unsigned Line, + StringRef Name, StringRef Value, StorageType Storage, + bool ShouldCreate = true) { + return getImpl(Context, MIType, Line, getCanonicalMDString(Context, Name), + getCanonicalMDString(Context, Value), Storage, ShouldCreate); + } + static DIMacro *getImpl(LLVMContext &Context, unsigned MIType, unsigned Line, + MDString *Name, MDString *Value, StorageType Storage, + bool ShouldCreate = true); + + TempDIMacro cloneImpl() const { + return getTemporary(getContext(), getMacinfoType(), getLine(), getName(), + getValue()); + } + +public: + DEFINE_MDNODE_GET(DIMacro, (unsigned MIType, unsigned Line, StringRef Name, + StringRef Value = ""), + (MIType, Line, Name, Value)) + DEFINE_MDNODE_GET(DIMacro, (unsigned MIType, unsigned Line, MDString *Name, + MDString *Value), + (MIType, Line, Name, Value)) + + TempDIMacro clone() const { return cloneImpl(); } + + unsigned getLine() const { return Line; } + + StringRef getName() const { return getStringOperand(0); } + StringRef getValue() const { return getStringOperand(1); } + + MDString *getRawName() const { return getOperandAs<MDString>(0); } + MDString *getRawValue() const { return getOperandAs<MDString>(1); } + + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == DIMacroKind; + } +}; + +class DIMacroFile : public DIMacroNode { + friend class LLVMContextImpl; + friend class MDNode; + + unsigned Line; + + DIMacroFile(LLVMContext &C, StorageType Storage, unsigned MIType, + unsigned Line, ArrayRef<Metadata *> Ops) + : DIMacroNode(C, DIMacroFileKind, Storage, MIType, Ops), Line(Line) {} + ~DIMacroFile() = default; + + static DIMacroFile *getImpl(LLVMContext &Context, unsigned MIType, + unsigned Line, DIFile *File, + DIMacroNodeArray Elements, StorageType Storage, + bool ShouldCreate = true) { + return getImpl(Context, MIType, Line, static_cast<Metadata *>(File), + Elements.get(), Storage, ShouldCreate); + } + + static DIMacroFile *getImpl(LLVMContext &Context, unsigned MIType, + unsigned Line, Metadata *File, Metadata *Elements, + StorageType Storage, bool ShouldCreate = true); + + TempDIMacroFile cloneImpl() const { + return getTemporary(getContext(), getMacinfoType(), getLine(), getFile(), + getElements()); + } + +public: + DEFINE_MDNODE_GET(DIMacroFile, (unsigned MIType, unsigned Line, DIFile *File, + DIMacroNodeArray Elements), + (MIType, Line, File, Elements)) + DEFINE_MDNODE_GET(DIMacroFile, (unsigned MIType, unsigned Line, + Metadata *File, Metadata *Elements), + (MIType, Line, File, Elements)) + + TempDIMacroFile clone() const { return cloneImpl(); } + + void replaceElements(DIMacroNodeArray Elements) { +#ifndef NDEBUG + for (DIMacroNode *Op : getElements()) + assert(std::find(Elements->op_begin(), Elements->op_end(), Op) && + "Lost a macro node during macro node list replacement"); +#endif + replaceOperandWith(1, Elements.get()); + } + + unsigned getLine() const { return Line; } + DIFile *getFile() const { return cast_or_null<DIFile>(getRawFile()); } + + DIMacroNodeArray getElements() const { + return cast_or_null<MDTuple>(getRawElements()); + } + + Metadata *getRawFile() const { return getOperand(0); } + Metadata *getRawElements() const { return getOperand(1); } + + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == DIMacroFileKind; + } +}; + } // end namespace llvm #undef DEFINE_MDNODE_GET_UNPACK_IMPL diff --git a/include/llvm/IR/DerivedTypes.h b/include/llvm/IR/DerivedTypes.h index 4a94499b4cf5..071e69b1e808 100644 --- a/include/llvm/IR/DerivedTypes.h +++ b/include/llvm/IR/DerivedTypes.h @@ -36,11 +36,12 @@ class StringRef; /// @brief Integer representation type class IntegerType : public Type { friend class LLVMContextImpl; - + protected: explicit IntegerType(LLVMContext &C, unsigned NumBits) : Type(C, IntegerTyID){ setSubclassData(NumBits); } + public: /// This enum is just used to hold constants we need for IntegerType. enum { @@ -90,6 +91,9 @@ public: } }; +unsigned Type::getIntegerBitWidth() const { + return cast<IntegerType>(this)->getBitWidth(); +} /// FunctionType - Class to represent function types /// @@ -108,7 +112,7 @@ public: /// FunctionType::get - Create a FunctionType taking no parameters. /// static FunctionType *get(Type *Result, bool isVarArg); - + /// isValidReturnType - Return true if the specified type is valid as a return /// type. static bool isValidReturnType(Type *RetTy); @@ -143,18 +147,30 @@ public: static_assert(AlignOf<FunctionType>::Alignment >= AlignOf<Type *>::Alignment, "Alignment sufficient for objects appended to FunctionType"); +bool Type::isFunctionVarArg() const { + return cast<FunctionType>(this)->isVarArg(); +} + +Type *Type::getFunctionParamType(unsigned i) const { + return cast<FunctionType>(this)->getParamType(i); +} + +unsigned Type::getFunctionNumParams() const { + return cast<FunctionType>(this)->getNumParams(); +} + /// CompositeType - Common super class of ArrayType, StructType, PointerType /// and VectorType. class CompositeType : public Type { protected: - explicit CompositeType(LLVMContext &C, TypeID tid) : Type(C, tid) { } -public: + explicit CompositeType(LLVMContext &C, TypeID tid) : Type(C, tid) {} +public: /// getTypeAtIndex - Given an index value into the type, return the type of /// the element. /// - Type *getTypeAtIndex(const Value *V); - Type *getTypeAtIndex(unsigned Idx); + Type *getTypeAtIndex(const Value *V) const; + Type *getTypeAtIndex(unsigned Idx) const; bool indexValid(const Value *V) const; bool indexValid(unsigned Idx) const; @@ -167,14 +183,13 @@ public: } }; - /// StructType - Class to represent struct types. There are two different kinds /// of struct types: Literal structs and Identified structs. /// /// Literal struct types (e.g. { i32, i32 }) are uniqued structurally, and must /// always have a body when created. You can get one of these by using one of /// the StructType::get() forms. -/// +/// /// Identified structs (e.g. %foo or %42) may optionally have a name and are not /// uniqued. The names for identified structs are managed at the LLVMContext /// level, so there can only be a single identified struct with a given name in @@ -205,23 +220,20 @@ class StructType : public CompositeType { /// pointer to the symbol table entry (maintained by LLVMContext) for the /// struct. This is null if the type is an literal struct or if it is /// a identified type that has an empty name. - /// + /// void *SymbolTableEntry; -public: +public: /// StructType::create - This creates an identified struct. static StructType *create(LLVMContext &Context, StringRef Name); static StructType *create(LLVMContext &Context); - - static StructType *create(ArrayRef<Type*> Elements, - StringRef Name, - bool isPacked = false); - static StructType *create(ArrayRef<Type*> Elements); - static StructType *create(LLVMContext &Context, - ArrayRef<Type*> Elements, - StringRef Name, + + static StructType *create(ArrayRef<Type *> Elements, StringRef Name, bool isPacked = false); - static StructType *create(LLVMContext &Context, ArrayRef<Type*> Elements); + static StructType *create(ArrayRef<Type *> Elements); + static StructType *create(LLVMContext &Context, ArrayRef<Type *> Elements, + StringRef Name, bool isPacked = false); + static StructType *create(LLVMContext &Context, ArrayRef<Type *> Elements); static StructType *create(StringRef Name, Type *elt1, ...) LLVM_END_WITH_NULL; /// StructType::get - This static method is the primary way to create a @@ -232,7 +244,7 @@ public: /// StructType::get - Create an empty structure type. /// static StructType *get(LLVMContext &Context, bool isPacked = false); - + /// StructType::get - This static method is a convenience method for creating /// structure types by specifying the elements as arguments. Note that this /// method always returns a non-packed struct, and requires at least one @@ -240,26 +252,26 @@ public: static StructType *get(Type *elt1, ...) LLVM_END_WITH_NULL; bool isPacked() const { return (getSubclassData() & SCDB_Packed) != 0; } - + /// isLiteral - Return true if this type is uniqued by structural /// equivalence, false if it is a struct definition. bool isLiteral() const { return (getSubclassData() & SCDB_IsLiteral) != 0; } - + /// isOpaque - Return true if this is a type with an identity that has no body /// specified yet. These prints as 'opaque' in .ll files. bool isOpaque() const { return (getSubclassData() & SCDB_HasBody) == 0; } /// isSized - Return true if this is a sized type. - bool isSized(SmallPtrSetImpl<const Type*> *Visited = nullptr) const; - + bool isSized(SmallPtrSetImpl<Type *> *Visited = nullptr) const; + /// hasName - Return true if this is a named struct that has a non-empty name. bool hasName() const { return SymbolTableEntry != nullptr; } - + /// getName - Return the name for this struct type if it has an identity. /// This may return an empty string for an unnamed struct type. Do not call /// this on an literal type. StringRef getName() const; - + /// setName - Change the name of this type to the specified name, or to a name /// with a suffix if there is a collision. Do not call this on an literal /// type. @@ -268,11 +280,10 @@ public: /// setBody - Specify a body for an opaque identified type. void setBody(ArrayRef<Type*> Elements, bool isPacked = false); void setBody(Type *elt1, ...) LLVM_END_WITH_NULL; - + /// isValidElementType - Return true if the specified type is valid as a /// element type. static bool isValidElementType(Type *ElemTy); - // Iterator access to the elements. typedef Type::subtype_iterator element_iterator; @@ -284,8 +295,8 @@ public: /// isLayoutIdentical - Return true if this is layout identical to the /// specified struct. - bool isLayoutIdentical(StructType *Other) const; - + bool isLayoutIdentical(StructType *Other) const; + /// Random access to the elements unsigned getNumElements() const { return NumContainedTys; } Type *getElementType(unsigned N) const { @@ -299,6 +310,18 @@ public: } }; +StringRef Type::getStructName() const { + return cast<StructType>(this)->getName(); +} + +unsigned Type::getStructNumElements() const { + return cast<StructType>(this)->getNumElements(); +} + +Type *Type::getStructElementType(unsigned N) const { + return cast<StructType>(this)->getElementType(N); +} + /// SequentialType - This is the superclass of the array, pointer and vector /// type classes. All of these represent "arrays" in memory. The array type /// represents a specifically sized array, pointer types are unsized/unknown @@ -330,6 +353,9 @@ public: } }; +Type *Type::getSequentialElementType() const { + return cast<SequentialType>(this)->getElementType(); +} /// ArrayType - Class to represent array types. /// @@ -339,6 +365,7 @@ class ArrayType : public SequentialType { ArrayType(const ArrayType &) = delete; const ArrayType &operator=(const ArrayType &) = delete; ArrayType(Type *ElType, uint64_t NumEl); + public: /// ArrayType::get - This static method is the primary way to construct an /// ArrayType @@ -357,6 +384,10 @@ public: } }; +uint64_t Type::getArrayNumElements() const { + return cast<ArrayType>(this)->getNumElements(); +} + /// VectorType - Class to represent vector types. /// class VectorType : public SequentialType { @@ -365,6 +396,7 @@ class VectorType : public SequentialType { VectorType(const VectorType &) = delete; const VectorType &operator=(const VectorType &) = delete; VectorType(Type *ElType, unsigned NumEl); + public: /// VectorType::get - This static method is the primary way to construct an /// VectorType. @@ -443,6 +475,9 @@ public: } }; +unsigned Type::getVectorNumElements() const { + return cast<VectorType>(this)->getNumElements(); +} /// PointerType - Class to represent pointers. /// @@ -450,6 +485,7 @@ class PointerType : public SequentialType { PointerType(const PointerType &) = delete; const PointerType &operator=(const PointerType &) = delete; explicit PointerType(Type *ElType, unsigned AddrSpace); + public: /// PointerType::get - This constructs a pointer to an object of the specified /// type in a numbered address space. @@ -477,6 +513,10 @@ public: } }; +unsigned Type::getPointerAddressSpace() const { + return cast<PointerType>(getScalarType())->getAddressSpace(); +} + } // End llvm namespace #endif diff --git a/include/llvm/IR/DiagnosticInfo.h b/include/llvm/IR/DiagnosticInfo.h index f38313f82ea7..f69955e5ed48 100644 --- a/include/llvm/IR/DiagnosticInfo.h +++ b/include/llvm/IR/DiagnosticInfo.h @@ -15,7 +15,6 @@ #ifndef LLVM_IR_DIAGNOSTICINFO_H #define LLVM_IR_DIAGNOSTICINFO_H -#include "llvm-c/Core.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Module.h" @@ -56,8 +55,11 @@ enum DiagnosticKind { DK_OptimizationRemark, DK_OptimizationRemarkMissed, DK_OptimizationRemarkAnalysis, + DK_OptimizationRemarkAnalysisFPCommute, + DK_OptimizationRemarkAnalysisAliasing, DK_OptimizationFailure, DK_MIRParser, + DK_PGOProfile, DK_FirstPluginKind }; @@ -99,6 +101,8 @@ public: /// The printed message must not end with '.' nor start with a severity /// keyword. virtual void print(DiagnosticPrinter &DP) const = 0; + + static const char *AlwaysPrint; }; typedef std::function<void(const DiagnosticInfo &)> DiagnosticHandlerFunction; @@ -210,19 +214,18 @@ public: /// Diagnostic information for the sample profiler. class DiagnosticInfoSampleProfile : public DiagnosticInfo { public: - DiagnosticInfoSampleProfile(const char *FileName, unsigned LineNum, + DiagnosticInfoSampleProfile(StringRef FileName, unsigned LineNum, const Twine &Msg, DiagnosticSeverity Severity = DS_Error) : DiagnosticInfo(DK_SampleProfile, Severity), FileName(FileName), LineNum(LineNum), Msg(Msg) {} - DiagnosticInfoSampleProfile(const char *FileName, const Twine &Msg, + DiagnosticInfoSampleProfile(StringRef FileName, const Twine &Msg, DiagnosticSeverity Severity = DS_Error) : DiagnosticInfo(DK_SampleProfile, Severity), FileName(FileName), LineNum(0), Msg(Msg) {} DiagnosticInfoSampleProfile(const Twine &Msg, DiagnosticSeverity Severity = DS_Error) - : DiagnosticInfo(DK_SampleProfile, Severity), FileName(nullptr), - LineNum(0), Msg(Msg) {} + : DiagnosticInfo(DK_SampleProfile, Severity), LineNum(0), Msg(Msg) {} /// \see DiagnosticInfo::print. void print(DiagnosticPrinter &DP) const override; @@ -231,13 +234,13 @@ public: return DI->getKind() == DK_SampleProfile; } - const char *getFileName() const { return FileName; } + StringRef getFileName() const { return FileName; } unsigned getLineNum() const { return LineNum; } const Twine &getMsg() const { return Msg; } private: /// Name of the input file associated with this diagnostic. - const char *FileName; + StringRef FileName; /// Line number where the diagnostic occurred. If 0, no line number will /// be emitted in the message. @@ -247,6 +250,31 @@ private: const Twine &Msg; }; +/// Diagnostic information for the PGO profiler. +class DiagnosticInfoPGOProfile : public DiagnosticInfo { +public: + DiagnosticInfoPGOProfile(const char *FileName, const Twine &Msg, + DiagnosticSeverity Severity = DS_Error) + : DiagnosticInfo(DK_PGOProfile, Severity), FileName(FileName), Msg(Msg) {} + + /// \see DiagnosticInfo::print. + void print(DiagnosticPrinter &DP) const override; + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_PGOProfile; + } + + const char *getFileName() const { return FileName; } + const Twine &getMsg() const { return Msg; } + +private: + /// Name of the input file associated with this diagnostic. + const char *FileName; + + /// Message to report. + const Twine &Msg; +}; + /// Common features for diagnostics dealing with optimization remarks. class DiagnosticInfoOptimizationBase : public DiagnosticInfo { public: @@ -267,10 +295,6 @@ public: /// \see DiagnosticInfo::print. void print(DiagnosticPrinter &DP) const override; - static bool classof(const DiagnosticInfo *DI) { - return DI->getKind() == DK_OptimizationRemark; - } - /// Return true if this optimization remark is enabled by one of /// of the LLVM command line flags (-pass-remarks, -pass-remarks-missed, /// or -pass-remarks-analysis). Note that this only handles the LLVM @@ -386,6 +410,69 @@ public: /// \see DiagnosticInfoOptimizationBase::isEnabled. bool isEnabled() const override; + +protected: + DiagnosticInfoOptimizationRemarkAnalysis(enum DiagnosticKind Kind, + const char *PassName, + const Function &Fn, + const DebugLoc &DLoc, + const Twine &Msg) + : DiagnosticInfoOptimizationBase(Kind, DS_Remark, PassName, Fn, DLoc, + Msg) {} +}; + +/// Diagnostic information for optimization analysis remarks related to +/// floating-point non-commutativity. +class DiagnosticInfoOptimizationRemarkAnalysisFPCommute + : public DiagnosticInfoOptimizationRemarkAnalysis { +public: + /// \p PassName is the name of the pass emitting this diagnostic. If + /// this name matches the regular expression given in -Rpass-analysis=, then + /// the diagnostic will be emitted. \p Fn is the function where the diagnostic + /// is being emitted. \p DLoc is the location information to use in the + /// diagnostic. If line table information is available, the diagnostic will + /// include the source code location. \p Msg is the message to show. The + /// front-end will append its own message related to options that address + /// floating-point non-commutativity. Note that this class does not copy this + /// message, so this reference must be valid for the whole life time of the + /// diagnostic. + DiagnosticInfoOptimizationRemarkAnalysisFPCommute(const char *PassName, + const Function &Fn, + const DebugLoc &DLoc, + const Twine &Msg) + : DiagnosticInfoOptimizationRemarkAnalysis( + DK_OptimizationRemarkAnalysisFPCommute, PassName, Fn, DLoc, Msg) {} + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_OptimizationRemarkAnalysisFPCommute; + } +}; + +/// Diagnostic information for optimization analysis remarks related to +/// pointer aliasing. +class DiagnosticInfoOptimizationRemarkAnalysisAliasing + : public DiagnosticInfoOptimizationRemarkAnalysis { +public: + /// \p PassName is the name of the pass emitting this diagnostic. If + /// this name matches the regular expression given in -Rpass-analysis=, then + /// the diagnostic will be emitted. \p Fn is the function where the diagnostic + /// is being emitted. \p DLoc is the location information to use in the + /// diagnostic. If line table information is available, the diagnostic will + /// include the source code location. \p Msg is the message to show. The + /// front-end will append its own message related to options that address + /// pointer aliasing legality. Note that this class does not copy this + /// message, so this reference must be valid for the whole life time of the + /// diagnostic. + DiagnosticInfoOptimizationRemarkAnalysisAliasing(const char *PassName, + const Function &Fn, + const DebugLoc &DLoc, + const Twine &Msg) + : DiagnosticInfoOptimizationRemarkAnalysis( + DK_OptimizationRemarkAnalysisAliasing, PassName, Fn, DLoc, Msg) {} + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_OptimizationRemarkAnalysisAliasing; + } }; /// Diagnostic information for machine IR parser. @@ -438,6 +525,30 @@ void emitOptimizationRemarkAnalysis(LLVMContext &Ctx, const char *PassName, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg); +/// Emit an optimization analysis remark related to messages about +/// floating-point non-commutativity. \p PassName is the name of the pass +/// emitting the message. If -Rpass-analysis= is given and \p PassName matches +/// the regular expression in -Rpass, then the remark will be emitted. \p Fn is +/// the function triggering the remark, \p DLoc is the debug location where the +/// diagnostic is generated. \p Msg is the message string to use. +void emitOptimizationRemarkAnalysisFPCommute(LLVMContext &Ctx, + const char *PassName, + const Function &Fn, + const DebugLoc &DLoc, + const Twine &Msg); + +/// Emit an optimization analysis remark related to messages about +/// pointer aliasing. \p PassName is the name of the pass emitting the message. +/// If -Rpass-analysis= is given and \p PassName matches the regular expression +/// in -Rpass, then the remark will be emitted. \p Fn is the function triggering +/// the remark, \p DLoc is the debug location where the diagnostic is generated. +/// \p Msg is the message string to use. +void emitOptimizationRemarkAnalysisAliasing(LLVMContext &Ctx, + const char *PassName, + const Function &Fn, + const DebugLoc &DLoc, + const Twine &Msg); + /// Diagnostic information for optimization failures. class DiagnosticInfoOptimizationFailure : public DiagnosticInfoOptimizationBase { diff --git a/include/llvm/IR/DiagnosticPrinter.h b/include/llvm/IR/DiagnosticPrinter.h index 735e3ad7a8b0..1bcd73738b66 100644 --- a/include/llvm/IR/DiagnosticPrinter.h +++ b/include/llvm/IR/DiagnosticPrinter.h @@ -63,7 +63,7 @@ protected: raw_ostream &Stream; public: - DiagnosticPrinterRawOStream(raw_ostream &Stream) : Stream(Stream) {}; + DiagnosticPrinterRawOStream(raw_ostream &Stream) : Stream(Stream) {} // Simple types. DiagnosticPrinter &operator<<(char C) override; diff --git a/include/llvm/IR/Dominators.h b/include/llvm/IR/Dominators.h index 27d989b0344c..37447c353b19 100644 --- a/include/llvm/IR/Dominators.h +++ b/include/llvm/IR/Dominators.h @@ -64,11 +64,30 @@ public: /// \brief Concrete subclass of DominatorTreeBase that is used to compute a /// normal dominator tree. +/// +/// Definition: A block is said to be forward statically reachable if there is +/// a path from the entry of the function to the block. A statically reachable +/// block may become statically unreachable during optimization. +/// +/// A forward unreachable block may appear in the dominator tree, or it may +/// not. If it does, dominance queries will return results as if all reachable +/// blocks dominate it. When asking for a Node corresponding to a potentially +/// unreachable block, calling code must handle the case where the block was +/// unreachable and the result of getNode() is nullptr. +/// +/// Generally, a block known to be unreachable when the dominator tree is +/// constructed will not be in the tree. One which becomes unreachable after +/// the dominator tree is initially constructed may still exist in the tree, +/// even if the tree is properly updated. Calling code should not rely on the +/// preceding statements; this is stated only to assist human understanding. class DominatorTree : public DominatorTreeBase<BasicBlock> { public: typedef DominatorTreeBase<BasicBlock> Base; DominatorTree() : DominatorTreeBase<BasicBlock>(false) {} + explicit DominatorTree(Function &F) : DominatorTreeBase<BasicBlock>(false) { + recalculate(F); + } DominatorTree(DominatorTree &&Arg) : Base(std::move(static_cast<Base &>(Arg))) {} @@ -122,31 +141,35 @@ public: // DominatorTree GraphTraits specializations so the DominatorTree can be // iterable by generic graph iterators. -template <> struct GraphTraits<DomTreeNode*> { - typedef DomTreeNode NodeType; - typedef NodeType::iterator ChildIteratorType; +template <class Node, class ChildIterator> struct DomTreeGraphTraitsBase { + typedef Node NodeType; + typedef ChildIterator ChildIteratorType; + typedef df_iterator<Node *, SmallPtrSet<NodeType *, 8>> nodes_iterator; - static NodeType *getEntryNode(NodeType *N) { - return N; - } + static NodeType *getEntryNode(NodeType *N) { return N; } static inline ChildIteratorType child_begin(NodeType *N) { return N->begin(); } - static inline ChildIteratorType child_end(NodeType *N) { - return N->end(); - } + static inline ChildIteratorType child_end(NodeType *N) { return N->end(); } - typedef df_iterator<DomTreeNode*> nodes_iterator; - - static nodes_iterator nodes_begin(DomTreeNode *N) { + static nodes_iterator nodes_begin(NodeType *N) { return df_begin(getEntryNode(N)); } - static nodes_iterator nodes_end(DomTreeNode *N) { + static nodes_iterator nodes_end(NodeType *N) { return df_end(getEntryNode(N)); } }; +template <> +struct GraphTraits<DomTreeNode *> + : public DomTreeGraphTraitsBase<DomTreeNode, DomTreeNode::iterator> {}; + +template <> +struct GraphTraits<const DomTreeNode *> + : public DomTreeGraphTraitsBase<const DomTreeNode, + DomTreeNode::const_iterator> {}; + template <> struct GraphTraits<DominatorTree*> : public GraphTraits<DomTreeNode*> { static NodeType *getEntryNode(DominatorTree *DT) { diff --git a/include/llvm/IR/Function.h b/include/llvm/IR/Function.h index ec9f4cad094a..2a983930bf4d 100644 --- a/include/llvm/IR/Function.h +++ b/include/llvm/IR/Function.h @@ -32,28 +32,16 @@ namespace llvm { class FunctionType; class LLVMContext; +class DISubprogram; -template<> struct ilist_traits<Argument> - : public SymbolTableListTraits<Argument, Function> { - - Argument *createSentinel() const { - return static_cast<Argument*>(&Sentinel); - } - static void destroySentinel(Argument*) {} - - Argument *provideInitialHead() const { return createSentinel(); } - Argument *ensureHead(Argument*) const { return createSentinel(); } - static void noteHead(Argument*, Argument*) {} - - static ValueSymbolTable *getSymTab(Function *ItemParent); -private: - mutable ilist_half_node<Argument> Sentinel; -}; +template <> +struct SymbolTableListSentinelTraits<Argument> + : public ilist_half_embedded_sentinel_traits<Argument> {}; class Function : public GlobalObject, public ilist_node<Function> { public: - typedef iplist<Argument> ArgumentListType; - typedef iplist<BasicBlock> BasicBlockListType; + typedef SymbolTableList<Argument> ArgumentListType; + typedef SymbolTableList<BasicBlock> BasicBlockListType; // BasicBlock iterators... typedef BasicBlockListType::iterator iterator; @@ -73,10 +61,12 @@ private: /* * Value::SubclassData * - * bit 0 : HasLazyArguments - * bit 1 : HasPrefixData - * bit 2 : HasPrologueData - * bit 3-6: CallingConvention + * bit 0 : HasLazyArguments + * bit 1 : HasPrefixData + * bit 2 : HasPrologueData + * bit 3 : HasPersonalityFn + * bits 4-13 : CallingConvention + * bits 14-15 : [reserved] */ /// Bits from GlobalObject::GlobalObjectSubclassData. @@ -90,7 +80,7 @@ private: (Value ? Mask : 0u)); } - friend class SymbolTableListTraits<Function, Module>; + friend class SymbolTableListTraits<Function>; void setParent(Module *parent); @@ -120,7 +110,7 @@ private: public: static Function *Create(FunctionType *Ty, LinkageTypes Linkage, const Twine &N = "", Module *M = nullptr) { - return new(1) Function(Ty, Linkage, N, M); + return new Function(Ty, Linkage, N, M); } ~Function() override; @@ -128,14 +118,6 @@ public: /// \brief Provide fast operand accessors DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - /// \brief Get the personality function associated with this function. - bool hasPersonalityFn() const { return getNumOperands() != 0; } - Constant *getPersonalityFn() const { - assert(hasPersonalityFn()); - return cast<Constant>(Op<0>()); - } - void setPersonalityFn(Constant *C); - Type *getReturnType() const; // Return the type of the ret val FunctionType *getFunctionType() const; // Return the FunctionType for me @@ -170,11 +152,13 @@ public: /// calling convention of this function. The enum values for the known /// calling conventions are defined in CallingConv.h. CallingConv::ID getCallingConv() const { - return static_cast<CallingConv::ID>(getSubclassDataFromValue() >> 3); + return static_cast<CallingConv::ID>((getSubclassDataFromValue() >> 4) & + CallingConv::MaxID); } void setCallingConv(CallingConv::ID CC) { - setValueSubclassData((getSubclassDataFromValue() & 7) | - (static_cast<unsigned>(CC) << 3)); + auto ID = static_cast<unsigned>(CC); + assert(!(ID & ~CallingConv::MaxID) && "Unsupported calling convention"); + setValueSubclassData((getSubclassDataFromValue() & 0xc00f) | (ID << 4)); } /// @brief Return the attribute list for this Function. @@ -267,13 +251,13 @@ public: uint64_t getDereferenceableBytes(unsigned i) const { return AttributeSets.getDereferenceableBytes(i); } - + /// @brief Extract the number of dereferenceable_or_null bytes for a call or /// parameter (0=unknown). uint64_t getDereferenceableOrNullBytes(unsigned i) const { return AttributeSets.getDereferenceableOrNullBytes(i); } - + /// @brief Determine if the function does not access memory. bool doesNotAccessMemory() const { return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, @@ -299,10 +283,28 @@ public: return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, Attribute::ArgMemOnly); } - void setOnlyAccessesArgMemory() { - addFnAttr(Attribute::ArgMemOnly); + void setOnlyAccessesArgMemory() { addFnAttr(Attribute::ArgMemOnly); } + + /// @brief Determine if the function may only access memory that is + /// inaccessible from the IR. + bool onlyAccessesInaccessibleMemory() const { + return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, + Attribute::InaccessibleMemOnly); + } + void setOnlyAccessesInaccessibleMemory() { + addFnAttr(Attribute::InaccessibleMemOnly); } - + + /// @brief Determine if the function may only access memory that is + // either inaccessible from the IR or pointed to by its arguments. + bool onlyAccessesInaccessibleMemOrArgMem() const { + return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, + Attribute::InaccessibleMemOrArgMemOnly); + } + void setOnlyAccessesInaccessibleMemOrArgMem() { + addFnAttr(Attribute::InaccessibleMemOrArgMemOnly); + } + /// @brief Determine if the function cannot return. bool doesNotReturn() const { return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, @@ -339,6 +341,15 @@ public: addFnAttr(Attribute::Convergent); } + /// Determine if the function is known not to recurse, directly or + /// indirectly. + bool doesNotRecurse() const { + return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, + Attribute::NoRecurse); + } + void setDoesNotRecurse() { + addFnAttr(Attribute::NoRecurse); + } /// @brief True if the ABI mandates (or the user requested) that this /// function be in a unwind table. @@ -362,7 +373,8 @@ public: AttributeSets.hasAttribute(2, Attribute::StructRet); } - /// @brief Determine if the parameter does not alias other parameters. + /// @brief Determine if the parameter or return value is marked with NoAlias + /// attribute. /// @param n The parameter to check. 1 is the first parameter, 0 is the return bool doesNotAlias(unsigned n) const { return AttributeSets.hasAttribute(n, Attribute::NoAlias); @@ -395,6 +407,14 @@ public: addAttribute(n, Attribute::ReadOnly); } + /// Optimize this function for minimum size (-Oz). + bool optForMinSize() const { return hasFnAttribute(Attribute::MinSize); }; + + /// Optimize this function for size (-Os) or minimum size (-Oz). + bool optForSize() const { + return hasFnAttribute(Attribute::OptimizeForSize) || optForMinSize(); + } + /// copyAttributesFrom - copy all additional attributes (those not needed to /// create a Function) from the Function Src to this one. void copyAttributesFrom(const GlobalValue *Src) override; @@ -417,7 +437,6 @@ public: /// void eraseFromParent() override; - /// Get the underlying elements of the Function... the basic block list is /// empty for external functions. /// @@ -429,13 +448,13 @@ public: CheckLazyArguments(); return ArgumentList; } - static iplist<Argument> Function::*getSublistAccess(Argument*) { + static ArgumentListType Function::*getSublistAccess(Argument*) { return &Function::ArgumentList; } const BasicBlockListType &getBasicBlockList() const { return BasicBlocks; } BasicBlockListType &getBasicBlockList() { return BasicBlocks; } - static iplist<BasicBlock> Function::*getSublistAccess(BasicBlock*) { + static BasicBlockListType Function::*getSublistAccess(BasicBlock*) { return &Function::BasicBlocks; } @@ -450,7 +469,6 @@ public: inline ValueSymbolTable &getValueSymbolTable() { return *SymTab; } inline const ValueSymbolTable &getValueSymbolTable() const { return *SymTab; } - //===--------------------------------------------------------------------===// // BasicBlock iterator forwarding functions // @@ -487,11 +505,11 @@ public: } iterator_range<arg_iterator> args() { - return iterator_range<arg_iterator>(arg_begin(), arg_end()); + return make_range(arg_begin(), arg_end()); } iterator_range<const_arg_iterator> args() const { - return iterator_range<const_arg_iterator>(arg_begin(), arg_end()); + return make_range(arg_begin(), arg_end()); } /// @} @@ -499,24 +517,33 @@ public: size_t arg_size() const; bool arg_empty() const; + /// \brief Check whether this function has a personality function. + bool hasPersonalityFn() const { + return getSubclassDataFromValue() & (1<<3); + } + + /// \brief Get the personality function associated with this function. + Constant *getPersonalityFn() const; + void setPersonalityFn(Constant *Fn); + + /// \brief Check whether this function has prefix data. bool hasPrefixData() const { return getSubclassDataFromValue() & (1<<1); } + /// \brief Get the prefix data associated with this function. Constant *getPrefixData() const; void setPrefixData(Constant *PrefixData); + /// \brief Check whether this function has prologue data. bool hasPrologueData() const { return getSubclassDataFromValue() & (1<<2); } + /// \brief Get the prologue data associated with this function. Constant *getPrologueData() const; void setPrologueData(Constant *PrologueData); - /// Print the function to an output stream with an optional - /// AssemblyAnnotationWriter. - void print(raw_ostream &OS, AssemblyAnnotationWriter *AAW = nullptr) const; - /// viewCFG - This function is meant for use from the debugger. You can just /// say 'call F->viewCFG()' and a ghostview window should pop up from the /// program, displaying the CFG of the current function with the code for each @@ -596,12 +623,27 @@ public: /// Drop all metadata from \c this not included in \c KnownIDs. void dropUnknownMetadata(ArrayRef<unsigned> KnownIDs); + /// \brief Set the attached subprogram. + /// + /// Calls \a setMetadata() with \a LLVMContext::MD_dbg. + void setSubprogram(DISubprogram *SP); + + /// \brief Get the attached subprogram. + /// + /// Calls \a getMetadata() with \a LLVMContext::MD_dbg and casts the result + /// to \a DISubprogram. + DISubprogram *getSubprogram() const; + private: + void allocHungoffUselist(); + template<int Idx> void setHungoffOperand(Constant *C); + // Shadow Value::setValueSubclassData with a private forwarding method so that // subclasses cannot accidentally use it. void setValueSubclassData(unsigned short D) { Value::setValueSubclassData(D); } + void setValueSubclassDataBit(unsigned Bit, bool On); bool hasMetadataHashEntry() const { return getGlobalObjectSubClassData() & HasMetadataHashEntryBit; @@ -613,18 +655,8 @@ private: void clearMetadata(); }; -inline ValueSymbolTable * -ilist_traits<BasicBlock>::getSymTab(Function *F) { - return F ? &F->getValueSymbolTable() : nullptr; -} - -inline ValueSymbolTable * -ilist_traits<Argument>::getSymTab(Function *F) { - return F ? &F->getValueSymbolTable() : nullptr; -} - template <> -struct OperandTraits<Function> : public OptionalOperandTraits<Function> {}; +struct OperandTraits<Function> : public HungoffOperandTraits<3> {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(Function, Value) diff --git a/include/llvm/IR/FunctionInfo.h b/include/llvm/IR/FunctionInfo.h new file mode 100644 index 000000000000..eba088a61bc0 --- /dev/null +++ b/include/llvm/IR/FunctionInfo.h @@ -0,0 +1,241 @@ +//===-- llvm/FunctionInfo.h - Function Info Index ---------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// @file +/// FunctionInfo.h This file contains the declarations the classes that hold +/// the function info index and summary. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_FUNCTIONINFO_H +#define LLVM_IR_FUNCTIONINFO_H + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +/// \brief Function summary information to aid decisions and implementation of +/// importing. +/// +/// This is a separate class from FunctionInfo to enable lazy reading of this +/// function summary information from the combined index file during imporing. +class FunctionSummary { +private: + /// \brief Path of module containing function IR, used to locate module when + /// importing this function. + /// + /// This is only used during parsing of the combined function index, or when + /// parsing the per-module index for creation of the combined function index, + /// not during writing of the per-module index which doesn't contain a + /// module path string table. + StringRef ModulePath; + + /// \brief Used to flag functions that have local linkage types and need to + /// have module identifier appended before placing into the combined + /// index, to disambiguate from other functions with the same name. + /// + /// This is only used in the per-module function index, as it is consumed + /// while creating the combined index. + bool IsLocalFunction; + + // The rest of the information is used to help decide whether importing + // is likely to be profitable. + // Other information will be added as the importing is tuned, such + // as hotness (when profile available), and other function characteristics. + + /// Number of instructions (ignoring debug instructions, e.g.) computed + /// during the initial compile step when the function index is first built. + unsigned InstCount; + +public: + /// Construct a summary object from summary data expected for all + /// summary records. + FunctionSummary(unsigned NumInsts) : InstCount(NumInsts) {} + + /// Set the path to the module containing this function, for use in + /// the combined index. + void setModulePath(StringRef ModPath) { ModulePath = ModPath; } + + /// Get the path to the module containing this function. + StringRef modulePath() const { return ModulePath; } + + /// Record whether this is a local function in the per-module index. + void setLocalFunction(bool IsLocal) { IsLocalFunction = IsLocal; } + + /// Check whether this was a local function, for use in creating + /// the combined index. + bool isLocalFunction() const { return IsLocalFunction; } + + /// Get the instruction count recorded for this function. + unsigned instCount() const { return InstCount; } +}; + +/// \brief Class to hold pointer to function summary and information required +/// for parsing it. +/// +/// For the per-module index, this holds the bitcode offset +/// of the corresponding function block. For the combined index, +/// after parsing of the \a ValueSymbolTable, this initially +/// holds the offset of the corresponding function summary bitcode +/// record. After parsing the associated summary information from the summary +/// block the \a FunctionSummary is populated and stored here. +class FunctionInfo { +private: + /// Function summary information used to help make ThinLTO importing + /// decisions. + std::unique_ptr<FunctionSummary> Summary; + + /// \brief The bitcode offset corresponding to either the associated + /// function's function body record, or its function summary record, + /// depending on whether this is a per-module or combined index. + /// + /// This bitcode offset is written to or read from the associated + /// \a ValueSymbolTable entry for the function. + /// For the per-module index this holds the bitcode offset of the + /// function's body record within bitcode module block in its module, + /// which is used during lazy function parsing or ThinLTO importing. + /// For the combined index this holds the offset of the corresponding + /// function summary record, to enable associating the combined index + /// VST records with the summary records. + uint64_t BitcodeIndex; + +public: + /// Constructor used during parsing of VST entries. + FunctionInfo(uint64_t FuncOffset) + : Summary(nullptr), BitcodeIndex(FuncOffset) {} + + /// Constructor used for per-module index bitcode writing. + FunctionInfo(uint64_t FuncOffset, + std::unique_ptr<FunctionSummary> FuncSummary) + : Summary(std::move(FuncSummary)), BitcodeIndex(FuncOffset) {} + + /// Record the function summary information parsed out of the function + /// summary block during parsing or combined index creation. + void setFunctionSummary(std::unique_ptr<FunctionSummary> FuncSummary) { + Summary = std::move(FuncSummary); + } + + /// Get the function summary recorded for this function. + FunctionSummary *functionSummary() const { return Summary.get(); } + + /// Get the bitcode index recorded for this function, depending on + /// the index type. + uint64_t bitcodeIndex() const { return BitcodeIndex; } + + /// Record the bitcode index for this function, depending on + /// the index type. + void setBitcodeIndex(uint64_t FuncOffset) { BitcodeIndex = FuncOffset; } +}; + +/// List of function info structures for a particular function name held +/// in the FunctionMap. Requires a vector in the case of multiple +/// COMDAT functions of the same name. +typedef std::vector<std::unique_ptr<FunctionInfo>> FunctionInfoList; + +/// Map from function name to corresponding function info structures. +typedef StringMap<FunctionInfoList> FunctionInfoMapTy; + +/// Type used for iterating through the function info map. +typedef FunctionInfoMapTy::const_iterator const_funcinfo_iterator; +typedef FunctionInfoMapTy::iterator funcinfo_iterator; + +/// String table to hold/own module path strings, which additionally holds the +/// module ID assigned to each module during the plugin step. The StringMap +/// makes a copy of and owns inserted strings. +typedef StringMap<uint64_t> ModulePathStringTableTy; + +/// Class to hold module path string table and function map, +/// and encapsulate methods for operating on them. +class FunctionInfoIndex { +private: + /// Map from function name to list of function information instances + /// for functions of that name (may be duplicates in the COMDAT case, e.g.). + FunctionInfoMapTy FunctionMap; + + /// Holds strings for combined index, mapping to the corresponding module ID. + ModulePathStringTableTy ModulePathStringTable; + +public: + FunctionInfoIndex() = default; + + // Disable the copy constructor and assignment operators, so + // no unexpected copying/moving occurs. + FunctionInfoIndex(const FunctionInfoIndex &) = delete; + void operator=(const FunctionInfoIndex &) = delete; + + funcinfo_iterator begin() { return FunctionMap.begin(); } + const_funcinfo_iterator begin() const { return FunctionMap.begin(); } + funcinfo_iterator end() { return FunctionMap.end(); } + const_funcinfo_iterator end() const { return FunctionMap.end(); } + + /// Get the list of function info objects for a given function. + const FunctionInfoList &getFunctionInfoList(StringRef FuncName) { + return FunctionMap[FuncName]; + } + + /// Get the list of function info objects for a given function. + const const_funcinfo_iterator findFunctionInfoList(StringRef FuncName) const { + return FunctionMap.find(FuncName); + } + + /// Add a function info for a function of the given name. + void addFunctionInfo(StringRef FuncName, std::unique_ptr<FunctionInfo> Info) { + FunctionMap[FuncName].push_back(std::move(Info)); + } + + /// Iterator to allow writer to walk through table during emission. + iterator_range<StringMap<uint64_t>::const_iterator> + modPathStringEntries() const { + return llvm::make_range(ModulePathStringTable.begin(), + ModulePathStringTable.end()); + } + + /// Get the module ID recorded for the given module path. + uint64_t getModuleId(const StringRef ModPath) const { + return ModulePathStringTable.lookup(ModPath); + } + + /// Add the given per-module index into this function index/summary, + /// assigning it the given module ID. Each module merged in should have + /// a unique ID, necessary for consistent renaming of promoted + /// static (local) variables. + void mergeFrom(std::unique_ptr<FunctionInfoIndex> Other, + uint64_t NextModuleId); + + /// Convenience method for creating a promoted global name + /// for the given value name of a local, and its original module's ID. + static std::string getGlobalNameForLocal(StringRef Name, uint64_t ModId) { + SmallString<256> NewName(Name); + NewName += ".llvm."; + raw_svector_ostream(NewName) << ModId; + return NewName.str(); + } + + /// Add a new module path, mapped to the given module Id, and return StringRef + /// owned by string table map. + StringRef addModulePath(StringRef ModPath, uint64_t ModId) { + return ModulePathStringTable.insert(std::make_pair(ModPath, ModId)) + .first->first(); + } + + /// Check if the given Module has any functions available for exporting + /// in the index. We consider any module present in the ModulePathStringTable + /// to have exported functions. + bool hasExportedFunctions(const Module &M) const { + return ModulePathStringTable.count(M.getModuleIdentifier()); + } +}; + +} // End llvm namespace + +#endif diff --git a/include/llvm/IR/GVMaterializer.h b/include/llvm/IR/GVMaterializer.h index 1d6c9157f0b8..6cb593c7a3da 100644 --- a/include/llvm/IR/GVMaterializer.h +++ b/include/llvm/IR/GVMaterializer.h @@ -18,12 +18,14 @@ #ifndef LLVM_IR_GVMATERIALIZER_H #define LLVM_IR_GVMATERIALIZER_H +#include "llvm/ADT/DenseMap.h" #include <system_error> #include <vector> namespace llvm { class Function; class GlobalValue; +class Metadata; class Module; class StructType; @@ -34,28 +36,25 @@ protected: public: virtual ~GVMaterializer(); - /// True if GV has been materialized and can be dematerialized back to - /// whatever backing store this GVMaterializer uses. - virtual bool isDematerializable(const GlobalValue *GV) const = 0; - /// Make sure the given GlobalValue is fully read. /// virtual std::error_code materialize(GlobalValue *GV) = 0; - /// If the given GlobalValue is read in, and if the GVMaterializer supports - /// it, release the memory for the GV, and set it up to be materialized - /// lazily. If the Materializer doesn't support this capability, this method - /// is a noop. - /// - virtual void dematerialize(GlobalValue *) {} - /// Make sure the entire Module has been completely read. /// - virtual std::error_code materializeModule(Module *M) = 0; + virtual std::error_code materializeModule() = 0; virtual std::error_code materializeMetadata() = 0; virtual void setStripDebugInfo() = 0; + /// Client should define this interface if the mapping between metadata + /// values and value ids needs to be preserved, e.g. across materializer + /// instantiations. If OnlyTempMD is true, only those that have remained + /// temporary metadata are recorded in the map. + virtual void + saveMetadataList(DenseMap<const Metadata *, unsigned> &MetadataToIDs, + bool OnlyTempMD) {} + virtual std::vector<StructType *> getIdentifiedStructTypes() const = 0; }; diff --git a/include/llvm/IR/GetElementPtrTypeIterator.h b/include/llvm/IR/GetElementPtrTypeIterator.h index 6bba0ae29e98..7cb13fa33aa6 100644 --- a/include/llvm/IR/GetElementPtrTypeIterator.h +++ b/include/llvm/IR/GetElementPtrTypeIterator.h @@ -78,7 +78,7 @@ namespace llvm { // current type directly. Type *operator->() const { return operator*(); } - Value *getOperand() const { return *OpIt; } + Value *getOperand() const { return const_cast<Value *>(&**OpIt); } generic_gep_type_iterator& operator++() { // Preincrement if (CurTy.getInt()) { diff --git a/include/llvm/IR/GlobalAlias.h b/include/llvm/IR/GlobalAlias.h index ce73b7af8ca1..b0772143309f 100644 --- a/include/llvm/IR/GlobalAlias.h +++ b/include/llvm/IR/GlobalAlias.h @@ -23,18 +23,17 @@ namespace llvm { class Module; -template<typename ValueSubClass, typename ItemParentClass> - class SymbolTableListTraits; +template <typename ValueSubClass> class SymbolTableListTraits; class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> { - friend class SymbolTableListTraits<GlobalAlias, Module>; + friend class SymbolTableListTraits<GlobalAlias>; void operator=(const GlobalAlias &) = delete; GlobalAlias(const GlobalAlias &) = delete; void setParent(Module *parent); - GlobalAlias(PointerType *Ty, LinkageTypes Linkage, const Twine &Name, - Constant *Aliasee, Module *Parent); + GlobalAlias(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, + const Twine &Name, Constant *Aliasee, Module *Parent); public: // allocate space for exactly one operand @@ -44,17 +43,19 @@ public: /// If a parent module is specified, the alias is automatically inserted into /// the end of the specified module's alias list. - static GlobalAlias *create(PointerType *Ty, LinkageTypes Linkage, - const Twine &Name, Constant *Aliasee, - Module *Parent); + static GlobalAlias *create(Type *Ty, unsigned AddressSpace, + LinkageTypes Linkage, const Twine &Name, + Constant *Aliasee, Module *Parent); // Without the Aliasee. - static GlobalAlias *create(PointerType *Ty, LinkageTypes Linkage, - const Twine &Name, Module *Parent); + static GlobalAlias *create(Type *Ty, unsigned AddressSpace, + LinkageTypes Linkage, const Twine &Name, + Module *Parent); // The module is taken from the Aliasee. - static GlobalAlias *create(PointerType *Ty, LinkageTypes Linkage, - const Twine &Name, GlobalValue *Aliasee); + static GlobalAlias *create(Type *Ty, unsigned AddressSpace, + LinkageTypes Linkage, const Twine &Name, + GlobalValue *Aliasee); // Type, Parent and AddressSpace taken from the Aliasee. static GlobalAlias *create(LinkageTypes Linkage, const Twine &Name, diff --git a/include/llvm/IR/GlobalObject.h b/include/llvm/IR/GlobalObject.h index f0552410b61d..ee111a046d73 100644 --- a/include/llvm/IR/GlobalObject.h +++ b/include/llvm/IR/GlobalObject.h @@ -27,9 +27,11 @@ class GlobalObject : public GlobalValue { GlobalObject(const GlobalObject &) = delete; protected: - GlobalObject(PointerType *Ty, ValueTy VTy, Use *Ops, unsigned NumOps, - LinkageTypes Linkage, const Twine &Name) - : GlobalValue(Ty, VTy, Ops, NumOps, Linkage, Name), ObjComdat(nullptr) { + GlobalObject(Type *Ty, ValueTy VTy, Use *Ops, unsigned NumOps, + LinkageTypes Linkage, const Twine &Name, + unsigned AddressSpace = 0) + : GlobalValue(Ty, VTy, Ops, NumOps, Linkage, Name, AddressSpace), + ObjComdat(nullptr) { setGlobalValueSubClassData(0); } diff --git a/include/llvm/IR/GlobalValue.h b/include/llvm/IR/GlobalValue.h index 2961369a7327..4fa4e7daeab0 100644 --- a/include/llvm/IR/GlobalValue.h +++ b/include/llvm/IR/GlobalValue.h @@ -65,15 +65,16 @@ public: }; protected: - GlobalValue(PointerType *Ty, ValueTy VTy, Use *Ops, unsigned NumOps, - LinkageTypes Linkage, const Twine &Name) - : Constant(Ty, VTy, Ops, NumOps), Linkage(Linkage), - Visibility(DefaultVisibility), UnnamedAddr(0), - DllStorageClass(DefaultStorageClass), + GlobalValue(Type *Ty, ValueTy VTy, Use *Ops, unsigned NumOps, + LinkageTypes Linkage, const Twine &Name, unsigned AddressSpace) + : Constant(PointerType::get(Ty, AddressSpace), VTy, Ops, NumOps), + ValueType(Ty), Linkage(Linkage), Visibility(DefaultVisibility), + UnnamedAddr(0), DllStorageClass(DefaultStorageClass), ThreadLocal(NotThreadLocal), IntID((Intrinsic::ID)0U), Parent(nullptr) { setName(Name); } + Type *ValueType; // Note: VC++ treats enums as signed, so an extra bit is required to prevent // Linkage and Visibility from turning into negative values. LinkageTypes Linkage : 5; // The linkage of this global @@ -184,7 +185,7 @@ public: /// Global values are always pointers. PointerType *getType() const { return cast<PointerType>(User::getType()); } - Type *getValueType() const { return getType()->getElementType(); } + Type *getValueType() const { return ValueType; } static LinkageTypes getLinkOnceLinkage(bool ODR) { return ODR ? LinkOnceODRLinkage : LinkOnceAnyLinkage; @@ -236,7 +237,8 @@ public: /// Whether the definition of this global may be discarded if it is not used /// in its compilation unit. static bool isDiscardableIfUnused(LinkageTypes Linkage) { - return isLinkOnceLinkage(Linkage) || isLocalLinkage(Linkage); + return isLinkOnceLinkage(Linkage) || isLocalLinkage(Linkage) || + isAvailableExternallyLinkage(Linkage); } /// Whether the definition of this global may be replaced by something @@ -320,21 +322,11 @@ public: /// function has been read in yet or not. bool isMaterializable() const; - /// Returns true if this function was loaded from a GVMaterializer that's - /// still attached to its Module and that knows how to dematerialize the - /// function. - bool isDematerializable() const; - /// Make sure this GlobalValue is fully read. If the module is corrupt, this /// returns true and fills in the optional string with information about the /// problem. If successful, this returns false. std::error_code materialize(); - /// If this GlobalValue is read in, and if the GVMaterializer supports it, - /// release the memory for the function, and set it up to be materialized - /// lazily. If !isDematerializable(), this method is a noop. - void dematerialize(); - /// @} /// Return true if the primary definition of this global value is outside of diff --git a/include/llvm/IR/GlobalVariable.h b/include/llvm/IR/GlobalVariable.h index a0159830ba3b..342bdc01bfbd 100644 --- a/include/llvm/IR/GlobalVariable.h +++ b/include/llvm/IR/GlobalVariable.h @@ -29,11 +29,10 @@ namespace llvm { class Module; class Constant; -template<typename ValueSubClass, typename ItemParentClass> - class SymbolTableListTraits; +template <typename ValueSubClass> class SymbolTableListTraits; class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> { - friend class SymbolTableListTraits<GlobalVariable, Module>; + friend class SymbolTableListTraits<GlobalVariable>; void *operator new(size_t, unsigned) = delete; void operator=(const GlobalVariable &) = delete; GlobalVariable(const GlobalVariable &) = delete; @@ -106,18 +105,13 @@ public: /// hasUniqueInitializer - Whether the global variable has an initializer, and /// any changes made to the initializer will turn up in the final executable. inline bool hasUniqueInitializer() const { - return hasInitializer() && - // It's not safe to modify initializers of global variables with weak - // linkage, because the linker might choose to discard the initializer and - // use the initializer from another instance of the global variable - // instead. It is wrong to modify the initializer of a global variable - // with *_odr linkage because then different instances of the global may - // have different initializers, breaking the One Definition Rule. - !isWeakForLinker() && - // It is not safe to modify initializers of global variables with the - // external_initializer marker since the value may be changed at runtime - // before C++ initializers are evaluated. - !isExternallyInitialized(); + return + // We need to be sure this is the definition that will actually be used + isStrongDefinitionForLinker() && + // It is not safe to modify initializers of global variables with the + // external_initializer marker since the value may be changed at runtime + // before C++ initializers are evaluated. + !isExternallyInitialized(); } /// getInitializer - Return the initializer for this global variable. It is diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index 6c67c79b6c0e..7fe04f2a091a 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -24,6 +24,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Operator.h" #include "llvm/IR/ValueHandle.h" @@ -51,6 +52,7 @@ protected: /// \brief Common base class shared among various IRBuilders. class IRBuilderBase { DebugLoc CurDbgLocation; + protected: BasicBlock *BB; BasicBlock::iterator InsertPt; @@ -58,8 +60,8 @@ protected: MDNode *DefaultFPMathTag; FastMathFlags FMF; -public: +public: IRBuilderBase(LLVMContext &context, MDNode *FPMathTag = nullptr) : Context(context), DefaultFPMathTag(FPMathTag), FMF() { ClearInsertionPoint(); @@ -73,7 +75,7 @@ public: /// inserted into a block. void ClearInsertionPoint() { BB = nullptr; - InsertPt = nullptr; + InsertPt.reset(nullptr); } BasicBlock *GetInsertBlock() const { return BB; } @@ -91,8 +93,8 @@ public: /// the specified instruction. void SetInsertPoint(Instruction *I) { BB = I->getParent(); - InsertPt = I; - assert(I != BB->end() && "Can't read debug loc from end()"); + InsertPt = I->getIterator(); + assert(InsertPt != BB->end() && "Can't read debug loc from end()"); SetCurrentDebugLocation(I->getDebugLoc()); } @@ -313,10 +315,8 @@ public: } /// \brief Fetch the type representing a 128-bit integer. - IntegerType *getInt128Ty() { - return Type::getInt128Ty(Context); - } - + IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); } + /// \brief Fetch the type representing an N-bit integer. IntegerType *getIntNTy(unsigned N) { return Type::getIntNTy(Context, N); @@ -426,7 +426,7 @@ public: /// \brief Create a call to Masked Load intrinsic CallInst *CreateMaskedLoad(Value *Ptr, unsigned Align, Value *Mask, - Value *PassThru = 0, const Twine &Name = ""); + Value *PassThru = nullptr, const Twine &Name = ""); /// \brief Create a call to Masked Store intrinsic CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align, @@ -445,6 +445,16 @@ public: ArrayRef<Value *> GCArgs, const Twine &Name = ""); + /// \brief Create a call to the experimental.gc.statepoint intrinsic to + /// start a new statepoint sequence. + CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, + Value *ActualCallee, uint32_t Flags, + ArrayRef<Use> CallArgs, + ArrayRef<Use> TransitionArgs, + ArrayRef<Use> DeoptArgs, + ArrayRef<Value *> GCArgs, + const Twine &Name = ""); + // \brief Conveninence function for the common case when CallArgs are filled // in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be // .get()'ed to get the Value pointer. @@ -463,6 +473,15 @@ public: ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name = ""); + /// brief Create an invoke to the experimental.gc.statepoint intrinsic to + /// start a new statepoint sequence. + InvokeInst *CreateGCStatepointInvoke( + uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, + BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, + ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs, + ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, + const Twine &Name = ""); + // Conveninence function for the common case when CallArgs are filled in using // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to // get the Value *. @@ -516,11 +535,11 @@ template<bool preserveNames = true, typename T = ConstantFolder, typename Inserter = IRBuilderDefaultInserter<preserveNames> > class IRBuilder : public IRBuilderBase, public Inserter { T Folder; + public: - IRBuilder(LLVMContext &C, const T &F, const Inserter &I = Inserter(), + IRBuilder(LLVMContext &C, const T &F, Inserter I = Inserter(), MDNode *FPMathTag = nullptr) - : IRBuilderBase(C, FPMathTag), Inserter(I), Folder(F) { - } + : IRBuilderBase(C, FPMathTag), Inserter(std::move(I)), Folder(F) {} explicit IRBuilder(LLVMContext &C, MDNode *FPMathTag = nullptr) : IRBuilderBase(C, FPMathTag), Folder() { @@ -578,12 +597,15 @@ public: //===--------------------------------------------------------------------===// private: - /// \brief Helper to add branch weight metadata onto an instruction. + /// \brief Helper to add branch weight and unpredictable metadata onto an + /// instruction. /// \returns The annotated instruction. template <typename InstTy> - InstTy *addBranchWeights(InstTy *I, MDNode *Weights) { + InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) { if (Weights) I->setMetadata(LLVMContext::MD_prof, Weights); + if (Unpredictable) + I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable); return I; } @@ -620,18 +642,20 @@ public: /// \brief Create a conditional 'br Cond, TrueDest, FalseDest' /// instruction. BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, - MDNode *BranchWeights = nullptr) { - return Insert(addBranchWeights(BranchInst::Create(True, False, Cond), - BranchWeights)); + MDNode *BranchWeights = nullptr, + MDNode *Unpredictable = nullptr) { + return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond), + BranchWeights, Unpredictable)); } /// \brief Create a switch instruction with the specified value, default dest, /// and with a hint for the number of cases that will be added (for efficient /// allocation). SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10, - MDNode *BranchWeights = nullptr) { - return Insert(addBranchWeights(SwitchInst::Create(V, Dest, NumCases), - BranchWeights)); + MDNode *BranchWeights = nullptr, + MDNode *Unpredictable = nullptr) { + return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases), + BranchWeights, Unpredictable)); } /// \brief Create an indirect branch instruction with the specified address @@ -667,11 +691,45 @@ public: return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, Args), Name); } + InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, + BasicBlock *UnwindDest, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> OpBundles, + const Twine &Name = "") { + return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, Args, + OpBundles), Name); + } ResumeInst *CreateResume(Value *Exn) { return Insert(ResumeInst::Create(Exn)); } + CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad, + BasicBlock *UnwindBB = nullptr) { + return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB)); + } + + CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB, + unsigned NumHandlers, + const Twine &Name = "") { + return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers), + Name); + } + + CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args, + const Twine &Name = "") { + return Insert(CatchPadInst::Create(ParentPad, Args), Name); + } + + CleanupPadInst *CreateCleanupPad(Value *ParentPad, + ArrayRef<Value *> Args = None, + const Twine &Name = "") { + return Insert(CleanupPadInst::Create(ParentPad, Args), Name); + } + + CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) { + return Insert(CatchReturnInst::Create(CatchPad, BB)); + } + UnreachableInst *CreateUnreachable() { return Insert(new UnreachableInst(Context)); } @@ -700,6 +758,7 @@ private: I->setFastMathFlags(FMF); return I; } + public: Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { @@ -1326,18 +1385,22 @@ public: const Twine &Name = "") { if (V->getType() == DestTy) return V; - if (V->getType()->isPointerTy() && DestTy->isIntegerTy()) + if (V->getType()->getScalarType()->isPointerTy() && + DestTy->getScalarType()->isIntegerTy()) return CreatePtrToInt(V, DestTy, Name); - if (V->getType()->isIntegerTy() && DestTy->isPointerTy()) + if (V->getType()->getScalarType()->isIntegerTy() && + DestTy->getScalarType()->isPointerTy()) return CreateIntToPtr(V, DestTy, Name); return CreateBitCast(V, DestTy, Name); } + private: // \brief Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a // compile time error, instead of converting the string to bool for the // isSigned parameter. Value *CreateIntCast(Value *, Type *, const char *) = delete; + public: Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") { if (V->getType() == DestTy) @@ -1465,18 +1528,30 @@ public: } CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args = None, + ArrayRef<OperandBundleDef> OpBundles = None, const Twine &Name = "") { - return Insert(CallInst::Create(Callee, Args), Name); + return Insert(CallInst::Create(Callee, Args, OpBundles), Name); + } + + CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args, + const Twine &Name, MDNode *FPMathTag = nullptr) { + PointerType *PTy = cast<PointerType>(Callee->getType()); + FunctionType *FTy = cast<FunctionType>(PTy->getElementType()); + return CreateCall(FTy, Callee, Args, Name, FPMathTag); } CallInst *CreateCall(llvm::FunctionType *FTy, Value *Callee, - ArrayRef<Value *> Args, const Twine &Name = "") { - return Insert(CallInst::Create(FTy, Callee, Args), Name); + ArrayRef<Value *> Args, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + CallInst *CI = CallInst::Create(FTy, Callee, Args); + if (isa<FPMathOperator>(CI)) + CI = cast<CallInst>(AddFPMathAttributes(CI, FPMathTag, FMF)); + return Insert(CI, Name); } CallInst *CreateCall(Function *Callee, ArrayRef<Value *> Args, - const Twine &Name = "") { - return CreateCall(Callee->getFunctionType(), Callee, Args, Name); + const Twine &Name = "", MDNode *FPMathTag = nullptr) { + return CreateCall(Callee->getFunctionType(), Callee, Args, Name, FPMathTag); } Value *CreateSelect(Value *C, Value *True, Value *False, @@ -1594,6 +1669,32 @@ public: Name); } + /// \brief Create an invariant.group.barrier intrinsic call, that stops + /// optimizer to propagate equality using invariant.group metadata. + /// If Ptr type is different from i8*, it's casted to i8* before call + /// and casted back to Ptr type after call. + Value *CreateInvariantGroupBarrier(Value *Ptr) { + Module *M = BB->getParent()->getParent(); + Function *FnInvariantGroupBarrier = Intrinsic::getDeclaration(M, + Intrinsic::invariant_group_barrier); + + Type *ArgumentAndReturnType = FnInvariantGroupBarrier->getReturnType(); + assert(ArgumentAndReturnType == + FnInvariantGroupBarrier->getFunctionType()->getParamType(0) && + "InvariantGroupBarrier should take and return the same type"); + Type *PtrType = Ptr->getType(); + + bool PtrTypeConversionNeeded = PtrType != ArgumentAndReturnType; + if (PtrTypeConversionNeeded) + Ptr = CreateBitCast(Ptr, ArgumentAndReturnType); + + CallInst *Fn = CreateCall(FnInvariantGroupBarrier, {Ptr}); + + if (PtrTypeConversionNeeded) + return CreateBitCast(Fn, PtrType); + return Fn; + } + /// \brief Return a vector value that contains \arg V broadcasted to \p /// NumElts elements. Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "") { @@ -1676,6 +1777,6 @@ public: // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRBuilder<>, LLVMBuilderRef) -} +} // end namespace llvm -#endif +#endif // LLVM_IR_IRBUILDER_H diff --git a/include/llvm/IR/IRPrintingPasses.h b/include/llvm/IR/IRPrintingPasses.h index 5f1d56f7e831..88b18e826daf 100644 --- a/include/llvm/IR/IRPrintingPasses.h +++ b/include/llvm/IR/IRPrintingPasses.h @@ -47,6 +47,12 @@ FunctionPass *createPrintFunctionPass(raw_ostream &OS, BasicBlockPass *createPrintBasicBlockPass(raw_ostream &OS, const std::string &Banner = ""); +/// Print out a name of an LLVM value without any prefixes. +/// +/// The name is surrounded with ""'s and escaped if it has any special or +/// non-printable characters in it. +void printLLVMNameWithoutPrefix(raw_ostream &OS, StringRef Name); + /// \brief Pass for printing a Module as LLVM's text IR assembly. /// /// Note: This pass is for use with the new pass manager. Use the create...Pass diff --git a/include/llvm/IR/InlineAsm.h b/include/llvm/IR/InlineAsm.h index 08b51021116c..d2e9e48539ce 100644 --- a/include/llvm/IR/InlineAsm.h +++ b/include/llvm/IR/InlineAsm.h @@ -44,11 +44,12 @@ private: void operator=(const InlineAsm&) = delete; std::string AsmString, Constraints; + FunctionType *FTy; bool HasSideEffects; bool IsAlignStack; AsmDialect Dialect; - InlineAsm(PointerType *Ty, const std::string &AsmString, + InlineAsm(FunctionType *Ty, const std::string &AsmString, const std::string &Constraints, bool hasSideEffects, bool isAlignStack, AsmDialect asmDialect); ~InlineAsm() override; @@ -56,15 +57,15 @@ private: /// When the ConstantUniqueMap merges two types and makes two InlineAsms /// identical, it destroys one of them with this method. void destroyConstant(); -public: +public: /// InlineAsm::get - Return the specified uniqued inline asm string. /// static InlineAsm *get(FunctionType *Ty, StringRef AsmString, StringRef Constraints, bool hasSideEffects, bool isAlignStack = false, AsmDialect asmDialect = AD_ATT); - + bool hasSideEffects() const { return HasSideEffects; } bool isAlignStack() const { return IsAlignStack; } AsmDialect getDialect() const { return Dialect; } @@ -74,11 +75,11 @@ public: PointerType *getType() const { return reinterpret_cast<PointerType*>(Value::getType()); } - + /// getFunctionType - InlineAsm's are always pointers to functions. /// FunctionType *getFunctionType() const; - + const std::string &getAsmString() const { return AsmString; } const std::string &getConstraintString() const { return Constraints; } @@ -88,15 +89,15 @@ public: /// static bool Verify(FunctionType *Ty, StringRef Constraints); - // Constraint String Parsing + // Constraint String Parsing enum ConstraintPrefix { isInput, // 'x' isOutput, // '=x' isClobber // '~x' }; - + typedef std::vector<std::string> ConstraintCodeVector; - + struct SubConstraintInfo { /// MatchingInput - If this is not -1, this is an output constraint where an /// input constraint is required to match it (e.g. "0"). The value is the @@ -113,80 +114,79 @@ public: typedef std::vector<SubConstraintInfo> SubConstraintInfoVector; struct ConstraintInfo; typedef std::vector<ConstraintInfo> ConstraintInfoVector; - + struct ConstraintInfo { /// Type - The basic type of the constraint: input/output/clobber /// ConstraintPrefix Type; - + /// isEarlyClobber - "&": output operand writes result before inputs are all /// read. This is only ever set for an output operand. - bool isEarlyClobber; - + bool isEarlyClobber; + /// MatchingInput - If this is not -1, this is an output constraint where an /// input constraint is required to match it (e.g. "0"). The value is the /// constraint number that matches this one (for example, if this is /// constraint #0 and constraint #4 has the value "0", this will be 4). signed char MatchingInput; - + /// hasMatchingInput - Return true if this is an output constraint that has /// a matching input constraint. bool hasMatchingInput() const { return MatchingInput != -1; } - + /// isCommutative - This is set to true for a constraint that is commutative /// with the next operand. bool isCommutative; - + /// isIndirect - True if this operand is an indirect operand. This means /// that the address of the source or destination is present in the call /// instruction, instead of it being returned or passed in explicitly. This /// is represented with a '*' in the asm string. bool isIndirect; - + /// Code - The constraint code, either the register name (in braces) or the /// constraint letter/number. ConstraintCodeVector Codes; - + /// isMultipleAlternative - '|': has multiple-alternative constraints. bool isMultipleAlternative; - + /// multipleAlternatives - If there are multiple alternative constraints, /// this array will contain them. Otherwise it will be empty. SubConstraintInfoVector multipleAlternatives; - + /// The currently selected alternative constraint index. unsigned currentAlternativeIndex; - - ///Default constructor. + + /// Default constructor. ConstraintInfo(); - + /// Parse - Analyze the specified string (e.g. "=*&{eax}") and fill in the /// fields in this structure. If the constraint string is not understood, /// return true, otherwise return false. bool Parse(StringRef Str, ConstraintInfoVector &ConstraintsSoFar); - + /// selectAlternative - Point this constraint to the alternative constraint /// indicated by the index. void selectAlternative(unsigned index); }; - + /// ParseConstraints - Split up the constraint string into the specific /// constraints and their prefixes. If this returns an empty vector, and if /// the constraint string itself isn't empty, there was an error parsing. static ConstraintInfoVector ParseConstraints(StringRef ConstraintString); - - /// ParseConstraints - Parse the constraints of this inlineasm object, + + /// ParseConstraints - Parse the constraints of this inlineasm object, /// returning them the same way that ParseConstraints(str) does. ConstraintInfoVector ParseConstraints() const { return ParseConstraints(Constraints); } - + // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Value *V) { return V->getValueID() == Value::InlineAsmVal; } - // These are helper methods for dealing with flags in the INLINEASM SDNode // in the backend. // @@ -203,7 +203,7 @@ public: // code. // Else: // Bit 30-16 - The register class ID to use for the operand. - + enum : uint32_t { // Fixed operands on an INLINEASM SDNode. Op_InputChain = 0, @@ -264,15 +264,15 @@ public: Flag_MatchingOperand = 0x80000000 }; - + static unsigned getFlagWord(unsigned Kind, unsigned NumOps) { assert(((NumOps << 3) & ~0xffff) == 0 && "Too many inline asm operands!"); assert(Kind >= Kind_RegUse && Kind <= Kind_Mem && "Invalid Kind"); return Kind | (NumOps << 3); } - + /// getFlagWordForMatchingOp - Augment an existing flag word returned by - /// getFlagWord with information indicating that this input operand is tied + /// getFlagWord with information indicating that this input operand is tied /// to a previous output operand. static unsigned getFlagWordForMatchingOp(unsigned InputFlag, unsigned MatchedOperandNo) { @@ -355,7 +355,6 @@ public: RC = High - 1; return true; } - }; } // End llvm namespace diff --git a/include/llvm/IR/InstIterator.h b/include/llvm/IR/InstIterator.h index f3ce6490fb66..1baca21c73af 100644 --- a/include/llvm/IR/InstIterator.h +++ b/include/llvm/IR/InstIterator.h @@ -115,19 +115,18 @@ private: } }; - -typedef InstIterator<iplist<BasicBlock>, - Function::iterator, BasicBlock::iterator, - Instruction> inst_iterator; -typedef InstIterator<const iplist<BasicBlock>, - Function::const_iterator, - BasicBlock::const_iterator, +typedef InstIterator<SymbolTableList<BasicBlock>, Function::iterator, + BasicBlock::iterator, Instruction> inst_iterator; +typedef InstIterator<const SymbolTableList<BasicBlock>, + Function::const_iterator, BasicBlock::const_iterator, const Instruction> const_inst_iterator; +typedef iterator_range<inst_iterator> inst_range; +typedef iterator_range<const_inst_iterator> const_inst_range; inline inst_iterator inst_begin(Function *F) { return inst_iterator(*F); } inline inst_iterator inst_end(Function *F) { return inst_iterator(*F, true); } -inline iterator_range<inst_iterator> inst_range(Function *F) { - return iterator_range<inst_iterator>(inst_begin(F), inst_end(F)); +inline inst_range instructions(Function *F) { + return inst_range(inst_begin(F), inst_end(F)); } inline const_inst_iterator inst_begin(const Function *F) { return const_inst_iterator(*F); @@ -135,13 +134,13 @@ inline const_inst_iterator inst_begin(const Function *F) { inline const_inst_iterator inst_end(const Function *F) { return const_inst_iterator(*F, true); } -inline iterator_range<const_inst_iterator> inst_range(const Function *F) { - return iterator_range<const_inst_iterator>(inst_begin(F), inst_end(F)); +inline const_inst_range instructions(const Function *F) { + return const_inst_range(inst_begin(F), inst_end(F)); } inline inst_iterator inst_begin(Function &F) { return inst_iterator(F); } inline inst_iterator inst_end(Function &F) { return inst_iterator(F, true); } -inline iterator_range<inst_iterator> inst_range(Function &F) { - return iterator_range<inst_iterator>(inst_begin(F), inst_end(F)); +inline inst_range instructions(Function &F) { + return inst_range(inst_begin(F), inst_end(F)); } inline const_inst_iterator inst_begin(const Function &F) { return const_inst_iterator(F); @@ -149,8 +148,8 @@ inline const_inst_iterator inst_begin(const Function &F) { inline const_inst_iterator inst_end(const Function &F) { return const_inst_iterator(F, true); } -inline iterator_range<const_inst_iterator> inst_range(const Function &F) { - return iterator_range<const_inst_iterator>(inst_begin(F), inst_end(F)); +inline const_inst_range instructions(const Function &F) { + return const_inst_range(inst_begin(F), inst_end(F)); } } // End llvm namespace diff --git a/include/llvm/IR/InstVisitor.h b/include/llvm/IR/InstVisitor.h index 581e860b8382..088d3e0fbfa5 100644 --- a/include/llvm/IR/InstVisitor.h +++ b/include/llvm/IR/InstVisitor.h @@ -169,6 +169,9 @@ public: RetTy visitIndirectBrInst(IndirectBrInst &I) { DELEGATE(TerminatorInst);} RetTy visitResumeInst(ResumeInst &I) { DELEGATE(TerminatorInst);} RetTy visitUnreachableInst(UnreachableInst &I) { DELEGATE(TerminatorInst);} + RetTy visitCleanupReturnInst(CleanupReturnInst &I) { DELEGATE(TerminatorInst);} + RetTy visitCatchReturnInst(CatchReturnInst &I) { DELEGATE(TerminatorInst); } + RetTy visitCatchSwitchInst(CatchSwitchInst &I) { DELEGATE(TerminatorInst);} RetTy visitICmpInst(ICmpInst &I) { DELEGATE(CmpInst);} RetTy visitFCmpInst(FCmpInst &I) { DELEGATE(CmpInst);} RetTy visitAllocaInst(AllocaInst &I) { DELEGATE(UnaryInstruction);} @@ -200,6 +203,9 @@ public: RetTy visitExtractValueInst(ExtractValueInst &I){ DELEGATE(UnaryInstruction);} RetTy visitInsertValueInst(InsertValueInst &I) { DELEGATE(Instruction); } RetTy visitLandingPadInst(LandingPadInst &I) { DELEGATE(Instruction); } + RetTy visitFuncletPadInst(FuncletPadInst &I) { DELEGATE(Instruction); } + RetTy visitCleanupPadInst(CleanupPadInst &I) { DELEGATE(FuncletPadInst); } + RetTy visitCatchPadInst(CatchPadInst &I) { DELEGATE(FuncletPadInst); } // Handle the special instrinsic instruction classes. RetTy visitDbgDeclareInst(DbgDeclareInst &I) { DELEGATE(DbgInfoIntrinsic);} diff --git a/include/llvm/IR/InstrTypes.h b/include/llvm/IR/InstrTypes.h index b791ded0e194..5091bb407833 100644 --- a/include/llvm/IR/InstrTypes.h +++ b/include/llvm/IR/InstrTypes.h @@ -16,9 +16,12 @@ #ifndef LLVM_IR_INSTRTYPES_H #define LLVM_IR_INSTRTYPES_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/Twine.h" +#include "llvm/IR/Attributes.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/OperandTraits.h" namespace llvm { @@ -51,8 +54,8 @@ protected: virtual BasicBlock *getSuccessorV(unsigned idx) const = 0; virtual unsigned getNumSuccessorsV() const = 0; virtual void setSuccessorV(unsigned idx, BasicBlock *B) = 0; -public: +public: /// Return the number of successors that this terminator has. unsigned getNumSuccessors() const { return getNumSuccessorsV(); @@ -75,8 +78,198 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } -}; + // \brief Returns true if this terminator relates to exception handling. + bool isExceptional() const { + switch (getOpcode()) { + case Instruction::CatchSwitch: + case Instruction::CatchRet: + case Instruction::CleanupRet: + case Instruction::Invoke: + case Instruction::Resume: + return true; + default: + return false; + } + } + + //===--------------------------------------------------------------------===// + // succ_iterator definition + //===--------------------------------------------------------------------===// + + template <class Term, class BB> // Successor Iterator + class SuccIterator : public std::iterator<std::random_access_iterator_tag, BB, + int, BB *, BB *> { + typedef std::iterator<std::random_access_iterator_tag, BB, int, BB *, BB *> + super; + + public: + typedef typename super::pointer pointer; + typedef typename super::reference reference; + + private: + Term TermInst; + unsigned idx; + typedef SuccIterator<Term, BB> Self; + + inline bool index_is_valid(unsigned idx) { + return idx < TermInst->getNumSuccessors(); + } + + /// \brief Proxy object to allow write access in operator[] + class SuccessorProxy { + Self it; + + public: + explicit SuccessorProxy(const Self &it) : it(it) {} + + SuccessorProxy(const SuccessorProxy &) = default; + + SuccessorProxy &operator=(SuccessorProxy r) { + *this = reference(r); + return *this; + } + + SuccessorProxy &operator=(reference r) { + it.TermInst->setSuccessor(it.idx, r); + return *this; + } + + operator reference() const { return *it; } + }; + + public: + // begin iterator + explicit inline SuccIterator(Term T) : TermInst(T), idx(0) {} + // end iterator + inline SuccIterator(Term T, bool) : TermInst(T) { + if (TermInst) + idx = TermInst->getNumSuccessors(); + else + // Term == NULL happens, if a basic block is not fully constructed and + // consequently getTerminator() returns NULL. In this case we construct + // a SuccIterator which describes a basic block that has zero + // successors. + // Defining SuccIterator for incomplete and malformed CFGs is especially + // useful for debugging. + idx = 0; + } + + /// This is used to interface between code that wants to + /// operate on terminator instructions directly. + unsigned getSuccessorIndex() const { return idx; } + + inline bool operator==(const Self &x) const { return idx == x.idx; } + inline bool operator!=(const Self &x) const { return !operator==(x); } + + inline reference operator*() const { return TermInst->getSuccessor(idx); } + inline pointer operator->() const { return operator*(); } + + inline Self &operator++() { + ++idx; + return *this; + } // Preincrement + + inline Self operator++(int) { // Postincrement + Self tmp = *this; + ++*this; + return tmp; + } + + inline Self &operator--() { + --idx; + return *this; + } // Predecrement + inline Self operator--(int) { // Postdecrement + Self tmp = *this; + --*this; + return tmp; + } + + inline bool operator<(const Self &x) const { + assert(TermInst == x.TermInst && + "Cannot compare iterators of different blocks!"); + return idx < x.idx; + } + + inline bool operator<=(const Self &x) const { + assert(TermInst == x.TermInst && + "Cannot compare iterators of different blocks!"); + return idx <= x.idx; + } + inline bool operator>=(const Self &x) const { + assert(TermInst == x.TermInst && + "Cannot compare iterators of different blocks!"); + return idx >= x.idx; + } + + inline bool operator>(const Self &x) const { + assert(TermInst == x.TermInst && + "Cannot compare iterators of different blocks!"); + return idx > x.idx; + } + + inline Self &operator+=(int Right) { + unsigned new_idx = idx + Right; + assert(index_is_valid(new_idx) && "Iterator index out of bound"); + idx = new_idx; + return *this; + } + + inline Self operator+(int Right) const { + Self tmp = *this; + tmp += Right; + return tmp; + } + + inline Self &operator-=(int Right) { return operator+=(-Right); } + + inline Self operator-(int Right) const { return operator+(-Right); } + + inline int operator-(const Self &x) const { + assert(TermInst == x.TermInst && + "Cannot work on iterators of different blocks!"); + int distance = idx - x.idx; + return distance; + } + + inline SuccessorProxy operator[](int offset) { + Self tmp = *this; + tmp += offset; + return SuccessorProxy(tmp); + } + + /// Get the source BB of this iterator. + inline BB *getSource() { + assert(TermInst && "Source not available, if basic block was malformed"); + return TermInst->getParent(); + } + }; + + typedef SuccIterator<TerminatorInst *, BasicBlock> succ_iterator; + typedef SuccIterator<const TerminatorInst *, const BasicBlock> + succ_const_iterator; + typedef llvm::iterator_range<succ_iterator> succ_range; + typedef llvm::iterator_range<succ_const_iterator> succ_const_range; + +private: + inline succ_iterator succ_begin() { return succ_iterator(this); } + inline succ_const_iterator succ_begin() const { + return succ_const_iterator(this); + } + inline succ_iterator succ_end() { return succ_iterator(this, true); } + inline succ_const_iterator succ_end() const { + return succ_const_iterator(this, true); + } + +public: + inline succ_range successors() { + return succ_range(succ_begin(), succ_end()); + } + inline succ_const_range successors() const { + return succ_const_range(succ_begin(), succ_end()); + } +}; //===----------------------------------------------------------------------===// // UnaryInstruction Class @@ -95,6 +288,7 @@ protected: : Instruction(Ty, iType, &Op<0>(), 1, IAE) { Op<0>() = V; } + public: // allocate space for exactly one operand void *operator new(size_t s) { @@ -133,6 +327,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(UnaryInstruction, Value) class BinaryOperator : public Instruction { void *operator new(size_t, unsigned) = delete; + protected: void init(BinaryOps iType); BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty, @@ -209,7 +404,7 @@ public: BO->setHasNoSignedWrap(true); return BO; } - + static BinaryOperator *CreateNUW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name = "") { BinaryOperator *BO = Create(Opc, V1, V2, Name); @@ -228,7 +423,7 @@ public: BO->setHasNoUnsignedWrap(true); return BO; } - + static BinaryOperator *CreateExact(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name = "") { BinaryOperator *BO = Create(Opc, V1, V2, Name); @@ -247,29 +442,29 @@ public: BO->setIsExact(true); return BO; } - -#define DEFINE_HELPERS(OPC, NUWNSWEXACT) \ - static BinaryOperator *Create ## NUWNSWEXACT ## OPC \ - (Value *V1, Value *V2, const Twine &Name = "") { \ - return Create ## NUWNSWEXACT(Instruction::OPC, V1, V2, Name); \ - } \ - static BinaryOperator *Create ## NUWNSWEXACT ## OPC \ - (Value *V1, Value *V2, const Twine &Name, BasicBlock *BB) { \ - return Create ## NUWNSWEXACT(Instruction::OPC, V1, V2, Name, BB); \ - } \ - static BinaryOperator *Create ## NUWNSWEXACT ## OPC \ - (Value *V1, Value *V2, const Twine &Name, Instruction *I) { \ - return Create ## NUWNSWEXACT(Instruction::OPC, V1, V2, Name, I); \ - } - - DEFINE_HELPERS(Add, NSW) // CreateNSWAdd - DEFINE_HELPERS(Add, NUW) // CreateNUWAdd - DEFINE_HELPERS(Sub, NSW) // CreateNSWSub - DEFINE_HELPERS(Sub, NUW) // CreateNUWSub - DEFINE_HELPERS(Mul, NSW) // CreateNSWMul - DEFINE_HELPERS(Mul, NUW) // CreateNUWMul - DEFINE_HELPERS(Shl, NSW) // CreateNSWShl - DEFINE_HELPERS(Shl, NUW) // CreateNUWShl + +#define DEFINE_HELPERS(OPC, NUWNSWEXACT) \ + static BinaryOperator *Create##NUWNSWEXACT##OPC(Value *V1, Value *V2, \ + const Twine &Name = "") { \ + return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name); \ + } \ + static BinaryOperator *Create##NUWNSWEXACT##OPC( \ + Value *V1, Value *V2, const Twine &Name, BasicBlock *BB) { \ + return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name, BB); \ + } \ + static BinaryOperator *Create##NUWNSWEXACT##OPC( \ + Value *V1, Value *V2, const Twine &Name, Instruction *I) { \ + return Create##NUWNSWEXACT(Instruction::OPC, V1, V2, Name, I); \ + } + + DEFINE_HELPERS(Add, NSW) // CreateNSWAdd + DEFINE_HELPERS(Add, NUW) // CreateNUWAdd + DEFINE_HELPERS(Sub, NSW) // CreateNSWSub + DEFINE_HELPERS(Sub, NUW) // CreateNUWSub + DEFINE_HELPERS(Mul, NSW) // CreateNSWMul + DEFINE_HELPERS(Mul, NUW) // CreateNUWMul + DEFINE_HELPERS(Shl, NSW) // CreateNSWShl + DEFINE_HELPERS(Shl, NUW) // CreateNUWShl DEFINE_HELPERS(SDiv, Exact) // CreateExactSDiv DEFINE_HELPERS(UDiv, Exact) // CreateExactUDiv @@ -277,7 +472,7 @@ public: DEFINE_HELPERS(LShr, Exact) // CreateExactLShr #undef DEFINE_HELPERS - + /// Helper functions to construct and inspect unary operations (NEG and NOT) /// via binary operators SUB and XOR: /// @@ -355,7 +550,7 @@ public: /// Convenience method to copy supported wrapping, exact, and fast-math flags /// from V to this instruction. void copyIRFlags(const Value *V); - + /// Logical 'and' of any supported wrapping, exact, and fast-math flags of /// V and this instruction. void andIRFlags(const Value *V); @@ -388,6 +583,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BinaryOperator, Value) /// @brief Base class of casting instructions. class CastInst : public UnaryInstruction { void anchor() override; + protected: /// @brief Constructor with insert-before-instruction semantics for subclasses CastInst(Type *Ty, unsigned iType, Value *S, @@ -401,6 +597,7 @@ protected: : UnaryInstruction(Ty, iType, S, InsertAtEnd) { setName(NameStr); } + public: /// Provides a way to construct any of the CastInst subclasses using an /// opcode instead of the subclass's constructor. The opcode must be in the @@ -490,7 +687,7 @@ public: Value *S, ///< The pointer value to be casted (operand 0) Type *Ty, ///< The type to which cast should be made const Twine &Name = "", ///< Name for the instruction - Instruction *InsertBefore = 0 ///< Place to insert the instruction + Instruction *InsertBefore = nullptr ///< Place to insert the instruction ); /// @brief Create a BitCast, a PtrToInt, or an IntToPTr cast instruction. @@ -503,7 +700,7 @@ public: Value *S, ///< The pointer value to be casted (operand 0) Type *Ty, ///< The type to which cast should be made const Twine &Name = "", ///< Name for the instruction - Instruction *InsertBefore = 0 ///< Place to insert the instruction + Instruction *InsertBefore = nullptr ///< Place to insert the instruction ); /// @brief Create a ZExt, BitCast, or Trunc for int -> int casts. @@ -677,18 +874,6 @@ public: /// This class is the base class for the comparison instructions. /// @brief Abstract base class of comparison instructions. class CmpInst : public Instruction { - void *operator new(size_t, unsigned) = delete; - CmpInst() = delete; -protected: - CmpInst(Type *ty, Instruction::OtherOps op, unsigned short pred, - Value *LHS, Value *RHS, const Twine &Name = "", - Instruction *InsertBefore = nullptr); - - CmpInst(Type *ty, Instruction::OtherOps op, unsigned short pred, - Value *LHS, Value *RHS, const Twine &Name, - BasicBlock *InsertAtEnd); - - void anchor() override; // Out of line virtual method. public: /// This enumeration lists the possible predicates for CmpInst subclasses. /// Values in the range 0-31 are reserved for FCmpInst, while values in the @@ -730,6 +915,22 @@ public: BAD_ICMP_PREDICATE = ICMP_SLE + 1 }; +private: + void *operator new(size_t, unsigned) = delete; + CmpInst() = delete; + +protected: + CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred, + Value *LHS, Value *RHS, const Twine &Name = "", + Instruction *InsertBefore = nullptr); + + CmpInst(Type *ty, Instruction::OtherOps op, Predicate pred, + Value *LHS, Value *RHS, const Twine &Name, + BasicBlock *InsertAtEnd); + + void anchor() override; // Out of line virtual method. + +public: // allocate space for exactly two operands void *operator new(size_t s) { return User::operator new(s, 2); @@ -740,7 +941,7 @@ public: /// The specified Instruction is allowed to be a dereferenced end iterator. /// @brief Create a CmpInst static CmpInst *Create(OtherOps Op, - unsigned short predicate, Value *S1, + Predicate predicate, Value *S1, Value *S2, const Twine &Name = "", Instruction *InsertBefore = nullptr); @@ -748,7 +949,7 @@ public: /// two operands. Also automatically insert this instruction to the end of /// the BasicBlock specified. /// @brief Create a CmpInst - static CmpInst *Create(OtherOps Op, unsigned short predicate, Value *S1, + static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name, BasicBlock *InsertAtEnd); /// @brief Get the opcode casted to the right type @@ -775,7 +976,6 @@ public: bool isFPPredicate() const { return isFPPredicate(getPredicate()); } bool isIntPredicate() const { return isIntPredicate(getPredicate()); } - /// For example, EQ -> NE, UGT -> ULE, SLT -> SGE, /// OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc. /// @returns the inverse predicate for the instruction's current predicate. @@ -833,6 +1033,19 @@ public: return isUnsigned(getPredicate()); } + /// For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert + /// @returns the signed version of the unsigned predicate pred. + /// @brief return the signed version of a predicate + static Predicate getSignedPredicate(Predicate pred); + + /// For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert + /// @returns the signed version of the predicate for this instruction (which + /// has to be an unsigned predicate). + /// @brief return the signed version of a predicate + Predicate getSignedPredicate() { + return getSignedPredicate(getPredicate()); + } + /// This is just a convenience. /// @brief Determine if this is true when both operands are the same. bool isTrueWhenEqual() const { @@ -847,23 +1060,23 @@ public: /// @returns true if the predicate is unsigned, false otherwise. /// @brief Determine if the predicate is an unsigned operation. - static bool isUnsigned(unsigned short predicate); + static bool isUnsigned(Predicate predicate); /// @returns true if the predicate is signed, false otherwise. /// @brief Determine if the predicate is an signed operation. - static bool isSigned(unsigned short predicate); + static bool isSigned(Predicate predicate); /// @brief Determine if the predicate is an ordered operation. - static bool isOrdered(unsigned short predicate); + static bool isOrdered(Predicate predicate); /// @brief Determine if the predicate is an unordered operation. - static bool isUnordered(unsigned short predicate); + static bool isUnordered(Predicate predicate); /// Determine if the predicate is true when comparing a value with itself. - static bool isTrueWhenEqual(unsigned short predicate); + static bool isTrueWhenEqual(Predicate predicate); /// Determine if the predicate is false when comparing a value with itself. - static bool isFalseWhenEqual(unsigned short predicate); + static bool isFalseWhenEqual(Predicate predicate); /// @brief Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { @@ -882,6 +1095,7 @@ public: } return Type::getInt1Ty(opnd_type->getContext()); } + private: // Shadow Value::setValueSubclassData with a private forwarding method so that // subclasses cannot accidentally use it. @@ -890,7 +1104,6 @@ private: } }; - // FIXME: these are redundant if CmpInst < BinaryOperator template <> struct OperandTraits<CmpInst> : public FixedNumOperandTraits<CmpInst, 2> { @@ -898,6 +1111,523 @@ struct OperandTraits<CmpInst> : public FixedNumOperandTraits<CmpInst, 2> { DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CmpInst, Value) -} // End llvm namespace +//===----------------------------------------------------------------------===// +// FuncletPadInst Class +//===----------------------------------------------------------------------===// +class FuncletPadInst : public Instruction { +private: + void init(Value *ParentPad, ArrayRef<Value *> Args, const Twine &NameStr); + + FuncletPadInst(const FuncletPadInst &CPI); + + explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad, + ArrayRef<Value *> Args, unsigned Values, + const Twine &NameStr, Instruction *InsertBefore); + explicit FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad, + ArrayRef<Value *> Args, unsigned Values, + const Twine &NameStr, BasicBlock *InsertAtEnd); + +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + friend class CatchPadInst; + friend class CleanupPadInst; + FuncletPadInst *cloneImpl() const; + +public: + /// Provide fast operand accessors + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + /// getNumArgOperands - Return the number of funcletpad arguments. + /// + unsigned getNumArgOperands() const { return getNumOperands() - 1; } + + /// Convenience accessors + + /// \brief Return the outer EH-pad this funclet is nested within. + /// + /// Note: This returns the associated CatchSwitchInst if this FuncletPadInst + /// is a CatchPadInst. + Value *getParentPad() const { return Op<-1>(); } + void setParentPad(Value *ParentPad) { + assert(ParentPad); + Op<-1>() = ParentPad; + } + + /// getArgOperand/setArgOperand - Return/set the i-th funcletpad argument. + /// + Value *getArgOperand(unsigned i) const { return getOperand(i); } + void setArgOperand(unsigned i, Value *v) { setOperand(i, v); } + + /// arg_operands - iteration adapter for range-for loops. + op_range arg_operands() { return op_range(op_begin(), op_end() - 1); } + + /// arg_operands - iteration adapter for range-for loops. + const_op_range arg_operands() const { + return const_op_range(op_begin(), op_end() - 1); + } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Instruction *I) { return I->isFuncletPad(); } + static inline bool classof(const Value *V) { + return isa<Instruction>(V) && classof(cast<Instruction>(V)); + } +}; + +template <> +struct OperandTraits<FuncletPadInst> + : public VariadicOperandTraits<FuncletPadInst, /*MINARITY=*/1> {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(FuncletPadInst, Value) + +/// \brief A lightweight accessor for an operand bundle meant to be passed +/// around by value. +struct OperandBundleUse { + ArrayRef<Use> Inputs; + + OperandBundleUse() {} + explicit OperandBundleUse(StringMapEntry<uint32_t> *Tag, ArrayRef<Use> Inputs) + : Inputs(Inputs), Tag(Tag) {} + + /// \brief Return true if the operand at index \p Idx in this operand bundle + /// has the attribute A. + bool operandHasAttr(unsigned Idx, Attribute::AttrKind A) const { + if (isDeoptOperandBundle()) + if (A == Attribute::ReadOnly || A == Attribute::NoCapture) + return Inputs[Idx]->getType()->isPointerTy(); + + // Conservative answer: no operands have any attributes. + return false; + }; + + /// \brief Return the tag of this operand bundle as a string. + StringRef getTagName() const { + return Tag->getKey(); + } + + /// \brief Return the tag of this operand bundle as an integer. + /// + /// Operand bundle tags are interned by LLVMContextImpl::getOrInsertBundleTag, + /// and this function returns the unique integer getOrInsertBundleTag + /// associated the tag of this operand bundle to. + uint32_t getTagID() const { + return Tag->getValue(); + } + + /// \brief Return true if this is a "deopt" operand bundle. + bool isDeoptOperandBundle() const { + return getTagID() == LLVMContext::OB_deopt; + } + + /// \brief Return true if this is a "funclet" operand bundle. + bool isFuncletOperandBundle() const { + return getTagID() == LLVMContext::OB_funclet; + } + +private: + /// \brief Pointer to an entry in LLVMContextImpl::getOrInsertBundleTag. + StringMapEntry<uint32_t> *Tag; +}; + +/// \brief A container for an operand bundle being viewed as a set of values +/// rather than a set of uses. +/// +/// Unlike OperandBundleUse, OperandBundleDefT owns the memory it carries, and +/// so it is possible to create and pass around "self-contained" instances of +/// OperandBundleDef and ConstOperandBundleDef. +template <typename InputTy> class OperandBundleDefT { + std::string Tag; + std::vector<InputTy> Inputs; + +public: + explicit OperandBundleDefT(std::string Tag, std::vector<InputTy> Inputs) + : Tag(std::move(Tag)), Inputs(std::move(Inputs)) {} + explicit OperandBundleDefT(std::string Tag, ArrayRef<InputTy> Inputs) + : Tag(std::move(Tag)), Inputs(Inputs) {} + + explicit OperandBundleDefT(const OperandBundleUse &OBU) { + Tag = OBU.getTagName(); + Inputs.insert(Inputs.end(), OBU.Inputs.begin(), OBU.Inputs.end()); + } + + ArrayRef<InputTy> inputs() const { return Inputs; } + + typedef typename std::vector<InputTy>::const_iterator input_iterator; + size_t input_size() const { return Inputs.size(); } + input_iterator input_begin() const { return Inputs.begin(); } + input_iterator input_end() const { return Inputs.end(); } + + StringRef getTag() const { return Tag; } +}; + +typedef OperandBundleDefT<Value *> OperandBundleDef; +typedef OperandBundleDefT<const Value *> ConstOperandBundleDef; + +/// \brief A mixin to add operand bundle functionality to llvm instruction +/// classes. +/// +/// OperandBundleUser uses the descriptor area co-allocated with the host User +/// to store some meta information about which operands are "normal" operands, +/// and which ones belong to some operand bundle. +/// +/// The layout of an operand bundle user is +/// +/// +-----------uint32_t End-------------------------------------+ +/// | | +/// | +--------uint32_t Begin--------------------+ | +/// | | | | +/// ^ ^ v v +/// |------|------|----|----|----|----|----|---------|----|---------|----|----- +/// | BOI0 | BOI1 | .. | DU | U0 | U1 | .. | BOI0_U0 | .. | BOI1_U0 | .. | Un +/// |------|------|----|----|----|----|----|---------|----|---------|----|----- +/// v v ^ ^ +/// | | | | +/// | +--------uint32_t Begin------------+ | +/// | | +/// +-----------uint32_t End-----------------------------+ +/// +/// +/// BOI0, BOI1 ... are descriptions of operand bundles in this User's use list. +/// These descriptions are installed and managed by this class, and they're all +/// instances of OperandBundleUser<T>::BundleOpInfo. +/// +/// DU is an additional descriptor installed by User's 'operator new' to keep +/// track of the 'BOI0 ... BOIN' co-allocation. OperandBundleUser does not +/// access or modify DU in any way, it's an implementation detail private to +/// User. +/// +/// The regular Use& vector for the User starts at U0. The operand bundle uses +/// are part of the Use& vector, just like normal uses. In the diagram above, +/// the operand bundle uses start at BOI0_U0. Each instance of BundleOpInfo has +/// information about a contiguous set of uses constituting an operand bundle, +/// and the total set of operand bundle uses themselves form a contiguous set of +/// uses (i.e. there are no gaps between uses corresponding to individual +/// operand bundles). +/// +/// This class does not know the location of the set of operand bundle uses +/// within the use list -- that is decided by the User using this class via the +/// BeginIdx argument in populateBundleOperandInfos. +/// +/// Currently operand bundle users with hung-off operands are not supported. +template <typename InstrTy, typename OpIteratorTy> class OperandBundleUser { +public: + /// \brief Return the number of operand bundles associated with this User. + unsigned getNumOperandBundles() const { + return std::distance(bundle_op_info_begin(), bundle_op_info_end()); + } + + /// \brief Return true if this User has any operand bundles. + bool hasOperandBundles() const { return getNumOperandBundles() != 0; } + + /// \brief Return the index of the first bundle operand in the Use array. + unsigned getBundleOperandsStartIndex() const { + assert(hasOperandBundles() && "Don't call otherwise!"); + return bundle_op_info_begin()->Begin; + } + + /// \brief Return the index of the last bundle operand in the Use array. + unsigned getBundleOperandsEndIndex() const { + assert(hasOperandBundles() && "Don't call otherwise!"); + return bundle_op_info_end()[-1].End; + } + + /// \brief Return the total number operands (not operand bundles) used by + /// every operand bundle in this OperandBundleUser. + unsigned getNumTotalBundleOperands() const { + if (!hasOperandBundles()) + return 0; + + unsigned Begin = getBundleOperandsStartIndex(); + unsigned End = getBundleOperandsEndIndex(); + + assert(Begin <= End && "Should be!"); + return End - Begin; + } + + /// \brief Return the operand bundle at a specific index. + OperandBundleUse getOperandBundleAt(unsigned Index) const { + assert(Index < getNumOperandBundles() && "Index out of bounds!"); + return operandBundleFromBundleOpInfo(*(bundle_op_info_begin() + Index)); + } + + /// \brief Return the number of operand bundles with the tag Name attached to + /// this instruction. + unsigned countOperandBundlesOfType(StringRef Name) const { + unsigned Count = 0; + for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) + if (getOperandBundleAt(i).getTagName() == Name) + Count++; + + return Count; + } + + /// \brief Return the number of operand bundles with the tag ID attached to + /// this instruction. + unsigned countOperandBundlesOfType(uint32_t ID) const { + unsigned Count = 0; + for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) + if (getOperandBundleAt(i).getTagID() == ID) + Count++; + + return Count; + } + + /// \brief Return an operand bundle by name, if present. + /// + /// It is an error to call this for operand bundle types that may have + /// multiple instances of them on the same instruction. + Optional<OperandBundleUse> getOperandBundle(StringRef Name) const { + assert(countOperandBundlesOfType(Name) < 2 && "Precondition violated!"); + + for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) { + OperandBundleUse U = getOperandBundleAt(i); + if (U.getTagName() == Name) + return U; + } + + return None; + } + + /// \brief Return an operand bundle by tag ID, if present. + /// + /// It is an error to call this for operand bundle types that may have + /// multiple instances of them on the same instruction. + Optional<OperandBundleUse> getOperandBundle(uint32_t ID) const { + assert(countOperandBundlesOfType(ID) < 2 && "Precondition violated!"); + + for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) { + OperandBundleUse U = getOperandBundleAt(i); + if (U.getTagID() == ID) + return U; + } + + return None; + } + + /// \brief Return the list of operand bundles attached to this instruction as + /// a vector of OperandBundleDefs. + /// + /// This function copies the OperandBundeUse instances associated with this + /// OperandBundleUser to a vector of OperandBundleDefs. Note: + /// OperandBundeUses and OperandBundleDefs are non-trivially *different* + /// representations of operand bundles (see documentation above). + void getOperandBundlesAsDefs(SmallVectorImpl<OperandBundleDef> &Defs) const { + for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) + Defs.emplace_back(getOperandBundleAt(i)); + } + + /// \brief Return the operand bundle for the operand at index OpIdx. + /// + /// It is an error to call this with an OpIdx that does not correspond to an + /// bundle operand. + OperandBundleUse getOperandBundleForOperand(unsigned OpIdx) const { + return operandBundleFromBundleOpInfo(getBundleOpInfoForOperand(OpIdx)); + } + + /// \brief Return true if this operand bundle user has operand bundles that + /// may read from the heap. + bool hasReadingOperandBundles() const { + // Implementation note: this is a conservative implementation of operand + // bundle semantics, where *any* operand bundle forces a callsite to be at + // least readonly. + return hasOperandBundles(); + } + + /// \brief Return true if this operand bundle user has operand bundles that + /// may write to the heap. + bool hasClobberingOperandBundles() const { + for (auto &BOI : bundle_op_infos()) { + if (BOI.Tag->second == LLVMContext::OB_deopt || + BOI.Tag->second == LLVMContext::OB_funclet) + continue; + + // This instruction has an operand bundle that is not known to us. + // Assume the worst. + return true; + } + + return false; + } + + /// \brief Return true if the bundle operand at index \p OpIdx has the + /// attribute \p A. + bool bundleOperandHasAttr(unsigned OpIdx, Attribute::AttrKind A) const { + auto &BOI = getBundleOpInfoForOperand(OpIdx); + auto OBU = operandBundleFromBundleOpInfo(BOI); + return OBU.operandHasAttr(OpIdx - BOI.Begin, A); + } + + /// \brief Return true if \p Other has the same sequence of operand bundle + /// tags with the same number of operands on each one of them as this + /// OperandBundleUser. + bool hasIdenticalOperandBundleSchema( + const OperandBundleUser<InstrTy, OpIteratorTy> &Other) const { + if (getNumOperandBundles() != Other.getNumOperandBundles()) + return false; + + return std::equal(bundle_op_info_begin(), bundle_op_info_end(), + Other.bundle_op_info_begin()); + }; + +protected: + /// \brief Is the function attribute S disallowed by some operand bundle on + /// this operand bundle user? + bool isFnAttrDisallowedByOpBundle(StringRef S) const { + // Operand bundles only possibly disallow readnone, readonly and argmenonly + // attributes. All String attributes are fine. + return false; + } + + /// \brief Is the function attribute A disallowed by some operand bundle on + /// this operand bundle user? + bool isFnAttrDisallowedByOpBundle(Attribute::AttrKind A) const { + switch (A) { + default: + return false; + + case Attribute::ArgMemOnly: + return hasReadingOperandBundles(); + + case Attribute::ReadNone: + return hasReadingOperandBundles(); + + case Attribute::ReadOnly: + return hasClobberingOperandBundles(); + } + + llvm_unreachable("switch has a default case!"); + } + + /// \brief Used to keep track of an operand bundle. See the main comment on + /// OperandBundleUser above. + struct BundleOpInfo { + /// \brief The operand bundle tag, interned by + /// LLVMContextImpl::getOrInsertBundleTag. + StringMapEntry<uint32_t> *Tag; + + /// \brief The index in the Use& vector where operands for this operand + /// bundle starts. + uint32_t Begin; + + /// \brief The index in the Use& vector where operands for this operand + /// bundle ends. + uint32_t End; + + bool operator==(const BundleOpInfo &Other) const { + return Tag == Other.Tag && Begin == Other.Begin && End == Other.End; + } + }; + + /// \brief Simple helper function to map a BundleOpInfo to an + /// OperandBundleUse. + OperandBundleUse + operandBundleFromBundleOpInfo(const BundleOpInfo &BOI) const { + auto op_begin = static_cast<const InstrTy *>(this)->op_begin(); + ArrayRef<Use> Inputs(op_begin + BOI.Begin, op_begin + BOI.End); + return OperandBundleUse(BOI.Tag, Inputs); + } + + typedef BundleOpInfo *bundle_op_iterator; + typedef const BundleOpInfo *const_bundle_op_iterator; + + /// \brief Return the start of the list of BundleOpInfo instances associated + /// with this OperandBundleUser. + bundle_op_iterator bundle_op_info_begin() { + if (!static_cast<InstrTy *>(this)->hasDescriptor()) + return nullptr; + + uint8_t *BytesBegin = static_cast<InstrTy *>(this)->getDescriptor().begin(); + return reinterpret_cast<bundle_op_iterator>(BytesBegin); + } + + /// \brief Return the start of the list of BundleOpInfo instances associated + /// with this OperandBundleUser. + const_bundle_op_iterator bundle_op_info_begin() const { + auto *NonConstThis = + const_cast<OperandBundleUser<InstrTy, OpIteratorTy> *>(this); + return NonConstThis->bundle_op_info_begin(); + } + + /// \brief Return the end of the list of BundleOpInfo instances associated + /// with this OperandBundleUser. + bundle_op_iterator bundle_op_info_end() { + if (!static_cast<InstrTy *>(this)->hasDescriptor()) + return nullptr; + + uint8_t *BytesEnd = static_cast<InstrTy *>(this)->getDescriptor().end(); + return reinterpret_cast<bundle_op_iterator>(BytesEnd); + } + + /// \brief Return the end of the list of BundleOpInfo instances associated + /// with this OperandBundleUser. + const_bundle_op_iterator bundle_op_info_end() const { + auto *NonConstThis = + const_cast<OperandBundleUser<InstrTy, OpIteratorTy> *>(this); + return NonConstThis->bundle_op_info_end(); + } + + /// \brief Return the range [\p bundle_op_info_begin, \p bundle_op_info_end). + iterator_range<bundle_op_iterator> bundle_op_infos() { + return make_range(bundle_op_info_begin(), bundle_op_info_end()); + } + + /// \brief Return the range [\p bundle_op_info_begin, \p bundle_op_info_end). + iterator_range<const_bundle_op_iterator> bundle_op_infos() const { + return make_range(bundle_op_info_begin(), bundle_op_info_end()); + } + + /// \brief Populate the BundleOpInfo instances and the Use& vector from \p + /// Bundles. Return the op_iterator pointing to the Use& one past the last + /// last bundle operand use. + /// + /// Each \p OperandBundleDef instance is tracked by a OperandBundleInfo + /// instance allocated in this User's descriptor. + OpIteratorTy populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles, + const unsigned BeginIndex) { + auto It = static_cast<InstrTy *>(this)->op_begin() + BeginIndex; + for (auto &B : Bundles) + It = std::copy(B.input_begin(), B.input_end(), It); + + auto *ContextImpl = static_cast<InstrTy *>(this)->getContext().pImpl; + auto BI = Bundles.begin(); + unsigned CurrentIndex = BeginIndex; + + for (auto &BOI : bundle_op_infos()) { + assert(BI != Bundles.end() && "Incorrect allocation?"); + + BOI.Tag = ContextImpl->getOrInsertBundleTag(BI->getTag()); + BOI.Begin = CurrentIndex; + BOI.End = CurrentIndex + BI->input_size(); + CurrentIndex = BOI.End; + BI++; + } + + assert(BI == Bundles.end() && "Incorrect allocation?"); + + return It; + } + + /// \brief Return the BundleOpInfo for the operand at index OpIdx. + /// + /// It is an error to call this with an OpIdx that does not correspond to an + /// bundle operand. + const BundleOpInfo &getBundleOpInfoForOperand(unsigned OpIdx) const { + for (auto &BOI : bundle_op_infos()) + if (BOI.Begin <= OpIdx && OpIdx < BOI.End) + return BOI; + + llvm_unreachable("Did not find operand bundle for operand!"); + } + + /// \brief Return the total number of values used in \p Bundles. + static unsigned CountBundleInputs(ArrayRef<OperandBundleDef> Bundles) { + unsigned Total = 0; + for (auto &B : Bundles) + Total += B.input_size(); + return Total; + } +}; + +} // end llvm namespace -#endif +#endif // LLVM_IR_INSTRTYPES_H diff --git a/include/llvm/IR/Instruction.def b/include/llvm/IR/Instruction.def index d46314cc761d..18711abb8060 100644 --- a/include/llvm/IR/Instruction.def +++ b/include/llvm/IR/Instruction.def @@ -1,21 +1,21 @@ //===-- llvm/Instruction.def - File that describes Instructions -*- C++ -*-===// -// +// // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. -// +// //===----------------------------------------------------------------------===// // // This file contains descriptions of the various LLVM instructions. This is -// used as a central place for enumerating the different instructions and +// used as a central place for enumerating the different instructions and // should eventually be the place to put comments about the instructions. // //===----------------------------------------------------------------------===// // NOTE: NO INCLUDE GUARD DESIRED! -// Provide definitions of macros so that users of this file do not have to +// Provide definitions of macros so that users of this file do not have to // define everything to use it... // #ifndef FIRST_TERM_INST @@ -74,6 +74,20 @@ #define LAST_CAST_INST(num) #endif +#ifndef FIRST_FUNCLETPAD_INST +#define FIRST_FUNCLETPAD_INST(num) +#endif +#ifndef HANDLE_FUNCLETPAD_INST +#ifndef HANDLE_INST +#define HANDLE_FUNCLETPAD_INST(num, opcode, Class) +#else +#define HANDLE_FUNCLETPAD_INST(num, opcode, Class) HANDLE_INST(num, opcode, Class) +#endif +#endif +#ifndef LAST_FUNCLETPAD_INST +#define LAST_FUNCLETPAD_INST(num) +#endif + #ifndef FIRST_OTHER_INST #define FIRST_OTHER_INST(num) #endif @@ -88,92 +102,99 @@ #define LAST_OTHER_INST(num) #endif - // Terminator Instructions - These instructions are used to terminate a basic // block of the program. Every basic block must end with one of these // instructions for it to be a well formed basic block. // FIRST_TERM_INST ( 1) -HANDLE_TERM_INST ( 1, Ret , ReturnInst) -HANDLE_TERM_INST ( 2, Br , BranchInst) -HANDLE_TERM_INST ( 3, Switch , SwitchInst) -HANDLE_TERM_INST ( 4, IndirectBr , IndirectBrInst) -HANDLE_TERM_INST ( 5, Invoke , InvokeInst) -HANDLE_TERM_INST ( 6, Resume , ResumeInst) -HANDLE_TERM_INST ( 7, Unreachable, UnreachableInst) - LAST_TERM_INST ( 7) +HANDLE_TERM_INST ( 1, Ret , ReturnInst) +HANDLE_TERM_INST ( 2, Br , BranchInst) +HANDLE_TERM_INST ( 3, Switch , SwitchInst) +HANDLE_TERM_INST ( 4, IndirectBr , IndirectBrInst) +HANDLE_TERM_INST ( 5, Invoke , InvokeInst) +HANDLE_TERM_INST ( 6, Resume , ResumeInst) +HANDLE_TERM_INST ( 7, Unreachable , UnreachableInst) +HANDLE_TERM_INST ( 8, CleanupRet , CleanupReturnInst) +HANDLE_TERM_INST ( 9, CatchRet , CatchReturnInst) +HANDLE_TERM_INST (10, CatchSwitch , CatchSwitchInst) + LAST_TERM_INST (10) // Standard binary operators... - FIRST_BINARY_INST( 8) -HANDLE_BINARY_INST( 8, Add , BinaryOperator) -HANDLE_BINARY_INST( 9, FAdd , BinaryOperator) -HANDLE_BINARY_INST(10, Sub , BinaryOperator) -HANDLE_BINARY_INST(11, FSub , BinaryOperator) -HANDLE_BINARY_INST(12, Mul , BinaryOperator) -HANDLE_BINARY_INST(13, FMul , BinaryOperator) -HANDLE_BINARY_INST(14, UDiv , BinaryOperator) -HANDLE_BINARY_INST(15, SDiv , BinaryOperator) -HANDLE_BINARY_INST(16, FDiv , BinaryOperator) -HANDLE_BINARY_INST(17, URem , BinaryOperator) -HANDLE_BINARY_INST(18, SRem , BinaryOperator) -HANDLE_BINARY_INST(19, FRem , BinaryOperator) + FIRST_BINARY_INST(11) +HANDLE_BINARY_INST(11, Add , BinaryOperator) +HANDLE_BINARY_INST(12, FAdd , BinaryOperator) +HANDLE_BINARY_INST(13, Sub , BinaryOperator) +HANDLE_BINARY_INST(14, FSub , BinaryOperator) +HANDLE_BINARY_INST(15, Mul , BinaryOperator) +HANDLE_BINARY_INST(16, FMul , BinaryOperator) +HANDLE_BINARY_INST(17, UDiv , BinaryOperator) +HANDLE_BINARY_INST(18, SDiv , BinaryOperator) +HANDLE_BINARY_INST(19, FDiv , BinaryOperator) +HANDLE_BINARY_INST(20, URem , BinaryOperator) +HANDLE_BINARY_INST(21, SRem , BinaryOperator) +HANDLE_BINARY_INST(22, FRem , BinaryOperator) // Logical operators (integer operands) -HANDLE_BINARY_INST(20, Shl , BinaryOperator) // Shift left (logical) -HANDLE_BINARY_INST(21, LShr , BinaryOperator) // Shift right (logical) -HANDLE_BINARY_INST(22, AShr , BinaryOperator) // Shift right (arithmetic) -HANDLE_BINARY_INST(23, And , BinaryOperator) -HANDLE_BINARY_INST(24, Or , BinaryOperator) -HANDLE_BINARY_INST(25, Xor , BinaryOperator) - LAST_BINARY_INST(25) +HANDLE_BINARY_INST(23, Shl , BinaryOperator) // Shift left (logical) +HANDLE_BINARY_INST(24, LShr , BinaryOperator) // Shift right (logical) +HANDLE_BINARY_INST(25, AShr , BinaryOperator) // Shift right (arithmetic) +HANDLE_BINARY_INST(26, And , BinaryOperator) +HANDLE_BINARY_INST(27, Or , BinaryOperator) +HANDLE_BINARY_INST(28, Xor , BinaryOperator) + LAST_BINARY_INST(28) // Memory operators... - FIRST_MEMORY_INST(26) -HANDLE_MEMORY_INST(26, Alloca, AllocaInst) // Stack management -HANDLE_MEMORY_INST(27, Load , LoadInst ) // Memory manipulation instrs -HANDLE_MEMORY_INST(28, Store , StoreInst ) -HANDLE_MEMORY_INST(29, GetElementPtr, GetElementPtrInst) -HANDLE_MEMORY_INST(30, Fence , FenceInst ) -HANDLE_MEMORY_INST(31, AtomicCmpXchg , AtomicCmpXchgInst ) -HANDLE_MEMORY_INST(32, AtomicRMW , AtomicRMWInst ) - LAST_MEMORY_INST(32) + FIRST_MEMORY_INST(29) +HANDLE_MEMORY_INST(29, Alloca, AllocaInst) // Stack management +HANDLE_MEMORY_INST(30, Load , LoadInst ) // Memory manipulation instrs +HANDLE_MEMORY_INST(31, Store , StoreInst ) +HANDLE_MEMORY_INST(32, GetElementPtr, GetElementPtrInst) +HANDLE_MEMORY_INST(33, Fence , FenceInst ) +HANDLE_MEMORY_INST(34, AtomicCmpXchg , AtomicCmpXchgInst ) +HANDLE_MEMORY_INST(35, AtomicRMW , AtomicRMWInst ) + LAST_MEMORY_INST(35) // Cast operators ... -// NOTE: The order matters here because CastInst::isEliminableCastPair +// NOTE: The order matters here because CastInst::isEliminableCastPair // NOTE: (see Instructions.cpp) encodes a table based on this ordering. - FIRST_CAST_INST(33) -HANDLE_CAST_INST(33, Trunc , TruncInst ) // Truncate integers -HANDLE_CAST_INST(34, ZExt , ZExtInst ) // Zero extend integers -HANDLE_CAST_INST(35, SExt , SExtInst ) // Sign extend integers -HANDLE_CAST_INST(36, FPToUI , FPToUIInst ) // floating point -> UInt -HANDLE_CAST_INST(37, FPToSI , FPToSIInst ) // floating point -> SInt -HANDLE_CAST_INST(38, UIToFP , UIToFPInst ) // UInt -> floating point -HANDLE_CAST_INST(39, SIToFP , SIToFPInst ) // SInt -> floating point -HANDLE_CAST_INST(40, FPTrunc , FPTruncInst ) // Truncate floating point -HANDLE_CAST_INST(41, FPExt , FPExtInst ) // Extend floating point -HANDLE_CAST_INST(42, PtrToInt, PtrToIntInst) // Pointer -> Integer -HANDLE_CAST_INST(43, IntToPtr, IntToPtrInst) // Integer -> Pointer -HANDLE_CAST_INST(44, BitCast , BitCastInst ) // Type cast -HANDLE_CAST_INST(45, AddrSpaceCast, AddrSpaceCastInst) // addrspace cast - LAST_CAST_INST(45) + FIRST_CAST_INST(36) +HANDLE_CAST_INST(36, Trunc , TruncInst ) // Truncate integers +HANDLE_CAST_INST(37, ZExt , ZExtInst ) // Zero extend integers +HANDLE_CAST_INST(38, SExt , SExtInst ) // Sign extend integers +HANDLE_CAST_INST(39, FPToUI , FPToUIInst ) // floating point -> UInt +HANDLE_CAST_INST(40, FPToSI , FPToSIInst ) // floating point -> SInt +HANDLE_CAST_INST(41, UIToFP , UIToFPInst ) // UInt -> floating point +HANDLE_CAST_INST(42, SIToFP , SIToFPInst ) // SInt -> floating point +HANDLE_CAST_INST(43, FPTrunc , FPTruncInst ) // Truncate floating point +HANDLE_CAST_INST(44, FPExt , FPExtInst ) // Extend floating point +HANDLE_CAST_INST(45, PtrToInt, PtrToIntInst) // Pointer -> Integer +HANDLE_CAST_INST(46, IntToPtr, IntToPtrInst) // Integer -> Pointer +HANDLE_CAST_INST(47, BitCast , BitCastInst ) // Type cast +HANDLE_CAST_INST(48, AddrSpaceCast, AddrSpaceCastInst) // addrspace cast + LAST_CAST_INST(48) + + FIRST_FUNCLETPAD_INST(49) +HANDLE_FUNCLETPAD_INST(49, CleanupPad, CleanupPadInst) +HANDLE_FUNCLETPAD_INST(50, CatchPad , CatchPadInst) + LAST_FUNCLETPAD_INST(50) // Other operators... - FIRST_OTHER_INST(46) -HANDLE_OTHER_INST(46, ICmp , ICmpInst ) // Integer comparison instruction -HANDLE_OTHER_INST(47, FCmp , FCmpInst ) // Floating point comparison instr. -HANDLE_OTHER_INST(48, PHI , PHINode ) // PHI node instruction -HANDLE_OTHER_INST(49, Call , CallInst ) // Call a function -HANDLE_OTHER_INST(50, Select , SelectInst ) // select instruction -HANDLE_OTHER_INST(51, UserOp1, Instruction) // May be used internally in a pass -HANDLE_OTHER_INST(52, UserOp2, Instruction) // Internal to passes only -HANDLE_OTHER_INST(53, VAArg , VAArgInst ) // vaarg instruction -HANDLE_OTHER_INST(54, ExtractElement, ExtractElementInst)// extract from vector -HANDLE_OTHER_INST(55, InsertElement, InsertElementInst) // insert into vector -HANDLE_OTHER_INST(56, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. -HANDLE_OTHER_INST(57, ExtractValue, ExtractValueInst)// extract from aggregate -HANDLE_OTHER_INST(58, InsertValue, InsertValueInst) // insert into aggregate -HANDLE_OTHER_INST(59, LandingPad, LandingPadInst) // Landing pad instruction. - LAST_OTHER_INST(59) + FIRST_OTHER_INST(51) +HANDLE_OTHER_INST(51, ICmp , ICmpInst ) // Integer comparison instruction +HANDLE_OTHER_INST(52, FCmp , FCmpInst ) // Floating point comparison instr. +HANDLE_OTHER_INST(53, PHI , PHINode ) // PHI node instruction +HANDLE_OTHER_INST(54, Call , CallInst ) // Call a function +HANDLE_OTHER_INST(55, Select , SelectInst ) // select instruction +HANDLE_OTHER_INST(56, UserOp1, Instruction) // May be used internally in a pass +HANDLE_OTHER_INST(57, UserOp2, Instruction) // Internal to passes only +HANDLE_OTHER_INST(58, VAArg , VAArgInst ) // vaarg instruction +HANDLE_OTHER_INST(59, ExtractElement, ExtractElementInst)// extract from vector +HANDLE_OTHER_INST(60, InsertElement, InsertElementInst) // insert into vector +HANDLE_OTHER_INST(61, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. +HANDLE_OTHER_INST(62, ExtractValue, ExtractValueInst)// extract from aggregate +HANDLE_OTHER_INST(63, InsertValue, InsertValueInst) // insert into aggregate +HANDLE_OTHER_INST(64, LandingPad, LandingPadInst) // Landing pad instruction. + LAST_OTHER_INST(64) #undef FIRST_TERM_INST #undef HANDLE_TERM_INST @@ -191,6 +212,10 @@ HANDLE_OTHER_INST(59, LandingPad, LandingPadInst) // Landing pad instruction. #undef HANDLE_CAST_INST #undef LAST_CAST_INST +#undef FIRST_FUNCLETPAD_INST +#undef HANDLE_FUNCLETPAD_INST +#undef LAST_FUNCLETPAD_INST + #undef FIRST_OTHER_INST #undef HANDLE_OTHER_INST #undef LAST_OTHER_INST diff --git a/include/llvm/IR/Instruction.h b/include/llvm/IR/Instruction.h index 31f363f70a5b..03c45497fa95 100644 --- a/include/llvm/IR/Instruction.h +++ b/include/llvm/IR/Instruction.h @@ -30,25 +30,11 @@ class BasicBlock; struct AAMDNodes; template <> -struct ilist_traits<Instruction> - : public SymbolTableListTraits<Instruction, BasicBlock> { +struct SymbolTableListSentinelTraits<Instruction> + : public ilist_half_embedded_sentinel_traits<Instruction> {}; - /// \brief Return a node that marks the end of a list. - /// - /// The sentinel is relative to this instance, so we use a non-static - /// method. - Instruction *createSentinel() const; - static void destroySentinel(Instruction *) {} - - Instruction *provideInitialHead() const { return createSentinel(); } - Instruction *ensureHead(Instruction *) const { return createSentinel(); } - static void noteHead(Instruction *, Instruction *) {} - -private: - mutable ilist_half_node<Instruction> Sentinel; -}; - -class Instruction : public User, public ilist_node<Instruction> { +class Instruction : public User, + public ilist_node_with_parent<Instruction, BasicBlock> { void operator=(const Instruction &) = delete; Instruction(const Instruction &) = delete; @@ -80,6 +66,13 @@ public: const Module *getModule() const; Module *getModule(); + /// \brief Return the function this instruction belongs to. + /// + /// Note: it is undefined behavior to call this on an instruction not + /// currently inserted into a function. + const Function *getFunction() const; + Function *getFunction(); + /// removeFromParent - This method unlinks 'this' from the containing basic /// block, but does not delete it. /// @@ -89,7 +82,7 @@ public: /// block and deletes it. /// /// \returns an iterator pointing to the element after the erased one - iplist<Instruction>::iterator eraseFromParent(); + SymbolTableList<Instruction>::iterator eraseFromParent(); /// Insert an unlinked instruction into a basic block immediately before /// the specified instruction. @@ -116,6 +109,7 @@ public: bool isBinaryOp() const { return isBinaryOp(getOpcode()); } bool isShift() { return isShift(getOpcode()); } bool isCast() const { return isCast(getOpcode()); } + bool isFuncletPad() const { return isFuncletPad(getOpcode()); } static const char* getOpcodeName(unsigned OpCode); @@ -148,6 +142,11 @@ public: return OpCode >= CastOpsBegin && OpCode < CastOpsEnd; } + /// @brief Determine if the OpCode is one of the FuncletPadInst instructions. + static inline bool isFuncletPad(unsigned OpCode) { + return OpCode >= FuncletPadOpsBegin && OpCode < FuncletPadOpsEnd; + } + //===--------------------------------------------------------------------===// // Metadata manipulation. //===--------------------------------------------------------------------===// @@ -204,20 +203,22 @@ public: void setMetadata(unsigned KindID, MDNode *Node); void setMetadata(StringRef Kind, MDNode *Node); - /// \brief Drop unknown metadata. + /// Drop all unknown metadata except for debug locations. + /// @{ /// Passes are required to drop metadata they don't understand. This is a /// convenience method for passes to do so. - void dropUnknownMetadata(ArrayRef<unsigned> KnownIDs); - void dropUnknownMetadata() { - return dropUnknownMetadata(None); + void dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs); + void dropUnknownNonDebugMetadata() { + return dropUnknownNonDebugMetadata(None); } - void dropUnknownMetadata(unsigned ID1) { - return dropUnknownMetadata(makeArrayRef(ID1)); + void dropUnknownNonDebugMetadata(unsigned ID1) { + return dropUnknownNonDebugMetadata(makeArrayRef(ID1)); } - void dropUnknownMetadata(unsigned ID1, unsigned ID2) { + void dropUnknownNonDebugMetadata(unsigned ID1, unsigned ID2) { unsigned IDs[] = {ID1, ID2}; - return dropUnknownMetadata(IDs); + return dropUnknownNonDebugMetadata(IDs); } + /// @} /// setAAMetadata - Sets the metadata on this instruction from the /// AAMDNodes structure. @@ -388,6 +389,19 @@ public: return mayWriteToMemory() || mayThrow() || !mayReturn(); } + /// \brief Return true if the instruction is a variety of EH-block. + bool isEHPad() const { + switch (getOpcode()) { + case Instruction::CatchSwitch: + case Instruction::CatchPad: + case Instruction::CleanupPad: + case Instruction::LandingPad: + return true; + default: + return false; + } + } + /// clone() - Create a copy of 'this' instruction that is identical in all /// ways except the following: /// * The instruction has no parent @@ -468,6 +482,13 @@ public: #include "llvm/IR/Instruction.def" }; + enum FuncletPadOps { +#define FIRST_FUNCLETPAD_INST(N) FuncletPadOpsBegin = N, +#define HANDLE_FUNCLETPAD_INST(N, OPC, CLASS) OPC = N, +#define LAST_FUNCLETPAD_INST(N) FuncletPadOpsEnd = N+1 +#include "llvm/IR/Instruction.def" + }; + enum OtherOps { #define FIRST_OTHER_INST(N) OtherOpsBegin = N, #define HANDLE_OTHER_INST(N, OPC, CLASS) OPC = N, @@ -489,7 +510,7 @@ private: (V ? HasMetadataBit : 0)); } - friend class SymbolTableListTraits<Instruction, BasicBlock>; + friend class SymbolTableListTraits<Instruction>; void setParent(BasicBlock *P); protected: // Instruction subclasses can stick up to 15 bits of stuff into the @@ -515,17 +536,6 @@ private: Instruction *cloneImpl() const; }; -inline Instruction *ilist_traits<Instruction>::createSentinel() const { - // Since i(p)lists always publicly derive from their corresponding traits, - // placing a data member in this class will augment the i(p)list. But since - // the NodeTy is expected to be publicly derive from ilist_node<NodeTy>, - // there is a legal viable downcast from it to NodeTy. We use this trick to - // superimpose an i(p)list with a "ghostly" NodeTy, which becomes the - // sentinel. Dereferencing the sentinel is forbidden (save the - // ilist_node<NodeTy>), so no one will ever notice the superposition. - return static_cast<Instruction *>(&Sentinel); -} - // Instruction* is only 4-byte aligned. template<> class PointerLikeTypeTraits<Instruction*> { diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h index 07d5f111b9e1..d781c7af36d7 100644 --- a/include/llvm/IR/Instructions.h +++ b/include/llvm/IR/Instructions.h @@ -18,6 +18,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" @@ -158,6 +159,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: // Shadow Instruction::setInstructionSubclassData with a private forwarding // method so that subclasses cannot accidentally use it. @@ -166,7 +168,6 @@ private: } }; - //===----------------------------------------------------------------------===// // LoadInst Class //===----------------------------------------------------------------------===// @@ -176,6 +177,7 @@ private: /// class LoadInst : public UnaryInstruction { void AssertOK(); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -290,7 +292,6 @@ public: return getPointerOperand()->getType()->getPointerAddressSpace(); } - // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Load; @@ -298,6 +299,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: // Shadow Instruction::setInstructionSubclassData with a private forwarding // method so that subclasses cannot accidentally use it. @@ -306,7 +308,6 @@ private: } }; - //===----------------------------------------------------------------------===// // StoreInst Class //===----------------------------------------------------------------------===// @@ -316,6 +317,7 @@ private: class StoreInst : public Instruction { void *operator new(size_t, unsigned) = delete; void AssertOK(); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -344,7 +346,6 @@ public: SynchronizationScope SynchScope, BasicBlock *InsertAtEnd); - /// isVolatile - Return true if this is a store to a volatile memory /// location. /// @@ -422,6 +423,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: // Shadow Instruction::setInstructionSubclassData with a private forwarding // method so that subclasses cannot accidentally use it. @@ -445,6 +447,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value) class FenceInst : public Instruction { void *operator new(size_t, unsigned) = delete; void Init(AtomicOrdering Ordering, SynchronizationScope SynchScope); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -496,6 +499,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: // Shadow Instruction::setInstructionSubclassData with a private forwarding // method so that subclasses cannot accidentally use it. @@ -517,6 +521,7 @@ class AtomicCmpXchgInst : public Instruction { void Init(Value *Ptr, Value *Cmp, Value *NewVal, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -648,6 +653,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: // Shadow Instruction::setInstructionSubclassData with a private forwarding // method so that subclasses cannot accidentally use it. @@ -673,6 +679,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value) /// class AtomicRMWInst : public Instruction { void *operator new(size_t, unsigned) = delete; + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -795,6 +802,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: void Init(BinOp Operation, Value *Ptr, Value *Val, AtomicOrdering Ordering, SynchronizationScope SynchScope); @@ -831,6 +839,8 @@ class GetElementPtrInst : public Instruction { Type *SourceElementType; Type *ResultElementType; + void anchor() override; + GetElementPtrInst(const GetElementPtrInst &GEPI); void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr); @@ -1078,10 +1088,8 @@ GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr, init(Ptr, IdxList, NameStr); } - DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value) - //===----------------------------------------------------------------------===// // ICmpInst Class //===----------------------------------------------------------------------===// @@ -1091,6 +1099,8 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value) /// must be identical types. /// \brief Represent an integer comparison operator. class ICmpInst: public CmpInst { + void anchor() override; + void AssertOK() { assert(getPredicate() >= CmpInst::FIRST_ICMP_PREDICATE && getPredicate() <= CmpInst::LAST_ICMP_PREDICATE && @@ -1226,7 +1236,6 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } - }; //===----------------------------------------------------------------------===// @@ -1350,62 +1359,102 @@ public: /// field to indicate whether or not this is a tail call. The rest of the bits /// hold the calling convention of the call. /// -class CallInst : public Instruction { +class CallInst : public Instruction, + public OperandBundleUser<CallInst, User::op_iterator> { AttributeSet AttributeList; ///< parameter attributes for call FunctionType *FTy; CallInst(const CallInst &CI); - void init(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr) { + void init(Value *Func, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr) { init(cast<FunctionType>( cast<PointerType>(Func->getType())->getElementType()), - Func, Args, NameStr); + Func, Args, Bundles, NameStr); } void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args, - const Twine &NameStr); + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr); void init(Value *Func, const Twine &NameStr); /// Construct a CallInst given a range of arguments. /// \brief Construct a CallInst from a range of arguments inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args, - const Twine &NameStr, Instruction *InsertBefore); - inline CallInst(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr, + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr, + Instruction *InsertBefore); + inline CallInst(Value *Func, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr, Instruction *InsertBefore) : CallInst(cast<FunctionType>( cast<PointerType>(Func->getType())->getElementType()), - Func, Args, NameStr, InsertBefore) {} + Func, Args, Bundles, NameStr, InsertBefore) {} + + inline CallInst(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr, + Instruction *InsertBefore) + : CallInst(Func, Args, None, NameStr, InsertBefore) {} /// Construct a CallInst given a range of arguments. /// \brief Construct a CallInst from a range of arguments inline CallInst(Value *Func, ArrayRef<Value *> Args, - const Twine &NameStr, BasicBlock *InsertAtEnd); + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr, + BasicBlock *InsertAtEnd); explicit CallInst(Value *F, const Twine &NameStr, Instruction *InsertBefore); CallInst(Value *F, const Twine &NameStr, BasicBlock *InsertAtEnd); + + friend class OperandBundleUser<CallInst, User::op_iterator>; + bool hasDescriptor() const { return HasDescriptor; } + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; CallInst *cloneImpl() const; public: - static CallInst *Create(Value *Func, - ArrayRef<Value *> Args, + static CallInst *Create(Value *Func, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles = None, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { return Create(cast<FunctionType>( cast<PointerType>(Func->getType())->getElementType()), - Func, Args, NameStr, InsertBefore); + Func, Args, Bundles, NameStr, InsertBefore); + } + static CallInst *Create(Value *Func, ArrayRef<Value *> Args, + const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + return Create(cast<FunctionType>( + cast<PointerType>(Func->getType())->getElementType()), + Func, Args, None, NameStr, InsertBefore); } static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args, - const Twine &NameStr = "", + const Twine &NameStr, Instruction *InsertBefore = nullptr) { return new (unsigned(Args.size() + 1)) - CallInst(Ty, Func, Args, NameStr, InsertBefore); + CallInst(Ty, Func, Args, None, NameStr, InsertBefore); } - static CallInst *Create(Value *Func, - ArrayRef<Value *> Args, + static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + const unsigned TotalOps = + unsigned(Args.size()) + CountBundleInputs(Bundles) + 1; + const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (TotalOps, DescriptorBytes) + CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore); + } + static CallInst *Create(Value *Func, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr, BasicBlock *InsertAtEnd) { - return new(unsigned(Args.size() + 1)) - CallInst(Func, Args, NameStr, InsertAtEnd); + const unsigned TotalOps = + unsigned(Args.size()) + CountBundleInputs(Bundles) + 1; + const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (TotalOps, DescriptorBytes) + CallInst(Func, Args, Bundles, NameStr, InsertAtEnd); + } + static CallInst *Create(Value *Func, ArrayRef<Value *> Args, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + return new (unsigned(Args.size() + 1)) + CallInst(Func, Args, None, NameStr, InsertAtEnd); } static CallInst *Create(Value *F, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { @@ -1415,6 +1464,16 @@ public: BasicBlock *InsertAtEnd) { return new(1) CallInst(F, NameStr, InsertAtEnd); } + + /// \brief Create a clone of \p CI with a different set of operand bundles and + /// insert it before \p InsertPt. + /// + /// The returned call instruction is identical \p CI in every way except that + /// the operand bundles for the new instruction are set to the operand bundles + /// in \p Bundles. + static CallInst *Create(CallInst *CI, ArrayRef<OperandBundleDef> Bundles, + Instruction *InsertPt = nullptr); + /// CreateMalloc - Generate the IR for a call to malloc: /// 1. Compute the malloc call's argument as the specified type's size, /// possibly multiplied by the array size if the array size is not @@ -1445,16 +1504,21 @@ public: } // Note that 'musttail' implies 'tail'. - enum TailCallKind { TCK_None = 0, TCK_Tail = 1, TCK_MustTail = 2 }; + enum TailCallKind { TCK_None = 0, TCK_Tail = 1, TCK_MustTail = 2, + TCK_NoTail = 3 }; TailCallKind getTailCallKind() const { return TailCallKind(getSubclassDataFromInstruction() & 3); } bool isTailCall() const { - return (getSubclassDataFromInstruction() & 3) != TCK_None; + unsigned Kind = getSubclassDataFromInstruction() & 3; + return Kind == TCK_Tail || Kind == TCK_MustTail; } bool isMustTailCall() const { return (getSubclassDataFromInstruction() & 3) == TCK_MustTail; } + bool isNoTailCall() const { + return (getSubclassDataFromInstruction() & 3) == TCK_NoTail; + } void setTailCall(bool isTC = true) { setInstructionSubclassData((getSubclassDataFromInstruction() & ~3) | unsigned(isTC ? TCK_Tail : TCK_None)); @@ -1469,28 +1533,58 @@ public: /// getNumArgOperands - Return the number of call arguments. /// - unsigned getNumArgOperands() const { return getNumOperands() - 1; } + unsigned getNumArgOperands() const { + return getNumOperands() - getNumTotalBundleOperands() - 1; + } /// getArgOperand/setArgOperand - Return/set the i-th call argument. /// - Value *getArgOperand(unsigned i) const { return getOperand(i); } - void setArgOperand(unsigned i, Value *v) { setOperand(i, v); } + Value *getArgOperand(unsigned i) const { + assert(i < getNumArgOperands() && "Out of bounds!"); + return getOperand(i); + } + void setArgOperand(unsigned i, Value *v) { + assert(i < getNumArgOperands() && "Out of bounds!"); + setOperand(i, v); + } + + /// \brief Return the iterator pointing to the beginning of the argument list. + op_iterator arg_begin() { return op_begin(); } - /// arg_operands - iteration adapter for range-for loops. + /// \brief Return the iterator pointing to the end of the argument list. + op_iterator arg_end() { + // [ call args ], [ operand bundles ], callee + return op_end() - getNumTotalBundleOperands() - 1; + }; + + /// \brief Iteration adapter for range-for loops. iterator_range<op_iterator> arg_operands() { - // The last operand in the op list is the callee - it's not one of the args - // so we don't want to iterate over it. - return iterator_range<op_iterator>(op_begin(), op_end() - 1); + return make_range(arg_begin(), arg_end()); } - /// arg_operands - iteration adapter for range-for loops. + /// \brief Return the iterator pointing to the beginning of the argument list. + const_op_iterator arg_begin() const { return op_begin(); } + + /// \brief Return the iterator pointing to the end of the argument list. + const_op_iterator arg_end() const { + // [ call args ], [ operand bundles ], callee + return op_end() - getNumTotalBundleOperands() - 1; + }; + + /// \brief Iteration adapter for range-for loops. iterator_range<const_op_iterator> arg_operands() const { - return iterator_range<const_op_iterator>(op_begin(), op_end() - 1); + return make_range(arg_begin(), arg_end()); } /// \brief Wrappers for getting the \c Use of a call argument. - const Use &getArgOperandUse(unsigned i) const { return getOperandUse(i); } - Use &getArgOperandUse(unsigned i) { return getOperandUse(i); } + const Use &getArgOperandUse(unsigned i) const { + assert(i < getNumArgOperands() && "Out of bounds!"); + return getOperandUse(i); + } + Use &getArgOperandUse(unsigned i) { + assert(i < getNumArgOperands() && "Out of bounds!"); + return getOperandUse(i); + } /// getCallingConv/setCallingConv - Get or set the calling convention of this /// function call. @@ -1498,8 +1592,10 @@ public: return static_cast<CallingConv::ID>(getSubclassDataFromInstruction() >> 2); } void setCallingConv(CallingConv::ID CC) { + auto ID = static_cast<unsigned>(CC); + assert(!(ID & ~CallingConv::MaxID) && "Unsupported calling convention"); setInstructionSubclassData((getSubclassDataFromInstruction() & 3) | - (static_cast<unsigned>(CC) << 2)); + (ID << 2)); } /// getAttributes - Return the parameter attributes for this call. @@ -1541,6 +1637,21 @@ public: /// \brief Determine whether the call or the callee has the given attributes. bool paramHasAttr(unsigned i, Attribute::AttrKind A) const; + /// \brief Return true if the data operand at index \p i has the attribute \p + /// A. + /// + /// Data operands include call arguments and values used in operand bundles, + /// but does not include the callee operand. This routine dispatches to the + /// underlying AttributeList or the OperandBundleUser as appropriate. + /// + /// The index \p i is interpreted as + /// + /// \p i == Attribute::ReturnIndex -> the return value + /// \p i in [1, arg_size + 1) -> argument number (\p i - 1) + /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index + /// (\p i - 1) in the operand list. + bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind A) const; + /// \brief Extract the alignment for a call or parameter (0=unknown). unsigned getParamAlignment(unsigned i) const { return AttributeList.getParamAlignment(i); @@ -1557,7 +1668,14 @@ public: uint64_t getDereferenceableOrNullBytes(unsigned i) const { return AttributeList.getDereferenceableOrNullBytes(i); } - + + /// @brief Determine if the parameter or return value is marked with NoAlias + /// attribute. + /// @param n The parameter to check. 1 is the first parameter, 0 is the return + bool doesNotAlias(unsigned n) const { + return AttributeList.hasAttribute(n, Attribute::NoAlias); + } + /// \brief Return true if the call should not be treated as a call to a /// builtin. bool isNoBuiltin() const { @@ -1622,9 +1740,18 @@ public: addAttribute(AttributeSet::FunctionIndex, Attribute::NoDuplicate); } + /// \brief Determine if the call is convergent + bool isConvergent() const { return hasFnAttr(Attribute::Convergent); } + void setConvergent() { + addAttribute(AttributeSet::FunctionIndex, Attribute::Convergent); + } + /// \brief Determine if the call returns a structure through first /// pointer argument. bool hasStructRetAttr() const { + if (getNumArgOperands() == 0) + return false; + // Be friendly and also check the callee. return paramHasAttr(1, Attribute::StructRet); } @@ -1671,12 +1798,17 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } -private: - template<typename AttrKind> - bool hasFnAttrImpl(AttrKind A) const { +private: + template <typename AttrKind> bool hasFnAttrImpl(AttrKind A) const { if (AttributeList.hasAttribute(AttributeSet::FunctionIndex, A)) return true; + + // Operand bundles override attributes on the called function, but don't + // override attributes directly present on the call instruction. + if (isFnAttrDisallowedByOpBundle(A)) + return false; + if (const Function *F = getCalledFunction()) return F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, A); return false; @@ -1694,24 +1826,28 @@ struct OperandTraits<CallInst> : public VariadicOperandTraits<CallInst, 1> { }; CallInst::CallInst(Value *Func, ArrayRef<Value *> Args, - const Twine &NameStr, BasicBlock *InsertAtEnd) - : Instruction(cast<FunctionType>(cast<PointerType>(Func->getType()) - ->getElementType())->getReturnType(), - Instruction::Call, - OperandTraits<CallInst>::op_end(this) - (Args.size() + 1), - unsigned(Args.size() + 1), InsertAtEnd) { - init(Func, Args, NameStr); + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr, + BasicBlock *InsertAtEnd) + : Instruction( + cast<FunctionType>(cast<PointerType>(Func->getType()) + ->getElementType())->getReturnType(), + Instruction::Call, OperandTraits<CallInst>::op_end(this) - + (Args.size() + CountBundleInputs(Bundles) + 1), + unsigned(Args.size() + CountBundleInputs(Bundles) + 1), InsertAtEnd) { + init(Func, Args, Bundles, NameStr); } CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args, - const Twine &NameStr, Instruction *InsertBefore) + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr, + Instruction *InsertBefore) : Instruction(Ty->getReturnType(), Instruction::Call, - OperandTraits<CallInst>::op_end(this) - (Args.size() + 1), - unsigned(Args.size() + 1), InsertBefore) { - init(Ty, Func, Args, NameStr); + OperandTraits<CallInst>::op_end(this) - + (Args.size() + CountBundleInputs(Bundles) + 1), + unsigned(Args.size() + CountBundleInputs(Bundles) + 1), + InsertBefore) { + init(Ty, Func, Args, Bundles, NameStr); } - // Note: if you get compile errors about private methods then // please update your code to use the high-level operand // interfaces. See line 943 above. @@ -1745,6 +1881,7 @@ class SelectInst : public Instruction { init(C, S1, S2); setName(NameStr); } + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -1845,6 +1982,7 @@ class ExtractElementInst : public Instruction { Instruction *InsertBefore = nullptr); ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr, BasicBlock *InsertAtEnd); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -1875,7 +2013,6 @@ public: return cast<VectorType>(getVectorOperand()->getType()); } - /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); @@ -1906,8 +2043,9 @@ class InsertElementInst : public Instruction { InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr = "", Instruction *InsertBefore = nullptr); - InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, - const Twine &NameStr, BasicBlock *InsertAtEnd); + InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr, + BasicBlock *InsertAtEnd); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -2020,7 +2158,6 @@ public: return Mask; } - // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { return I->getOpcode() == Instruction::ShuffleVector; @@ -2063,9 +2200,8 @@ class ExtractValueInst : public UnaryInstruction { const Twine &NameStr, BasicBlock *InsertAtEnd); // allocate space for exactly one operand - void *operator new(size_t s) { - return User::operator new(s, 1); - } + void *operator new(size_t s) { return User::operator new(s, 1); } + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -2096,7 +2232,7 @@ public: inline idx_iterator idx_begin() const { return Indices.begin(); } inline idx_iterator idx_end() const { return Indices.end(); } inline iterator_range<idx_iterator> indices() const { - return iterator_range<idx_iterator>(idx_begin(), idx_end()); + return make_range(idx_begin(), idx_end()); } Value *getAggregateOperand() { @@ -2147,7 +2283,6 @@ ExtractValueInst::ExtractValueInst(Value *Agg, init(Idxs, NameStr); } - //===----------------------------------------------------------------------===// // InsertValueInst Class //===----------------------------------------------------------------------===// @@ -2177,11 +2312,12 @@ class InsertValueInst : public Instruction { /// Constructors - These two constructors are convenience methods because one /// and two index insertvalue instructions are so common. - InsertValueInst(Value *Agg, Value *Val, - unsigned Idx, const Twine &NameStr = "", - Instruction *InsertBefore = nullptr); InsertValueInst(Value *Agg, Value *Val, unsigned Idx, - const Twine &NameStr, BasicBlock *InsertAtEnd); + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr); + InsertValueInst(Value *Agg, Value *Val, unsigned Idx, const Twine &NameStr, + BasicBlock *InsertAtEnd); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -2213,7 +2349,7 @@ public: inline idx_iterator idx_begin() const { return Indices.begin(); } inline idx_iterator idx_end() const { return Indices.end(); } inline iterator_range<idx_iterator> indices() const { - return iterator_range<idx_iterator>(idx_begin(), idx_end()); + return make_range(idx_begin(), idx_end()); } Value *getAggregateOperand() { @@ -2294,6 +2430,8 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueInst, Value) // scientist's overactive imagination. // class PHINode : public Instruction { + void anchor() override; + void *operator new(size_t, unsigned) = delete; /// ReservedSpace - The number of operands actually allocated. NumOperands is /// the number actually in use. @@ -2319,6 +2457,7 @@ class PHINode : public Instruction { setName(NameStr); allocHungoffUses(ReservedSpace); } + protected: // allocHungoffUses - this is more complicated than the generic // User::allocHungoffUses, because we have to allocate Uses for the incoming @@ -2387,6 +2526,9 @@ public: return getOperand(i); } void setIncomingValue(unsigned i, Value *V) { + assert(V && "PHI node got a null value!"); + assert(getType() == V->getType() && + "All operands to PHI node must be the same type as the PHI node!"); setOperand(i, V); } static unsigned getOperandNumForIncomingValue(unsigned i) { @@ -2418,16 +2560,13 @@ public: } void setIncomingBlock(unsigned i, BasicBlock *BB) { + assert(BB && "PHI node got a null basic block!"); block_begin()[i] = BB; } /// addIncoming - Add an incoming value to the end of the PHI list /// void addIncoming(Value *V, BasicBlock *BB) { - assert(V && "PHI node got a null value!"); - assert(BB && "PHI node got a null basic block!"); - assert(getType() == V->getType() && - "All operands to PHI node must be the same type as the PHI node!"); if (getNumOperands() == ReservedSpace) growOperands(); // Get more space! // Initialize some new operands. @@ -2479,7 +2618,8 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } - private: + +private: void growOperands(); }; @@ -2506,8 +2646,10 @@ class LandingPadInst : public Instruction { /// the number actually in use. unsigned ReservedSpace; LandingPadInst(const LandingPadInst &LP); + public: enum ClauseType { Catch, Filter }; + private: void *operator new(size_t, unsigned) = delete; // Allocate space for exactly zero operands. @@ -2618,6 +2760,7 @@ private: Instruction *InsertBefore = nullptr); ReturnInst(LLVMContext &C, Value *retVal, BasicBlock *InsertAtEnd); explicit ReturnInst(LLVMContext &C, BasicBlock *InsertAtEnd); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -2654,7 +2797,8 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } - private: + +private: BasicBlock *getSuccessorV(unsigned idx) const override; unsigned getNumSuccessorsV() const override; void setSuccessorV(unsigned idx, BasicBlock *B) override; @@ -2693,6 +2837,7 @@ class BranchInst : public TerminatorInst { BranchInst(BasicBlock *IfTrue, BasicBlock *InsertAtEnd); BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond, BasicBlock *InsertAtEnd); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -2740,7 +2885,7 @@ public: void setSuccessor(unsigned idx, BasicBlock *NewSucc) { assert(idx < getNumSuccessors() && "Successor # out of range for Branch!"); - *(&Op<-1>() - idx) = (Value*)NewSucc; + *(&Op<-1>() - idx) = NewSucc; } /// \brief Swap the successors of this branch instruction. @@ -2757,6 +2902,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: BasicBlock *getSuccessorV(unsigned idx) const override; unsigned getNumSuccessorsV() const override; @@ -2803,25 +2949,23 @@ class SwitchInst : public TerminatorInst { /// constructor also autoinserts at the end of the specified BasicBlock. SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases, BasicBlock *InsertAtEnd); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; SwitchInst *cloneImpl() const; public: - // -2 static const unsigned DefaultPseudoIndex = static_cast<unsigned>(~0L-1); template <class SwitchInstTy, class ConstantIntTy, class BasicBlockTy> class CaseIteratorT { protected: - SwitchInstTy *SI; unsigned Index; public: - typedef CaseIteratorT<SwitchInstTy, ConstantIntTy, BasicBlockTy> Self; /// Initializes case iterator for given SwitchInst and for given @@ -2912,8 +3056,7 @@ public: typedef CaseIteratorT<SwitchInst, ConstantInt, BasicBlock> ParentTy; public: - - CaseIt(const ParentTy& Src) : ParentTy(Src) {} + CaseIt(const ParentTy &Src) : ParentTy(Src) {} CaseIt(SwitchInst *SI, unsigned CaseNum) : ParentTy(SI, CaseNum) {} /// Sets the new value for current case. @@ -2983,12 +3126,12 @@ public: /// cases - iteration adapter for range-for loops. iterator_range<CaseIt> cases() { - return iterator_range<CaseIt>(case_begin(), case_end()); + return make_range(case_begin(), case_end()); } /// cases - iteration adapter for range-for loops. iterator_range<ConstCaseIt> cases() const { - return iterator_range<ConstCaseIt>(case_begin(), case_end()); + return make_range(case_begin(), case_end()); } /// Returns an iterator that points to the default case. @@ -3056,7 +3199,7 @@ public: } void setSuccessor(unsigned idx, BasicBlock *NewSucc) { assert(idx < getNumSuccessors() && "Successor # out of range for switch!"); - setOperand(idx*2+1, (Value*)NewSucc); + setOperand(idx * 2 + 1, NewSucc); } // Methods for support type inquiry through isa, cast, and dyn_cast: @@ -3066,6 +3209,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: BasicBlock *getSuccessorV(unsigned idx) const override; unsigned getNumSuccessorsV() const override; @@ -3078,7 +3222,6 @@ struct OperandTraits<SwitchInst> : public HungoffOperandTraits<2> { DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SwitchInst, Value) - //===----------------------------------------------------------------------===// // IndirectBrInst Class //===----------------------------------------------------------------------===// @@ -3111,6 +3254,7 @@ class IndirectBrInst : public TerminatorInst { /// here to make memory allocation more efficient. This constructor also /// autoinserts at the end of the specified BasicBlock. IndirectBrInst(Value *Address, unsigned NumDests, BasicBlock *InsertAtEnd); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -3134,7 +3278,6 @@ public: const Value *getAddress() const { return getOperand(0); } void setAddress(Value *V) { setOperand(0, V); } - /// getNumDestinations - return the number of possible destinations in this /// indirectbr instruction. unsigned getNumDestinations() const { return getNumOperands()-1; } @@ -3156,7 +3299,7 @@ public: return cast<BasicBlock>(getOperand(i+1)); } void setSuccessor(unsigned i, BasicBlock *NewSucc) { - setOperand(i+1, (Value*)NewSucc); + setOperand(i + 1, NewSucc); } // Methods for support type inquiry through isa, cast, and dyn_cast: @@ -3166,6 +3309,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: BasicBlock *getSuccessorV(unsigned idx) const override; unsigned getNumSuccessorsV() const override; @@ -3178,7 +3322,6 @@ struct OperandTraits<IndirectBrInst> : public HungoffOperandTraits<1> { DEFINE_TRANSPARENT_OPERAND_ACCESSORS(IndirectBrInst, Value) - //===----------------------------------------------------------------------===// // InvokeInst Class //===----------------------------------------------------------------------===// @@ -3186,72 +3329,123 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(IndirectBrInst, Value) /// InvokeInst - Invoke instruction. The SubclassData field is used to hold the /// calling convention of the call. /// -class InvokeInst : public TerminatorInst { +class InvokeInst : public TerminatorInst, + public OperandBundleUser<InvokeInst, User::op_iterator> { AttributeSet AttributeList; FunctionType *FTy; InvokeInst(const InvokeInst &BI); void init(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef<Value *> Args, const Twine &NameStr) { + ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles, + const Twine &NameStr) { init(cast<FunctionType>( cast<PointerType>(Func->getType())->getElementType()), - Func, IfNormal, IfException, Args, NameStr); + Func, IfNormal, IfException, Args, Bundles, NameStr); } void init(FunctionType *FTy, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef<Value *> Args, - const Twine &NameStr); + ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr); /// Construct an InvokeInst given a range of arguments. /// /// \brief Construct an InvokeInst from a range of arguments inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef<Value *> Args, unsigned Values, - const Twine &NameStr, Instruction *InsertBefore) + ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles, + unsigned Values, const Twine &NameStr, + Instruction *InsertBefore) : InvokeInst(cast<FunctionType>( cast<PointerType>(Func->getType())->getElementType()), - Func, IfNormal, IfException, Args, Values, NameStr, + Func, IfNormal, IfException, Args, Bundles, Values, NameStr, InsertBefore) {} inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef<Value *> Args, - unsigned Values, const Twine &NameStr, - Instruction *InsertBefore); + ArrayRef<OperandBundleDef> Bundles, unsigned Values, + const Twine &NameStr, Instruction *InsertBefore); /// Construct an InvokeInst given a range of arguments. /// /// \brief Construct an InvokeInst from a range of arguments inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef<Value *> Args, unsigned Values, - const Twine &NameStr, BasicBlock *InsertAtEnd); + ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles, + unsigned Values, const Twine &NameStr, + BasicBlock *InsertAtEnd); + + friend class OperandBundleUser<InvokeInst, User::op_iterator>; + bool hasDescriptor() const { return HasDescriptor; } + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; InvokeInst *cloneImpl() const; public: - static InvokeInst *Create(Value *Func, - BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef<Value *> Args, const Twine &NameStr = "", + static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef<Value *> Args, + const Twine &NameStr, Instruction *InsertBefore = nullptr) { return Create(cast<FunctionType>( cast<PointerType>(Func->getType())->getElementType()), - Func, IfNormal, IfException, Args, NameStr, InsertBefore); + Func, IfNormal, IfException, Args, None, NameStr, + InsertBefore); } - static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, + static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles = None, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { + return Create(cast<FunctionType>( + cast<PointerType>(Func->getType())->getElementType()), + Func, IfNormal, IfException, Args, Bundles, NameStr, + InsertBefore); + } + static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef<Value *> Args, + const Twine &NameStr, + Instruction *InsertBefore = nullptr) { unsigned Values = unsigned(Args.size()) + 3; - return new (Values) InvokeInst(Ty, Func, IfNormal, IfException, Args, + return new (Values) InvokeInst(Ty, Func, IfNormal, IfException, Args, None, Values, NameStr, InsertBefore); } + static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + unsigned Values = unsigned(Args.size()) + CountBundleInputs(Bundles) + 3; + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (Values, DescriptorBytes) + InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, Values, + NameStr, InsertBefore); + } static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef<Value *> Args, const Twine &NameStr, BasicBlock *InsertAtEnd) { unsigned Values = unsigned(Args.size()) + 3; - return new(Values) InvokeInst(Func, IfNormal, IfException, Args, - Values, NameStr, InsertAtEnd); + return new (Values) InvokeInst(Func, IfNormal, IfException, Args, None, + Values, NameStr, InsertAtEnd); + } + static InvokeInst *Create(Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + unsigned Values = unsigned(Args.size()) + CountBundleInputs(Bundles) + 3; + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (Values, DescriptorBytes) + InvokeInst(Func, IfNormal, IfException, Args, Bundles, Values, NameStr, + InsertAtEnd); } + /// \brief Create a clone of \p II with a different set of operand bundles and + /// insert it before \p InsertPt. + /// + /// The returned invoke instruction is identical to \p II in every way except + /// that the operand bundles for the new instruction are set to the operand + /// bundles in \p Bundles. + static InvokeInst *Create(InvokeInst *II, ArrayRef<OperandBundleDef> Bundles, + Instruction *InsertPt = nullptr); + /// Provide fast operand accessors DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); @@ -3264,26 +3458,58 @@ public: /// getNumArgOperands - Return the number of invoke arguments. /// - unsigned getNumArgOperands() const { return getNumOperands() - 3; } + unsigned getNumArgOperands() const { + return getNumOperands() - getNumTotalBundleOperands() - 3; + } /// getArgOperand/setArgOperand - Return/set the i-th invoke argument. /// - Value *getArgOperand(unsigned i) const { return getOperand(i); } - void setArgOperand(unsigned i, Value *v) { setOperand(i, v); } + Value *getArgOperand(unsigned i) const { + assert(i < getNumArgOperands() && "Out of bounds!"); + return getOperand(i); + } + void setArgOperand(unsigned i, Value *v) { + assert(i < getNumArgOperands() && "Out of bounds!"); + setOperand(i, v); + } + + /// \brief Return the iterator pointing to the beginning of the argument list. + op_iterator arg_begin() { return op_begin(); } - /// arg_operands - iteration adapter for range-for loops. + /// \brief Return the iterator pointing to the end of the argument list. + op_iterator arg_end() { + // [ invoke args ], [ operand bundles ], normal dest, unwind dest, callee + return op_end() - getNumTotalBundleOperands() - 3; + }; + + /// \brief Iteration adapter for range-for loops. iterator_range<op_iterator> arg_operands() { - return iterator_range<op_iterator>(op_begin(), op_end() - 3); + return make_range(arg_begin(), arg_end()); } - /// arg_operands - iteration adapter for range-for loops. + /// \brief Return the iterator pointing to the beginning of the argument list. + const_op_iterator arg_begin() const { return op_begin(); } + + /// \brief Return the iterator pointing to the end of the argument list. + const_op_iterator arg_end() const { + // [ invoke args ], [ operand bundles ], normal dest, unwind dest, callee + return op_end() - getNumTotalBundleOperands() - 3; + }; + + /// \brief Iteration adapter for range-for loops. iterator_range<const_op_iterator> arg_operands() const { - return iterator_range<const_op_iterator>(op_begin(), op_end() - 3); + return make_range(arg_begin(), arg_end()); } /// \brief Wrappers for getting the \c Use of a invoke argument. - const Use &getArgOperandUse(unsigned i) const { return getOperandUse(i); } - Use &getArgOperandUse(unsigned i) { return getOperandUse(i); } + const Use &getArgOperandUse(unsigned i) const { + assert(i < getNumArgOperands() && "Out of bounds!"); + return getOperandUse(i); + } + Use &getArgOperandUse(unsigned i) { + assert(i < getNumArgOperands() && "Out of bounds!"); + return getOperandUse(i); + } /// getCallingConv/setCallingConv - Get or set the calling convention of this /// function call. @@ -3291,7 +3517,9 @@ public: return static_cast<CallingConv::ID>(getSubclassDataFromInstruction()); } void setCallingConv(CallingConv::ID CC) { - setInstructionSubclassData(static_cast<unsigned>(CC)); + auto ID = static_cast<unsigned>(CC); + assert(!(ID & ~CallingConv::MaxID) && "Unsupported calling convention"); + setInstructionSubclassData(ID); } /// getAttributes - Return the parameter attributes for this invoke. @@ -3325,6 +3553,22 @@ public: /// \brief Determine whether the call or the callee has the given attributes. bool paramHasAttr(unsigned i, Attribute::AttrKind A) const; + /// \brief Return true if the data operand at index \p i has the attribute \p + /// A. + /// + /// Data operands include invoke arguments and values used in operand bundles, + /// but does not include the invokee operand, or the two successor blocks. + /// This routine dispatches to the underlying AttributeList or the + /// OperandBundleUser as appropriate. + /// + /// The index \p i is interpreted as + /// + /// \p i == Attribute::ReturnIndex -> the return value + /// \p i in [1, arg_size + 1) -> argument number (\p i - 1) + /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index + /// (\p i - 1) in the operand list. + bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind A) const; + /// \brief Extract the alignment for a call or parameter (0=unknown). unsigned getParamAlignment(unsigned i) const { return AttributeList.getParamAlignment(i); @@ -3335,13 +3579,20 @@ public: uint64_t getDereferenceableBytes(unsigned i) const { return AttributeList.getDereferenceableBytes(i); } - + /// \brief Extract the number of dereferenceable_or_null bytes for a call or /// parameter (0=unknown). uint64_t getDereferenceableOrNullBytes(unsigned i) const { return AttributeList.getDereferenceableOrNullBytes(i); } + /// @brief Determine if the parameter or return value is marked with NoAlias + /// attribute. + /// @param n The parameter to check. 1 is the first parameter, 0 is the return + bool doesNotAlias(unsigned n) const { + return AttributeList.hasAttribute(n, Attribute::NoAlias); + } + /// \brief Return true if the call should not be treated as a call to a /// builtin. bool isNoBuiltin() const { @@ -3403,6 +3654,9 @@ public: /// \brief Determine if the call returns a structure through first /// pointer argument. bool hasStructRetAttr() const { + if (getNumArgOperands() == 0) + return false; + // Be friendly and also check the callee. return paramHasAttr(1, Attribute::StructRet); } @@ -3495,23 +3749,23 @@ struct OperandTraits<InvokeInst> : public VariadicOperandTraits<InvokeInst, 3> { InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef<Value *> Args, - unsigned Values, const Twine &NameStr, - Instruction *InsertBefore) + ArrayRef<OperandBundleDef> Bundles, unsigned Values, + const Twine &NameStr, Instruction *InsertBefore) : TerminatorInst(Ty->getReturnType(), Instruction::Invoke, OperandTraits<InvokeInst>::op_end(this) - Values, Values, InsertBefore) { - init(Ty, Func, IfNormal, IfException, Args, NameStr); + init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr); } -InvokeInst::InvokeInst(Value *Func, - BasicBlock *IfNormal, BasicBlock *IfException, - ArrayRef<Value *> Args, unsigned Values, +InvokeInst::InvokeInst(Value *Func, BasicBlock *IfNormal, + BasicBlock *IfException, ArrayRef<Value *> Args, + ArrayRef<OperandBundleDef> Bundles, unsigned Values, const Twine &NameStr, BasicBlock *InsertAtEnd) - : TerminatorInst(cast<FunctionType>(cast<PointerType>(Func->getType()) - ->getElementType())->getReturnType(), - Instruction::Invoke, - OperandTraits<InvokeInst>::op_end(this) - Values, - Values, InsertAtEnd) { - init(Func, IfNormal, IfException, Args, NameStr); + : TerminatorInst( + cast<FunctionType>(cast<PointerType>(Func->getType()) + ->getElementType())->getReturnType(), + Instruction::Invoke, OperandTraits<InvokeInst>::op_end(this) - Values, + Values, InsertAtEnd) { + init(Func, IfNormal, IfException, Args, Bundles, NameStr); } DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InvokeInst, Value) @@ -3528,6 +3782,7 @@ class ResumeInst : public TerminatorInst { explicit ResumeInst(Value *Exn, Instruction *InsertBefore=nullptr); ResumeInst(Value *Exn, BasicBlock *InsertAtEnd); + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -3556,6 +3811,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: BasicBlock *getSuccessorV(unsigned idx) const override; unsigned getNumSuccessorsV() const override; @@ -3570,6 +3826,430 @@ struct OperandTraits<ResumeInst> : DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value) //===----------------------------------------------------------------------===// +// CatchSwitchInst Class +//===----------------------------------------------------------------------===// +class CatchSwitchInst : public TerminatorInst { + void *operator new(size_t, unsigned) = delete; + /// ReservedSpace - The number of operands actually allocated. NumOperands is + /// the number actually in use. + unsigned ReservedSpace; + // Operand[0] = Outer scope + // Operand[1] = Unwind block destination + // Operand[n] = BasicBlock to go to on match + CatchSwitchInst(const CatchSwitchInst &CSI); + void init(Value *ParentPad, BasicBlock *UnwindDest, unsigned NumReserved); + void growOperands(unsigned Size); + // allocate space for exactly zero operands + void *operator new(size_t s) { return User::operator new(s); } + /// CatchSwitchInst ctor - Create a new switch instruction, specifying a + /// default destination. The number of additional handlers can be specified + /// here to make memory allocation more efficient. + /// This constructor can also autoinsert before another instruction. + CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest, + unsigned NumHandlers, const Twine &NameStr, + Instruction *InsertBefore); + + /// CatchSwitchInst ctor - Create a new switch instruction, specifying a + /// default destination. The number of additional handlers can be specified + /// here to make memory allocation more efficient. + /// This constructor also autoinserts at the end of the specified BasicBlock. + CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest, + unsigned NumHandlers, const Twine &NameStr, + BasicBlock *InsertAtEnd); + +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + CatchSwitchInst *cloneImpl() const; + +public: + static CatchSwitchInst *Create(Value *ParentPad, BasicBlock *UnwindDest, + unsigned NumHandlers, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + return new CatchSwitchInst(ParentPad, UnwindDest, NumHandlers, NameStr, + InsertBefore); + } + static CatchSwitchInst *Create(Value *ParentPad, BasicBlock *UnwindDest, + unsigned NumHandlers, const Twine &NameStr, + BasicBlock *InsertAtEnd) { + return new CatchSwitchInst(ParentPad, UnwindDest, NumHandlers, NameStr, + InsertAtEnd); + } + + /// Provide fast operand accessors + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + // Accessor Methods for CatchSwitch stmt + Value *getParentPad() const { return getOperand(0); } + void setParentPad(Value *ParentPad) { setOperand(0, ParentPad); } + + // Accessor Methods for CatchSwitch stmt + bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; } + bool unwindsToCaller() const { return !hasUnwindDest(); } + BasicBlock *getUnwindDest() const { + if (hasUnwindDest()) + return cast<BasicBlock>(getOperand(1)); + return nullptr; + } + void setUnwindDest(BasicBlock *UnwindDest) { + assert(UnwindDest); + assert(hasUnwindDest()); + setOperand(1, UnwindDest); + } + + /// getNumHandlers - return the number of 'handlers' in this catchswitch + /// instruction, except the default handler + unsigned getNumHandlers() const { + if (hasUnwindDest()) + return getNumOperands() - 2; + return getNumOperands() - 1; + } + +private: + static BasicBlock *handler_helper(Value *V) { return cast<BasicBlock>(V); } + static const BasicBlock *handler_helper(const Value *V) { + return cast<BasicBlock>(V); + } + +public: + typedef std::pointer_to_unary_function<Value *, BasicBlock *> DerefFnTy; + typedef mapped_iterator<op_iterator, DerefFnTy> handler_iterator; + typedef iterator_range<handler_iterator> handler_range; + + + typedef std::pointer_to_unary_function<const Value *, const BasicBlock *> + ConstDerefFnTy; + typedef mapped_iterator<const_op_iterator, ConstDerefFnTy> const_handler_iterator; + typedef iterator_range<const_handler_iterator> const_handler_range; + + /// Returns an iterator that points to the first handler in CatchSwitchInst. + handler_iterator handler_begin() { + op_iterator It = op_begin() + 1; + if (hasUnwindDest()) + ++It; + return handler_iterator(It, DerefFnTy(handler_helper)); + } + /// Returns an iterator that points to the first handler in the + /// CatchSwitchInst. + const_handler_iterator handler_begin() const { + const_op_iterator It = op_begin() + 1; + if (hasUnwindDest()) + ++It; + return const_handler_iterator(It, ConstDerefFnTy(handler_helper)); + } + + /// Returns a read-only iterator that points one past the last + /// handler in the CatchSwitchInst. + handler_iterator handler_end() { + return handler_iterator(op_end(), DerefFnTy(handler_helper)); + } + /// Returns an iterator that points one past the last handler in the + /// CatchSwitchInst. + const_handler_iterator handler_end() const { + return const_handler_iterator(op_end(), ConstDerefFnTy(handler_helper)); + } + + /// handlers - iteration adapter for range-for loops. + handler_range handlers() { + return make_range(handler_begin(), handler_end()); + } + + /// handlers - iteration adapter for range-for loops. + const_handler_range handlers() const { + return make_range(handler_begin(), handler_end()); + } + + /// addHandler - Add an entry to the switch instruction... + /// Note: + /// This action invalidates handler_end(). Old handler_end() iterator will + /// point to the added handler. + void addHandler(BasicBlock *Dest); + + unsigned getNumSuccessors() const { return getNumOperands() - 1; } + BasicBlock *getSuccessor(unsigned Idx) const { + assert(Idx < getNumSuccessors() && + "Successor # out of range for catchswitch!"); + return cast<BasicBlock>(getOperand(Idx + 1)); + } + void setSuccessor(unsigned Idx, BasicBlock *NewSucc) { + assert(Idx < getNumSuccessors() && + "Successor # out of range for catchswitch!"); + setOperand(Idx + 1, NewSucc); + } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Instruction *I) { + return I->getOpcode() == Instruction::CatchSwitch; + } + static inline bool classof(const Value *V) { + return isa<Instruction>(V) && classof(cast<Instruction>(V)); + } + +private: + BasicBlock *getSuccessorV(unsigned Idx) const override; + unsigned getNumSuccessorsV() const override; + void setSuccessorV(unsigned Idx, BasicBlock *B) override; +}; + +template <> +struct OperandTraits<CatchSwitchInst> : public HungoffOperandTraits<2> {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchSwitchInst, Value) + +//===----------------------------------------------------------------------===// +// CleanupPadInst Class +//===----------------------------------------------------------------------===// +class CleanupPadInst : public FuncletPadInst { +private: + explicit CleanupPadInst(Value *ParentPad, ArrayRef<Value *> Args, + unsigned Values, const Twine &NameStr, + Instruction *InsertBefore) + : FuncletPadInst(Instruction::CleanupPad, ParentPad, Args, Values, + NameStr, InsertBefore) {} + explicit CleanupPadInst(Value *ParentPad, ArrayRef<Value *> Args, + unsigned Values, const Twine &NameStr, + BasicBlock *InsertAtEnd) + : FuncletPadInst(Instruction::CleanupPad, ParentPad, Args, Values, + NameStr, InsertAtEnd) {} + +public: + static CleanupPadInst *Create(Value *ParentPad, ArrayRef<Value *> Args = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + unsigned Values = 1 + Args.size(); + return new (Values) + CleanupPadInst(ParentPad, Args, Values, NameStr, InsertBefore); + } + static CleanupPadInst *Create(Value *ParentPad, ArrayRef<Value *> Args, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + unsigned Values = 1 + Args.size(); + return new (Values) + CleanupPadInst(ParentPad, Args, Values, NameStr, InsertAtEnd); + } + + /// \brief Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Instruction *I) { + return I->getOpcode() == Instruction::CleanupPad; + } + static inline bool classof(const Value *V) { + return isa<Instruction>(V) && classof(cast<Instruction>(V)); + } +}; + +//===----------------------------------------------------------------------===// +// CatchPadInst Class +//===----------------------------------------------------------------------===// +class CatchPadInst : public FuncletPadInst { +private: + explicit CatchPadInst(Value *CatchSwitch, ArrayRef<Value *> Args, + unsigned Values, const Twine &NameStr, + Instruction *InsertBefore) + : FuncletPadInst(Instruction::CatchPad, CatchSwitch, Args, Values, + NameStr, InsertBefore) {} + explicit CatchPadInst(Value *CatchSwitch, ArrayRef<Value *> Args, + unsigned Values, const Twine &NameStr, + BasicBlock *InsertAtEnd) + : FuncletPadInst(Instruction::CatchPad, CatchSwitch, Args, Values, + NameStr, InsertAtEnd) {} + +public: + static CatchPadInst *Create(Value *CatchSwitch, ArrayRef<Value *> Args, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + unsigned Values = 1 + Args.size(); + return new (Values) + CatchPadInst(CatchSwitch, Args, Values, NameStr, InsertBefore); + } + static CatchPadInst *Create(Value *CatchSwitch, ArrayRef<Value *> Args, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + unsigned Values = 1 + Args.size(); + return new (Values) + CatchPadInst(CatchSwitch, Args, Values, NameStr, InsertAtEnd); + } + + /// Convenience accessors + CatchSwitchInst *getCatchSwitch() const { + return cast<CatchSwitchInst>(Op<-1>()); + } + void setCatchSwitch(Value *CatchSwitch) { + assert(CatchSwitch); + Op<-1>() = CatchSwitch; + } + + /// \brief Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Instruction *I) { + return I->getOpcode() == Instruction::CatchPad; + } + static inline bool classof(const Value *V) { + return isa<Instruction>(V) && classof(cast<Instruction>(V)); + } +}; + +//===----------------------------------------------------------------------===// +// CatchReturnInst Class +//===----------------------------------------------------------------------===// + +class CatchReturnInst : public TerminatorInst { + CatchReturnInst(const CatchReturnInst &RI); + + void init(Value *CatchPad, BasicBlock *BB); + CatchReturnInst(Value *CatchPad, BasicBlock *BB, Instruction *InsertBefore); + CatchReturnInst(Value *CatchPad, BasicBlock *BB, BasicBlock *InsertAtEnd); + +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + CatchReturnInst *cloneImpl() const; + +public: + static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB, + Instruction *InsertBefore = nullptr) { + assert(CatchPad); + assert(BB); + return new (2) CatchReturnInst(CatchPad, BB, InsertBefore); + } + static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB, + BasicBlock *InsertAtEnd) { + assert(CatchPad); + assert(BB); + return new (2) CatchReturnInst(CatchPad, BB, InsertAtEnd); + } + + /// Provide fast operand accessors + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + /// Convenience accessors. + CatchPadInst *getCatchPad() const { return cast<CatchPadInst>(Op<0>()); } + void setCatchPad(CatchPadInst *CatchPad) { + assert(CatchPad); + Op<0>() = CatchPad; + } + + BasicBlock *getSuccessor() const { return cast<BasicBlock>(Op<1>()); } + void setSuccessor(BasicBlock *NewSucc) { + assert(NewSucc); + Op<1>() = NewSucc; + } + unsigned getNumSuccessors() const { return 1; } + + Value *getParentPad() const { + return getCatchPad()->getCatchSwitch()->getParentPad(); + } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Instruction *I) { + return (I->getOpcode() == Instruction::CatchRet); + } + static inline bool classof(const Value *V) { + return isa<Instruction>(V) && classof(cast<Instruction>(V)); + } + +private: + BasicBlock *getSuccessorV(unsigned Idx) const override; + unsigned getNumSuccessorsV() const override; + void setSuccessorV(unsigned Idx, BasicBlock *B) override; +}; + +template <> +struct OperandTraits<CatchReturnInst> + : public FixedNumOperandTraits<CatchReturnInst, 2> {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchReturnInst, Value) + +//===----------------------------------------------------------------------===// +// CleanupReturnInst Class +//===----------------------------------------------------------------------===// + +class CleanupReturnInst : public TerminatorInst { +private: + CleanupReturnInst(const CleanupReturnInst &RI); + + void init(Value *CleanupPad, BasicBlock *UnwindBB); + CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values, + Instruction *InsertBefore = nullptr); + CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values, + BasicBlock *InsertAtEnd); + +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + CleanupReturnInst *cloneImpl() const; + +public: + static CleanupReturnInst *Create(Value *CleanupPad, + BasicBlock *UnwindBB = nullptr, + Instruction *InsertBefore = nullptr) { + assert(CleanupPad); + unsigned Values = 1; + if (UnwindBB) + ++Values; + return new (Values) + CleanupReturnInst(CleanupPad, UnwindBB, Values, InsertBefore); + } + static CleanupReturnInst *Create(Value *CleanupPad, BasicBlock *UnwindBB, + BasicBlock *InsertAtEnd) { + assert(CleanupPad); + unsigned Values = 1; + if (UnwindBB) + ++Values; + return new (Values) + CleanupReturnInst(CleanupPad, UnwindBB, Values, InsertAtEnd); + } + + /// Provide fast operand accessors + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; } + bool unwindsToCaller() const { return !hasUnwindDest(); } + + /// Convenience accessor. + CleanupPadInst *getCleanupPad() const { + return cast<CleanupPadInst>(Op<0>()); + } + void setCleanupPad(CleanupPadInst *CleanupPad) { + assert(CleanupPad); + Op<0>() = CleanupPad; + } + + unsigned getNumSuccessors() const { return hasUnwindDest() ? 1 : 0; } + + BasicBlock *getUnwindDest() const { + return hasUnwindDest() ? cast<BasicBlock>(Op<1>()) : nullptr; + } + void setUnwindDest(BasicBlock *NewDest) { + assert(NewDest); + assert(hasUnwindDest()); + Op<1>() = NewDest; + } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Instruction *I) { + return (I->getOpcode() == Instruction::CleanupRet); + } + static inline bool classof(const Value *V) { + return isa<Instruction>(V) && classof(cast<Instruction>(V)); + } + +private: + BasicBlock *getSuccessorV(unsigned Idx) const override; + unsigned getNumSuccessorsV() const override; + void setSuccessorV(unsigned Idx, BasicBlock *B) override; + + // Shadow Instruction::setInstructionSubclassData with a private forwarding + // method so that subclasses cannot accidentally use it. + void setInstructionSubclassData(unsigned short D) { + Instruction::setInstructionSubclassData(D); + } +}; + +template <> +struct OperandTraits<CleanupReturnInst> + : public VariadicOperandTraits<CleanupReturnInst, /*MINARITY=*/1> {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CleanupReturnInst, Value) + +//===----------------------------------------------------------------------===// // UnreachableInst Class //===----------------------------------------------------------------------===// @@ -3580,6 +4260,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value) /// class UnreachableInst : public TerminatorInst { void *operator new(size_t, unsigned) = delete; + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -3602,6 +4283,7 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + private: BasicBlock *getSuccessorV(unsigned idx) const override; unsigned getNumSuccessorsV() const override; diff --git a/include/llvm/IR/IntrinsicInst.h b/include/llvm/IR/IntrinsicInst.h index 2c8b6eb6f39a..169bcc021984 100644 --- a/include/llvm/IR/IntrinsicInst.h +++ b/include/llvm/IR/IntrinsicInst.h @@ -372,6 +372,39 @@ namespace llvm { return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3))); } }; -} + + /// This represents the llvm.instrprof_value_profile intrinsic. + class InstrProfValueProfileInst : public IntrinsicInst { + public: + static inline bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_value_profile; + } + static inline bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + + GlobalVariable *getName() const { + return cast<GlobalVariable>( + const_cast<Value *>(getArgOperand(0))->stripPointerCasts()); + } + + ConstantInt *getHash() const { + return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1))); + } + + Value *getTargetValue() const { + return cast<Value>(const_cast<Value *>(getArgOperand(2))); + } + + ConstantInt *getValueKind() const { + return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3))); + } + + // Returns the value site index. + ConstantInt *getIndex() const { + return cast<ConstantInt>(const_cast<Value *>(getArgOperand(4))); + } + }; +} // namespace llvm #endif diff --git a/include/llvm/IR/Intrinsics.h b/include/llvm/IR/Intrinsics.h index 43b8325107fa..314e2aaecf4b 100644 --- a/include/llvm/IR/Intrinsics.h +++ b/include/llvm/IR/Intrinsics.h @@ -79,7 +79,7 @@ namespace Intrinsic { /// intrinsic. This is returned by getIntrinsicInfoTableEntries. struct IITDescriptor { enum IITDescriptorKind { - Void, VarArg, MMX, Metadata, Half, Float, Double, + Void, VarArg, MMX, Token, Metadata, Half, Float, Double, Integer, Vector, Pointer, Struct, Argument, ExtendArgument, TruncArgument, HalfVecArgument, SameVecWidthArgument, PtrToArgument, VecOfPtrsToElt diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td index bbae720b4e12..5a95ddced538 100644 --- a/include/llvm/IR/Intrinsics.td +++ b/include/llvm/IR/Intrinsics.td @@ -73,8 +73,8 @@ def IntrNoReturn : IntrinsicProperty; // Parallels the noduplicate attribute on LLVM IR functions. def IntrNoDuplicate : IntrinsicProperty; -// IntrConvergent - Calls to this intrinsic are convergent and may only be -// moved to control equivalent blocks. +// IntrConvergent - Calls to this intrinsic are convergent and may not be made +// control-dependent on any additional values. // Parallels the convergent attribute on LLVM IR functions. def IntrConvergent : IntrinsicProperty; @@ -150,16 +150,20 @@ def llvm_anyptr_ty : LLVMAnyPointerType<llvm_i8_ty>; // (space)i8* def llvm_empty_ty : LLVMType<OtherVT>; // { } def llvm_descriptor_ty : LLVMPointerType<llvm_empty_ty>; // { }* def llvm_metadata_ty : LLVMType<MetadataVT>; // !{...} +def llvm_token_ty : LLVMType<token>; // token def llvm_x86mmx_ty : LLVMType<x86mmx>; def llvm_ptrx86mmx_ty : LLVMPointerType<llvm_x86mmx_ty>; // <1 x i64>* -def llvm_v2i1_ty : LLVMType<v2i1>; // 2 x i1 -def llvm_v4i1_ty : LLVMType<v4i1>; // 4 x i1 -def llvm_v8i1_ty : LLVMType<v8i1>; // 8 x i1 -def llvm_v16i1_ty : LLVMType<v16i1>; // 16 x i1 -def llvm_v32i1_ty : LLVMType<v32i1>; // 32 x i1 -def llvm_v64i1_ty : LLVMType<v64i1>; // 64 x i1 +def llvm_v2i1_ty : LLVMType<v2i1>; // 2 x i1 +def llvm_v4i1_ty : LLVMType<v4i1>; // 4 x i1 +def llvm_v8i1_ty : LLVMType<v8i1>; // 8 x i1 +def llvm_v16i1_ty : LLVMType<v16i1>; // 16 x i1 +def llvm_v32i1_ty : LLVMType<v32i1>; // 32 x i1 +def llvm_v64i1_ty : LLVMType<v64i1>; // 64 x i1 +def llvm_v512i1_ty : LLVMType<v512i1>; // 512 x i1 +def llvm_v1024i1_ty : LLVMType<v1024i1>; //1024 x i1 + def llvm_v1i8_ty : LLVMType<v1i8>; // 1 x i8 def llvm_v2i8_ty : LLVMType<v2i8>; // 2 x i8 def llvm_v4i8_ty : LLVMType<v4i8>; // 4 x i8 @@ -167,6 +171,8 @@ def llvm_v8i8_ty : LLVMType<v8i8>; // 8 x i8 def llvm_v16i8_ty : LLVMType<v16i8>; // 16 x i8 def llvm_v32i8_ty : LLVMType<v32i8>; // 32 x i8 def llvm_v64i8_ty : LLVMType<v64i8>; // 64 x i8 +def llvm_v128i8_ty : LLVMType<v128i8>; //128 x i8 +def llvm_v256i8_ty : LLVMType<v256i8>; //256 x i8 def llvm_v1i16_ty : LLVMType<v1i16>; // 1 x i16 def llvm_v2i16_ty : LLVMType<v2i16>; // 2 x i16 @@ -174,17 +180,23 @@ def llvm_v4i16_ty : LLVMType<v4i16>; // 4 x i16 def llvm_v8i16_ty : LLVMType<v8i16>; // 8 x i16 def llvm_v16i16_ty : LLVMType<v16i16>; // 16 x i16 def llvm_v32i16_ty : LLVMType<v32i16>; // 32 x i16 +def llvm_v64i16_ty : LLVMType<v64i16>; // 64 x i16 +def llvm_v128i16_ty : LLVMType<v128i16>; //128 x i16 def llvm_v1i32_ty : LLVMType<v1i32>; // 1 x i32 def llvm_v2i32_ty : LLVMType<v2i32>; // 2 x i32 def llvm_v4i32_ty : LLVMType<v4i32>; // 4 x i32 def llvm_v8i32_ty : LLVMType<v8i32>; // 8 x i32 def llvm_v16i32_ty : LLVMType<v16i32>; // 16 x i32 +def llvm_v32i32_ty : LLVMType<v32i32>; // 32 x i32 +def llvm_v64i32_ty : LLVMType<v64i32>; // 64 x i32 + def llvm_v1i64_ty : LLVMType<v1i64>; // 1 x i64 def llvm_v2i64_ty : LLVMType<v2i64>; // 2 x i64 def llvm_v4i64_ty : LLVMType<v4i64>; // 4 x i64 def llvm_v8i64_ty : LLVMType<v8i64>; // 8 x i64 def llvm_v16i64_ty : LLVMType<v16i64>; // 16 x i64 +def llvm_v32i64_ty : LLVMType<v32i64>; // 32 x i64 def llvm_v1i128_ty : LLVMType<v1i128>; // 1 x i128 @@ -292,6 +304,8 @@ def int_stacksave : Intrinsic<[llvm_ptr_ty]>, def int_stackrestore : Intrinsic<[], [llvm_ptr_ty]>, GCCBuiltin<"__builtin_stack_restore">; +def int_get_dynamic_area_offset : Intrinsic<[llvm_anyint_ty]>; + // IntrReadWriteArgMem is more pessimistic than strictly necessary for prefetch, // however it does conveniently prevent the prefetch from being reordered // with respect to nearby accesses to the same memory. @@ -319,6 +333,14 @@ def int_instrprof_increment : Intrinsic<[], llvm_i32_ty, llvm_i32_ty], []>; +// A call to profile runtime for value profiling of target expressions +// through instrumentation based profiling. +def int_instrprof_value_profile : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i64_ty, llvm_i32_ty, + llvm_i32_ty], + []>; + //===------------------- Standard C Library Intrinsics --------------------===// // @@ -399,6 +421,7 @@ let Properties = [IntrNoMem] in { def int_ctpop: Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; def int_ctlz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; def int_cttz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; + def int_bitreverse : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; } //===------------------------ Debugger Intrinsics -------------------------===// @@ -428,17 +451,13 @@ def int_eh_typeid_for : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; def int_eh_return_i32 : Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty]>; def int_eh_return_i64 : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty]>; -// eh.begincatch takes a pointer returned by a landingpad instruction and -// copies the exception object into the memory pointed to by the second -// parameter. If the second parameter is null, no copy occurs. -def int_eh_begincatch : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], - [NoCapture<0>, NoCapture<1>]>; -def int_eh_endcatch : Intrinsic<[], []>; +// eh.exceptionpointer returns the pointer to the exception caught by +// the given `catchpad`. +def int_eh_exceptionpointer : Intrinsic<[llvm_anyptr_ty], [llvm_token_ty], + [IntrNoMem]>; -// Represents the list of actions to take when an exception is thrown. -def int_eh_actions : Intrinsic<[llvm_ptr_ty], [llvm_vararg_ty], []>; - -def int_eh_exceptioncode : Intrinsic<[llvm_i32_ty], [], [IntrReadMem]>; +// Gets the exception code from a catchpad token. Only used on some platforms. +def int_eh_exceptioncode : Intrinsic<[llvm_i32_ty], [llvm_token_ty], [IntrNoMem]>; // __builtin_unwind_init is an undocumented GCC intrinsic that causes all // callee-saved registers to be saved and restored (regardless of whether they @@ -455,6 +474,7 @@ let Properties = [IntrNoMem] in { def int_eh_sjlj_functioncontext : Intrinsic<[], [llvm_ptr_ty]>; def int_eh_sjlj_setjmp : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty]>; def int_eh_sjlj_longjmp : Intrinsic<[], [llvm_ptr_ty], [IntrNoReturn]>; +def int_eh_sjlj_setup_dispatch : Intrinsic<[], []>; //===---------------- Generic Variable Attribute Intrinsics----------------===// // @@ -523,6 +543,10 @@ def int_invariant_end : Intrinsic<[], llvm_ptr_ty], [IntrReadWriteArgMem, NoCapture<2>]>; +def int_invariant_group_barrier : Intrinsic<[llvm_ptr_ty], + [llvm_ptr_ty], + [IntrNoMem]>; + //===------------------------ Stackmap Intrinsics -------------------------===// // def int_experimental_stackmap : Intrinsic<[], @@ -543,21 +567,17 @@ def int_experimental_patchpoint_i64 : Intrinsic<[llvm_i64_ty], //===------------------------ Garbage Collection Intrinsics ---------------===// // These are documented in docs/Statepoint.rst -def int_experimental_gc_statepoint : Intrinsic<[llvm_i32_ty], +def int_experimental_gc_statepoint : Intrinsic<[llvm_token_ty], [llvm_i64_ty, llvm_i32_ty, llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_vararg_ty], [Throws]>; -def int_experimental_gc_result : Intrinsic<[llvm_any_ty], [llvm_i32_ty]>; +def int_experimental_gc_result : Intrinsic<[llvm_any_ty], [llvm_token_ty], + [IntrReadMem]>; def int_experimental_gc_relocate : Intrinsic<[llvm_anyptr_ty], - [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>; - -// Deprecated: will be removed in a couple of weeks -def int_experimental_gc_result_int : Intrinsic<[llvm_anyint_ty], [llvm_i32_ty]>; -def int_experimental_gc_result_float : Intrinsic<[llvm_anyfloat_ty], - [llvm_i32_ty]>; -def int_experimental_gc_result_ptr : Intrinsic<[llvm_anyptr_ty], [llvm_i32_ty]>; + [llvm_token_ty, llvm_i32_ty, llvm_i32_ty], + [IntrReadMem]>; //===-------------------------- Other Intrinsics --------------------------===// // diff --git a/include/llvm/IR/IntrinsicsAArch64.td b/include/llvm/IR/IntrinsicsAArch64.td index 7d69ed52171c..578f259aae14 100644 --- a/include/llvm/IR/IntrinsicsAArch64.td +++ b/include/llvm/IR/IntrinsicsAArch64.td @@ -13,6 +13,9 @@ let TargetPrefix = "aarch64" in { +def int_aarch64_thread_pointer : GCCBuiltin<"__builtin_thread_pointer">, + Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; + def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>; def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>; def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>; diff --git a/include/llvm/IR/IntrinsicsAMDGPU.td b/include/llvm/IR/IntrinsicsAMDGPU.td index 510e5ad2d9b4..84582e8b9925 100644 --- a/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/include/llvm/IR/IntrinsicsAMDGPU.td @@ -33,6 +33,14 @@ defm int_r600_read_tgid : R600ReadPreloadRegisterIntrinsic_xyz < "__builtin_r600_read_tgid">; defm int_r600_read_tidig : R600ReadPreloadRegisterIntrinsic_xyz < "__builtin_r600_read_tidig">; + +def int_r600_rat_store_typed : + // 1st parameter: Data + // 2nd parameter: Index + // 3rd parameter: Constant RAT ID + Intrinsic<[], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], []>, + GCCBuiltin<"__builtin_r600_rat_store_typed">; + } // End TargetPrefix = "r600" let TargetPrefix = "AMDGPU" in { @@ -83,3 +91,67 @@ def int_AMDGPU_read_workdim : AMDGPUReadPreloadRegisterIntrinsic < "__builtin_amdgpu_read_workdim">; } // End TargetPrefix = "AMDGPU" + +let TargetPrefix = "amdgcn" in { + +// SI only +def int_amdgcn_buffer_wbinvl1_sc : + GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1_sc">, + Intrinsic<[], [], []>; + +// On CI+ +def int_amdgcn_buffer_wbinvl1_vol : + GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1_vol">, + Intrinsic<[], [], []>; + +def int_amdgcn_buffer_wbinvl1 : + GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1">, + Intrinsic<[], [], []>; + +def int_amdgcn_s_dcache_inv : + GCCBuiltin<"__builtin_amdgcn_s_dcache_inv">, + Intrinsic<[], [], []>; + +// CI+ +def int_amdgcn_s_dcache_inv_vol : + GCCBuiltin<"__builtin_amdgcn_s_dcache_inv_vol">, + Intrinsic<[], [], []>; + +// VI +def int_amdgcn_s_dcache_wb : + GCCBuiltin<"__builtin_amdgcn_s_dcache_wb">, + Intrinsic<[], [], []>; + +// VI +def int_amdgcn_s_dcache_wb_vol : + GCCBuiltin<"__builtin_amdgcn_s_dcache_wb_vol">, + Intrinsic<[], [], []>; + +def int_amdgcn_dispatch_ptr : + GCCBuiltin<"__builtin_amdgcn_dispatch_ptr">, + Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 2>], [], [IntrNoMem]>; + +// __builtin_amdgcn_interp_p1 <i>, <attr_chan>, <attr>, <m0> +def int_amdgcn_interp_p1 : + GCCBuiltin<"__builtin_amdgcn_interp_p1">, + Intrinsic<[llvm_float_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; // This intrinsic reads from lds, but the memory + // values are constant, so it behaves like IntrNoMem. + +// __builtin_amdgcn_interp_p2 <p1>, <j>, <attr_chan>, <attr>, <m0> +def int_amdgcn_interp_p2 : + GCCBuiltin<"__builtin_amdgcn_interp_p2">, + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; // See int_amdgcn_v_interp_p1 for why this is + // IntrNoMem. + +def int_amdgcn_mbcnt_lo : + GCCBuiltin<"__builtin_amdgcn_mbcnt_lo">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + +def int_amdgcn_mbcnt_hi : + GCCBuiltin<"__builtin_amdgcn_mbcnt_hi">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; +} diff --git a/include/llvm/IR/IntrinsicsARM.td b/include/llvm/IR/IntrinsicsARM.td index 1dff80878592..c1d911cefee2 100644 --- a/include/llvm/IR/IntrinsicsARM.td +++ b/include/llvm/IR/IntrinsicsARM.td @@ -405,36 +405,36 @@ def int_arm_neon_vrintp : Neon_1Arg_Intrinsic; // De-interleaving vector loads from N-element structures. // Source operands are the address and alignment. def int_arm_neon_vld1 : Intrinsic<[llvm_anyvector_ty], - [llvm_ptr_ty, llvm_i32_ty], + [llvm_anyptr_ty, llvm_i32_ty], [IntrReadArgMem]>; def int_arm_neon_vld2 : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], - [llvm_ptr_ty, llvm_i32_ty], + [llvm_anyptr_ty, llvm_i32_ty], [IntrReadArgMem]>; def int_arm_neon_vld3 : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], - [llvm_ptr_ty, llvm_i32_ty], + [llvm_anyptr_ty, llvm_i32_ty], [IntrReadArgMem]>; def int_arm_neon_vld4 : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], - [llvm_ptr_ty, llvm_i32_ty], + [llvm_anyptr_ty, llvm_i32_ty], [IntrReadArgMem]>; // Vector load N-element structure to one lane. // Source operands are: the address, the N input vectors (since only one // lane is assigned), the lane number, and the alignment. def int_arm_neon_vld2lane : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], - [llvm_ptr_ty, LLVMMatchType<0>, + [llvm_anyptr_ty, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], [IntrReadArgMem]>; def int_arm_neon_vld3lane : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], - [llvm_ptr_ty, LLVMMatchType<0>, + [llvm_anyptr_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], [IntrReadArgMem]>; def int_arm_neon_vld4lane : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], - [llvm_ptr_ty, LLVMMatchType<0>, + [llvm_anyptr_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], [IntrReadArgMem]>; @@ -442,38 +442,38 @@ def int_arm_neon_vld4lane : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, // Interleaving vector stores from N-element structures. // Source operands are: the address, the N vectors, and the alignment. def int_arm_neon_vst1 : Intrinsic<[], - [llvm_ptr_ty, llvm_anyvector_ty, + [llvm_anyptr_ty, llvm_anyvector_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; def int_arm_neon_vst2 : Intrinsic<[], - [llvm_ptr_ty, llvm_anyvector_ty, - LLVMMatchType<0>, llvm_i32_ty], + [llvm_anyptr_ty, llvm_anyvector_ty, + LLVMMatchType<1>, llvm_i32_ty], [IntrReadWriteArgMem]>; def int_arm_neon_vst3 : Intrinsic<[], - [llvm_ptr_ty, llvm_anyvector_ty, - LLVMMatchType<0>, LLVMMatchType<0>, + [llvm_anyptr_ty, llvm_anyvector_ty, + LLVMMatchType<1>, LLVMMatchType<1>, llvm_i32_ty], [IntrReadWriteArgMem]>; def int_arm_neon_vst4 : Intrinsic<[], - [llvm_ptr_ty, llvm_anyvector_ty, - LLVMMatchType<0>, LLVMMatchType<0>, - LLVMMatchType<0>, llvm_i32_ty], + [llvm_anyptr_ty, llvm_anyvector_ty, + LLVMMatchType<1>, LLVMMatchType<1>, + LLVMMatchType<1>, llvm_i32_ty], [IntrReadWriteArgMem]>; // Vector store N-element structure from one lane. // Source operands are: the address, the N vectors, the lane number, and // the alignment. def int_arm_neon_vst2lane : Intrinsic<[], - [llvm_ptr_ty, llvm_anyvector_ty, - LLVMMatchType<0>, llvm_i32_ty, + [llvm_anyptr_ty, llvm_anyvector_ty, + LLVMMatchType<1>, llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; def int_arm_neon_vst3lane : Intrinsic<[], - [llvm_ptr_ty, llvm_anyvector_ty, - LLVMMatchType<0>, LLVMMatchType<0>, + [llvm_anyptr_ty, llvm_anyvector_ty, + LLVMMatchType<1>, LLVMMatchType<1>, llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; def int_arm_neon_vst4lane : Intrinsic<[], - [llvm_ptr_ty, llvm_anyvector_ty, - LLVMMatchType<0>, LLVMMatchType<0>, - LLVMMatchType<0>, llvm_i32_ty, + [llvm_anyptr_ty, llvm_anyvector_ty, + LLVMMatchType<1>, LLVMMatchType<1>, + LLVMMatchType<1>, llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; // Vector bitwise select. diff --git a/include/llvm/IR/IntrinsicsHexagon.td b/include/llvm/IR/IntrinsicsHexagon.td index 78ee651d20f9..ca6fcbd44337 100644 --- a/include/llvm/IR/IntrinsicsHexagon.td +++ b/include/llvm/IR/IntrinsicsHexagon.td @@ -32,14 +32,16 @@ class Hexagon_qi_mem_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, [llvm_i1_ty], [llvm_ptr_ty], [IntrNoMem]>; + // // DEF_FUNCTION_TYPE_1(void_ftype_SI,BT_VOID,BT_INT) -> // Hexagon_void_si_Intrinsic<string GCCIntSuffix> // class Hexagon_void_si_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, - [llvm_void_ty], [llvm_i32_ty], - [IntrNoMem]>; + [], [llvm_ptr_ty], + []>; + // // DEF_FUNCTION_TYPE_1(HI_ftype_SI,BT_I16,BT_INT) -> // Hexagon_hi_si_Intrinsic<string GCCIntSuffix> @@ -458,6 +460,11 @@ class Hexagon_mem_memdisisi_Intrinsic<string GCCIntSuffix> llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; +class Hexagon_v256_v256v256_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], + [IntrReadWriteArgMem]>; + // // Hexagon_sf_df_Intrinsic<string GCCIntSuffix> // @@ -756,6 +763,12 @@ def int_hexagon_circ_stb : Hexagon_mem_memsisisi_Intrinsic<"circ_stb">; +def int_hexagon_mm256i_vaddw : +Hexagon_v256_v256v256_Intrinsic<"_mm256i_vaddw">; + + +// This one above will not be auto-generated, +// so make sure, you don't overwrite this one. // // BUILTIN_INFO(HEXAGON.C2_cmpeq,QI_ftype_SISI,2) // @@ -4946,6 +4959,11 @@ Hexagon_di_di_Intrinsic<"HEXAGON_S2_interleave">; // def int_hexagon_S2_deinterleave : Hexagon_di_di_Intrinsic<"HEXAGON_S2_deinterleave">; +// +// BUILTIN_INFO(HEXAGON.dcfetch_A,v_ftype_DI*,1) +// +def int_hexagon_prefetch : +Hexagon_void_si_Intrinsic<"HEXAGON_prefetch">; def llvm_ptr32_ty : LLVMPointerType<llvm_i32_ty>; def llvm_ptr64_ty : LLVMPointerType<llvm_i64_ty>; @@ -4964,3 +4982,4392 @@ Hexagon_Intrinsic<"HEXAGON_S2_storew_locked", [llvm_i32_ty], def int_hexagon_S4_stored_locked : Hexagon_Intrinsic<"HEXAGON_S4_stored_locked", [llvm_i32_ty], [llvm_ptr64_ty, llvm_i64_ty], [IntrReadWriteArgMem, NoCapture<0>]>; + +// V60 + +class Hexagon_v2048v2048_Intrinsic_T<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty], + [IntrNoMem]>; + +// tag : V6_hi_W +// tag : V6_lo_W +class Hexagon_v512v1024_Intrinsic_T<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v32i32_ty], + [IntrNoMem]>; + +// tag : V6_hi_W_128B +// tag : V6_lo_W_128B +class Hexagon_v1024v2048_Intrinsic_T<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v64i32_ty], + [IntrNoMem]>; + +class Hexagon_v1024v1024_Intrinsic_T<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty], + [IntrNoMem]>; + +// BUILTIN_INFO(HEXAGON.V6_hi_W,VI_ftype_VI,1) +// tag : V6_hi +def int_hexagon_V6_hi : +Hexagon_v512v1024_Intrinsic_T<"HEXAGON_V6_hi">; + +// BUILTIN_INFO(HEXAGON.V6_lo_W,VI_ftype_VI,1) +// tag : V6_lo +def int_hexagon_V6_lo : +Hexagon_v512v1024_Intrinsic_T<"HEXAGON_V6_lo">; + +// BUILTIN_INFO(HEXAGON.V6_hi_W,VI_ftype_VI,1) +// tag : V6_hi_128B +def int_hexagon_V6_hi_128B : +Hexagon_v1024v2048_Intrinsic_T<"HEXAGON_V6_hi_128B">; + +// BUILTIN_INFO(HEXAGON.V6_lo_W,VI_ftype_VI,1) +// tag : V6_lo_128B +def int_hexagon_V6_lo_128B : +Hexagon_v1024v2048_Intrinsic_T<"HEXAGON_V6_lo_128B">; + +// BUILTIN_INFO(HEXAGON.V6_vassignp,VI_ftype_VI,1) +// tag : V6_vassignp +def int_hexagon_V6_vassignp : +Hexagon_v1024v1024_Intrinsic_T<"HEXAGON_V6_vassignp">; + +// BUILTIN_INFO(HEXAGON.V6_vassignp,VI_ftype_VI,1) +// tag : V6_vassignp_128B +def int_hexagon_V6_vassignp_128B : +Hexagon_v2048v2048_Intrinsic_T<"HEXAGON_V6_vassignp_128B">; + + + +// +// Hexagon_iii_Intrinsic<string GCCIntSuffix> +// tag : S6_rol_i_r +class Hexagon_iii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i32_ty], [llvm_i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_LLiLLii_Intrinsic<string GCCIntSuffix> +// tag : S6_rol_i_p +class Hexagon_LLiLLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i64_ty], [llvm_i64_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_iiii_Intrinsic<string GCCIntSuffix> +// tag : S6_rol_i_r_acc +class Hexagon_iiii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i32_ty], [llvm_i32_ty,llvm_i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_LLiLLiLLii_Intrinsic<string GCCIntSuffix> +// tag : S6_rol_i_p_acc +class Hexagon_LLiLLiLLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i64_ty], [llvm_i64_ty,llvm_i64_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_valignb +class Hexagon_v512v512v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_valignb_128B +class Hexagon_v1024v1024v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vror +class Hexagon_v512v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vror_128B +class Hexagon_v1024v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vunpackub +class Hexagon_v1024v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vunpackub_128B +class Hexagon_v2048v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vunpackob +class Hexagon_v1024v1024v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vunpackob_128B +class Hexagon_v2048v2048v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vpackeb +class Hexagon_v512v512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vpackeb_128B +class Hexagon_v1024v1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048i_Intrinsic<string GCCIntSuffix> +// tag : V6_vdmpybus_dv_128B +class Hexagon_v2048v2048i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048v2048i_Intrinsic<string GCCIntSuffix> +// tag : V6_vdmpybus_dv_acc_128B +class Hexagon_v2048v2048v2048i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v64i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512v512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vdmpyhvsat_acc +class Hexagon_v512v512v512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vdmpyhvsat_acc_128B +class Hexagon_v1024v1024v1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vdmpyhisat +class Hexagon_v512v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v2048i_Intrinsic<string GCCIntSuffix> +// tag : V6_vdmpyhisat_128B +class Hexagon_v1024v2048i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v64i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vdmpyhisat_acc +class Hexagon_v512v512v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v2048i_Intrinsic<string GCCIntSuffix> +// tag : V6_vdmpyhisat_acc_128B +class Hexagon_v1024v1024v2048i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v64i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024ii_Intrinsic<string GCCIntSuffix> +// tag : V6_vrmpyubi +class Hexagon_v1024v1024ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048ii_Intrinsic<string GCCIntSuffix> +// tag : V6_vrmpyubi_128B +class Hexagon_v2048v2048ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v1024ii_Intrinsic<string GCCIntSuffix> +// tag : V6_vrmpyubi_acc +class Hexagon_v1024v1024v1024ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048v2048ii_Intrinsic<string GCCIntSuffix> +// tag : V6_vrmpyubi_acc_128B +class Hexagon_v2048v2048v2048ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v64i32_ty,llvm_i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048v2048_Intrinsic<string GCCIntSuffix> +// tag : V6_vaddb_dv_128B +class Hexagon_v2048v2048v2048_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v64i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vaddubh +class Hexagon_v1024v512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vaddubh_128B +class Hexagon_v2048v1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vd0 +class Hexagon_v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [], + [IntrNoMem]>; + +// +// Hexagon_v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vd0_128B +class Hexagon_v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [], + [IntrNoMem]>; + +// +// Hexagon_v512v64iv512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vaddbq +class Hexagon_v512v64iv512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v512i1_ty,llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v128iv1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vaddbq_128B +class Hexagon_v1024v128iv1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v1024i1_ty,llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vabsh +class Hexagon_v512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vabsh_128B +class Hexagon_v1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpybv_acc +class Hexagon_v1024v1024v512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048v1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpybv_acc_128B +class Hexagon_v2048v2048v1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpyub +class Hexagon_v1024v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpyub_128B +class Hexagon_v2048v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpyub_acc +class Hexagon_v1024v1024v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vmpyub_acc_128B +class Hexagon_v2048v2048v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v64ii_Intrinsic<string GCCIntSuffix> +// tag : V6_vandqrt +class Hexagon_v512v64ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v512i1_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v128ii_Intrinsic<string GCCIntSuffix> +// tag : V6_vandqrt_128B +class Hexagon_v1024v128ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v1024i1_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512v64ii_Intrinsic<string GCCIntSuffix> +// tag : V6_vandqrt_acc +class Hexagon_v512v512v64ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v512i1_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v128ii_Intrinsic<string GCCIntSuffix> +// tag : V6_vandqrt_acc_128B +class Hexagon_v1024v1024v128ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v1024i1_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v64iv512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vandvrt +class Hexagon_v64iv512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v512i1_ty], [llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v128iv1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vandvrt_128B +class Hexagon_v128iv1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v1024i1_ty], [llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v64iv64iv512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vandvrt_acc +class Hexagon_v64iv64iv512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v512i1_ty], [llvm_v512i1_ty,llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v128iv128iv1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vandvrt_acc_128B +class Hexagon_v128iv128iv1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v1024i1_ty], [llvm_v1024i1_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v64iv512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vgtw +class Hexagon_v64iv512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v512i1_ty], [llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v128iv1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vgtw_128B +class Hexagon_v128iv1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v1024i1_ty], [llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v64iv64iv512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vgtw_and +class Hexagon_v64iv64iv512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v512i1_ty], [llvm_v512i1_ty,llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v128iv128iv1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vgtw_and_128B +class Hexagon_v128iv128iv1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v1024i1_ty], [llvm_v1024i1_ty,llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v64iv64iv64i_Intrinsic<string GCCIntSuffix> +// tag : V6_pred_or +class Hexagon_v64iv64iv64i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v512i1_ty], [llvm_v512i1_ty,llvm_v512i1_ty], + [IntrNoMem]>; + +// +// Hexagon_v128iv128iv128i_Intrinsic<string GCCIntSuffix> +// tag : V6_pred_or_128B +class Hexagon_v128iv128iv128i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v1024i1_ty], [llvm_v1024i1_ty,llvm_v1024i1_ty], + [IntrNoMem]>; + +// +// Hexagon_v64iv64i_Intrinsic<string GCCIntSuffix> +// tag : V6_pred_not +class Hexagon_v64iv64i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v512i1_ty], [llvm_v512i1_ty], + [IntrNoMem]>; + +// +// Hexagon_v128iv128i_Intrinsic<string GCCIntSuffix> +// tag : V6_pred_not_128B +class Hexagon_v128iv128i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v1024i1_ty], [llvm_v1024i1_ty], + [IntrNoMem]>; + +// +// Hexagon_v64ii_Intrinsic<string GCCIntSuffix> +// tag : V6_pred_scalar2 +class Hexagon_v64ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v512i1_ty], [llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v128ii_Intrinsic<string GCCIntSuffix> +// tag : V6_pred_scalar2_128B +class Hexagon_v128ii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v1024i1_ty], [llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v64iv512v512_Intrinsic<string GCCIntSuffix> +// tag : V6_vswap +class Hexagon_v1024v64iv512v512_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v512i1_ty,llvm_v16i32_ty,llvm_v16i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v128iv1024v1024_Intrinsic<string GCCIntSuffix> +// tag : V6_vswap_128B +class Hexagon_v2048v128iv1024v1024_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v1024i1_ty,llvm_v32i32_ty,llvm_v32i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v512v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vshuffvdd +class Hexagon_v1024v512v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v1024v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vshuffvdd_128B +class Hexagon_v2048v1024v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + + +// +// Hexagon_iv512i_Intrinsic<string GCCIntSuffix> +// tag : V6_extractw +class Hexagon_iv512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i32_ty], [llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_iv1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_extractw_128B +class Hexagon_iv1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i32_ty], [llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_lvsplatw +class Hexagon_v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_lvsplatw_128B +class Hexagon_v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512LLii_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutb +class Hexagon_v512v512LLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_i64_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024LLii_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutb_128B +class Hexagon_v1024v1024LLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_i64_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512v512LLii_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutb_acc +class Hexagon_v512v512v512LLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_i64_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v1024LLii_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutb_acc_128B +class Hexagon_v1024v1024v1024LLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_i64_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048LLii_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutb_dv_128B +class Hexagon_v2048v2048LLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_i64_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048v2048LLii_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutb_dv_acc_128B +class Hexagon_v2048v2048v2048LLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v64i32_ty,llvm_i64_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v512v512v512v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutvvb_oracc +class Hexagon_v512v512v512v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v1024v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutvvb_oracc_128B +class Hexagon_v1024v1024v1024v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v1024v1024v512v512i_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutvwh_oracc +class Hexagon_v1024v1024v512v512i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v16i32_ty,llvm_v16i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_v2048v2048v1024v1024i_Intrinsic<string GCCIntSuffix> +// tag : V6_vlutvwh_oracc_128B +class Hexagon_v2048v2048v1024v1024i_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v32i32_ty,llvm_v32i32_ty,llvm_i32_ty], + [IntrNoMem]>; + +// +// Hexagon_LLiLLiLLi_Intrinsic<string GCCIntSuffix> +// tag : M6_vabsdiffb +class Hexagon_LLiLLiLLi_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i64_ty], [llvm_i64_ty,llvm_i64_ty], + [IntrNoMem]>; + +// +// Hexagon_LLii_Intrinsic<string GCCIntSuffix> +// tag : S6_vsplatrbp +class Hexagon_LLii_Intrinsic<string GCCIntSuffix> + : Hexagon_Intrinsic<GCCIntSuffix, + [llvm_i64_ty], [llvm_i32_ty], + [IntrNoMem]>; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_r,SI_ftype_SISI,2) +// tag : S6_rol_i_r +def int_hexagon_S6_rol_i_r : +Hexagon_iii_Intrinsic<"HEXAGON_S6_rol_i_r">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_p,DI_ftype_DISI,2) +// tag : S6_rol_i_p +def int_hexagon_S6_rol_i_p : +Hexagon_LLiLLii_Intrinsic<"HEXAGON_S6_rol_i_p">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_r_acc,SI_ftype_SISISI,3) +// tag : S6_rol_i_r_acc +def int_hexagon_S6_rol_i_r_acc : +Hexagon_iiii_Intrinsic<"HEXAGON_S6_rol_i_r_acc">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_p_acc,DI_ftype_DIDISI,3) +// tag : S6_rol_i_p_acc +def int_hexagon_S6_rol_i_p_acc : +Hexagon_LLiLLiLLii_Intrinsic<"HEXAGON_S6_rol_i_p_acc">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_r_nac,SI_ftype_SISISI,3) +// tag : S6_rol_i_r_nac +def int_hexagon_S6_rol_i_r_nac : +Hexagon_iiii_Intrinsic<"HEXAGON_S6_rol_i_r_nac">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_p_nac,DI_ftype_DIDISI,3) +// tag : S6_rol_i_p_nac +def int_hexagon_S6_rol_i_p_nac : +Hexagon_LLiLLiLLii_Intrinsic<"HEXAGON_S6_rol_i_p_nac">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_r_xacc,SI_ftype_SISISI,3) +// tag : S6_rol_i_r_xacc +def int_hexagon_S6_rol_i_r_xacc : +Hexagon_iiii_Intrinsic<"HEXAGON_S6_rol_i_r_xacc">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_p_xacc,DI_ftype_DIDISI,3) +// tag : S6_rol_i_p_xacc +def int_hexagon_S6_rol_i_p_xacc : +Hexagon_LLiLLiLLii_Intrinsic<"HEXAGON_S6_rol_i_p_xacc">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_r_and,SI_ftype_SISISI,3) +// tag : S6_rol_i_r_and +def int_hexagon_S6_rol_i_r_and : +Hexagon_iiii_Intrinsic<"HEXAGON_S6_rol_i_r_and">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_r_or,SI_ftype_SISISI,3) +// tag : S6_rol_i_r_or +def int_hexagon_S6_rol_i_r_or : +Hexagon_iiii_Intrinsic<"HEXAGON_S6_rol_i_r_or">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_p_and,DI_ftype_DIDISI,3) +// tag : S6_rol_i_p_and +def int_hexagon_S6_rol_i_p_and : +Hexagon_LLiLLiLLii_Intrinsic<"HEXAGON_S6_rol_i_p_and">; + +// +// BUILTIN_INFO(HEXAGON.S6_rol_i_p_or,DI_ftype_DIDISI,3) +// tag : S6_rol_i_p_or +def int_hexagon_S6_rol_i_p_or : +Hexagon_LLiLLiLLii_Intrinsic<"HEXAGON_S6_rol_i_p_or">; + +// +// BUILTIN_INFO(HEXAGON.S2_cabacencbin,DI_ftype_DIDIQI,3) +// tag : S2_cabacencbin +def int_hexagon_S2_cabacencbin : +Hexagon_LLiLLiLLii_Intrinsic<"HEXAGON_S2_cabacencbin">; + +// +// BUILTIN_INFO(HEXAGON.V6_valignb,VI_ftype_VIVISI,3) +// tag : V6_valignb +def int_hexagon_V6_valignb : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_valignb">; + +// +// BUILTIN_INFO(HEXAGON.V6_valignb_128B,VI_ftype_VIVISI,3) +// tag : V6_valignb_128B +def int_hexagon_V6_valignb_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_valignb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlalignb,VI_ftype_VIVISI,3) +// tag : V6_vlalignb +def int_hexagon_V6_vlalignb : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vlalignb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlalignb_128B,VI_ftype_VIVISI,3) +// tag : V6_vlalignb_128B +def int_hexagon_V6_vlalignb_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vlalignb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_valignbi,VI_ftype_VIVISI,3) +// tag : V6_valignbi +def int_hexagon_V6_valignbi : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_valignbi">; + +// +// BUILTIN_INFO(HEXAGON.V6_valignbi_128B,VI_ftype_VIVISI,3) +// tag : V6_valignbi_128B +def int_hexagon_V6_valignbi_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_valignbi_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlalignbi,VI_ftype_VIVISI,3) +// tag : V6_vlalignbi +def int_hexagon_V6_vlalignbi : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vlalignbi">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlalignbi_128B,VI_ftype_VIVISI,3) +// tag : V6_vlalignbi_128B +def int_hexagon_V6_vlalignbi_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vlalignbi_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vror,VI_ftype_VISI,2) +// tag : V6_vror +def int_hexagon_V6_vror : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vror">; + +// +// BUILTIN_INFO(HEXAGON.V6_vror_128B,VI_ftype_VISI,2) +// tag : V6_vror_128B +def int_hexagon_V6_vror_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vror_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackub,VD_ftype_VI,1) +// tag : V6_vunpackub +def int_hexagon_V6_vunpackub : +Hexagon_v1024v512_Intrinsic<"HEXAGON_V6_vunpackub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackub_128B,VD_ftype_VI,1) +// tag : V6_vunpackub_128B +def int_hexagon_V6_vunpackub_128B : +Hexagon_v2048v1024_Intrinsic<"HEXAGON_V6_vunpackub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackb,VD_ftype_VI,1) +// tag : V6_vunpackb +def int_hexagon_V6_vunpackb : +Hexagon_v1024v512_Intrinsic<"HEXAGON_V6_vunpackb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackb_128B,VD_ftype_VI,1) +// tag : V6_vunpackb_128B +def int_hexagon_V6_vunpackb_128B : +Hexagon_v2048v1024_Intrinsic<"HEXAGON_V6_vunpackb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackuh,VD_ftype_VI,1) +// tag : V6_vunpackuh +def int_hexagon_V6_vunpackuh : +Hexagon_v1024v512_Intrinsic<"HEXAGON_V6_vunpackuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackuh_128B,VD_ftype_VI,1) +// tag : V6_vunpackuh_128B +def int_hexagon_V6_vunpackuh_128B : +Hexagon_v2048v1024_Intrinsic<"HEXAGON_V6_vunpackuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackh,VD_ftype_VI,1) +// tag : V6_vunpackh +def int_hexagon_V6_vunpackh : +Hexagon_v1024v512_Intrinsic<"HEXAGON_V6_vunpackh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackh_128B,VD_ftype_VI,1) +// tag : V6_vunpackh_128B +def int_hexagon_V6_vunpackh_128B : +Hexagon_v2048v1024_Intrinsic<"HEXAGON_V6_vunpackh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackob,VD_ftype_VDVI,2) +// tag : V6_vunpackob +def int_hexagon_V6_vunpackob : +Hexagon_v1024v1024v512_Intrinsic<"HEXAGON_V6_vunpackob">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackob_128B,VD_ftype_VDVI,2) +// tag : V6_vunpackob_128B +def int_hexagon_V6_vunpackob_128B : +Hexagon_v2048v2048v1024_Intrinsic<"HEXAGON_V6_vunpackob_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackoh,VD_ftype_VDVI,2) +// tag : V6_vunpackoh +def int_hexagon_V6_vunpackoh : +Hexagon_v1024v1024v512_Intrinsic<"HEXAGON_V6_vunpackoh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vunpackoh_128B,VD_ftype_VDVI,2) +// tag : V6_vunpackoh_128B +def int_hexagon_V6_vunpackoh_128B : +Hexagon_v2048v2048v1024_Intrinsic<"HEXAGON_V6_vunpackoh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackeb,VI_ftype_VIVI,2) +// tag : V6_vpackeb +def int_hexagon_V6_vpackeb : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vpackeb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackeb_128B,VI_ftype_VIVI,2) +// tag : V6_vpackeb_128B +def int_hexagon_V6_vpackeb_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vpackeb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackeh,VI_ftype_VIVI,2) +// tag : V6_vpackeh +def int_hexagon_V6_vpackeh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vpackeh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackeh_128B,VI_ftype_VIVI,2) +// tag : V6_vpackeh_128B +def int_hexagon_V6_vpackeh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vpackeh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackob,VI_ftype_VIVI,2) +// tag : V6_vpackob +def int_hexagon_V6_vpackob : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vpackob">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackob_128B,VI_ftype_VIVI,2) +// tag : V6_vpackob_128B +def int_hexagon_V6_vpackob_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vpackob_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackoh,VI_ftype_VIVI,2) +// tag : V6_vpackoh +def int_hexagon_V6_vpackoh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vpackoh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackoh_128B,VI_ftype_VIVI,2) +// tag : V6_vpackoh_128B +def int_hexagon_V6_vpackoh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vpackoh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackhub_sat,VI_ftype_VIVI,2) +// tag : V6_vpackhub_sat +def int_hexagon_V6_vpackhub_sat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vpackhub_sat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackhub_sat_128B,VI_ftype_VIVI,2) +// tag : V6_vpackhub_sat_128B +def int_hexagon_V6_vpackhub_sat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vpackhub_sat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackhb_sat,VI_ftype_VIVI,2) +// tag : V6_vpackhb_sat +def int_hexagon_V6_vpackhb_sat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vpackhb_sat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackhb_sat_128B,VI_ftype_VIVI,2) +// tag : V6_vpackhb_sat_128B +def int_hexagon_V6_vpackhb_sat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vpackhb_sat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackwuh_sat,VI_ftype_VIVI,2) +// tag : V6_vpackwuh_sat +def int_hexagon_V6_vpackwuh_sat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vpackwuh_sat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackwuh_sat_128B,VI_ftype_VIVI,2) +// tag : V6_vpackwuh_sat_128B +def int_hexagon_V6_vpackwuh_sat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vpackwuh_sat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackwh_sat,VI_ftype_VIVI,2) +// tag : V6_vpackwh_sat +def int_hexagon_V6_vpackwh_sat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vpackwh_sat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpackwh_sat_128B,VI_ftype_VIVI,2) +// tag : V6_vpackwh_sat_128B +def int_hexagon_V6_vpackwh_sat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vpackwh_sat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vzb,VD_ftype_VI,1) +// tag : V6_vzb +def int_hexagon_V6_vzb : +Hexagon_v1024v512_Intrinsic<"HEXAGON_V6_vzb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vzb_128B,VD_ftype_VI,1) +// tag : V6_vzb_128B +def int_hexagon_V6_vzb_128B : +Hexagon_v2048v1024_Intrinsic<"HEXAGON_V6_vzb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsb,VD_ftype_VI,1) +// tag : V6_vsb +def int_hexagon_V6_vsb : +Hexagon_v1024v512_Intrinsic<"HEXAGON_V6_vsb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsb_128B,VD_ftype_VI,1) +// tag : V6_vsb_128B +def int_hexagon_V6_vsb_128B : +Hexagon_v2048v1024_Intrinsic<"HEXAGON_V6_vsb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vzh,VD_ftype_VI,1) +// tag : V6_vzh +def int_hexagon_V6_vzh : +Hexagon_v1024v512_Intrinsic<"HEXAGON_V6_vzh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vzh_128B,VD_ftype_VI,1) +// tag : V6_vzh_128B +def int_hexagon_V6_vzh_128B : +Hexagon_v2048v1024_Intrinsic<"HEXAGON_V6_vzh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsh,VD_ftype_VI,1) +// tag : V6_vsh +def int_hexagon_V6_vsh : +Hexagon_v1024v512_Intrinsic<"HEXAGON_V6_vsh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsh_128B,VD_ftype_VI,1) +// tag : V6_vsh_128B +def int_hexagon_V6_vsh_128B : +Hexagon_v2048v1024_Intrinsic<"HEXAGON_V6_vsh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpybus,VI_ftype_VISI,2) +// tag : V6_vdmpybus +def int_hexagon_V6_vdmpybus : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vdmpybus">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpybus_128B,VI_ftype_VISI,2) +// tag : V6_vdmpybus_128B +def int_hexagon_V6_vdmpybus_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpybus_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpybus_acc,VI_ftype_VIVISI,3) +// tag : V6_vdmpybus_acc +def int_hexagon_V6_vdmpybus_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vdmpybus_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpybus_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vdmpybus_acc_128B +def int_hexagon_V6_vdmpybus_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpybus_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpybus_dv,VD_ftype_VDSI,2) +// tag : V6_vdmpybus_dv +def int_hexagon_V6_vdmpybus_dv : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpybus_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpybus_dv_128B,VD_ftype_VDSI,2) +// tag : V6_vdmpybus_dv_128B +def int_hexagon_V6_vdmpybus_dv_128B : +Hexagon_v2048v2048i_Intrinsic<"HEXAGON_V6_vdmpybus_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpybus_dv_acc,VD_ftype_VDVDSI,3) +// tag : V6_vdmpybus_dv_acc +def int_hexagon_V6_vdmpybus_dv_acc : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpybus_dv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpybus_dv_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vdmpybus_dv_acc_128B +def int_hexagon_V6_vdmpybus_dv_acc_128B : +Hexagon_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vdmpybus_dv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhb,VI_ftype_VISI,2) +// tag : V6_vdmpyhb +def int_hexagon_V6_vdmpyhb : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vdmpyhb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhb_128B,VI_ftype_VISI,2) +// tag : V6_vdmpyhb_128B +def int_hexagon_V6_vdmpyhb_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpyhb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhb_acc,VI_ftype_VIVISI,3) +// tag : V6_vdmpyhb_acc +def int_hexagon_V6_vdmpyhb_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vdmpyhb_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhb_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vdmpyhb_acc_128B +def int_hexagon_V6_vdmpyhb_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpyhb_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhb_dv,VD_ftype_VDSI,2) +// tag : V6_vdmpyhb_dv +def int_hexagon_V6_vdmpyhb_dv : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpyhb_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhb_dv_128B,VD_ftype_VDSI,2) +// tag : V6_vdmpyhb_dv_128B +def int_hexagon_V6_vdmpyhb_dv_128B : +Hexagon_v2048v2048i_Intrinsic<"HEXAGON_V6_vdmpyhb_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhb_dv_acc,VD_ftype_VDVDSI,3) +// tag : V6_vdmpyhb_dv_acc +def int_hexagon_V6_vdmpyhb_dv_acc : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpyhb_dv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhb_dv_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vdmpyhb_dv_acc_128B +def int_hexagon_V6_vdmpyhb_dv_acc_128B : +Hexagon_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vdmpyhb_dv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhvsat,VI_ftype_VIVI,2) +// tag : V6_vdmpyhvsat +def int_hexagon_V6_vdmpyhvsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vdmpyhvsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhvsat_128B,VI_ftype_VIVI,2) +// tag : V6_vdmpyhvsat_128B +def int_hexagon_V6_vdmpyhvsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vdmpyhvsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhvsat_acc,VI_ftype_VIVIVI,3) +// tag : V6_vdmpyhvsat_acc +def int_hexagon_V6_vdmpyhvsat_acc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vdmpyhvsat_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhvsat_acc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vdmpyhvsat_acc_128B +def int_hexagon_V6_vdmpyhvsat_acc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vdmpyhvsat_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsat,VI_ftype_VISI,2) +// tag : V6_vdmpyhsat +def int_hexagon_V6_vdmpyhsat : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vdmpyhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsat_128B,VI_ftype_VISI,2) +// tag : V6_vdmpyhsat_128B +def int_hexagon_V6_vdmpyhsat_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpyhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsat_acc,VI_ftype_VIVISI,3) +// tag : V6_vdmpyhsat_acc +def int_hexagon_V6_vdmpyhsat_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vdmpyhsat_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsat_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vdmpyhsat_acc_128B +def int_hexagon_V6_vdmpyhsat_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpyhsat_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhisat,VI_ftype_VDSI,2) +// tag : V6_vdmpyhisat +def int_hexagon_V6_vdmpyhisat : +Hexagon_v512v1024i_Intrinsic<"HEXAGON_V6_vdmpyhisat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhisat_128B,VI_ftype_VDSI,2) +// tag : V6_vdmpyhisat_128B +def int_hexagon_V6_vdmpyhisat_128B : +Hexagon_v1024v2048i_Intrinsic<"HEXAGON_V6_vdmpyhisat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhisat_acc,VI_ftype_VIVDSI,3) +// tag : V6_vdmpyhisat_acc +def int_hexagon_V6_vdmpyhisat_acc : +Hexagon_v512v512v1024i_Intrinsic<"HEXAGON_V6_vdmpyhisat_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhisat_acc_128B,VI_ftype_VIVDSI,3) +// tag : V6_vdmpyhisat_acc_128B +def int_hexagon_V6_vdmpyhisat_acc_128B : +Hexagon_v1024v1024v2048i_Intrinsic<"HEXAGON_V6_vdmpyhisat_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsusat,VI_ftype_VISI,2) +// tag : V6_vdmpyhsusat +def int_hexagon_V6_vdmpyhsusat : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vdmpyhsusat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsusat_128B,VI_ftype_VISI,2) +// tag : V6_vdmpyhsusat_128B +def int_hexagon_V6_vdmpyhsusat_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpyhsusat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsusat_acc,VI_ftype_VIVISI,3) +// tag : V6_vdmpyhsusat_acc +def int_hexagon_V6_vdmpyhsusat_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vdmpyhsusat_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsusat_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vdmpyhsusat_acc_128B +def int_hexagon_V6_vdmpyhsusat_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vdmpyhsusat_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsuisat,VI_ftype_VDSI,2) +// tag : V6_vdmpyhsuisat +def int_hexagon_V6_vdmpyhsuisat : +Hexagon_v512v1024i_Intrinsic<"HEXAGON_V6_vdmpyhsuisat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsuisat_128B,VI_ftype_VDSI,2) +// tag : V6_vdmpyhsuisat_128B +def int_hexagon_V6_vdmpyhsuisat_128B : +Hexagon_v1024v2048i_Intrinsic<"HEXAGON_V6_vdmpyhsuisat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsuisat_acc,VI_ftype_VIVDSI,3) +// tag : V6_vdmpyhsuisat_acc +def int_hexagon_V6_vdmpyhsuisat_acc : +Hexagon_v512v512v1024i_Intrinsic<"HEXAGON_V6_vdmpyhsuisat_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdmpyhsuisat_acc_128B,VI_ftype_VIVDSI,3) +// tag : V6_vdmpyhsuisat_acc_128B +def int_hexagon_V6_vdmpyhsuisat_acc_128B : +Hexagon_v1024v1024v2048i_Intrinsic<"HEXAGON_V6_vdmpyhsuisat_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpyb,VD_ftype_VDSI,2) +// tag : V6_vtmpyb +def int_hexagon_V6_vtmpyb : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vtmpyb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpyb_128B,VD_ftype_VDSI,2) +// tag : V6_vtmpyb_128B +def int_hexagon_V6_vtmpyb_128B : +Hexagon_v2048v2048i_Intrinsic<"HEXAGON_V6_vtmpyb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpyb_acc,VD_ftype_VDVDSI,3) +// tag : V6_vtmpyb_acc +def int_hexagon_V6_vtmpyb_acc : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vtmpyb_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpyb_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vtmpyb_acc_128B +def int_hexagon_V6_vtmpyb_acc_128B : +Hexagon_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vtmpyb_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpybus,VD_ftype_VDSI,2) +// tag : V6_vtmpybus +def int_hexagon_V6_vtmpybus : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vtmpybus">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpybus_128B,VD_ftype_VDSI,2) +// tag : V6_vtmpybus_128B +def int_hexagon_V6_vtmpybus_128B : +Hexagon_v2048v2048i_Intrinsic<"HEXAGON_V6_vtmpybus_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpybus_acc,VD_ftype_VDVDSI,3) +// tag : V6_vtmpybus_acc +def int_hexagon_V6_vtmpybus_acc : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vtmpybus_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpybus_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vtmpybus_acc_128B +def int_hexagon_V6_vtmpybus_acc_128B : +Hexagon_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vtmpybus_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpyhb,VD_ftype_VDSI,2) +// tag : V6_vtmpyhb +def int_hexagon_V6_vtmpyhb : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vtmpyhb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpyhb_128B,VD_ftype_VDSI,2) +// tag : V6_vtmpyhb_128B +def int_hexagon_V6_vtmpyhb_128B : +Hexagon_v2048v2048i_Intrinsic<"HEXAGON_V6_vtmpyhb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpyhb_acc,VD_ftype_VDVDSI,3) +// tag : V6_vtmpyhb_acc +def int_hexagon_V6_vtmpyhb_acc : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vtmpyhb_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vtmpyhb_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vtmpyhb_acc_128B +def int_hexagon_V6_vtmpyhb_acc_128B : +Hexagon_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vtmpyhb_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyub,VI_ftype_VISI,2) +// tag : V6_vrmpyub +def int_hexagon_V6_vrmpyub : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vrmpyub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyub_128B,VI_ftype_VISI,2) +// tag : V6_vrmpyub_128B +def int_hexagon_V6_vrmpyub_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vrmpyub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyub_acc,VI_ftype_VIVISI,3) +// tag : V6_vrmpyub_acc +def int_hexagon_V6_vrmpyub_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vrmpyub_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyub_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vrmpyub_acc_128B +def int_hexagon_V6_vrmpyub_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vrmpyub_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyubv,VI_ftype_VIVI,2) +// tag : V6_vrmpyubv +def int_hexagon_V6_vrmpyubv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vrmpyubv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyubv_128B,VI_ftype_VIVI,2) +// tag : V6_vrmpyubv_128B +def int_hexagon_V6_vrmpyubv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vrmpyubv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyubv_acc,VI_ftype_VIVIVI,3) +// tag : V6_vrmpyubv_acc +def int_hexagon_V6_vrmpyubv_acc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vrmpyubv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyubv_acc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vrmpyubv_acc_128B +def int_hexagon_V6_vrmpyubv_acc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vrmpyubv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybv,VI_ftype_VIVI,2) +// tag : V6_vrmpybv +def int_hexagon_V6_vrmpybv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vrmpybv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybv_128B,VI_ftype_VIVI,2) +// tag : V6_vrmpybv_128B +def int_hexagon_V6_vrmpybv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vrmpybv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybv_acc,VI_ftype_VIVIVI,3) +// tag : V6_vrmpybv_acc +def int_hexagon_V6_vrmpybv_acc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vrmpybv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybv_acc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vrmpybv_acc_128B +def int_hexagon_V6_vrmpybv_acc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vrmpybv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyubi,VD_ftype_VDSISI,3) +// tag : V6_vrmpyubi +def int_hexagon_V6_vrmpyubi : +Hexagon_v1024v1024ii_Intrinsic<"HEXAGON_V6_vrmpyubi">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyubi_128B,VD_ftype_VDSISI,3) +// tag : V6_vrmpyubi_128B +def int_hexagon_V6_vrmpyubi_128B : +Hexagon_v2048v2048ii_Intrinsic<"HEXAGON_V6_vrmpyubi_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyubi_acc,VD_ftype_VDVDSISI,4) +// tag : V6_vrmpyubi_acc +def int_hexagon_V6_vrmpyubi_acc : +Hexagon_v1024v1024v1024ii_Intrinsic<"HEXAGON_V6_vrmpyubi_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpyubi_acc_128B,VD_ftype_VDVDSISI,4) +// tag : V6_vrmpyubi_acc_128B +def int_hexagon_V6_vrmpyubi_acc_128B : +Hexagon_v2048v2048v2048ii_Intrinsic<"HEXAGON_V6_vrmpyubi_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybus,VI_ftype_VISI,2) +// tag : V6_vrmpybus +def int_hexagon_V6_vrmpybus : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vrmpybus">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybus_128B,VI_ftype_VISI,2) +// tag : V6_vrmpybus_128B +def int_hexagon_V6_vrmpybus_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vrmpybus_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybus_acc,VI_ftype_VIVISI,3) +// tag : V6_vrmpybus_acc +def int_hexagon_V6_vrmpybus_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vrmpybus_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybus_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vrmpybus_acc_128B +def int_hexagon_V6_vrmpybus_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vrmpybus_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybusi,VD_ftype_VDSISI,3) +// tag : V6_vrmpybusi +def int_hexagon_V6_vrmpybusi : +Hexagon_v1024v1024ii_Intrinsic<"HEXAGON_V6_vrmpybusi">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybusi_128B,VD_ftype_VDSISI,3) +// tag : V6_vrmpybusi_128B +def int_hexagon_V6_vrmpybusi_128B : +Hexagon_v2048v2048ii_Intrinsic<"HEXAGON_V6_vrmpybusi_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybusi_acc,VD_ftype_VDVDSISI,4) +// tag : V6_vrmpybusi_acc +def int_hexagon_V6_vrmpybusi_acc : +Hexagon_v1024v1024v1024ii_Intrinsic<"HEXAGON_V6_vrmpybusi_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybusi_acc_128B,VD_ftype_VDVDSISI,4) +// tag : V6_vrmpybusi_acc_128B +def int_hexagon_V6_vrmpybusi_acc_128B : +Hexagon_v2048v2048v2048ii_Intrinsic<"HEXAGON_V6_vrmpybusi_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybusv,VI_ftype_VIVI,2) +// tag : V6_vrmpybusv +def int_hexagon_V6_vrmpybusv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vrmpybusv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybusv_128B,VI_ftype_VIVI,2) +// tag : V6_vrmpybusv_128B +def int_hexagon_V6_vrmpybusv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vrmpybusv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybusv_acc,VI_ftype_VIVIVI,3) +// tag : V6_vrmpybusv_acc +def int_hexagon_V6_vrmpybusv_acc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vrmpybusv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrmpybusv_acc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vrmpybusv_acc_128B +def int_hexagon_V6_vrmpybusv_acc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vrmpybusv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdsaduh,VD_ftype_VDSI,2) +// tag : V6_vdsaduh +def int_hexagon_V6_vdsaduh : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vdsaduh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdsaduh_128B,VD_ftype_VDSI,2) +// tag : V6_vdsaduh_128B +def int_hexagon_V6_vdsaduh_128B : +Hexagon_v2048v2048i_Intrinsic<"HEXAGON_V6_vdsaduh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdsaduh_acc,VD_ftype_VDVDSI,3) +// tag : V6_vdsaduh_acc +def int_hexagon_V6_vdsaduh_acc : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vdsaduh_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdsaduh_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vdsaduh_acc_128B +def int_hexagon_V6_vdsaduh_acc_128B : +Hexagon_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vdsaduh_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrsadubi,VD_ftype_VDSISI,3) +// tag : V6_vrsadubi +def int_hexagon_V6_vrsadubi : +Hexagon_v1024v1024ii_Intrinsic<"HEXAGON_V6_vrsadubi">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrsadubi_128B,VD_ftype_VDSISI,3) +// tag : V6_vrsadubi_128B +def int_hexagon_V6_vrsadubi_128B : +Hexagon_v2048v2048ii_Intrinsic<"HEXAGON_V6_vrsadubi_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrsadubi_acc,VD_ftype_VDVDSISI,4) +// tag : V6_vrsadubi_acc +def int_hexagon_V6_vrsadubi_acc : +Hexagon_v1024v1024v1024ii_Intrinsic<"HEXAGON_V6_vrsadubi_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrsadubi_acc_128B,VD_ftype_VDVDSISI,4) +// tag : V6_vrsadubi_acc_128B +def int_hexagon_V6_vrsadubi_acc_128B : +Hexagon_v2048v2048v2048ii_Intrinsic<"HEXAGON_V6_vrsadubi_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrw,VI_ftype_VISI,2) +// tag : V6_vasrw +def int_hexagon_V6_vasrw : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vasrw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrw_128B,VI_ftype_VISI,2) +// tag : V6_vasrw_128B +def int_hexagon_V6_vasrw_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vasrw_128B">; + + +// +// BUILTIN_INFO(HEXAGON.V6_vaslw,VI_ftype_VISI,2) +// tag : V6_vaslw +def int_hexagon_V6_vaslw : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vaslw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslw_128B,VI_ftype_VISI,2) +// tag : V6_vaslw_128B +def int_hexagon_V6_vaslw_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vaslw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlsrw,VI_ftype_VISI,2) +// tag : V6_vlsrw +def int_hexagon_V6_vlsrw : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vlsrw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlsrw_128B,VI_ftype_VISI,2) +// tag : V6_vlsrw_128B +def int_hexagon_V6_vlsrw_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vlsrw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwv,VI_ftype_VIVI,2) +// tag : V6_vasrwv +def int_hexagon_V6_vasrwv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vasrwv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwv_128B,VI_ftype_VIVI,2) +// tag : V6_vasrwv_128B +def int_hexagon_V6_vasrwv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vasrwv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslwv,VI_ftype_VIVI,2) +// tag : V6_vaslwv +def int_hexagon_V6_vaslwv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vaslwv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslwv_128B,VI_ftype_VIVI,2) +// tag : V6_vaslwv_128B +def int_hexagon_V6_vaslwv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaslwv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlsrwv,VI_ftype_VIVI,2) +// tag : V6_vlsrwv +def int_hexagon_V6_vlsrwv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vlsrwv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlsrwv_128B,VI_ftype_VIVI,2) +// tag : V6_vlsrwv_128B +def int_hexagon_V6_vlsrwv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vlsrwv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrh,VI_ftype_VISI,2) +// tag : V6_vasrh +def int_hexagon_V6_vasrh : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vasrh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrh_128B,VI_ftype_VISI,2) +// tag : V6_vasrh_128B +def int_hexagon_V6_vasrh_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vasrh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslh,VI_ftype_VISI,2) +// tag : V6_vaslh +def int_hexagon_V6_vaslh : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vaslh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslh_128B,VI_ftype_VISI,2) +// tag : V6_vaslh_128B +def int_hexagon_V6_vaslh_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vaslh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlsrh,VI_ftype_VISI,2) +// tag : V6_vlsrh +def int_hexagon_V6_vlsrh : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vlsrh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlsrh_128B,VI_ftype_VISI,2) +// tag : V6_vlsrh_128B +def int_hexagon_V6_vlsrh_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vlsrh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrhv,VI_ftype_VIVI,2) +// tag : V6_vasrhv +def int_hexagon_V6_vasrhv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vasrhv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrhv_128B,VI_ftype_VIVI,2) +// tag : V6_vasrhv_128B +def int_hexagon_V6_vasrhv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vasrhv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslhv,VI_ftype_VIVI,2) +// tag : V6_vaslhv +def int_hexagon_V6_vaslhv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vaslhv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslhv_128B,VI_ftype_VIVI,2) +// tag : V6_vaslhv_128B +def int_hexagon_V6_vaslhv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaslhv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlsrhv,VI_ftype_VIVI,2) +// tag : V6_vlsrhv +def int_hexagon_V6_vlsrhv : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vlsrhv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlsrhv_128B,VI_ftype_VIVI,2) +// tag : V6_vlsrhv_128B +def int_hexagon_V6_vlsrhv_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vlsrhv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwh,VI_ftype_VIVISI,3) +// tag : V6_vasrwh +def int_hexagon_V6_vasrwh : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrwh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwh_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrwh_128B +def int_hexagon_V6_vasrwh_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrwh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwhsat,VI_ftype_VIVISI,3) +// tag : V6_vasrwhsat +def int_hexagon_V6_vasrwhsat : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrwhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwhsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrwhsat_128B +def int_hexagon_V6_vasrwhsat_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrwhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwhrndsat,VI_ftype_VIVISI,3) +// tag : V6_vasrwhrndsat +def int_hexagon_V6_vasrwhrndsat : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrwhrndsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwhrndsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrwhrndsat_128B +def int_hexagon_V6_vasrwhrndsat_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrwhrndsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwuhsat,VI_ftype_VIVISI,3) +// tag : V6_vasrwuhsat +def int_hexagon_V6_vasrwuhsat : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrwuhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrwuhsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrwuhsat_128B +def int_hexagon_V6_vasrwuhsat_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrwuhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vroundwh,VI_ftype_VIVI,2) +// tag : V6_vroundwh +def int_hexagon_V6_vroundwh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vroundwh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vroundwh_128B,VI_ftype_VIVI,2) +// tag : V6_vroundwh_128B +def int_hexagon_V6_vroundwh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vroundwh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vroundwuh,VI_ftype_VIVI,2) +// tag : V6_vroundwuh +def int_hexagon_V6_vroundwuh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vroundwuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vroundwuh_128B,VI_ftype_VIVI,2) +// tag : V6_vroundwuh_128B +def int_hexagon_V6_vroundwuh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vroundwuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrhubsat,VI_ftype_VIVISI,3) +// tag : V6_vasrhubsat +def int_hexagon_V6_vasrhubsat : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrhubsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrhubsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrhubsat_128B +def int_hexagon_V6_vasrhubsat_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrhubsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrhubrndsat,VI_ftype_VIVISI,3) +// tag : V6_vasrhubrndsat +def int_hexagon_V6_vasrhubrndsat : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrhubrndsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrhubrndsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrhubrndsat_128B +def int_hexagon_V6_vasrhubrndsat_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrhubrndsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrhbrndsat,VI_ftype_VIVISI,3) +// tag : V6_vasrhbrndsat +def int_hexagon_V6_vasrhbrndsat : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrhbrndsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrhbrndsat_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrhbrndsat_128B +def int_hexagon_V6_vasrhbrndsat_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrhbrndsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vroundhb,VI_ftype_VIVI,2) +// tag : V6_vroundhb +def int_hexagon_V6_vroundhb : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vroundhb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vroundhb_128B,VI_ftype_VIVI,2) +// tag : V6_vroundhb_128B +def int_hexagon_V6_vroundhb_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vroundhb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vroundhub,VI_ftype_VIVI,2) +// tag : V6_vroundhub +def int_hexagon_V6_vroundhub : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vroundhub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vroundhub_128B,VI_ftype_VIVI,2) +// tag : V6_vroundhub_128B +def int_hexagon_V6_vroundhub_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vroundhub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslw_acc,VI_ftype_VIVISI,3) +// tag : V6_vaslw_acc +def int_hexagon_V6_vaslw_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vaslw_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaslw_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vaslw_acc_128B +def int_hexagon_V6_vaslw_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vaslw_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrw_acc,VI_ftype_VIVISI,3) +// tag : V6_vasrw_acc +def int_hexagon_V6_vasrw_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vasrw_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vasrw_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vasrw_acc_128B +def int_hexagon_V6_vasrw_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vasrw_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddb,VI_ftype_VIVI,2) +// tag : V6_vaddb +def int_hexagon_V6_vaddb : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vaddb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddb_128B,VI_ftype_VIVI,2) +// tag : V6_vaddb_128B +def int_hexagon_V6_vaddb_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubb,VI_ftype_VIVI,2) +// tag : V6_vsubb +def int_hexagon_V6_vsubb : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsubb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubb_128B,VI_ftype_VIVI,2) +// tag : V6_vsubb_128B +def int_hexagon_V6_vsubb_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddb_dv,VD_ftype_VDVD,2) +// tag : V6_vaddb_dv +def int_hexagon_V6_vaddb_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddb_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddb_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vaddb_dv_128B +def int_hexagon_V6_vaddb_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vaddb_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubb_dv,VD_ftype_VDVD,2) +// tag : V6_vsubb_dv +def int_hexagon_V6_vsubb_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubb_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubb_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vsubb_dv_128B +def int_hexagon_V6_vsubb_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vsubb_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddh,VI_ftype_VIVI,2) +// tag : V6_vaddh +def int_hexagon_V6_vaddh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vaddh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddh_128B,VI_ftype_VIVI,2) +// tag : V6_vaddh_128B +def int_hexagon_V6_vaddh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubh,VI_ftype_VIVI,2) +// tag : V6_vsubh +def int_hexagon_V6_vsubh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsubh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubh_128B,VI_ftype_VIVI,2) +// tag : V6_vsubh_128B +def int_hexagon_V6_vsubh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddh_dv,VD_ftype_VDVD,2) +// tag : V6_vaddh_dv +def int_hexagon_V6_vaddh_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddh_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddh_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vaddh_dv_128B +def int_hexagon_V6_vaddh_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vaddh_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubh_dv,VD_ftype_VDVD,2) +// tag : V6_vsubh_dv +def int_hexagon_V6_vsubh_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubh_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubh_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vsubh_dv_128B +def int_hexagon_V6_vsubh_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vsubh_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddw,VI_ftype_VIVI,2) +// tag : V6_vaddw +def int_hexagon_V6_vaddw : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vaddw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddw_128B,VI_ftype_VIVI,2) +// tag : V6_vaddw_128B +def int_hexagon_V6_vaddw_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubw,VI_ftype_VIVI,2) +// tag : V6_vsubw +def int_hexagon_V6_vsubw : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsubw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubw_128B,VI_ftype_VIVI,2) +// tag : V6_vsubw_128B +def int_hexagon_V6_vsubw_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddw_dv,VD_ftype_VDVD,2) +// tag : V6_vaddw_dv +def int_hexagon_V6_vaddw_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddw_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddw_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vaddw_dv_128B +def int_hexagon_V6_vaddw_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vaddw_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubw_dv,VD_ftype_VDVD,2) +// tag : V6_vsubw_dv +def int_hexagon_V6_vsubw_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubw_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubw_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vsubw_dv_128B +def int_hexagon_V6_vsubw_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vsubw_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddubsat,VI_ftype_VIVI,2) +// tag : V6_vaddubsat +def int_hexagon_V6_vaddubsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vaddubsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddubsat_128B,VI_ftype_VIVI,2) +// tag : V6_vaddubsat_128B +def int_hexagon_V6_vaddubsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddubsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddubsat_dv,VD_ftype_VDVD,2) +// tag : V6_vaddubsat_dv +def int_hexagon_V6_vaddubsat_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddubsat_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddubsat_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vaddubsat_dv_128B +def int_hexagon_V6_vaddubsat_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vaddubsat_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsububsat,VI_ftype_VIVI,2) +// tag : V6_vsububsat +def int_hexagon_V6_vsububsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsububsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsububsat_128B,VI_ftype_VIVI,2) +// tag : V6_vsububsat_128B +def int_hexagon_V6_vsububsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsububsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsububsat_dv,VD_ftype_VDVD,2) +// tag : V6_vsububsat_dv +def int_hexagon_V6_vsububsat_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsububsat_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsububsat_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vsububsat_dv_128B +def int_hexagon_V6_vsububsat_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vsububsat_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vadduhsat,VI_ftype_VIVI,2) +// tag : V6_vadduhsat +def int_hexagon_V6_vadduhsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vadduhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vadduhsat_128B,VI_ftype_VIVI,2) +// tag : V6_vadduhsat_128B +def int_hexagon_V6_vadduhsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vadduhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vadduhsat_dv,VD_ftype_VDVD,2) +// tag : V6_vadduhsat_dv +def int_hexagon_V6_vadduhsat_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vadduhsat_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vadduhsat_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vadduhsat_dv_128B +def int_hexagon_V6_vadduhsat_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vadduhsat_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubuhsat,VI_ftype_VIVI,2) +// tag : V6_vsubuhsat +def int_hexagon_V6_vsubuhsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsubuhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubuhsat_128B,VI_ftype_VIVI,2) +// tag : V6_vsubuhsat_128B +def int_hexagon_V6_vsubuhsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubuhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubuhsat_dv,VD_ftype_VDVD,2) +// tag : V6_vsubuhsat_dv +def int_hexagon_V6_vsubuhsat_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubuhsat_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubuhsat_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vsubuhsat_dv_128B +def int_hexagon_V6_vsubuhsat_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vsubuhsat_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhsat,VI_ftype_VIVI,2) +// tag : V6_vaddhsat +def int_hexagon_V6_vaddhsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vaddhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhsat_128B,VI_ftype_VIVI,2) +// tag : V6_vaddhsat_128B +def int_hexagon_V6_vaddhsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhsat_dv,VD_ftype_VDVD,2) +// tag : V6_vaddhsat_dv +def int_hexagon_V6_vaddhsat_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddhsat_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhsat_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vaddhsat_dv_128B +def int_hexagon_V6_vaddhsat_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vaddhsat_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhsat,VI_ftype_VIVI,2) +// tag : V6_vsubhsat +def int_hexagon_V6_vsubhsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsubhsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhsat_128B,VI_ftype_VIVI,2) +// tag : V6_vsubhsat_128B +def int_hexagon_V6_vsubhsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubhsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhsat_dv,VD_ftype_VDVD,2) +// tag : V6_vsubhsat_dv +def int_hexagon_V6_vsubhsat_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubhsat_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhsat_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vsubhsat_dv_128B +def int_hexagon_V6_vsubhsat_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vsubhsat_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddwsat,VI_ftype_VIVI,2) +// tag : V6_vaddwsat +def int_hexagon_V6_vaddwsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vaddwsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddwsat_128B,VI_ftype_VIVI,2) +// tag : V6_vaddwsat_128B +def int_hexagon_V6_vaddwsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddwsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddwsat_dv,VD_ftype_VDVD,2) +// tag : V6_vaddwsat_dv +def int_hexagon_V6_vaddwsat_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vaddwsat_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddwsat_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vaddwsat_dv_128B +def int_hexagon_V6_vaddwsat_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vaddwsat_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubwsat,VI_ftype_VIVI,2) +// tag : V6_vsubwsat +def int_hexagon_V6_vsubwsat : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsubwsat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubwsat_128B,VI_ftype_VIVI,2) +// tag : V6_vsubwsat_128B +def int_hexagon_V6_vsubwsat_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubwsat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubwsat_dv,VD_ftype_VDVD,2) +// tag : V6_vsubwsat_dv +def int_hexagon_V6_vsubwsat_dv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsubwsat_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubwsat_dv_128B,VD_ftype_VDVD,2) +// tag : V6_vsubwsat_dv_128B +def int_hexagon_V6_vsubwsat_dv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vsubwsat_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgub,VI_ftype_VIVI,2) +// tag : V6_vavgub +def int_hexagon_V6_vavgub : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vavgub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgub_128B,VI_ftype_VIVI,2) +// tag : V6_vavgub_128B +def int_hexagon_V6_vavgub_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavgub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgubrnd,VI_ftype_VIVI,2) +// tag : V6_vavgubrnd +def int_hexagon_V6_vavgubrnd : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vavgubrnd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgubrnd_128B,VI_ftype_VIVI,2) +// tag : V6_vavgubrnd_128B +def int_hexagon_V6_vavgubrnd_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavgubrnd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavguh,VI_ftype_VIVI,2) +// tag : V6_vavguh +def int_hexagon_V6_vavguh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vavguh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavguh_128B,VI_ftype_VIVI,2) +// tag : V6_vavguh_128B +def int_hexagon_V6_vavguh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavguh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavguhrnd,VI_ftype_VIVI,2) +// tag : V6_vavguhrnd +def int_hexagon_V6_vavguhrnd : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vavguhrnd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavguhrnd_128B,VI_ftype_VIVI,2) +// tag : V6_vavguhrnd_128B +def int_hexagon_V6_vavguhrnd_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavguhrnd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgh,VI_ftype_VIVI,2) +// tag : V6_vavgh +def int_hexagon_V6_vavgh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vavgh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgh_128B,VI_ftype_VIVI,2) +// tag : V6_vavgh_128B +def int_hexagon_V6_vavgh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavgh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavghrnd,VI_ftype_VIVI,2) +// tag : V6_vavghrnd +def int_hexagon_V6_vavghrnd : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vavghrnd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavghrnd_128B,VI_ftype_VIVI,2) +// tag : V6_vavghrnd_128B +def int_hexagon_V6_vavghrnd_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavghrnd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnavgh,VI_ftype_VIVI,2) +// tag : V6_vnavgh +def int_hexagon_V6_vnavgh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vnavgh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnavgh_128B,VI_ftype_VIVI,2) +// tag : V6_vnavgh_128B +def int_hexagon_V6_vnavgh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vnavgh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgw,VI_ftype_VIVI,2) +// tag : V6_vavgw +def int_hexagon_V6_vavgw : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vavgw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgw_128B,VI_ftype_VIVI,2) +// tag : V6_vavgw_128B +def int_hexagon_V6_vavgw_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavgw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgwrnd,VI_ftype_VIVI,2) +// tag : V6_vavgwrnd +def int_hexagon_V6_vavgwrnd : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vavgwrnd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vavgwrnd_128B,VI_ftype_VIVI,2) +// tag : V6_vavgwrnd_128B +def int_hexagon_V6_vavgwrnd_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vavgwrnd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnavgw,VI_ftype_VIVI,2) +// tag : V6_vnavgw +def int_hexagon_V6_vnavgw : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vnavgw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnavgw_128B,VI_ftype_VIVI,2) +// tag : V6_vnavgw_128B +def int_hexagon_V6_vnavgw_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vnavgw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsdiffub,VI_ftype_VIVI,2) +// tag : V6_vabsdiffub +def int_hexagon_V6_vabsdiffub : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vabsdiffub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsdiffub_128B,VI_ftype_VIVI,2) +// tag : V6_vabsdiffub_128B +def int_hexagon_V6_vabsdiffub_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vabsdiffub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsdiffuh,VI_ftype_VIVI,2) +// tag : V6_vabsdiffuh +def int_hexagon_V6_vabsdiffuh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vabsdiffuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsdiffuh_128B,VI_ftype_VIVI,2) +// tag : V6_vabsdiffuh_128B +def int_hexagon_V6_vabsdiffuh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vabsdiffuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsdiffh,VI_ftype_VIVI,2) +// tag : V6_vabsdiffh +def int_hexagon_V6_vabsdiffh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vabsdiffh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsdiffh_128B,VI_ftype_VIVI,2) +// tag : V6_vabsdiffh_128B +def int_hexagon_V6_vabsdiffh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vabsdiffh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsdiffw,VI_ftype_VIVI,2) +// tag : V6_vabsdiffw +def int_hexagon_V6_vabsdiffw : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vabsdiffw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsdiffw_128B,VI_ftype_VIVI,2) +// tag : V6_vabsdiffw_128B +def int_hexagon_V6_vabsdiffw_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vabsdiffw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnavgub,VI_ftype_VIVI,2) +// tag : V6_vnavgub +def int_hexagon_V6_vnavgub : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vnavgub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnavgub_128B,VI_ftype_VIVI,2) +// tag : V6_vnavgub_128B +def int_hexagon_V6_vnavgub_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vnavgub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddubh,VD_ftype_VIVI,2) +// tag : V6_vaddubh +def int_hexagon_V6_vaddubh : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vaddubh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddubh_128B,VD_ftype_VIVI,2) +// tag : V6_vaddubh_128B +def int_hexagon_V6_vaddubh_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vaddubh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsububh,VD_ftype_VIVI,2) +// tag : V6_vsububh +def int_hexagon_V6_vsububh : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vsububh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsububh_128B,VD_ftype_VIVI,2) +// tag : V6_vsububh_128B +def int_hexagon_V6_vsububh_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vsububh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhw,VD_ftype_VIVI,2) +// tag : V6_vaddhw +def int_hexagon_V6_vaddhw : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vaddhw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhw_128B,VD_ftype_VIVI,2) +// tag : V6_vaddhw_128B +def int_hexagon_V6_vaddhw_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vaddhw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhw,VD_ftype_VIVI,2) +// tag : V6_vsubhw +def int_hexagon_V6_vsubhw : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vsubhw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhw_128B,VD_ftype_VIVI,2) +// tag : V6_vsubhw_128B +def int_hexagon_V6_vsubhw_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vsubhw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vadduhw,VD_ftype_VIVI,2) +// tag : V6_vadduhw +def int_hexagon_V6_vadduhw : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vadduhw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vadduhw_128B,VD_ftype_VIVI,2) +// tag : V6_vadduhw_128B +def int_hexagon_V6_vadduhw_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vadduhw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubuhw,VD_ftype_VIVI,2) +// tag : V6_vsubuhw +def int_hexagon_V6_vsubuhw : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vsubuhw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubuhw_128B,VD_ftype_VIVI,2) +// tag : V6_vsubuhw_128B +def int_hexagon_V6_vsubuhw_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vsubuhw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vd0,VI_ftype_,0) +// tag : V6_vd0 +def int_hexagon_V6_vd0 : +Hexagon_v512_Intrinsic<"HEXAGON_V6_vd0">; + +// +// BUILTIN_INFO(HEXAGON.V6_vd0_128B,VI_ftype_,0) +// tag : V6_vd0_128B +def int_hexagon_V6_vd0_128B : +Hexagon_v1024_Intrinsic<"HEXAGON_V6_vd0_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddbq,VI_ftype_QVVIVI,3) +// tag : V6_vaddbq +def int_hexagon_V6_vaddbq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vaddbq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddbq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vaddbq_128B +def int_hexagon_V6_vaddbq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vaddbq_128B">; + + +// +// BUILTIN_INFO(HEXAGON.V6_vsubbq,VI_ftype_QVVIVI,3) +// tag : V6_vsubbq +def int_hexagon_V6_vsubbq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vsubbq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubbq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vsubbq_128B +def int_hexagon_V6_vsubbq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vsubbq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddbnq,VI_ftype_QVVIVI,3) +// tag : V6_vaddbnq +def int_hexagon_V6_vaddbnq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vaddbnq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddbnq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vaddbnq_128B +def int_hexagon_V6_vaddbnq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vaddbnq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubbnq,VI_ftype_QVVIVI,3) +// tag : V6_vsubbnq +def int_hexagon_V6_vsubbnq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vsubbnq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubbnq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vsubbnq_128B +def int_hexagon_V6_vsubbnq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vsubbnq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhq,VI_ftype_QVVIVI,3) +// tag : V6_vaddhq +def int_hexagon_V6_vaddhq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vaddhq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vaddhq_128B +def int_hexagon_V6_vaddhq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vaddhq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhq,VI_ftype_QVVIVI,3) +// tag : V6_vsubhq +def int_hexagon_V6_vsubhq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vsubhq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vsubhq_128B +def int_hexagon_V6_vsubhq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vsubhq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhnq,VI_ftype_QVVIVI,3) +// tag : V6_vaddhnq +def int_hexagon_V6_vaddhnq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vaddhnq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddhnq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vaddhnq_128B +def int_hexagon_V6_vaddhnq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vaddhnq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhnq,VI_ftype_QVVIVI,3) +// tag : V6_vsubhnq +def int_hexagon_V6_vsubhnq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vsubhnq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubhnq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vsubhnq_128B +def int_hexagon_V6_vsubhnq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vsubhnq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddwq,VI_ftype_QVVIVI,3) +// tag : V6_vaddwq +def int_hexagon_V6_vaddwq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vaddwq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddwq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vaddwq_128B +def int_hexagon_V6_vaddwq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vaddwq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubwq,VI_ftype_QVVIVI,3) +// tag : V6_vsubwq +def int_hexagon_V6_vsubwq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vsubwq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubwq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vsubwq_128B +def int_hexagon_V6_vsubwq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vsubwq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddwnq,VI_ftype_QVVIVI,3) +// tag : V6_vaddwnq +def int_hexagon_V6_vaddwnq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vaddwnq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vaddwnq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vaddwnq_128B +def int_hexagon_V6_vaddwnq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vaddwnq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubwnq,VI_ftype_QVVIVI,3) +// tag : V6_vsubwnq +def int_hexagon_V6_vsubwnq : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vsubwnq">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsubwnq_128B,VI_ftype_QVVIVI,3) +// tag : V6_vsubwnq_128B +def int_hexagon_V6_vsubwnq_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vsubwnq_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsh,VI_ftype_VI,1) +// tag : V6_vabsh +def int_hexagon_V6_vabsh : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vabsh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsh_128B,VI_ftype_VI,1) +// tag : V6_vabsh_128B +def int_hexagon_V6_vabsh_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vabsh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsh_sat,VI_ftype_VI,1) +// tag : V6_vabsh_sat +def int_hexagon_V6_vabsh_sat : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vabsh_sat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsh_sat_128B,VI_ftype_VI,1) +// tag : V6_vabsh_sat_128B +def int_hexagon_V6_vabsh_sat_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vabsh_sat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsw,VI_ftype_VI,1) +// tag : V6_vabsw +def int_hexagon_V6_vabsw : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vabsw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsw_128B,VI_ftype_VI,1) +// tag : V6_vabsw_128B +def int_hexagon_V6_vabsw_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vabsw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsw_sat,VI_ftype_VI,1) +// tag : V6_vabsw_sat +def int_hexagon_V6_vabsw_sat : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vabsw_sat">; + +// +// BUILTIN_INFO(HEXAGON.V6_vabsw_sat_128B,VI_ftype_VI,1) +// tag : V6_vabsw_sat_128B +def int_hexagon_V6_vabsw_sat_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vabsw_sat_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybv,VD_ftype_VIVI,2) +// tag : V6_vmpybv +def int_hexagon_V6_vmpybv : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vmpybv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybv_128B,VD_ftype_VIVI,2) +// tag : V6_vmpybv_128B +def int_hexagon_V6_vmpybv_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpybv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybv_acc,VD_ftype_VDVIVI,3) +// tag : V6_vmpybv_acc +def int_hexagon_V6_vmpybv_acc : +Hexagon_v1024v1024v512v512_Intrinsic<"HEXAGON_V6_vmpybv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybv_acc_128B,VD_ftype_VDVIVI,3) +// tag : V6_vmpybv_acc_128B +def int_hexagon_V6_vmpybv_acc_128B : +Hexagon_v2048v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpybv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyubv,VD_ftype_VIVI,2) +// tag : V6_vmpyubv +def int_hexagon_V6_vmpyubv : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vmpyubv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyubv_128B,VD_ftype_VIVI,2) +// tag : V6_vmpyubv_128B +def int_hexagon_V6_vmpyubv_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpyubv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyubv_acc,VD_ftype_VDVIVI,3) +// tag : V6_vmpyubv_acc +def int_hexagon_V6_vmpyubv_acc : +Hexagon_v1024v1024v512v512_Intrinsic<"HEXAGON_V6_vmpyubv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyubv_acc_128B,VD_ftype_VDVIVI,3) +// tag : V6_vmpyubv_acc_128B +def int_hexagon_V6_vmpyubv_acc_128B : +Hexagon_v2048v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpyubv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybusv,VD_ftype_VIVI,2) +// tag : V6_vmpybusv +def int_hexagon_V6_vmpybusv : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vmpybusv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybusv_128B,VD_ftype_VIVI,2) +// tag : V6_vmpybusv_128B +def int_hexagon_V6_vmpybusv_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpybusv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybusv_acc,VD_ftype_VDVIVI,3) +// tag : V6_vmpybusv_acc +def int_hexagon_V6_vmpybusv_acc : +Hexagon_v1024v1024v512v512_Intrinsic<"HEXAGON_V6_vmpybusv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybusv_acc_128B,VD_ftype_VDVIVI,3) +// tag : V6_vmpybusv_acc_128B +def int_hexagon_V6_vmpybusv_acc_128B : +Hexagon_v2048v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpybusv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabusv,VD_ftype_VDVD,2) +// tag : V6_vmpabusv +def int_hexagon_V6_vmpabusv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpabusv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabusv_128B,VD_ftype_VDVD,2) +// tag : V6_vmpabusv_128B +def int_hexagon_V6_vmpabusv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vmpabusv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabuuv,VD_ftype_VDVD,2) +// tag : V6_vmpabuuv +def int_hexagon_V6_vmpabuuv : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpabuuv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabuuv_128B,VD_ftype_VDVD,2) +// tag : V6_vmpabuuv_128B +def int_hexagon_V6_vmpabuuv_128B : +Hexagon_v2048v2048v2048_Intrinsic<"HEXAGON_V6_vmpabuuv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhv,VD_ftype_VIVI,2) +// tag : V6_vmpyhv +def int_hexagon_V6_vmpyhv : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vmpyhv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhv_128B,VD_ftype_VIVI,2) +// tag : V6_vmpyhv_128B +def int_hexagon_V6_vmpyhv_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpyhv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhv_acc,VD_ftype_VDVIVI,3) +// tag : V6_vmpyhv_acc +def int_hexagon_V6_vmpyhv_acc : +Hexagon_v1024v1024v512v512_Intrinsic<"HEXAGON_V6_vmpyhv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhv_acc_128B,VD_ftype_VDVIVI,3) +// tag : V6_vmpyhv_acc_128B +def int_hexagon_V6_vmpyhv_acc_128B : +Hexagon_v2048v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpyhv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuhv,VD_ftype_VIVI,2) +// tag : V6_vmpyuhv +def int_hexagon_V6_vmpyuhv : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vmpyuhv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuhv_128B,VD_ftype_VIVI,2) +// tag : V6_vmpyuhv_128B +def int_hexagon_V6_vmpyuhv_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpyuhv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuhv_acc,VD_ftype_VDVIVI,3) +// tag : V6_vmpyuhv_acc +def int_hexagon_V6_vmpyuhv_acc : +Hexagon_v1024v1024v512v512_Intrinsic<"HEXAGON_V6_vmpyuhv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuhv_acc_128B,VD_ftype_VDVIVI,3) +// tag : V6_vmpyuhv_acc_128B +def int_hexagon_V6_vmpyuhv_acc_128B : +Hexagon_v2048v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpyuhv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhvsrs,VI_ftype_VIVI,2) +// tag : V6_vmpyhvsrs +def int_hexagon_V6_vmpyhvsrs : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmpyhvsrs">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhvsrs_128B,VI_ftype_VIVI,2) +// tag : V6_vmpyhvsrs_128B +def int_hexagon_V6_vmpyhvsrs_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyhvsrs_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhus,VD_ftype_VIVI,2) +// tag : V6_vmpyhus +def int_hexagon_V6_vmpyhus : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vmpyhus">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhus_128B,VD_ftype_VIVI,2) +// tag : V6_vmpyhus_128B +def int_hexagon_V6_vmpyhus_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpyhus_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhus_acc,VD_ftype_VDVIVI,3) +// tag : V6_vmpyhus_acc +def int_hexagon_V6_vmpyhus_acc : +Hexagon_v1024v1024v512v512_Intrinsic<"HEXAGON_V6_vmpyhus_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhus_acc_128B,VD_ftype_VDVIVI,3) +// tag : V6_vmpyhus_acc_128B +def int_hexagon_V6_vmpyhus_acc_128B : +Hexagon_v2048v2048v1024v1024_Intrinsic<"HEXAGON_V6_vmpyhus_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyih,VI_ftype_VIVI,2) +// tag : V6_vmpyih +def int_hexagon_V6_vmpyih : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmpyih">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyih_128B,VI_ftype_VIVI,2) +// tag : V6_vmpyih_128B +def int_hexagon_V6_vmpyih_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyih_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyih_acc,VI_ftype_VIVIVI,3) +// tag : V6_vmpyih_acc +def int_hexagon_V6_vmpyih_acc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vmpyih_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyih_acc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vmpyih_acc_128B +def int_hexagon_V6_vmpyih_acc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyih_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyewuh,VI_ftype_VIVI,2) +// tag : V6_vmpyewuh +def int_hexagon_V6_vmpyewuh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmpyewuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyewuh_128B,VI_ftype_VIVI,2) +// tag : V6_vmpyewuh_128B +def int_hexagon_V6_vmpyewuh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyewuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyowh,VI_ftype_VIVI,2) +// tag : V6_vmpyowh +def int_hexagon_V6_vmpyowh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmpyowh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyowh_128B,VI_ftype_VIVI,2) +// tag : V6_vmpyowh_128B +def int_hexagon_V6_vmpyowh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyowh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyowh_rnd,VI_ftype_VIVI,2) +// tag : V6_vmpyowh_rnd +def int_hexagon_V6_vmpyowh_rnd : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmpyowh_rnd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyowh_rnd_128B,VI_ftype_VIVI,2) +// tag : V6_vmpyowh_rnd_128B +def int_hexagon_V6_vmpyowh_rnd_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyowh_rnd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyowh_sacc,VI_ftype_VIVIVI,3) +// tag : V6_vmpyowh_sacc +def int_hexagon_V6_vmpyowh_sacc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vmpyowh_sacc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyowh_sacc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vmpyowh_sacc_128B +def int_hexagon_V6_vmpyowh_sacc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyowh_sacc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyowh_rnd_sacc,VI_ftype_VIVIVI,3) +// tag : V6_vmpyowh_rnd_sacc +def int_hexagon_V6_vmpyowh_rnd_sacc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vmpyowh_rnd_sacc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyowh_rnd_sacc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vmpyowh_rnd_sacc_128B +def int_hexagon_V6_vmpyowh_rnd_sacc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyowh_rnd_sacc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyieoh,VI_ftype_VIVI,2) +// tag : V6_vmpyieoh +def int_hexagon_V6_vmpyieoh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmpyieoh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyieoh_128B,VI_ftype_VIVI,2) +// tag : V6_vmpyieoh_128B +def int_hexagon_V6_vmpyieoh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyieoh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiewuh,VI_ftype_VIVI,2) +// tag : V6_vmpyiewuh +def int_hexagon_V6_vmpyiewuh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmpyiewuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiewuh_128B,VI_ftype_VIVI,2) +// tag : V6_vmpyiewuh_128B +def int_hexagon_V6_vmpyiewuh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyiewuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiowh,VI_ftype_VIVI,2) +// tag : V6_vmpyiowh +def int_hexagon_V6_vmpyiowh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmpyiowh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiowh_128B,VI_ftype_VIVI,2) +// tag : V6_vmpyiowh_128B +def int_hexagon_V6_vmpyiowh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyiowh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiewh_acc,VI_ftype_VIVIVI,3) +// tag : V6_vmpyiewh_acc +def int_hexagon_V6_vmpyiewh_acc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vmpyiewh_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiewh_acc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vmpyiewh_acc_128B +def int_hexagon_V6_vmpyiewh_acc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyiewh_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiewuh_acc,VI_ftype_VIVIVI,3) +// tag : V6_vmpyiewuh_acc +def int_hexagon_V6_vmpyiewuh_acc : +Hexagon_v512v512v512v512_Intrinsic<"HEXAGON_V6_vmpyiewuh_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiewuh_acc_128B,VI_ftype_VIVIVI,3) +// tag : V6_vmpyiewuh_acc_128B +def int_hexagon_V6_vmpyiewuh_acc_128B : +Hexagon_v1024v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmpyiewuh_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyub,VD_ftype_VISI,2) +// tag : V6_vmpyub +def int_hexagon_V6_vmpyub : +Hexagon_v1024v512i_Intrinsic<"HEXAGON_V6_vmpyub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyub_128B,VD_ftype_VISI,2) +// tag : V6_vmpyub_128B +def int_hexagon_V6_vmpyub_128B : +Hexagon_v2048v1024i_Intrinsic<"HEXAGON_V6_vmpyub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyub_acc,VD_ftype_VDVISI,3) +// tag : V6_vmpyub_acc +def int_hexagon_V6_vmpyub_acc : +Hexagon_v1024v1024v512i_Intrinsic<"HEXAGON_V6_vmpyub_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyub_acc_128B,VD_ftype_VDVISI,3) +// tag : V6_vmpyub_acc_128B +def int_hexagon_V6_vmpyub_acc_128B : +Hexagon_v2048v2048v1024i_Intrinsic<"HEXAGON_V6_vmpyub_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybus,VD_ftype_VISI,2) +// tag : V6_vmpybus +def int_hexagon_V6_vmpybus : +Hexagon_v1024v512i_Intrinsic<"HEXAGON_V6_vmpybus">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybus_128B,VD_ftype_VISI,2) +// tag : V6_vmpybus_128B +def int_hexagon_V6_vmpybus_128B : +Hexagon_v2048v1024i_Intrinsic<"HEXAGON_V6_vmpybus_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybus_acc,VD_ftype_VDVISI,3) +// tag : V6_vmpybus_acc +def int_hexagon_V6_vmpybus_acc : +Hexagon_v1024v1024v512i_Intrinsic<"HEXAGON_V6_vmpybus_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpybus_acc_128B,VD_ftype_VDVISI,3) +// tag : V6_vmpybus_acc_128B +def int_hexagon_V6_vmpybus_acc_128B : +Hexagon_v2048v2048v1024i_Intrinsic<"HEXAGON_V6_vmpybus_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabus,VD_ftype_VDSI,2) +// tag : V6_vmpabus +def int_hexagon_V6_vmpabus : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpabus">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabus_128B,VD_ftype_VDSI,2) +// tag : V6_vmpabus_128B +def int_hexagon_V6_vmpabus_128B : +Hexagon_v2048v2048i_Intrinsic<"HEXAGON_V6_vmpabus_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabus_acc,VD_ftype_VDVDSI,3) +// tag : V6_vmpabus_acc +def int_hexagon_V6_vmpabus_acc : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vmpabus_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpabus_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vmpabus_acc_128B +def int_hexagon_V6_vmpabus_acc_128B : +Hexagon_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vmpabus_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpahb,VD_ftype_VDSI,2) +// tag : V6_vmpahb +def int_hexagon_V6_vmpahb : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpahb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpahb_128B,VD_ftype_VDSI,2) +// tag : V6_vmpahb_128B +def int_hexagon_V6_vmpahb_128B : +Hexagon_v2048v2048i_Intrinsic<"HEXAGON_V6_vmpahb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpahb_acc,VD_ftype_VDVDSI,3) +// tag : V6_vmpahb_acc +def int_hexagon_V6_vmpahb_acc : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vmpahb_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpahb_acc_128B,VD_ftype_VDVDSI,3) +// tag : V6_vmpahb_acc_128B +def int_hexagon_V6_vmpahb_acc_128B : +Hexagon_v2048v2048v2048i_Intrinsic<"HEXAGON_V6_vmpahb_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyh,VD_ftype_VISI,2) +// tag : V6_vmpyh +def int_hexagon_V6_vmpyh : +Hexagon_v1024v512i_Intrinsic<"HEXAGON_V6_vmpyh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyh_128B,VD_ftype_VISI,2) +// tag : V6_vmpyh_128B +def int_hexagon_V6_vmpyh_128B : +Hexagon_v2048v1024i_Intrinsic<"HEXAGON_V6_vmpyh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhsat_acc,VD_ftype_VDVISI,3) +// tag : V6_vmpyhsat_acc +def int_hexagon_V6_vmpyhsat_acc : +Hexagon_v1024v1024v512i_Intrinsic<"HEXAGON_V6_vmpyhsat_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhsat_acc_128B,VD_ftype_VDVISI,3) +// tag : V6_vmpyhsat_acc_128B +def int_hexagon_V6_vmpyhsat_acc_128B : +Hexagon_v2048v2048v1024i_Intrinsic<"HEXAGON_V6_vmpyhsat_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhss,VI_ftype_VISI,2) +// tag : V6_vmpyhss +def int_hexagon_V6_vmpyhss : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vmpyhss">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhss_128B,VI_ftype_VISI,2) +// tag : V6_vmpyhss_128B +def int_hexagon_V6_vmpyhss_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyhss_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhsrs,VI_ftype_VISI,2) +// tag : V6_vmpyhsrs +def int_hexagon_V6_vmpyhsrs : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vmpyhsrs">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyhsrs_128B,VI_ftype_VISI,2) +// tag : V6_vmpyhsrs_128B +def int_hexagon_V6_vmpyhsrs_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyhsrs_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuh,VD_ftype_VISI,2) +// tag : V6_vmpyuh +def int_hexagon_V6_vmpyuh : +Hexagon_v1024v512i_Intrinsic<"HEXAGON_V6_vmpyuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuh_128B,VD_ftype_VISI,2) +// tag : V6_vmpyuh_128B +def int_hexagon_V6_vmpyuh_128B : +Hexagon_v2048v1024i_Intrinsic<"HEXAGON_V6_vmpyuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuh_acc,VD_ftype_VDVISI,3) +// tag : V6_vmpyuh_acc +def int_hexagon_V6_vmpyuh_acc : +Hexagon_v1024v1024v512i_Intrinsic<"HEXAGON_V6_vmpyuh_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyuh_acc_128B,VD_ftype_VDVISI,3) +// tag : V6_vmpyuh_acc_128B +def int_hexagon_V6_vmpyuh_acc_128B : +Hexagon_v2048v2048v1024i_Intrinsic<"HEXAGON_V6_vmpyuh_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyihb,VI_ftype_VISI,2) +// tag : V6_vmpyihb +def int_hexagon_V6_vmpyihb : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vmpyihb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyihb_128B,VI_ftype_VISI,2) +// tag : V6_vmpyihb_128B +def int_hexagon_V6_vmpyihb_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyihb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyihb_acc,VI_ftype_VIVISI,3) +// tag : V6_vmpyihb_acc +def int_hexagon_V6_vmpyihb_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vmpyihb_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyihb_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vmpyihb_acc_128B +def int_hexagon_V6_vmpyihb_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyihb_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiwb,VI_ftype_VISI,2) +// tag : V6_vmpyiwb +def int_hexagon_V6_vmpyiwb : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vmpyiwb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiwb_128B,VI_ftype_VISI,2) +// tag : V6_vmpyiwb_128B +def int_hexagon_V6_vmpyiwb_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyiwb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiwb_acc,VI_ftype_VIVISI,3) +// tag : V6_vmpyiwb_acc +def int_hexagon_V6_vmpyiwb_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vmpyiwb_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiwb_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vmpyiwb_acc_128B +def int_hexagon_V6_vmpyiwb_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyiwb_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiwh,VI_ftype_VISI,2) +// tag : V6_vmpyiwh +def int_hexagon_V6_vmpyiwh : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vmpyiwh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiwh_128B,VI_ftype_VISI,2) +// tag : V6_vmpyiwh_128B +def int_hexagon_V6_vmpyiwh_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyiwh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiwh_acc,VI_ftype_VIVISI,3) +// tag : V6_vmpyiwh_acc +def int_hexagon_V6_vmpyiwh_acc : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vmpyiwh_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmpyiwh_acc_128B,VI_ftype_VIVISI,3) +// tag : V6_vmpyiwh_acc_128B +def int_hexagon_V6_vmpyiwh_acc_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vmpyiwh_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vand,VI_ftype_VIVI,2) +// tag : V6_vand +def int_hexagon_V6_vand : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vand">; + +// +// BUILTIN_INFO(HEXAGON.V6_vand_128B,VI_ftype_VIVI,2) +// tag : V6_vand_128B +def int_hexagon_V6_vand_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vand_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vor,VI_ftype_VIVI,2) +// tag : V6_vor +def int_hexagon_V6_vor : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vor">; + +// +// BUILTIN_INFO(HEXAGON.V6_vor_128B,VI_ftype_VIVI,2) +// tag : V6_vor_128B +def int_hexagon_V6_vor_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vxor,VI_ftype_VIVI,2) +// tag : V6_vxor +def int_hexagon_V6_vxor : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vxor">; + +// +// BUILTIN_INFO(HEXAGON.V6_vxor_128B,VI_ftype_VIVI,2) +// tag : V6_vxor_128B +def int_hexagon_V6_vxor_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vxor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnot,VI_ftype_VI,1) +// tag : V6_vnot +def int_hexagon_V6_vnot : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vnot">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnot_128B,VI_ftype_VI,1) +// tag : V6_vnot_128B +def int_hexagon_V6_vnot_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vnot_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vandqrt,VI_ftype_QVSI,2) +// tag : V6_vandqrt +def int_hexagon_V6_vandqrt : +Hexagon_v512v64ii_Intrinsic<"HEXAGON_V6_vandqrt">; + +// +// BUILTIN_INFO(HEXAGON.V6_vandqrt_128B,VI_ftype_QVSI,2) +// tag : V6_vandqrt_128B +def int_hexagon_V6_vandqrt_128B : +Hexagon_v1024v128ii_Intrinsic<"HEXAGON_V6_vandqrt_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vandqrt_acc,VI_ftype_VIQVSI,3) +// tag : V6_vandqrt_acc +def int_hexagon_V6_vandqrt_acc : +Hexagon_v512v512v64ii_Intrinsic<"HEXAGON_V6_vandqrt_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vandqrt_acc_128B,VI_ftype_VIQVSI,3) +// tag : V6_vandqrt_acc_128B +def int_hexagon_V6_vandqrt_acc_128B : +Hexagon_v1024v1024v128ii_Intrinsic<"HEXAGON_V6_vandqrt_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vandvrt,QV_ftype_VISI,2) +// tag : V6_vandvrt +def int_hexagon_V6_vandvrt : +Hexagon_v64iv512i_Intrinsic<"HEXAGON_V6_vandvrt">; + +// +// BUILTIN_INFO(HEXAGON.V6_vandvrt_128B,QV_ftype_VISI,2) +// tag : V6_vandvrt_128B +def int_hexagon_V6_vandvrt_128B : +Hexagon_v128iv1024i_Intrinsic<"HEXAGON_V6_vandvrt_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vandvrt_acc,QV_ftype_QVVISI,3) +// tag : V6_vandvrt_acc +def int_hexagon_V6_vandvrt_acc : +Hexagon_v64iv64iv512i_Intrinsic<"HEXAGON_V6_vandvrt_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vandvrt_acc_128B,QV_ftype_QVVISI,3) +// tag : V6_vandvrt_acc_128B +def int_hexagon_V6_vandvrt_acc_128B : +Hexagon_v128iv128iv1024i_Intrinsic<"HEXAGON_V6_vandvrt_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtw,QV_ftype_VIVI,2) +// tag : V6_vgtw +def int_hexagon_V6_vgtw : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_vgtw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtw_128B,QV_ftype_VIVI,2) +// tag : V6_vgtw_128B +def int_hexagon_V6_vgtw_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtw_and,QV_ftype_QVVIVI,3) +// tag : V6_vgtw_and +def int_hexagon_V6_vgtw_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtw_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtw_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtw_and_128B +def int_hexagon_V6_vgtw_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtw_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtw_or,QV_ftype_QVVIVI,3) +// tag : V6_vgtw_or +def int_hexagon_V6_vgtw_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtw_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtw_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtw_or_128B +def int_hexagon_V6_vgtw_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtw_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtw_xor,QV_ftype_QVVIVI,3) +// tag : V6_vgtw_xor +def int_hexagon_V6_vgtw_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtw_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtw_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtw_xor_128B +def int_hexagon_V6_vgtw_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtw_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqw,QV_ftype_VIVI,2) +// tag : V6_veqw +def int_hexagon_V6_veqw : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_veqw">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqw_128B,QV_ftype_VIVI,2) +// tag : V6_veqw_128B +def int_hexagon_V6_veqw_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_veqw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqw_and,QV_ftype_QVVIVI,3) +// tag : V6_veqw_and +def int_hexagon_V6_veqw_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqw_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqw_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqw_and_128B +def int_hexagon_V6_veqw_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqw_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqw_or,QV_ftype_QVVIVI,3) +// tag : V6_veqw_or +def int_hexagon_V6_veqw_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqw_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqw_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqw_or_128B +def int_hexagon_V6_veqw_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqw_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqw_xor,QV_ftype_QVVIVI,3) +// tag : V6_veqw_xor +def int_hexagon_V6_veqw_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqw_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqw_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqw_xor_128B +def int_hexagon_V6_veqw_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqw_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgth,QV_ftype_VIVI,2) +// tag : V6_vgth +def int_hexagon_V6_vgth : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_vgth">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgth_128B,QV_ftype_VIVI,2) +// tag : V6_vgth_128B +def int_hexagon_V6_vgth_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_vgth_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgth_and,QV_ftype_QVVIVI,3) +// tag : V6_vgth_and +def int_hexagon_V6_vgth_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgth_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgth_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgth_and_128B +def int_hexagon_V6_vgth_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgth_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgth_or,QV_ftype_QVVIVI,3) +// tag : V6_vgth_or +def int_hexagon_V6_vgth_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgth_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgth_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgth_or_128B +def int_hexagon_V6_vgth_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgth_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgth_xor,QV_ftype_QVVIVI,3) +// tag : V6_vgth_xor +def int_hexagon_V6_vgth_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgth_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgth_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgth_xor_128B +def int_hexagon_V6_vgth_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgth_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqh,QV_ftype_VIVI,2) +// tag : V6_veqh +def int_hexagon_V6_veqh : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_veqh">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqh_128B,QV_ftype_VIVI,2) +// tag : V6_veqh_128B +def int_hexagon_V6_veqh_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_veqh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqh_and,QV_ftype_QVVIVI,3) +// tag : V6_veqh_and +def int_hexagon_V6_veqh_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqh_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqh_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqh_and_128B +def int_hexagon_V6_veqh_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqh_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqh_or,QV_ftype_QVVIVI,3) +// tag : V6_veqh_or +def int_hexagon_V6_veqh_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqh_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqh_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqh_or_128B +def int_hexagon_V6_veqh_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqh_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqh_xor,QV_ftype_QVVIVI,3) +// tag : V6_veqh_xor +def int_hexagon_V6_veqh_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqh_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqh_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqh_xor_128B +def int_hexagon_V6_veqh_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqh_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtb,QV_ftype_VIVI,2) +// tag : V6_vgtb +def int_hexagon_V6_vgtb : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_vgtb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtb_128B,QV_ftype_VIVI,2) +// tag : V6_vgtb_128B +def int_hexagon_V6_vgtb_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtb_and,QV_ftype_QVVIVI,3) +// tag : V6_vgtb_and +def int_hexagon_V6_vgtb_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtb_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtb_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtb_and_128B +def int_hexagon_V6_vgtb_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtb_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtb_or,QV_ftype_QVVIVI,3) +// tag : V6_vgtb_or +def int_hexagon_V6_vgtb_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtb_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtb_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtb_or_128B +def int_hexagon_V6_vgtb_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtb_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtb_xor,QV_ftype_QVVIVI,3) +// tag : V6_vgtb_xor +def int_hexagon_V6_vgtb_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtb_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtb_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtb_xor_128B +def int_hexagon_V6_vgtb_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtb_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqb,QV_ftype_VIVI,2) +// tag : V6_veqb +def int_hexagon_V6_veqb : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_veqb">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqb_128B,QV_ftype_VIVI,2) +// tag : V6_veqb_128B +def int_hexagon_V6_veqb_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_veqb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqb_and,QV_ftype_QVVIVI,3) +// tag : V6_veqb_and +def int_hexagon_V6_veqb_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqb_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqb_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqb_and_128B +def int_hexagon_V6_veqb_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqb_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqb_or,QV_ftype_QVVIVI,3) +// tag : V6_veqb_or +def int_hexagon_V6_veqb_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqb_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqb_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqb_or_128B +def int_hexagon_V6_veqb_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqb_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqb_xor,QV_ftype_QVVIVI,3) +// tag : V6_veqb_xor +def int_hexagon_V6_veqb_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_veqb_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_veqb_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_veqb_xor_128B +def int_hexagon_V6_veqb_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_veqb_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuw,QV_ftype_VIVI,2) +// tag : V6_vgtuw +def int_hexagon_V6_vgtuw : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_vgtuw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuw_128B,QV_ftype_VIVI,2) +// tag : V6_vgtuw_128B +def int_hexagon_V6_vgtuw_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtuw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuw_and,QV_ftype_QVVIVI,3) +// tag : V6_vgtuw_and +def int_hexagon_V6_vgtuw_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtuw_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuw_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtuw_and_128B +def int_hexagon_V6_vgtuw_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtuw_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuw_or,QV_ftype_QVVIVI,3) +// tag : V6_vgtuw_or +def int_hexagon_V6_vgtuw_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtuw_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuw_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtuw_or_128B +def int_hexagon_V6_vgtuw_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtuw_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuw_xor,QV_ftype_QVVIVI,3) +// tag : V6_vgtuw_xor +def int_hexagon_V6_vgtuw_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtuw_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuw_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtuw_xor_128B +def int_hexagon_V6_vgtuw_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtuw_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuh,QV_ftype_VIVI,2) +// tag : V6_vgtuh +def int_hexagon_V6_vgtuh : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_vgtuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuh_128B,QV_ftype_VIVI,2) +// tag : V6_vgtuh_128B +def int_hexagon_V6_vgtuh_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuh_and,QV_ftype_QVVIVI,3) +// tag : V6_vgtuh_and +def int_hexagon_V6_vgtuh_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtuh_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuh_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtuh_and_128B +def int_hexagon_V6_vgtuh_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtuh_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuh_or,QV_ftype_QVVIVI,3) +// tag : V6_vgtuh_or +def int_hexagon_V6_vgtuh_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtuh_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuh_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtuh_or_128B +def int_hexagon_V6_vgtuh_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtuh_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuh_xor,QV_ftype_QVVIVI,3) +// tag : V6_vgtuh_xor +def int_hexagon_V6_vgtuh_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtuh_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtuh_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtuh_xor_128B +def int_hexagon_V6_vgtuh_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtuh_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtub,QV_ftype_VIVI,2) +// tag : V6_vgtub +def int_hexagon_V6_vgtub : +Hexagon_v64iv512v512_Intrinsic<"HEXAGON_V6_vgtub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtub_128B,QV_ftype_VIVI,2) +// tag : V6_vgtub_128B +def int_hexagon_V6_vgtub_128B : +Hexagon_v128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtub_and,QV_ftype_QVVIVI,3) +// tag : V6_vgtub_and +def int_hexagon_V6_vgtub_and : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtub_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtub_and_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtub_and_128B +def int_hexagon_V6_vgtub_and_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtub_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtub_or,QV_ftype_QVVIVI,3) +// tag : V6_vgtub_or +def int_hexagon_V6_vgtub_or : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtub_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtub_or_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtub_or_128B +def int_hexagon_V6_vgtub_or_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtub_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtub_xor,QV_ftype_QVVIVI,3) +// tag : V6_vgtub_xor +def int_hexagon_V6_vgtub_xor : +Hexagon_v64iv64iv512v512_Intrinsic<"HEXAGON_V6_vgtub_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_vgtub_xor_128B,QV_ftype_QVVIVI,3) +// tag : V6_vgtub_xor_128B +def int_hexagon_V6_vgtub_xor_128B : +Hexagon_v128iv128iv1024v1024_Intrinsic<"HEXAGON_V6_vgtub_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_or,QV_ftype_QVQV,2) +// tag : V6_pred_or +def int_hexagon_V6_pred_or : +Hexagon_v64iv64iv64i_Intrinsic<"HEXAGON_V6_pred_or">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_or_128B,QV_ftype_QVQV,2) +// tag : V6_pred_or_128B +def int_hexagon_V6_pred_or_128B : +Hexagon_v128iv128iv128i_Intrinsic<"HEXAGON_V6_pred_or_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_and,QV_ftype_QVQV,2) +// tag : V6_pred_and +def int_hexagon_V6_pred_and : +Hexagon_v64iv64iv64i_Intrinsic<"HEXAGON_V6_pred_and">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_and_128B,QV_ftype_QVQV,2) +// tag : V6_pred_and_128B +def int_hexagon_V6_pred_and_128B : +Hexagon_v128iv128iv128i_Intrinsic<"HEXAGON_V6_pred_and_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_not,QV_ftype_QV,1) +// tag : V6_pred_not +def int_hexagon_V6_pred_not : +Hexagon_v64iv64i_Intrinsic<"HEXAGON_V6_pred_not">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_not_128B,QV_ftype_QV,1) +// tag : V6_pred_not_128B +def int_hexagon_V6_pred_not_128B : +Hexagon_v128iv128i_Intrinsic<"HEXAGON_V6_pred_not_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_xor,QV_ftype_QVQV,2) +// tag : V6_pred_xor +def int_hexagon_V6_pred_xor : +Hexagon_v64iv64iv64i_Intrinsic<"HEXAGON_V6_pred_xor">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_xor_128B,QV_ftype_QVQV,2) +// tag : V6_pred_xor_128B +def int_hexagon_V6_pred_xor_128B : +Hexagon_v128iv128iv128i_Intrinsic<"HEXAGON_V6_pred_xor_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_and_n,QV_ftype_QVQV,2) +// tag : V6_pred_and_n +def int_hexagon_V6_pred_and_n : +Hexagon_v64iv64iv64i_Intrinsic<"HEXAGON_V6_pred_and_n">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_and_n_128B,QV_ftype_QVQV,2) +// tag : V6_pred_and_n_128B +def int_hexagon_V6_pred_and_n_128B : +Hexagon_v128iv128iv128i_Intrinsic<"HEXAGON_V6_pred_and_n_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_or_n,QV_ftype_QVQV,2) +// tag : V6_pred_or_n +def int_hexagon_V6_pred_or_n : +Hexagon_v64iv64iv64i_Intrinsic<"HEXAGON_V6_pred_or_n">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_or_n_128B,QV_ftype_QVQV,2) +// tag : V6_pred_or_n_128B +def int_hexagon_V6_pred_or_n_128B : +Hexagon_v128iv128iv128i_Intrinsic<"HEXAGON_V6_pred_or_n_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_scalar2,QV_ftype_SI,1) +// tag : V6_pred_scalar2 +def int_hexagon_V6_pred_scalar2 : +Hexagon_v64ii_Intrinsic<"HEXAGON_V6_pred_scalar2">; + +// +// BUILTIN_INFO(HEXAGON.V6_pred_scalar2_128B,QV_ftype_SI,1) +// tag : V6_pred_scalar2_128B +def int_hexagon_V6_pred_scalar2_128B : +Hexagon_v128ii_Intrinsic<"HEXAGON_V6_pred_scalar2_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmux,VI_ftype_QVVIVI,3) +// tag : V6_vmux +def int_hexagon_V6_vmux : +Hexagon_v512v64iv512v512_Intrinsic<"HEXAGON_V6_vmux">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmux_128B,VI_ftype_QVVIVI,3) +// tag : V6_vmux_128B +def int_hexagon_V6_vmux_128B : +Hexagon_v1024v128iv1024v1024_Intrinsic<"HEXAGON_V6_vmux_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vswap,VD_ftype_QVVIVI,3) +// tag : V6_vswap +def int_hexagon_V6_vswap : +Hexagon_v1024v64iv512v512_Intrinsic<"HEXAGON_V6_vswap">; + +// +// BUILTIN_INFO(HEXAGON.V6_vswap_128B,VD_ftype_QVVIVI,3) +// tag : V6_vswap_128B +def int_hexagon_V6_vswap_128B : +Hexagon_v2048v128iv1024v1024_Intrinsic<"HEXAGON_V6_vswap_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmaxub,VI_ftype_VIVI,2) +// tag : V6_vmaxub +def int_hexagon_V6_vmaxub : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmaxub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmaxub_128B,VI_ftype_VIVI,2) +// tag : V6_vmaxub_128B +def int_hexagon_V6_vmaxub_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmaxub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vminub,VI_ftype_VIVI,2) +// tag : V6_vminub +def int_hexagon_V6_vminub : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vminub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vminub_128B,VI_ftype_VIVI,2) +// tag : V6_vminub_128B +def int_hexagon_V6_vminub_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vminub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmaxuh,VI_ftype_VIVI,2) +// tag : V6_vmaxuh +def int_hexagon_V6_vmaxuh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmaxuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmaxuh_128B,VI_ftype_VIVI,2) +// tag : V6_vmaxuh_128B +def int_hexagon_V6_vmaxuh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmaxuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vminuh,VI_ftype_VIVI,2) +// tag : V6_vminuh +def int_hexagon_V6_vminuh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vminuh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vminuh_128B,VI_ftype_VIVI,2) +// tag : V6_vminuh_128B +def int_hexagon_V6_vminuh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vminuh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmaxh,VI_ftype_VIVI,2) +// tag : V6_vmaxh +def int_hexagon_V6_vmaxh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmaxh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmaxh_128B,VI_ftype_VIVI,2) +// tag : V6_vmaxh_128B +def int_hexagon_V6_vmaxh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmaxh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vminh,VI_ftype_VIVI,2) +// tag : V6_vminh +def int_hexagon_V6_vminh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vminh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vminh_128B,VI_ftype_VIVI,2) +// tag : V6_vminh_128B +def int_hexagon_V6_vminh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vminh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmaxw,VI_ftype_VIVI,2) +// tag : V6_vmaxw +def int_hexagon_V6_vmaxw : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vmaxw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vmaxw_128B,VI_ftype_VIVI,2) +// tag : V6_vmaxw_128B +def int_hexagon_V6_vmaxw_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vmaxw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vminw,VI_ftype_VIVI,2) +// tag : V6_vminw +def int_hexagon_V6_vminw : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vminw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vminw_128B,VI_ftype_VIVI,2) +// tag : V6_vminw_128B +def int_hexagon_V6_vminw_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vminw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsathub,VI_ftype_VIVI,2) +// tag : V6_vsathub +def int_hexagon_V6_vsathub : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsathub">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsathub_128B,VI_ftype_VIVI,2) +// tag : V6_vsathub_128B +def int_hexagon_V6_vsathub_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsathub_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsatwh,VI_ftype_VIVI,2) +// tag : V6_vsatwh +def int_hexagon_V6_vsatwh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vsatwh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vsatwh_128B,VI_ftype_VIVI,2) +// tag : V6_vsatwh_128B +def int_hexagon_V6_vsatwh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vsatwh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffeb,VI_ftype_VIVI,2) +// tag : V6_vshuffeb +def int_hexagon_V6_vshuffeb : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vshuffeb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffeb_128B,VI_ftype_VIVI,2) +// tag : V6_vshuffeb_128B +def int_hexagon_V6_vshuffeb_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vshuffeb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffob,VI_ftype_VIVI,2) +// tag : V6_vshuffob +def int_hexagon_V6_vshuffob : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vshuffob">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffob_128B,VI_ftype_VIVI,2) +// tag : V6_vshuffob_128B +def int_hexagon_V6_vshuffob_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vshuffob_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshufeh,VI_ftype_VIVI,2) +// tag : V6_vshufeh +def int_hexagon_V6_vshufeh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vshufeh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshufeh_128B,VI_ftype_VIVI,2) +// tag : V6_vshufeh_128B +def int_hexagon_V6_vshufeh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vshufeh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshufoh,VI_ftype_VIVI,2) +// tag : V6_vshufoh +def int_hexagon_V6_vshufoh : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vshufoh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshufoh_128B,VI_ftype_VIVI,2) +// tag : V6_vshufoh_128B +def int_hexagon_V6_vshufoh_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vshufoh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffvdd,VD_ftype_VIVISI,3) +// tag : V6_vshuffvdd +def int_hexagon_V6_vshuffvdd : +Hexagon_v1024v512v512i_Intrinsic<"HEXAGON_V6_vshuffvdd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffvdd_128B,VD_ftype_VIVISI,3) +// tag : V6_vshuffvdd_128B +def int_hexagon_V6_vshuffvdd_128B : +Hexagon_v2048v1024v1024i_Intrinsic<"HEXAGON_V6_vshuffvdd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdealvdd,VD_ftype_VIVISI,3) +// tag : V6_vdealvdd +def int_hexagon_V6_vdealvdd : +Hexagon_v1024v512v512i_Intrinsic<"HEXAGON_V6_vdealvdd">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdealvdd_128B,VD_ftype_VIVISI,3) +// tag : V6_vdealvdd_128B +def int_hexagon_V6_vdealvdd_128B : +Hexagon_v2048v1024v1024i_Intrinsic<"HEXAGON_V6_vdealvdd_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshufoeh,VD_ftype_VIVI,2) +// tag : V6_vshufoeh +def int_hexagon_V6_vshufoeh : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vshufoeh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshufoeh_128B,VD_ftype_VIVI,2) +// tag : V6_vshufoeh_128B +def int_hexagon_V6_vshufoeh_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vshufoeh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshufoeb,VD_ftype_VIVI,2) +// tag : V6_vshufoeb +def int_hexagon_V6_vshufoeb : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vshufoeb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshufoeb_128B,VD_ftype_VIVI,2) +// tag : V6_vshufoeb_128B +def int_hexagon_V6_vshufoeb_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vshufoeb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdealh,VI_ftype_VI,1) +// tag : V6_vdealh +def int_hexagon_V6_vdealh : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vdealh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdealh_128B,VI_ftype_VI,1) +// tag : V6_vdealh_128B +def int_hexagon_V6_vdealh_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vdealh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdealb,VI_ftype_VI,1) +// tag : V6_vdealb +def int_hexagon_V6_vdealb : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vdealb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdealb_128B,VI_ftype_VI,1) +// tag : V6_vdealb_128B +def int_hexagon_V6_vdealb_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vdealb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdealb4w,VI_ftype_VIVI,2) +// tag : V6_vdealb4w +def int_hexagon_V6_vdealb4w : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vdealb4w">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdealb4w_128B,VI_ftype_VIVI,2) +// tag : V6_vdealb4w_128B +def int_hexagon_V6_vdealb4w_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vdealb4w_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffh,VI_ftype_VI,1) +// tag : V6_vshuffh +def int_hexagon_V6_vshuffh : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vshuffh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffh_128B,VI_ftype_VI,1) +// tag : V6_vshuffh_128B +def int_hexagon_V6_vshuffh_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vshuffh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffb,VI_ftype_VI,1) +// tag : V6_vshuffb +def int_hexagon_V6_vshuffb : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vshuffb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vshuffb_128B,VI_ftype_VI,1) +// tag : V6_vshuffb_128B +def int_hexagon_V6_vshuffb_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vshuffb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_extractw,SI_ftype_VISI,2) +// tag : V6_extractw +def int_hexagon_V6_extractw : +Hexagon_iv512i_Intrinsic<"HEXAGON_V6_extractw">; + +// +// BUILTIN_INFO(HEXAGON.V6_extractw_128B,SI_ftype_VISI,2) +// tag : V6_extractw_128B +def int_hexagon_V6_extractw_128B : +Hexagon_iv1024i_Intrinsic<"HEXAGON_V6_extractw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vinsertwr,VI_ftype_VISI,2) +// tag : V6_vinsertwr +def int_hexagon_V6_vinsertwr : +Hexagon_v512v512i_Intrinsic<"HEXAGON_V6_vinsertwr">; + +// +// BUILTIN_INFO(HEXAGON.V6_vinsertwr_128B,VI_ftype_VISI,2) +// tag : V6_vinsertwr_128B +def int_hexagon_V6_vinsertwr_128B : +Hexagon_v1024v1024i_Intrinsic<"HEXAGON_V6_vinsertwr_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_lvsplatw,VI_ftype_SI,1) +// tag : V6_lvsplatw +def int_hexagon_V6_lvsplatw : +Hexagon_v512i_Intrinsic<"HEXAGON_V6_lvsplatw">; + +// +// BUILTIN_INFO(HEXAGON.V6_lvsplatw_128B,VI_ftype_SI,1) +// tag : V6_lvsplatw_128B +def int_hexagon_V6_lvsplatw_128B : +Hexagon_v1024i_Intrinsic<"HEXAGON_V6_lvsplatw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vassign,VI_ftype_VI,1) +// tag : V6_vassign +def int_hexagon_V6_vassign : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vassign">; + +// +// BUILTIN_INFO(HEXAGON.V6_vassign_128B,VI_ftype_VI,1) +// tag : V6_vassign_128B +def int_hexagon_V6_vassign_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vassign_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vcombine,VD_ftype_VIVI,2) +// tag : V6_vcombine +def int_hexagon_V6_vcombine : +Hexagon_v1024v512v512_Intrinsic<"HEXAGON_V6_vcombine">; + +// +// BUILTIN_INFO(HEXAGON.V6_vcombine_128B,VD_ftype_VIVI,2) +// tag : V6_vcombine_128B +def int_hexagon_V6_vcombine_128B : +Hexagon_v2048v1024v1024_Intrinsic<"HEXAGON_V6_vcombine_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutb,VI_ftype_VIDISI,3) +// tag : V6_vlutb +def int_hexagon_V6_vlutb : +Hexagon_v512v512LLii_Intrinsic<"HEXAGON_V6_vlutb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutb_128B,VI_ftype_VIDISI,3) +// tag : V6_vlutb_128B +def int_hexagon_V6_vlutb_128B : +Hexagon_v1024v1024LLii_Intrinsic<"HEXAGON_V6_vlutb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutb_acc,VI_ftype_VIVIDISI,4) +// tag : V6_vlutb_acc +def int_hexagon_V6_vlutb_acc : +Hexagon_v512v512v512LLii_Intrinsic<"HEXAGON_V6_vlutb_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutb_acc_128B,VI_ftype_VIVIDISI,4) +// tag : V6_vlutb_acc_128B +def int_hexagon_V6_vlutb_acc_128B : +Hexagon_v1024v1024v1024LLii_Intrinsic<"HEXAGON_V6_vlutb_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutb_dv,VD_ftype_VDDISI,3) +// tag : V6_vlutb_dv +def int_hexagon_V6_vlutb_dv : +Hexagon_v1024v1024LLii_Intrinsic<"HEXAGON_V6_vlutb_dv">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutb_dv_128B,VD_ftype_VDDISI,3) +// tag : V6_vlutb_dv_128B +def int_hexagon_V6_vlutb_dv_128B : +Hexagon_v2048v2048LLii_Intrinsic<"HEXAGON_V6_vlutb_dv_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutb_dv_acc,VD_ftype_VDVDDISI,4) +// tag : V6_vlutb_dv_acc +def int_hexagon_V6_vlutb_dv_acc : +Hexagon_v1024v1024v1024LLii_Intrinsic<"HEXAGON_V6_vlutb_dv_acc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutb_dv_acc_128B,VD_ftype_VDVDDISI,4) +// tag : V6_vlutb_dv_acc_128B +def int_hexagon_V6_vlutb_dv_acc_128B : +Hexagon_v2048v2048v2048LLii_Intrinsic<"HEXAGON_V6_vlutb_dv_acc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdelta,VI_ftype_VIVI,2) +// tag : V6_vdelta +def int_hexagon_V6_vdelta : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vdelta">; + +// +// BUILTIN_INFO(HEXAGON.V6_vdelta_128B,VI_ftype_VIVI,2) +// tag : V6_vdelta_128B +def int_hexagon_V6_vdelta_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vdelta_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrdelta,VI_ftype_VIVI,2) +// tag : V6_vrdelta +def int_hexagon_V6_vrdelta : +Hexagon_v512v512v512_Intrinsic<"HEXAGON_V6_vrdelta">; + +// +// BUILTIN_INFO(HEXAGON.V6_vrdelta_128B,VI_ftype_VIVI,2) +// tag : V6_vrdelta_128B +def int_hexagon_V6_vrdelta_128B : +Hexagon_v1024v1024v1024_Intrinsic<"HEXAGON_V6_vrdelta_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vcl0w,VI_ftype_VI,1) +// tag : V6_vcl0w +def int_hexagon_V6_vcl0w : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vcl0w">; + +// +// BUILTIN_INFO(HEXAGON.V6_vcl0w_128B,VI_ftype_VI,1) +// tag : V6_vcl0w_128B +def int_hexagon_V6_vcl0w_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vcl0w_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vcl0h,VI_ftype_VI,1) +// tag : V6_vcl0h +def int_hexagon_V6_vcl0h : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vcl0h">; + +// +// BUILTIN_INFO(HEXAGON.V6_vcl0h_128B,VI_ftype_VI,1) +// tag : V6_vcl0h_128B +def int_hexagon_V6_vcl0h_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vcl0h_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnormamtw,VI_ftype_VI,1) +// tag : V6_vnormamtw +def int_hexagon_V6_vnormamtw : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vnormamtw">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnormamtw_128B,VI_ftype_VI,1) +// tag : V6_vnormamtw_128B +def int_hexagon_V6_vnormamtw_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vnormamtw_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnormamth,VI_ftype_VI,1) +// tag : V6_vnormamth +def int_hexagon_V6_vnormamth : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vnormamth">; + +// +// BUILTIN_INFO(HEXAGON.V6_vnormamth_128B,VI_ftype_VI,1) +// tag : V6_vnormamth_128B +def int_hexagon_V6_vnormamth_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vnormamth_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpopcounth,VI_ftype_VI,1) +// tag : V6_vpopcounth +def int_hexagon_V6_vpopcounth : +Hexagon_v512v512_Intrinsic<"HEXAGON_V6_vpopcounth">; + +// +// BUILTIN_INFO(HEXAGON.V6_vpopcounth_128B,VI_ftype_VI,1) +// tag : V6_vpopcounth_128B +def int_hexagon_V6_vpopcounth_128B : +Hexagon_v1024v1024_Intrinsic<"HEXAGON_V6_vpopcounth_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutvvb,VI_ftype_VIVISI,3) +// tag : V6_vlutvvb +def int_hexagon_V6_vlutvvb : +Hexagon_v512v512v512i_Intrinsic<"HEXAGON_V6_vlutvvb">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutvvb_128B,VI_ftype_VIVISI,3) +// tag : V6_vlutvvb_128B +def int_hexagon_V6_vlutvvb_128B : +Hexagon_v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vlutvvb_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutvvb_oracc,VI_ftype_VIVIVISI,4) +// tag : V6_vlutvvb_oracc +def int_hexagon_V6_vlutvvb_oracc : +Hexagon_v512v512v512v512i_Intrinsic<"HEXAGON_V6_vlutvvb_oracc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutvvb_oracc_128B,VI_ftype_VIVIVISI,4) +// tag : V6_vlutvvb_oracc_128B +def int_hexagon_V6_vlutvvb_oracc_128B : +Hexagon_v1024v1024v1024v1024i_Intrinsic<"HEXAGON_V6_vlutvvb_oracc_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutvwh,VD_ftype_VIVISI,3) +// tag : V6_vlutvwh +def int_hexagon_V6_vlutvwh : +Hexagon_v1024v512v512i_Intrinsic<"HEXAGON_V6_vlutvwh">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutvwh_128B,VD_ftype_VIVISI,3) +// tag : V6_vlutvwh_128B +def int_hexagon_V6_vlutvwh_128B : +Hexagon_v2048v1024v1024i_Intrinsic<"HEXAGON_V6_vlutvwh_128B">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutvwh_oracc,VD_ftype_VDVIVISI,4) +// tag : V6_vlutvwh_oracc +def int_hexagon_V6_vlutvwh_oracc : +Hexagon_v1024v1024v512v512i_Intrinsic<"HEXAGON_V6_vlutvwh_oracc">; + +// +// BUILTIN_INFO(HEXAGON.V6_vlutvwh_oracc_128B,VD_ftype_VDVIVISI,4) +// tag : V6_vlutvwh_oracc_128B +def int_hexagon_V6_vlutvwh_oracc_128B : +Hexagon_v2048v2048v1024v1024i_Intrinsic<"HEXAGON_V6_vlutvwh_oracc_128B">; + +// +// BUILTIN_INFO(HEXAGON.M6_vabsdiffb,DI_ftype_DIDI,2) +// tag : M6_vabsdiffb +def int_hexagon_M6_vabsdiffb : +Hexagon_LLiLLiLLi_Intrinsic<"HEXAGON_M6_vabsdiffb">; + +// +// BUILTIN_INFO(HEXAGON.M6_vabsdiffub,DI_ftype_DIDI,2) +// tag : M6_vabsdiffub +def int_hexagon_M6_vabsdiffub : +Hexagon_LLiLLiLLi_Intrinsic<"HEXAGON_M6_vabsdiffub">; + +// +// BUILTIN_INFO(HEXAGON.S6_vsplatrbp,DI_ftype_SI,1) +// tag : S6_vsplatrbp +def int_hexagon_S6_vsplatrbp : +Hexagon_LLii_Intrinsic<"HEXAGON_S6_vsplatrbp">; + +// +// BUILTIN_INFO(HEXAGON.S6_vtrunehb_ppp,DI_ftype_DIDI,2) +// tag : S6_vtrunehb_ppp +def int_hexagon_S6_vtrunehb_ppp : +Hexagon_LLiLLiLLi_Intrinsic<"HEXAGON_S6_vtrunehb_ppp">; + +// +// BUILTIN_INFO(HEXAGON.S6_vtrunohb_ppp,DI_ftype_DIDI,2) +// tag : S6_vtrunohb_ppp +def int_hexagon_S6_vtrunohb_ppp : +Hexagon_LLiLLiLLi_Intrinsic<"HEXAGON_S6_vtrunohb_ppp">; diff --git a/include/llvm/IR/IntrinsicsPowerPC.td b/include/llvm/IR/IntrinsicsPowerPC.td index eb8f1e6cd079..06dfc329fe32 100644 --- a/include/llvm/IR/IntrinsicsPowerPC.td +++ b/include/llvm/IR/IntrinsicsPowerPC.td @@ -710,21 +710,39 @@ def int_ppc_vsx_xvrsqrtedp : GCCBuiltin<"__builtin_vsx_xvrsqrtedp">, def int_ppc_vsx_xvcmpeqdp : PowerPC_VSX_Intrinsic<"xvcmpeqdp", [llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; +def int_ppc_vsx_xvcmpeqdp_p : GCCBuiltin<"__builtin_vsx_xvcmpeqdp_p">, + Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2f64_ty,llvm_v2f64_ty], + [IntrNoMem]>; def int_ppc_vsx_xvcmpeqsp : PowerPC_VSX_Intrinsic<"xvcmpeqsp", [llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; +def int_ppc_vsx_xvcmpeqsp_p : GCCBuiltin<"__builtin_vsx_xvcmpeqsp_p">, + Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], + [IntrNoMem]>; def int_ppc_vsx_xvcmpgedp : PowerPC_VSX_Intrinsic<"xvcmpgedp", [llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; +def int_ppc_vsx_xvcmpgedp_p : GCCBuiltin<"__builtin_vsx_xvcmpgedp_p">, + Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2f64_ty,llvm_v2f64_ty], + [IntrNoMem]>; def int_ppc_vsx_xvcmpgesp : PowerPC_VSX_Intrinsic<"xvcmpgesp", [llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; +def int_ppc_vsx_xvcmpgesp_p : GCCBuiltin<"__builtin_vsx_xvcmpgesp_p">, + Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], + [IntrNoMem]>; def int_ppc_vsx_xvcmpgtdp : PowerPC_VSX_Intrinsic<"xvcmpgtdp", [llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; +def int_ppc_vsx_xvcmpgtdp_p : GCCBuiltin<"__builtin_vsx_xvcmpgtdp_p">, + Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v2f64_ty,llvm_v2f64_ty], + [IntrNoMem]>; def int_ppc_vsx_xvcmpgtsp : PowerPC_VSX_Intrinsic<"xvcmpgtsp", [llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; +def int_ppc_vsx_xvcmpgtsp_p : GCCBuiltin<"__builtin_vsx_xvcmpgtsp_p">, + Intrinsic<[llvm_i32_ty],[llvm_i32_ty,llvm_v4f32_ty,llvm_v4f32_ty], + [IntrNoMem]>; def int_ppc_vsx_xxleqv : PowerPC_VSX_Intrinsic<"xxleqv", [llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; diff --git a/include/llvm/IR/IntrinsicsWebAssembly.td b/include/llvm/IR/IntrinsicsWebAssembly.td index 3ccde4742384..3953aef43dad 100644 --- a/include/llvm/IR/IntrinsicsWebAssembly.td +++ b/include/llvm/IR/IntrinsicsWebAssembly.td @@ -13,4 +13,10 @@ //===----------------------------------------------------------------------===// let TargetPrefix = "wasm" in { // All intrinsics start with "llvm.wasm.". + +// Note that memory_size is not IntrNoMem because it must be sequenced with +// respect to grow_memory calls. +def int_wasm_memory_size : Intrinsic<[llvm_anyint_ty], [], [IntrReadMem]>; +def int_wasm_grow_memory : Intrinsic<[], [llvm_anyint_ty], []>; + } diff --git a/include/llvm/IR/IntrinsicsX86.td b/include/llvm/IR/IntrinsicsX86.td index a3bc4af84308..18390f853510 100644 --- a/include/llvm/IR/IntrinsicsX86.td +++ b/include/llvm/IR/IntrinsicsX86.td @@ -22,10 +22,8 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { def int_x86_seh_lsda : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty], [IntrNoMem]>; - // Restores the frame, base, and stack pointers as necessary after recovering - // from an exception. Any block resuming control flow in the parent function - // should call this before accessing any stack memory. - def int_x86_seh_restoreframe : Intrinsic<[], [], []>; + // Marks the EH registration node created in LLVM IR prior to code generation. + def int_x86_seh_ehregnode : Intrinsic<[], [llvm_ptr_ty], []>; // Given a pointer to the end of an EH registration object, returns the true // parent frame address that can be used with llvm.localrecover. @@ -1406,6 +1404,78 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpermil_pd_128 : + GCCBuiltin<"__builtin_ia32_vpermilpd_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermil_pd_256 : + GCCBuiltin<"__builtin_ia32_vpermilpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermil_pd_512 : + GCCBuiltin<"__builtin_ia32_vpermilpd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermil_ps_128 : + GCCBuiltin<"__builtin_ia32_vpermilps_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermil_ps_256 : + GCCBuiltin<"__builtin_ia32_vpermilps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermil_ps_512 : + GCCBuiltin<"__builtin_ia32_vpermilps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermilvar_pd_256 : + GCCBuiltin<"__builtin_ia32_vpermilvarpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermilvar_pd_512 : + GCCBuiltin<"__builtin_ia32_vpermilvarpd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermilvar_pd_128 : + GCCBuiltin<"__builtin_ia32_vpermilvarpd_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermilvar_ps_256 : + GCCBuiltin<"__builtin_ia32_vpermilvarps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermilvar_ps_512 : + GCCBuiltin<"__builtin_ia32_vpermilvarps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermilvar_ps_128 : + GCCBuiltin<"__builtin_ia32_vpermilvarps_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pshuf_b_128 : GCCBuiltin<"__builtin_ia32_pshufb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -1423,8 +1493,145 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; -} + def int_x86_avx512_mask_shuf_f32x4_256 : + GCCBuiltin<"__builtin_ia32_shuf_f32x4_256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_f32x4 : + GCCBuiltin<"__builtin_ia32_shuf_f32x4_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_f64x2_256 : + GCCBuiltin<"__builtin_ia32_shuf_f64x2_256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_f64x2 : + GCCBuiltin<"__builtin_ia32_shuf_f64x2_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_i32x4_256 : + GCCBuiltin<"__builtin_ia32_shuf_i32x4_256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_i32x4 : + GCCBuiltin<"__builtin_ia32_shuf_i32x4_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_i64x2_256 : + GCCBuiltin<"__builtin_ia32_shuf_i64x2_256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_i64x2 : + GCCBuiltin<"__builtin_ia32_shuf_i64x2_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_pd_128 : + GCCBuiltin<"__builtin_ia32_shufpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_pd_256 : + GCCBuiltin<"__builtin_ia32_shufpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_pd_512 : + GCCBuiltin<"__builtin_ia32_shufpd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_ps_128 : + GCCBuiltin<"__builtin_ia32_shufps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_ps_256 : + GCCBuiltin<"__builtin_ia32_shufps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_shuf_ps_512 : + GCCBuiltin<"__builtin_ia32_shufps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movshdup_128 : + GCCBuiltin<"__builtin_ia32_movshdup128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movshdup_256 : + GCCBuiltin<"__builtin_ia32_movshdup256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movshdup_512 : + GCCBuiltin<"__builtin_ia32_movshdup512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movsldup_128 : + GCCBuiltin<"__builtin_ia32_movsldup128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movsldup_256 : + GCCBuiltin<"__builtin_ia32_movsldup256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movsldup_512 : + GCCBuiltin<"__builtin_ia32_movsldup512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movddup_128 : + GCCBuiltin<"__builtin_ia32_movddup128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movddup_256 : + GCCBuiltin<"__builtin_ia32_movddup256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_movddup_512 : + GCCBuiltin<"__builtin_ia32_movddup512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; +} // Vector blend let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". @@ -1526,6 +1733,38 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_mask_ptestm_q_512 : GCCBuiltin<"__builtin_ia32_ptestmq512">, Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_fpclass_pd_128 : + GCCBuiltin<"__builtin_ia32_fpclasspd128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fpclass_pd_256 : + GCCBuiltin<"__builtin_ia32_fpclasspd256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4f64_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fpclass_pd_512 : + GCCBuiltin<"__builtin_ia32_fpclasspd512_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fpclass_ps_128 : + GCCBuiltin<"__builtin_ia32_fpclassps128_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fpclass_ps_256 : + GCCBuiltin<"__builtin_ia32_fpclassps256_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v8f32_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fpclass_ps_512 : + GCCBuiltin<"__builtin_ia32_fpclassps512_mask">, + Intrinsic<[llvm_i16_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fpclass_sd : + GCCBuiltin<"__builtin_ia32_fpclasssd">, + Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fpclass_ss : + GCCBuiltin<"__builtin_ia32_fpclassss">, + Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; } // Vector extract sign mask @@ -1573,16 +1812,16 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Conditional load ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_maskload_pd : GCCBuiltin<"__builtin_ia32_maskloadpd">, - Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty, llvm_v2f64_ty], + Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty, llvm_v2i64_ty], [IntrReadArgMem]>; def int_x86_avx_maskload_ps : GCCBuiltin<"__builtin_ia32_maskloadps">, - Intrinsic<[llvm_v4f32_ty], [llvm_ptr_ty, llvm_v4f32_ty], + Intrinsic<[llvm_v4f32_ty], [llvm_ptr_ty, llvm_v4i32_ty], [IntrReadArgMem]>; def int_x86_avx_maskload_pd_256 : GCCBuiltin<"__builtin_ia32_maskloadpd256">, - Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty, llvm_v4f64_ty], + Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty, llvm_v4i64_ty], [IntrReadArgMem]>; def int_x86_avx_maskload_ps_256 : GCCBuiltin<"__builtin_ia32_maskloadps256">, - Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty, llvm_v8f32_ty], + Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty, llvm_v8i32_ty], [IntrReadArgMem]>; def int_x86_avx512_mask_loadu_ps_512 : GCCBuiltin<"__builtin_ia32_loadups512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], @@ -1596,24 +1835,31 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_mask_load_pd_512 : GCCBuiltin<"__builtin_ia32_loadapd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrReadArgMem]>; + + def int_x86_avx512_mask_move_ss : GCCBuiltin<"__builtin_ia32_movss_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_move_sd : GCCBuiltin<"__builtin_ia32_movsd_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; } // Conditional store ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_maskstore_pd : GCCBuiltin<"__builtin_ia32_maskstorepd">, Intrinsic<[], [llvm_ptr_ty, - llvm_v2f64_ty, llvm_v2f64_ty], [IntrReadWriteArgMem]>; + llvm_v2i64_ty, llvm_v2f64_ty], [IntrReadWriteArgMem]>; def int_x86_avx_maskstore_ps : GCCBuiltin<"__builtin_ia32_maskstoreps">, Intrinsic<[], [llvm_ptr_ty, - llvm_v4f32_ty, llvm_v4f32_ty], [IntrReadWriteArgMem]>; + llvm_v4i32_ty, llvm_v4f32_ty], [IntrReadWriteArgMem]>; def int_x86_avx_maskstore_pd_256 : GCCBuiltin<"__builtin_ia32_maskstorepd256">, Intrinsic<[], [llvm_ptr_ty, - llvm_v4f64_ty, llvm_v4f64_ty], [IntrReadWriteArgMem]>; + llvm_v4i64_ty, llvm_v4f64_ty], [IntrReadWriteArgMem]>; def int_x86_avx_maskstore_ps_256 : GCCBuiltin<"__builtin_ia32_maskstoreps256">, Intrinsic<[], [llvm_ptr_ty, - llvm_v8f32_ty, llvm_v8f32_ty], [IntrReadWriteArgMem]>; + llvm_v8i32_ty, llvm_v8f32_ty], [IntrReadWriteArgMem]>; def int_x86_avx512_mask_storeu_ps_512 : GCCBuiltin<"__builtin_ia32_storeups512_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], @@ -1946,6 +2192,25 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrl_w_128 : GCCBuiltin<"__builtin_ia32_psrlw128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrl_w_256 : GCCBuiltin<"__builtin_ia32_psrlw256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + llvm_v8i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrl_w_512 : GCCBuiltin<"__builtin_ia32_psrlw512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + llvm_v8i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrl_wi_128 : GCCBuiltin<"__builtin_ia32_psrlwi128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrl_wi_256 : GCCBuiltin<"__builtin_ia32_psrlwi256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psrl_wi_512 : GCCBuiltin<"__builtin_ia32_psrlwi512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_psll_d : GCCBuiltin<"__builtin_ia32_pslld512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; @@ -2167,39 +2432,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector load with broadcast let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_vbroadcast_ss_ps : - GCCBuiltin<"__builtin_ia32_vbroadcastss_ps">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx2_vbroadcast_sd_pd_256 : - GCCBuiltin<"__builtin_ia32_vbroadcastsd_pd256">, - Intrinsic<[llvm_v4f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_avx2_vbroadcast_ss_ps_256 : - GCCBuiltin<"__builtin_ia32_vbroadcastss_ps256">, - Intrinsic<[llvm_v8f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx2_pbroadcastb_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastb128">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_avx2_pbroadcastb_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastb256">, - Intrinsic<[llvm_v32i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_avx2_pbroadcastw_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx2_pbroadcastw_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v8i16_ty], [IntrNoMem]>; - def int_x86_avx2_pbroadcastd_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastd128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx2_pbroadcastd_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastd256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; - def int_x86_avx2_pbroadcastq_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastq128">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; - def int_x86_avx2_pbroadcastq_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastq256">, - Intrinsic<[llvm_v4i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; def int_x86_avx512_mask_pbroadcast_d_gpr_512 : GCCBuiltin<"__builtin_ia32_pbroadcastd512_gpr_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_i32_ty, llvm_v16i32_ty, @@ -2220,7 +2452,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx2_permps : GCCBuiltin<"__builtin_ia32_permvarsf256">, - Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty], + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx2_vperm2i128 : GCCBuiltin<"__builtin_ia32_permti256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, @@ -2231,20 +2463,124 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_mask_vextractf32x4_512 : GCCBuiltin<"__builtin_ia32_extractf32x4_mask">, - Intrinsic<[llvm_v4f32_ty], [llvm_v16f32_ty, llvm_i8_ty, - llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4f32_ty], [llvm_v16f32_ty, llvm_i32_ty, + llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_vextracti32x4_512 : GCCBuiltin<"__builtin_ia32_extracti32x4_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v16i32_ty, llvm_i8_ty, - llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v16i32_ty, llvm_i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextractf32x4_256 : + GCCBuiltin<"__builtin_ia32_extractf32x4_256_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v8f32_ty, llvm_i32_ty, + llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextracti32x4_256 : + GCCBuiltin<"__builtin_ia32_extracti32x4_256_mask">, + Intrinsic<[llvm_v4i32_ty], [llvm_v8i32_ty, llvm_i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextractf64x2_256 : + GCCBuiltin<"__builtin_ia32_extractf64x2_256_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v4f64_ty, llvm_i32_ty, + llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextracti64x2_256 : + GCCBuiltin<"__builtin_ia32_extracti64x2_256_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v4i64_ty, llvm_i32_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextractf64x2_512 : + GCCBuiltin<"__builtin_ia32_extractf64x2_512_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v8f64_ty, llvm_i32_ty, + llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextracti64x2_512 : + GCCBuiltin<"__builtin_ia32_extracti64x2_512_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v8i64_ty, llvm_i32_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextractf32x8_512 : + GCCBuiltin<"__builtin_ia32_extractf32x8_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v16f32_ty, llvm_i32_ty, + llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vextracti32x8_512 : + GCCBuiltin<"__builtin_ia32_extracti32x8_mask">, + Intrinsic<[llvm_v8i32_ty],[llvm_v16i32_ty, llvm_i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_vextractf64x4_512 : GCCBuiltin<"__builtin_ia32_extractf64x4_mask">, - Intrinsic<[llvm_v4f64_ty], [llvm_v8f64_ty, llvm_i8_ty, - llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4f64_ty], [llvm_v8f64_ty, llvm_i32_ty, + llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_vextracti64x4_512 : GCCBuiltin<"__builtin_ia32_extracti64x4_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v8i64_ty, llvm_i8_ty, - llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i64_ty], [llvm_v8i64_ty, llvm_i32_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_insertf32x4_256 : + GCCBuiltin<"__builtin_ia32_insertf32x4_256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_insertf32x4_512 : + GCCBuiltin<"__builtin_ia32_insertf32x4_512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_insertf32x8_512 : + GCCBuiltin<"__builtin_ia32_insertf32x8_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v8f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_insertf64x2_256 : + GCCBuiltin<"__builtin_ia32_insertf64x2_256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_insertf64x2_512 : + GCCBuiltin<"__builtin_ia32_insertf64x2_512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_insertf64x4_512 : + GCCBuiltin<"__builtin_ia32_insertf64x4_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v4f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_inserti32x4_256 : + GCCBuiltin<"__builtin_ia32_inserti32x4_256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_inserti32x4_512 : + GCCBuiltin<"__builtin_ia32_inserti32x4_512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_v16i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_inserti32x8_512 : + GCCBuiltin<"__builtin_ia32_inserti32x8_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_inserti64x2_256 : + GCCBuiltin<"__builtin_ia32_inserti64x2_256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_inserti64x2_512 : + GCCBuiltin<"__builtin_ia32_inserti64x2_512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_inserti64x4_512 : + GCCBuiltin<"__builtin_ia32_inserti64x4_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; } // Conditional load ops @@ -2354,6 +2690,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_psll_dq_512 : GCCBuiltin<"__builtin_ia32_pslldq512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_psrl_dq_512 : GCCBuiltin<"__builtin_ia32_psrldq512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i32_ty], + [IntrNoMem]>; } // Gather ops @@ -3545,6 +3887,43 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". } //===----------------------------------------------------------------------===// +// XSAVE +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_xsave : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xsave64 : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xrstor : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xrstor64 : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xsaveopt : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xsaveopt64 : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xrstors : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xrstors64 : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xsavec : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xsavec64 : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xsaves : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; + def int_x86_xsaves64 : + Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], []>; +} + +//===----------------------------------------------------------------------===// +// Support protection key +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_rdpkru : GCCBuiltin <"__builtin_ia32_rdpkru">, + Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; + def int_x86_wrpkru : GCCBuiltin<"__builtin_ia32_wrpkru">, + Intrinsic<[], [llvm_i32_ty], [IntrNoMem]>; +} +//===----------------------------------------------------------------------===// // Half float conversion let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". @@ -3561,9 +3940,21 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_mask_vcvtph2ps_512 : GCCBuiltin<"__builtin_ia32_vcvtph2ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16i16_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vcvtph2ps_256 : GCCBuiltin<"__builtin_ia32_vcvtph2ps256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8i16_ty, llvm_v8f32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vcvtph2ps_128 : GCCBuiltin<"__builtin_ia32_vcvtph2ps_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v8i16_ty, llvm_v4f32_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_vcvtps2ph_512 : GCCBuiltin<"__builtin_ia32_vcvtps2ph512_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vcvtps2ph_256 : GCCBuiltin<"__builtin_ia32_vcvtps2ph256_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty, llvm_i32_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vcvtps2ph_128 : GCCBuiltin<"__builtin_ia32_vcvtps2ph_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_i32_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; } //===----------------------------------------------------------------------===// @@ -3657,6 +4048,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_kunpck_bw : GCCBuiltin<"__builtin_ia32_kunpckhi">, Intrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_kunpck_wd : GCCBuiltin<"__builtin_ia32_kunpcksi">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_kunpck_dq : GCCBuiltin<"__builtin_ia32_kunpckdi">, + Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem]>; def int_x86_avx512_kortestz_w : GCCBuiltin<"__builtin_ia32_kortestzhi">, Intrinsic<[llvm_i32_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; @@ -3671,10 +4068,14 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; def int_x86_avx512_cvtss2usi64 : GCCBuiltin<"__builtin_ia32_cvtss2usi64">, Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx512_cvttss2usi : GCCBuiltin<"__builtin_ia32_cvttss2usi">, - Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx512_cvttss2usi64 : GCCBuiltin<"__builtin_ia32_cvttss2usi64">, - Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty], [IntrNoMem]>; + def int_x86_avx512_cvttss2si : GCCBuiltin<"__builtin_ia32_vcvttss2si32">, + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvttss2si64 : GCCBuiltin<"__builtin_ia32_vcvttss2si64">, + Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvttss2usi : GCCBuiltin<"__builtin_ia32_vcvttss2usi32">, + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvttss2usi64 : GCCBuiltin<"__builtin_ia32_vcvttss2usi64">, + Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtusi2ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss32">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; @@ -3686,10 +4087,14 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; def int_x86_avx512_cvtsd2usi64 : GCCBuiltin<"__builtin_ia32_cvtsd2usi64">, Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_avx512_cvttsd2usi : GCCBuiltin<"__builtin_ia32_cvttsd2usi">, - Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_avx512_cvttsd2usi64 : GCCBuiltin<"__builtin_ia32_cvttsd2usi64">, - Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; + def int_x86_avx512_cvttsd2si : GCCBuiltin<"__builtin_ia32_vcvttsd2si32">, + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvttsd2si64 : GCCBuiltin<"__builtin_ia32_vcvttsd2si64">, + Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvttsd2usi : GCCBuiltin<"__builtin_ia32_vcvttsd2usi32">, + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvttsd2usi64 : GCCBuiltin<"__builtin_ia32_vcvttsd2usi64">, + Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtusi2sd : GCCBuiltin<"__builtin_ia32_cvtusi2sd32">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; @@ -3698,17 +4103,74 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtsi2ss32 : GCCBuiltin<"__builtin_ia32_cvtsi2ss32">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, + llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtsi2ss64 : GCCBuiltin<"__builtin_ia32_cvtsi2ss64">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, + llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtsi2sd32 : GCCBuiltin<"__builtin_ia32_cvtsi2sd32">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, + llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtsi2sd64 : GCCBuiltin<"__builtin_ia32_cvtsi2sd64">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, - llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, + llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_cvtb2mask_128 : GCCBuiltin<"__builtin_ia32_cvtb2mask128">, + Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty], [IntrNoMem]>; + def int_x86_avx512_cvtb2mask_256 : GCCBuiltin<"__builtin_ia32_cvtb2mask256">, + Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty], [IntrNoMem]>; + def int_x86_avx512_cvtb2mask_512 : GCCBuiltin<"__builtin_ia32_cvtb2mask512">, + Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty], [IntrNoMem]>; + + def int_x86_avx512_cvtw2mask_128 : GCCBuiltin<"__builtin_ia32_cvtw2mask128">, + Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty], [IntrNoMem]>; + def int_x86_avx512_cvtw2mask_256 : GCCBuiltin<"__builtin_ia32_cvtw2mask256">, + Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty], [IntrNoMem]>; + def int_x86_avx512_cvtw2mask_512 : GCCBuiltin<"__builtin_ia32_cvtw2mask512">, + Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty], [IntrNoMem]>; + + def int_x86_avx512_cvtd2mask_128 : GCCBuiltin<"__builtin_ia32_cvtd2mask128">, + Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvtd2mask_256 : GCCBuiltin<"__builtin_ia32_cvtd2mask256">, + Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvtd2mask_512 : GCCBuiltin<"__builtin_ia32_cvtd2mask512">, + Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty], [IntrNoMem]>; + + def int_x86_avx512_cvtq2mask_128 : GCCBuiltin<"__builtin_ia32_cvtq2mask128">, + Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty], [IntrNoMem]>; + def int_x86_avx512_cvtq2mask_256 : GCCBuiltin<"__builtin_ia32_cvtq2mask256">, + Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty], [IntrNoMem]>; + def int_x86_avx512_cvtq2mask_512 : GCCBuiltin<"__builtin_ia32_cvtq2mask512">, + Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty], [IntrNoMem]>; + + def int_x86_avx512_cvtmask2b_128 : GCCBuiltin<"__builtin_ia32_cvtmask2b128">, + Intrinsic<[llvm_v16i8_ty], [llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_cvtmask2b_256 : GCCBuiltin<"__builtin_ia32_cvtmask2b256">, + Intrinsic<[llvm_v32i8_ty], [llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_cvtmask2b_512 : GCCBuiltin<"__builtin_ia32_cvtmask2b512">, + Intrinsic<[llvm_v64i8_ty], [llvm_i64_ty], [IntrNoMem]>; + + def int_x86_avx512_cvtmask2w_128 : GCCBuiltin<"__builtin_ia32_cvtmask2w128">, + Intrinsic<[llvm_v8i16_ty], [llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_cvtmask2w_256 : GCCBuiltin<"__builtin_ia32_cvtmask2w256">, + Intrinsic<[llvm_v16i16_ty], [llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_cvtmask2w_512 : GCCBuiltin<"__builtin_ia32_cvtmask2w512">, + Intrinsic<[llvm_v32i16_ty], [llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_cvtmask2d_128 : GCCBuiltin<"__builtin_ia32_cvtmask2d128">, + Intrinsic<[llvm_v4i32_ty], [llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_cvtmask2d_256 : GCCBuiltin<"__builtin_ia32_cvtmask2d256">, + Intrinsic<[llvm_v8i32_ty], [llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_cvtmask2d_512 : GCCBuiltin<"__builtin_ia32_cvtmask2d512">, + Intrinsic<[llvm_v16i32_ty], [llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_cvtmask2q_128 : GCCBuiltin<"__builtin_ia32_cvtmask2q128">, + Intrinsic<[llvm_v2i64_ty], [llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_cvtmask2q_256 : GCCBuiltin<"__builtin_ia32_cvtmask2q256">, + Intrinsic<[llvm_v4i64_ty], [llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_cvtmask2q_512 : GCCBuiltin<"__builtin_ia32_cvtmask2q512">, + Intrinsic<[llvm_v8i64_ty], [llvm_i8_ty], [IntrNoMem]>; + } // Pack ops. @@ -3751,53 +4213,761 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; } +// Unpack ops. +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_avx512_mask_unpckh_pd_128 : + GCCBuiltin<"__builtin_ia32_unpckhpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckh_pd_256 : + GCCBuiltin<"__builtin_ia32_unpckhpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckh_pd_512 : + GCCBuiltin<"__builtin_ia32_unpckhpd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckh_ps_128 : + GCCBuiltin<"__builtin_ia32_unpckhps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckh_ps_256 : + GCCBuiltin<"__builtin_ia32_unpckhps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckh_ps_512 : + GCCBuiltin<"__builtin_ia32_unpckhps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckl_pd_128 : + GCCBuiltin<"__builtin_ia32_unpcklpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckl_pd_256 : + GCCBuiltin<"__builtin_ia32_unpcklpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckl_pd_512 : + GCCBuiltin<"__builtin_ia32_unpcklpd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckl_ps_128 : + GCCBuiltin<"__builtin_ia32_unpcklps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckl_ps_256 : + GCCBuiltin<"__builtin_ia32_unpcklps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_unpckl_ps_512 : + GCCBuiltin<"__builtin_ia32_unpcklps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhb_w_128 : + GCCBuiltin<"__builtin_ia32_punpckhbw128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhb_w_256 : + GCCBuiltin<"__builtin_ia32_punpckhbw256_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhb_w_512 : + GCCBuiltin<"__builtin_ia32_punpckhbw512_mask">, + Intrinsic<[llvm_v64i8_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhd_q_128 : + GCCBuiltin<"__builtin_ia32_punpckhdq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhd_q_256 : + GCCBuiltin<"__builtin_ia32_punpckhdq256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhd_q_512 : + GCCBuiltin<"__builtin_ia32_punpckhdq512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhqd_q_128 : + GCCBuiltin<"__builtin_ia32_punpckhqdq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhqd_q_256 : + GCCBuiltin<"__builtin_ia32_punpckhqdq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhqd_q_512 : + GCCBuiltin<"__builtin_ia32_punpckhqdq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhw_d_128 : + GCCBuiltin<"__builtin_ia32_punpckhwd128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhw_d_256 : + GCCBuiltin<"__builtin_ia32_punpckhwd256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckhw_d_512 : + GCCBuiltin<"__builtin_ia32_punpckhwd512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklb_w_128 : + GCCBuiltin<"__builtin_ia32_punpcklbw128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklb_w_256 : + GCCBuiltin<"__builtin_ia32_punpcklbw256_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklb_w_512 : + GCCBuiltin<"__builtin_ia32_punpcklbw512_mask">, + Intrinsic<[llvm_v64i8_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckld_q_128 : + GCCBuiltin<"__builtin_ia32_punpckldq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckld_q_256 : + GCCBuiltin<"__builtin_ia32_punpckldq256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpckld_q_512 : + GCCBuiltin<"__builtin_ia32_punpckldq512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklqd_q_128 : + GCCBuiltin<"__builtin_ia32_punpcklqdq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklqd_q_256 : + GCCBuiltin<"__builtin_ia32_punpcklqdq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklqd_q_512 : + GCCBuiltin<"__builtin_ia32_punpcklqdq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklw_d_128 : + GCCBuiltin<"__builtin_ia32_punpcklwd128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklw_d_256 : + GCCBuiltin<"__builtin_ia32_punpcklwd256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_punpcklw_d_512 : + GCCBuiltin<"__builtin_ia32_punpcklwd512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrNoMem]>; +} + // Vector convert let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_mask_cvttps2dq_512: GCCBuiltin<"__builtin_ia32_cvttps2dq512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2udq_512: GCCBuiltin<"__builtin_ia32_cvttps2udq512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2dq_512: GCCBuiltin<"__builtin_ia32_cvttpd2dq512_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2udq_512: GCCBuiltin<"__builtin_ia32_cvttpd2udq512_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_rndscale_ps_512: GCCBuiltin<"__builtin_ia32_rndscaleps_mask">, + def int_x86_avx512_mask_cvtdq2pd_128 : + GCCBuiltin<"__builtin_ia32_cvtdq2pd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v4i32_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtdq2pd_256 : + GCCBuiltin<"__builtin_ia32_cvtdq2pd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4i32_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtdq2pd_512 : + GCCBuiltin<"__builtin_ia32_cvtdq2pd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8i32_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtdq2ps_128 : + GCCBuiltin<"__builtin_ia32_cvtdq2ps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtdq2ps_256 : + GCCBuiltin<"__builtin_ia32_cvtdq2ps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtdq2ps_512 : + GCCBuiltin<"__builtin_ia32_cvtdq2ps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2dq_128 : + GCCBuiltin<"__builtin_ia32_cvtpd2dq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2dq_256 : + GCCBuiltin<"__builtin_ia32_cvtpd2dq256_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2dq_512 : + GCCBuiltin<"__builtin_ia32_cvtpd2dq512_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2ps_256 : + GCCBuiltin<"__builtin_ia32_cvtpd2ps256_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f64_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2ps_512 : + GCCBuiltin<"__builtin_ia32_cvtpd2ps512_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtsd2ss_round : + GCCBuiltin<"__builtin_ia32_cvtsd2ss_round">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtss2sd_round : + GCCBuiltin<"__builtin_ia32_cvtss2sd_round">, + Intrinsic<[llvm_v2f64_ty], + [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2ps : + GCCBuiltin<"__builtin_ia32_cvtpd2ps_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2qq_128 : + GCCBuiltin<"__builtin_ia32_cvtpd2qq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2qq_256 : + GCCBuiltin<"__builtin_ia32_cvtpd2qq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2qq_512 : + GCCBuiltin<"__builtin_ia32_cvtpd2qq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2udq_128 : + GCCBuiltin<"__builtin_ia32_cvtpd2udq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2udq_256 : + GCCBuiltin<"__builtin_ia32_cvtpd2udq256_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2udq_512 : + GCCBuiltin<"__builtin_ia32_cvtpd2udq512_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2uqq_128 : + GCCBuiltin<"__builtin_ia32_cvtpd2uqq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2uqq_256 : + GCCBuiltin<"__builtin_ia32_cvtpd2uqq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtpd2uqq_512 : + GCCBuiltin<"__builtin_ia32_cvtpd2uqq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2dq_128 : + GCCBuiltin<"__builtin_ia32_cvtps2dq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2dq_256 : + GCCBuiltin<"__builtin_ia32_cvtps2dq256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2dq_512 : + GCCBuiltin<"__builtin_ia32_cvtps2dq512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2pd_128 : + GCCBuiltin<"__builtin_ia32_cvtps2pd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v4f32_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2pd_256 : + GCCBuiltin<"__builtin_ia32_cvtps2pd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f32_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2pd_512 : + GCCBuiltin<"__builtin_ia32_cvtps2pd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2qq_128 : + GCCBuiltin<"__builtin_ia32_cvtps2qq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2qq_256 : + GCCBuiltin<"__builtin_ia32_cvtps2qq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2qq_512 : + GCCBuiltin<"__builtin_ia32_cvtps2qq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2udq_128 : + GCCBuiltin<"__builtin_ia32_cvtps2udq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2udq_256 : + GCCBuiltin<"__builtin_ia32_cvtps2udq256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2udq_512 : + GCCBuiltin<"__builtin_ia32_cvtps2udq512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2uqq_128 : + GCCBuiltin<"__builtin_ia32_cvtps2uqq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2uqq_256 : + GCCBuiltin<"__builtin_ia32_cvtps2uqq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtps2uqq_512 : + GCCBuiltin<"__builtin_ia32_cvtps2uqq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtqq2pd_128 : + GCCBuiltin<"__builtin_ia32_cvtqq2pd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtqq2pd_256 : + GCCBuiltin<"__builtin_ia32_cvtqq2pd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtqq2pd_512 : + GCCBuiltin<"__builtin_ia32_cvtqq2pd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtqq2ps_128 : + GCCBuiltin<"__builtin_ia32_cvtqq2ps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtqq2ps_256 : + GCCBuiltin<"__builtin_ia32_cvtqq2ps256_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtqq2ps_512 : + GCCBuiltin<"__builtin_ia32_cvtqq2ps512_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8i64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2dq_128 : + GCCBuiltin<"__builtin_ia32_cvttpd2dq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2dq_256 : + GCCBuiltin<"__builtin_ia32_cvttpd2dq256_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2dq_512 : + GCCBuiltin<"__builtin_ia32_cvttpd2dq512_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2qq_128 : + GCCBuiltin<"__builtin_ia32_cvttpd2qq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2qq_256 : + GCCBuiltin<"__builtin_ia32_cvttpd2qq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2qq_512 : + GCCBuiltin<"__builtin_ia32_cvttpd2qq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2udq_128 : + GCCBuiltin<"__builtin_ia32_cvttpd2udq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2udq_256 : + GCCBuiltin<"__builtin_ia32_cvttpd2udq256_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2udq_512 : + GCCBuiltin<"__builtin_ia32_cvttpd2udq512_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2uqq_128 : + GCCBuiltin<"__builtin_ia32_cvttpd2uqq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2uqq_256 : + GCCBuiltin<"__builtin_ia32_cvttpd2uqq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttpd2uqq_512 : + GCCBuiltin<"__builtin_ia32_cvttpd2uqq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2dq_128 : + GCCBuiltin<"__builtin_ia32_cvttps2dq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2dq_256 : + GCCBuiltin<"__builtin_ia32_cvttps2dq256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2dq_512 : + GCCBuiltin<"__builtin_ia32_cvttps2dq512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2qq_128 : + GCCBuiltin<"__builtin_ia32_cvttps2qq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2qq_256 : + GCCBuiltin<"__builtin_ia32_cvttps2qq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2qq_512 : + GCCBuiltin<"__builtin_ia32_cvttps2qq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2udq_128 : + GCCBuiltin<"__builtin_ia32_cvttps2udq128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2udq_256 : + GCCBuiltin<"__builtin_ia32_cvttps2udq256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2udq_512 : + GCCBuiltin<"__builtin_ia32_cvttps2udq512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2uqq_128 : + GCCBuiltin<"__builtin_ia32_cvttps2uqq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2uqq_256 : + GCCBuiltin<"__builtin_ia32_cvttps2uqq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvttps2uqq_512 : + GCCBuiltin<"__builtin_ia32_cvttps2uqq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtudq2pd_128 : + GCCBuiltin<"__builtin_ia32_cvtudq2pd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v4i32_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtudq2pd_256 : + GCCBuiltin<"__builtin_ia32_cvtudq2pd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4i32_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtudq2pd_512 : + GCCBuiltin<"__builtin_ia32_cvtudq2pd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8i32_ty, llvm_v8f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtudq2ps_128 : + GCCBuiltin<"__builtin_ia32_cvtudq2ps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtudq2ps_256 : + GCCBuiltin<"__builtin_ia32_cvtudq2ps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtudq2ps_512 : + GCCBuiltin<"__builtin_ia32_cvtudq2ps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtuqq2pd_128 : + GCCBuiltin<"__builtin_ia32_cvtuqq2pd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtuqq2pd_256 : + GCCBuiltin<"__builtin_ia32_cvtuqq2pd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtuqq2pd_512 : + GCCBuiltin<"__builtin_ia32_cvtuqq2pd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtuqq2ps_128 : + GCCBuiltin<"__builtin_ia32_cvtuqq2ps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtuqq2ps_256 : + GCCBuiltin<"__builtin_ia32_cvtuqq2ps256_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_cvtuqq2ps_512 : + GCCBuiltin<"__builtin_ia32_cvtuqq2ps512_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8i64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_rndscale_pd_128 : GCCBuiltin<"__builtin_ia32_rndscalepd_128_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, + llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_rndscale_pd_256 : GCCBuiltin<"__builtin_ia32_rndscalepd_256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, + llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_rndscale_pd_512 : GCCBuiltin<"__builtin_ia32_rndscalepd_mask">, + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_rndscale_ps_128 : GCCBuiltin<"__builtin_ia32_rndscaleps_128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, + llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_rndscale_ps_256 : GCCBuiltin<"__builtin_ia32_rndscaleps_256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, + llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_rndscale_ps_512 : GCCBuiltin<"__builtin_ia32_rndscaleps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_rndscale_pd_512: GCCBuiltin<"__builtin_ia32_rndscalepd_mask">, + def int_x86_avx512_mask_reduce_pd_128 : GCCBuiltin<"__builtin_ia32_reducepd128_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, + llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_reduce_pd_256 : GCCBuiltin<"__builtin_ia32_reducepd256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, + llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_reduce_pd_512 : GCCBuiltin<"__builtin_ia32_reducepd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2dq_512: GCCBuiltin<"__builtin_ia32_cvtps2dq512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2dq_512: GCCBuiltin<"__builtin_ia32_cvtpd2dq512_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2udq_512: GCCBuiltin<"__builtin_ia32_cvtps2udq512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2udq_512: GCCBuiltin<"__builtin_ia32_cvtpd2udq512_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtdq2ps_512 : GCCBuiltin<"__builtin_ia32_cvtdq2ps512_mask">, - Intrinsic<[llvm_v16f32_ty], [llvm_v16i32_ty, llvm_v16f32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtdq2pd_512 : GCCBuiltin<"__builtin_ia32_cvtdq2pd512_mask">, - Intrinsic<[llvm_v8f64_ty], [llvm_v8i32_ty, llvm_v8f64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtudq2ps_512 : GCCBuiltin<"__builtin_ia32_cvtudq2ps512_mask">, - Intrinsic<[llvm_v16f32_ty], [llvm_v16i32_ty, llvm_v16f32_ty, + def int_x86_avx512_mask_reduce_ps_128 : GCCBuiltin<"__builtin_ia32_reduceps128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, + llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_reduce_ps_256 : GCCBuiltin<"__builtin_ia32_reduceps256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, + llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_reduce_ps_512 : GCCBuiltin<"__builtin_ia32_reduceps512_mask">, + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtudq2pd_512 : GCCBuiltin<"__builtin_ia32_cvtudq2pd512_mask">, - Intrinsic<[llvm_v8f64_ty], [llvm_v8i32_ty, llvm_v8f64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2ps_512 : GCCBuiltin<"__builtin_ia32_cvtpd2ps512_mask">, - Intrinsic<[llvm_v8f32_ty], [llvm_v8f64_ty, llvm_v8f32_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; +def int_x86_avx512_mask_range_pd_128 : GCCBuiltin<"__builtin_ia32_rangepd128_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, + llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; +def int_x86_avx512_mask_range_pd_256 : GCCBuiltin<"__builtin_ia32_rangepd256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty, + llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; +def int_x86_avx512_mask_range_pd_512 : GCCBuiltin<"__builtin_ia32_rangepd512_mask">, + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty, + llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; +def int_x86_avx512_mask_range_ps_128 : GCCBuiltin<"__builtin_ia32_rangeps128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, + llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; +def int_x86_avx512_mask_range_ps_256 : GCCBuiltin<"__builtin_ia32_rangeps256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, + llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; +def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mask">, + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty, + llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; } // Vector load with broadcast @@ -3805,28 +4975,183 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_vbroadcast_ss_512 : GCCBuiltin<"__builtin_ia32_vbroadcastss512">, Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty], [IntrReadArgMem]>; - def int_x86_avx512_vbroadcast_ss_ps_512 : - GCCBuiltin<"__builtin_ia32_vbroadcastss_ps512">, - Intrinsic<[llvm_v16f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_broadcast_ss_ps_512 : + GCCBuiltin<"__builtin_ia32_broadcastss512">, + Intrinsic<[llvm_v16f32_ty], [llvm_v4f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_broadcast_ss_ps_256 : + GCCBuiltin<"__builtin_ia32_broadcastss256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v4f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_broadcast_ss_ps_128 : + GCCBuiltin<"__builtin_ia32_broadcastss128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_vbroadcast_sd_512 : GCCBuiltin<"__builtin_ia32_vbroadcastsd512">, Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty], [IntrReadArgMem]>; - def int_x86_avx512_vbroadcast_sd_pd_512 : - GCCBuiltin<"__builtin_ia32_vbroadcastsd_pd512">, - Intrinsic<[llvm_v8f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; - + def int_x86_avx512_mask_broadcast_sd_pd_512 : + GCCBuiltin<"__builtin_ia32_broadcastsd512">, + Intrinsic<[llvm_v8f64_ty], [llvm_v2f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_broadcast_sd_pd_256 : + GCCBuiltin<"__builtin_ia32_broadcastsd256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v2f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_pbroadcastb_128 : + GCCBuiltin<"__builtin_ia32_pbroadcastb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastb_256 : + GCCBuiltin<"__builtin_ia32_pbroadcastb256_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v16i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastb_512 : + GCCBuiltin<"__builtin_ia32_pbroadcastb512_mask">, + Intrinsic<[llvm_v64i8_ty], + [llvm_v16i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastw_128 : + GCCBuiltin<"__builtin_ia32_pbroadcastw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastw_256 : + GCCBuiltin<"__builtin_ia32_pbroadcastw256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v8i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastw_512 : + GCCBuiltin<"__builtin_ia32_pbroadcastw512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v8i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastd_128 : + GCCBuiltin<"__builtin_ia32_pbroadcastd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastd_256 : + GCCBuiltin<"__builtin_ia32_pbroadcastd256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_pbroadcastd_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastd512">, - Intrinsic<[llvm_v16i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; + GCCBuiltin<"__builtin_ia32_pbroadcastd512">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastq_128 : + GCCBuiltin<"__builtin_ia32_pbroadcastq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastq_256 : + GCCBuiltin<"__builtin_ia32_pbroadcastq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v2i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastq_512 : + GCCBuiltin<"__builtin_ia32_pbroadcastq512">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v2i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcastf32x2_256 : + GCCBuiltin<"__builtin_ia32_broadcastf32x2_256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v4f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcastf32x2_512 : + GCCBuiltin<"__builtin_ia32_broadcastf32x2_512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v4f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti32x2_128 : + GCCBuiltin<"__builtin_ia32_broadcasti32x2_128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti32x2_256 : + GCCBuiltin<"__builtin_ia32_broadcasti32x2_256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti32x2_512 : + GCCBuiltin<"__builtin_ia32_broadcasti32x2_512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcastf32x4_256 : + GCCBuiltin<"__builtin_ia32_broadcastf32x4_256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v4f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcastf32x4_512 : + GCCBuiltin<"__builtin_ia32_broadcastf32x4_512">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v4f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcastf32x8_512 : + GCCBuiltin<"__builtin_ia32_broadcastf32x8_512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v8f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcastf64x2_256 : + GCCBuiltin<"__builtin_ia32_broadcastf64x2_256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v2f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcastf64x2_512 : + GCCBuiltin<"__builtin_ia32_broadcastf64x2_512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v2f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcastf64x4_512 : + GCCBuiltin<"__builtin_ia32_broadcastf64x4_512">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v4f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti32x4_256 : + GCCBuiltin<"__builtin_ia32_broadcasti32x4_256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti32x4_512 : + GCCBuiltin<"__builtin_ia32_broadcasti32x4_512">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti32x8_512 : + GCCBuiltin<"__builtin_ia32_broadcasti32x8_512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v8i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti64x2_256 : + GCCBuiltin<"__builtin_ia32_broadcasti64x2_256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v2i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti64x2_512 : + GCCBuiltin<"__builtin_ia32_broadcasti64x2_512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v2i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_broadcasti64x4_512 : + GCCBuiltin<"__builtin_ia32_broadcasti64x4_512">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v4i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_pbroadcastd_i32_512 : Intrinsic<[llvm_v16i32_ty], [llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastq_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastq512">, - Intrinsic<[llvm_v8i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; def int_x86_avx512_pbroadcastq_i64_512 : Intrinsic<[llvm_v8i64_ty], [llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_broadcastmw_512 : + GCCBuiltin<"__builtin_ia32_broadcastmw512">, + Intrinsic<[llvm_v16i32_ty], [llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_broadcastmw_256 : + GCCBuiltin<"__builtin_ia32_broadcastmw256">, + Intrinsic<[llvm_v8i32_ty], [llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_broadcastmw_128 : + GCCBuiltin<"__builtin_ia32_broadcastmw128">, + Intrinsic<[llvm_v4i32_ty], [llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_broadcastmb_512 : + GCCBuiltin<"__builtin_ia32_broadcastmb512">, + Intrinsic<[llvm_v8i64_ty], [llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_broadcastmb_256 : + GCCBuiltin<"__builtin_ia32_broadcastmb256">, + Intrinsic<[llvm_v4i64_ty], [llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_broadcastmb_128 : + GCCBuiltin<"__builtin_ia32_broadcastmb128">, + Intrinsic<[llvm_v2i64_ty], [llvm_i8_ty], [IntrNoMem]>; } // Vector sign and zero extend @@ -4071,15 +5396,36 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_mask">, + def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_round">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_rndscale_sd : GCCBuiltin<"__builtin_ia32_rndscalesd_mask">, + def int_x86_avx512_mask_rndscale_sd : GCCBuiltin<"__builtin_ia32_rndscalesd_round">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - + def int_x86_avx512_mask_range_ss : GCCBuiltin<"__builtin_ia32_rangess128_round">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_range_sd : GCCBuiltin<"__builtin_ia32_rangesd128_round">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_reduce_ss : GCCBuiltin<"__builtin_ia32_reducess">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_reduce_sd : GCCBuiltin<"__builtin_ia32_reducesd">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_scalef_sd : GCCBuiltin<"__builtin_ia32_scalefsd_round">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, + llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_scalef_ss : GCCBuiltin<"__builtin_ia32_scalefss_round">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, + llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_scalef_pd_128 : GCCBuiltin<"__builtin_ia32_scalefpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; @@ -4099,12 +5445,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_sqrt_ss : GCCBuiltin<"__builtin_ia32_sqrtrndss">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], - [IntrNoMem]>; - def int_x86_avx512_sqrt_sd : GCCBuiltin<"__builtin_ia32_sqrtrndsd">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], - [IntrNoMem]>; + def int_x86_avx512_mask_sqrt_ss : GCCBuiltin<"__builtin_ia32_sqrtss_round">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_sqrt_sd : GCCBuiltin<"__builtin_ia32_sqrtsd_round">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_sqrt_pd_128 : GCCBuiltin<"__builtin_ia32_sqrtpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, @@ -4143,29 +5489,108 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss_mask">, + def int_x86_avx512_mask_getexp_ss : GCCBuiltin<"__builtin_ia32_getexpss_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_getexp_sd : GCCBuiltin<"__builtin_ia32_getexpsd_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_getmant_pd_128 : + GCCBuiltin<"__builtin_ia32_getmantpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty,llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_getmant_pd_256 : + GCCBuiltin<"__builtin_ia32_getmantpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty,llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_getmant_pd_512 : + GCCBuiltin<"__builtin_ia32_getmantpd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty,llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty,llvm_i32_ty ], + [IntrNoMem]>; + + def int_x86_avx512_mask_getmant_ps_128 : + GCCBuiltin<"__builtin_ia32_getmantps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_getmant_ps_256 : + GCCBuiltin<"__builtin_ia32_getmantps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_getmant_ps_512 : + GCCBuiltin<"__builtin_ia32_getmantps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty,llvm_i32_ty, llvm_v16f32_ty,llvm_i16_ty,llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_getmant_ss : + GCCBuiltin<"__builtin_ia32_getmantss_round">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_getmant_sd : + GCCBuiltin<"__builtin_ia32_getmantsd_round">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, + llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_sd : GCCBuiltin<"__builtin_ia32_rsqrt14sd_mask">, + def int_x86_avx512_rsqrt14_sd : GCCBuiltin<"__builtin_ia32_rsqrt14sd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_rsqrt14_pd_128 : GCCBuiltin<"__builtin_ia32_rsqrt14pd128_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_rsqrt14_pd_256 : GCCBuiltin<"__builtin_ia32_rsqrt14pd256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_rsqrt14_pd_512 : GCCBuiltin<"__builtin_ia32_rsqrt14pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_rsqrt14_ps_128 : GCCBuiltin<"__builtin_ia32_rsqrt14ps128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_rsqrt14_ps_256 : GCCBuiltin<"__builtin_ia32_rsqrt14ps256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_rsqrt14_ps_512 : GCCBuiltin<"__builtin_ia32_rsqrt14ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_ss : GCCBuiltin<"__builtin_ia32_rcp14ss_mask">, + def int_x86_avx512_rcp14_ss : GCCBuiltin<"__builtin_ia32_rcp14ss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_sd : GCCBuiltin<"__builtin_ia32_rcp14sd_mask">, + def int_x86_avx512_rcp14_sd : GCCBuiltin<"__builtin_ia32_rcp14sd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_rcp14_pd_128 : GCCBuiltin<"__builtin_ia32_rcp14pd128_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_rcp14_pd_256 : GCCBuiltin<"__builtin_ia32_rcp14pd256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_rcp14_pd_512 : GCCBuiltin<"__builtin_ia32_rcp14pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_rcp14_ps_128 : GCCBuiltin<"__builtin_ia32_rcp14ps128_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_rcp14_ps_256 : GCCBuiltin<"__builtin_ia32_rcp14ps256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, + llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_rcp14_ps_512 : GCCBuiltin<"__builtin_ia32_rcp14ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; @@ -4183,11 +5608,11 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rcp28_ss : GCCBuiltin<"__builtin_ia32_rcp28ss_mask">, + def int_x86_avx512_rcp28_ss : GCCBuiltin<"__builtin_ia32_rcp28ss_round">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rcp28_sd : GCCBuiltin<"__builtin_ia32_rcp28sd_mask">, + def int_x86_avx512_rcp28_sd : GCCBuiltin<"__builtin_ia32_rcp28sd_round">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -4199,14 +5624,17 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt28_ss : GCCBuiltin<"__builtin_ia32_rsqrt28ss_mask">, + def int_x86_avx512_rsqrt28_ss : GCCBuiltin<"__builtin_ia32_rsqrt28ss_round">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt28_sd : GCCBuiltin<"__builtin_ia32_rsqrt28sd_mask">, + def int_x86_avx512_rsqrt28_sd : GCCBuiltin<"__builtin_ia32_rsqrt28sd_round">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; +def int_x86_avx512_psad_bw_512 : GCCBuiltin<"__builtin_ia32_psadbw512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty], + [IntrNoMem]>; } // FP logical ops let TargetPrefix = "x86" in { @@ -4511,6 +5939,54 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_pavg_w_256 : GCCBuiltin<"__builtin_ia32_pavgw256_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmaddw_d_128 : + GCCBuiltin<"__builtin_ia32_pmaddwd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmaddw_d_256 : + GCCBuiltin<"__builtin_ia32_pmaddwd256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmaddw_d_512 : + GCCBuiltin<"__builtin_ia32_pmaddwd512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmaddubs_w_128 : + GCCBuiltin<"__builtin_ia32_pmaddubsw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmaddubs_w_256 : + GCCBuiltin<"__builtin_ia32_pmaddubsw256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmaddubs_w_512 : + GCCBuiltin<"__builtin_ia32_pmaddubsw512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_dbpsadbw_128 : + GCCBuiltin<"__builtin_ia32_dbpsadbw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_v8i16_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_dbpsadbw_256 : + GCCBuiltin<"__builtin_ia32_dbpsadbw256_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty, llvm_v16i16_ty, + llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_dbpsadbw_512 : + GCCBuiltin<"__builtin_ia32_dbpsadbw512_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty, llvm_v32i16_ty, + llvm_i32_ty], [IntrNoMem]>; } // Gather and Scatter ops @@ -4807,27 +6283,71 @@ let TargetPrefix = "x86" in { llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; } -// AVX-512 conflict detection +// AVX-512 conflict detection instruction +// Instructions that count the number of leading zero bits let TargetPrefix = "x86" in { + def int_x86_avx512_mask_conflict_d_128 : + GCCBuiltin<"__builtin_ia32_vpconflictsi_128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_conflict_d_256 : + GCCBuiltin<"__builtin_ia32_vpconflictsi_256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; def int_x86_avx512_mask_conflict_d_512 : GCCBuiltin<"__builtin_ia32_vpconflictsi_512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, - llvm_v16i32_ty, llvm_i16_ty], + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_conflict_q_128 : + GCCBuiltin<"__builtin_ia32_vpconflictdi_128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_conflict_q_256 : + GCCBuiltin<"__builtin_ia32_vpconflictdi_256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_conflict_q_512 : GCCBuiltin<"__builtin_ia32_vpconflictdi_512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_v8i64_ty, llvm_i8_ty], + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_lzcnt_d_128 : + GCCBuiltin<"__builtin_ia32_vplzcntd_128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_lzcnt_d_256 : + GCCBuiltin<"__builtin_ia32_vplzcntd_256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_lzcnt_d_512 : GCCBuiltin<"__builtin_ia32_vplzcntd_512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, - llvm_v16i32_ty, llvm_i16_ty], + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_lzcnt_q_128 : + GCCBuiltin<"__builtin_ia32_vplzcntq_128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_lzcnt_q_256 : + GCCBuiltin<"__builtin_ia32_vplzcntq_256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_lzcnt_q_512 : GCCBuiltin<"__builtin_ia32_vplzcntq_512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_v8i64_ty, llvm_i8_ty], + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } @@ -4911,20 +6431,70 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". } let TargetPrefix = "x86" in { - def int_x86_avx512_mask_valign_q_512 : GCCBuiltin<"__builtin_ia32_alignq512_mask">, + def int_x86_avx512_mask_valign_q_512 : + GCCBuiltin<"__builtin_ia32_alignq512_mask">, Intrinsic<[llvm_v8i64_ty], - [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_v8i64_ty, + llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_valign_d_512 : GCCBuiltin<"__builtin_ia32_alignd512_mask">, + def int_x86_avx512_mask_valign_d_512 : + GCCBuiltin<"__builtin_ia32_alignd512_mask">, Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_v16i32_ty, + llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_valign_q_256 : + GCCBuiltin<"__builtin_ia32_alignq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_v4i64_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_valign_d_256 : + GCCBuiltin<"__builtin_ia32_alignd256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_v8i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_valign_q_128 : + GCCBuiltin<"__builtin_ia32_alignq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_v2i64_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_valign_d_128 : + GCCBuiltin<"__builtin_ia32_alignd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_palignr_128 : + GCCBuiltin<"__builtin_ia32_palignr128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, + llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_palignr_256 : + GCCBuiltin<"__builtin_ia32_palignr256_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty, llvm_v32i8_ty, + llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_palignr_512 : + GCCBuiltin<"__builtin_ia32_palignr512_mask">, + Intrinsic<[llvm_v64i8_ty], + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty, llvm_v64i8_ty, + llvm_i64_ty], [IntrNoMem]>; } // Compares let TargetPrefix = "x86" in { // 512-bit + def int_x86_avx512_vcomi_sd : GCCBuiltin<"__builtin_ia32_vcomisd">, + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, + llvm_v2f64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_vcomi_ss : GCCBuiltin<"__builtin_ia32_vcomiss">, + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, + llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pcmpeq_b_512 : GCCBuiltin<"__builtin_ia32_pcmpeqb512_mask">, Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; @@ -5288,6 +6858,626 @@ let TargetPrefix = "x86" in { llvm_i8_ty], [IntrReadArgMem]>; } + +// truncate +let TargetPrefix = "x86" in { + def int_x86_avx512_mask_pmov_qb_128 : + GCCBuiltin<"__builtin_ia32_pmovqb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qb_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovqb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qb_128 : + GCCBuiltin<"__builtin_ia32_pmovsqb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qb_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovsqb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qb_128 : + GCCBuiltin<"__builtin_ia32_pmovusqb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v2i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qb_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovusqb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_qb_256 : + GCCBuiltin<"__builtin_ia32_pmovqb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qb_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovqb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qb_256 : + GCCBuiltin<"__builtin_ia32_pmovsqb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qb_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovsqb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qb_256 : + GCCBuiltin<"__builtin_ia32_pmovusqb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v4i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qb_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovusqb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_qb_512 : + GCCBuiltin<"__builtin_ia32_pmovqb512_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qb_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovqb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qb_512 : + GCCBuiltin<"__builtin_ia32_pmovsqb512_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qb_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovsqb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qb_512 : + GCCBuiltin<"__builtin_ia32_pmovusqb512_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i64_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qb_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovusqb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_qw_128 : + GCCBuiltin<"__builtin_ia32_pmovqw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qw_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovqw128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qw_128 : + GCCBuiltin<"__builtin_ia32_pmovsqw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qw_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovsqw128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qw_128 : + GCCBuiltin<"__builtin_ia32_pmovusqw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v2i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qw_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovusqw128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_qw_256 : + GCCBuiltin<"__builtin_ia32_pmovqw256_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qw_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovqw256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qw_256 : + GCCBuiltin<"__builtin_ia32_pmovsqw256_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qw_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovsqw256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qw_256 : + GCCBuiltin<"__builtin_ia32_pmovusqw256_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v4i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qw_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovusqw256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_qw_512 : + GCCBuiltin<"__builtin_ia32_pmovqw512_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qw_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovqw512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qw_512 : + GCCBuiltin<"__builtin_ia32_pmovsqw512_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qw_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovsqw512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qw_512 : + GCCBuiltin<"__builtin_ia32_pmovusqw512_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i64_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qw_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovusqw512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_qd_128 : + GCCBuiltin<"__builtin_ia32_pmovqd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qd_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovqd128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qd_128 : + GCCBuiltin<"__builtin_ia32_pmovsqd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qd_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovsqd128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qd_128 : + GCCBuiltin<"__builtin_ia32_pmovusqd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qd_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovusqd128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_qd_256 : + GCCBuiltin<"__builtin_ia32_pmovqd256_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qd_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovqd256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qd_256 : + GCCBuiltin<"__builtin_ia32_pmovsqd256_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qd_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovsqd256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qd_256 : + GCCBuiltin<"__builtin_ia32_pmovusqd256_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qd_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovusqd256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_qd_512 : + GCCBuiltin<"__builtin_ia32_pmovqd512_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_qd_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovqd512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_qd_512 : + GCCBuiltin<"__builtin_ia32_pmovsqd512_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_qd_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovsqd512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_qd_512 : + GCCBuiltin<"__builtin_ia32_pmovusqd512_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_qd_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovusqd512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_db_128 : + GCCBuiltin<"__builtin_ia32_pmovdb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_db_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovdb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_db_128 : + GCCBuiltin<"__builtin_ia32_pmovsdb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_db_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovsdb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_db_128 : + GCCBuiltin<"__builtin_ia32_pmovusdb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v4i32_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_db_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovusdb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_db_256 : + GCCBuiltin<"__builtin_ia32_pmovdb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_db_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovdb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_db_256 : + GCCBuiltin<"__builtin_ia32_pmovsdb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_db_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovsdb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_db_256 : + GCCBuiltin<"__builtin_ia32_pmovusdb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i32_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_db_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovusdb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_db_512 : + GCCBuiltin<"__builtin_ia32_pmovdb512_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_db_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovdb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_db_512 : + GCCBuiltin<"__builtin_ia32_pmovsdb512_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_db_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovsdb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_db_512 : + GCCBuiltin<"__builtin_ia32_pmovusdb512_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i32_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_db_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovusdb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_dw_128 : + GCCBuiltin<"__builtin_ia32_pmovdw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_dw_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovdw128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_dw_128 : + GCCBuiltin<"__builtin_ia32_pmovsdw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_dw_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovsdw128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_dw_128 : + GCCBuiltin<"__builtin_ia32_pmovusdw128_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v4i32_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_dw_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovusdw128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_dw_256 : + GCCBuiltin<"__builtin_ia32_pmovdw256_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_dw_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovdw256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_dw_256 : + GCCBuiltin<"__builtin_ia32_pmovsdw256_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_dw_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovsdw256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_dw_256 : + GCCBuiltin<"__builtin_ia32_pmovusdw256_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_v8i32_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_dw_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovusdw256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_dw_512 : + GCCBuiltin<"__builtin_ia32_pmovdw512_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_dw_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovdw512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_dw_512 : + GCCBuiltin<"__builtin_ia32_pmovsdw512_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_dw_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovsdw512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_dw_512 : + GCCBuiltin<"__builtin_ia32_pmovusdw512_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_v16i32_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_dw_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovusdw512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_wb_128 : + GCCBuiltin<"__builtin_ia32_pmovwb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_wb_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovwb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_wb_128 : + GCCBuiltin<"__builtin_ia32_pmovswb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_wb_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovswb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_wb_128 : + GCCBuiltin<"__builtin_ia32_pmovuswb128_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v8i16_ty, llvm_v16i8_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_wb_mem_128 : + GCCBuiltin<"__builtin_ia32_pmovuswb128mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_wb_256 : + GCCBuiltin<"__builtin_ia32_pmovwb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_wb_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovwb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_wb_256 : + GCCBuiltin<"__builtin_ia32_pmovswb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_wb_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovswb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_wb_256 : + GCCBuiltin<"__builtin_ia32_pmovuswb256_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_wb_mem_256 : + GCCBuiltin<"__builtin_ia32_pmovuswb256mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmov_wb_512 : + GCCBuiltin<"__builtin_ia32_pmovwb512_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmov_wb_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovwb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovs_wb_512 : + GCCBuiltin<"__builtin_ia32_pmovswb512_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovs_wb_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovswb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrReadWriteArgMem]>; + def int_x86_avx512_mask_pmovus_wb_512 : + GCCBuiltin<"__builtin_ia32_pmovuswb512_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_pmovus_wb_mem_512 : + GCCBuiltin<"__builtin_ia32_pmovuswb512mem_mask">, + Intrinsic<[], + [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], + [IntrReadWriteArgMem]>; +} + +// Bitwise ternary logic +let TargetPrefix = "x86" in { + def int_x86_avx512_mask_pternlog_d_128 : + GCCBuiltin<"__builtin_ia32_pternlogd128_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_maskz_pternlog_d_128 : + GCCBuiltin<"__builtin_ia32_pternlogd128_maskz">, + Intrinsic<[llvm_v4i32_ty], + [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pternlog_d_256 : + GCCBuiltin<"__builtin_ia32_pternlogd256_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_maskz_pternlog_d_256 : + GCCBuiltin<"__builtin_ia32_pternlogd256_maskz">, + Intrinsic<[llvm_v8i32_ty], + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pternlog_d_512 : + GCCBuiltin<"__builtin_ia32_pternlogd512_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, + llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_maskz_pternlog_d_512 : + GCCBuiltin<"__builtin_ia32_pternlogd512_maskz">, + Intrinsic<[llvm_v16i32_ty], + [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, + llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pternlog_q_128 : + GCCBuiltin<"__builtin_ia32_pternlogq128_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_maskz_pternlog_q_128 : + GCCBuiltin<"__builtin_ia32_pternlogq128_maskz">, + Intrinsic<[llvm_v2i64_ty], + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pternlog_q_256 : + GCCBuiltin<"__builtin_ia32_pternlogq256_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_maskz_pternlog_q_256 : + GCCBuiltin<"__builtin_ia32_pternlogq256_maskz">, + Intrinsic<[llvm_v4i64_ty], + [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pternlog_q_512 : + GCCBuiltin<"__builtin_ia32_pternlogq512_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; + + def int_x86_avx512_maskz_pternlog_q_512 : + GCCBuiltin<"__builtin_ia32_pternlogq512_maskz">, + Intrinsic<[llvm_v8i64_ty], + [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, + llvm_i8_ty], [IntrNoMem]>; +} + // Misc. let TargetPrefix = "x86" in { def int_x86_avx512_mask_cmp_ps_512 : @@ -5314,6 +7504,14 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_cmppd128_mask">, Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_ss : + GCCBuiltin<"__builtin_ia32_cmpss_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_v4f32_ty, + llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_cmp_sd : + GCCBuiltin<"__builtin_ia32_cmpsd_mask">, + Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_v2f64_ty, + llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_movntdqa : GCCBuiltin<"__builtin_ia32_movntdqa512">, diff --git a/include/llvm/IR/LLVMContext.h b/include/llvm/IR/LLVMContext.h index e6c22090ab6d..c546fc3d1ee0 100644 --- a/include/llvm/IR/LLVMContext.h +++ b/include/llvm/IR/LLVMContext.h @@ -15,7 +15,6 @@ #ifndef LLVM_IR_LLVMCONTEXT_H #define LLVM_IR_LLVMCONTEXT_H -#include "llvm-c/Core.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Options.h" @@ -60,7 +59,20 @@ public: MD_mem_parallel_loop_access = 10, // "llvm.mem.parallel_loop_access" MD_nonnull = 11, // "nonnull" MD_dereferenceable = 12, // "dereferenceable" - MD_dereferenceable_or_null = 13 // "dereferenceable_or_null" + MD_dereferenceable_or_null = 13, // "dereferenceable_or_null" + MD_make_implicit = 14, // "make.implicit" + MD_unpredictable = 15, // "unpredictable" + MD_invariant_group = 16, // "invariant.group" + MD_align = 17 // "align" + }; + + /// Known operand bundle tag IDs, which always have the same value. All + /// operand bundle tags that LLVM has special knowledge of are listed here. + /// Additionally, this scheme allows LLVM to efficiently check for specific + /// operand bundle tags without comparing strings. + enum { + OB_deopt = 0, // "deopt" + OB_funclet = 1, // "funclet" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. @@ -71,6 +83,15 @@ public: /// custom metadata IDs registered in this LLVMContext. void getMDKindNames(SmallVectorImpl<StringRef> &Result) const; + /// getOperandBundleTags - Populate client supplied SmallVector with the + /// bundle tags registered in this LLVMContext. The bundle tags are ordered + /// by increasing bundle IDs. + /// \see LLVMContext::getOperandBundleTagID + void getOperandBundleTags(SmallVectorImpl<StringRef> &Result) const; + + /// getOperandBundleTagID - Maps a bundle tag to an integer ID. Every bundle + /// tag registered with an LLVMContext has an unique ID. + uint32_t getOperandBundleTagID(StringRef Tag) const; typedef void (*InlineAsmDiagHandlerTy)(const SMDiagnostic&, void *Context, unsigned LocCookie); diff --git a/include/llvm/IR/LegacyPassManagers.h b/include/llvm/IR/LegacyPassManagers.h index 7f7889ad5fb3..b8e33478d6a9 100644 --- a/include/llvm/IR/LegacyPassManagers.h +++ b/include/llvm/IR/LegacyPassManagers.h @@ -16,6 +16,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Pass.h" @@ -118,6 +119,7 @@ class PassManagerPrettyStackEntry : public PrettyStackTraceEntry { Pass *P; Value *V; Module *M; + public: explicit PassManagerPrettyStackEntry(Pass *p) : P(p), V(nullptr), M(nullptr) {} // When P is releaseMemory'd. @@ -130,7 +132,6 @@ public: void print(raw_ostream &OS) const override; }; - //===----------------------------------------------------------------------===// // PMStack // @@ -158,7 +159,6 @@ private: std::vector<PMDataManager *> S; }; - //===----------------------------------------------------------------------===// // PMTopLevelManager // @@ -204,10 +204,7 @@ public: virtual ~PMTopLevelManager(); /// Add immutable pass and initialize it. - inline void addImmutablePass(ImmutablePass *P) { - P->initializePass(); - ImmutablePasses.push_back(P); - } + void addImmutablePass(ImmutablePass *P); inline SmallVectorImpl<ImmutablePass *>& getImmutablePasses() { return ImmutablePasses; @@ -231,12 +228,10 @@ public: PMStack activeStack; protected: - /// Collection of pass managers SmallVector<PMDataManager *, 8> PassManagers; private: - /// Collection of pass managers that are not directly maintained /// by this pass manager SmallVector<PMDataManager *, 8> IndirectPassManagers; @@ -253,7 +248,46 @@ private: /// Immutable passes are managed by top level manager. SmallVector<ImmutablePass *, 16> ImmutablePasses; - DenseMap<Pass *, AnalysisUsage *> AnUsageMap; + /// Map from ID to immutable passes. + SmallDenseMap<AnalysisID, ImmutablePass *, 8> ImmutablePassMap; + + + /// A wrapper around AnalysisUsage for the purpose of uniqueing. The wrapper + /// is used to avoid needing to make AnalysisUsage itself a folding set node. + struct AUFoldingSetNode : public FoldingSetNode { + AnalysisUsage AU; + AUFoldingSetNode(const AnalysisUsage &AU) : AU(AU) {} + void Profile(FoldingSetNodeID &ID) const { + Profile(ID, AU); + } + static void Profile(FoldingSetNodeID &ID, const AnalysisUsage &AU) { + // TODO: We could consider sorting the dependency arrays within the + // AnalysisUsage (since they are conceptually unordered). + ID.AddBoolean(AU.getPreservesAll()); + auto ProfileVec = [&](const SmallVectorImpl<AnalysisID>& Vec) { + ID.AddInteger(Vec.size()); + for(AnalysisID AID : Vec) + ID.AddPointer(AID); + }; + ProfileVec(AU.getRequiredSet()); + ProfileVec(AU.getRequiredTransitiveSet()); + ProfileVec(AU.getPreservedSet()); + ProfileVec(AU.getUsedSet()); + } + }; + + // Contains all of the unique combinations of AnalysisUsage. This is helpful + // when we have multiple instances of the same pass since they'll usually + // have the same analysis usage and can share storage. + FoldingSet<AUFoldingSetNode> UniqueAnalysisUsages; + + // Allocator used for allocating UAFoldingSetNodes. This handles deletion of + // all allocated nodes in one fell swoop. + SpecificBumpPtrAllocator<AUFoldingSetNode> AUFoldingSetNodeAllocator; + + // Maps from a pass to it's associated entry in UniqueAnalysisUsages. Does + // not own the storage associated with either key or value.. + DenseMap<Pass *, AnalysisUsage*> AnUsageMap; /// Collection of PassInfo objects found via analysis IDs and in this top /// level manager. This is used to memoize queries to the pass registry. @@ -262,8 +296,6 @@ private: mutable DenseMap<AnalysisID, const PassInfo *> AnalysisPassInfos; }; - - //===----------------------------------------------------------------------===// // PMDataManager @@ -271,7 +303,6 @@ private: /// used by pass managers. class PMDataManager { public: - explicit PMDataManager() : TPM(nullptr), Depth(0) { initializeAnalysisInfo(); } @@ -319,13 +350,12 @@ public: // passes that are managed by this manager. bool preserveHigherLevelAnalysis(Pass *P); - - /// Populate RequiredPasses with analysis pass that are required by - /// pass P and are available. Populate ReqPassNotAvailable with analysis - /// pass that are required by pass P but are not available. - void collectRequiredAnalysis(SmallVectorImpl<Pass *> &RequiredPasses, - SmallVectorImpl<AnalysisID> &ReqPassNotAvailable, - Pass *P); + /// Populate UsedPasses with analysis pass that are used or required by pass + /// P and are available. Populate ReqPassNotAvailable with analysis pass that + /// are required by pass P but are not available. + void collectRequiredAndUsedAnalyses( + SmallVectorImpl<Pass *> &UsedPasses, + SmallVectorImpl<AnalysisID> &ReqPassNotAvailable, Pass *P); /// All Required analyses should be available to the pass as it runs! Here /// we fill in the AnalysisImpls member of the pass so that it can @@ -351,6 +381,7 @@ public: enum PassDebuggingString S2, StringRef Msg); void dumpRequiredSet(const Pass *P) const; void dumpPreservedSet(const Pass *P) const; + void dumpUsedSet(const Pass *P) const; unsigned getNumContainedPasses() const { return (unsigned)PassVector.size(); @@ -374,7 +405,6 @@ public: } protected: - // Top level manager. PMTopLevelManager *TPM; @@ -439,9 +469,9 @@ public: /// doFinalization - Overrides ModulePass doFinalization for global /// finalization tasks - /// + /// using ModulePass::doFinalization; - + /// doFinalization - Run all of the finalizers for the function passes. /// bool doFinalization(Module &M) override; @@ -473,7 +503,6 @@ public: }; Timer *getPassTimer(Pass *); - } #endif diff --git a/include/llvm/IR/MDBuilder.h b/include/llvm/IR/MDBuilder.h index ceb1c736e5c7..35341e3271ff 100644 --- a/include/llvm/IR/MDBuilder.h +++ b/include/llvm/IR/MDBuilder.h @@ -60,6 +60,9 @@ public: /// \brief Return metadata containing a number of branch weights. MDNode *createBranchWeights(ArrayRef<uint32_t> Weights); + /// Return metadata specifying that a branch or switch is unpredictable. + MDNode *createUnpredictable(); + /// Return metadata containing the entry count for a function. MDNode *createFunctionEntryCount(uint64_t Count); diff --git a/include/llvm/IR/Mangler.h b/include/llvm/IR/Mangler.h index b72b259097c3..ea2f0c3f09f3 100644 --- a/include/llvm/IR/Mangler.h +++ b/include/llvm/IR/Mangler.h @@ -15,12 +15,12 @@ #define LLVM_IR_MANGLER_H #include "llvm/ADT/DenseMap.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/Support/raw_ostream.h" namespace llvm { class DataLayout; -class GlobalValue; template <typename T> class SmallVectorImpl; class Twine; diff --git a/include/llvm/IR/Metadata.def b/include/llvm/IR/Metadata.def index 857e4637d1e4..b1d22178e262 100644 --- a/include/llvm/IR/Metadata.def +++ b/include/llvm/IR/Metadata.def @@ -13,7 +13,8 @@ #if !(defined HANDLE_METADATA || defined HANDLE_METADATA_LEAF || \ defined HANDLE_METADATA_BRANCH || defined HANDLE_MDNODE_LEAF || \ - defined HANDLE_MDNODE_BRANCH || \ + defined HANDLE_MDNODE_LEAF_UNIQUABLE || defined HANDLE_MDNODE_BRANCH || \ + defined HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE || \ defined HANDLE_SPECIALIZED_MDNODE_LEAF || \ defined HANDLE_SPECIALIZED_MDNODE_BRANCH) #error "Missing macro definition of HANDLE_METADATA*" @@ -34,6 +35,24 @@ #define HANDLE_METADATA_BRANCH(CLASS) HANDLE_METADATA(CLASS) #endif +// Handler for specialized and uniquable leaf nodes under MDNode. Defers to +// HANDLE_MDNODE_LEAF_UNIQUABLE if it's defined, otherwise to +// HANDLE_SPECIALIZED_MDNODE_LEAF. +#ifndef HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE +#ifdef HANDLE_MDNODE_LEAF_UNIQUABLE +#define HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(CLASS) \ + HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) +#else +#define HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(CLASS) \ + HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) +#endif +#endif + +// Handler for leaf nodes under MDNode. +#ifndef HANDLE_MDNODE_LEAF_UNIQUABLE +#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) HANDLE_MDNODE_LEAF(CLASS) +#endif + // Handler for leaf nodes under MDNode. #ifndef HANDLE_MDNODE_LEAF #define HANDLE_MDNODE_LEAF(CLASS) HANDLE_METADATA_LEAF(CLASS) @@ -59,43 +78,46 @@ HANDLE_METADATA_BRANCH(ValueAsMetadata) HANDLE_METADATA_LEAF(ConstantAsMetadata) HANDLE_METADATA_LEAF(LocalAsMetadata) HANDLE_MDNODE_BRANCH(MDNode) -HANDLE_MDNODE_LEAF(MDTuple) -HANDLE_SPECIALIZED_MDNODE_LEAF(DILocation) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIExpression) +HANDLE_MDNODE_LEAF_UNIQUABLE(MDTuple) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DILocation) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIExpression) HANDLE_SPECIALIZED_MDNODE_BRANCH(DINode) -HANDLE_SPECIALIZED_MDNODE_LEAF(GenericDINode) -HANDLE_SPECIALIZED_MDNODE_LEAF(DISubrange) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIEnumerator) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(GenericDINode) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DISubrange) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIEnumerator) HANDLE_SPECIALIZED_MDNODE_BRANCH(DIScope) HANDLE_SPECIALIZED_MDNODE_BRANCH(DIType) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIBasicType) -HANDLE_SPECIALIZED_MDNODE_BRANCH(DIDerivedTypeBase) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIDerivedType) -HANDLE_SPECIALIZED_MDNODE_BRANCH(DICompositeTypeBase) -HANDLE_SPECIALIZED_MDNODE_LEAF(DICompositeType) -HANDLE_SPECIALIZED_MDNODE_LEAF(DISubroutineType) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIFile) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIBasicType) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIDerivedType) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DICompositeType) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DISubroutineType) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIFile) HANDLE_SPECIALIZED_MDNODE_LEAF(DICompileUnit) HANDLE_SPECIALIZED_MDNODE_BRANCH(DILocalScope) -HANDLE_SPECIALIZED_MDNODE_LEAF(DISubprogram) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DISubprogram) HANDLE_SPECIALIZED_MDNODE_BRANCH(DILexicalBlockBase) -HANDLE_SPECIALIZED_MDNODE_LEAF(DILexicalBlock) -HANDLE_SPECIALIZED_MDNODE_LEAF(DILexicalBlockFile) -HANDLE_SPECIALIZED_MDNODE_LEAF(DINamespace) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIModule) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DILexicalBlock) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DILexicalBlockFile) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DINamespace) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIModule) HANDLE_SPECIALIZED_MDNODE_BRANCH(DITemplateParameter) -HANDLE_SPECIALIZED_MDNODE_LEAF(DITemplateTypeParameter) -HANDLE_SPECIALIZED_MDNODE_LEAF(DITemplateValueParameter) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DITemplateTypeParameter) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DITemplateValueParameter) HANDLE_SPECIALIZED_MDNODE_BRANCH(DIVariable) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIGlobalVariable) -HANDLE_SPECIALIZED_MDNODE_LEAF(DILocalVariable) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIObjCProperty) -HANDLE_SPECIALIZED_MDNODE_LEAF(DIImportedEntity) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIGlobalVariable) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DILocalVariable) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIObjCProperty) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIImportedEntity) +HANDLE_SPECIALIZED_MDNODE_BRANCH(DIMacroNode) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIMacro) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIMacroFile) #undef HANDLE_METADATA #undef HANDLE_METADATA_LEAF #undef HANDLE_METADATA_BRANCH #undef HANDLE_MDNODE_LEAF +#undef HANDLE_MDNODE_LEAF_UNIQUABLE #undef HANDLE_MDNODE_BRANCH #undef HANDLE_SPECIALIZED_MDNODE_LEAF +#undef HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE #undef HANDLE_SPECIALIZED_MDNODE_BRANCH diff --git a/include/llvm/IR/Metadata.h b/include/llvm/IR/Metadata.h index c639625bf16c..2ea591383f82 100644 --- a/include/llvm/IR/Metadata.h +++ b/include/llvm/IR/Metadata.h @@ -18,10 +18,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Constant.h" -#include "llvm/IR/MetadataTracking.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Value.h" #include "llvm/Support/ErrorHandling.h" #include <type_traits> @@ -32,9 +33,6 @@ class LLVMContext; class Module; class ModuleSlotTracker; -template<typename ValueSubClass, typename ItemParentClass> - class SymbolTableListTraits; - enum LLVMConstants : uint32_t { DEBUG_METADATA_VERSION = 3 // Current debug info version number. }; @@ -86,7 +84,9 @@ public: DIImportedEntityKind, ConstantAsMetadataKind, LocalAsMetadataKind, - MDStringKind + MDStringKind, + DIMacroKind, + DIMacroFileKind }; protected: @@ -126,9 +126,10 @@ public: /// If \c M is provided, metadata nodes will be numbered canonically; /// otherwise, pointer addresses are substituted. /// @{ - void print(raw_ostream &OS, const Module *M = nullptr) const; - void print(raw_ostream &OS, ModuleSlotTracker &MST, - const Module *M = nullptr) const; + void print(raw_ostream &OS, const Module *M = nullptr, + bool IsForDebug = false) const; + void print(raw_ostream &OS, ModuleSlotTracker &MST, const Module *M = nullptr, + bool IsForDebug = false) const; /// @} /// \brief Print as operand. @@ -196,6 +197,77 @@ private: void untrack(); }; +/// \brief API for tracking metadata references through RAUW and deletion. +/// +/// Shared API for updating \a Metadata pointers in subclasses that support +/// RAUW. +/// +/// This API is not meant to be used directly. See \a TrackingMDRef for a +/// user-friendly tracking reference. +class MetadataTracking { +public: + /// \brief Track the reference to metadata. + /// + /// Register \c MD with \c *MD, if the subclass supports tracking. If \c *MD + /// gets RAUW'ed, \c MD will be updated to the new address. If \c *MD gets + /// deleted, \c MD will be set to \c nullptr. + /// + /// If tracking isn't supported, \c *MD will not change. + /// + /// \return true iff tracking is supported by \c MD. + static bool track(Metadata *&MD) { + return track(&MD, *MD, static_cast<Metadata *>(nullptr)); + } + + /// \brief Track the reference to metadata for \a Metadata. + /// + /// As \a track(Metadata*&), but with support for calling back to \c Owner to + /// tell it that its operand changed. This could trigger \c Owner being + /// re-uniqued. + static bool track(void *Ref, Metadata &MD, Metadata &Owner) { + return track(Ref, MD, &Owner); + } + + /// \brief Track the reference to metadata for \a MetadataAsValue. + /// + /// As \a track(Metadata*&), but with support for calling back to \c Owner to + /// tell it that its operand changed. This could trigger \c Owner being + /// re-uniqued. + static bool track(void *Ref, Metadata &MD, MetadataAsValue &Owner) { + return track(Ref, MD, &Owner); + } + + /// \brief Stop tracking a reference to metadata. + /// + /// Stops \c *MD from tracking \c MD. + static void untrack(Metadata *&MD) { untrack(&MD, *MD); } + static void untrack(void *Ref, Metadata &MD); + + /// \brief Move tracking from one reference to another. + /// + /// Semantically equivalent to \c untrack(MD) followed by \c track(New), + /// except that ownership callbacks are maintained. + /// + /// Note: it is an error if \c *MD does not equal \c New. + /// + /// \return true iff tracking is supported by \c MD. + static bool retrack(Metadata *&MD, Metadata *&New) { + return retrack(&MD, *MD, &New); + } + static bool retrack(void *Ref, Metadata &MD, void *New); + + /// \brief Check whether metadata is replaceable. + static bool isReplaceable(const Metadata &MD); + + typedef PointerUnion<MetadataAsValue *, Metadata *> OwnerTy; + +private: + /// \brief Track a reference to metadata for an owner. + /// + /// Generalized version of tracking. + static bool track(void *Ref, Metadata &MD, OwnerTy Owner); +}; + /// \brief Shared implementation of use-lists for replaceable metadata. /// /// Most metadata cannot be RAUW'ed. This is a shared implementation of @@ -572,10 +644,12 @@ struct AAMDNodes { template<> struct DenseMapInfo<AAMDNodes> { static inline AAMDNodes getEmptyKey() { - return AAMDNodes(DenseMapInfo<MDNode *>::getEmptyKey(), 0, 0); + return AAMDNodes(DenseMapInfo<MDNode *>::getEmptyKey(), + nullptr, nullptr); } static inline AAMDNodes getTombstoneKey() { - return AAMDNodes(DenseMapInfo<MDNode *>::getTombstoneKey(), 0, 0); + return AAMDNodes(DenseMapInfo<MDNode *>::getTombstoneKey(), + nullptr, nullptr); } static unsigned getHashValue(const AAMDNodes &Val) { return DenseMapInfo<MDNode *>::getHashValue(Val.TBAA) ^ @@ -830,10 +904,11 @@ public: /// \brief Resolve cycles. /// /// Once all forward declarations have been resolved, force cycles to be - /// resolved. + /// resolved. If \p MDMaterialized is true, then any temporary metadata + /// is ignored, otherwise it asserts when encountering temporary metadata. /// /// \pre No operands (or operands' operands, etc.) have \a isTemporary(). - void resolveCycles(); + void resolveCycles(bool MDMaterialized = true); /// \brief Replace a temporary node with a permanent one. /// @@ -881,6 +956,7 @@ protected: void storeDistinctInContext(); template <class T, class StoreT> static T *storeImpl(T *N, StorageType Storage, StoreT &Store); + template <class T> static T *storeImpl(T *N, StorageType Storage); private: void handleChangedOperand(void *Ref, Metadata *New); @@ -913,13 +989,13 @@ private: N->recalculateHash(); } template <class NodeTy> - static void dispatchRecalculateHash(NodeTy *N, std::false_type) {} + static void dispatchRecalculateHash(NodeTy *, std::false_type) {} template <class NodeTy> static void dispatchResetHash(NodeTy *N, std::true_type) { N->setHash(0); } template <class NodeTy> - static void dispatchResetHash(NodeTy *N, std::false_type) {} + static void dispatchResetHash(NodeTy *, std::false_type) {} public: typedef const MDOperand *op_iterator; @@ -963,6 +1039,8 @@ public: static MDNode *getMostGenericFPMath(MDNode *A, MDNode *B); static MDNode *getMostGenericRange(MDNode *A, MDNode *B); static MDNode *getMostGenericAliasScope(MDNode *A, MDNode *B); + static MDNode *getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B); + }; /// \brief Tuple of metadata. @@ -1125,7 +1203,6 @@ public: /// /// TODO: Inherit from Metadata. class NamedMDNode : public ilist_node<NamedMDNode> { - friend class SymbolTableListTraits<NamedMDNode, Module>; friend struct ilist_traits<NamedMDNode>; friend class LLVMContextImpl; friend class Module; @@ -1193,7 +1270,7 @@ public: void addOperand(MDNode *M); void setOperand(unsigned I, MDNode *New); StringRef getName() const; - void print(raw_ostream &ROS) const; + void print(raw_ostream &ROS, bool IsForDebug = false) const; void dump() const; // --------------------------------------------------------------------------- @@ -1208,13 +1285,13 @@ public: const_op_iterator op_end() const { return const_op_iterator(this, getNumOperands()); } inline iterator_range<op_iterator> operands() { - return iterator_range<op_iterator>(op_begin(), op_end()); + return make_range(op_begin(), op_end()); } inline iterator_range<const_op_iterator> operands() const { - return iterator_range<const_op_iterator>(op_begin(), op_end()); + return make_range(op_begin(), op_end()); } }; } // end llvm namespace -#endif +#endif // LLVM_IR_METADATA_H diff --git a/include/llvm/IR/MetadataTracking.h b/include/llvm/IR/MetadataTracking.h deleted file mode 100644 index 541d9b3b1245..000000000000 --- a/include/llvm/IR/MetadataTracking.h +++ /dev/null @@ -1,99 +0,0 @@ -//===- llvm/IR/MetadataTracking.h - Metadata tracking ---------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Low-level functions to enable tracking of metadata that could RAUW. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_IR_METADATATRACKING_H -#define LLVM_IR_METADATATRACKING_H - -#include "llvm/ADT/PointerUnion.h" -#include "llvm/Support/Casting.h" -#include <type_traits> - -namespace llvm { - -class Metadata; -class MetadataAsValue; - -/// \brief API for tracking metadata references through RAUW and deletion. -/// -/// Shared API for updating \a Metadata pointers in subclasses that support -/// RAUW. -/// -/// This API is not meant to be used directly. See \a TrackingMDRef for a -/// user-friendly tracking reference. -class MetadataTracking { -public: - /// \brief Track the reference to metadata. - /// - /// Register \c MD with \c *MD, if the subclass supports tracking. If \c *MD - /// gets RAUW'ed, \c MD will be updated to the new address. If \c *MD gets - /// deleted, \c MD will be set to \c nullptr. - /// - /// If tracking isn't supported, \c *MD will not change. - /// - /// \return true iff tracking is supported by \c MD. - static bool track(Metadata *&MD) { - return track(&MD, *MD, static_cast<Metadata *>(nullptr)); - } - - /// \brief Track the reference to metadata for \a Metadata. - /// - /// As \a track(Metadata*&), but with support for calling back to \c Owner to - /// tell it that its operand changed. This could trigger \c Owner being - /// re-uniqued. - static bool track(void *Ref, Metadata &MD, Metadata &Owner) { - return track(Ref, MD, &Owner); - } - - /// \brief Track the reference to metadata for \a MetadataAsValue. - /// - /// As \a track(Metadata*&), but with support for calling back to \c Owner to - /// tell it that its operand changed. This could trigger \c Owner being - /// re-uniqued. - static bool track(void *Ref, Metadata &MD, MetadataAsValue &Owner) { - return track(Ref, MD, &Owner); - } - - /// \brief Stop tracking a reference to metadata. - /// - /// Stops \c *MD from tracking \c MD. - static void untrack(Metadata *&MD) { untrack(&MD, *MD); } - static void untrack(void *Ref, Metadata &MD); - - /// \brief Move tracking from one reference to another. - /// - /// Semantically equivalent to \c untrack(MD) followed by \c track(New), - /// except that ownership callbacks are maintained. - /// - /// Note: it is an error if \c *MD does not equal \c New. - /// - /// \return true iff tracking is supported by \c MD. - static bool retrack(Metadata *&MD, Metadata *&New) { - return retrack(&MD, *MD, &New); - } - static bool retrack(void *Ref, Metadata &MD, void *New); - - /// \brief Check whether metadata is replaceable. - static bool isReplaceable(const Metadata &MD); - - typedef PointerUnion<MetadataAsValue *, Metadata *> OwnerTy; - -private: - /// \brief Track a reference to metadata for an owner. - /// - /// Generalized version of tracking. - static bool track(void *Ref, Metadata &MD, OwnerTy Owner); -}; - -} // end namespace llvm - -#endif diff --git a/include/llvm/IR/Module.h b/include/llvm/IR/Module.h index 1668b95c8bd1..942f68543cb6 100644 --- a/include/llvm/IR/Module.h +++ b/include/llvm/IR/Module.h @@ -15,6 +15,7 @@ #ifndef LLVM_IR_MODULE_H #define LLVM_IR_MODULE_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/DataLayout.h" @@ -34,54 +35,6 @@ class LLVMContext; class RandomNumberGenerator; class StructType; -template<> struct ilist_traits<Function> - : public SymbolTableListTraits<Function, Module> { - - // createSentinel is used to get hold of the node that marks the end of the - // list... (same trick used here as in ilist_traits<Instruction>) - Function *createSentinel() const { - return static_cast<Function*>(&Sentinel); - } - static void destroySentinel(Function*) {} - - Function *provideInitialHead() const { return createSentinel(); } - Function *ensureHead(Function*) const { return createSentinel(); } - static void noteHead(Function*, Function*) {} - -private: - mutable ilist_node<Function> Sentinel; -}; - -template<> struct ilist_traits<GlobalVariable> - : public SymbolTableListTraits<GlobalVariable, Module> { - // createSentinel is used to create a node that marks the end of the list. - GlobalVariable *createSentinel() const { - return static_cast<GlobalVariable*>(&Sentinel); - } - static void destroySentinel(GlobalVariable*) {} - - GlobalVariable *provideInitialHead() const { return createSentinel(); } - GlobalVariable *ensureHead(GlobalVariable*) const { return createSentinel(); } - static void noteHead(GlobalVariable*, GlobalVariable*) {} -private: - mutable ilist_node<GlobalVariable> Sentinel; -}; - -template<> struct ilist_traits<GlobalAlias> - : public SymbolTableListTraits<GlobalAlias, Module> { - // createSentinel is used to create a node that marks the end of the list. - GlobalAlias *createSentinel() const { - return static_cast<GlobalAlias*>(&Sentinel); - } - static void destroySentinel(GlobalAlias*) {} - - GlobalAlias *provideInitialHead() const { return createSentinel(); } - GlobalAlias *ensureHead(GlobalAlias*) const { return createSentinel(); } - static void noteHead(GlobalAlias*, GlobalAlias*) {} -private: - mutable ilist_node<GlobalAlias> Sentinel; -}; - template<> struct ilist_traits<NamedMDNode> : public ilist_default_traits<NamedMDNode> { // createSentinel is used to get hold of a node that marks the end of @@ -96,6 +49,7 @@ template<> struct ilist_traits<NamedMDNode> static void noteHead(NamedMDNode*, NamedMDNode*) {} void addNodeToList(NamedMDNode *) {} void removeNodeFromList(NamedMDNode *) {} + private: mutable ilist_node<NamedMDNode> Sentinel; }; @@ -116,11 +70,11 @@ class Module { /// @{ public: /// The type for the list of global variables. - typedef iplist<GlobalVariable> GlobalListType; + typedef SymbolTableList<GlobalVariable> GlobalListType; /// The type for the list of functions. - typedef iplist<Function> FunctionListType; + typedef SymbolTableList<Function> FunctionListType; /// The type for the list of aliases. - typedef iplist<GlobalAlias> AliasListType; + typedef SymbolTableList<GlobalAlias> AliasListType; /// The type for the list of named metadata. typedef ilist<NamedMDNode> NamedMDListType; /// The type of the comdat "symbol" table. @@ -328,6 +282,11 @@ public: /// registered in this LLVMContext. void getMDKindNames(SmallVectorImpl<StringRef> &Result) const; + /// Populate client supplied SmallVector with the bundle tags registered in + /// this LLVMContext. The bundle tags are ordered by increasing bundle IDs. + /// \see LLVMContext::getOperandBundleTagID + void getOperandBundleTags(SmallVectorImpl<StringRef> &Result) const; + /// Return the type with the specified name, or null if there is none by that /// name. StructType *getTypeByName(StringRef Name) const; @@ -472,7 +431,7 @@ public: /// Sets the GVMaterializer to GVM. This module must not yet have a /// Materializer. To reset the materializer for a module that already has one, - /// call MaterializeAllPermanently first. Destroying this module will destroy + /// call materializeAll first. Destroying this module will destroy /// its materializer without materializing any more GlobalValues. Without /// destroying the Module, there is no way to detach or destroy a materializer /// without materializing all the GVs it controls, to avoid leaving orphan @@ -480,27 +439,16 @@ public: void setMaterializer(GVMaterializer *GVM); /// Retrieves the GVMaterializer, if any, for this Module. GVMaterializer *getMaterializer() const { return Materializer.get(); } - - /// Returns true if this GV was loaded from this Module's GVMaterializer and - /// the GVMaterializer knows how to dematerialize the GV. - bool isDematerializable(const GlobalValue *GV) const; + bool isMaterialized() const { return !getMaterializer(); } /// Make sure the GlobalValue is fully read. If the module is corrupt, this /// returns true and fills in the optional string with information about the /// problem. If successful, this returns false. std::error_code materialize(GlobalValue *GV); - /// If the GlobalValue is read in, and if the GVMaterializer supports it, - /// release the memory for the function, and set it up to be materialized - /// lazily. If !isDematerializable(), this method is a no-op. - void dematerialize(GlobalValue *GV); - - /// Make sure all GlobalValues in this Module are fully read. - std::error_code materializeAll(); /// Make sure all GlobalValues in this Module are fully read and clear the - /// Materializer. If the module is corrupt, this DOES NOT clear the old /// Materializer. - std::error_code materializeAllPermanently(); + std::error_code materializeAll(); std::error_code materializeMetadata(); @@ -556,10 +504,10 @@ public: bool global_empty() const { return GlobalList.empty(); } iterator_range<global_iterator> globals() { - return iterator_range<global_iterator>(global_begin(), global_end()); + return make_range(global_begin(), global_end()); } iterator_range<const_global_iterator> globals() const { - return iterator_range<const_global_iterator>(global_begin(), global_end()); + return make_range(global_begin(), global_end()); } /// @} @@ -578,10 +526,10 @@ public: bool empty() const { return FunctionList.empty(); } iterator_range<iterator> functions() { - return iterator_range<iterator>(begin(), end()); + return make_range(begin(), end()); } iterator_range<const_iterator> functions() const { - return iterator_range<const_iterator>(begin(), end()); + return make_range(begin(), end()); } /// @} @@ -596,10 +544,10 @@ public: bool alias_empty() const { return AliasList.empty(); } iterator_range<alias_iterator> aliases() { - return iterator_range<alias_iterator>(alias_begin(), alias_end()); + return make_range(alias_begin(), alias_end()); } iterator_range<const_alias_iterator> aliases() const { - return iterator_range<const_alias_iterator>(alias_begin(), alias_end()); + return make_range(alias_begin(), alias_end()); } /// @} @@ -620,12 +568,10 @@ public: bool named_metadata_empty() const { return NamedMDList.empty(); } iterator_range<named_metadata_iterator> named_metadata() { - return iterator_range<named_metadata_iterator>(named_metadata_begin(), - named_metadata_end()); + return make_range(named_metadata_begin(), named_metadata_end()); } iterator_range<const_named_metadata_iterator> named_metadata() const { - return iterator_range<const_named_metadata_iterator>(named_metadata_begin(), - named_metadata_end()); + return make_range(named_metadata_begin(), named_metadata_end()); } /// Destroy ConstantArrays in LLVMContext if they are not used. @@ -646,11 +592,12 @@ public: /// uselistorder directives so that use-lists can be recreated when reading /// the assembly. void print(raw_ostream &OS, AssemblyAnnotationWriter *AAW, - bool ShouldPreserveUseListOrder = false) const; + bool ShouldPreserveUseListOrder = false, + bool IsForDebug = false) const; /// Dump the module to stderr (for debugging). void dump() const; - + /// This function causes all the subinstructions to "let go" of all references /// that they are maintaining. This allows one to 'delete' a whole class at /// a time, even though there may be circular references... first all @@ -666,6 +613,10 @@ public: /// \brief Returns the Dwarf Version by checking module flags. unsigned getDwarfVersion() const; + /// \brief Returns the CodeView Version by checking module flags. + /// Returns zero if not present in module. + unsigned getCodeViewFlag() const; + /// @} /// @name Utility functions for querying and setting PIC level /// @{ @@ -676,6 +627,16 @@ public: /// \brief Set the PIC level (small or large model) void setPICLevel(PICLevel::Level PL); /// @} + + /// @name Utility functions for querying and setting PGO counts + /// @{ + + /// \brief Set maximum function count in PGO mode + void setMaximumFunctionCount(uint64_t); + + /// \brief Returns maximum function count in PGO mode + Optional<uint64_t> getMaximumFunctionCount(); + /// @} }; /// An raw_ostream inserter for modules. @@ -693,7 +654,7 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Module, LLVMModuleRef) inline Module *unwrap(LLVMModuleProviderRef MP) { return reinterpret_cast<Module*>(MP); } - + } // End llvm namespace #endif diff --git a/include/llvm/IR/ModuleSlotTracker.h b/include/llvm/IR/ModuleSlotTracker.h index c37dcecf8e40..49730a66bdf6 100644 --- a/include/llvm/IR/ModuleSlotTracker.h +++ b/include/llvm/IR/ModuleSlotTracker.h @@ -17,6 +17,7 @@ namespace llvm { class Module; class Function; class SlotTracker; +class Value; /// Manage lifetime of a slot tracker for printing IR. /// @@ -61,6 +62,13 @@ public: /// Purge the currently incorporated function and incorporate \c F. If \c F /// is currently incorporated, this is a no-op. void incorporateFunction(const Function &F); + + /// Return the slot number of the specified local value. + /// + /// A function that defines this value should be incorporated prior to calling + /// this method. + /// Return -1 if the value is not in the function's SlotTracker. + int getLocalSlot(const Value *V); }; } // end namespace llvm diff --git a/include/llvm/IR/PassManager.h b/include/llvm/IR/PassManager.h index 4166babd63e5..2ceb53d21b7a 100644 --- a/include/llvm/IR/PassManager.h +++ b/include/llvm/IR/PassManager.h @@ -203,7 +203,8 @@ public: for (unsigned Idx = 0, Size = Passes.size(); Idx != Size; ++Idx) { if (DebugLogging) - dbgs() << "Running pass: " << Passes[Idx]->name() << "\n"; + dbgs() << "Running pass: " << Passes[Idx]->name() << " on " + << IR.getName() << "\n"; PreservedAnalyses PassPA = Passes[Idx]->run(IR, AM); diff --git a/include/llvm/IR/PatternMatch.h b/include/llvm/IR/PatternMatch.h index 41154e6441a9..f4d7d8c44416 100644 --- a/include/llvm/IR/PatternMatch.h +++ b/include/llvm/IR/PatternMatch.h @@ -1272,6 +1272,46 @@ inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0, return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1); } +template <typename Opnd_t> struct Signum_match { + Opnd_t Val; + Signum_match(const Opnd_t &V) : Val(V) {} + + template <typename OpTy> bool match(OpTy *V) { + unsigned TypeSize = V->getType()->getScalarSizeInBits(); + if (TypeSize == 0) + return false; + + unsigned ShiftWidth = TypeSize - 1; + Value *OpL = nullptr, *OpR = nullptr; + + // This is the representation of signum we match: + // + // signum(x) == (x >> 63) | (-x >>u 63) + // + // An i1 value is its own signum, so it's correct to match + // + // signum(x) == (x >> 0) | (-x >>u 0) + // + // for i1 values. + + auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth)); + auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth)); + auto Signum = m_Or(LHS, RHS); + + return Signum.match(V) && OpL == OpR && Val.match(OpL); + } +}; + +/// \brief Matches a signum pattern. +/// +/// signum(x) = +/// x > 0 -> 1 +/// x == 0 -> 0 +/// x < 0 -> -1 +template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { + return Signum_match<Val_t>(V); +} + } // end namespace PatternMatch } // end namespace llvm diff --git a/include/llvm/IR/Statepoint.h b/include/llvm/IR/Statepoint.h index 4ab1f8497adb..7310c5697a7e 100644 --- a/include/llvm/IR/Statepoint.h +++ b/include/llvm/IR/Statepoint.h @@ -173,7 +173,7 @@ public: /// range adapter for call arguments iterator_range<arg_iterator> call_args() const { - return iterator_range<arg_iterator>(arg_begin(), arg_end()); + return make_range(arg_begin(), arg_end()); } /// \brief Return true if the call or the callee has the given attribute. @@ -201,8 +201,7 @@ public: /// range adapter for GC transition arguments iterator_range<arg_iterator> gc_transition_args() const { - return iterator_range<arg_iterator>(gc_transition_args_begin(), - gc_transition_args_end()); + return make_range(gc_transition_args_begin(), gc_transition_args_end()); } /// Number of additional arguments excluding those intended @@ -225,7 +224,7 @@ public: /// range adapter for vm state arguments iterator_range<arg_iterator> vm_state_args() const { - return iterator_range<arg_iterator>(vm_state_begin(), vm_state_end()); + return make_range(vm_state_begin(), vm_state_end()); } typename CallSiteTy::arg_iterator gc_args_begin() const { @@ -235,9 +234,13 @@ public: return getCallSite().arg_end(); } + unsigned gcArgsStartIdx() const { + return gc_args_begin() - getInstruction()->op_begin(); + } + /// range adapter for gc arguments iterator_range<arg_iterator> gc_args() const { - return iterator_range<arg_iterator>(gc_args_begin(), gc_args_end()); + return make_range(gc_args_begin(), gc_args_end()); } /// Get list of all gc reloactes linked to this statepoint @@ -320,7 +323,7 @@ public: bool isTiedToInvoke() const { const Value *Token = RelocateCS.getArgument(0); - return isa<ExtractValueInst>(Token) || isa<InvokeInst>(Token); + return isa<LandingPadInst>(Token) || isa<InvokeInst>(Token); } /// Get enclosed relocate intrinsic @@ -332,7 +335,7 @@ public: // This takes care both of relocates for call statepoints and relocates // on normal path of invoke statepoint. - if (!isa<ExtractValueInst>(Token)) { + if (!isa<LandingPadInst>(Token)) { return cast<Instruction>(Token); } @@ -396,16 +399,10 @@ StatepointBase<FunTy, InstructionTy, ValueTy, CallSiteTy>::getRelocates() LandingPadInst *LandingPad = cast<InvokeInst>(getInstruction())->getLandingPadInst(); - // Search for extract value from landingpad instruction to which - // gc relocates will be attached + // Search for gc relocates that are attached to this landingpad. for (const User *LandingPadUser : LandingPad->users()) { - if (!isa<ExtractValueInst>(LandingPadUser)) - continue; - - // gc relocates should be attached to this extract value - for (const User *U : LandingPadUser->users()) - if (isGCRelocate(U)) - Result.push_back(GCRelocateOperands(U)); + if (isGCRelocate(LandingPadUser)) + Result.push_back(GCRelocateOperands(LandingPadUser)); } return Result; } diff --git a/include/llvm/IR/SymbolTableListTraits.h b/include/llvm/IR/SymbolTableListTraits.h index 0a5149c3d938..5fc48d10d63f 100644 --- a/include/llvm/IR/SymbolTableListTraits.h +++ b/include/llvm/IR/SymbolTableListTraits.h @@ -29,31 +29,66 @@ namespace llvm { class ValueSymbolTable; - -template<typename NodeTy> class ilist_iterator; -template<typename NodeTy, typename Traits> class iplist; -template<typename Ty> struct ilist_traits; + +template <typename NodeTy> class ilist_iterator; +template <typename NodeTy, typename Traits> class iplist; +template <typename Ty> struct ilist_traits; + +template <typename NodeTy> +struct SymbolTableListSentinelTraits + : public ilist_embedded_sentinel_traits<NodeTy> {}; + +/// Template metafunction to get the parent type for a symbol table list. +/// +/// Implementations create a typedef called \c type so that we only need a +/// single template parameter for the list and traits. +template <typename NodeTy> struct SymbolTableListParentType {}; +class Argument; +class BasicBlock; +class Function; +class Instruction; +class GlobalVariable; +class GlobalAlias; +class Module; +#define DEFINE_SYMBOL_TABLE_PARENT_TYPE(NODE, PARENT) \ + template <> struct SymbolTableListParentType<NODE> { typedef PARENT type; }; +DEFINE_SYMBOL_TABLE_PARENT_TYPE(Instruction, BasicBlock) +DEFINE_SYMBOL_TABLE_PARENT_TYPE(BasicBlock, Function) +DEFINE_SYMBOL_TABLE_PARENT_TYPE(Argument, Function) +DEFINE_SYMBOL_TABLE_PARENT_TYPE(Function, Module) +DEFINE_SYMBOL_TABLE_PARENT_TYPE(GlobalVariable, Module) +DEFINE_SYMBOL_TABLE_PARENT_TYPE(GlobalAlias, Module) +#undef DEFINE_SYMBOL_TABLE_PARENT_TYPE + +template <typename NodeTy> class SymbolTableList; // ValueSubClass - The type of objects that I hold, e.g. Instruction. // ItemParentClass - The type of object that owns the list, e.g. BasicBlock. // -template<typename ValueSubClass, typename ItemParentClass> -class SymbolTableListTraits : public ilist_default_traits<ValueSubClass> { - typedef ilist_traits<ValueSubClass> TraitsClass; +template <typename ValueSubClass> +class SymbolTableListTraits + : public ilist_nextprev_traits<ValueSubClass>, + public SymbolTableListSentinelTraits<ValueSubClass>, + public ilist_node_traits<ValueSubClass> { + typedef SymbolTableList<ValueSubClass> ListTy; + typedef + typename SymbolTableListParentType<ValueSubClass>::type ItemParentClass; + public: SymbolTableListTraits() {} +private: /// getListOwner - Return the object that owns this list. If this is a list /// of instructions, it returns the BasicBlock that owns them. ItemParentClass *getListOwner() { size_t Offset(size_t(&((ItemParentClass*)nullptr->*ItemParentClass:: getSublistAccess(static_cast<ValueSubClass*>(nullptr))))); - iplist<ValueSubClass>* Anchor(static_cast<iplist<ValueSubClass>*>(this)); + ListTy *Anchor(static_cast<ListTy *>(this)); return reinterpret_cast<ItemParentClass*>(reinterpret_cast<char*>(Anchor)- Offset); } - static iplist<ValueSubClass> &getList(ItemParentClass *Par) { + static ListTy &getList(ItemParentClass *Par) { return Par->*(Par->getSublistAccess((ValueSubClass*)nullptr)); } @@ -61,9 +96,10 @@ public: return Par ? toPtr(Par->getValueSymbolTable()) : nullptr; } +public: void addNodeToList(ValueSubClass *V); void removeNodeFromList(ValueSubClass *V); - void transferNodesFromList(ilist_traits<ValueSubClass> &L2, + void transferNodesFromList(SymbolTableListTraits &L2, ilist_iterator<ValueSubClass> first, ilist_iterator<ValueSubClass> last); //private: @@ -73,6 +109,14 @@ public: static ValueSymbolTable *toPtr(ValueSymbolTable &R) { return &R; } }; +/// List that automatically updates parent links and symbol tables. +/// +/// When nodes are inserted into and removed from this list, the associated +/// symbol table will be automatically updated. Similarly, parent links get +/// updated automatically. +template <typename NodeTy> +class SymbolTableList : public iplist<NodeTy, SymbolTableListTraits<NodeTy>> {}; + } // End llvm namespace #endif diff --git a/include/llvm/IR/TrackingMDRef.h b/include/llvm/IR/TrackingMDRef.h index e24112154e16..97efaff7a377 100644 --- a/include/llvm/IR/TrackingMDRef.h +++ b/include/llvm/IR/TrackingMDRef.h @@ -14,15 +14,11 @@ #ifndef LLVM_IR_TRACKINGMDREF_H #define LLVM_IR_TRACKINGMDREF_H -#include "llvm/IR/MetadataTracking.h" +#include "llvm/IR/Metadata.h" #include "llvm/Support/Casting.h" namespace llvm { -class Metadata; -class MDNode; -class ValueAsMetadata; - /// \brief Tracking metadata reference. /// /// This class behaves like \a TrackingVH, but for metadata. diff --git a/include/llvm/IR/Type.h b/include/llvm/IR/Type.h index 6ab0bd0631a0..b2920dd3de63 100644 --- a/include/llvm/IR/Type.h +++ b/include/llvm/IR/Type.h @@ -15,7 +15,6 @@ #ifndef LLVM_IR_TYPE_H #define LLVM_IR_TYPE_H -#include "llvm-c/Core.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/CBindingWrapping.h" @@ -38,10 +37,10 @@ template<class GraphType> struct GraphTraits; /// they are never changed. Also note that only one instance of a particular /// type is ever created. Thus seeing if two types are equal is a matter of /// doing a trivial pointer comparison. To enforce that no two equal instances -/// are created, Type instances can only be created via static factory methods +/// are created, Type instances can only be created via static factory methods /// in class Type and in derived classes. Once allocated, Types are never /// free'd. -/// +/// class Type { public: //===--------------------------------------------------------------------===// @@ -63,45 +62,36 @@ public: LabelTyID, ///< 7: Labels MetadataTyID, ///< 8: Metadata X86_MMXTyID, ///< 9: MMX vectors (64 bits, X86 specific) + TokenTyID, ///< 10: Tokens // Derived types... see DerivedTypes.h file. // Make sure FirstDerivedTyID stays up to date! - IntegerTyID, ///< 10: Arbitrary bit width integers - FunctionTyID, ///< 11: Functions - StructTyID, ///< 12: Structures - ArrayTyID, ///< 13: Arrays - PointerTyID, ///< 14: Pointers - VectorTyID ///< 15: SIMD 'packed' format, or other vector type + IntegerTyID, ///< 11: Arbitrary bit width integers + FunctionTyID, ///< 12: Functions + StructTyID, ///< 13: Structures + ArrayTyID, ///< 14: Arrays + PointerTyID, ///< 15: Pointers + VectorTyID ///< 16: SIMD 'packed' format, or other vector type }; private: /// Context - This refers to the LLVMContext in which this type was uniqued. LLVMContext &Context; - // Due to Ubuntu GCC bug 910363: - // https://bugs.launchpad.net/ubuntu/+source/gcc-4.5/+bug/910363 - // Bitpack ID and SubclassData manually. - // Note: TypeID : low 8 bit; SubclassData : high 24 bit. - uint32_t IDAndSubclassData; + TypeID ID : 8; // The current base type of this type. + unsigned SubclassData : 24; // Space for subclasses to store data. protected: friend class LLVMContextImpl; explicit Type(LLVMContext &C, TypeID tid) - : Context(C), IDAndSubclassData(0), - NumContainedTys(0), ContainedTys(nullptr) { - setTypeID(tid); - } + : Context(C), ID(tid), SubclassData(0), + NumContainedTys(0), ContainedTys(nullptr) {} ~Type() = default; - void setTypeID(TypeID ID) { - IDAndSubclassData = (ID & 0xFF) | (IDAndSubclassData & 0xFFFFFF00); - assert(getTypeID() == ID && "TypeID data too large for field"); - } - - unsigned getSubclassData() const { return IDAndSubclassData >> 8; } - + unsigned getSubclassData() const { return SubclassData; } + void setSubclassData(unsigned val) { - IDAndSubclassData = (IDAndSubclassData & 0xFF) | (val << 8); + SubclassData = val; // Ensure we don't have any accidental truncation. assert(getSubclassData() == val && "Subclass data too large for field"); } @@ -118,7 +108,7 @@ protected: Type * const *ContainedTys; public: - void print(raw_ostream &O) const; + void print(raw_ostream &O, bool IsForDebug = false) const; void dump() const; /// getContext - Return the LLVMContext in which this type was uniqued. @@ -131,7 +121,7 @@ public: /// getTypeID - Return the type id for the type. This will return one /// of the TypeID enum elements defined above. /// - TypeID getTypeID() const { return (TypeID)(IDAndSubclassData & 0xFF); } + TypeID getTypeID() const { return ID; } /// isVoidTy - Return true if this is 'void'. bool isVoidTy() const { return getTypeID() == VoidTyID; } @@ -141,7 +131,7 @@ public: /// isFloatTy - Return true if this is 'float', a 32-bit IEEE fp type. bool isFloatTy() const { return getTypeID() == FloatTyID; } - + /// isDoubleTy - Return true if this is 'double', a 64-bit IEEE fp type. bool isDoubleTy() const { return getTypeID() == DoubleTyID; } @@ -181,16 +171,19 @@ public: /// isFPOrFPVectorTy - Return true if this is a FP type or a vector of FP. /// bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); } - + /// isLabelTy - Return true if this is 'label'. bool isLabelTy() const { return getTypeID() == LabelTyID; } /// isMetadataTy - Return true if this is 'metadata'. bool isMetadataTy() const { return getTypeID() == MetadataTyID; } + /// isTokenTy - Return true if this is 'token'. + bool isTokenTy() const { return getTypeID() == TokenTyID; } + /// isIntegerTy - True if this is an instance of IntegerType. /// - bool isIntegerTy() const { return getTypeID() == IntegerTyID; } + bool isIntegerTy() const { return getTypeID() == IntegerTyID; } /// isIntegerTy - Return true if this is an IntegerType of the given width. bool isIntegerTy(unsigned Bitwidth) const; @@ -199,7 +192,7 @@ public: /// integer types. /// bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); } - + /// isFunctionTy - True if this is an instance of FunctionType. /// bool isFunctionTy() const { return getTypeID() == FunctionTyID; } @@ -220,14 +213,14 @@ public: /// pointer types. /// bool isPtrOrPtrVectorTy() const { return getScalarType()->isPointerTy(); } - + /// isVectorTy - True if this is an instance of VectorType. /// bool isVectorTy() const { return getTypeID() == VectorTyID; } - /// canLosslesslyBitCastTo - Return true if this type could be converted - /// with a lossless BitCast to type 'Ty'. For example, i8* to i32*. BitCasts - /// are valid for types of the same size only where no re-interpretation of + /// canLosslesslyBitCastTo - Return true if this type could be converted + /// with a lossless BitCast to type 'Ty'. For example, i8* to i32*. BitCasts + /// are valid for types of the same size only where no re-interpretation of /// the bits is done. /// @brief Determine if this type could be losslessly bitcast to Ty bool canLosslesslyBitCastTo(Type *Ty) const; @@ -265,7 +258,7 @@ public: /// get the actual size for a particular target, it is reasonable to use the /// DataLayout subsystem to do this. /// - bool isSized(SmallPtrSetImpl<const Type*> *Visited = nullptr) const { + bool isSized(SmallPtrSetImpl<Type*> *Visited = nullptr) const { // If it's a primitive, it is always sized. if (getTypeID() == IntegerTyID || isFloatingPointTy() || getTypeID() == PointerTyID || @@ -304,8 +297,7 @@ public: /// getScalarType - If this is a vector type, return the element type, /// otherwise return 'this'. - const Type *getScalarType() const LLVM_READONLY; - Type *getScalarType() LLVM_READONLY; + Type *getScalarType() const LLVM_READONLY; //===--------------------------------------------------------------------===// // Type Iteration support. @@ -344,30 +336,30 @@ public: // example) is shorthand for cast<VectorType>(Ty)->getNumElements(). This is // only intended to cover the core methods that are frequently used, helper // methods should not be added here. - - unsigned getIntegerBitWidth() const; - - Type *getFunctionParamType(unsigned i) const; - unsigned getFunctionNumParams() const; - bool isFunctionVarArg() const; - - StringRef getStructName() const; - unsigned getStructNumElements() const; - Type *getStructElementType(unsigned N) const; - - Type *getSequentialElementType() const; - - uint64_t getArrayNumElements() const; + + inline unsigned getIntegerBitWidth() const; + + inline Type *getFunctionParamType(unsigned i) const; + inline unsigned getFunctionNumParams() const; + inline bool isFunctionVarArg() const; + + inline StringRef getStructName() const; + inline unsigned getStructNumElements() const; + inline Type *getStructElementType(unsigned N) const; + + inline Type *getSequentialElementType() const; + + inline uint64_t getArrayNumElements() const; Type *getArrayElementType() const { return getSequentialElementType(); } - unsigned getVectorNumElements() const; + inline unsigned getVectorNumElements() const; Type *getVectorElementType() const { return getSequentialElementType(); } Type *getPointerElementType() const { return getSequentialElementType(); } /// \brief Get the address space of this pointer or pointer vector type. - unsigned getPointerAddressSpace() const; - + inline unsigned getPointerAddressSpace() const; + //===--------------------------------------------------------------------===// // Static members exported by the Type class itself. Useful for getting // instances of Type. @@ -389,6 +381,7 @@ public: static Type *getFP128Ty(LLVMContext &C); static Type *getPPC_FP128Ty(LLVMContext &C); static Type *getX86_MMXTy(LLVMContext &C); + static Type *getTokenTy(LLVMContext &C); static IntegerType *getIntNTy(LLVMContext &C, unsigned N); static IntegerType *getInt1Ty(LLVMContext &C); static IntegerType *getInt8Ty(LLVMContext &C); @@ -396,7 +389,7 @@ public: static IntegerType *getInt32Ty(LLVMContext &C); static IntegerType *getInt64Ty(LLVMContext &C); static IntegerType *getInt128Ty(LLVMContext &C); - + //===--------------------------------------------------------------------===// // Convenience methods for getting pointer types with one of the above builtin // types as pointee. @@ -417,13 +410,13 @@ public: /// getPointerTo - Return a pointer to the current type. This is equivalent /// to PointerType::get(Foo, AddrSpace). - PointerType *getPointerTo(unsigned AddrSpace = 0); + PointerType *getPointerTo(unsigned AddrSpace = 0) const; private: /// isSizedDerivedType - Derived types like structures and arrays are sized /// iff all of the members of the type are sized as well. Since asking for /// their size is relatively uncommon, move this operation out of line. - bool isSizedDerivedType(SmallPtrSetImpl<const Type*> *Visited = nullptr) const; + bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const; }; // Printing of types. @@ -439,13 +432,11 @@ template <> struct isa_impl<PointerType, Type> { } }; - //===----------------------------------------------------------------------===// // Provide specializations of GraphTraits to be able to treat a type as a // graph of sub types. - -template <> struct GraphTraits<Type*> { +template <> struct GraphTraits<Type *> { typedef Type NodeType; typedef Type::subtype_iterator ChildIteratorType; @@ -483,7 +474,7 @@ inline Type **unwrap(LLVMTypeRef* Tys) { inline LLVMTypeRef *wrap(Type **Tys) { return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys)); } - + } // End llvm namespace #endif diff --git a/include/llvm/IR/TypeFinder.h b/include/llvm/IR/TypeFinder.h index 73a63ad0349e..5f3854377c16 100644 --- a/include/llvm/IR/TypeFinder.h +++ b/include/llvm/IR/TypeFinder.h @@ -7,7 +7,7 @@ // //===----------------------------------------------------------------------===// // -// This file contains the declaration of the TypeFinder class. +// This file contains the declaration of the TypeFinder class. // //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/Use.h b/include/llvm/IR/Use.h index 160d71b03e7f..a738677f8e5b 100644 --- a/include/llvm/IR/Use.h +++ b/include/llvm/IR/Use.h @@ -25,7 +25,6 @@ #ifndef LLVM_IR_USE_H #define LLVM_IR_USE_H -#include "llvm-c/Core.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Compiler.h" diff --git a/include/llvm/IR/UseListOrder.h b/include/llvm/IR/UseListOrder.h index b7c2418d348d..1cabf03d1b00 100644 --- a/include/llvm/IR/UseListOrder.h +++ b/include/llvm/IR/UseListOrder.h @@ -34,7 +34,7 @@ struct UseListOrder { UseListOrder(const Value *V, const Function *F, size_t ShuffleSize) : V(V), F(F), Shuffle(ShuffleSize) {} - UseListOrder() : V(0), F(0) {} + UseListOrder() : V(nullptr), F(nullptr) {} UseListOrder(UseListOrder &&X) : V(X.V), F(X.F), Shuffle(std::move(X.Shuffle)) {} UseListOrder &operator=(UseListOrder &&X) { @@ -53,4 +53,4 @@ typedef std::vector<UseListOrder> UseListOrderStack; } // end namespace llvm -#endif +#endif // LLVM_IR_USELISTORDER_H diff --git a/include/llvm/IR/User.h b/include/llvm/IR/User.h index 93614fab5759..885ae197d228 100644 --- a/include/llvm/IR/User.h +++ b/include/llvm/IR/User.h @@ -19,6 +19,7 @@ #ifndef LLVM_IR_USER_H #define LLVM_IR_USER_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Value.h" @@ -39,6 +40,9 @@ class User : public Value { friend struct HungoffOperandTraits; virtual void anchor(); + LLVM_ATTRIBUTE_ALWAYS_INLINE inline static void * + allocateFixedOperandUser(size_t, unsigned, unsigned); + protected: /// Allocate a User with an operand pointer co-allocated. /// @@ -51,7 +55,17 @@ protected: /// This is used for subclasses which have a fixed number of operands. void *operator new(size_t Size, unsigned Us); - User(Type *ty, unsigned vty, Use *OpList, unsigned NumOps) + /// Allocate a User with the operands co-allocated. If DescBytes is non-zero + /// then allocate an additional DescBytes bytes before the operands. These + /// bytes can be accessed by calling getDescriptor. + /// + /// DescBytes needs to be divisible by sizeof(void *). The allocated + /// descriptor, if any, is aligned to sizeof(void *) bytes. + /// + /// This is used for subclasses which have a fixed number of operands. + void *operator new(size_t Size, unsigned Us, unsigned DescBytes); + + User(Type *ty, unsigned vty, Use *, unsigned NumOps) : Value(ty, vty) { assert(NumOps < (1u << NumUserOperandsBits) && "Too many operands"); NumUserOperands = NumOps; @@ -137,6 +151,12 @@ public: unsigned getNumOperands() const { return NumUserOperands; } + /// Returns the descriptor co-allocated with this User instance. + ArrayRef<const uint8_t> getDescriptor() const; + + /// Returns the descriptor co-allocated with this User instance. + MutableArrayRef<uint8_t> getDescriptor(); + /// Set the number of operands on a GlobalVariable. /// /// GlobalVariable always allocates space for a single operands, but @@ -150,19 +170,6 @@ public: NumUserOperands = NumOps; } - /// Set the number of operands on a Function. - /// - /// Function always allocates space for a single operands, but - /// doesn't always use it. - /// - /// FIXME: As that the number of operands is used to find the start of - /// the allocated memory in operator delete, we need to always think we have - /// 1 operand before delete. - void setFunctionNumOperands(unsigned NumOps) { - assert(NumOps <= 1 && "Function can only have 0 or 1 operands"); - NumUserOperands = NumOps; - } - /// \brief Subclasses with hung off uses need to manage the operand count /// themselves. In these instances, the operand count isn't used to find the /// OperandList, so there's no issue in having the operand count change. @@ -213,7 +220,7 @@ public: return value_op_iterator(op_end()); } iterator_range<value_op_iterator> operand_values() { - return iterator_range<value_op_iterator>(value_op_begin(), value_op_end()); + return make_range(value_op_begin(), value_op_end()); } /// \brief Drop all references to operands. diff --git a/include/llvm/IR/Value.def b/include/llvm/IR/Value.def index c2a0639603ed..4c5d452fc3c3 100644 --- a/include/llvm/IR/Value.def +++ b/include/llvm/IR/Value.def @@ -70,6 +70,7 @@ HANDLE_CONSTANT(ConstantArray) HANDLE_CONSTANT(ConstantStruct) HANDLE_CONSTANT(ConstantVector) HANDLE_CONSTANT(ConstantPointerNull) +HANDLE_CONSTANT(ConstantTokenNone) HANDLE_METADATA_VALUE(MetadataAsValue) HANDLE_INLINE_ASM_VALUE(InlineAsm) @@ -79,7 +80,7 @@ HANDLE_INSTRUCTION(Instruction) // don't add new values here! HANDLE_CONSTANT_MARKER(ConstantFirstVal, Function) -HANDLE_CONSTANT_MARKER(ConstantLastVal, ConstantPointerNull) +HANDLE_CONSTANT_MARKER(ConstantLastVal, ConstantTokenNone) #undef HANDLE_GLOBAL_VALUE #undef HANDLE_CONSTANT diff --git a/include/llvm/IR/Value.h b/include/llvm/IR/Value.h index 17a80c82d1bc..bb7ff278fdef 100644 --- a/include/llvm/IR/Value.h +++ b/include/llvm/IR/Value.h @@ -14,7 +14,6 @@ #ifndef LLVM_IR_VALUE_H #define LLVM_IR_VALUE_H -#include "llvm-c/Core.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Use.h" #include "llvm/Support/CBindingWrapping.h" @@ -104,12 +103,13 @@ protected: /// /// Note, this should *NOT* be used directly by any class other than User. /// User uses this value to find the Use list. - enum : unsigned { NumUserOperandsBits = 29 }; + enum : unsigned { NumUserOperandsBits = 28 }; unsigned NumUserOperands : NumUserOperandsBits; bool IsUsedByMD : 1; bool HasName : 1; bool HasHungOffUses : 1; + bool HasDescriptor : 1; private: template <typename UseT> // UseT == 'Use' or 'const Use' @@ -201,8 +201,9 @@ public: /// \brief Implement operator<< on Value. /// @{ - void print(raw_ostream &O) const; - void print(raw_ostream &O, ModuleSlotTracker &MST) const; + void print(raw_ostream &O, bool IsForDebug = false) const; + void print(raw_ostream &O, ModuleSlotTracker &MST, + bool IsForDebug = false) const; /// @} /// \brief Print the name of this Value out to the specified raw_ostream. @@ -272,36 +273,91 @@ public: //---------------------------------------------------------------------- // Methods for handling the chain of uses of this Value. // - bool use_empty() const { return UseList == nullptr; } + // Materializing a function can introduce new uses, so these methods come in + // two variants: + // The methods that start with materialized_ check the uses that are + // currently known given which functions are materialized. Be very careful + // when using them since you might not get all uses. + // The methods that don't start with materialized_ assert that modules is + // fully materialized. +#ifdef NDEBUG + void assertModuleIsMaterialized() const {} +#else + void assertModuleIsMaterialized() const; +#endif + + bool use_empty() const { + assertModuleIsMaterialized(); + return UseList == nullptr; + } - typedef use_iterator_impl<Use> use_iterator; + typedef use_iterator_impl<Use> use_iterator; typedef use_iterator_impl<const Use> const_use_iterator; - use_iterator use_begin() { return use_iterator(UseList); } - const_use_iterator use_begin() const { return const_use_iterator(UseList); } - use_iterator use_end() { return use_iterator(); } - const_use_iterator use_end() const { return const_use_iterator(); } + use_iterator materialized_use_begin() { return use_iterator(UseList); } + const_use_iterator materialized_use_begin() const { + return const_use_iterator(UseList); + } + use_iterator use_begin() { + assertModuleIsMaterialized(); + return materialized_use_begin(); + } + const_use_iterator use_begin() const { + assertModuleIsMaterialized(); + return materialized_use_begin(); + } + use_iterator use_end() { return use_iterator(); } + const_use_iterator use_end() const { return const_use_iterator(); } + iterator_range<use_iterator> materialized_uses() { + return make_range(materialized_use_begin(), use_end()); + } + iterator_range<const_use_iterator> materialized_uses() const { + return make_range(materialized_use_begin(), use_end()); + } iterator_range<use_iterator> uses() { - return iterator_range<use_iterator>(use_begin(), use_end()); + assertModuleIsMaterialized(); + return materialized_uses(); } iterator_range<const_use_iterator> uses() const { - return iterator_range<const_use_iterator>(use_begin(), use_end()); + assertModuleIsMaterialized(); + return materialized_uses(); } - bool user_empty() const { return UseList == nullptr; } + bool user_empty() const { + assertModuleIsMaterialized(); + return UseList == nullptr; + } - typedef user_iterator_impl<User> user_iterator; + typedef user_iterator_impl<User> user_iterator; typedef user_iterator_impl<const User> const_user_iterator; - user_iterator user_begin() { return user_iterator(UseList); } - const_user_iterator user_begin() const { return const_user_iterator(UseList); } - user_iterator user_end() { return user_iterator(); } - const_user_iterator user_end() const { return const_user_iterator(); } - User *user_back() { return *user_begin(); } - const User *user_back() const { return *user_begin(); } + user_iterator materialized_user_begin() { return user_iterator(UseList); } + const_user_iterator materialized_user_begin() const { + return const_user_iterator(UseList); + } + user_iterator user_begin() { + assertModuleIsMaterialized(); + return materialized_user_begin(); + } + const_user_iterator user_begin() const { + assertModuleIsMaterialized(); + return materialized_user_begin(); + } + user_iterator user_end() { return user_iterator(); } + const_user_iterator user_end() const { return const_user_iterator(); } + User *user_back() { + assertModuleIsMaterialized(); + return *materialized_user_begin(); + } + const User *user_back() const { + assertModuleIsMaterialized(); + return *materialized_user_begin(); + } iterator_range<user_iterator> users() { - return iterator_range<user_iterator>(user_begin(), user_end()); + assertModuleIsMaterialized(); + return make_range(materialized_user_begin(), user_end()); } iterator_range<const_user_iterator> users() const { - return iterator_range<const_user_iterator>(user_begin(), user_end()); + assertModuleIsMaterialized(); + return make_range(materialized_user_begin(), user_end()); } /// \brief Return true if there is exactly one user of this value. @@ -493,7 +549,28 @@ private: template <class Compare> static Use *mergeUseLists(Use *L, Use *R, Compare Cmp) { Use *Merged; - mergeUseListsImpl(L, R, &Merged, Cmp); + Use **Next = &Merged; + + for (;;) { + if (!L) { + *Next = R; + break; + } + if (!R) { + *Next = L; + break; + } + if (Cmp(*R, *L)) { + *Next = R; + Next = &R->Next; + R = R->Next; + } else { + *Next = L; + Next = &L->Next; + L = L->Next; + } + } + return Merged; } @@ -586,25 +663,6 @@ template <class Compare> void Value::sortUseList(Compare Cmp) { } } -template <class Compare> -void Value::mergeUseListsImpl(Use *L, Use *R, Use **Next, Compare Cmp) { - if (!L) { - *Next = R; - return; - } - if (!R) { - *Next = L; - return; - } - if (Cmp(*R, *L)) { - *Next = R; - mergeUseListsImpl(L, R->Next, &R->Next, Cmp); - return; - } - *Next = L; - mergeUseListsImpl(L->Next, R, &L->Next, Cmp); -} - // isa - Provide some specializations of isa so that we don't have to include // the subtype header files to test to see if the value is a subclass... // diff --git a/include/llvm/IR/ValueHandle.h b/include/llvm/IR/ValueHandle.h index 53fa80a626aa..3c2805913ef5 100644 --- a/include/llvm/IR/ValueHandle.h +++ b/include/llvm/IR/ValueHandle.h @@ -52,13 +52,21 @@ protected: Weak }; + ValueHandleBase(const ValueHandleBase &RHS) + : ValueHandleBase(RHS.PrevPair.getInt(), RHS) {} + + ValueHandleBase(HandleBaseKind Kind, const ValueHandleBase &RHS) + : PrevPair(nullptr, Kind), Next(nullptr), V(RHS.V) { + if (isValid(V)) + AddToExistingUseList(RHS.getPrevPtr()); + } + private: PointerIntPair<ValueHandleBase**, 2, HandleBaseKind> PrevPair; ValueHandleBase *Next; Value* V; - ValueHandleBase(const ValueHandleBase&) = delete; public: explicit ValueHandleBase(HandleBaseKind Kind) : PrevPair(nullptr, Kind), Next(nullptr), V(nullptr) {} @@ -67,11 +75,7 @@ public: if (isValid(V)) AddToUseList(); } - ValueHandleBase(HandleBaseKind Kind, const ValueHandleBase &RHS) - : PrevPair(nullptr, Kind), Next(nullptr), V(RHS.V) { - if (isValid(V)) - AddToExistingUseList(RHS.getPrevPtr()); - } + ~ValueHandleBase() { if (isValid(V)) RemoveFromUseList(); @@ -145,6 +149,8 @@ public: WeakVH(const WeakVH &RHS) : ValueHandleBase(Weak, RHS) {} + WeakVH &operator=(const WeakVH &RHS) = default; + Value *operator=(Value *RHS) { return ValueHandleBase::operator=(RHS); } @@ -314,7 +320,6 @@ class TrackingVH : public ValueHandleBase { public: TrackingVH() : ValueHandleBase(Tracking) {} TrackingVH(ValueTy *P) : ValueHandleBase(Tracking, GetAsValue(P)) {} - TrackingVH(const TrackingVH &RHS) : ValueHandleBase(Tracking, RHS) {} operator ValueTy*() const { return getValPtr(); @@ -324,10 +329,6 @@ public: setValPtr(RHS); return getValPtr(); } - ValueTy *operator=(const TrackingVH<ValueTy> &RHS) { - setValPtr(RHS.getValPtr()); - return getValPtr(); - } ValueTy *operator->() const { return getValPtr(); } ValueTy &operator*() const { return *getValPtr(); } @@ -339,15 +340,13 @@ public: /// when the underlying Value has RAUW called on it or is destroyed. This /// class can be used as the key of a map, as long as the user takes it out of /// the map before calling setValPtr() (since the map has to rearrange itself -/// when the pointer changes). Unlike ValueHandleBase, this class has a vtable -/// and a virtual destructor. +/// when the pointer changes). Unlike ValueHandleBase, this class has a vtable. class CallbackVH : public ValueHandleBase { virtual void anchor(); protected: - CallbackVH(const CallbackVH &RHS) - : ValueHandleBase(Callback, RHS) {} - - virtual ~CallbackVH() {} + ~CallbackVH() = default; + CallbackVH(const CallbackVH &) = default; + CallbackVH &operator=(const CallbackVH &) = default; void setValPtr(Value *P) { ValueHandleBase::operator=(P); diff --git a/include/llvm/IR/ValueMap.h b/include/llvm/IR/ValueMap.h index 4d00b637609c..ad518ac053b2 100644 --- a/include/llvm/IR/ValueMap.h +++ b/include/llvm/IR/ValueMap.h @@ -214,8 +214,8 @@ private: // This CallbackVH updates its ValueMap when the contained Value changes, // according to the user's preferences expressed through the Config object. -template<typename KeyT, typename ValueT, typename Config> -class ValueMapCallbackVH : public CallbackVH { +template <typename KeyT, typename ValueT, typename Config> +class ValueMapCallbackVH final : public CallbackVH { friend class ValueMap<KeyT, ValueT, Config>; friend struct DenseMapInfo<ValueMapCallbackVH>; typedef ValueMap<KeyT, ValueT, Config> ValueMapT; diff --git a/include/llvm/IR/ValueSymbolTable.h b/include/llvm/IR/ValueSymbolTable.h index bf1fade1ccef..65bd7fc2fec1 100644 --- a/include/llvm/IR/ValueSymbolTable.h +++ b/include/llvm/IR/ValueSymbolTable.h @@ -14,13 +14,13 @@ #ifndef LLVM_IR_VALUESYMBOLTABLE_H #define LLVM_IR_VALUESYMBOLTABLE_H +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" #include "llvm/IR/Value.h" #include "llvm/Support/DataTypes.h" namespace llvm { - template<typename ValueSubClass, typename ItemParentClass> - class SymbolTableListTraits; + template <typename ValueSubClass> class SymbolTableListTraits; class BasicBlock; class Function; class NamedMDNode; @@ -33,12 +33,12 @@ namespace llvm { /// class ValueSymbolTable { friend class Value; - friend class SymbolTableListTraits<Argument, Function>; - friend class SymbolTableListTraits<BasicBlock, Function>; - friend class SymbolTableListTraits<Instruction, BasicBlock>; - friend class SymbolTableListTraits<Function, Module>; - friend class SymbolTableListTraits<GlobalVariable, Module>; - friend class SymbolTableListTraits<GlobalAlias, Module>; + friend class SymbolTableListTraits<Argument>; + friend class SymbolTableListTraits<BasicBlock>; + friend class SymbolTableListTraits<Instruction>; + friend class SymbolTableListTraits<Function>; + friend class SymbolTableListTraits<GlobalVariable>; + friend class SymbolTableListTraits<GlobalAlias>; /// @name Types /// @{ public: @@ -55,7 +55,6 @@ public: /// @name Constructors /// @{ public: - ValueSymbolTable() : vmap(0), LastUnique(0) {} ~ValueSymbolTable(); @@ -63,9 +62,8 @@ public: /// @name Accessors /// @{ public: - /// This method finds the value with the given \p Name in the - /// the symbol table. + /// the symbol table. /// @returns the value associated with the \p Name /// @brief Lookup a named Value. Value *lookup(StringRef Name) const { return vmap.lookup(Name); } @@ -97,30 +95,32 @@ public: /// @brief Get a const_iterator to the end of the symbol table. inline const_iterator end() const { return vmap.end(); } - -/// @} -/// @name Mutators -/// @{ + + /// @} + /// @name Mutators + /// @{ private: + ValueName *makeUniqueName(Value *V, SmallString<256> &UniqueName); + /// This method adds the provided value \p N to the symbol table. The Value - /// must have a name which is used to place the value in the symbol table. + /// must have a name which is used to place the value in the symbol table. /// If the inserted name conflicts, this renames the value. /// @brief Add a named value to the symbol table void reinsertValue(Value *V); - + /// createValueName - This method attempts to create a value name and insert /// it into the symbol table with the specified name. If it conflicts, it /// auto-renames the name and returns that instead. ValueName *createValueName(StringRef Name, Value *V); - + /// This method removes a value from the symbol table. It leaves the /// ValueName attached to the value, but it is no longer inserted in the /// symtab. void removeValueName(ValueName *V); - -/// @} -/// @name Internal Data -/// @{ + + /// @} + /// @name Internal Data + /// @{ private: ValueMap vmap; ///< The map that holds the symbol table. mutable uint32_t LastUnique; ///< Counter for tracking unique names diff --git a/include/llvm/IRReader/IRReader.h b/include/llvm/IRReader/IRReader.h index 2d9ace0b62a0..523cd3d6df72 100644 --- a/include/llvm/IRReader/IRReader.h +++ b/include/llvm/IRReader/IRReader.h @@ -27,10 +27,11 @@ class LLVMContext; /// If the given file holds a bitcode image, return a Module /// for it which does lazy deserialization of function bodies. Otherwise, /// attempt to parse it as LLVM Assembly and return a fully populated -/// Module. -std::unique_ptr<Module> getLazyIRFileModule(StringRef Filename, - SMDiagnostic &Err, - LLVMContext &Context); +/// Module. The ShouldLazyLoadMetadata flag is passed down to the bitcode +/// reader to optionally enable lazy metadata loading. +std::unique_ptr<Module> +getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, + bool ShouldLazyLoadMetadata = false); /// If the given MemoryBuffer holds a bitcode image, return a Module /// for it. Otherwise, attempt to parse it as LLVM Assembly and return diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index e3b9a95f0a3d..cb2b1394e92b 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -53,9 +53,6 @@ void initializeInstrumentation(PassRegistry&); /// initializeAnalysis - Initialize all passes linked into the Analysis library. void initializeAnalysis(PassRegistry&); -/// initializeIPA - Initialize all passes linked into the IPA library. -void initializeIPA(PassRegistry&); - /// initializeCodeGen - Initialize all passes linked into the CodeGen library. void initializeCodeGen(PassRegistry&); @@ -64,11 +61,8 @@ void initializeTarget(PassRegistry&); void initializeAAEvalPass(PassRegistry&); void initializeAddDiscriminatorsPass(PassRegistry&); -void initializeADCEPass(PassRegistry&); +void initializeADCELegacyPassPass(PassRegistry&); void initializeBDCEPass(PassRegistry&); -void initializeAliasAnalysisAnalysisGroup(PassRegistry&); -void initializeAliasAnalysisCounterPass(PassRegistry&); -void initializeAliasDebuggerPass(PassRegistry&); void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlwaysInlinerPass(PassRegistry&); void initializeArgPromotionPass(PassRegistry&); @@ -76,13 +70,13 @@ void initializeAtomicExpandPass(PassRegistry&); void initializeSampleProfileLoaderPass(PassRegistry&); void initializeAlignmentFromAssumptionsPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); -void initializeBasicAliasAnalysisPass(PassRegistry&); +void initializeBasicAAWrapperPassPass(PassRegistry&); void initializeCallGraphWrapperPassPass(PassRegistry &); void initializeBlockExtractorPassPass(PassRegistry&); -void initializeBlockFrequencyInfoPass(PassRegistry&); +void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry&); void initializeBoundsCheckingPass(PassRegistry&); void initializeBranchFolderPassPass(PassRegistry&); -void initializeBranchProbabilityInfoPass(PassRegistry&); +void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&); void initializeBreakCriticalEdgesPass(PassRegistry&); void initializeCallGraphPrinterPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); @@ -90,7 +84,8 @@ void initializeCFGOnlyPrinterPass(PassRegistry&); void initializeCFGOnlyViewerPass(PassRegistry&); void initializeCFGPrinterPass(PassRegistry&); void initializeCFGSimplifyPassPass(PassRegistry&); -void initializeCFLAliasAnalysisPass(PassRegistry&); +void initializeCFLAAWrapperPassPass(PassRegistry&); +void initializeExternalAAWrapperPassPass(PassRegistry&); void initializeForwardControlFlowIntegrityPass(PassRegistry&); void initializeFlattenCFGPassPass(PassRegistry&); void initializeStructurizeCFGPass(PassRegistry&); @@ -102,6 +97,7 @@ void initializeConstantPropagationPass(PassRegistry&); void initializeMachineCopyPropagationPass(PassRegistry&); void initializeCostModelAnalysisPass(PassRegistry&); void initializeCorrelatedValuePropagationPass(PassRegistry&); +void initializeCrossDSOCFIPass(PassRegistry&); void initializeDAEPass(PassRegistry&); void initializeDAHPass(PassRegistry&); void initializeDCEPass(PassRegistry&); @@ -120,7 +116,10 @@ void initializeDominatorTreeWrapperPassPass(PassRegistry&); void initializeEarlyIfConverterPass(PassRegistry&); void initializeEdgeBundlesPass(PassRegistry&); void initializeExpandPostRAPass(PassRegistry&); +void initializeAAResultsWrapperPassPass(PassRegistry &); void initializeGCOVProfilerPass(PassRegistry&); +void initializePGOInstrumentationGenPass(PassRegistry&); +void initializePGOInstrumentationUsePass(PassRegistry&); void initializeInstrProfilingPass(PassRegistry&); void initializeAddressSanitizerPass(PassRegistry&); void initializeAddressSanitizerModulePass(PassRegistry&); @@ -132,19 +131,21 @@ void initializeScalarizerPass(PassRegistry&); void initializeEarlyCSELegacyPassPass(PassRegistry &); void initializeEliminateAvailableExternallyPass(PassRegistry&); void initializeExpandISelPseudosPass(PassRegistry&); +void initializeForceFunctionAttrsLegacyPassPass(PassRegistry&); void initializeFunctionAttrsPass(PassRegistry&); void initializeGCMachineCodeAnalysisPass(PassRegistry&); void initializeGCModuleInfoPass(PassRegistry&); void initializeGVNPass(PassRegistry&); void initializeGlobalDCEPass(PassRegistry&); void initializeGlobalOptPass(PassRegistry&); -void initializeGlobalsModRefPass(PassRegistry&); +void initializeGlobalsAAWrapperPassPass(PassRegistry&); void initializeIPCPPass(PassRegistry&); void initializeIPSCCPPass(PassRegistry&); void initializeIVUsersPass(PassRegistry&); void initializeIfConverterPass(PassRegistry&); void initializeInductiveRangeCheckEliminationPass(PassRegistry&); void initializeIndVarSimplifyPass(PassRegistry&); +void initializeInferFunctionAttrsLegacyPassPass(PassRegistry&); void initializeInlineCostAnalysisPass(PassRegistry&); void initializeInstructionCombiningPassPass(PassRegistry&); void initializeInstCountPass(PassRegistry&); @@ -155,7 +156,6 @@ void initializeJumpThreadingPass(PassRegistry&); void initializeLCSSAPass(PassRegistry&); void initializeLICMPass(PassRegistry&); void initializeLazyValueInfoPass(PassRegistry&); -void initializeLibCallAliasAnalysisPass(PassRegistry&); void initializeLintPass(PassRegistry&); void initializeLiveDebugVariablesPass(PassRegistry&); void initializeLiveIntervalsPass(PassRegistry&); @@ -210,7 +210,7 @@ void initializeMergeFunctionsPass(PassRegistry&); void initializeModuleDebugInfoPrinterPass(PassRegistry&); void initializeNaryReassociatePass(PassRegistry&); void initializeNoAAPass(PassRegistry&); -void initializeObjCARCAliasAnalysisPass(PassRegistry&); +void initializeObjCARCAAWrapperPassPass(PassRegistry&); void initializeObjCARCAPElimPass(PassRegistry&); void initializeObjCARCExpandPass(PassRegistry&); void initializeObjCARCContractPass(PassRegistry&); @@ -245,14 +245,14 @@ void initializeRegionViewerPass(PassRegistry&); void initializeRewriteStatepointsForGCPass(PassRegistry&); void initializeSafeStackPass(PassRegistry&); void initializeSCCPPass(PassRegistry&); -void initializeSROAPass(PassRegistry&); +void initializeSROALegacyPassPass(PassRegistry&); void initializeSROA_DTPass(PassRegistry&); void initializeSROA_SSAUpPass(PassRegistry&); -void initializeScalarEvolutionAliasAnalysisPass(PassRegistry&); -void initializeScalarEvolutionPass(PassRegistry&); +void initializeSCEVAAWrapperPassPass(PassRegistry&); +void initializeScalarEvolutionWrapperPassPass(PassRegistry&); void initializeShrinkWrapPass(PassRegistry &); void initializeSimpleInlinerPass(PassRegistry&); -void initializeShadowStackGCLoweringPass(PassRegistry&); +void initializeShadowStackGCLoweringPass(PassRegistry&); void initializeRegisterCoalescerPass(PassRegistry&); void initializeSingleLoopExtractorPass(PassRegistry&); void initializeSinkingPass(PassRegistry&); @@ -265,7 +265,7 @@ void initializeStackColoringPass(PassRegistry&); void initializeStackSlotColoringPass(PassRegistry&); void initializeStraightLineStrengthReducePass(PassRegistry &); void initializeStripDeadDebugInfoPass(PassRegistry&); -void initializeStripDeadPrototypesPassPass(PassRegistry&); +void initializeStripDeadPrototypesLegacyPassPass(PassRegistry&); void initializeStripDebugDeclarePass(PassRegistry&); void initializeStripNonDebugSymbolsPass(PassRegistry&); void initializeStripSymbolsPass(PassRegistry&); @@ -276,8 +276,8 @@ void initializeTargetTransformInfoWrapperPassPass(PassRegistry &); void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &); void initializeAssumptionCacheTrackerPass(PassRegistry &); void initializeTwoAddressInstructionPassPass(PassRegistry&); -void initializeTypeBasedAliasAnalysisPass(PassRegistry&); -void initializeScopedNoAliasAAPass(PassRegistry&); +void initializeTypeBasedAAWrapperPassPass(PassRegistry&); +void initializeScopedNoAliasAAWrapperPassPass(PassRegistry&); void initializeUnifyFunctionExitNodesPass(PassRegistry&); void initializeUnreachableBlockElimPass(PassRegistry&); void initializeUnreachableMachineBlockElimPass(PassRegistry&); @@ -294,6 +294,7 @@ void initializeBBVectorizePass(PassRegistry&); void initializeMachineFunctionPrinterPassPass(PassRegistry&); void initializeMIRPrintingPassPass(PassRegistry&); void initializeStackMapLivenessPass(PassRegistry&); +void initializeLiveDebugValuesPass(PassRegistry&); void initializeMachineCombinerPass(PassRegistry &); void initializeLoadCombinePass(PassRegistry&); void initializeRewriteSymbolsPass(PassRegistry&); @@ -304,6 +305,10 @@ void initializeDwarfEHPreparePass(PassRegistry&); void initializeFloat2IntPass(PassRegistry&); void initializeLoopDistributePass(PassRegistry&); void initializeSjLjEHPreparePass(PassRegistry&); +void initializeDemandedBitsPass(PassRegistry&); +void initializeFuncletLayoutPass(PassRegistry &); +void initializeLoopLoadEliminationPass(PassRegistry&); +void initializeFunctionImportPassPass(PassRegistry &); } #endif diff --git a/include/llvm/LTO/LTOCodeGenerator.h b/include/llvm/LTO/LTOCodeGenerator.h index 0c46fc048a43..3820b211a381 100644 --- a/include/llvm/LTO/LTOCodeGenerator.h +++ b/include/llvm/LTO/LTOCodeGenerator.h @@ -39,7 +39,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" -#include "llvm/Linker/Linker.h" +#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include <string> #include <vector> @@ -48,6 +48,7 @@ namespace llvm { class LLVMContext; class DiagnosticInfo; class GlobalValue; + class Linker; class Mangler; class MemoryBuffer; class TargetLibraryInfo; @@ -61,121 +62,135 @@ namespace llvm { struct LTOCodeGenerator { static const char *getVersionString(); - LTOCodeGenerator(); - LTOCodeGenerator(std::unique_ptr<LLVMContext> Context); + LTOCodeGenerator(LLVMContext &Context); ~LTOCodeGenerator(); - // Merge given module, return true on success. + /// Merge given module. Return true on success. bool addModule(struct LTOModule *); - // Set the destination module. - void setModule(struct LTOModule *); + /// Set the destination module. + void setModule(std::unique_ptr<LTOModule> M); - void setTargetOptions(TargetOptions options); + void setTargetOptions(TargetOptions Options); void setDebugInfo(lto_debug_model); - void setCodePICModel(lto_codegen_model); + void setCodePICModel(Reloc::Model Model) { RelocModel = Model; } + + /// Set the file type to be emitted (assembly or object code). + /// The default is TargetMachine::CGFT_ObjectFile. + void setFileType(TargetMachine::CodeGenFileType FT) { FileType = FT; } - void setCpu(const char *mCpu) { MCpu = mCpu; } - void setAttr(const char *mAttr) { MAttr = mAttr; } - void setOptLevel(unsigned optLevel) { OptLevel = optLevel; } + void setCpu(const char *MCpu) { this->MCpu = MCpu; } + void setAttr(const char *MAttr) { this->MAttr = MAttr; } + void setOptLevel(unsigned OptLevel); void setShouldInternalize(bool Value) { ShouldInternalize = Value; } void setShouldEmbedUselists(bool Value) { ShouldEmbedUselists = Value; } - void addMustPreserveSymbol(StringRef sym) { MustPreserveSymbols[sym] = 1; } - - // To pass options to the driver and optimization passes. These options are - // not necessarily for debugging purpose (The function name is misleading). - // This function should be called before LTOCodeGenerator::compilexxx(), - // and LTOCodeGenerator::writeMergedModules(). - void setCodeGenDebugOptions(const char *opts); - - // Parse the options set in setCodeGenDebugOptions. Like - // setCodeGenDebugOptions, this must be called before - // LTOCodeGenerator::compilexxx() and LTOCodeGenerator::writeMergedModules() + void addMustPreserveSymbol(StringRef Sym) { MustPreserveSymbols[Sym] = 1; } + + /// Pass options to the driver and optimization passes. + /// + /// These options are not necessarily for debugging purpose (the function + /// name is misleading). This function should be called before + /// LTOCodeGenerator::compilexxx(), and + /// LTOCodeGenerator::writeMergedModules(). + void setCodeGenDebugOptions(const char *Opts); + + /// Parse the options set in setCodeGenDebugOptions. + /// + /// Like \a setCodeGenDebugOptions(), this must be called before + /// LTOCodeGenerator::compilexxx() and + /// LTOCodeGenerator::writeMergedModules(). void parseCodeGenDebugOptions(); - // Write the merged module to the file specified by the given path. - // Return true on success. - bool writeMergedModules(const char *path, std::string &errMsg); - - // Compile the merged module into a *single* object file; the path to object - // file is returned to the caller via argument "name". Return true on - // success. - // - // NOTE that it is up to the linker to remove the intermediate object file. - // Do not try to remove the object file in LTOCodeGenerator's destructor - // as we don't who (LTOCodeGenerator or the obj file) will last longer. - bool compile_to_file(const char **name, - bool disableInline, - bool disableGVNLoadPRE, - bool disableVectorization, - std::string &errMsg); - - // As with compile_to_file(), this function compiles the merged module into - // single object file. Instead of returning the object-file-path to the caller - // (linker), it brings the object to a buffer, and return the buffer to the - // caller. This function should delete intermediate object file once its content - // is brought to memory. Return NULL if the compilation was not successful. - std::unique_ptr<MemoryBuffer> compile(bool disableInline, - bool disableGVNLoadPRE, - bool disableVectorization, - std::string &errMsg); - - // Optimizes the merged module. Returns true on success. - bool optimize(bool disableInline, - bool disableGVNLoadPRE, - bool disableVectorization, - std::string &errMsg); - - // Compiles the merged optimized module into a single object file. It brings - // the object to a buffer, and returns the buffer to the caller. Return NULL - // if the compilation was not successful. - std::unique_ptr<MemoryBuffer> compileOptimized(std::string &errMsg); + /// Write the merged module to the file specified by the given path. Return + /// true on success. + bool writeMergedModules(const char *Path); + + /// Compile the merged module into a *single* output file; the path to output + /// file is returned to the caller via argument "name". Return true on + /// success. + /// + /// \note It is up to the linker to remove the intermediate output file. Do + /// not try to remove the object file in LTOCodeGenerator's destructor as we + /// don't who (LTOCodeGenerator or the output file) will last longer. + bool compile_to_file(const char **Name, bool DisableVerify, + bool DisableInline, bool DisableGVNLoadPRE, + bool DisableVectorization); + + /// As with compile_to_file(), this function compiles the merged module into + /// single output file. Instead of returning the output file path to the + /// caller (linker), it brings the output to a buffer, and returns the buffer + /// to the caller. This function should delete the intermediate file once + /// its content is brought to memory. Return NULL if the compilation was not + /// successful. + std::unique_ptr<MemoryBuffer> compile(bool DisableVerify, bool DisableInline, + bool DisableGVNLoadPRE, + bool DisableVectorization); + + /// Optimizes the merged module. Returns true on success. + bool optimize(bool DisableVerify, bool DisableInline, bool DisableGVNLoadPRE, + bool DisableVectorization); + + /// Compiles the merged optimized module into a single output file. It brings + /// the output to a buffer, and returns the buffer to the caller. Return NULL + /// if the compilation was not successful. + std::unique_ptr<MemoryBuffer> compileOptimized(); + + /// Compile the merged optimized module into out.size() output files each + /// representing a linkable partition of the module. If out contains more + /// than one element, code generation is done in parallel with out.size() + /// threads. Output files will be written to members of out. Returns true on + /// success. + bool compileOptimized(ArrayRef<raw_pwrite_stream *> Out); void setDiagnosticHandler(lto_diagnostic_handler_t, void *); LLVMContext &getContext() { return Context; } + void resetMergedModule() { MergedModule.reset(); } + private: void initializeLTOPasses(); - bool compileOptimized(raw_pwrite_stream &out, std::string &errMsg); - bool compileOptimizedToFile(const char **name, std::string &errMsg); + bool compileOptimizedToFile(const char **Name); void applyScopeRestrictions(); void applyRestriction(GlobalValue &GV, ArrayRef<StringRef> Libcalls, std::vector<const char *> &MustPreserveList, SmallPtrSetImpl<GlobalValue *> &AsmUsed, Mangler &Mangler); - bool determineTarget(std::string &errMsg); + bool determineTarget(); static void DiagnosticHandler(const DiagnosticInfo &DI, void *Context); void DiagnosticHandler2(const DiagnosticInfo &DI); + void emitError(const std::string &ErrMsg); + typedef StringMap<uint8_t> StringSet; - void destroyMergedModule(); - std::unique_ptr<LLVMContext> OwnedContext; LLVMContext &Context; - Linker IRLinker; - TargetMachine *TargetMach = nullptr; + std::unique_ptr<Module> MergedModule; + std::unique_ptr<Linker> TheLinker; + std::unique_ptr<TargetMachine> TargetMach; bool EmitDwarfDebugInfo = false; bool ScopeRestrictionsDone = false; - lto_codegen_model CodeModel = LTO_CODEGEN_PIC_MODEL_DEFAULT; + Reloc::Model RelocModel = Reloc::Default; StringSet MustPreserveSymbols; StringSet AsmUndefinedRefs; - std::vector<char *> CodegenOptions; + std::vector<std::string> CodegenOptions; + std::string FeatureStr; std::string MCpu; std::string MAttr; std::string NativeObjectPath; TargetOptions Options; + CodeGenOpt::Level CGOptLevel = CodeGenOpt::Default; unsigned OptLevel = 2; lto_diagnostic_handler_t DiagHandler = nullptr; void *DiagContext = nullptr; - LTOModule *OwnedModule = nullptr; bool ShouldInternalize = true; bool ShouldEmbedUselists = false; + TargetMachine::CodeGenFileType FileType = TargetMachine::CGFT_ObjectFile; }; } #endif diff --git a/include/llvm/LTO/LTOModule.h b/include/llvm/LTO/LTOModule.h index c4e2be627399..97b5865bd47f 100644 --- a/include/llvm/LTO/LTOModule.h +++ b/include/llvm/LTO/LTOModule.h @@ -74,6 +74,11 @@ public: static bool isBitcodeForTarget(MemoryBuffer *memBuffer, StringRef triplePrefix); + /// Returns a string representing the producer identification stored in the + /// bitcode, or "" if the bitcode does not contains any. + /// + static std::string getProducerString(MemoryBuffer *Buffer); + /// Create a MemoryBuffer from a memory range with an optional name. static std::unique_ptr<MemoryBuffer> makeBuffer(const void *mem, size_t length, StringRef name = ""); @@ -86,25 +91,24 @@ public: /// InitializeAllTargetMCs(); /// InitializeAllAsmPrinters(); /// InitializeAllAsmParsers(); - static LTOModule *createFromFile(const char *path, TargetOptions options, - std::string &errMsg); - static LTOModule *createFromOpenFile(int fd, const char *path, size_t size, - TargetOptions options, - std::string &errMsg); - static LTOModule *createFromOpenFileSlice(int fd, const char *path, - size_t map_size, off_t offset, - TargetOptions options, - std::string &errMsg); - static LTOModule *createFromBuffer(const void *mem, size_t length, - TargetOptions options, std::string &errMsg, - StringRef path = ""); - - static LTOModule *createInLocalContext(const void *mem, size_t length, - TargetOptions options, - std::string &errMsg, StringRef path); - static LTOModule *createInContext(const void *mem, size_t length, - TargetOptions options, std::string &errMsg, - StringRef path, LLVMContext *Context); + static ErrorOr<std::unique_ptr<LTOModule>> + createFromFile(LLVMContext &Context, const char *path, TargetOptions options); + static ErrorOr<std::unique_ptr<LTOModule>> + createFromOpenFile(LLVMContext &Context, int fd, const char *path, + size_t size, TargetOptions options); + static ErrorOr<std::unique_ptr<LTOModule>> + createFromOpenFileSlice(LLVMContext &Context, int fd, const char *path, + size_t map_size, off_t offset, TargetOptions options); + static ErrorOr<std::unique_ptr<LTOModule>> + createFromBuffer(LLVMContext &Context, const void *mem, size_t length, + TargetOptions options, StringRef path = ""); + + static ErrorOr<std::unique_ptr<LTOModule>> + createInLocalContext(const void *mem, size_t length, TargetOptions options, + StringRef path); + static ErrorOr<std::unique_ptr<LTOModule>> + createInContext(const void *mem, size_t length, TargetOptions options, + StringRef path, LLVMContext *Context); const Module &getModule() const { return const_cast<LTOModule*>(this)->getModule(); @@ -113,6 +117,8 @@ public: return IRFile->getModule(); } + std::unique_ptr<Module> takeModule() { return IRFile->takeModule(); } + /// Return the Module's target triple. const std::string &getTargetTriple() { return getModule().getTargetTriple(); @@ -163,7 +169,7 @@ private: /// Parse the symbols from the module and model-level ASM and add them to /// either the defined or undefined lists. - bool parseSymbols(std::string &errMsg); + void parseSymbols(); /// Add a symbol which isn't defined just yet to a list to be resolved later. void addPotentialUndefinedSymbol(const object::BasicSymbolRef &Sym, @@ -200,8 +206,9 @@ private: bool objcClassNameFromExpression(const Constant *c, std::string &name); /// Create an LTOModule (private version). - static LTOModule *makeLTOModule(MemoryBufferRef Buffer, TargetOptions options, - std::string &errMsg, LLVMContext *Context); + static ErrorOr<std::unique_ptr<LTOModule>> + makeLTOModule(MemoryBufferRef Buffer, TargetOptions options, + LLVMContext *Context); }; } #endif diff --git a/include/llvm/LibDriver/LibDriver.h b/include/llvm/LibDriver/LibDriver.h index aaaa7b7d21c3..09495650c1b9 100644 --- a/include/llvm/LibDriver/LibDriver.h +++ b/include/llvm/LibDriver/LibDriver.h @@ -8,7 +8,7 @@ //===----------------------------------------------------------------------===// // // Defines an interface to a lib.exe-compatible driver that also understands -// bitcode files. Used by llvm-lib and lld-link2 /lib. +// bitcode files. Used by llvm-lib and lld-link /lib. // //===----------------------------------------------------------------------===// diff --git a/include/llvm/LinkAllPasses.h b/include/llvm/LinkAllPasses.h index cea5530db3b8..29fcd93a2a1c 100644 --- a/include/llvm/LinkAllPasses.h +++ b/include/llvm/LinkAllPasses.h @@ -17,8 +17,11 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasSetTracker.h" +#include "llvm/Analysis/BasicAliasAnalysis.h" +#include "llvm/Analysis/CFLAliasAnalysis.h" #include "llvm/Analysis/CallPrinter.h" #include "llvm/Analysis/DomPrinter.h" +#include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/IntervalPartition.h" #include "llvm/Analysis/Lint.h" #include "llvm/Analysis/Passes.h" @@ -26,6 +29,9 @@ #include "llvm/Analysis/RegionPass.h" #include "llvm/Analysis/RegionPrinter.h" #include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" +#include "llvm/Analysis/ScopedNoAliasAA.h" +#include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/Passes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRPrintingPasses.h" @@ -52,21 +58,18 @@ namespace { (void) llvm::createAAEvalPass(); (void) llvm::createAggressiveDCEPass(); (void) llvm::createBitTrackingDCEPass(); - (void) llvm::createAliasAnalysisCounterPass(); - (void) llvm::createAliasDebugger(); (void) llvm::createArgumentPromotionPass(); (void) llvm::createAlignmentFromAssumptionsPass(); - (void) llvm::createBasicAliasAnalysisPass(); - (void) llvm::createLibCallAliasAnalysisPass(nullptr); - (void) llvm::createScalarEvolutionAliasAnalysisPass(); - (void) llvm::createTypeBasedAliasAnalysisPass(); - (void) llvm::createScopedNoAliasAAPass(); + (void) llvm::createBasicAAWrapperPass(); + (void) llvm::createSCEVAAWrapperPass(); + (void) llvm::createTypeBasedAAWrapperPass(); + (void) llvm::createScopedNoAliasAAWrapperPass(); (void) llvm::createBoundsCheckingPass(); (void) llvm::createBreakCriticalEdgesPass(); (void) llvm::createCallGraphPrinterPass(); (void) llvm::createCallGraphViewerPass(); (void) llvm::createCFGSimplificationPass(); - (void) llvm::createCFLAliasAnalysisPass(); + (void) llvm::createCFLAAWrapperPass(); (void) llvm::createStructurizeCFGPass(); (void) llvm::createConstantMergePass(); (void) llvm::createConstantPropagationPass(); @@ -82,12 +85,15 @@ namespace { (void) llvm::createDomOnlyViewerPass(); (void) llvm::createDomViewerPass(); (void) llvm::createGCOVProfilerPass(); + (void) llvm::createPGOInstrumentationGenPass(); + (void) llvm::createPGOInstrumentationUsePass(); (void) llvm::createInstrProfilingPass(); + (void) llvm::createFunctionImportPass(); (void) llvm::createFunctionInliningPass(); (void) llvm::createAlwaysInlinerPass(); (void) llvm::createGlobalDCEPass(); (void) llvm::createGlobalOptimizerPass(); - (void) llvm::createGlobalsModRefPass(); + (void) llvm::createGlobalsAAWrapperPass(); (void) llvm::createIPConstantPropagationPass(); (void) llvm::createIPSCCPPass(); (void) llvm::createInductiveRangeCheckEliminationPass(); @@ -110,8 +116,7 @@ namespace { (void) llvm::createLowerInvokePass(); (void) llvm::createLowerSwitchPass(); (void) llvm::createNaryReassociatePass(); - (void) llvm::createNoAAPass(); - (void) llvm::createObjCARCAliasAnalysisPass(); + (void) llvm::createObjCARCAAWrapperPass(); (void) llvm::createObjCARCAPElimPass(); (void) llvm::createObjCARCExpandPass(); (void) llvm::createObjCARCContractPass(); @@ -179,7 +184,7 @@ namespace { (void) llvm::createEliminateAvailableExternallyPass(); (void)new llvm::IntervalPartition(); - (void)new llvm::ScalarEvolution(); + (void)new llvm::ScalarEvolutionWrapperPass(); ((llvm::Function*)nullptr)->viewCFGOnly(); llvm::RGPassManager RGM; ((llvm::RegionPass*)nullptr)->runOnRegion((llvm::Region*)nullptr, RGM); diff --git a/include/llvm/Linker/IRMover.h b/include/llvm/Linker/IRMover.h new file mode 100644 index 000000000000..a964cc4b72c5 --- /dev/null +++ b/include/llvm/Linker/IRMover.h @@ -0,0 +1,76 @@ +//===- IRMover.h ------------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LINKER_IRMOVER_H +#define LLVM_LINKER_IRMOVER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include <functional> + +namespace llvm { +class GlobalValue; +class MDNode; +class Module; +class StructType; +class Type; + +class IRMover { + struct StructTypeKeyInfo { + struct KeyTy { + ArrayRef<Type *> ETypes; + bool IsPacked; + KeyTy(ArrayRef<Type *> E, bool P); + KeyTy(const StructType *ST); + bool operator==(const KeyTy &that) const; + bool operator!=(const KeyTy &that) const; + }; + static StructType *getEmptyKey(); + static StructType *getTombstoneKey(); + static unsigned getHashValue(const KeyTy &Key); + static unsigned getHashValue(const StructType *ST); + static bool isEqual(const KeyTy &LHS, const StructType *RHS); + static bool isEqual(const StructType *LHS, const StructType *RHS); + }; + +public: + class IdentifiedStructTypeSet { + // The set of opaque types is the composite module. + DenseSet<StructType *> OpaqueStructTypes; + + // The set of identified but non opaque structures in the composite module. + DenseSet<StructType *, StructTypeKeyInfo> NonOpaqueStructTypes; + + public: + void addNonOpaque(StructType *Ty); + void switchToNonOpaque(StructType *Ty); + void addOpaque(StructType *Ty); + StructType *findNonOpaque(ArrayRef<Type *> ETypes, bool IsPacked); + bool hasType(StructType *Ty); + }; + + IRMover(Module &M); + + typedef std::function<void(GlobalValue &)> ValueAdder; + /// Move in the provide values. The source is destroyed. + /// Returns true on error. + bool move(Module &Src, ArrayRef<GlobalValue *> ValuesToLink, + std::function<void(GlobalValue &GV, ValueAdder Add)> AddLazyFor, + DenseMap<unsigned, MDNode *> *ValIDToTempMDMap = nullptr, + bool IsMetadataLinkingPostpass = false); + Module &getModule() { return Composite; } + +private: + Module &Composite; + IdentifiedStructTypeSet IdentifiedStructTypes; +}; + +} // End llvm namespace + +#endif diff --git a/include/llvm/Linker/Linker.h b/include/llvm/Linker/Linker.h index c43b90e9cd26..dde3f73883ca 100644 --- a/include/llvm/Linker/Linker.h +++ b/include/llvm/Linker/Linker.h @@ -10,10 +10,8 @@ #ifndef LLVM_LINKER_LINKER_H #define LLVM_LINKER_LINKER_H -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/FunctionInfo.h" +#include "llvm/Linker/IRMover.h" namespace llvm { class Module; @@ -25,71 +23,55 @@ class Type; /// module since it is assumed that the user of this class will want to do /// something with it after the linking. class Linker { -public: - struct StructTypeKeyInfo { - struct KeyTy { - ArrayRef<Type *> ETypes; - bool IsPacked; - KeyTy(ArrayRef<Type *> E, bool P); - KeyTy(const StructType *ST); - bool operator==(const KeyTy &that) const; - bool operator!=(const KeyTy &that) const; - }; - static StructType *getEmptyKey(); - static StructType *getTombstoneKey(); - static unsigned getHashValue(const KeyTy &Key); - static unsigned getHashValue(const StructType *ST); - static bool isEqual(const KeyTy &LHS, const StructType *RHS); - static bool isEqual(const StructType *LHS, const StructType *RHS); - }; - - typedef DenseSet<StructType *, StructTypeKeyInfo> NonOpaqueStructTypeSet; - typedef DenseSet<StructType *> OpaqueStructTypeSet; - - struct IdentifiedStructTypeSet { - // The set of opaque types is the composite module. - OpaqueStructTypeSet OpaqueStructTypes; + IRMover Mover; - // The set of identified but non opaque structures in the composite module. - NonOpaqueStructTypeSet NonOpaqueStructTypes; - - void addNonOpaque(StructType *Ty); - void switchToNonOpaque(StructType *Ty); - void addOpaque(StructType *Ty); - StructType *findNonOpaque(ArrayRef<Type *> ETypes, bool IsPacked); - bool hasType(StructType *Ty); +public: + enum Flags { + None = 0, + OverrideFromSrc = (1 << 0), + LinkOnlyNeeded = (1 << 1), + InternalizeLinkedSymbols = (1 << 2) }; - Linker(Module *M, DiagnosticHandlerFunction DiagnosticHandler); - Linker(Module *M); - ~Linker(); - - Module *getModule() const { return Composite; } - void deleteModule(); + Linker(Module &M); - /// \brief Link \p Src into the composite. The source is destroyed. + /// \brief Link \p Src into the composite. + /// /// Passing OverrideSymbols as true will have symbols from Src /// shadow those in the Dest. + /// For ThinLTO function importing/exporting the \p FunctionInfoIndex + /// is passed. If \p FunctionsToImport is provided, only the functions that + /// are part of the set will be imported from the source module. + /// The \p ValIDToTempMDMap is populated by the linker when function + /// importing is performed. + /// /// Returns true on error. - bool linkInModule(Module *Src, bool OverrideSymbols = false); - - /// \brief Set the composite to the passed-in module. - void setModule(Module *Dst); - - static bool LinkModules(Module *Dest, Module *Src, - DiagnosticHandlerFunction DiagnosticHandler); - - static bool LinkModules(Module *Dest, Module *Src); - -private: - void init(Module *M, DiagnosticHandlerFunction DiagnosticHandler); - Module *Composite; - - IdentifiedStructTypeSet IdentifiedStructTypes; - - DiagnosticHandlerFunction DiagnosticHandler; + bool linkInModule(std::unique_ptr<Module> Src, unsigned Flags = Flags::None, + const FunctionInfoIndex *Index = nullptr, + DenseSet<const GlobalValue *> *FunctionsToImport = nullptr, + DenseMap<unsigned, MDNode *> *ValIDToTempMDMap = nullptr); + + /// This exists to implement the deprecated LLVMLinkModules C api. Don't use + /// for anything else. + bool linkInModuleForCAPI(Module &Src); + + static bool linkModules(Module &Dest, std::unique_ptr<Module> Src, + unsigned Flags = Flags::None); + + /// \brief Link metadata from \p Src into the composite. The source is + /// destroyed. + /// + /// The \p ValIDToTempMDMap sound have been populated earlier during function + /// importing from \p Src. + bool linkInMetadata(Module &Src, + DenseMap<unsigned, MDNode *> *ValIDToTempMDMap); }; +/// Create a new module with exported local functions renamed and promoted +/// for ThinLTO. +std::unique_ptr<Module> renameModuleForThinLTO(std::unique_ptr<Module> M, + const FunctionInfoIndex *Index); + } // End llvm namespace #endif diff --git a/include/llvm/MC/ConstantPools.h b/include/llvm/MC/ConstantPools.h index 9aa4663ba0fc..552e1443e7d0 100644 --- a/include/llvm/MC/ConstantPools.h +++ b/include/llvm/MC/ConstantPools.h @@ -17,6 +17,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/SMLoc.h" namespace llvm { class MCContext; @@ -26,11 +27,12 @@ class MCStreamer; class MCSymbol; struct ConstantPoolEntry { - ConstantPoolEntry(MCSymbol *L, const MCExpr *Val, unsigned Sz) - : Label(L), Value(Val), Size(Sz) {} + ConstantPoolEntry(MCSymbol *L, const MCExpr *Val, unsigned Sz, SMLoc Loc_) + : Label(L), Value(Val), Size(Sz), Loc(Loc_) {} MCSymbol *Label; const MCExpr *Value; unsigned Size; + SMLoc Loc; }; // A class to keep track of assembler-generated constant pools that are use to @@ -49,7 +51,7 @@ public: // // \returns a MCExpr that references the newly inserted value const MCExpr *addEntry(const MCExpr *Value, MCContext &Context, - unsigned Size); + unsigned Size, SMLoc Loc); // Emit the contents of the constant pool using the provided streamer. void emitEntries(MCStreamer &Streamer); @@ -80,7 +82,7 @@ public: void emitAll(MCStreamer &Streamer); void emitForCurrentSection(MCStreamer &Streamer); const MCExpr *addEntry(MCStreamer &Streamer, const MCExpr *Expr, - unsigned Size); + unsigned Size, SMLoc Loc); private: ConstantPool *getConstantPool(MCSection *Section); diff --git a/include/llvm/MC/MCAsmBackend.h b/include/llvm/MC/MCAsmBackend.h index 2bfad2d355b8..51312ff80447 100644 --- a/include/llvm/MC/MCAsmBackend.h +++ b/include/llvm/MC/MCAsmBackend.h @@ -67,6 +67,11 @@ public: /// Get the number of target specific fixup kinds. virtual unsigned getNumFixupKinds() const = 0; + /// Map a relocation name used in .reloc to a fixup kind. + /// Returns true and sets MappedKind if Name is successfully mapped. + /// Otherwise returns false and leaves MappedKind unchanged. + virtual bool getFixupKind(StringRef Name, MCFixupKind &MappedKind) const; + /// Get information on a fixup kind. virtual const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const; diff --git a/include/llvm/MC/MCAsmInfo.h b/include/llvm/MC/MCAsmInfo.h index 9bb0fa63c523..384584ef4ef0 100644 --- a/include/llvm/MC/MCAsmInfo.h +++ b/include/llvm/MC/MCAsmInfo.h @@ -414,6 +414,15 @@ public: /// syntactically correct. virtual bool isValidUnquotedName(StringRef Name) const; + /// Return true if the .section directive should be omitted when + /// emitting \p SectionName. For example: + /// + /// shouldOmitSectionDirective(".text") + /// + /// returns false => .section .text,#alloc,#execinstr + /// returns true => .text + virtual bool shouldOmitSectionDirective(StringRef SectionName) const; + bool usesSunStyleELFSectionSwitchSyntax() const { return SunStyleELFSectionSwitchSyntax; } diff --git a/include/llvm/MC/MCAssembler.h b/include/llvm/MC/MCAssembler.h index 0642af837e7e..c0bd12875839 100644 --- a/include/llvm/MC/MCAssembler.h +++ b/include/llvm/MC/MCAssembler.h @@ -10,23 +10,18 @@ #ifndef LLVM_MC_MCASSEMBLER_H #define LLVM_MC_MCASSEMBLER_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator.h" #include "llvm/MC/MCDirectives.h" +#include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCFixup.h" +#include "llvm/MC/MCFragment.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCLinkerOptimizationHint.h" -#include "llvm/MC/MCSection.h" #include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/DataTypes.h" -#include <algorithm> -#include <vector> // FIXME: Shouldn't be needed. +#include "llvm/MC/MCSymbol.h" namespace llvm { class raw_ostream; @@ -42,476 +37,6 @@ class MCSubtargetInfo; class MCValue; class MCAsmBackend; -class MCFragment : public ilist_node<MCFragment> { - friend class MCAsmLayout; - - MCFragment(const MCFragment &) = delete; - void operator=(const MCFragment &) = delete; - -public: - enum FragmentType : uint8_t { - FT_Align, - FT_Data, - FT_CompactEncodedInst, - FT_Fill, - FT_Relaxable, - FT_Org, - FT_Dwarf, - FT_DwarfFrame, - FT_LEB, - FT_SafeSEH - }; - -private: - FragmentType Kind; - -protected: - bool HasInstructions; - -private: - /// \brief Should this fragment be aligned to the end of a bundle? - bool AlignToBundleEnd; - - uint8_t BundlePadding; - - /// LayoutOrder - The layout order of this fragment. - unsigned LayoutOrder; - - /// The data for the section this fragment is in. - MCSection *Parent; - - /// Atom - The atom this fragment is in, as represented by it's defining - /// symbol. - const MCSymbol *Atom; - - /// \name Assembler Backend Data - /// @{ - // - // FIXME: This could all be kept private to the assembler implementation. - - /// Offset - The offset of this fragment in its section. This is ~0 until - /// initialized. - uint64_t Offset; - - /// @} - -protected: - MCFragment(FragmentType Kind, bool HasInstructions, - uint8_t BundlePadding, MCSection *Parent = nullptr); - - ~MCFragment(); -private: - - // This is a friend so that the sentinal can be created. - friend struct ilist_sentinel_traits<MCFragment>; - MCFragment(); - -public: - /// Destroys the current fragment. - /// - /// This must be used instead of delete as MCFragment is non-virtual. - /// This method will dispatch to the appropriate subclass. - void destroy(); - - FragmentType getKind() const { return Kind; } - - MCSection *getParent() const { return Parent; } - void setParent(MCSection *Value) { Parent = Value; } - - const MCSymbol *getAtom() const { return Atom; } - void setAtom(const MCSymbol *Value) { Atom = Value; } - - unsigned getLayoutOrder() const { return LayoutOrder; } - void setLayoutOrder(unsigned Value) { LayoutOrder = Value; } - - /// \brief Does this fragment have instructions emitted into it? By default - /// this is false, but specific fragment types may set it to true. - bool hasInstructions() const { return HasInstructions; } - - /// \brief Should this fragment be placed at the end of an aligned bundle? - bool alignToBundleEnd() const { return AlignToBundleEnd; } - void setAlignToBundleEnd(bool V) { AlignToBundleEnd = V; } - - /// \brief Get the padding size that must be inserted before this fragment. - /// Used for bundling. By default, no padding is inserted. - /// Note that padding size is restricted to 8 bits. This is an optimization - /// to reduce the amount of space used for each fragment. In practice, larger - /// padding should never be required. - uint8_t getBundlePadding() const { return BundlePadding; } - - /// \brief Set the padding size for this fragment. By default it's a no-op, - /// and only some fragments have a meaningful implementation. - void setBundlePadding(uint8_t N) { BundlePadding = N; } - - void dump(); -}; - -/// Interface implemented by fragments that contain encoded instructions and/or -/// data. -/// -class MCEncodedFragment : public MCFragment { -protected: - MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions, - MCSection *Sec) - : MCFragment(FType, HasInstructions, 0, Sec) {} - -public: - static bool classof(const MCFragment *F) { - MCFragment::FragmentType Kind = F->getKind(); - switch (Kind) { - default: - return false; - case MCFragment::FT_Relaxable: - case MCFragment::FT_CompactEncodedInst: - case MCFragment::FT_Data: - return true; - } - } -}; - -/// Interface implemented by fragments that contain encoded instructions and/or -/// data. -/// -template<unsigned ContentsSize> -class MCEncodedFragmentWithContents : public MCEncodedFragment { - SmallVector<char, ContentsSize> Contents; - -protected: - MCEncodedFragmentWithContents(MCFragment::FragmentType FType, - bool HasInstructions, - MCSection *Sec) - : MCEncodedFragment(FType, HasInstructions, Sec) {} - -public: - SmallVectorImpl<char> &getContents() { return Contents; } - const SmallVectorImpl<char> &getContents() const { return Contents; } -}; - -/// Interface implemented by fragments that contain encoded instructions and/or -/// data and also have fixups registered. -/// -template<unsigned ContentsSize, unsigned FixupsSize> -class MCEncodedFragmentWithFixups : - public MCEncodedFragmentWithContents<ContentsSize> { - - /// Fixups - The list of fixups in this fragment. - SmallVector<MCFixup, FixupsSize> Fixups; - -protected: - MCEncodedFragmentWithFixups(MCFragment::FragmentType FType, - bool HasInstructions, - MCSection *Sec) - : MCEncodedFragmentWithContents<ContentsSize>(FType, HasInstructions, - Sec) {} - -public: - typedef SmallVectorImpl<MCFixup>::const_iterator const_fixup_iterator; - typedef SmallVectorImpl<MCFixup>::iterator fixup_iterator; - - SmallVectorImpl<MCFixup> &getFixups() { return Fixups; } - const SmallVectorImpl<MCFixup> &getFixups() const { return Fixups; } - - fixup_iterator fixup_begin() { return Fixups.begin(); } - const_fixup_iterator fixup_begin() const { return Fixups.begin(); } - - fixup_iterator fixup_end() { return Fixups.end(); } - const_fixup_iterator fixup_end() const { return Fixups.end(); } - - static bool classof(const MCFragment *F) { - MCFragment::FragmentType Kind = F->getKind(); - return Kind == MCFragment::FT_Relaxable || Kind == MCFragment::FT_Data; - } -}; - -/// Fragment for data and encoded instructions. -/// -class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> { -public: - MCDataFragment(MCSection *Sec = nullptr) - : MCEncodedFragmentWithFixups<32, 4>(FT_Data, false, Sec) {} - - void setHasInstructions(bool V) { HasInstructions = V; } - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_Data; - } -}; - -/// This is a compact (memory-size-wise) fragment for holding an encoded -/// instruction (non-relaxable) that has no fixups registered. When applicable, -/// it can be used instead of MCDataFragment and lead to lower memory -/// consumption. -/// -class MCCompactEncodedInstFragment : public MCEncodedFragmentWithContents<4> { -public: - MCCompactEncodedInstFragment(MCSection *Sec = nullptr) - : MCEncodedFragmentWithContents(FT_CompactEncodedInst, true, Sec) { - } - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_CompactEncodedInst; - } -}; - -/// A relaxable fragment holds on to its MCInst, since it may need to be -/// relaxed during the assembler layout and relaxation stage. -/// -class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> { - - /// Inst - The instruction this is a fragment for. - MCInst Inst; - - /// STI - The MCSubtargetInfo in effect when the instruction was encoded. - /// Keep a copy instead of a reference to make sure that updates to STI - /// in the assembler are not seen here. - const MCSubtargetInfo STI; - -public: - MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI, - MCSection *Sec = nullptr) - : MCEncodedFragmentWithFixups(FT_Relaxable, true, Sec), - Inst(Inst), STI(STI) {} - - const MCInst &getInst() const { return Inst; } - void setInst(const MCInst &Value) { Inst = Value; } - - const MCSubtargetInfo &getSubtargetInfo() { return STI; } - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_Relaxable; - } -}; - -class MCAlignFragment : public MCFragment { - - /// Alignment - The alignment to ensure, in bytes. - unsigned Alignment; - - /// EmitNops - Flag to indicate that (optimal) NOPs should be emitted instead - /// of using the provided value. The exact interpretation of this flag is - /// target dependent. - bool EmitNops : 1; - - /// Value - Value to use for filling padding bytes. - int64_t Value; - - /// ValueSize - The size of the integer (in bytes) of \p Value. - unsigned ValueSize; - - /// MaxBytesToEmit - The maximum number of bytes to emit; if the alignment - /// cannot be satisfied in this width then this fragment is ignored. - unsigned MaxBytesToEmit; - -public: - MCAlignFragment(unsigned Alignment, int64_t Value, unsigned ValueSize, - unsigned MaxBytesToEmit, MCSection *Sec = nullptr) - : MCFragment(FT_Align, false, 0, Sec), Alignment(Alignment), - EmitNops(false), Value(Value), - ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {} - - /// \name Accessors - /// @{ - - unsigned getAlignment() const { return Alignment; } - - int64_t getValue() const { return Value; } - - unsigned getValueSize() const { return ValueSize; } - - unsigned getMaxBytesToEmit() const { return MaxBytesToEmit; } - - bool hasEmitNops() const { return EmitNops; } - void setEmitNops(bool Value) { EmitNops = Value; } - - /// @} - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_Align; - } -}; - -class MCFillFragment : public MCFragment { - - /// Value - Value to use for filling bytes. - int64_t Value; - - /// ValueSize - The size (in bytes) of \p Value to use when filling, or 0 if - /// this is a virtual fill fragment. - unsigned ValueSize; - - /// Size - The number of bytes to insert. - uint64_t Size; - -public: - MCFillFragment(int64_t Value, unsigned ValueSize, uint64_t Size, - MCSection *Sec = nullptr) - : MCFragment(FT_Fill, false, 0, Sec), Value(Value), ValueSize(ValueSize), - Size(Size) { - assert((!ValueSize || (Size % ValueSize) == 0) && - "Fill size must be a multiple of the value size!"); - } - - /// \name Accessors - /// @{ - - int64_t getValue() const { return Value; } - - unsigned getValueSize() const { return ValueSize; } - - uint64_t getSize() const { return Size; } - - /// @} - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_Fill; - } -}; - -class MCOrgFragment : public MCFragment { - - /// Offset - The offset this fragment should start at. - const MCExpr *Offset; - - /// Value - Value to use for filling bytes. - int8_t Value; - -public: - MCOrgFragment(const MCExpr &Offset, int8_t Value, MCSection *Sec = nullptr) - : MCFragment(FT_Org, false, 0, Sec), Offset(&Offset), Value(Value) {} - - /// \name Accessors - /// @{ - - const MCExpr &getOffset() const { return *Offset; } - - uint8_t getValue() const { return Value; } - - /// @} - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_Org; - } -}; - -class MCLEBFragment : public MCFragment { - - /// Value - The value this fragment should contain. - const MCExpr *Value; - - /// IsSigned - True if this is a sleb128, false if uleb128. - bool IsSigned; - - SmallString<8> Contents; - -public: - MCLEBFragment(const MCExpr &Value_, bool IsSigned_, MCSection *Sec = nullptr) - : MCFragment(FT_LEB, false, 0, Sec), Value(&Value_), IsSigned(IsSigned_) { - Contents.push_back(0); - } - - /// \name Accessors - /// @{ - - const MCExpr &getValue() const { return *Value; } - - bool isSigned() const { return IsSigned; } - - SmallString<8> &getContents() { return Contents; } - const SmallString<8> &getContents() const { return Contents; } - - /// @} - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_LEB; - } -}; - -class MCDwarfLineAddrFragment : public MCFragment { - - /// LineDelta - the value of the difference between the two line numbers - /// between two .loc dwarf directives. - int64_t LineDelta; - - /// AddrDelta - The expression for the difference of the two symbols that - /// make up the address delta between two .loc dwarf directives. - const MCExpr *AddrDelta; - - SmallString<8> Contents; - -public: - MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta, - MCSection *Sec = nullptr) - : MCFragment(FT_Dwarf, false, 0, Sec), LineDelta(LineDelta), - AddrDelta(&AddrDelta) { - Contents.push_back(0); - } - - /// \name Accessors - /// @{ - - int64_t getLineDelta() const { return LineDelta; } - - const MCExpr &getAddrDelta() const { return *AddrDelta; } - - SmallString<8> &getContents() { return Contents; } - const SmallString<8> &getContents() const { return Contents; } - - /// @} - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_Dwarf; - } -}; - -class MCDwarfCallFrameFragment : public MCFragment { - - /// AddrDelta - The expression for the difference of the two symbols that - /// make up the address delta between two .cfi_* dwarf directives. - const MCExpr *AddrDelta; - - SmallString<8> Contents; - -public: - MCDwarfCallFrameFragment(const MCExpr &AddrDelta, MCSection *Sec = nullptr) - : MCFragment(FT_DwarfFrame, false, 0, Sec), AddrDelta(&AddrDelta) { - Contents.push_back(0); - } - - /// \name Accessors - /// @{ - - const MCExpr &getAddrDelta() const { return *AddrDelta; } - - SmallString<8> &getContents() { return Contents; } - const SmallString<8> &getContents() const { return Contents; } - - /// @} - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_DwarfFrame; - } -}; - -class MCSafeSEHFragment : public MCFragment { - const MCSymbol *Sym; - -public: - MCSafeSEHFragment(const MCSymbol *Sym, MCSection *Sec = nullptr) - : MCFragment(FT_SafeSEH, false, 0, Sec), Sym(Sym) {} - - /// \name Accessors - /// @{ - - const MCSymbol *getSymbol() { return Sym; } - const MCSymbol *getSymbol() const { return Sym; } - - /// @} - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_SafeSEH; - } -}; - // FIXME: This really doesn't belong here. See comments below. struct IndirectSymbolData { MCSymbol *Symbol; @@ -575,8 +100,6 @@ private: MCObjectWriter &Writer; - raw_ostream &OS; - SectionListType Sections; SymbolDataListType Symbols; @@ -591,6 +114,8 @@ private: /// List of declared file names std::vector<std::string> FileNames; + MCDwarfLineTableParams LTParams; + /// The set of function symbols for which a .thumb_func directive has /// been seen. // @@ -607,6 +132,7 @@ private: unsigned RelaxAll : 1; unsigned SubsectionsViaSymbols : 1; + unsigned IncrementalLinkerCompatible : 1; /// ELF specific e_header flags // It would be good if there were an MCELFAssembler class to hold this. @@ -712,16 +238,13 @@ public: public: /// Construct a new assembler instance. - /// - /// \param OS The stream to output to. // // FIXME: How are we going to parameterize this? Two obvious options are stay // concrete and require clients to pass in a target like object. The other // option is to make this abstract, and have targets provide concrete // implementations as we do with AsmParser. MCAssembler(MCContext &Context_, MCAsmBackend &Backend_, - MCCodeEmitter &Emitter_, MCObjectWriter &Writer_, - raw_ostream &OS); + MCCodeEmitter &Emitter_, MCObjectWriter &Writer_); ~MCAssembler(); /// Reuse an assembler instance @@ -736,15 +259,28 @@ public: MCObjectWriter &getWriter() const { return Writer; } + MCDwarfLineTableParams getDWARFLinetableParams() const { return LTParams; } + void setDWARFLinetableParams(MCDwarfLineTableParams P) { LTParams = P; } + /// Finish - Do final processing and write the object to the output stream. /// \p Writer is used for custom object writer (as the MCJIT does), /// if not specified it is automatically created from backend. void Finish(); + // Layout all section and prepare them for emission. + void layout(MCAsmLayout &Layout); + // FIXME: This does not belong here. bool getSubsectionsViaSymbols() const { return SubsectionsViaSymbols; } void setSubsectionsViaSymbols(bool Value) { SubsectionsViaSymbols = Value; } + bool isIncrementalLinkerCompatible() const { + return IncrementalLinkerCompatible; + } + void setIncrementalLinkerCompatible(bool Value) { + IncrementalLinkerCompatible = Value; + } + bool getRelaxAll() const { return RelaxAll; } void setRelaxAll(bool Value) { RelaxAll = Value; } @@ -856,13 +392,7 @@ public: /// \name Backend Data Access /// @{ - bool registerSection(MCSection &Section) { - if (Section.isRegistered()) - return false; - Sections.push_back(&Section); - Section.setIsRegistered(true); - return true; - } + bool registerSection(MCSection &Section); void registerSymbol(const MCSymbol &Symbol, bool *Created = nullptr); diff --git a/include/llvm/MC/MCContext.h b/include/llvm/MC/MCContext.h index 41169e9a12a0..e5a9afd9968c 100644 --- a/include/llvm/MC/MCContext.h +++ b/include/llvm/MC/MCContext.h @@ -17,6 +17,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/Twine.h" #include "llvm/MC/MCDwarf.h" +#include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/SectionKind.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" @@ -71,6 +72,10 @@ namespace llvm { /// objects. BumpPtrAllocator Allocator; + SpecificBumpPtrAllocator<MCSectionCOFF> COFFAllocator; + SpecificBumpPtrAllocator<MCSectionELF> ELFAllocator; + SpecificBumpPtrAllocator<MCSectionMachO> MachOAllocator; + /// Bindings of names to symbols. SymbolTable Symbols; @@ -108,7 +113,7 @@ namespace llvm { /// directive is used or it is an error. char *SecureLogFile; /// The stream that gets written to for the .secure_log_unique directive. - raw_ostream *SecureLog; + std::unique_ptr<raw_fd_ostream> SecureLog; /// Boolean toggled when .secure_log_unique / .secure_log_reset is seen to /// catch errors if .secure_log_unique appears twice without /// .secure_log_reset appearing between them. @@ -203,9 +208,13 @@ namespace llvm { std::map<COFFSectionKey, MCSectionCOFF *> COFFUniquingMap; StringMap<bool> ELFRelSecNames; + SpecificBumpPtrAllocator<MCSubtargetInfo> MCSubtargetAllocator; + /// Do automatic reset in destructor bool AutoReset; + bool HadError; + MCSymbol *createSymbolImpl(const StringMapEntry<bool> *Name, bool CanBeUnnamed); MCSymbol *createSymbol(StringRef Name, bool AlwaysAddSuffix, @@ -376,6 +385,9 @@ namespace llvm { MCSectionCOFF *getAssociativeCOFFSection(MCSectionCOFF *Sec, const MCSymbol *KeySym); + // Create and save a copy of STI and return a reference to the copy. + MCSubtargetInfo &getSubtargetCopy(const MCSubtargetInfo &STI); + /// @} /// \name Dwarf Management @@ -494,9 +506,11 @@ namespace llvm { /// @} char *getSecureLogFile() { return SecureLogFile; } - raw_ostream *getSecureLog() { return SecureLog; } + raw_fd_ostream *getSecureLog() { return SecureLog.get(); } bool getSecureLogUsed() { return SecureLogUsed; } - void setSecureLog(raw_ostream *Value) { SecureLog = Value; } + void setSecureLog(std::unique_ptr<raw_fd_ostream> Value) { + SecureLog = std::move(Value); + } void setSecureLogUsed(bool Value) { SecureLogUsed = Value; } void *allocate(unsigned Size, unsigned Align = 8) { @@ -504,11 +518,13 @@ namespace llvm { } void deallocate(void *Ptr) {} + bool hadError() { return HadError; } + void reportError(SMLoc L, const Twine &Msg); // Unrecoverable error has occurred. Display the best diagnostic we can // and bail via exit(1). For now, most MC backend errors are unrecoverable. // FIXME: We should really do something about that. LLVM_ATTRIBUTE_NORETURN void reportFatalError(SMLoc L, - const Twine &Msg) const; + const Twine &Msg); }; } // end namespace llvm @@ -538,7 +554,7 @@ namespace llvm { /// allocator supports it). /// \return The allocated memory. Could be NULL. inline void *operator new(size_t Bytes, llvm::MCContext &C, - size_t Alignment = 8) throw() { + size_t Alignment = 8) LLVM_NOEXCEPT { return C.allocate(Bytes, Alignment); } /// \brief Placement delete companion to the new above. @@ -547,8 +563,8 @@ inline void *operator new(size_t Bytes, llvm::MCContext &C, /// invoking it directly; see the new operator for more details. This operator /// is called implicitly by the compiler if a placement new expression using /// the MCContext throws in the object constructor. -inline void operator delete(void *Ptr, llvm::MCContext &C, size_t) - throw () { +inline void operator delete(void *Ptr, llvm::MCContext &C, + size_t) LLVM_NOEXCEPT { C.deallocate(Ptr); } @@ -571,8 +587,8 @@ inline void operator delete(void *Ptr, llvm::MCContext &C, size_t) /// \param Alignment The alignment of the allocated memory (if the underlying /// allocator supports it). /// \return The allocated memory. Could be NULL. -inline void *operator new[](size_t Bytes, llvm::MCContext& C, - size_t Alignment = 8) throw() { +inline void *operator new[](size_t Bytes, llvm::MCContext &C, + size_t Alignment = 8) LLVM_NOEXCEPT { return C.allocate(Bytes, Alignment); } @@ -582,7 +598,7 @@ inline void *operator new[](size_t Bytes, llvm::MCContext& C, /// invoking it directly; see the new[] operator for more details. This operator /// is called implicitly by the compiler if a placement new[] expression using /// the MCContext throws in the object constructor. -inline void operator delete[](void *Ptr, llvm::MCContext &C) throw () { +inline void operator delete[](void *Ptr, llvm::MCContext &C) LLVM_NOEXCEPT { C.deallocate(Ptr); } diff --git a/include/llvm/MC/MCDirectives.h b/include/llvm/MC/MCDirectives.h index f9d66e0b15d7..326b2a1ac061 100644 --- a/include/llvm/MC/MCDirectives.h +++ b/include/llvm/MC/MCDirectives.h @@ -62,7 +62,9 @@ enum MCDataRegionType { enum MCVersionMinType { MCVM_IOSVersionMin, ///< .ios_version_min - MCVM_OSXVersionMin ///< .macosx_version_min + MCVM_OSXVersionMin, ///< .macosx_version_min + MCVM_TvOSVersionMin, ///< .tvos_version_min + MCVM_WatchOSVersionMin, ///< .watchos_version_min }; } // end namespace llvm diff --git a/include/llvm/MC/MCDwarf.h b/include/llvm/MC/MCDwarf.h index 1e72dfee4ad1..8a50863a0c39 100644 --- a/include/llvm/MC/MCDwarf.h +++ b/include/llvm/MC/MCDwarf.h @@ -19,6 +19,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/MC/MCSection.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Dwarf.h" #include "llvm/Support/raw_ostream.h" @@ -31,7 +32,6 @@ namespace llvm { class MCAsmBackend; class MCContext; class MCObjectStreamer; -class MCSection; class MCStreamer; class MCSymbol; class SourceMgr; @@ -182,6 +182,19 @@ public: } }; +struct MCDwarfLineTableParams { + /// First special line opcode - leave room for the standard opcodes. + /// Note: If you want to change this, you'll have to update the + /// "StandardOpcodeLengths" table that is emitted in + /// \c Emit(). + uint8_t DWARF2LineOpcodeBase = 13; + /// Minimum line offset in a special line info. opcode. The value + /// -5 was chosen to give a reasonable range of values. + int8_t DWARF2LineBase = -5; + /// Range of line offsets in a special line info. opcode. + uint8_t DWARF2LineRange = 14; +}; + struct MCDwarfLineTableHeader { MCSymbol *Label; SmallVector<std::string, 3> MCDwarfDirs; @@ -192,9 +205,11 @@ struct MCDwarfLineTableHeader { MCDwarfLineTableHeader() : Label(nullptr) {} unsigned getFile(StringRef &Directory, StringRef &FileName, unsigned FileNumber = 0); - std::pair<MCSymbol *, MCSymbol *> Emit(MCStreamer *MCOS) const; + std::pair<MCSymbol *, MCSymbol *> Emit(MCStreamer *MCOS, + MCDwarfLineTableParams Params) const; std::pair<MCSymbol *, MCSymbol *> - Emit(MCStreamer *MCOS, ArrayRef<char> SpecialOpcodeLengths) const; + Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params, + ArrayRef<char> SpecialOpcodeLengths) const; }; class MCDwarfDwoLineTable { @@ -206,7 +221,7 @@ public: unsigned getFile(StringRef Directory, StringRef FileName) { return Header.getFile(Directory, FileName); } - void Emit(MCStreamer &MCOS) const; + void Emit(MCStreamer &MCOS, MCDwarfLineTableParams Params) const; }; class MCDwarfLineTable { @@ -215,10 +230,10 @@ class MCDwarfLineTable { public: // This emits the Dwarf file and the line tables for all Compile Units. - static void Emit(MCObjectStreamer *MCOS); + static void Emit(MCObjectStreamer *MCOS, MCDwarfLineTableParams Params); // This emits the Dwarf file and the line tables for a given Compile Unit. - void EmitCU(MCObjectStreamer *MCOS) const; + void EmitCU(MCObjectStreamer *MCOS, MCDwarfLineTableParams Params) const; unsigned getFile(StringRef &Directory, StringRef &FileName, unsigned FileNumber = 0); @@ -262,11 +277,12 @@ public: class MCDwarfLineAddr { public: /// Utility function to encode a Dwarf pair of LineDelta and AddrDeltas. - static void Encode(MCContext &Context, int64_t LineDelta, uint64_t AddrDelta, - raw_ostream &OS); + static void Encode(MCContext &Context, MCDwarfLineTableParams Params, + int64_t LineDelta, uint64_t AddrDelta, raw_ostream &OS); /// Utility function to emit the encoding to a streamer. - static void Emit(MCStreamer *MCOS, int64_t LineDelta, uint64_t AddrDelta); + static void Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params, + int64_t LineDelta, uint64_t AddrDelta); }; class MCGenDwarfInfo { @@ -324,7 +340,8 @@ public: OpRestore, OpUndefined, OpRegister, - OpWindowSave + OpWindowSave, + OpGnuArgsSize }; private: @@ -438,6 +455,11 @@ public: return MCCFIInstruction(OpEscape, L, 0, 0, Vals); } + /// \brief A special wrapper for .cfi_escape that indicates GNU_ARGS_SIZE + static MCCFIInstruction createGnuArgsSize(MCSymbol *L, int Size) { + return MCCFIInstruction(OpGnuArgsSize, L, 0, Size, ""); + } + OpType getOperation() const { return Operation; } MCSymbol *getLabel() const { return Label; } @@ -457,7 +479,7 @@ public: int getOffset() const { assert(Operation == OpDefCfa || Operation == OpOffset || Operation == OpRelOffset || Operation == OpDefCfaOffset || - Operation == OpAdjustCfaOffset); + Operation == OpAdjustCfaOffset || Operation == OpGnuArgsSize); return Offset; } diff --git a/include/llvm/MC/MCELFObjectWriter.h b/include/llvm/MC/MCELFObjectWriter.h index 01f694d3b756..193dac018b2b 100644 --- a/include/llvm/MC/MCELFObjectWriter.h +++ b/include/llvm/MC/MCELFObjectWriter.h @@ -57,8 +57,6 @@ public: case Triple::PS4: case Triple::FreeBSD: return ELF::ELFOSABI_FREEBSD; - case Triple::Linux: - return ELF::ELFOSABI_LINUX; default: return ELF::ELFOSABI_NONE; } diff --git a/include/llvm/MC/MCELFStreamer.h b/include/llvm/MC/MCELFStreamer.h index a5b257f5958b..6eb2c2c343ff 100644 --- a/include/llvm/MC/MCELFStreamer.h +++ b/include/llvm/MC/MCELFStreamer.h @@ -36,7 +36,6 @@ public: /// state management void reset() override { SeenIdent = false; - LocalCommons.clear(); BundleGroups.clear(); MCObjectStreamer::reset(); } @@ -69,7 +68,7 @@ public: void EmitTBSSSymbol(MCSection *Section, MCSymbol *Symbol, uint64_t Size, unsigned ByteAlignment = 0) override; void EmitValueImpl(const MCExpr *Value, unsigned Size, - const SMLoc &Loc = SMLoc()) override; + SMLoc Loc = SMLoc()) override; void EmitFileDirective(StringRef Filename) override; @@ -77,8 +76,6 @@ public: void EmitValueToAlignment(unsigned, int64_t, unsigned, unsigned) override; - void Flush() override; - void FinishImpl() override; void EmitBundleAlignMode(unsigned AlignPow2) override; @@ -97,14 +94,6 @@ private: bool SeenIdent; - struct LocalCommon { - const MCSymbol *Symbol; - uint64_t Size; - unsigned ByteAlignment; - }; - - std::vector<LocalCommon> LocalCommons; - /// BundleGroups - The stack of fragments holding the bundle-locked /// instructions. llvm::SmallVector<MCDataFragment *, 4> BundleGroups; diff --git a/include/llvm/MC/MCExpr.h b/include/llvm/MC/MCExpr.h index b3a607351a82..1d6bdef0af27 100644 --- a/include/llvm/MC/MCExpr.h +++ b/include/llvm/MC/MCExpr.h @@ -20,6 +20,7 @@ class MCAsmLayout; class MCAssembler; class MCContext; class MCFixup; +class MCFragment; class MCSection; class MCStreamer; class MCSymbol; @@ -115,7 +116,7 @@ public: /// currently defined as the absolute section for constants, or /// otherwise the section associated with the first defined symbol in the /// expression. - MCSection *findAssociatedSection() const; + MCFragment *findAssociatedFragment() const; /// @} }; @@ -187,6 +188,7 @@ public: VK_WEAKREF, // The link between the symbols in .weakref foo, bar VK_ARM_NONE, + VK_ARM_GOT_PREL, VK_ARM_TARGET1, VK_ARM_TARGET2, VK_ARM_PREL31, @@ -556,7 +558,7 @@ public: const MCAsmLayout *Layout, const MCFixup *Fixup) const = 0; virtual void visitUsedExpr(MCStreamer& Streamer) const = 0; - virtual MCSection *findAssociatedSection() const = 0; + virtual MCFragment *findAssociatedFragment() const = 0; virtual void fixELFSymbolsInTLSFixups(MCAssembler &) const = 0; diff --git a/include/llvm/MC/MCFixedLenDisassembler.h b/include/llvm/MC/MCFixedLenDisassembler.h index ad99943df2c3..ad34d9494bb9 100644 --- a/include/llvm/MC/MCFixedLenDisassembler.h +++ b/include/llvm/MC/MCFixedLenDisassembler.h @@ -22,6 +22,8 @@ enum DecoderOps { // uleb128 Val, uint16_t NumToSkip) OPC_CheckPredicate, // OPC_CheckPredicate(uleb128 PIdx, uint16_t NumToSkip) OPC_Decode, // OPC_Decode(uleb128 Opcode, uleb128 DIdx) + OPC_TryDecode, // OPC_TryDecode(uleb128 Opcode, uleb128 DIdx, + // uint16_t NumToSkip) OPC_SoftFail, // OPC_SoftFail(uleb128 PMask, uleb128 NMask) OPC_Fail // OPC_Fail() }; diff --git a/include/llvm/MC/MCFragment.h b/include/llvm/MC/MCFragment.h new file mode 100644 index 000000000000..7d6db525ce61 --- /dev/null +++ b/include/llvm/MC/MCFragment.h @@ -0,0 +1,506 @@ +//===- MCFragment.h - Fragment type hierarchy -------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCFRAGMENT_H +#define LLVM_MC_MCFRAGMENT_H + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/ilist.h" +#include "llvm/ADT/ilist_node.h" +#include "llvm/ADT/iterator.h" +#include "llvm/MC/MCFixup.h" +#include "llvm/MC/MCInst.h" + +namespace llvm { +class MCSection; +class MCSymbol; +class MCSubtargetInfo; + +class MCFragment : public ilist_node_with_parent<MCFragment, MCSection> { + friend class MCAsmLayout; + + MCFragment(const MCFragment &) = delete; + void operator=(const MCFragment &) = delete; + +public: + enum FragmentType : uint8_t { + FT_Align, + FT_Data, + FT_CompactEncodedInst, + FT_Fill, + FT_Relaxable, + FT_Org, + FT_Dwarf, + FT_DwarfFrame, + FT_LEB, + FT_SafeSEH, + FT_Dummy + }; + +private: + FragmentType Kind; + +protected: + bool HasInstructions; + +private: + /// \brief Should this fragment be aligned to the end of a bundle? + bool AlignToBundleEnd; + + uint8_t BundlePadding; + + /// LayoutOrder - The layout order of this fragment. + unsigned LayoutOrder; + + /// The data for the section this fragment is in. + MCSection *Parent; + + /// Atom - The atom this fragment is in, as represented by it's defining + /// symbol. + const MCSymbol *Atom; + + /// \name Assembler Backend Data + /// @{ + // + // FIXME: This could all be kept private to the assembler implementation. + + /// Offset - The offset of this fragment in its section. This is ~0 until + /// initialized. + uint64_t Offset; + + /// @} + +protected: + MCFragment(FragmentType Kind, bool HasInstructions, + uint8_t BundlePadding, MCSection *Parent = nullptr); + + ~MCFragment(); +private: + + // This is a friend so that the sentinal can be created. + friend struct ilist_sentinel_traits<MCFragment>; + MCFragment(); + +public: + /// Destroys the current fragment. + /// + /// This must be used instead of delete as MCFragment is non-virtual. + /// This method will dispatch to the appropriate subclass. + void destroy(); + + FragmentType getKind() const { return Kind; } + + MCSection *getParent() const { return Parent; } + void setParent(MCSection *Value) { Parent = Value; } + + const MCSymbol *getAtom() const { return Atom; } + void setAtom(const MCSymbol *Value) { Atom = Value; } + + unsigned getLayoutOrder() const { return LayoutOrder; } + void setLayoutOrder(unsigned Value) { LayoutOrder = Value; } + + /// \brief Does this fragment have instructions emitted into it? By default + /// this is false, but specific fragment types may set it to true. + bool hasInstructions() const { return HasInstructions; } + + /// \brief Should this fragment be placed at the end of an aligned bundle? + bool alignToBundleEnd() const { return AlignToBundleEnd; } + void setAlignToBundleEnd(bool V) { AlignToBundleEnd = V; } + + /// \brief Get the padding size that must be inserted before this fragment. + /// Used for bundling. By default, no padding is inserted. + /// Note that padding size is restricted to 8 bits. This is an optimization + /// to reduce the amount of space used for each fragment. In practice, larger + /// padding should never be required. + uint8_t getBundlePadding() const { return BundlePadding; } + + /// \brief Set the padding size for this fragment. By default it's a no-op, + /// and only some fragments have a meaningful implementation. + void setBundlePadding(uint8_t N) { BundlePadding = N; } + + /// \brief Return true if given frgment has FT_Dummy type. + bool isDummy() const { return Kind == FT_Dummy; } + + void dump(); +}; + +class MCDummyFragment : public MCFragment { +public: + explicit MCDummyFragment(MCSection *Sec) + : MCFragment(FT_Dummy, false, 0, Sec){}; + static bool classof(const MCFragment *F) { return F->getKind() == FT_Dummy; } +}; + +/// Interface implemented by fragments that contain encoded instructions and/or +/// data. +/// +class MCEncodedFragment : public MCFragment { +protected: + MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions, + MCSection *Sec) + : MCFragment(FType, HasInstructions, 0, Sec) {} + +public: + static bool classof(const MCFragment *F) { + MCFragment::FragmentType Kind = F->getKind(); + switch (Kind) { + default: + return false; + case MCFragment::FT_Relaxable: + case MCFragment::FT_CompactEncodedInst: + case MCFragment::FT_Data: + return true; + } + } +}; + +/// Interface implemented by fragments that contain encoded instructions and/or +/// data. +/// +template<unsigned ContentsSize> +class MCEncodedFragmentWithContents : public MCEncodedFragment { + SmallVector<char, ContentsSize> Contents; + +protected: + MCEncodedFragmentWithContents(MCFragment::FragmentType FType, + bool HasInstructions, + MCSection *Sec) + : MCEncodedFragment(FType, HasInstructions, Sec) {} + +public: + SmallVectorImpl<char> &getContents() { return Contents; } + const SmallVectorImpl<char> &getContents() const { return Contents; } +}; + +/// Interface implemented by fragments that contain encoded instructions and/or +/// data and also have fixups registered. +/// +template<unsigned ContentsSize, unsigned FixupsSize> +class MCEncodedFragmentWithFixups : + public MCEncodedFragmentWithContents<ContentsSize> { + + /// Fixups - The list of fixups in this fragment. + SmallVector<MCFixup, FixupsSize> Fixups; + +protected: + MCEncodedFragmentWithFixups(MCFragment::FragmentType FType, + bool HasInstructions, + MCSection *Sec) + : MCEncodedFragmentWithContents<ContentsSize>(FType, HasInstructions, + Sec) {} + +public: + typedef SmallVectorImpl<MCFixup>::const_iterator const_fixup_iterator; + typedef SmallVectorImpl<MCFixup>::iterator fixup_iterator; + + SmallVectorImpl<MCFixup> &getFixups() { return Fixups; } + const SmallVectorImpl<MCFixup> &getFixups() const { return Fixups; } + + fixup_iterator fixup_begin() { return Fixups.begin(); } + const_fixup_iterator fixup_begin() const { return Fixups.begin(); } + + fixup_iterator fixup_end() { return Fixups.end(); } + const_fixup_iterator fixup_end() const { return Fixups.end(); } + + static bool classof(const MCFragment *F) { + MCFragment::FragmentType Kind = F->getKind(); + return Kind == MCFragment::FT_Relaxable || Kind == MCFragment::FT_Data; + } +}; + +/// Fragment for data and encoded instructions. +/// +class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> { +public: + MCDataFragment(MCSection *Sec = nullptr) + : MCEncodedFragmentWithFixups<32, 4>(FT_Data, false, Sec) {} + + void setHasInstructions(bool V) { HasInstructions = V; } + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_Data; + } +}; + +/// This is a compact (memory-size-wise) fragment for holding an encoded +/// instruction (non-relaxable) that has no fixups registered. When applicable, +/// it can be used instead of MCDataFragment and lead to lower memory +/// consumption. +/// +class MCCompactEncodedInstFragment : public MCEncodedFragmentWithContents<4> { +public: + MCCompactEncodedInstFragment(MCSection *Sec = nullptr) + : MCEncodedFragmentWithContents(FT_CompactEncodedInst, true, Sec) { + } + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_CompactEncodedInst; + } +}; + +/// A relaxable fragment holds on to its MCInst, since it may need to be +/// relaxed during the assembler layout and relaxation stage. +/// +class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> { + + /// Inst - The instruction this is a fragment for. + MCInst Inst; + + /// STI - The MCSubtargetInfo in effect when the instruction was encoded. + const MCSubtargetInfo &STI; + +public: + MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI, + MCSection *Sec = nullptr) + : MCEncodedFragmentWithFixups(FT_Relaxable, true, Sec), + Inst(Inst), STI(STI) {} + + const MCInst &getInst() const { return Inst; } + void setInst(const MCInst &Value) { Inst = Value; } + + const MCSubtargetInfo &getSubtargetInfo() { return STI; } + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_Relaxable; + } +}; + +class MCAlignFragment : public MCFragment { + + /// Alignment - The alignment to ensure, in bytes. + unsigned Alignment; + + /// EmitNops - Flag to indicate that (optimal) NOPs should be emitted instead + /// of using the provided value. The exact interpretation of this flag is + /// target dependent. + bool EmitNops : 1; + + /// Value - Value to use for filling padding bytes. + int64_t Value; + + /// ValueSize - The size of the integer (in bytes) of \p Value. + unsigned ValueSize; + + /// MaxBytesToEmit - The maximum number of bytes to emit; if the alignment + /// cannot be satisfied in this width then this fragment is ignored. + unsigned MaxBytesToEmit; + +public: + MCAlignFragment(unsigned Alignment, int64_t Value, unsigned ValueSize, + unsigned MaxBytesToEmit, MCSection *Sec = nullptr) + : MCFragment(FT_Align, false, 0, Sec), Alignment(Alignment), + EmitNops(false), Value(Value), + ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {} + + /// \name Accessors + /// @{ + + unsigned getAlignment() const { return Alignment; } + + int64_t getValue() const { return Value; } + + unsigned getValueSize() const { return ValueSize; } + + unsigned getMaxBytesToEmit() const { return MaxBytesToEmit; } + + bool hasEmitNops() const { return EmitNops; } + void setEmitNops(bool Value) { EmitNops = Value; } + + /// @} + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_Align; + } +}; + +class MCFillFragment : public MCFragment { + + /// Value - Value to use for filling bytes. + int64_t Value; + + /// ValueSize - The size (in bytes) of \p Value to use when filling, or 0 if + /// this is a virtual fill fragment. + unsigned ValueSize; + + /// Size - The number of bytes to insert. + uint64_t Size; + +public: + MCFillFragment(int64_t Value, unsigned ValueSize, uint64_t Size, + MCSection *Sec = nullptr) + : MCFragment(FT_Fill, false, 0, Sec), Value(Value), ValueSize(ValueSize), + Size(Size) { + assert((!ValueSize || (Size % ValueSize) == 0) && + "Fill size must be a multiple of the value size!"); + } + + /// \name Accessors + /// @{ + + int64_t getValue() const { return Value; } + + unsigned getValueSize() const { return ValueSize; } + + uint64_t getSize() const { return Size; } + + /// @} + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_Fill; + } +}; + +class MCOrgFragment : public MCFragment { + + /// Offset - The offset this fragment should start at. + const MCExpr *Offset; + + /// Value - Value to use for filling bytes. + int8_t Value; + +public: + MCOrgFragment(const MCExpr &Offset, int8_t Value, MCSection *Sec = nullptr) + : MCFragment(FT_Org, false, 0, Sec), Offset(&Offset), Value(Value) {} + + /// \name Accessors + /// @{ + + const MCExpr &getOffset() const { return *Offset; } + + uint8_t getValue() const { return Value; } + + /// @} + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_Org; + } +}; + +class MCLEBFragment : public MCFragment { + + /// Value - The value this fragment should contain. + const MCExpr *Value; + + /// IsSigned - True if this is a sleb128, false if uleb128. + bool IsSigned; + + SmallString<8> Contents; + +public: + MCLEBFragment(const MCExpr &Value_, bool IsSigned_, MCSection *Sec = nullptr) + : MCFragment(FT_LEB, false, 0, Sec), Value(&Value_), IsSigned(IsSigned_) { + Contents.push_back(0); + } + + /// \name Accessors + /// @{ + + const MCExpr &getValue() const { return *Value; } + + bool isSigned() const { return IsSigned; } + + SmallString<8> &getContents() { return Contents; } + const SmallString<8> &getContents() const { return Contents; } + + /// @} + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_LEB; + } +}; + +class MCDwarfLineAddrFragment : public MCFragment { + + /// LineDelta - the value of the difference between the two line numbers + /// between two .loc dwarf directives. + int64_t LineDelta; + + /// AddrDelta - The expression for the difference of the two symbols that + /// make up the address delta between two .loc dwarf directives. + const MCExpr *AddrDelta; + + SmallString<8> Contents; + +public: + MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta, + MCSection *Sec = nullptr) + : MCFragment(FT_Dwarf, false, 0, Sec), LineDelta(LineDelta), + AddrDelta(&AddrDelta) { + Contents.push_back(0); + } + + /// \name Accessors + /// @{ + + int64_t getLineDelta() const { return LineDelta; } + + const MCExpr &getAddrDelta() const { return *AddrDelta; } + + SmallString<8> &getContents() { return Contents; } + const SmallString<8> &getContents() const { return Contents; } + + /// @} + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_Dwarf; + } +}; + +class MCDwarfCallFrameFragment : public MCFragment { + + /// AddrDelta - The expression for the difference of the two symbols that + /// make up the address delta between two .cfi_* dwarf directives. + const MCExpr *AddrDelta; + + SmallString<8> Contents; + +public: + MCDwarfCallFrameFragment(const MCExpr &AddrDelta, MCSection *Sec = nullptr) + : MCFragment(FT_DwarfFrame, false, 0, Sec), AddrDelta(&AddrDelta) { + Contents.push_back(0); + } + + /// \name Accessors + /// @{ + + const MCExpr &getAddrDelta() const { return *AddrDelta; } + + SmallString<8> &getContents() { return Contents; } + const SmallString<8> &getContents() const { return Contents; } + + /// @} + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_DwarfFrame; + } +}; + +class MCSafeSEHFragment : public MCFragment { + const MCSymbol *Sym; + +public: + MCSafeSEHFragment(const MCSymbol *Sym, MCSection *Sec = nullptr) + : MCFragment(FT_SafeSEH, false, 0, Sec), Sym(Sym) {} + + /// \name Accessors + /// @{ + + const MCSymbol *getSymbol() { return Sym; } + const MCSymbol *getSymbol() const { return Sym; } + + /// @} + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_SafeSEH; + } +}; + +} // end namespace llvm + +#endif diff --git a/include/llvm/MC/MCInstrDesc.h b/include/llvm/MC/MCInstrDesc.h index 6a582e82d00e..88aab73d4058 100644 --- a/include/llvm/MC/MCInstrDesc.h +++ b/include/llvm/MC/MCInstrDesc.h @@ -15,12 +15,12 @@ #ifndef LLVM_MC_MCINSTRDESC_H #define LLVM_MC_MCINSTRDESC_H +#include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/DataTypes.h" #include <string> namespace llvm { class MCInst; - class MCRegisterInfo; class MCSubtargetInfo; class FeatureBitset; @@ -137,16 +137,16 @@ enum Flag { /// directly to describe itself. class MCInstrDesc { public: - unsigned short Opcode; // The opcode number - unsigned short NumOperands; // Num of args (may be more if variable_ops) - unsigned char NumDefs; // Num of args that are definitions - unsigned char Size; // Number of bytes in encoding. - unsigned short SchedClass; // enum identifying instr sched class - uint64_t Flags; // Flags identifying machine instr class - uint64_t TSFlags; // Target Specific Flag values - const uint16_t *ImplicitUses; // Registers implicitly read by this instr - const uint16_t *ImplicitDefs; // Registers implicitly defined by this instr - const MCOperandInfo *OpInfo; // 'NumOperands' entries about operands + unsigned short Opcode; // The opcode number + unsigned short NumOperands; // Num of args (may be more if variable_ops) + unsigned char NumDefs; // Num of args that are definitions + unsigned char Size; // Number of bytes in encoding. + unsigned short SchedClass; // enum identifying instr sched class + uint64_t Flags; // Flags identifying machine instr class + uint64_t TSFlags; // Target Specific Flag values + const MCPhysReg *ImplicitUses; // Registers implicitly read by this instr + const MCPhysReg *ImplicitDefs; // Registers implicitly defined by this instr + const MCOperandInfo *OpInfo; // 'NumOperands' entries about operands // Subtarget feature that this is deprecated on, if any // -1 implies this is not deprecated by any single feature. It may still be // deprecated due to a "complex" reason, below. @@ -336,8 +336,8 @@ public: /// \brief Return true if this instruction is convergent. /// - /// Convergent instructions may only be moved to locations that are - /// control-equivalent to their original positions. + /// Convergent instructions may not be made control-dependent on any + /// additional values. bool isConvergent() const { return Flags & (1 << MCID::Convergent); } //===--------------------------------------------------------------------===// @@ -472,7 +472,7 @@ public: /// marked as implicitly reading the 'CL' register, which it always does. /// /// This method returns null if the instruction has no implicit uses. - const uint16_t *getImplicitUses() const { return ImplicitUses; } + const MCPhysReg *getImplicitUses() const { return ImplicitUses; } /// \brief Return the number of implicit uses this instruction has. unsigned getNumImplicitUses() const { @@ -494,7 +494,7 @@ public: /// EAX/EDX/EFLAGS registers. /// /// This method returns null if the instruction has no implicit defs. - const uint16_t *getImplicitDefs() const { return ImplicitDefs; } + const MCPhysReg *getImplicitDefs() const { return ImplicitDefs; } /// \brief Return the number of implicit defs this instruct has. unsigned getNumImplicitDefs() const { @@ -509,7 +509,7 @@ public: /// \brief Return true if this instruction implicitly /// uses the specified physical register. bool hasImplicitUseOfPhysReg(unsigned Reg) const { - if (const uint16_t *ImpUses = ImplicitUses) + if (const MCPhysReg *ImpUses = ImplicitUses) for (; *ImpUses; ++ImpUses) if (*ImpUses == Reg) return true; diff --git a/include/llvm/MC/MCInstrItineraries.h b/include/llvm/MC/MCInstrItineraries.h index 161705de7c4e..b2871a9805e1 100644 --- a/include/llvm/MC/MCInstrItineraries.h +++ b/include/llvm/MC/MCInstrItineraries.h @@ -38,7 +38,7 @@ namespace llvm { /// /// { 2, x|y, 1 } /// indicates that the stage occupies either FU x or FU y for 2 -/// consecuative cycles and that the next stage starts one cycle +/// consecutive cycles and that the next stage starts one cycle /// after this stage starts. That is, the stage requirements /// overlap in time. /// diff --git a/include/llvm/MC/MCLinkerOptimizationHint.h b/include/llvm/MC/MCLinkerOptimizationHint.h index 4b6f7ecc9fba..a519c4b71b03 100644 --- a/include/llvm/MC/MCLinkerOptimizationHint.h +++ b/include/llvm/MC/MCLinkerOptimizationHint.h @@ -160,7 +160,7 @@ class MCLOHContainer { public: typedef SmallVectorImpl<MCLOHDirective> LOHDirectives; - MCLOHContainer() : EmitSize(0) {}; + MCLOHContainer() : EmitSize(0) {} /// Const accessor to the directives. const LOHDirectives &getDirectives() const { diff --git a/include/llvm/MC/MCMachObjectWriter.h b/include/llvm/MC/MCMachObjectWriter.h index 175d73e72c10..cd3db957afc1 100644 --- a/include/llvm/MC/MCMachObjectWriter.h +++ b/include/llvm/MC/MCMachObjectWriter.h @@ -13,6 +13,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallString.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCSection.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Support/DataTypes.h" @@ -104,7 +105,7 @@ class MachObjectWriter : public MCObjectWriter { /// \name Symbol Table Data /// @{ - StringTableBuilder StringTable; + StringTableBuilder StringTable{StringTableBuilder::MachO}; std::vector<MachSymbolData> LocalSymbolData; std::vector<MachSymbolData> ExternalSymbolData; std::vector<MachSymbolData> UndefinedSymbolData; @@ -159,19 +160,21 @@ public: /// @} - void writeHeader(unsigned NumLoadCommands, unsigned LoadCommandsSize, - bool SubsectionsViaSymbols); + void writeHeader(MachO::HeaderFileType Type, unsigned NumLoadCommands, + unsigned LoadCommandsSize, bool SubsectionsViaSymbols); /// Write a segment load command. /// /// \param NumSections The number of sections in this segment. /// \param SectionDataSize The total size of the sections. - void writeSegmentLoadCommand(unsigned NumSections, uint64_t VMSize, + void writeSegmentLoadCommand(StringRef Name, unsigned NumSections, + uint64_t VMAddr, uint64_t VMSize, uint64_t SectionDataStartOffset, - uint64_t SectionDataSize); + uint64_t SectionDataSize, uint32_t MaxProt, + uint32_t InitProt); - void writeSection(const MCAssembler &Asm, const MCAsmLayout &Layout, - const MCSection &Sec, uint64_t FileOffset, + void writeSection(const MCAsmLayout &Layout, const MCSection &Sec, + uint64_t VMAddr, uint64_t FileOffset, unsigned Flags, uint64_t RelocationsStart, unsigned NumRelocations); void writeSymtabLoadCommand(uint32_t SymbolOffset, uint32_t NumSymbols, @@ -246,6 +249,11 @@ public: const MCAsmLayout &Layout) override; bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, + const MCSymbol &A, + const MCSymbol &B, + bool InSet) const override; + + bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, const MCSymbol &SymA, const MCFragment &FB, bool InSet, bool IsPCRel) const override; diff --git a/include/llvm/MC/MCObjectFileInfo.h b/include/llvm/MC/MCObjectFileInfo.h index 99e3f92bfe26..cf2c3f12bb6b 100644 --- a/include/llvm/MC/MCObjectFileInfo.h +++ b/include/llvm/MC/MCObjectFileInfo.h @@ -35,16 +35,18 @@ protected: /// without an associated EH frame section. bool SupportsCompactUnwindWithoutEHFrame; - /// Some encoding values for EH. + /// OmitDwarfIfHaveCompactUnwind - True if the target object file + /// supports having some functions with compact unwind and other with + /// dwarf unwind. + bool OmitDwarfIfHaveCompactUnwind; + + /// PersonalityEncoding, LSDAEncoding, TTypeEncoding - Some encoding values + /// for EH. unsigned PersonalityEncoding; unsigned LSDAEncoding; unsigned FDECFIEncoding; unsigned TTypeEncoding; - /// Section flags for eh_frame - unsigned EHSectionType; - unsigned EHSectionFlags; - /// Compact unwind encoding indicating that we should emit only an EH frame. unsigned CompactUnwindDwarfEHFrameOnly; @@ -114,6 +116,10 @@ protected: MCSection *DwarfStrOffDWOSection; MCSection *DwarfAddrSection; + // These are for Fission DWP files. + MCSection *DwarfCUIndexSection; + MCSection *DwarfTUIndexSection; + /// Section for newer gnu pubnames. MCSection *DwarfGnuPubNamesSection; /// Section for newer gnu pubtypes. @@ -147,10 +153,7 @@ protected: MCSection *EHFrameSection; // ELF specific sections. - MCSection *DataRelSection; - const MCSection *DataRelLocalSection; MCSection *DataRelROSection; - MCSection *DataRelROLocalSection; MCSection *MergeableConst4Section; MCSection *MergeableConst8Section; MCSection *MergeableConst16Section; @@ -200,6 +203,10 @@ public: bool getSupportsCompactUnwindWithoutEHFrame() const { return SupportsCompactUnwindWithoutEHFrame; } + bool getOmitDwarfIfHaveCompactUnwind() const { + return OmitDwarfIfHaveCompactUnwind; + } + bool getCommDirectiveSupportsAlignment() const { return CommDirectiveSupportsAlignment; } @@ -216,6 +223,7 @@ public: MCSection *getTextSection() const { return TextSection; } MCSection *getDataSection() const { return DataSection; } MCSection *getBSSSection() const { return BSSSection; } + MCSection *getReadOnlySection() const { return ReadOnlySection; } MCSection *getLSDASection() const { return LSDASection; } MCSection *getCompactUnwindSection() const { return CompactUnwindSection; } MCSection *getDwarfAbbrevSection() const { return DwarfAbbrevSection; } @@ -258,6 +266,8 @@ public: MCSection *getDwarfLocDWOSection() const { return DwarfLocDWOSection; } MCSection *getDwarfStrOffDWOSection() const { return DwarfStrOffDWOSection; } MCSection *getDwarfAddrSection() const { return DwarfAddrSection; } + MCSection *getDwarfCUIndexSection() const { return DwarfCUIndexSection; } + MCSection *getDwarfTUIndexSection() const { return DwarfTUIndexSection; } MCSection *getCOFFDebugSymbolsSection() const { return COFFDebugSymbolsSection; @@ -271,12 +281,7 @@ public: MCSection *getFaultMapSection() const { return FaultMapSection; } // ELF specific sections. - MCSection *getDataRelSection() const { return DataRelSection; } - const MCSection *getDataRelLocalSection() const { - return DataRelLocalSection; - } MCSection *getDataRelROSection() const { return DataRelROSection; } - MCSection *getDataRelROLocalSection() const { return DataRelROLocalSection; } const MCSection *getMergeableConst4Section() const { return MergeableConst4Section; } @@ -325,8 +330,6 @@ public: MCSection *getSXDataSection() const { return SXDataSection; } MCSection *getEHFrameSection() { - if (!EHFrameSection) - InitEHFrameSection(); return EHFrameSection; } @@ -346,9 +349,6 @@ private: void initELFMCObjectFileInfo(Triple T); void initCOFFMCObjectFileInfo(Triple T); - /// Initialize EHFrameSection on demand. - void InitEHFrameSection(); - public: const Triple &getTargetTriple() const { return TT; } }; diff --git a/include/llvm/MC/MCObjectStreamer.h b/include/llvm/MC/MCObjectStreamer.h index ce1fc80f2cf2..9fe2fda21353 100644 --- a/include/llvm/MC/MCObjectStreamer.h +++ b/include/llvm/MC/MCObjectStreamer.h @@ -92,7 +92,7 @@ public: void EmitLabel(MCSymbol *Symbol) override; void EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) override; void EmitValueImpl(const MCExpr *Value, unsigned Size, - const SMLoc &Loc = SMLoc()) override; + SMLoc Loc = SMLoc()) override; void EmitULEB128Value(const MCExpr *Value) override; void EmitSLEB128Value(const MCExpr *Value) override; void EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override; @@ -112,7 +112,7 @@ public: unsigned MaxBytesToEmit = 0) override; void EmitCodeAlignment(unsigned ByteAlignment, unsigned MaxBytesToEmit = 0) override; - bool EmitValueToOffset(const MCExpr *Offset, unsigned char Value) override; + void emitValueToOffset(const MCExpr *Offset, unsigned char Value) override; void EmitDwarfLocDirective(unsigned FileNo, unsigned Line, unsigned Column, unsigned Flags, unsigned Isa, unsigned Discriminator, @@ -124,8 +124,9 @@ public: const MCSymbol *Label); void EmitGPRel32Value(const MCExpr *Value) override; void EmitGPRel64Value(const MCExpr *Value) override; + bool EmitRelocDirective(const MCExpr &Offset, StringRef Name, + const MCExpr *Expr, SMLoc Loc) override; void EmitFill(uint64_t NumBytes, uint8_t FillValue) override; - void EmitZeros(uint64_t NumBytes) override; void FinishImpl() override; /// Emit the absolute difference between two symbols if possible. diff --git a/include/llvm/MC/MCObjectWriter.h b/include/llvm/MC/MCObjectWriter.h index 2211673efc31..63c833ac20d6 100644 --- a/include/llvm/MC/MCObjectWriter.h +++ b/include/llvm/MC/MCObjectWriter.h @@ -40,14 +40,18 @@ class MCObjectWriter { MCObjectWriter(const MCObjectWriter &) = delete; void operator=(const MCObjectWriter &) = delete; -protected: - raw_pwrite_stream &OS; + raw_pwrite_stream *OS; +protected: unsigned IsLittleEndian : 1; protected: // Can only create subclasses. MCObjectWriter(raw_pwrite_stream &OS, bool IsLittleEndian) - : OS(OS), IsLittleEndian(IsLittleEndian) {} + : OS(&OS), IsLittleEndian(IsLittleEndian) {} + + unsigned getInitialOffset() { + return OS->tell(); + } public: virtual ~MCObjectWriter(); @@ -57,7 +61,8 @@ public: bool isLittleEndian() const { return IsLittleEndian; } - raw_ostream &getStream() { return OS; } + raw_pwrite_stream &getStream() { return *OS; } + void setStream(raw_pwrite_stream &NewOS) { OS = &NewOS; } /// \name High-Level API /// @{ @@ -92,6 +97,11 @@ public: bool InSet) const; virtual bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, + const MCSymbol &A, + const MCSymbol &B, + bool InSet) const; + + virtual bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm, const MCSymbol &SymA, const MCFragment &FB, bool InSet, @@ -113,30 +123,30 @@ public: /// \name Binary Output /// @{ - void write8(uint8_t Value) { OS << char(Value); } + void write8(uint8_t Value) { *OS << char(Value); } void writeLE16(uint16_t Value) { - support::endian::Writer<support::little>(OS).write(Value); + support::endian::Writer<support::little>(*OS).write(Value); } void writeLE32(uint32_t Value) { - support::endian::Writer<support::little>(OS).write(Value); + support::endian::Writer<support::little>(*OS).write(Value); } void writeLE64(uint64_t Value) { - support::endian::Writer<support::little>(OS).write(Value); + support::endian::Writer<support::little>(*OS).write(Value); } void writeBE16(uint16_t Value) { - support::endian::Writer<support::big>(OS).write(Value); + support::endian::Writer<support::big>(*OS).write(Value); } void writeBE32(uint32_t Value) { - support::endian::Writer<support::big>(OS).write(Value); + support::endian::Writer<support::big>(*OS).write(Value); } void writeBE64(uint64_t Value) { - support::endian::Writer<support::big>(OS).write(Value); + support::endian::Writer<support::big>(*OS).write(Value); } void write16(uint16_t Value) { @@ -164,9 +174,9 @@ public: const char Zeros[16] = {0}; for (unsigned i = 0, e = N / 16; i != e; ++i) - OS << StringRef(Zeros, 16); + *OS << StringRef(Zeros, 16); - OS << StringRef(Zeros, N % 16); + *OS << StringRef(Zeros, N % 16); } void writeBytes(const SmallVectorImpl<char> &ByteVec, @@ -180,7 +190,7 @@ public: assert( (ZeroFillSize == 0 || Str.size() <= ZeroFillSize) && "data size greater than fill size, unexpected large write will occur"); - OS << Str; + *OS << Str; if (ZeroFillSize) WriteZeros(ZeroFillSize - Str.size()); } diff --git a/include/llvm/MC/MCParser/AsmLexer.h b/include/llvm/MC/MCParser/AsmLexer.h index 62d39b26c860..1bb6d212784e 100644 --- a/include/llvm/MC/MCParser/AsmLexer.h +++ b/include/llvm/MC/MCParser/AsmLexer.h @@ -47,7 +47,8 @@ public: StringRef LexUntilEndOfStatement() override; StringRef LexUntilEndOfLine(); - const AsmToken peekTok(bool ShouldSkipSpace = true) override; + size_t peekTokens(MutableArrayRef<AsmToken> Buf, + bool ShouldSkipSpace = true) override; bool isAtStartOfComment(const char *Ptr); bool isAtStatementSeparator(const char *Ptr); diff --git a/include/llvm/MC/MCParser/MCAsmLexer.h b/include/llvm/MC/MCParser/MCAsmLexer.h index 71f15b37c331..55279f49529a 100644 --- a/include/llvm/MC/MCParser/MCAsmLexer.h +++ b/include/llvm/MC/MCParser/MCAsmLexer.h @@ -118,7 +118,7 @@ public: /// lexers. class MCAsmLexer { /// The current token, stored in the base class for faster access. - AsmToken CurTok; + SmallVector<AsmToken, 1> CurTok; /// The location and description of the current error SMLoc ErrLoc; @@ -135,7 +135,7 @@ protected: // Can only create subclasses. virtual AsmToken LexToken() = 0; - void SetError(const SMLoc &errLoc, const std::string &err) { + void SetError(SMLoc errLoc, const std::string &err) { ErrLoc = errLoc; Err = err; } @@ -148,7 +148,15 @@ public: /// The lexer will continuosly return the end-of-file token once the end of /// the main input file has been reached. const AsmToken &Lex() { - return CurTok = LexToken(); + assert(!CurTok.empty()); + CurTok.erase(CurTok.begin()); + if (CurTok.empty()) + CurTok.emplace_back(LexToken()); + return CurTok.front(); + } + + void UnLex(AsmToken const &Token) { + CurTok.insert(CurTok.begin(), Token); } virtual StringRef LexUntilEndOfStatement() = 0; @@ -158,14 +166,28 @@ public: /// Get the current (last) lexed token. const AsmToken &getTok() const { - return CurTok; + return CurTok[0]; } /// Look ahead at the next token to be lexed. - virtual const AsmToken peekTok(bool ShouldSkipSpace = true) = 0; + const AsmToken peekTok(bool ShouldSkipSpace = true) { + AsmToken Tok; + + MutableArrayRef<AsmToken> Buf(Tok); + size_t ReadCount = peekTokens(Buf, ShouldSkipSpace); + + assert(ReadCount == 1); + (void)ReadCount; + + return Tok; + } + + /// Look ahead an arbitrary number of tokens. + virtual size_t peekTokens(MutableArrayRef<AsmToken> Buf, + bool ShouldSkipSpace = true) = 0; /// Get the current error location - const SMLoc &getErrLoc() { + SMLoc getErrLoc() { return ErrLoc; } @@ -175,13 +197,13 @@ public: } /// Get the kind of current token. - AsmToken::TokenKind getKind() const { return CurTok.getKind(); } + AsmToken::TokenKind getKind() const { return getTok().getKind(); } /// Check if the current token has kind \p K. - bool is(AsmToken::TokenKind K) const { return CurTok.is(K); } + bool is(AsmToken::TokenKind K) const { return getTok().is(K); } /// Check if the current token has kind \p K. - bool isNot(AsmToken::TokenKind K) const { return CurTok.isNot(K); } + bool isNot(AsmToken::TokenKind K) const { return getTok().isNot(K); } /// Set whether spaces should be ignored by the lexer void setSkipSpace(bool val) { SkipSpace = val; } diff --git a/include/llvm/MC/MCParser/MCAsmParserExtension.h b/include/llvm/MC/MCParser/MCAsmParserExtension.h index 077fd21e073c..30b25dcfdaec 100644 --- a/include/llvm/MC/MCParser/MCAsmParserExtension.h +++ b/include/llvm/MC/MCParser/MCAsmParserExtension.h @@ -71,6 +71,9 @@ public: bool Error(SMLoc L, const Twine &Msg) { return getParser().Error(L, Msg); } + void Note(SMLoc L, const Twine &Msg) { + getParser().Note(L, Msg); + } bool TokError(const Twine &Msg) { return getParser().TokError(Msg); } diff --git a/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/include/llvm/MC/MCParser/MCParsedAsmOperand.h index a25108a0effb..a90d280c240c 100644 --- a/include/llvm/MC/MCParser/MCParsedAsmOperand.h +++ b/include/llvm/MC/MCParser/MCParsedAsmOperand.h @@ -30,8 +30,16 @@ class MCParsedAsmOperand { /// MS-style inline assembly. std::string Constraint; +protected: + // This only seems to need to be movable (by ARMOperand) but ARMOperand has + // lots of members and MSVC doesn't support defaulted move ops, so to avoid + // that verbosity, just rely on defaulted copy ops. It's only the Constraint + // string member that would benefit from movement anyway. + MCParsedAsmOperand(const MCParsedAsmOperand &RHS) = default; + MCParsedAsmOperand &operator=(const MCParsedAsmOperand &) = default; + MCParsedAsmOperand() = default; + public: - MCParsedAsmOperand() {} virtual ~MCParsedAsmOperand() {} void setConstraint(StringRef C) { Constraint = C.str(); } diff --git a/include/llvm/MC/MCRegisterInfo.h b/include/llvm/MC/MCRegisterInfo.h index 8e25ee18e08d..a4d5e0867232 100644 --- a/include/llvm/MC/MCRegisterInfo.h +++ b/include/llvm/MC/MCRegisterInfo.h @@ -632,7 +632,7 @@ private: unsigned Reg; const MCRegisterInfo *MCRI; bool IncludeSelf; - + MCRegUnitIterator RI; MCRegUnitRootIterator RRI; MCSuperRegIterator SI; @@ -652,10 +652,8 @@ public: } } - bool isValid() const { - return RI.isValid(); - } - + bool isValid() const { return RI.isValid(); } + unsigned operator*() const { assert (SI.isValid() && "Cannot dereference an invalid iterator."); return *SI; diff --git a/include/llvm/MC/MCSchedule.h b/include/llvm/MC/MCSchedule.h index c09791631056..d7f9b69a9a2c 100644 --- a/include/llvm/MC/MCSchedule.h +++ b/include/llvm/MC/MCSchedule.h @@ -183,7 +183,7 @@ struct MCSchedModel { // takes to recover from a branch misprediction. unsigned MispredictPenalty; static const unsigned DefaultMispredictPenalty = 10; - + bool PostRAScheduler; // default value is false bool CompleteModel; @@ -206,6 +206,9 @@ struct MCSchedModel { /// scheduling class (itinerary class or SchedRW list). bool isComplete() const { return CompleteModel; } + /// Return true if machine supports out of order execution. + bool isOutOfOrder() const { return MicroOpBufferSize > 1; } + unsigned getNumProcResourceKinds() const { return NumProcResourceKinds; } diff --git a/include/llvm/MC/MCSection.h b/include/llvm/MC/MCSection.h index 2d0d4dfc5913..09a98929113a 100644 --- a/include/llvm/MC/MCSection.h +++ b/include/llvm/MC/MCSection.h @@ -18,12 +18,13 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" +#include "llvm/MC/MCFragment.h" #include "llvm/MC/SectionKind.h" #include "llvm/Support/Compiler.h" namespace llvm { -class MCAssembler; class MCAsmInfo; +class MCAssembler; class MCContext; class MCExpr; class MCFragment; @@ -92,6 +93,8 @@ private: unsigned IsRegistered : 1; + MCDummyFragment DummyFragment; + FragmentListType Fragments; /// Mapping from subsection number to insertion point for subsection numbers @@ -102,10 +105,9 @@ protected: MCSection(SectionVariant V, SectionKind K, MCSymbol *Begin); SectionVariant Variant; SectionKind Kind; + ~MCSection(); public: - virtual ~MCSection(); - SectionKind getKind() const { return Kind; } SectionVariant getVariant() const { return Variant; } @@ -152,6 +154,14 @@ public: return const_cast<MCSection *>(this)->getFragmentList(); } + /// Support for MCFragment::getNextNode(). + static FragmentListType MCSection::*getSublistAccess(MCFragment *) { + return &MCSection::Fragments; + } + + const MCDummyFragment &getDummyFragment() const { return DummyFragment; } + MCDummyFragment &getDummyFragment() { return DummyFragment; } + MCSection::iterator begin(); MCSection::const_iterator begin() const { return const_cast<MCSection *>(this)->begin(); diff --git a/include/llvm/MC/MCSectionCOFF.h b/include/llvm/MC/MCSectionCOFF.h index 237f6d31fb1b..d94682c8c381 100644 --- a/include/llvm/MC/MCSectionCOFF.h +++ b/include/llvm/MC/MCSectionCOFF.h @@ -16,66 +16,63 @@ #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCSection.h" -#include "llvm/Support/COFF.h" namespace llvm { class MCSymbol; -/// MCSectionCOFF - This represents a section on Windows - class MCSectionCOFF : public MCSection { - // The memory for this string is stored in the same MCContext as *this. - StringRef SectionName; - - // FIXME: The following fields should not be mutable, but are for now so - // the asm parser can honor the .linkonce directive. - - /// Characteristics - This is the Characteristics field of a section, - /// drawn from the enums below. - mutable unsigned Characteristics; - - /// The COMDAT symbol of this section. Only valid if this is a COMDAT - /// section. Two COMDAT sections are merged if they have the same - /// COMDAT symbol. - MCSymbol *COMDATSymbol; - - /// Selection - This is the Selection field for the section symbol, if - /// it is a COMDAT section (Characteristics & IMAGE_SCN_LNK_COMDAT) != 0 - mutable int Selection; - - private: - friend class MCContext; - MCSectionCOFF(StringRef Section, unsigned Characteristics, - MCSymbol *COMDATSymbol, int Selection, SectionKind K, - MCSymbol *Begin) - : MCSection(SV_COFF, K, Begin), SectionName(Section), - Characteristics(Characteristics), COMDATSymbol(COMDATSymbol), - Selection(Selection) { - assert ((Characteristics & 0x00F00000) == 0 && - "alignment must not be set upon section creation"); - } - ~MCSectionCOFF() override; - - public: - /// ShouldOmitSectionDirective - Decides whether a '.section' directive - /// should be printed before the section name - bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; - - StringRef getSectionName() const { return SectionName; } - unsigned getCharacteristics() const { return Characteristics; } - MCSymbol *getCOMDATSymbol() const { return COMDATSymbol; } - int getSelection() const { return Selection; } - - void setSelection(int Selection) const; - - void PrintSwitchToSection(const MCAsmInfo &MAI, raw_ostream &OS, - const MCExpr *Subsection) const override; - bool UseCodeAlign() const override; - bool isVirtualSection() const override; - - static bool classof(const MCSection *S) { - return S->getVariant() == SV_COFF; - } - }; +/// This represents a section on Windows +class MCSectionCOFF final : public MCSection { + // The memory for this string is stored in the same MCContext as *this. + StringRef SectionName; + + // FIXME: The following fields should not be mutable, but are for now so the + // asm parser can honor the .linkonce directive. + + /// This is the Characteristics field of a section, drawn from the enums + /// below. + mutable unsigned Characteristics; + + /// The COMDAT symbol of this section. Only valid if this is a COMDAT section. + /// Two COMDAT sections are merged if they have the same COMDAT symbol. + MCSymbol *COMDATSymbol; + + /// This is the Selection field for the section symbol, if it is a COMDAT + /// section (Characteristics & IMAGE_SCN_LNK_COMDAT) != 0 + mutable int Selection; + +private: + friend class MCContext; + MCSectionCOFF(StringRef Section, unsigned Characteristics, + MCSymbol *COMDATSymbol, int Selection, SectionKind K, + MCSymbol *Begin) + : MCSection(SV_COFF, K, Begin), SectionName(Section), + Characteristics(Characteristics), COMDATSymbol(COMDATSymbol), + Selection(Selection) { + assert((Characteristics & 0x00F00000) == 0 && + "alignment must not be set upon section creation"); + } + +public: + ~MCSectionCOFF(); + + /// Decides whether a '.section' directive should be printed before the + /// section name + bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; + + StringRef getSectionName() const { return SectionName; } + unsigned getCharacteristics() const { return Characteristics; } + MCSymbol *getCOMDATSymbol() const { return COMDATSymbol; } + int getSelection() const { return Selection; } + + void setSelection(int Selection) const; + + void PrintSwitchToSection(const MCAsmInfo &MAI, raw_ostream &OS, + const MCExpr *Subsection) const override; + bool UseCodeAlign() const override; + bool isVirtualSection() const override; + + static bool classof(const MCSection *S) { return S->getVariant() == SV_COFF; } +}; } // end namespace llvm diff --git a/include/llvm/MC/MCSectionELF.h b/include/llvm/MC/MCSectionELF.h index f6730371fe15..b3bb3ad4e02c 100644 --- a/include/llvm/MC/MCSectionELF.h +++ b/include/llvm/MC/MCSectionELF.h @@ -25,25 +25,24 @@ namespace llvm { class MCSymbol; -/// MCSectionELF - This represents a section on linux, lots of unix variants -/// and some bare metal systems. -class MCSectionELF : public MCSection { - /// SectionName - This is the name of the section. The referenced memory is - /// owned by TargetLoweringObjectFileELF's ELFUniqueMap. +/// This represents a section on linux, lots of unix variants and some bare +/// metal systems. +class MCSectionELF final : public MCSection { + /// This is the name of the section. The referenced memory is owned by + /// TargetLoweringObjectFileELF's ELFUniqueMap. StringRef SectionName; - /// Type - This is the sh_type field of a section, drawn from the enums below. + /// This is the sh_type field of a section, drawn from the enums below. unsigned Type; - /// Flags - This is the sh_flags field of a section, drawn from the enums. - /// below. + /// This is the sh_flags field of a section, drawn from the enums below. unsigned Flags; unsigned UniqueID; - /// EntrySize - The size of each entry in this section. This size only - /// makes sense for sections that contain fixed-sized entries. If a - /// section does not contain fixed-sized entries 'EntrySize' will be 0. + /// The size of each entry in this section. This size only makes sense for + /// sections that contain fixed-sized entries. If a section does not contain + /// fixed-sized entries 'EntrySize' will be 0. unsigned EntrySize; const MCSymbolELF *Group; @@ -62,14 +61,14 @@ private: if (Group) Group->setIsSignature(); } - ~MCSectionELF() override; void setSectionName(StringRef Name) { SectionName = Name; } public: + ~MCSectionELF(); - /// ShouldOmitSectionDirective - Decides whether a '.section' directive - /// should be printed before the section name + /// Decides whether a '.section' directive should be printed before the + /// section name bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; StringRef getSectionName() const { return SectionName; } diff --git a/include/llvm/MC/MCSectionMachO.h b/include/llvm/MC/MCSectionMachO.h index 97227517c82d..658dfcda7268 100644 --- a/include/llvm/MC/MCSectionMachO.h +++ b/include/llvm/MC/MCSectionMachO.h @@ -20,19 +20,18 @@ namespace llvm { -/// MCSectionMachO - This represents a section on a Mach-O system (used by -/// Mac OS X). On a Mac system, these are also described in -/// /usr/include/mach-o/loader.h. -class MCSectionMachO : public MCSection { +/// This represents a section on a Mach-O system (used by Mac OS X). On a Mac +/// system, these are also described in /usr/include/mach-o/loader.h. +class MCSectionMachO final : public MCSection { char SegmentName[16]; // Not necessarily null terminated! char SectionName[16]; // Not necessarily null terminated! - /// TypeAndAttributes - This is the SECTION_TYPE and SECTION_ATTRIBUTES - /// field of a section, drawn from the enums below. + /// This is the SECTION_TYPE and SECTION_ATTRIBUTES field of a section, drawn + /// from the enums below. unsigned TypeAndAttributes; - /// Reserved2 - The 'reserved2' field of a section, used to represent the - /// size of stubs, for example. + /// The 'reserved2' field of a section, used to represent the size of stubs, + /// for example. unsigned Reserved2; MCSectionMachO(StringRef Segment, StringRef Section, unsigned TAA, @@ -64,12 +63,12 @@ public: return (TypeAndAttributes & Value) != 0; } - /// ParseSectionSpecifier - Parse the section specifier indicated by "Spec". - /// This is a string that can appear after a .section directive in a mach-o - /// flavored .s file. If successful, this fills in the specified Out - /// parameters and returns an empty string. When an invalid section - /// specifier is present, this returns a string indicating the problem. - /// If no TAA was parsed, TAA is not altered, and TAAWasSet becomes false. + /// Parse the section specifier indicated by "Spec". This is a string that can + /// appear after a .section directive in a mach-o flavored .s file. If + /// successful, this fills in the specified Out parameters and returns an + /// empty string. When an invalid section specifier is present, this returns + /// a string indicating the problem. If no TAA was parsed, TAA is not altered, + /// and TAAWasSet becomes false. static std::string ParseSectionSpecifier(StringRef Spec, // In. StringRef &Segment, // Out. StringRef &Section, // Out. diff --git a/include/llvm/MC/MCStreamer.h b/include/llvm/MC/MCStreamer.h index 6b9b8a153845..494f02dfad3e 100644 --- a/include/llvm/MC/MCStreamer.h +++ b/include/llvm/MC/MCStreamer.h @@ -19,6 +19,7 @@ #include "llvm/MC/MCDirectives.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCLinkerOptimizationHint.h" +#include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCWinEH.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/SMLoc.h" @@ -33,7 +34,6 @@ class MCInst; class MCInstPrinter; class MCSection; class MCStreamer; -class MCSymbol; class MCSymbolELF; class MCSymbolRefExpr; class MCSubtargetInfo; @@ -134,7 +134,7 @@ public: /// Callback used to implement the ldr= pseudo. /// Add a new entry to the constant pool for the current section and return an /// MCExpr that can be used to refer to the constant pool location. - const MCExpr *addConstantPoolEntry(const MCExpr *); + const MCExpr *addConstantPoolEntry(const MCExpr *, SMLoc Loc); /// Callback used to implemnt the .ltorg directive. /// Emit contents of constant pool for the current section. @@ -358,7 +358,7 @@ public: /// /// Each emitted symbol will be tracked in the ordering table, /// so we can sort on them later. - void AssignSection(MCSymbol *Symbol, MCSection *Section); + void AssignFragment(MCSymbol *Symbol, MCFragment *Fragment); /// \brief Emit a label for \p Symbol into the current section. /// @@ -522,10 +522,9 @@ public: /// match a native machine width. /// \param Loc - The location of the expression for error reporting. virtual void EmitValueImpl(const MCExpr *Value, unsigned Size, - const SMLoc &Loc = SMLoc()); + SMLoc Loc = SMLoc()); - void EmitValue(const MCExpr *Value, unsigned Size, - const SMLoc &Loc = SMLoc()); + void EmitValue(const MCExpr *Value, unsigned Size, SMLoc Loc = SMLoc()); /// \brief Special case of EmitValue that avoids the client having /// to pass in a MCExpr for constant integers. @@ -568,7 +567,7 @@ public: /// \brief Emit NumBytes worth of zeros. /// This function properly handles data in virtual sections. - virtual void EmitZeros(uint64_t NumBytes); + void EmitZeros(uint64_t NumBytes); /// \brief Emit some number of copies of \p Value until the byte alignment \p /// ByteAlignment is reached. @@ -612,9 +611,7 @@ public: /// \param Offset - The offset to reach. This may be an expression, but the /// expression must be associated with the current section. /// \param Value - The value to use when filling bytes. - /// \return false on success, true if the offset was invalid. - virtual bool EmitValueToOffset(const MCExpr *Offset, - unsigned char Value = 0); + virtual void emitValueToOffset(const MCExpr *Offset, unsigned char Value = 0); /// @} @@ -662,6 +659,7 @@ public: virtual void EmitCFIRelOffset(int64_t Register, int64_t Offset); virtual void EmitCFIAdjustCfaOffset(int64_t Adjustment); virtual void EmitCFIEscape(StringRef Values); + virtual void EmitCFIGnuArgsSize(int64_t Size); virtual void EmitCFISignalFrame(); virtual void EmitCFIUndefined(int64_t Register); virtual void EmitCFIRegister(int64_t Register1, int64_t Register2); @@ -682,6 +680,16 @@ public: virtual void EmitWinEHHandler(const MCSymbol *Sym, bool Unwind, bool Except); virtual void EmitWinEHHandlerData(); + virtual void EmitSyntaxDirective(); + + /// \brief Emit a .reloc directive. + /// Returns true if the relocation could not be emitted because Name is not + /// known. + virtual bool EmitRelocDirective(const MCExpr &Offset, StringRef Name, + const MCExpr *Expr, SMLoc Loc) { + return true; + } + /// \brief Emit the given \p Instruction into the current section. virtual void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI); @@ -704,9 +712,6 @@ public: /// the hasRawTextSupport() predicate. By default this aborts. void EmitRawText(const Twine &String); - /// \brief Causes any cached state to be written out. - virtual void Flush() {} - /// \brief Streamer specific finalization. virtual void FinishImpl(); /// \brief Finish emission of machine code. diff --git a/include/llvm/MC/MCSubtargetInfo.h b/include/llvm/MC/MCSubtargetInfo.h index d5ad4eebf9ef..446feefc4500 100644 --- a/include/llvm/MC/MCSubtargetInfo.h +++ b/include/llvm/MC/MCSubtargetInfo.h @@ -86,8 +86,9 @@ protected: void InitMCProcessorInfo(StringRef CPU, StringRef FS); public: - /// Set the features to the default for the given CPU. - void setDefaultFeatures(StringRef CPU); + /// Set the features to the default for the given CPU with an appended feature + /// string. + void setDefaultFeatures(StringRef CPU, StringRef FS); /// ToggleFeature - Toggle a feature and returns the re-computed feature /// bits. This version does not change the implied bits. @@ -159,11 +160,8 @@ public: /// Check whether the CPU string is valid. bool isCPUStringValid(StringRef CPU) const { - auto Found = std::find_if(ProcDesc.begin(), ProcDesc.end(), - [=](const SubtargetFeatureKV &KV) { - return CPU == KV.Key; - }); - return Found != ProcDesc.end(); + auto Found = std::lower_bound(ProcDesc.begin(), ProcDesc.end(), CPU); + return Found != ProcDesc.end() && StringRef(Found->Key) == CPU; } }; diff --git a/include/llvm/MC/MCSymbol.h b/include/llvm/MC/MCSymbol.h index b2910dfccd63..c51ecfcb0c5c 100644 --- a/include/llvm/MC/MCSymbol.h +++ b/include/llvm/MC/MCSymbol.h @@ -56,19 +56,17 @@ protected: SymContentsCommon, }; - // Special sentinal value for the absolute pseudo section. - // - // FIXME: Use a PointerInt wrapper for this? - static MCSection *AbsolutePseudoSection; + // Special sentinal value for the absolute pseudo fragment. + static MCFragment *AbsolutePseudoFragment; /// If a symbol has a Fragment, the section is implied, so we only need /// one pointer. + /// The special AbsolutePseudoFragment value is for absolute symbols. + /// If this is a variable symbol, this caches the variable value's fragment. /// FIXME: We might be able to simplify this by having the asm streamer create /// dummy fragments. /// If this is a section, then it gives the symbol is defined in. This is null - /// for undefined symbols, and the special AbsolutePseudoSection value for - /// absolute symbols. If this is a variable symbol, this caches the variable - /// value's section. + /// for undefined symbols. /// /// If this is a fragment, then it gives the fragment this symbol's value is /// relative to, if any. @@ -76,8 +74,7 @@ protected: /// For the 'HasName' integer, this is true if this symbol is named. /// A named symbol will have a pointer to the name allocated in the bytes /// immediately prior to the MCSymbol. - mutable PointerIntPair<PointerUnion<MCSection *, MCFragment *>, 1> - SectionOrFragmentAndHasName; + mutable PointerIntPair<MCFragment *, 1> FragmentAndHasName; /// IsTemporary - True if this is an assembler temporary label, which /// typically does not survive in the .o file's symbol table. Usually @@ -155,7 +152,7 @@ protected: // MCContext creates and uniques these. Kind(Kind), IsUsedInReloc(false), SymbolContents(SymContentsUnset), CommonAlignLog2(0), Flags(0) { Offset = 0; - SectionOrFragmentAndHasName.setInt(!!Name); + FragmentAndHasName.setInt(!!Name); if (Name) getNameEntryPtr() = Name; } @@ -179,20 +176,17 @@ private: MCSymbol(const MCSymbol &) = delete; void operator=(const MCSymbol &) = delete; - MCSection *getSectionPtr() const { - if (MCFragment *F = getFragment()) + MCSection *getSectionPtr(bool SetUsed = true) const { + if (MCFragment *F = getFragment(SetUsed)) { + assert(F != AbsolutePseudoFragment); return F->getParent(); - const auto &SectionOrFragment = SectionOrFragmentAndHasName.getPointer(); - assert(!SectionOrFragment.is<MCFragment *>() && "Section or null expected"); - MCSection *Section = SectionOrFragment.dyn_cast<MCSection *>(); - if (Section || !isVariable()) - return Section; - return Section = getVariableValue()->findAssociatedSection(); + } + return nullptr; } /// \brief Get a reference to the name field. Requires that we have a name const StringMapEntry<bool> *&getNameEntryPtr() { - assert(SectionOrFragmentAndHasName.getInt() && "Name is required"); + assert(FragmentAndHasName.getInt() && "Name is required"); NameEntryStorageTy *Name = reinterpret_cast<NameEntryStorageTy *>(this); return (*(Name - 1)).NameEntry; } @@ -203,7 +197,7 @@ private: public: /// getName - Get the symbol name. StringRef getName() const { - if (!SectionOrFragmentAndHasName.getInt()) + if (!FragmentAndHasName.getInt()) return StringRef(); return getNameEntryPtr()->first(); @@ -223,7 +217,7 @@ public: /// isUsed - Check if this is used. bool isUsed() const { return IsUsed; } - void setUsed(bool Value) const { IsUsed = Value; } + void setUsed(bool Value) const { IsUsed |= Value; } /// \brief Check if this symbol is redefinable. bool isRedefinable() const { return IsRedefinable; } @@ -248,37 +242,38 @@ public: /// isDefined - Check if this symbol is defined (i.e., it has an address). /// /// Defined symbols are either absolute or in some section. - bool isDefined() const { return getSectionPtr() != nullptr; } + bool isDefined(bool SetUsed = true) const { + return getFragment(SetUsed) != nullptr; + } /// isInSection - Check if this symbol is defined in some section (i.e., it /// is defined but not absolute). - bool isInSection() const { return isDefined() && !isAbsolute(); } + bool isInSection(bool SetUsed = true) const { + return isDefined(SetUsed) && !isAbsolute(SetUsed); + } /// isUndefined - Check if this symbol undefined (i.e., implicitly defined). - bool isUndefined() const { return !isDefined(); } + bool isUndefined(bool SetUsed = true) const { return !isDefined(SetUsed); } /// isAbsolute - Check if this is an absolute symbol. - bool isAbsolute() const { return getSectionPtr() == AbsolutePseudoSection; } + bool isAbsolute(bool SetUsed = true) const { + return getFragment(SetUsed) == AbsolutePseudoFragment; + } /// Get the section associated with a defined, non-absolute symbol. - MCSection &getSection() const { - assert(isInSection() && "Invalid accessor!"); - return *getSectionPtr(); + MCSection &getSection(bool SetUsed = true) const { + assert(isInSection(SetUsed) && "Invalid accessor!"); + return *getSectionPtr(SetUsed); } - /// Mark the symbol as defined in the section \p S. - void setSection(MCSection &S) { - assert(!isVariable() && "Cannot set section of variable"); - assert(!SectionOrFragmentAndHasName.getPointer().is<MCFragment *>() && - "Section or null expected"); - SectionOrFragmentAndHasName.setPointer(&S); + /// Mark the symbol as defined in the fragment \p F. + void setFragment(MCFragment *F) const { + assert(!isVariable() && "Cannot set fragment of variable"); + FragmentAndHasName.setPointer(F); } /// Mark the symbol as undefined. - void setUndefined() { - SectionOrFragmentAndHasName.setPointer( - PointerUnion<MCSection *, MCFragment *>()); - } + void setUndefined() { FragmentAndHasName.setPointer(nullptr); } bool isELF() const { return Kind == SymbolKindELF; } @@ -295,10 +290,10 @@ public: return SymbolContents == SymContentsVariable; } - /// getVariableValue() - Get the value for variable symbols. - const MCExpr *getVariableValue() const { + /// getVariableValue - Get the value for variable symbols. + const MCExpr *getVariableValue(bool SetUsed = true) const { assert(isVariable() && "Invalid accessor!"); - IsUsed = true; + IsUsed |= SetUsed; return Value; } @@ -379,11 +374,13 @@ public: return SymbolContents == SymContentsCommon; } - MCFragment *getFragment() const { - return SectionOrFragmentAndHasName.getPointer().dyn_cast<MCFragment *>(); - } - void setFragment(MCFragment *Value) const { - SectionOrFragmentAndHasName.setPointer(Value); + MCFragment *getFragment(bool SetUsed = true) const { + MCFragment *Fragment = FragmentAndHasName.getPointer(); + if (Fragment || !isVariable()) + return Fragment; + Fragment = getVariableValue(SetUsed)->findAssociatedFragment(); + FragmentAndHasName.setPointer(Fragment); + return Fragment; } bool isExternal() const { return IsExternal; } diff --git a/include/llvm/MC/MCTargetAsmParser.h b/include/llvm/MC/MCTargetAsmParser.h index 36db3914f017..03b2dc9a282c 100644 --- a/include/llvm/MC/MCTargetAsmParser.h +++ b/include/llvm/MC/MCTargetAsmParser.h @@ -20,6 +20,7 @@ class AsmToken; class MCInst; class MCParsedAsmOperand; class MCStreamer; +class MCSubtargetInfo; class SMLoc; class StringRef; template <typename T> class SmallVectorImpl; @@ -29,6 +30,7 @@ typedef SmallVectorImpl<std::unique_ptr<MCParsedAsmOperand>> OperandVector; enum AsmRewriteKind { AOK_Delete = 0, // Rewrite should be ignored. AOK_Align, // Rewrite align as .align. + AOK_EVEN, // Rewrite even as .even. AOK_DotOperator, // Rewrite a dot operator expression as an immediate. // E.g., [eax].foo.bar -> [eax].8 AOK_Emit, // Rewrite _emit as .byte. @@ -44,6 +46,7 @@ enum AsmRewriteKind { const char AsmRewritePrecedence [] = { 0, // AOK_Delete 2, // AOK_Align + 2, // AOK_EVEN 2, // AOK_DotOperator 2, // AOK_Emit 4, // AOK_Imm @@ -92,7 +95,10 @@ private: MCTargetAsmParser(const MCTargetAsmParser &) = delete; void operator=(const MCTargetAsmParser &) = delete; protected: // Can only create subclasses. - MCTargetAsmParser(); + MCTargetAsmParser(MCTargetOptions const &, const MCSubtargetInfo &STI); + + /// Create a copy of STI and return a non-const reference to it. + MCSubtargetInfo ©STI(); /// AvailableFeatures - The current set of available features. uint64_t AvailableFeatures; @@ -107,9 +113,14 @@ protected: // Can only create subclasses. /// Set of options which affects instrumentation of inline assembly. MCTargetOptions MCOptions; + /// Current STI. + const MCSubtargetInfo *STI; + public: ~MCTargetAsmParser() override; + const MCSubtargetInfo &getSTI() const; + uint64_t getAvailableFeatures() const { return AvailableFeatures; } void setAvailableFeatures(uint64_t Value) { AvailableFeatures = Value; } @@ -143,6 +154,10 @@ public: /// \return True on failure. virtual bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc, OperandVector &Operands) = 0; + virtual bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name, + AsmToken Token, OperandVector &Operands) { + return ParseInstruction(Info, Name, Token.getLoc(), Operands); + } /// ParseDirective - Parse a target specific assembler directive /// @@ -156,10 +171,6 @@ public: /// \param DirectiveID - the identifier token of the directive. virtual bool ParseDirective(AsmToken DirectiveID) = 0; - /// mnemonicIsValid - This returns true if this is a valid mnemonic and false - /// otherwise. - virtual bool mnemonicIsValid(StringRef Mnemonic, unsigned VariantID) = 0; - /// MatchAndEmitInstruction - Recognize a series of operands of a parsed /// instruction as an actual MCInst and emit it to the specified MCStreamer. /// This returns false on success and returns true on failure to match. @@ -192,13 +203,18 @@ public: virtual void convertToMapAndConstraints(unsigned Kind, const OperandVector &Operands) = 0; + // Return whether this parser uses assignment statements with equals tokens + virtual bool equalIsAsmAssignment() { return true; }; + // Return whether this start of statement identifier is a label + virtual bool isLabel(AsmToken &Token) { return true; }; + virtual const MCExpr *applyModifierToExpr(const MCExpr *E, MCSymbolRefExpr::VariantKind, MCContext &Ctx) { return nullptr; } - virtual void onLabelParsed(MCSymbol *Symbol) { }; + virtual void onLabelParsed(MCSymbol *Symbol) { } }; } // End llvm namespace diff --git a/include/llvm/MC/MCTargetOptions.h b/include/llvm/MC/MCTargetOptions.h index 7f4f23eda27f..4b66a750cb7d 100644 --- a/include/llvm/MC/MCTargetOptions.h +++ b/include/llvm/MC/MCTargetOptions.h @@ -29,8 +29,10 @@ public: bool MCRelaxAll : 1; bool MCNoExecStack : 1; bool MCFatalWarnings : 1; + bool MCNoWarn : 1; bool MCSaveTempLabels : 1; bool MCUseDwarfDirectory : 1; + bool MCIncrementalLinkerCompatible : 1; bool ShowMCEncoding : 1; bool ShowMCInst : 1; bool AsmVerbose : 1; @@ -49,8 +51,10 @@ inline bool operator==(const MCTargetOptions &LHS, const MCTargetOptions &RHS) { ARE_EQUAL(MCRelaxAll) && ARE_EQUAL(MCNoExecStack) && ARE_EQUAL(MCFatalWarnings) && + ARE_EQUAL(MCNoWarn) && ARE_EQUAL(MCSaveTempLabels) && ARE_EQUAL(MCUseDwarfDirectory) && + ARE_EQUAL(MCIncrementalLinkerCompatible) && ARE_EQUAL(ShowMCEncoding) && ARE_EQUAL(ShowMCInst) && ARE_EQUAL(AsmVerbose) && diff --git a/include/llvm/MC/MCTargetOptionsCommandFlags.h b/include/llvm/MC/MCTargetOptionsCommandFlags.h index af23a92e6e99..5180208d33b6 100644 --- a/include/llvm/MC/MCTargetOptionsCommandFlags.h +++ b/include/llvm/MC/MCTargetOptionsCommandFlags.h @@ -33,6 +33,12 @@ cl::opt<bool> RelaxAll("mc-relax-all", cl::desc("When used with filetype=obj, " "relax all fixups in the emitted object file")); +cl::opt<bool> IncrementalLinkerCompatible( + "incremental-linker-compatible", + cl::desc( + "When used with filetype=obj, " + "emit an object file which can be used with an incremental linker")); + cl::opt<int> DwarfVersion("dwarf-version", cl::desc("Dwarf version"), cl::init(0)); @@ -40,6 +46,12 @@ cl::opt<bool> ShowMCInst("asm-show-inst", cl::desc("Emit internal instruction representation to " "assembly file")); +cl::opt<bool> FatalWarnings("fatal-warnings", + cl::desc("Treat warnings as errors")); + +cl::opt<bool> NoWarn("no-warn", cl::desc("Suppress all warnings")); +cl::alias NoWarnW("W", cl::desc("Alias for --no-warn"), cl::aliasopt(NoWarn)); + cl::opt<std::string> ABIName("target-abi", cl::Hidden, cl::desc("The name of the ABI to be targeted from the backend."), @@ -50,9 +62,12 @@ static inline MCTargetOptions InitMCTargetOptionsFromFlags() { Options.SanitizeAddress = (AsmInstrumentation == MCTargetOptions::AsmInstrumentationAddress); Options.MCRelaxAll = RelaxAll; + Options.MCIncrementalLinkerCompatible = IncrementalLinkerCompatible; Options.DwarfVersion = DwarfVersion; Options.ShowMCInst = ShowMCInst; Options.ABIName = ABIName; + Options.MCFatalWarnings = FatalWarnings; + Options.MCNoWarn = NoWarn; return Options; } diff --git a/include/llvm/MC/MCValue.h b/include/llvm/MC/MCValue.h index 6bdf43685f21..ead08fd90ca0 100644 --- a/include/llvm/MC/MCValue.h +++ b/include/llvm/MC/MCValue.h @@ -35,10 +35,6 @@ class raw_ostream; /// relocation modifiers apply to the closest symbol or the whole /// expression. /// -/// In the general form, SymbolB can only be defined if SymbolA is, and both -/// must be in the same (non-external) section. The latter constraint is not -/// enforced, since a symbol's section may not be known at construction. -/// /// Note that this class must remain a simple POD value class, because we need /// it to live in unions etc. class MCValue { @@ -67,7 +63,6 @@ public: const MCSymbolRefExpr *SymB = nullptr, int64_t Val = 0, uint32_t RefKind = 0) { MCValue R; - assert((!SymB || SymA) && "Invalid relocatable MCValue!"); R.Cst = Val; R.SymA = SymA; R.SymB = SymB; diff --git a/include/llvm/MC/MCWinCOFFStreamer.h b/include/llvm/MC/MCWinCOFFStreamer.h index 6fbc754f1125..fe1ada9b9e5b 100644 --- a/include/llvm/MC/MCWinCOFFStreamer.h +++ b/include/llvm/MC/MCWinCOFFStreamer.h @@ -73,7 +73,7 @@ protected: void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo &STI) override; private: - LLVM_ATTRIBUTE_NORETURN void FatalError(const Twine &Msg) const; + void Error(const Twine &Msg) const; }; } diff --git a/include/llvm/MC/MachineLocation.h b/include/llvm/MC/MachineLocation.h index 2a18615eff62..4b5cf4357793 100644 --- a/include/llvm/MC/MachineLocation.h +++ b/include/llvm/MC/MachineLocation.h @@ -68,10 +68,6 @@ public: Register = R; Offset = O; } - -#ifndef NDEBUG - void dump(); -#endif }; inline bool operator!=(const MachineLocation &LHS, const MachineLocation &RHS) { diff --git a/include/llvm/MC/SectionKind.h b/include/llvm/MC/SectionKind.h index 9e8b68f4340c..b09b93cfc377 100644 --- a/include/llvm/MC/SectionKind.h +++ b/include/llvm/MC/SectionKind.h @@ -1,4 +1,4 @@ -//===-- llvm/Target/TargetLoweringObjectFile.h - Object Info ----*- C++ -*-===// +//===-- llvm/MC/SectionKind.h - Classification of sections ------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,11 +6,6 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -// -// This file implements classes used to handle lowerings specific to common -// object file formats. -// -//===----------------------------------------------------------------------===// #ifndef LLVM_MC_SECTIONKIND_H #define LLVM_MC_SECTIONKIND_H @@ -99,21 +94,8 @@ class SectionKind { /// marked 'constant'. Common, - /// DataRel - This is the most general form of data that is written - /// to by the program, it can have random relocations to arbitrary - /// globals. - DataRel, - - /// DataRelLocal - This is writeable data that has a non-zero - /// initializer and has relocations in it, but all of the - /// relocations are known to be within the final linked image - /// the global is linked into. - DataRelLocal, - - /// DataNoRel - This is writeable data that has a non-zero - /// initializer, but whose initializer is known to have no - /// relocations. - DataNoRel, + /// This is writeable data that has a non-zero initializer. + Data, /// ReadOnlyWithRel - These are global variables that are never /// written to by the program, but that have relocations, so they @@ -121,15 +103,7 @@ class SectionKind { /// can write to them. If it chooses to, the dynamic linker can /// mark the pages these globals end up on as read-only after it is /// done with its relocation phase. - ReadOnlyWithRel, - - /// ReadOnlyWithRelLocal - This is data that is readonly by the - /// program, but must be writeable so that the dynamic linker - /// can perform relocations in it. This is used when we know - /// that all the relocations are to globals in this final - /// linked image. - ReadOnlyWithRelLocal - + ReadOnlyWithRel } K : 8; public: @@ -169,7 +143,7 @@ public: bool isThreadData() const { return K == ThreadData; } bool isGlobalWriteableData() const { - return isBSS() || isCommon() || isDataRel() || isReadOnlyWithRel(); + return isBSS() || isCommon() || isData() || isReadOnlyWithRel(); } bool isBSS() const { return K == BSS || K == BSSLocal || K == BSSExtern; } @@ -178,22 +152,10 @@ public: bool isCommon() const { return K == Common; } - bool isDataRel() const { - return K == DataRel || K == DataRelLocal || K == DataNoRel; - } - - bool isDataRelLocal() const { - return K == DataRelLocal || K == DataNoRel; - } - - bool isDataNoRel() const { return K == DataNoRel; } + bool isData() const { return K == Data; } bool isReadOnlyWithRel() const { - return K == ReadOnlyWithRel || K == ReadOnlyWithRelLocal; - } - - bool isReadOnlyWithRelLocal() const { - return K == ReadOnlyWithRelLocal; + return K == ReadOnlyWithRel; } private: static SectionKind get(Kind K) { @@ -224,13 +186,8 @@ public: static SectionKind getBSSLocal() { return get(BSSLocal); } static SectionKind getBSSExtern() { return get(BSSExtern); } static SectionKind getCommon() { return get(Common); } - static SectionKind getDataRel() { return get(DataRel); } - static SectionKind getDataRelLocal() { return get(DataRelLocal); } - static SectionKind getDataNoRel() { return get(DataNoRel); } + static SectionKind getData() { return get(Data); } static SectionKind getReadOnlyWithRel() { return get(ReadOnlyWithRel); } - static SectionKind getReadOnlyWithRelLocal(){ - return get(ReadOnlyWithRelLocal); - } }; } // end namespace llvm diff --git a/include/llvm/MC/StringTableBuilder.h b/include/llvm/MC/StringTableBuilder.h index 897d449254ea..adde86b45583 100644 --- a/include/llvm/MC/StringTableBuilder.h +++ b/include/llvm/MC/StringTableBuilder.h @@ -11,53 +11,51 @@ #define LLVM_MC_STRINGTABLEBUILDER_H #include "llvm/ADT/SmallString.h" -#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/DenseMap.h" #include <cassert> namespace llvm { /// \brief Utility for building string tables with deduplicated suffixes. class StringTableBuilder { +public: + enum Kind { ELF, WinCOFF, MachO, RAW }; + +private: SmallString<256> StringTable; - StringMap<size_t> StringIndexMap; + DenseMap<StringRef, size_t> StringIndexMap; + size_t Size = 0; + Kind K; public: - /// \brief Add a string to the builder. Returns a StringRef to the internal - /// copy of s. Can only be used before the table is finalized. - StringRef add(StringRef s) { - assert(!isFinalized()); - return StringIndexMap.insert(std::make_pair(s, 0)).first->first(); - } + StringTableBuilder(Kind K); - enum Kind { - ELF, - WinCOFF, - MachO - }; + /// \brief Add a string to the builder. Returns the position of S in the + /// table. The position will be changed if finalize is used. + /// Can only be used before the table is finalized. + size_t add(StringRef S); /// \brief Analyze the strings and build the final table. No more strings can /// be added after this point. - void finalize(Kind kind); + void finalize(); /// \brief Retrieve the string table data. Can only be used after the table /// is finalized. - StringRef data() { + StringRef data() const { assert(isFinalized()); return StringTable; } /// \brief Get the offest of a string in the string table. Can only be used /// after the table is finalized. - size_t getOffset(StringRef s) { - assert(isFinalized()); - assert(StringIndexMap.count(s) && "String is not in table!"); - return StringIndexMap[s]; - } + size_t getOffset(StringRef S) const; + const DenseMap<StringRef, size_t> &getMap() const { return StringIndexMap; } + size_t getSize() const { return Size; } void clear(); private: - bool isFinalized() { + bool isFinalized() const { return !StringTable.empty(); } }; diff --git a/include/llvm/MC/SubtargetFeature.h b/include/llvm/MC/SubtargetFeature.h index 2fb9b4ae2503..0d97b226d728 100644 --- a/include/llvm/MC/SubtargetFeature.h +++ b/include/llvm/MC/SubtargetFeature.h @@ -30,7 +30,7 @@ namespace llvm { // A container class for subtarget features. // This is convenient because std::bitset does not have a constructor // with an initializer list of set bits. -const unsigned MAX_SUBTARGET_FEATURES = 64; +const unsigned MAX_SUBTARGET_FEATURES = 128; class FeatureBitset : public std::bitset<MAX_SUBTARGET_FEATURES> { public: // Cannot inherit constructors because it's not supported by VC++.. diff --git a/include/llvm/Object/Archive.h b/include/llvm/Object/Archive.h index 597f0d48c118..8dd042a2533f 100644 --- a/include/llvm/Object/Archive.h +++ b/include/llvm/Object/Archive.h @@ -37,7 +37,7 @@ struct ArchiveMemberHeader { llvm::StringRef getName() const; /// Members are not larger than 4GB. - uint32_t getSize() const; + ErrorOr<uint32_t> getSize() const; sys::fs::perms getAccessMode() const; sys::TimeValue getLastModified() const; @@ -52,6 +52,7 @@ class Archive : public Binary { virtual void anchor(); public: class Child { + friend Archive; const Archive *Parent; /// \brief Includes header but not padding byte. StringRef Data; @@ -62,19 +63,19 @@ public: return reinterpret_cast<const ArchiveMemberHeader *>(Data.data()); } + bool isThinMember() const; + public: - Child(const Archive *Parent, const char *Start); + Child(const Archive *Parent, const char *Start, std::error_code *EC); + Child(const Archive *Parent, StringRef Data, uint16_t StartOfFile); bool operator ==(const Child &other) const { assert(Parent == other.Parent); return Data.begin() == other.Data.begin(); } - bool operator <(const Child &other) const { - return Data.begin() < other.Data.begin(); - } - - Child getNext() const; + const Archive *getParent() const { return Parent; } + ErrorOr<Child> getNext() const; ErrorOr<StringRef> getName() const; StringRef getRawName() const { return getHeader()->getName(); } @@ -90,9 +91,9 @@ public: return getHeader()->getAccessMode(); } /// \return the size of the archive member without the header or padding. - uint64_t getSize() const; + ErrorOr<uint64_t> getSize() const; /// \return the size in the archive header for this member. - uint64_t getRawSize() const; + ErrorOr<uint64_t> getRawSize() const; ErrorOr<StringRef> getBuffer() const; uint64_t getChildOffset() const; @@ -104,28 +105,32 @@ public: }; class child_iterator { - Child child; + ErrorOr<Child> child; public: - child_iterator() : child(Child(nullptr, nullptr)) {} + child_iterator() : child(Child(nullptr, nullptr, nullptr)) {} child_iterator(const Child &c) : child(c) {} - const Child *operator->() const { return &child; } - const Child &operator*() const { return child; } + child_iterator(std::error_code EC) : child(EC) {} + const ErrorOr<Child> *operator->() const { return &child; } + const ErrorOr<Child> &operator*() const { return child; } bool operator==(const child_iterator &other) const { - return child == other.child; + // We ignore error states so that comparisions with end() work, which + // allows range loops. + if (child.getError() || other.child.getError()) + return false; + return *child == *other.child; } bool operator!=(const child_iterator &other) const { return !(*this == other); } - bool operator<(const child_iterator &other) const { - return child < other.child; - } - + // Code in loops with child_iterators must check for errors on each loop + // iteration. And if there is an error break out of the loop. child_iterator &operator++() { // Preincrement - child = child.getNext(); + assert(child && "Can't increment iterator with error"); + child = child->getNext(); return *this; } }; @@ -145,7 +150,7 @@ public: , SymbolIndex(symi) , StringIndex(stri) {} StringRef getName() const; - ErrorOr<child_iterator> getMember() const; + ErrorOr<Child> getMember() const; Symbol getNext() const; }; @@ -186,14 +191,13 @@ public: child_iterator child_begin(bool SkipInternal = true) const; child_iterator child_end() const; iterator_range<child_iterator> children(bool SkipInternal = true) const { - return iterator_range<child_iterator>(child_begin(SkipInternal), - child_end()); + return make_range(child_begin(SkipInternal), child_end()); } symbol_iterator symbol_begin() const; symbol_iterator symbol_end() const; iterator_range<symbol_iterator> symbols() const { - return iterator_range<symbol_iterator>(symbol_begin(), symbol_end()); + return make_range(symbol_begin(), symbol_end()); } // Cast methods. @@ -205,18 +209,17 @@ public: child_iterator findSym(StringRef name) const; bool hasSymbolTable() const; - child_iterator getSymbolTableChild() const { return SymbolTable; } - StringRef getSymbolTable() const { - // We know that the symbol table is not an external file, - // so we just assert there is no error. - return *SymbolTable->getBuffer(); - } + StringRef getSymbolTable() const { return SymbolTable; } uint32_t getNumberOfSymbols() const; private: - child_iterator SymbolTable; - child_iterator StringTable; - child_iterator FirstRegular; + StringRef SymbolTable; + StringRef StringTable; + + StringRef FirstRegularData; + uint16_t FirstRegularStartOfFile = -1; + void setFirstRegular(const Child &C); + unsigned Format : 2; unsigned IsThin : 1; mutable std::vector<std::unique_ptr<MemoryBuffer>> ThinBuffers; diff --git a/include/llvm/Object/ArchiveWriter.h b/include/llvm/Object/ArchiveWriter.h index 3648d0c77fb5..b5d2ba358080 100644 --- a/include/llvm/Object/ArchiveWriter.h +++ b/include/llvm/Object/ArchiveWriter.h @@ -24,17 +24,15 @@ class NewArchiveIterator { bool IsNewMember; StringRef Name; - object::Archive::child_iterator OldI; - - StringRef NewFilename; + object::Archive::Child OldMember; public: - NewArchiveIterator(object::Archive::child_iterator I, StringRef Name); - NewArchiveIterator(StringRef I, StringRef Name); + NewArchiveIterator(const object::Archive::Child &OldMember, StringRef Name); + NewArchiveIterator(StringRef FileName); bool isNewMember() const; StringRef getName() const; - object::Archive::child_iterator getOld() const; + const object::Archive::Child &getOld() const; StringRef getNew() const; llvm::ErrorOr<int> getFD(sys::fs::file_status &NewStatus) const; @@ -43,7 +41,8 @@ public: std::pair<StringRef, std::error_code> writeArchive(StringRef ArcName, std::vector<NewArchiveIterator> &NewMembers, - bool WriteSymtab, object::Archive::Kind Kind, bool Deterministic); + bool WriteSymtab, object::Archive::Kind Kind, bool Deterministic, + bool Thin); } #endif diff --git a/include/llvm/Object/Binary.h b/include/llvm/Object/Binary.h index a3d6d0d4d428..a0d1127781f6 100644 --- a/include/llvm/Object/Binary.h +++ b/include/llvm/Object/Binary.h @@ -41,7 +41,9 @@ protected: enum { ID_Archive, ID_MachOUniversalBinary, - ID_IR, // LLVM IR + ID_COFFImportFile, + ID_IR, // LLVM IR + ID_FunctionIndex, // Function summary index // Object and children. ID_StartObjects, @@ -113,10 +115,16 @@ public: return TypeID == ID_COFF; } + bool isCOFFImportFile() const { + return TypeID == ID_COFFImportFile; + } + bool isIR() const { return TypeID == ID_IR; } + bool isFunctionIndex() const { return TypeID == ID_FunctionIndex; } + bool isLittleEndian() const { return !(TypeID == ID_ELF32B || TypeID == ID_ELF64B || TypeID == ID_MachO32B || TypeID == ID_MachO64B); diff --git a/include/llvm/Object/COFF.h b/include/llvm/Object/COFF.h index 025a9dbc6bc0..1b0e2e36bd5e 100644 --- a/include/llvm/Object/COFF.h +++ b/include/llvm/Object/COFF.h @@ -653,8 +653,7 @@ protected: uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override; uint32_t getSymbolFlags(DataRefImpl Symb) const override; SymbolRef::Type getSymbolType(DataRefImpl Symb) const override; - std::error_code getSymbolSection(DataRefImpl Symb, - section_iterator &Res) const override; + ErrorOr<section_iterator> getSymbolSection(DataRefImpl Symb) const override; void moveSectionNext(DataRefImpl &Sec) const override; std::error_code getSectionName(DataRefImpl Sec, StringRef &Res) const override; @@ -774,6 +773,7 @@ public: std::error_code getSectionContents(const coff_section *Sec, ArrayRef<uint8_t> &Res) const; + uint64_t getImageBase() const; std::error_code getVaPtr(uint64_t VA, uintptr_t &Res) const; std::error_code getRvaPtr(uint32_t Rva, uintptr_t &Res) const; std::error_code getHintName(uint32_t Rva, uint16_t &Hint, diff --git a/include/llvm/Object/COFFImportFile.h b/include/llvm/Object/COFFImportFile.h new file mode 100644 index 000000000000..b04a44ea60d2 --- /dev/null +++ b/include/llvm/Object/COFFImportFile.h @@ -0,0 +1,74 @@ +//===- COFFImportFile.h - COFF short import file implementation -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// COFF short import file is a special kind of file which contains +// only symbol names for DLL-exported symbols. This class implements +// SymbolicFile interface for the file. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_COFF_IMPORT_FILE_H +#define LLVM_OBJECT_COFF_IMPORT_FILE_H + +#include "llvm/Object/COFF.h" +#include "llvm/Object/IRObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace object { + +class COFFImportFile : public SymbolicFile { +public: + COFFImportFile(MemoryBufferRef Source) + : SymbolicFile(ID_COFFImportFile, Source) {} + + static inline bool classof(Binary const *V) { return V->isCOFFImportFile(); } + + void moveSymbolNext(DataRefImpl &Symb) const override { ++Symb.p; } + + std::error_code printSymbolName(raw_ostream &OS, + DataRefImpl Symb) const override { + if (Symb.p == 0) + OS << "__imp_"; + OS << StringRef(Data.getBufferStart() + sizeof(coff_import_header)); + return std::error_code(); + } + + uint32_t getSymbolFlags(DataRefImpl Symb) const override { + return SymbolRef::SF_Global; + } + + basic_symbol_iterator symbol_begin_impl() const override { + return BasicSymbolRef(DataRefImpl(), this); + } + + basic_symbol_iterator symbol_end_impl() const override { + DataRefImpl Symb; + Symb.p = isCode() ? 2 : 1; + return BasicSymbolRef(Symb, this); + } + + const coff_import_header *getCOFFImportHeader() const { + return reinterpret_cast<const object::coff_import_header *>( + Data.getBufferStart()); + } + +private: + bool isCode() const { + return getCOFFImportHeader()->getType() == COFF::IMPORT_CODE; + } +}; + +} // namespace object +} // namespace llvm + +#endif diff --git a/include/llvm/Object/ELF.h b/include/llvm/Object/ELF.h index cc271851e6b0..b0eaa3f5ed4d 100644 --- a/include/llvm/Object/ELF.h +++ b/include/llvm/Object/ELF.h @@ -14,25 +14,9 @@ #ifndef LLVM_OBJECT_ELF_H #define LLVM_OBJECT_ELF_H -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/IntervalMap.h" -#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" #include "llvm/Object/ELFTypes.h" -#include "llvm/Object/Error.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/ELF.h" -#include "llvm/Support/Endian.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ErrorOr.h" #include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <limits> -#include <utility> namespace llvm { namespace object { @@ -56,78 +40,6 @@ public: typedef typename std::conditional<ELFT::Is64Bits, uint64_t, uint32_t>::type uintX_t; - /// \brief Iterate over constant sized entities. - template <class EntT> - class ELFEntityIterator { - public: - typedef ptrdiff_t difference_type; - typedef EntT value_type; - typedef std::forward_iterator_tag iterator_category; - typedef value_type &reference; - typedef value_type *pointer; - - /// \brief Default construct iterator. - ELFEntityIterator() : EntitySize(0), Current(nullptr) {} - ELFEntityIterator(uintX_t EntSize, const char *Start) - : EntitySize(EntSize), Current(Start) {} - - reference operator *() { - assert(Current && "Attempted to dereference an invalid iterator!"); - return *reinterpret_cast<pointer>(Current); - } - - pointer operator ->() { - assert(Current && "Attempted to dereference an invalid iterator!"); - return reinterpret_cast<pointer>(Current); - } - - bool operator ==(const ELFEntityIterator &Other) { - return Current == Other.Current; - } - - bool operator !=(const ELFEntityIterator &Other) { - return !(*this == Other); - } - - ELFEntityIterator &operator ++() { - assert(Current && "Attempted to increment an invalid iterator!"); - Current += EntitySize; - return *this; - } - - ELFEntityIterator &operator+(difference_type n) { - assert(Current && "Attempted to increment an invalid iterator!"); - Current += (n * EntitySize); - return *this; - } - - ELFEntityIterator &operator-(difference_type n) { - assert(Current && "Attempted to subtract an invalid iterator!"); - Current -= (n * EntitySize); - return *this; - } - - ELFEntityIterator operator ++(int) { - ELFEntityIterator Tmp = *this; - ++*this; - return Tmp; - } - - difference_type operator -(const ELFEntityIterator &Other) const { - assert(EntitySize == Other.EntitySize && - "Subtracting iterators of different EntitySize!"); - return (Current - Other.Current) / EntitySize; - } - - const char *get() const { return Current; } - - uintX_t getEntSize() const { return EntitySize; } - - private: - uintX_t EntitySize; - const char *Current; - }; - typedef Elf_Ehdr_Impl<ELFT> Elf_Ehdr; typedef Elf_Shdr_Impl<ELFT> Elf_Shdr; typedef Elf_Sym_Impl<ELFT> Elf_Sym; @@ -141,98 +53,22 @@ public: typedef Elf_Vernaux_Impl<ELFT> Elf_Vernaux; typedef Elf_Versym_Impl<ELFT> Elf_Versym; typedef Elf_Hash_Impl<ELFT> Elf_Hash; - typedef ELFEntityIterator<const Elf_Dyn> Elf_Dyn_Iter; - typedef iterator_range<Elf_Dyn_Iter> Elf_Dyn_Range; - typedef ELFEntityIterator<const Elf_Rela> Elf_Rela_Iter; - typedef ELFEntityIterator<const Elf_Rel> Elf_Rel_Iter; + typedef Elf_GnuHash_Impl<ELFT> Elf_GnuHash; + typedef iterator_range<const Elf_Dyn *> Elf_Dyn_Range; typedef iterator_range<const Elf_Shdr *> Elf_Shdr_Range; - - /// \brief Archive files are 2 byte aligned, so we need this for - /// PointerIntPair to work. - template <typename T> - class ArchivePointerTypeTraits { - public: - static inline const void *getAsVoidPointer(T *P) { return P; } - static inline T *getFromVoidPointer(const void *P) { - return static_cast<T *>(P); - } - enum { NumLowBitsAvailable = 1 }; - }; - typedef iterator_range<const Elf_Sym *> Elf_Sym_Range; -private: - typedef SmallVector<const Elf_Shdr *, 2> Sections_t; - typedef DenseMap<unsigned, unsigned> IndexMap_t; - - StringRef Buf; - const uint8_t *base() const { return reinterpret_cast<const uint8_t *>(Buf.data()); } +private: + + StringRef Buf; + const Elf_Ehdr *Header; const Elf_Shdr *SectionHeaderTable = nullptr; StringRef DotShstrtab; // Section header string table. - StringRef DotStrtab; // Symbol header string table. - const Elf_Shdr *dot_symtab_sec = nullptr; // Symbol table section. - const Elf_Shdr *DotDynSymSec = nullptr; // Dynamic symbol table section. - const Elf_Hash *HashTable = nullptr; - - const Elf_Shdr *SymbolTableSectionHeaderIndex = nullptr; - DenseMap<const Elf_Sym *, ELF::Elf64_Word> ExtendedSymbolTable; - - const Elf_Shdr *dot_gnu_version_sec = nullptr; // .gnu.version - const Elf_Shdr *dot_gnu_version_r_sec = nullptr; // .gnu.version_r - const Elf_Shdr *dot_gnu_version_d_sec = nullptr; // .gnu.version_d - - /// \brief Represents a region described by entries in the .dynamic table. - struct DynRegionInfo { - DynRegionInfo() : Addr(nullptr), Size(0), EntSize(0) {} - /// \brief Address in current address space. - const void *Addr; - /// \brief Size in bytes of the region. - uintX_t Size; - /// \brief Size of each entity in the region. - uintX_t EntSize; - }; - - DynRegionInfo DynamicRegion; - DynRegionInfo DynHashRegion; - DynRegionInfo DynStrRegion; - DynRegionInfo DynRelaRegion; - - // Pointer to SONAME entry in dynamic string table - // This is set the first time getLoadName is called. - mutable const char *dt_soname = nullptr; - - // Records for each version index the corresponding Verdef or Vernaux entry. - // This is filled the first time LoadVersionMap() is called. - class VersionMapEntry : public PointerIntPair<const void*, 1> { - public: - // If the integer is 0, this is an Elf_Verdef*. - // If the integer is 1, this is an Elf_Vernaux*. - VersionMapEntry() : PointerIntPair<const void*, 1>(nullptr, 0) { } - VersionMapEntry(const Elf_Verdef *verdef) - : PointerIntPair<const void*, 1>(verdef, 0) { } - VersionMapEntry(const Elf_Vernaux *vernaux) - : PointerIntPair<const void*, 1>(vernaux, 1) { } - bool isNull() const { return getPointer() == nullptr; } - bool isVerdef() const { return !isNull() && getInt() == 0; } - bool isVernaux() const { return !isNull() && getInt() == 1; } - const Elf_Verdef *getVerdef() const { - return isVerdef() ? (const Elf_Verdef*)getPointer() : nullptr; - } - const Elf_Vernaux *getVernaux() const { - return isVernaux() ? (const Elf_Vernaux*)getPointer() : nullptr; - } - }; - mutable SmallVector<VersionMapEntry, 16> VersionMap; - void LoadVersionDefs(const Elf_Shdr *sec) const; - void LoadVersionNeeds(const Elf_Shdr *ec) const; - void LoadVersionMap() const; - - void scanDynamicTable(); public: template<typename T> @@ -240,25 +76,20 @@ public: template <typename T> const T *getEntry(const Elf_Shdr *Section, uint32_t Entry) const; - const Elf_Shdr *getDotSymtabSec() const { return dot_symtab_sec; } - const Elf_Shdr *getDotDynSymSec() const { return DotDynSymSec; } - const Elf_Hash *getHashTable() const { return HashTable; } - ErrorOr<StringRef> getStringTable(const Elf_Shdr *Section) const; - const char *getDynamicString(uintX_t Offset) const; - ErrorOr<StringRef> getSymbolVersion(const Elf_Shdr *section, - const Elf_Sym *Symb, - bool &IsDefault) const; + ErrorOr<StringRef> getStringTableForSymtab(const Elf_Shdr &Section) const; + + ErrorOr<ArrayRef<Elf_Word>> getSHNDXTable(const Elf_Shdr &Section) const; + void VerifyStrTab(const Elf_Shdr *sh) const; StringRef getRelocationTypeName(uint32_t Type) const; void getRelocationTypeName(uint32_t Type, SmallVectorImpl<char> &Result) const; - /// \brief Get the symbol table section and symbol for a given relocation. - template <class RelT> - std::pair<const Elf_Shdr *, const Elf_Sym *> - getRelocationSymbol(const Elf_Shdr *RelSec, const RelT *Rel) const; + /// \brief Get the symbol for a given relocation. + const Elf_Sym *getRelocationSymbol(const Elf_Rel *Rel, + const Elf_Shdr *SymTab) const; ELFFile(StringRef Object, std::error_code &EC); @@ -273,111 +104,116 @@ public: Header->getDataEncoding() == ELF::ELFDATA2LSB; } + ErrorOr<const Elf_Dyn *> dynamic_table_begin(const Elf_Phdr *Phdr) const; + ErrorOr<const Elf_Dyn *> dynamic_table_end(const Elf_Phdr *Phdr) const; + ErrorOr<Elf_Dyn_Range> dynamic_table(const Elf_Phdr *Phdr) const { + ErrorOr<const Elf_Dyn *> Begin = dynamic_table_begin(Phdr); + if (std::error_code EC = Begin.getError()) + return EC; + ErrorOr<const Elf_Dyn *> End = dynamic_table_end(Phdr); + if (std::error_code EC = End.getError()) + return EC; + return make_range(*Begin, *End); + } + const Elf_Shdr *section_begin() const; const Elf_Shdr *section_end() const; Elf_Shdr_Range sections() const { return make_range(section_begin(), section_end()); } - const Elf_Sym *symbol_begin() const; - const Elf_Sym *symbol_end() const; - Elf_Sym_Range symbols() const { - return make_range(symbol_begin(), symbol_end()); - } - - Elf_Dyn_Iter dynamic_table_begin() const; - /// \param NULLEnd use one past the first DT_NULL entry as the end instead of - /// the section size. - Elf_Dyn_Iter dynamic_table_end(bool NULLEnd = false) const; - Elf_Dyn_Range dynamic_table(bool NULLEnd = false) const { - return make_range(dynamic_table_begin(), dynamic_table_end(NULLEnd)); - } - - const Elf_Sym *dynamic_symbol_begin() const { - if (!DotDynSymSec) + const Elf_Sym *symbol_begin(const Elf_Shdr *Sec) const { + if (!Sec) return nullptr; - if (DotDynSymSec->sh_entsize != sizeof(Elf_Sym)) + if (Sec->sh_entsize != sizeof(Elf_Sym)) report_fatal_error("Invalid symbol size"); - return reinterpret_cast<const Elf_Sym *>(base() + DotDynSymSec->sh_offset); + return reinterpret_cast<const Elf_Sym *>(base() + Sec->sh_offset); } - - const Elf_Sym *dynamic_symbol_end() const { - if (!DotDynSymSec) + const Elf_Sym *symbol_end(const Elf_Shdr *Sec) const { + if (!Sec) return nullptr; - return reinterpret_cast<const Elf_Sym *>(base() + DotDynSymSec->sh_offset + - DotDynSymSec->sh_size); + uint64_t Size = Sec->sh_size; + if (Size % sizeof(Elf_Sym)) + report_fatal_error("Invalid symbol table size"); + return symbol_begin(Sec) + Size / sizeof(Elf_Sym); } - - Elf_Sym_Range dynamic_symbols() const { - return make_range(dynamic_symbol_begin(), dynamic_symbol_end()); + Elf_Sym_Range symbols(const Elf_Shdr *Sec) const { + return make_range(symbol_begin(Sec), symbol_end(Sec)); } - Elf_Rela_Iter dyn_rela_begin() const { - if (DynRelaRegion.Addr) - return Elf_Rela_Iter(DynRelaRegion.EntSize, - (const char *)DynRelaRegion.Addr); - return Elf_Rela_Iter(0, nullptr); + typedef iterator_range<const Elf_Rela *> Elf_Rela_Range; + + const Elf_Rela *rela_begin(const Elf_Shdr *sec) const { + if (sec->sh_entsize != sizeof(Elf_Rela)) + report_fatal_error("Invalid relocation entry size"); + return reinterpret_cast<const Elf_Rela *>(base() + sec->sh_offset); } - Elf_Rela_Iter dyn_rela_end() const { - if (DynRelaRegion.Addr) - return Elf_Rela_Iter( - DynRelaRegion.EntSize, - (const char *)DynRelaRegion.Addr + DynRelaRegion.Size); - return Elf_Rela_Iter(0, nullptr); + const Elf_Rela *rela_end(const Elf_Shdr *sec) const { + uint64_t Size = sec->sh_size; + if (Size % sizeof(Elf_Rela)) + report_fatal_error("Invalid relocation table size"); + return rela_begin(sec) + Size / sizeof(Elf_Rela); } - Elf_Rela_Iter rela_begin(const Elf_Shdr *sec) const { - return Elf_Rela_Iter(sec->sh_entsize, - (const char *)(base() + sec->sh_offset)); + Elf_Rela_Range relas(const Elf_Shdr *Sec) const { + return make_range(rela_begin(Sec), rela_end(Sec)); } - Elf_Rela_Iter rela_end(const Elf_Shdr *sec) const { - return Elf_Rela_Iter( - sec->sh_entsize, - (const char *)(base() + sec->sh_offset + sec->sh_size)); + const Elf_Rel *rel_begin(const Elf_Shdr *sec) const { + if (sec->sh_entsize != sizeof(Elf_Rel)) + report_fatal_error("Invalid relocation entry size"); + return reinterpret_cast<const Elf_Rel *>(base() + sec->sh_offset); } - Elf_Rel_Iter rel_begin(const Elf_Shdr *sec) const { - return Elf_Rel_Iter(sec->sh_entsize, - (const char *)(base() + sec->sh_offset)); + const Elf_Rel *rel_end(const Elf_Shdr *sec) const { + uint64_t Size = sec->sh_size; + if (Size % sizeof(Elf_Rel)) + report_fatal_error("Invalid relocation table size"); + return rel_begin(sec) + Size / sizeof(Elf_Rel); } - Elf_Rel_Iter rel_end(const Elf_Shdr *sec) const { - return Elf_Rel_Iter(sec->sh_entsize, - (const char *)(base() + sec->sh_offset + sec->sh_size)); + typedef iterator_range<const Elf_Rel *> Elf_Rel_Range; + Elf_Rel_Range rels(const Elf_Shdr *Sec) const { + return make_range(rel_begin(Sec), rel_end(Sec)); } /// \brief Iterate over program header table. - typedef ELFEntityIterator<const Elf_Phdr> Elf_Phdr_Iter; + const Elf_Phdr *program_header_begin() const { + if (Header->e_phnum && Header->e_phentsize != sizeof(Elf_Phdr)) + report_fatal_error("Invalid program header size"); + return reinterpret_cast<const Elf_Phdr *>(base() + Header->e_phoff); + } - Elf_Phdr_Iter program_header_begin() const { - return Elf_Phdr_Iter(Header->e_phentsize, - (const char*)base() + Header->e_phoff); + const Elf_Phdr *program_header_end() const { + return program_header_begin() + Header->e_phnum; } - Elf_Phdr_Iter program_header_end() const { - return Elf_Phdr_Iter(Header->e_phentsize, - (const char*)base() + - Header->e_phoff + - (Header->e_phnum * Header->e_phentsize)); + typedef iterator_range<const Elf_Phdr *> Elf_Phdr_Range; + + const Elf_Phdr_Range program_headers() const { + return make_range(program_header_begin(), program_header_end()); } uint64_t getNumSections() const; uintX_t getStringTableIndex() const; - ELF::Elf64_Word getExtendedSymbolTableIndex(const Elf_Sym *symb) const; + uint32_t getExtendedSymbolTableIndex(const Elf_Sym *Sym, + const Elf_Shdr *SymTab, + ArrayRef<Elf_Word> ShndxTable) const; const Elf_Ehdr *getHeader() const { return Header; } - ErrorOr<const Elf_Shdr *> getSection(const Elf_Sym *symb) const; + ErrorOr<const Elf_Shdr *> getSection(const Elf_Sym *Sym, + const Elf_Shdr *SymTab, + ArrayRef<Elf_Word> ShndxTable) const; ErrorOr<const Elf_Shdr *> getSection(uint32_t Index) const; - const Elf_Sym *getSymbol(uint32_t index) const; - ErrorOr<StringRef> getStaticSymbolName(const Elf_Sym *Symb) const; - ErrorOr<StringRef> getDynamicSymbolName(const Elf_Sym *Symb) const; - ErrorOr<StringRef> getSymbolName(const Elf_Sym *Symb, bool IsDynamic) const; + const Elf_Sym *getSymbol(const Elf_Shdr *Sec, uint32_t Index) const { + return &*(symbol_begin(Sec) + Index); + } ErrorOr<StringRef> getSectionName(const Elf_Shdr *Section) const; + template <typename T> + ErrorOr<ArrayRef<T>> getSectionContentsAsArray(const Elf_Shdr *Sec) const; ErrorOr<ArrayRef<uint8_t> > getSectionContents(const Elf_Shdr *Sec) const; - StringRef getLoadName() const; }; typedef ELFFile<ELFType<support::little, false>> ELF32LEFile; @@ -385,118 +221,50 @@ typedef ELFFile<ELFType<support::little, true>> ELF64LEFile; typedef ELFFile<ELFType<support::big, false>> ELF32BEFile; typedef ELFFile<ELFType<support::big, true>> ELF64BEFile; -// Iterate through the version definitions, and place each Elf_Verdef -// in the VersionMap according to its index. template <class ELFT> -void ELFFile<ELFT>::LoadVersionDefs(const Elf_Shdr *sec) const { - unsigned vd_size = sec->sh_size; // Size of section in bytes - unsigned vd_count = sec->sh_info; // Number of Verdef entries - const char *sec_start = (const char*)base() + sec->sh_offset; - const char *sec_end = sec_start + vd_size; - // The first Verdef entry is at the start of the section. - const char *p = sec_start; - for (unsigned i = 0; i < vd_count; i++) { - if (p + sizeof(Elf_Verdef) > sec_end) - report_fatal_error("Section ended unexpectedly while scanning " - "version definitions."); - const Elf_Verdef *vd = reinterpret_cast<const Elf_Verdef *>(p); - if (vd->vd_version != ELF::VER_DEF_CURRENT) - report_fatal_error("Unexpected verdef version"); - size_t index = vd->vd_ndx & ELF::VERSYM_VERSION; - if (index >= VersionMap.size()) - VersionMap.resize(index + 1); - VersionMap[index] = VersionMapEntry(vd); - p += vd->vd_next; - } -} +uint32_t ELFFile<ELFT>::getExtendedSymbolTableIndex( + const Elf_Sym *Sym, const Elf_Shdr *SymTab, + ArrayRef<Elf_Word> ShndxTable) const { + assert(Sym->st_shndx == ELF::SHN_XINDEX); + unsigned Index = Sym - symbol_begin(SymTab); -// Iterate through the versions needed section, and place each Elf_Vernaux -// in the VersionMap according to its index. -template <class ELFT> -void ELFFile<ELFT>::LoadVersionNeeds(const Elf_Shdr *sec) const { - unsigned vn_size = sec->sh_size; // Size of section in bytes - unsigned vn_count = sec->sh_info; // Number of Verneed entries - const char *sec_start = (const char *)base() + sec->sh_offset; - const char *sec_end = sec_start + vn_size; - // The first Verneed entry is at the start of the section. - const char *p = sec_start; - for (unsigned i = 0; i < vn_count; i++) { - if (p + sizeof(Elf_Verneed) > sec_end) - report_fatal_error("Section ended unexpectedly while scanning " - "version needed records."); - const Elf_Verneed *vn = reinterpret_cast<const Elf_Verneed *>(p); - if (vn->vn_version != ELF::VER_NEED_CURRENT) - report_fatal_error("Unexpected verneed version"); - // Iterate through the Vernaux entries - const char *paux = p + vn->vn_aux; - for (unsigned j = 0; j < vn->vn_cnt; j++) { - if (paux + sizeof(Elf_Vernaux) > sec_end) - report_fatal_error("Section ended unexpected while scanning auxiliary " - "version needed records."); - const Elf_Vernaux *vna = reinterpret_cast<const Elf_Vernaux *>(paux); - size_t index = vna->vna_other & ELF::VERSYM_VERSION; - if (index >= VersionMap.size()) - VersionMap.resize(index + 1); - VersionMap[index] = VersionMapEntry(vna); - paux += vna->vna_next; - } - p += vn->vn_next; - } -} - -template <class ELFT> -void ELFFile<ELFT>::LoadVersionMap() const { - // If there is no dynamic symtab or version table, there is nothing to do. - if (!DotDynSymSec || !dot_gnu_version_sec) - return; - - // Has the VersionMap already been loaded? - if (VersionMap.size() > 0) - return; - - // The first two version indexes are reserved. - // Index 0 is LOCAL, index 1 is GLOBAL. - VersionMap.push_back(VersionMapEntry()); - VersionMap.push_back(VersionMapEntry()); - - if (dot_gnu_version_d_sec) - LoadVersionDefs(dot_gnu_version_d_sec); - - if (dot_gnu_version_r_sec) - LoadVersionNeeds(dot_gnu_version_r_sec); -} - -template <class ELFT> -ELF::Elf64_Word -ELFFile<ELFT>::getExtendedSymbolTableIndex(const Elf_Sym *symb) const { - assert(symb->st_shndx == ELF::SHN_XINDEX); - return ExtendedSymbolTable.lookup(symb); + // The size of the table was checked in getSHNDXTable. + return ShndxTable[Index]; } template <class ELFT> ErrorOr<const typename ELFFile<ELFT>::Elf_Shdr *> -ELFFile<ELFT>::getSection(const Elf_Sym *symb) const { - uint32_t Index = symb->st_shndx; +ELFFile<ELFT>::getSection(const Elf_Sym *Sym, const Elf_Shdr *SymTab, + ArrayRef<Elf_Word> ShndxTable) const { + uint32_t Index = Sym->st_shndx; if (Index == ELF::SHN_XINDEX) - return getSection(ExtendedSymbolTable.lookup(symb)); + return getSection(getExtendedSymbolTableIndex(Sym, SymTab, ShndxTable)); + if (Index == ELF::SHN_UNDEF || Index >= ELF::SHN_LORESERVE) return nullptr; - return getSection(symb->st_shndx); + return getSection(Sym->st_shndx); } template <class ELFT> -const typename ELFFile<ELFT>::Elf_Sym * -ELFFile<ELFT>::getSymbol(uint32_t Index) const { - return &*(symbol_begin() + Index); +template <typename T> +ErrorOr<ArrayRef<T>> +ELFFile<ELFT>::getSectionContentsAsArray(const Elf_Shdr *Sec) const { + uintX_t Offset = Sec->sh_offset; + uintX_t Size = Sec->sh_size; + + if (Size % sizeof(T)) + return object_error::parse_failed; + if (Offset + Size > Buf.size()) + return object_error::parse_failed; + + const T *Start = reinterpret_cast<const T *>(base() + Offset); + return makeArrayRef(Start, Size / sizeof(T)); } template <class ELFT> -ErrorOr<ArrayRef<uint8_t> > +ErrorOr<ArrayRef<uint8_t>> ELFFile<ELFT>::getSectionContents(const Elf_Shdr *Sec) const { - if (Sec->sh_offset + Sec->sh_size > Buf.size()) - return object_error::parse_failed; - const uint8_t *Start = base() + Sec->sh_offset; - return makeArrayRef(Start, Sec->sh_size); + return getSectionContentsAsArray<uint8_t>(Sec); } template <class ELFT> @@ -536,18 +304,13 @@ void ELFFile<ELFT>::getRelocationTypeName(uint32_t Type, } template <class ELFT> -template <class RelT> -std::pair<const typename ELFFile<ELFT>::Elf_Shdr *, - const typename ELFFile<ELFT>::Elf_Sym *> -ELFFile<ELFT>::getRelocationSymbol(const Elf_Shdr *Sec, const RelT *Rel) const { - if (!Sec->sh_link) - return std::make_pair(nullptr, nullptr); - ErrorOr<const Elf_Shdr *> SymTableOrErr = getSection(Sec->sh_link); - if (std::error_code EC = SymTableOrErr.getError()) - report_fatal_error(EC.message()); - const Elf_Shdr *SymTable = *SymTableOrErr; - return std::make_pair( - SymTable, getEntry<Elf_Sym>(SymTable, Rel->getSymbol(isMips64EL()))); +const typename ELFFile<ELFT>::Elf_Sym * +ELFFile<ELFT>::getRelocationSymbol(const Elf_Rel *Rel, + const Elf_Shdr *SymTab) const { + uint32_t Index = Rel->getSymbol(isMips64EL()); + if (Index == 0) + return nullptr; + return getEntry<Elf_Sym>(SymTab, Index); } template <class ELFT> @@ -584,10 +347,8 @@ ELFFile<ELFT>::ELFFile(StringRef Object, std::error_code &EC) Header = reinterpret_cast<const Elf_Ehdr *>(base()); - if (Header->e_shoff == 0) { - scanDynamicTable(); + if (Header->e_shoff == 0) return; - } const uint64_t SectionTableOffset = Header->e_shoff; @@ -608,185 +369,25 @@ ELFFile<ELFT>::ELFFile(StringRef Object, std::error_code &EC) return; } - // Scan sections for special sections. - - for (const Elf_Shdr &Sec : sections()) { - switch (Sec.sh_type) { - case ELF::SHT_HASH: - if (HashTable) { - EC = object_error::parse_failed; - return; - } - HashTable = reinterpret_cast<const Elf_Hash *>(base() + Sec.sh_offset); - break; - case ELF::SHT_SYMTAB_SHNDX: - if (SymbolTableSectionHeaderIndex) { - // More than one .symtab_shndx! - EC = object_error::parse_failed; - return; - } - SymbolTableSectionHeaderIndex = &Sec; - break; - case ELF::SHT_SYMTAB: { - if (dot_symtab_sec) { - // More than one .symtab! - EC = object_error::parse_failed; - return; - } - dot_symtab_sec = &Sec; - ErrorOr<const Elf_Shdr *> SectionOrErr = getSection(Sec.sh_link); - if ((EC = SectionOrErr.getError())) - return; - ErrorOr<StringRef> SymtabOrErr = getStringTable(*SectionOrErr); - if ((EC = SymtabOrErr.getError())) - return; - DotStrtab = *SymtabOrErr; - } break; - case ELF::SHT_DYNSYM: { - if (DotDynSymSec) { - // More than one .dynsym! - EC = object_error::parse_failed; - return; - } - DotDynSymSec = &Sec; - ErrorOr<const Elf_Shdr *> SectionOrErr = getSection(Sec.sh_link); - if ((EC = SectionOrErr.getError())) - return; - ErrorOr<StringRef> SymtabOrErr = getStringTable(*SectionOrErr); - if ((EC = SymtabOrErr.getError())) - return; - DynStrRegion.Addr = SymtabOrErr->data(); - DynStrRegion.Size = SymtabOrErr->size(); - DynStrRegion.EntSize = 1; - break; - } - case ELF::SHT_DYNAMIC: - if (DynamicRegion.Addr) { - // More than one .dynamic! - EC = object_error::parse_failed; - return; - } - DynamicRegion.Addr = base() + Sec.sh_offset; - DynamicRegion.Size = Sec.sh_size; - DynamicRegion.EntSize = Sec.sh_entsize; - break; - case ELF::SHT_GNU_versym: - if (dot_gnu_version_sec != nullptr) { - // More than one .gnu.version section! - EC = object_error::parse_failed; - return; - } - dot_gnu_version_sec = &Sec; - break; - case ELF::SHT_GNU_verdef: - if (dot_gnu_version_d_sec != nullptr) { - // More than one .gnu.version_d section! - EC = object_error::parse_failed; - return; - } - dot_gnu_version_d_sec = &Sec; - break; - case ELF::SHT_GNU_verneed: - if (dot_gnu_version_r_sec != nullptr) { - // More than one .gnu.version_r section! - EC = object_error::parse_failed; - return; - } - dot_gnu_version_r_sec = &Sec; - break; - } - } - // Get string table sections. - ErrorOr<const Elf_Shdr *> StrTabSecOrErr = getSection(getStringTableIndex()); - if ((EC = StrTabSecOrErr.getError())) - return; + uintX_t StringTableIndex = getStringTableIndex(); + if (StringTableIndex) { + ErrorOr<const Elf_Shdr *> StrTabSecOrErr = getSection(StringTableIndex); + if ((EC = StrTabSecOrErr.getError())) + return; - ErrorOr<StringRef> SymtabOrErr = getStringTable(*StrTabSecOrErr); - if ((EC = SymtabOrErr.getError())) - return; - DotShstrtab = *SymtabOrErr; - - // Build symbol name side-mapping if there is one. - if (SymbolTableSectionHeaderIndex) { - const Elf_Word *ShndxTable = reinterpret_cast<const Elf_Word*>(base() + - SymbolTableSectionHeaderIndex->sh_offset); - for (const Elf_Sym &S : symbols()) { - if (*ShndxTable != ELF::SHN_UNDEF) - ExtendedSymbolTable[&S] = *ShndxTable; - ++ShndxTable; - } + ErrorOr<StringRef> StringTableOrErr = getStringTable(*StrTabSecOrErr); + if ((EC = StringTableOrErr.getError())) + return; + DotShstrtab = *StringTableOrErr; } - scanDynamicTable(); - EC = std::error_code(); } template <class ELFT> -void ELFFile<ELFT>::scanDynamicTable() { - // Build load-address to file-offset map. - typedef IntervalMap< - uintX_t, uintptr_t, - IntervalMapImpl::NodeSizer<uintX_t, uintptr_t>::LeafSize, - IntervalMapHalfOpenInfo<uintX_t>> LoadMapT; - typename LoadMapT::Allocator Alloc; - // Allocate the IntervalMap on the heap to work around MSVC bug where the - // stack doesn't get realigned despite LoadMap having alignment 8 (PR24113). - std::unique_ptr<LoadMapT> LoadMap(new LoadMapT(Alloc)); - - for (Elf_Phdr_Iter PhdrI = program_header_begin(), - PhdrE = program_header_end(); - PhdrI != PhdrE; ++PhdrI) { - if (PhdrI->p_type == ELF::PT_DYNAMIC) { - DynamicRegion.Addr = base() + PhdrI->p_offset; - DynamicRegion.Size = PhdrI->p_filesz; - DynamicRegion.EntSize = sizeof(Elf_Dyn); - continue; - } - if (PhdrI->p_type != ELF::PT_LOAD) - continue; - if (PhdrI->p_filesz == 0) - continue; - LoadMap->insert(PhdrI->p_vaddr, PhdrI->p_vaddr + PhdrI->p_filesz, - PhdrI->p_offset); - } - - auto toMappedAddr = [&](uint64_t VAddr) -> const uint8_t * { - auto I = LoadMap->find(VAddr); - if (I == LoadMap->end()) - return nullptr; - return this->base() + I.value() + (VAddr - I.start()); - }; - - for (Elf_Dyn_Iter DynI = dynamic_table_begin(), DynE = dynamic_table_end(); - DynI != DynE; ++DynI) { - switch (DynI->d_tag) { - case ELF::DT_HASH: - if (HashTable) - continue; - HashTable = - reinterpret_cast<const Elf_Hash *>(toMappedAddr(DynI->getPtr())); - break; - case ELF::DT_STRTAB: - if (!DynStrRegion.Addr) - DynStrRegion.Addr = toMappedAddr(DynI->getPtr()); - break; - case ELF::DT_STRSZ: - if (!DynStrRegion.Size) - DynStrRegion.Size = DynI->getVal(); - break; - case ELF::DT_RELA: - if (!DynRelaRegion.Addr) - DynRelaRegion.Addr = toMappedAddr(DynI->getPtr()); - break; - case ELF::DT_RELASZ: - DynRelaRegion.Size = DynI->getVal(); - break; - case ELF::DT_RELAENT: - DynRelaRegion.EntSize = DynI->getVal(); - } - } +static bool compareAddr(uint64_t VAddr, const Elf_Phdr_Impl<ELFT> *Phdr) { + return VAddr < Phdr->p_vaddr; } template <class ELFT> @@ -803,64 +404,31 @@ const typename ELFFile<ELFT>::Elf_Shdr *ELFFile<ELFT>::section_end() const { } template <class ELFT> -const typename ELFFile<ELFT>::Elf_Sym *ELFFile<ELFT>::symbol_begin() const { - if (!dot_symtab_sec) +ErrorOr<const typename ELFFile<ELFT>::Elf_Dyn *> +ELFFile<ELFT>::dynamic_table_begin(const Elf_Phdr *Phdr) const { + if (!Phdr) return nullptr; - if (dot_symtab_sec->sh_entsize != sizeof(Elf_Sym)) - report_fatal_error("Invalid symbol size"); - return reinterpret_cast<const Elf_Sym *>(base() + dot_symtab_sec->sh_offset); + assert(Phdr->p_type == ELF::PT_DYNAMIC && "Got the wrong program header"); + uintX_t Offset = Phdr->p_offset; + if (Offset > Buf.size()) + return object_error::parse_failed; + return reinterpret_cast<const Elf_Dyn *>(base() + Offset); } template <class ELFT> -const typename ELFFile<ELFT>::Elf_Sym *ELFFile<ELFT>::symbol_end() const { - if (!dot_symtab_sec) +ErrorOr<const typename ELFFile<ELFT>::Elf_Dyn *> +ELFFile<ELFT>::dynamic_table_end(const Elf_Phdr *Phdr) const { + if (!Phdr) return nullptr; - return reinterpret_cast<const Elf_Sym *>(base() + dot_symtab_sec->sh_offset + - dot_symtab_sec->sh_size); -} - -template <class ELFT> -typename ELFFile<ELFT>::Elf_Dyn_Iter -ELFFile<ELFT>::dynamic_table_begin() const { - if (DynamicRegion.Addr) - return Elf_Dyn_Iter(DynamicRegion.EntSize, - (const char *)DynamicRegion.Addr); - return Elf_Dyn_Iter(0, nullptr); -} - -template <class ELFT> -typename ELFFile<ELFT>::Elf_Dyn_Iter -ELFFile<ELFT>::dynamic_table_end(bool NULLEnd) const { - if (!DynamicRegion.Addr) - return Elf_Dyn_Iter(0, nullptr); - Elf_Dyn_Iter Ret(DynamicRegion.EntSize, - (const char *)DynamicRegion.Addr + DynamicRegion.Size); - - if (NULLEnd) { - Elf_Dyn_Iter Start = dynamic_table_begin(); - while (Start != Ret && Start->getTag() != ELF::DT_NULL) - ++Start; - - // Include the DT_NULL. - if (Start != Ret) - ++Start; - Ret = Start; - } - return Ret; -} - -template <class ELFT> -StringRef ELFFile<ELFT>::getLoadName() const { - if (!dt_soname) { - dt_soname = ""; - // Find the DT_SONAME entry - for (const auto &Entry : dynamic_table()) - if (Entry.getTag() == ELF::DT_SONAME) { - dt_soname = getDynamicString(Entry.getVal()); - break; - } - } - return dt_soname; + assert(Phdr->p_type == ELF::PT_DYNAMIC && "Got the wrong program header"); + uintX_t Size = Phdr->p_filesz; + if (Size % sizeof(Elf_Dyn)) + return object_error::elf_invalid_dynamic_table_size; + // FIKME: Check for overflow? + uintX_t End = Phdr->p_offset + Size; + if (End > Buf.size()) + return object_error::parse_failed; + return reinterpret_cast<const Elf_Dyn *>(base() + End); } template <class ELFT> @@ -908,127 +476,52 @@ ELFFile<ELFT>::getStringTable(const Elf_Shdr *Section) const { } template <class ELFT> -const char *ELFFile<ELFT>::getDynamicString(uintX_t Offset) const { - if (Offset >= DynStrRegion.Size) - return nullptr; - return (const char *)DynStrRegion.Addr + Offset; -} - -template <class ELFT> -ErrorOr<StringRef> -ELFFile<ELFT>::getStaticSymbolName(const Elf_Sym *Symb) const { - return Symb->getName(DotStrtab); +ErrorOr<ArrayRef<typename ELFFile<ELFT>::Elf_Word>> +ELFFile<ELFT>::getSHNDXTable(const Elf_Shdr &Section) const { + assert(Section.sh_type == ELF::SHT_SYMTAB_SHNDX); + const Elf_Word *ShndxTableBegin = + reinterpret_cast<const Elf_Word *>(base() + Section.sh_offset); + uintX_t Size = Section.sh_size; + if (Size % sizeof(uint32_t)) + return object_error::parse_failed; + uintX_t NumSymbols = Size / sizeof(uint32_t); + const Elf_Word *ShndxTableEnd = ShndxTableBegin + NumSymbols; + if (reinterpret_cast<const char *>(ShndxTableEnd) > Buf.end()) + return object_error::parse_failed; + ErrorOr<const Elf_Shdr *> SymTableOrErr = getSection(Section.sh_link); + if (std::error_code EC = SymTableOrErr.getError()) + return EC; + const Elf_Shdr &SymTable = **SymTableOrErr; + if (SymTable.sh_type != ELF::SHT_SYMTAB && + SymTable.sh_type != ELF::SHT_DYNSYM) + return object_error::parse_failed; + if (NumSymbols != (SymTable.sh_size / sizeof(Elf_Sym))) + return object_error::parse_failed; + return makeArrayRef(ShndxTableBegin, ShndxTableEnd); } template <class ELFT> ErrorOr<StringRef> -ELFFile<ELFT>::getDynamicSymbolName(const Elf_Sym *Symb) const { - return StringRef(getDynamicString(Symb->st_name)); -} - -template <class ELFT> -ErrorOr<StringRef> ELFFile<ELFT>::getSymbolName(const Elf_Sym *Symb, - bool IsDynamic) const { - if (IsDynamic) - return getDynamicSymbolName(Symb); - return getStaticSymbolName(Symb); +ELFFile<ELFT>::getStringTableForSymtab(const Elf_Shdr &Sec) const { + if (Sec.sh_type != ELF::SHT_SYMTAB && Sec.sh_type != ELF::SHT_DYNSYM) + return object_error::parse_failed; + ErrorOr<const Elf_Shdr *> SectionOrErr = getSection(Sec.sh_link); + if (std::error_code EC = SectionOrErr.getError()) + return EC; + return getStringTable(*SectionOrErr); } template <class ELFT> ErrorOr<StringRef> ELFFile<ELFT>::getSectionName(const Elf_Shdr *Section) const { uint32_t Offset = Section->sh_name; + if (Offset == 0) + return StringRef(); if (Offset >= DotShstrtab.size()) return object_error::parse_failed; return StringRef(DotShstrtab.data() + Offset); } -template <class ELFT> -ErrorOr<StringRef> ELFFile<ELFT>::getSymbolVersion(const Elf_Shdr *section, - const Elf_Sym *symb, - bool &IsDefault) const { - StringRef StrTab; - if (section) { - ErrorOr<StringRef> StrTabOrErr = getStringTable(section); - if (std::error_code EC = StrTabOrErr.getError()) - return EC; - StrTab = *StrTabOrErr; - } - // Handle non-dynamic symbols. - if (section != DotDynSymSec && section != nullptr) { - // Non-dynamic symbols can have versions in their names - // A name of the form 'foo@V1' indicates version 'V1', non-default. - // A name of the form 'foo@@V2' indicates version 'V2', default version. - ErrorOr<StringRef> SymName = symb->getName(StrTab); - if (!SymName) - return SymName; - StringRef Name = *SymName; - size_t atpos = Name.find('@'); - if (atpos == StringRef::npos) { - IsDefault = false; - return StringRef(""); - } - ++atpos; - if (atpos < Name.size() && Name[atpos] == '@') { - IsDefault = true; - ++atpos; - } else { - IsDefault = false; - } - return Name.substr(atpos); - } - - // This is a dynamic symbol. Look in the GNU symbol version table. - if (!dot_gnu_version_sec) { - // No version table. - IsDefault = false; - return StringRef(""); - } - - // Determine the position in the symbol table of this entry. - size_t entry_index = - (reinterpret_cast<uintptr_t>(symb) - DotDynSymSec->sh_offset - - reinterpret_cast<uintptr_t>(base())) / - sizeof(Elf_Sym); - - // Get the corresponding version index entry - const Elf_Versym *vs = getEntry<Elf_Versym>(dot_gnu_version_sec, entry_index); - size_t version_index = vs->vs_index & ELF::VERSYM_VERSION; - - // Special markers for unversioned symbols. - if (version_index == ELF::VER_NDX_LOCAL || - version_index == ELF::VER_NDX_GLOBAL) { - IsDefault = false; - return StringRef(""); - } - - // Lookup this symbol in the version table - LoadVersionMap(); - if (version_index >= VersionMap.size() || VersionMap[version_index].isNull()) - return object_error::parse_failed; - const VersionMapEntry &entry = VersionMap[version_index]; - - // Get the version name string - size_t name_offset; - if (entry.isVerdef()) { - // The first Verdaux entry holds the name. - name_offset = entry.getVerdef()->getAux()->vda_name; - } else { - name_offset = entry.getVernaux()->vna_name; - } - - // Set IsDefault - if (entry.isVerdef()) { - IsDefault = !(vs->vs_index & ELF::VERSYM_HIDDEN); - } else { - IsDefault = false; - } - - if (name_offset >= DynStrRegion.Size) - return object_error::parse_failed; - return StringRef(getDynamicString(name_offset)); -} - /// This function returns the hash value for a symbol in the .dynsym section /// Name of the API remains consistent as specified in the libelf /// REF : http://www.sco.com/developers/gabi/latest/ch5.dynamic.html#hash diff --git a/include/llvm/Object/ELFObjectFile.h b/include/llvm/Object/ELFObjectFile.h index 6e8ace427a20..5823848aaacb 100644 --- a/include/llvm/Object/ELFObjectFile.h +++ b/include/llvm/Object/ELFObjectFile.h @@ -189,11 +189,13 @@ public: typedef typename ELFFile<ELFT>::Elf_Rela Elf_Rela; typedef typename ELFFile<ELFT>::Elf_Dyn Elf_Dyn; - typedef typename ELFFile<ELFT>::Elf_Dyn_Iter Elf_Dyn_Iter; - protected: ELFFile<ELFT> EF; + const Elf_Shdr *DotDynSymSec = nullptr; // Dynamic symbol table section. + const Elf_Shdr *DotSymtabSec = nullptr; // Symbol table section. + ArrayRef<Elf_Word> ShndxTable; + void moveSymbolNext(DataRefImpl &Symb) const override; ErrorOr<StringRef> getSymbolName(DataRefImpl Symb) const override; ErrorOr<uint64_t> getSymbolAddress(DataRefImpl Symb) const override; @@ -204,9 +206,9 @@ protected: uint8_t getSymbolOther(DataRefImpl Symb) const override; uint8_t getSymbolELFType(DataRefImpl Symb) const override; SymbolRef::Type getSymbolType(DataRefImpl Symb) const override; - section_iterator getSymbolSection(const Elf_Sym *Symb) const; - std::error_code getSymbolSection(DataRefImpl Symb, - section_iterator &Res) const override; + ErrorOr<section_iterator> getSymbolSection(const Elf_Sym *Symb, + const Elf_Shdr *SymTab) const; + ErrorOr<section_iterator> getSymbolSection(DataRefImpl Symb) const override; void moveSectionNext(DataRefImpl &Sec) const override; std::error_code getSectionName(DataRefImpl Sec, @@ -240,10 +242,6 @@ protected: return *EF.getSection(Rel.d.a); } - const Elf_Sym *toELFSymIter(DataRefImpl Sym) const { - return EF.template getEntry<Elf_Sym>(Sym.d.a, Sym.d.b); - } - DataRefImpl toDRI(const Elf_Shdr *SymTable, unsigned SymbolNum) const { DataRefImpl DRI; if (!SymTable) { @@ -273,9 +271,9 @@ protected: return DRI; } - DataRefImpl toDRI(Elf_Dyn_Iter Dyn) const { + DataRefImpl toDRI(const Elf_Dyn *Dyn) const { DataRefImpl DRI; - DRI.p = reinterpret_cast<uintptr_t>(Dyn.get()); + DRI.p = reinterpret_cast<uintptr_t>(Dyn); return DRI; } @@ -304,7 +302,13 @@ public: const Elf_Rel *getRel(DataRefImpl Rel) const; const Elf_Rela *getRela(DataRefImpl Rela) const; - const Elf_Sym *getSymbol(DataRefImpl Symb) const; + const Elf_Sym *getSymbol(DataRefImpl Sym) const { + return EF.template getEntry<Elf_Sym>(Sym.d.a, Sym.d.b); + } + + const Elf_Shdr *getSection(DataRefImpl Sec) const { + return reinterpret_cast<const Elf_Shdr *>(Sec.p); + } basic_symbol_iterator symbol_begin_impl() const override; basic_symbol_iterator symbol_end_impl() const override; @@ -320,7 +324,6 @@ public: uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; unsigned getArch() const override; - StringRef getLoadName() const; std::error_code getPlatformFlags(unsigned &Result) const override { Result = EF.getHeader()->e_flags; @@ -352,7 +355,7 @@ void ELFObjectFile<ELFT>::moveSymbolNext(DataRefImpl &Sym) const { template <class ELFT> ErrorOr<StringRef> ELFObjectFile<ELFT>::getSymbolName(DataRefImpl Sym) const { - const Elf_Sym *ESym = toELFSymIter(Sym); + const Elf_Sym *ESym = getSymbol(Sym); const Elf_Shdr *SymTableSec = *EF.getSection(Sym.d.a); const Elf_Shdr *StringTableSec = *EF.getSection(SymTableSec->sh_link); StringRef SymTable = *EF.getStringTable(StringTableSec); @@ -361,12 +364,12 @@ ErrorOr<StringRef> ELFObjectFile<ELFT>::getSymbolName(DataRefImpl Sym) const { template <class ELFT> uint64_t ELFObjectFile<ELFT>::getSectionFlags(DataRefImpl Sec) const { - return toELFShdrIter(Sec)->sh_flags; + return getSection(Sec)->sh_flags; } template <class ELFT> uint32_t ELFObjectFile<ELFT>::getSectionType(DataRefImpl Sec) const { - return toELFShdrIter(Sec)->sh_type; + return getSection(Sec)->sh_type; } template <class ELFT> @@ -398,9 +401,11 @@ ELFObjectFile<ELFT>::getSymbolAddress(DataRefImpl Symb) const { } const Elf_Ehdr *Header = EF.getHeader(); + const Elf_Shdr *SymTab = *EF.getSection(Symb.d.a); if (Header->e_type == ELF::ET_REL) { - ErrorOr<const Elf_Shdr *> SectionOrErr = EF.getSection(ESym); + ErrorOr<const Elf_Shdr *> SectionOrErr = + EF.getSection(ESym, SymTab, ShndxTable); if (std::error_code EC = SectionOrErr.getError()) return EC; const Elf_Shdr *Section = *SectionOrErr; @@ -413,7 +418,7 @@ ELFObjectFile<ELFT>::getSymbolAddress(DataRefImpl Symb) const { template <class ELFT> uint32_t ELFObjectFile<ELFT>::getSymbolAlignment(DataRefImpl Symb) const { - const Elf_Sym *Sym = toELFSymIter(Symb); + const Elf_Sym *Sym = getSymbol(Symb); if (Sym->st_shndx == ELF::SHN_COMMON) return Sym->st_value; return 0; @@ -421,22 +426,22 @@ uint32_t ELFObjectFile<ELFT>::getSymbolAlignment(DataRefImpl Symb) const { template <class ELFT> uint64_t ELFObjectFile<ELFT>::getSymbolSize(DataRefImpl Sym) const { - return toELFSymIter(Sym)->st_size; + return getSymbol(Sym)->st_size; } template <class ELFT> uint64_t ELFObjectFile<ELFT>::getCommonSymbolSizeImpl(DataRefImpl Symb) const { - return toELFSymIter(Symb)->st_size; + return getSymbol(Symb)->st_size; } template <class ELFT> uint8_t ELFObjectFile<ELFT>::getSymbolOther(DataRefImpl Symb) const { - return toELFSymIter(Symb)->st_other; + return getSymbol(Symb)->st_other; } template <class ELFT> uint8_t ELFObjectFile<ELFT>::getSymbolELFType(DataRefImpl Symb) const { - return toELFSymIter(Symb)->getType(); + return getSymbol(Symb)->getType(); } template <class ELFT> @@ -463,7 +468,7 @@ SymbolRef::Type ELFObjectFile<ELFT>::getSymbolType(DataRefImpl Symb) const { template <class ELFT> uint32_t ELFObjectFile<ELFT>::getSymbolFlags(DataRefImpl Sym) const { - const Elf_Sym *ESym = toELFSymIter(Sym); + const Elf_Sym *ESym = getSymbol(Sym); uint32_t Result = SymbolRef::SF_None; @@ -477,7 +482,8 @@ uint32_t ELFObjectFile<ELFT>::getSymbolFlags(DataRefImpl Sym) const { Result |= SymbolRef::SF_Absolute; if (ESym->getType() == ELF::STT_FILE || ESym->getType() == ELF::STT_SECTION || - ESym == EF.symbol_begin() || ESym == EF.dynamic_symbol_begin()) + ESym == EF.symbol_begin(DotSymtabSec) || + ESym == EF.symbol_begin(DotDynSymSec)) Result |= SymbolRef::SF_FormatSpecific; if (EF.getHeader()->e_machine == ELF::EM_ARM) { @@ -505,11 +511,12 @@ uint32_t ELFObjectFile<ELFT>::getSymbolFlags(DataRefImpl Sym) const { } template <class ELFT> -section_iterator -ELFObjectFile<ELFT>::getSymbolSection(const Elf_Sym *ESym) const { - ErrorOr<const Elf_Shdr *> ESecOrErr = EF.getSection(ESym); +ErrorOr<section_iterator> +ELFObjectFile<ELFT>::getSymbolSection(const Elf_Sym *ESym, + const Elf_Shdr *SymTab) const { + ErrorOr<const Elf_Shdr *> ESecOrErr = EF.getSection(ESym, SymTab, ShndxTable); if (std::error_code EC = ESecOrErr.getError()) - report_fatal_error(EC.message()); + return EC; const Elf_Shdr *ESec = *ESecOrErr; if (!ESec) @@ -521,23 +528,23 @@ ELFObjectFile<ELFT>::getSymbolSection(const Elf_Sym *ESym) const { } template <class ELFT> -std::error_code -ELFObjectFile<ELFT>::getSymbolSection(DataRefImpl Symb, - section_iterator &Res) const { - Res = getSymbolSection(getSymbol(Symb)); - return std::error_code(); +ErrorOr<section_iterator> +ELFObjectFile<ELFT>::getSymbolSection(DataRefImpl Symb) const { + const Elf_Sym *Sym = getSymbol(Symb); + const Elf_Shdr *SymTab = *EF.getSection(Symb.d.a); + return getSymbolSection(Sym, SymTab); } template <class ELFT> void ELFObjectFile<ELFT>::moveSectionNext(DataRefImpl &Sec) const { - const Elf_Shdr *ESec = toELFShdrIter(Sec); + const Elf_Shdr *ESec = getSection(Sec); Sec = toDRI(++ESec); } template <class ELFT> std::error_code ELFObjectFile<ELFT>::getSectionName(DataRefImpl Sec, StringRef &Result) const { - ErrorOr<StringRef> Name = EF.getSectionName(&*toELFShdrIter(Sec)); + ErrorOr<StringRef> Name = EF.getSectionName(&*getSection(Sec)); if (!Name) return Name.getError(); Result = *Name; @@ -546,50 +553,50 @@ std::error_code ELFObjectFile<ELFT>::getSectionName(DataRefImpl Sec, template <class ELFT> uint64_t ELFObjectFile<ELFT>::getSectionAddress(DataRefImpl Sec) const { - return toELFShdrIter(Sec)->sh_addr; + return getSection(Sec)->sh_addr; } template <class ELFT> uint64_t ELFObjectFile<ELFT>::getSectionSize(DataRefImpl Sec) const { - return toELFShdrIter(Sec)->sh_size; + return getSection(Sec)->sh_size; } template <class ELFT> std::error_code ELFObjectFile<ELFT>::getSectionContents(DataRefImpl Sec, StringRef &Result) const { - const Elf_Shdr *EShdr = toELFShdrIter(Sec); + const Elf_Shdr *EShdr = getSection(Sec); Result = StringRef((const char *)base() + EShdr->sh_offset, EShdr->sh_size); return std::error_code(); } template <class ELFT> uint64_t ELFObjectFile<ELFT>::getSectionAlignment(DataRefImpl Sec) const { - return toELFShdrIter(Sec)->sh_addralign; + return getSection(Sec)->sh_addralign; } template <class ELFT> bool ELFObjectFile<ELFT>::isSectionText(DataRefImpl Sec) const { - return toELFShdrIter(Sec)->sh_flags & ELF::SHF_EXECINSTR; + return getSection(Sec)->sh_flags & ELF::SHF_EXECINSTR; } template <class ELFT> bool ELFObjectFile<ELFT>::isSectionData(DataRefImpl Sec) const { - const Elf_Shdr *EShdr = toELFShdrIter(Sec); + const Elf_Shdr *EShdr = getSection(Sec); return EShdr->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE) && EShdr->sh_type == ELF::SHT_PROGBITS; } template <class ELFT> bool ELFObjectFile<ELFT>::isSectionBSS(DataRefImpl Sec) const { - const Elf_Shdr *EShdr = toELFShdrIter(Sec); + const Elf_Shdr *EShdr = getSection(Sec); return EShdr->sh_flags & (ELF::SHF_ALLOC | ELF::SHF_WRITE) && EShdr->sh_type == ELF::SHT_NOBITS; } template <class ELFT> bool ELFObjectFile<ELFT>::isSectionVirtual(DataRefImpl Sec) const { - return toELFShdrIter(Sec)->sh_type == ELF::SHT_NOBITS; + return getSection(Sec)->sh_type == ELF::SHT_NOBITS; } template <class ELFT> @@ -636,7 +643,7 @@ ELFObjectFile<ELFT>::getRelocatedSection(DataRefImpl Sec) const { if (EF.getHeader()->e_type != ELF::ET_REL) return section_end(); - const Elf_Shdr *EShdr = toELFShdrIter(Sec); + const Elf_Shdr *EShdr = getSection(Sec); uintX_t Type = EShdr->sh_type; if (Type != ELF::SHT_REL && Type != ELF::SHT_RELA) return section_end(); @@ -668,9 +675,9 @@ ELFObjectFile<ELFT>::getRelocationSymbol(DataRefImpl Rel) const { bool IsDyn = Rel.d.b & 1; DataRefImpl SymbolData; if (IsDyn) - SymbolData = toDRI(EF.getDotDynSymSec(), symbolIdx); + SymbolData = toDRI(DotDynSymSec, symbolIdx); else - SymbolData = toDRI(EF.getDotSymtabSec(), symbolIdx); + SymbolData = toDRI(DotSymtabSec, symbolIdx); return symbol_iterator(SymbolRef(SymbolData, this)); } @@ -715,12 +722,6 @@ ELFObjectFile<ELFT>::getRelocationAddend(DataRefImpl Rel) const { } template <class ELFT> -const typename ELFFile<ELFT>::Elf_Sym * -ELFObjectFile<ELFT>::getSymbol(DataRefImpl Symb) const { - return &*toELFSymIter(Symb); -} - -template <class ELFT> const typename ELFObjectFile<ELFT>::Elf_Rel * ELFObjectFile<ELFT>::getRel(DataRefImpl Rel) const { assert(getRelSection(Rel)->sh_type == ELF::SHT_REL); @@ -737,21 +738,51 @@ ELFObjectFile<ELFT>::getRela(DataRefImpl Rela) const { template <class ELFT> ELFObjectFile<ELFT>::ELFObjectFile(MemoryBufferRef Object, std::error_code &EC) : ELFObjectFileBase( - getELFType(static_cast<endianness>(ELFT::TargetEndianness) == - support::little, - ELFT::Is64Bits), + getELFType(ELFT::TargetEndianness == support::little, ELFT::Is64Bits), Object), - EF(Data.getBuffer(), EC) {} + EF(Data.getBuffer(), EC) { + if (EC) + return; + for (const Elf_Shdr &Sec : EF.sections()) { + switch (Sec.sh_type) { + case ELF::SHT_DYNSYM: { + if (DotDynSymSec) { + // More than one .dynsym! + EC = object_error::parse_failed; + return; + } + DotDynSymSec = &Sec; + break; + } + case ELF::SHT_SYMTAB: { + if (DotSymtabSec) { + // More than one .dynsym! + EC = object_error::parse_failed; + return; + } + DotSymtabSec = &Sec; + break; + } + case ELF::SHT_SYMTAB_SHNDX: { + ErrorOr<ArrayRef<Elf_Word>> TableOrErr = EF.getSHNDXTable(Sec); + if ((EC = TableOrErr.getError())) + return; + ShndxTable = *TableOrErr; + break; + } + } + } +} template <class ELFT> basic_symbol_iterator ELFObjectFile<ELFT>::symbol_begin_impl() const { - DataRefImpl Sym = toDRI(EF.getDotSymtabSec(), 0); + DataRefImpl Sym = toDRI(DotSymtabSec, 0); return basic_symbol_iterator(SymbolRef(Sym, this)); } template <class ELFT> basic_symbol_iterator ELFObjectFile<ELFT>::symbol_end_impl() const { - const Elf_Shdr *SymTab = EF.getDotSymtabSec(); + const Elf_Shdr *SymTab = DotSymtabSec; if (!SymTab) return symbol_begin_impl(); DataRefImpl Sym = toDRI(SymTab, SymTab->sh_size / sizeof(Elf_Sym)); @@ -760,13 +791,13 @@ basic_symbol_iterator ELFObjectFile<ELFT>::symbol_end_impl() const { template <class ELFT> elf_symbol_iterator ELFObjectFile<ELFT>::dynamic_symbol_begin() const { - DataRefImpl Sym = toDRI(EF.getDotDynSymSec(), 0); + DataRefImpl Sym = toDRI(DotDynSymSec, 0); return symbol_iterator(SymbolRef(Sym, this)); } template <class ELFT> elf_symbol_iterator ELFObjectFile<ELFT>::dynamic_symbol_end() const { - const Elf_Shdr *SymTab = EF.getDotDynSymSec(); + const Elf_Shdr *SymTab = DotDynSymSec; DataRefImpl Sym = toDRI(SymTab, SymTab->sh_size / sizeof(Elf_Sym)); return basic_symbol_iterator(SymbolRef(Sym, this)); } @@ -782,19 +813,6 @@ section_iterator ELFObjectFile<ELFT>::section_end() const { } template <class ELFT> -StringRef ELFObjectFile<ELFT>::getLoadName() const { - Elf_Dyn_Iter DI = EF.dynamic_table_begin(); - Elf_Dyn_Iter DE = EF.dynamic_table_end(); - - while (DI != DE && DI->getTag() != ELF::DT_SONAME) - ++DI; - - if (DI != DE) - return EF.getDynamicString(DI->getVal()); - return ""; -} - -template <class ELFT> uint8_t ELFObjectFile<ELFT>::getBytesInAddress() const { return ELFT::Is64Bits ? 8 : 4; } @@ -807,10 +825,14 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const { switch (EF.getHeader()->e_machine) { case ELF::EM_386: return "ELF32-i386"; + case ELF::EM_IAMCU: + return "ELF32-iamcu"; case ELF::EM_X86_64: return "ELF32-x86-64"; case ELF::EM_ARM: return (IsLittleEndian ? "ELF32-arm-little" : "ELF32-arm-big"); + case ELF::EM_AVR: + return "ELF32-avr"; case ELF::EM_HEXAGON: return "ELF32-hexagon"; case ELF::EM_MIPS: @@ -853,6 +875,7 @@ unsigned ELFObjectFile<ELFT>::getArch() const { bool IsLittleEndian = ELFT::TargetEndianness == support::little; switch (EF.getHeader()->e_machine) { case ELF::EM_386: + case ELF::EM_IAMCU: return Triple::x86; case ELF::EM_X86_64: return Triple::x86_64; @@ -860,6 +883,8 @@ unsigned ELFObjectFile<ELFT>::getArch() const { return Triple::aarch64; case ELF::EM_ARM: return Triple::arm; + case ELF::EM_AVR: + return Triple::avr; case ELF::EM_HEXAGON: return Triple::hexagon; case ELF::EM_MIPS: diff --git a/include/llvm/Object/ELFTypes.h b/include/llvm/Object/ELFTypes.h index 27e987ba2852..07b312a7d77c 100644 --- a/include/llvm/Object/ELFTypes.h +++ b/include/llvm/Object/ELFTypes.h @@ -12,7 +12,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/Object/Error.h" -#include "llvm/Support/DataTypes.h" #include "llvm/Support/ELF.h" #include "llvm/Support/Endian.h" #include "llvm/Support/ErrorOr.h" @@ -307,14 +306,18 @@ struct Elf_Dyn_Base<ELFType<TargetEndianness, true>> { } d_un; }; -/// Elf_Dyn_Impl: This inherits from Elf_Dyn_Base, adding getters and setters. +/// Elf_Dyn_Impl: This inherits from Elf_Dyn_Base, adding getters. template <class ELFT> struct Elf_Dyn_Impl : Elf_Dyn_Base<ELFT> { using Elf_Dyn_Base<ELFT>::d_tag; using Elf_Dyn_Base<ELFT>::d_un; - int64_t getTag() const { return d_tag; } - uint64_t getVal() const { return d_un.d_val; } - uint64_t getPtr() const { return d_un.d_ptr; } + typedef typename std::conditional<ELFT::Is64Bits, + int64_t, int32_t>::type intX_t; + typedef typename std::conditional<ELFT::Is64Bits, + uint64_t, uint32_t>::type uintX_t; + intX_t getTag() const { return d_tag; } + uintX_t getVal() const { return d_un.d_val; } + uintX_t getPtr() const { return d_un.d_ptr; } }; // Elf_Rel: Elf Relocation @@ -481,6 +484,30 @@ struct Elf_Hash_Impl { } }; +// .gnu.hash section +template <class ELFT> +struct Elf_GnuHash_Impl { + LLVM_ELF_IMPORT_TYPES_ELFT(ELFT) + Elf_Word nbuckets; + Elf_Word symndx; + Elf_Word maskwords; + Elf_Word shift2; + + ArrayRef<Elf_Off> filter() const { + return ArrayRef<Elf_Off>(reinterpret_cast<const Elf_Off *>(&shift2 + 1), + maskwords); + } + + ArrayRef<Elf_Word> buckets() const { + return ArrayRef<Elf_Word>( + reinterpret_cast<const Elf_Word *>(filter().end()), nbuckets); + } + + ArrayRef<Elf_Word> values(unsigned DynamicSymCount) const { + return ArrayRef<Elf_Word>(buckets().end(), DynamicSymCount - symndx); + } +}; + // MIPS .reginfo section template <class ELFT> struct Elf_Mips_RegInfo; diff --git a/include/llvm/Object/Error.h b/include/llvm/Object/Error.h index aa320bb51a46..0f79a6ed0dd8 100644 --- a/include/llvm/Object/Error.h +++ b/include/llvm/Object/Error.h @@ -30,6 +30,7 @@ enum class object_error { string_table_non_null_end, invalid_section_index, bitcode_section_not_found, + elf_invalid_dynamic_table_size, macho_small_load_command, macho_load_segment_too_many_sections, macho_load_segment_too_small, diff --git a/include/llvm/Object/FunctionIndexObjectFile.h b/include/llvm/Object/FunctionIndexObjectFile.h new file mode 100644 index 000000000000..74b461dc7cc7 --- /dev/null +++ b/include/llvm/Object/FunctionIndexObjectFile.h @@ -0,0 +1,110 @@ +//===- FunctionIndexObjectFile.h - Function index file implementation -----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares the FunctionIndexObjectFile template class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_FUNCTIONINDEXOBJECTFILE_H +#define LLVM_OBJECT_FUNCTIONINDEXOBJECTFILE_H + +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/Object/SymbolicFile.h" + +namespace llvm { +class FunctionInfoIndex; +class Module; + +namespace object { +class ObjectFile; + +/// This class is used to read just the function summary index related +/// sections out of the given object (which may contain a single module's +/// bitcode or be a combined index bitcode file). It builds a FunctionInfoIndex +/// object. +class FunctionIndexObjectFile : public SymbolicFile { + std::unique_ptr<FunctionInfoIndex> Index; + +public: + FunctionIndexObjectFile(MemoryBufferRef Object, + std::unique_ptr<FunctionInfoIndex> I); + ~FunctionIndexObjectFile() override; + + // TODO: Walk through FunctionMap entries for function symbols. + // However, currently these interfaces are not used by any consumers. + void moveSymbolNext(DataRefImpl &Symb) const override { + llvm_unreachable("not implemented"); + } + std::error_code printSymbolName(raw_ostream &OS, + DataRefImpl Symb) const override { + llvm_unreachable("not implemented"); + return std::error_code(); + } + uint32_t getSymbolFlags(DataRefImpl Symb) const override { + llvm_unreachable("not implemented"); + return 0; + } + basic_symbol_iterator symbol_begin_impl() const override { + llvm_unreachable("not implemented"); + return basic_symbol_iterator(BasicSymbolRef()); + } + basic_symbol_iterator symbol_end_impl() const override { + llvm_unreachable("not implemented"); + return basic_symbol_iterator(BasicSymbolRef()); + } + + const FunctionInfoIndex &getIndex() const { + return const_cast<FunctionIndexObjectFile *>(this)->getIndex(); + } + FunctionInfoIndex &getIndex() { return *Index; } + std::unique_ptr<FunctionInfoIndex> takeIndex(); + + static inline bool classof(const Binary *v) { return v->isFunctionIndex(); } + + /// \brief Finds and returns bitcode embedded in the given object file, or an + /// error code if not found. + static ErrorOr<MemoryBufferRef> findBitcodeInObject(const ObjectFile &Obj); + + /// \brief Finds and returns bitcode in the given memory buffer (which may + /// be either a bitcode file or a native object file with embedded bitcode), + /// or an error code if not found. + static ErrorOr<MemoryBufferRef> + findBitcodeInMemBuffer(MemoryBufferRef Object); + + /// \brief Looks for function summary in the given memory buffer, + /// returns true if found, else false. + static bool + hasFunctionSummaryInMemBuffer(MemoryBufferRef Object, + DiagnosticHandlerFunction DiagnosticHandler); + + /// \brief Parse function index in the given memory buffer. + /// Return new FunctionIndexObjectFile instance containing parsed function + /// summary/index. + static ErrorOr<std::unique_ptr<FunctionIndexObjectFile>> + create(MemoryBufferRef Object, DiagnosticHandlerFunction DiagnosticHandler, + bool IsLazy = false); + + /// \brief Parse the function summary information for function with the + /// given name out of the given buffer. Parsed information is + /// stored on the index object saved in this object. + std::error_code + findFunctionSummaryInMemBuffer(MemoryBufferRef Object, + DiagnosticHandlerFunction DiagnosticHandler, + StringRef FunctionName); +}; +} + +/// Parse the function index out of an IR file and return the function +/// index object if found, or nullptr if not. +ErrorOr<std::unique_ptr<FunctionInfoIndex>> +getFunctionIndexForFile(StringRef Path, + DiagnosticHandlerFunction DiagnosticHandler); +} + +#endif diff --git a/include/llvm/Object/MachO.h b/include/llvm/Object/MachO.h index 489ecef5c996..e02ce3b21416 100644 --- a/include/llvm/Object/MachO.h +++ b/include/llvm/Object/MachO.h @@ -100,7 +100,7 @@ private: }; typedef content_iterator<ExportEntry> export_iterator; -/// MachORebaseEntry encapsulates the current state in the decompression of +/// MachORebaseEntry encapsulates the current state in the decompression of /// rebasing opcodes. This allows you to iterate through the compressed table of /// rebasing using: /// for (const llvm::object::MachORebaseEntry &Entry : Obj->rebaseTable()) { @@ -116,7 +116,7 @@ public: bool operator==(const MachORebaseEntry &) const; void moveNext(); - + private: friend class MachOObjectFile; void moveToFirst(); @@ -210,8 +210,7 @@ public: uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override; SymbolRef::Type getSymbolType(DataRefImpl Symb) const override; uint32_t getSymbolFlags(DataRefImpl Symb) const override; - std::error_code getSymbolSection(DataRefImpl Symb, - section_iterator &Res) const override; + ErrorOr<section_iterator> getSymbolSection(DataRefImpl Symb) const override; unsigned getSymbolSectionID(SymbolRef Symb) const; unsigned getSectionID(SectionRef Sec) const; @@ -423,6 +422,24 @@ public: return v->isMachO(); } + static uint32_t + getVersionMinMajor(MachO::version_min_command &C, bool SDK) { + uint32_t VersionOrSDK = (SDK) ? C.sdk : C.version; + return (VersionOrSDK >> 16) & 0xffff; + } + + static uint32_t + getVersionMinMinor(MachO::version_min_command &C, bool SDK) { + uint32_t VersionOrSDK = (SDK) ? C.sdk : C.version; + return (VersionOrSDK >> 8) & 0xff; + } + + static uint32_t + getVersionMinUpdate(MachO::version_min_command &C, bool SDK) { + uint32_t VersionOrSDK = (SDK) ? C.sdk : C.version; + return VersionOrSDK & 0xff; + } + private: uint64_t getSymbolValueImpl(DataRefImpl Symb) const override; @@ -504,4 +521,3 @@ inline const ObjectFile *DiceRef::getObjectFile() const { } #endif - diff --git a/include/llvm/Object/ObjectFile.h b/include/llvm/Object/ObjectFile.h index 8dd525626218..ce0c891ee0c2 100644 --- a/include/llvm/Object/ObjectFile.h +++ b/include/llvm/Object/ObjectFile.h @@ -100,8 +100,7 @@ public: relocation_iterator relocation_begin() const; relocation_iterator relocation_end() const; iterator_range<relocation_iterator> relocations() const { - return iterator_range<relocation_iterator>(relocation_begin(), - relocation_end()); + return make_range(relocation_begin(), relocation_end()); } section_iterator getRelocatedSection() const; @@ -147,7 +146,7 @@ public: /// @brief Get section this symbol is defined in reference to. Result is /// end_sections() if it is undefined or is an absolute symbol. - std::error_code getSection(section_iterator &Result) const; + ErrorOr<section_iterator> getSection() const; const ObjectFile *getObject() const; }; @@ -202,8 +201,8 @@ protected: virtual uint32_t getSymbolAlignment(DataRefImpl Symb) const; virtual uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const = 0; virtual SymbolRef::Type getSymbolType(DataRefImpl Symb) const = 0; - virtual std::error_code getSymbolSection(DataRefImpl Symb, - section_iterator &Res) const = 0; + virtual ErrorOr<section_iterator> + getSymbolSection(DataRefImpl Symb) const = 0; // Same as above for SectionRef. friend class SectionRef; @@ -323,8 +322,8 @@ inline uint64_t SymbolRef::getCommonSize() const { return getObject()->getCommonSymbolSize(getRawDataRefImpl()); } -inline std::error_code SymbolRef::getSection(section_iterator &Result) const { - return getObject()->getSymbolSection(getRawDataRefImpl(), Result); +inline ErrorOr<section_iterator> SymbolRef::getSection() const { + return getObject()->getSymbolSection(getRawDataRefImpl()); } inline SymbolRef::Type SymbolRef::getType() const { diff --git a/include/llvm/Object/SymbolicFile.h b/include/llvm/Object/SymbolicFile.h index 537997ac6318..0c5b38111a9c 100644 --- a/include/llvm/Object/SymbolicFile.h +++ b/include/llvm/Object/SymbolicFile.h @@ -15,6 +15,7 @@ #define LLVM_OBJECT_SYMBOLICFILE_H #include "llvm/Object/Binary.h" +#include "llvm/Support/Format.h" namespace llvm { namespace object { @@ -29,6 +30,12 @@ union DataRefImpl { DataRefImpl() { std::memset(this, 0, sizeof(DataRefImpl)); } }; +template <typename OStream> +OStream& operator<<(OStream &OS, const DataRefImpl &D) { + OS << "(" << format("0x%x8", D.p) << " (" << format("0x%x8", D.d.a) << ", " << format("0x%x8", D.d.b) << "))"; + return OS; +} + inline bool operator==(const DataRefImpl &a, const DataRefImpl &b) { // Check bitwise identical. This is the only legal way to compare a union w/o // knowing which member is in use. @@ -94,6 +101,7 @@ public: // (e.g. section symbols) SF_Thumb = 1U << 8, // Thumb symbol in a 32-bit ARM binary SF_Hidden = 1U << 9, // Symbol has hidden visibility + SF_Const = 1U << 10, // Symbol value is constant }; BasicSymbolRef() : OwningObject(nullptr) { } diff --git a/include/llvm/Option/Arg.h b/include/llvm/Option/Arg.h index e1b72b6267cf..99d329693de2 100644 --- a/include/llvm/Option/Arg.h +++ b/include/llvm/Option/Arg.h @@ -113,6 +113,7 @@ public: /// when rendered as a input (e.g., Xlinker). void renderAsInput(const ArgList &Args, ArgStringList &Output) const; + void print(raw_ostream &O) const; void dump() const; /// \brief Return a formatted version of the argument and diff --git a/include/llvm/Option/ArgList.h b/include/llvm/Option/ArgList.h index ef4005761b75..89771b5c3cf1 100644 --- a/include/llvm/Option/ArgList.h +++ b/include/llvm/Option/ArgList.h @@ -259,6 +259,9 @@ public: void AddLastArg(ArgStringList &Output, OptSpecifier Id0, OptSpecifier Id1) const; + /// AddAllArgs - Render all arguments matching any of the given ids. + void AddAllArgs(ArgStringList &Output, ArrayRef<OptSpecifier> Ids) const; + /// AddAllArgs - Render all arguments matching the given ids. void AddAllArgs(ArgStringList &Output, OptSpecifier Id0, OptSpecifier Id1 = 0U, OptSpecifier Id2 = 0U) const; @@ -303,6 +306,9 @@ public: const char *GetOrMakeJoinedArgString(unsigned Index, StringRef LHS, StringRef RHS) const; + void print(raw_ostream &O) const; + void dump() const; + /// @} }; diff --git a/include/llvm/Option/OptTable.h b/include/llvm/Option/OptTable.h index 96f51cf3317d..390e52774fea 100644 --- a/include/llvm/Option/OptTable.h +++ b/include/llvm/Option/OptTable.h @@ -50,8 +50,7 @@ public: private: /// \brief The static option information table. - const Info *OptionInfos; - unsigned NumOptionInfos; + ArrayRef<Info> OptionInfos; bool IgnoreCase; unsigned TheInputOptionID; @@ -74,14 +73,13 @@ private: } protected: - OptTable(const Info *OptionInfos, unsigned NumOptionInfos, - bool IgnoreCase = false); + OptTable(ArrayRef<Info> OptionInfos, bool IgnoreCase = false); public: ~OptTable(); /// \brief Return the total number of option classes. - unsigned getNumOptions() const { return NumOptionInfos; } + unsigned getNumOptions() const { return OptionInfos.size(); } /// \brief Get the given Opt's Option instance, lazily creating it /// if necessary. diff --git a/include/llvm/Option/Option.h b/include/llvm/Option/Option.h index 09be26c7cf20..494987a135ef 100644 --- a/include/llvm/Option/Option.h +++ b/include/llvm/Option/Option.h @@ -195,6 +195,7 @@ public: /// start. Arg *accept(const ArgList &Args, unsigned &Index, unsigned ArgSize) const; + void print(raw_ostream &O) const; void dump() const; }; diff --git a/include/llvm/PassAnalysisSupport.h b/include/llvm/PassAnalysisSupport.h index 0b318fc8fb07..492a4ef464f8 100644 --- a/include/llvm/PassAnalysisSupport.h +++ b/include/llvm/PassAnalysisSupport.h @@ -36,11 +36,17 @@ namespace llvm { /// class AnalysisUsage { public: - typedef SmallVector<AnalysisID, 32> VectorType; + typedef SmallVectorImpl<AnalysisID> VectorType; private: /// Sets of analyses required and preserved by a pass - VectorType Required, RequiredTransitive, Preserved; + // TODO: It's not clear that SmallVector is an appropriate data structure for + // this usecase. The sizes were picked to minimize wasted space, but are + // otherwise fairly meaningless. + SmallVector<AnalysisID, 8> Required; + SmallVector<AnalysisID, 2> RequiredTransitive; + SmallVector<AnalysisID, 2> Preserved; + SmallVector<AnalysisID, 0> Used; bool PreservesAll; public: @@ -72,14 +78,32 @@ public: Preserved.push_back(&ID); return *this; } - ///@} - /// Add the specified Pass class to the set of analyses preserved by this pass. template<class PassClass> AnalysisUsage &addPreserved() { Preserved.push_back(&PassClass::ID); return *this; } + ///@} + + ///@{ + /// Add the specified ID to the set of analyses used by this pass if they are + /// available.. + AnalysisUsage &addUsedIfAvailableID(const void *ID) { + Used.push_back(ID); + return *this; + } + AnalysisUsage &addUsedIfAvailableID(char &ID) { + Used.push_back(&ID); + return *this; + } + /// Add the specified Pass class to the set of analyses used by this pass. + template<class PassClass> + AnalysisUsage &addUsedIfAvailable() { + Used.push_back(&PassClass::ID); + return *this; + } + ///@} /// Add the Pass with the specified argument string to the set of analyses /// preserved by this pass. If no such Pass exists, do nothing. This can be @@ -108,6 +132,7 @@ public: return RequiredTransitive; } const VectorType &getPreservedSet() const { return Preserved; } + const VectorType &getUsedSet() const { return Used; } }; //===----------------------------------------------------------------------===// diff --git a/include/llvm/PassInfo.h b/include/llvm/PassInfo.h index d10761831b3a..cee4ade323e4 100644 --- a/include/llvm/PassInfo.h +++ b/include/llvm/PassInfo.h @@ -33,13 +33,13 @@ public: typedef Pass *(*TargetMachineCtor_t)(TargetMachine *); private: - const char *const PassName; // Nice name for Pass - const char *const PassArgument; // Command Line argument to run this pass - const void *PassID; - const bool IsCFGOnlyPass; // Pass only looks at the CFG. - const bool IsAnalysis; // True if an analysis pass. - const bool IsAnalysisGroup; // True if an analysis group. - std::vector<const PassInfo*> ItfImpl;// Interfaces implemented by this pass + const char *const PassName; // Nice name for Pass + const char *const PassArgument; // Command Line argument to run this pass + const void *PassID; + const bool IsCFGOnlyPass; // Pass only looks at the CFG. + const bool IsAnalysis; // True if an analysis pass. + const bool IsAnalysisGroup; // True if an analysis group. + std::vector<const PassInfo *> ItfImpl; // Interfaces implemented by this pass NormalCtor_t NormalCtor; TargetMachineCtor_t TargetMachineCtor; @@ -50,18 +50,16 @@ public: PassInfo(const char *name, const char *arg, const void *pi, NormalCtor_t normal, bool isCFGOnly, bool is_analysis, TargetMachineCtor_t machine = nullptr) - : PassName(name), PassArgument(arg), PassID(pi), - IsCFGOnlyPass(isCFGOnly), - IsAnalysis(is_analysis), IsAnalysisGroup(false), NormalCtor(normal), - TargetMachineCtor(machine) {} + : PassName(name), PassArgument(arg), PassID(pi), IsCFGOnlyPass(isCFGOnly), + IsAnalysis(is_analysis), IsAnalysisGroup(false), NormalCtor(normal), + TargetMachineCtor(machine) {} /// PassInfo ctor - Do not call this directly, this should only be invoked /// through RegisterPass. This version is for use by analysis groups; it /// does not auto-register the pass. PassInfo(const char *name, const void *pi) - : PassName(name), PassArgument(""), PassID(pi), - IsCFGOnlyPass(false), - IsAnalysis(false), IsAnalysisGroup(true), NormalCtor(nullptr), - TargetMachineCtor(nullptr) {} + : PassName(name), PassArgument(""), PassID(pi), IsCFGOnlyPass(false), + IsAnalysis(false), IsAnalysisGroup(true), NormalCtor(nullptr), + TargetMachineCtor(nullptr) {} /// getPassName - Return the friendly name for the pass, never returns null /// @@ -78,10 +76,8 @@ public: const void *getTypeInfo() const { return PassID; } /// Return true if this PassID implements the specified ID pointer. - bool isPassID(const void *IDPtr) const { - return PassID == IDPtr; - } - + bool isPassID(const void *IDPtr) const { return PassID == IDPtr; } + /// isAnalysisGroup - Return true if this is an analysis group, not a normal /// pass. /// @@ -91,7 +87,7 @@ public: /// isCFGOnlyPass - return true if this pass only looks at the CFG for the /// function. bool isCFGOnlyPass() const { return IsCFGOnlyPass; } - + /// getNormalCtor - Return a pointer to a function, that when called, creates /// an instance of the pass and returns it. This pointer may be null if there /// is no default constructor for the pass. diff --git a/include/llvm/PassRegistry.h b/include/llvm/PassRegistry.h index 8c28ef5e7e61..e7fe1f53a4d4 100644 --- a/include/llvm/PassRegistry.h +++ b/include/llvm/PassRegistry.h @@ -17,7 +17,6 @@ #ifndef LLVM_PASSREGISTRY_H #define LLVM_PASSREGISTRY_H -#include "llvm-c/Core.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" diff --git a/include/llvm/PassSupport.h b/include/llvm/PassSupport.h index 6cb6516412e8..7c3d49f02e8f 100644 --- a/include/llvm/PassSupport.h +++ b/include/llvm/PassSupport.h @@ -26,7 +26,7 @@ #include "llvm/PassInfo.h" #include "llvm/PassRegistry.h" #include "llvm/Support/Atomic.h" -#include "llvm/Support/Valgrind.h" +#include "llvm/Support/Compiler.h" #include <vector> namespace llvm { diff --git a/include/llvm/ProfileData/CoverageMapping.h b/include/llvm/ProfileData/CoverageMapping.h index 3488e793d84f..3790e1358449 100644 --- a/include/llvm/ProfileData/CoverageMapping.h +++ b/include/llvm/ProfileData/CoverageMapping.h @@ -104,7 +104,7 @@ struct CounterExpression { }; /// \brief A Counter expression builder is used to construct the -/// counter expressions. It avoids unecessary duplication +/// counter expressions. It avoids unnecessary duplication /// and simplifies algebraic expressions. class CounterExpressionBuilder { /// \brief A list of all the counter expressions @@ -236,7 +236,7 @@ class CounterMappingContext { public: CounterMappingContext(ArrayRef<CounterExpression> Expressions, - ArrayRef<uint64_t> CounterValues = ArrayRef<uint64_t>()) + ArrayRef<uint64_t> CounterValues = None) : Expressions(Expressions), CounterValues(CounterValues) {} void setCounts(ArrayRef<uint64_t> Counts) { CounterValues = Counts; } @@ -443,7 +443,7 @@ public: /// \brief Get the list of function instantiations in the file. /// - /// Fucntions that are instantiated more than once, such as C++ template + /// Functions that are instantiated more than once, such as C++ template /// specializations, have distinct coverage records for each instantiation. std::vector<const FunctionRecord *> getInstantiations(StringRef Filename); diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index 77055ba87268..4688759a3bd1 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -16,34 +16,310 @@ #ifndef LLVM_PROFILEDATA_INSTRPROF_H_ #define LLVM_PROFILEDATA_INSTRPROF_H_ +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/ProfileData/InstrProfData.inc" +#include "llvm/Support/Endian.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/MD5.h" #include <cstdint> +#include <list> #include <system_error> #include <vector> +#define INSTR_PROF_INDEX_VERSION 3 namespace llvm { + +class Function; +class GlobalVariable; +class Module; + +/// Return the name of data section containing profile counter variables. +inline StringRef getInstrProfCountersSectionName(bool AddSegment) { + return AddSegment ? "__DATA," INSTR_PROF_CNTS_SECT_NAME_STR + : INSTR_PROF_CNTS_SECT_NAME_STR; +} + +/// Return the name of data section containing names of instrumented +/// functions. +inline StringRef getInstrProfNameSectionName(bool AddSegment) { + return AddSegment ? "__DATA," INSTR_PROF_NAME_SECT_NAME_STR + : INSTR_PROF_NAME_SECT_NAME_STR; +} + +/// Return the name of the data section containing per-function control +/// data. +inline StringRef getInstrProfDataSectionName(bool AddSegment) { + return AddSegment ? "__DATA," INSTR_PROF_DATA_SECT_NAME_STR + : INSTR_PROF_DATA_SECT_NAME_STR; +} + +/// Return the name profile runtime entry point to do value profiling +/// for a given site. +inline StringRef getInstrProfValueProfFuncName() { + return INSTR_PROF_VALUE_PROF_FUNC_STR; +} + +/// Return the name of the section containing function coverage mapping +/// data. +inline StringRef getInstrProfCoverageSectionName(bool AddSegment) { + return AddSegment ? "__DATA,__llvm_covmap" : "__llvm_covmap"; +} + +/// Return the name prefix of variables containing instrumented function names. +inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; } + +/// Return the name prefix of variables containing per-function control data. +inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } + +/// Return the name prefix of profile counter variables. +inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; } + +/// Return the name prefix of the COMDAT group for instrumentation variables +/// associated with a COMDAT function. +inline StringRef getInstrProfComdatPrefix() { return "__profv_"; } + +/// Return the name of a covarage mapping variable (internal linkage) +/// for each instrumented source module. Such variables are allocated +/// in the __llvm_covmap section. +inline StringRef getCoverageMappingVarName() { + return "__llvm_coverage_mapping"; +} + +/// Return the name of function that registers all the per-function control +/// data at program startup time by calling __llvm_register_function. This +/// function has internal linkage and is called by __llvm_profile_init +/// runtime method. This function is not generated for these platforms: +/// Darwin, Linux, and FreeBSD. +inline StringRef getInstrProfRegFuncsName() { + return "__llvm_profile_register_functions"; +} + +/// Return the name of the runtime interface that registers per-function control +/// data for one instrumented function. +inline StringRef getInstrProfRegFuncName() { + return "__llvm_profile_register_function"; +} + +/// Return the name of the runtime initialization method that is generated by +/// the compiler. The function calls __llvm_profile_register_functions and +/// __llvm_profile_override_default_filename functions if needed. This function +/// has internal linkage and invoked at startup time via init_array. +inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; } + +/// Return the name of the hook variable defined in profile runtime library. +/// A reference to the variable causes the linker to link in the runtime +/// initialization module (which defines the hook variable). +inline StringRef getInstrProfRuntimeHookVarName() { + return "__llvm_profile_runtime"; +} + +/// Return the name of the compiler generated function that references the +/// runtime hook variable. The function is a weak global. +inline StringRef getInstrProfRuntimeHookVarUseFuncName() { + return "__llvm_profile_runtime_user"; +} + +/// Return the name of the profile runtime interface that overrides the default +/// profile data file name. +inline StringRef getInstrProfFileOverriderFuncName() { + return "__llvm_profile_override_default_filename"; +} + +/// Return the modified name for function \c F suitable to be +/// used the key for profile lookup. +std::string getPGOFuncName(const Function &F, + uint64_t Version = INSTR_PROF_INDEX_VERSION); + +/// Return the modified name for a function suitable to be +/// used the key for profile lookup. The function's original +/// name is \c RawFuncName and has linkage of type \c Linkage. +/// The function is defined in module \c FileName. +std::string getPGOFuncName(StringRef RawFuncName, + GlobalValue::LinkageTypes Linkage, + StringRef FileName, + uint64_t Version = INSTR_PROF_INDEX_VERSION); + +/// Create and return the global variable for function name used in PGO +/// instrumentation. \c FuncName is the name of the function returned +/// by \c getPGOFuncName call. +GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName); + +/// Create and return the global variable for function name used in PGO +/// instrumentation. /// \c FuncName is the name of the function +/// returned by \c getPGOFuncName call, \c M is the owning module, +/// and \c Linkage is the linkage of the instrumented function. +GlobalVariable *createPGOFuncNameVar(Module &M, + GlobalValue::LinkageTypes Linkage, + StringRef FuncName); + +/// Given a PGO function name, remove the filename prefix and return +/// the original (static) function name. +StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName); + const std::error_category &instrprof_category(); enum class instrprof_error { - success = 0, - eof, - bad_magic, - bad_header, - unsupported_version, - unsupported_hash_type, - too_large, - truncated, - malformed, - unknown_function, - hash_mismatch, - count_mismatch, - counter_overflow + success = 0, + eof, + unrecognized_format, + bad_magic, + bad_header, + unsupported_version, + unsupported_hash_type, + too_large, + truncated, + malformed, + unknown_function, + hash_mismatch, + count_mismatch, + counter_overflow, + value_site_count_mismatch }; inline std::error_code make_error_code(instrprof_error E) { return std::error_code(static_cast<int>(E), instrprof_category()); } +inline instrprof_error MergeResult(instrprof_error &Accumulator, + instrprof_error Result) { + // Prefer first error encountered as later errors may be secondary effects of + // the initial problem. + if (Accumulator == instrprof_error::success && + Result != instrprof_error::success) + Accumulator = Result; + return Accumulator; +} + +enum InstrProfValueKind : uint32_t { +#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, +#include "llvm/ProfileData/InstrProfData.inc" +}; + +namespace object { +class SectionRef; +} + +namespace IndexedInstrProf { +uint64_t ComputeHash(StringRef K); +} + +/// A symbol table used for function PGO name look-up with keys +/// (such as pointers, md5hash values) to the function. A function's +/// PGO name or name's md5hash are used in retrieving the profile +/// data of the function. See \c getPGOFuncName() method for details +/// on how PGO name is formed. +class InstrProfSymtab { +public: + typedef std::vector<std::pair<uint64_t, uint64_t>> AddrHashMap; + +private: + StringRef Data; + uint64_t Address; + // A map from MD5 hash keys to function name strings. + std::vector<std::pair<uint64_t, std::string>> HashNameMap; + // A map from function runtime address to function name MD5 hash. + // This map is only populated and used by raw instr profile reader. + AddrHashMap AddrToMD5Map; + +public: + InstrProfSymtab() : Data(), Address(0), HashNameMap(), AddrToMD5Map() {} + + /// Create InstrProfSymtab from an object file section which + /// contains function PGO names that are uncompressed. + /// This interface is used by CoverageMappingReader. + std::error_code create(object::SectionRef &Section); + /// This interface is used by reader of CoverageMapping test + /// format. + inline std::error_code create(StringRef D, uint64_t BaseAddr); + /// Create InstrProfSymtab from a set of names iteratable from + /// \p IterRange. This interface is used by IndexedProfReader. + template <typename NameIterRange> void create(const NameIterRange &IterRange); + // If the symtab is created by a series of calls to \c addFuncName, \c + // finalizeSymtab needs to be called before looking up function names. + // This is required because the underlying map is a vector (for space + // efficiency) which needs to be sorted. + inline void finalizeSymtab(); + /// Update the symtab by adding \p FuncName to the table. This interface + /// is used by the raw and text profile readers. + void addFuncName(StringRef FuncName) { + HashNameMap.push_back(std::make_pair( + IndexedInstrProf::ComputeHash(FuncName), FuncName.str())); + } + /// Map a function address to its name's MD5 hash. This interface + /// is only used by the raw profiler reader. + void mapAddress(uint64_t Addr, uint64_t MD5Val) { + AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val)); + } + AddrHashMap &getAddrHashMap() { return AddrToMD5Map; } + /// Return function's PGO name from the function name's symabol + /// address in the object file. If an error occurs, Return + /// an empty string. + StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize); + /// Return function's PGO name from the name's md5 hash value. + /// If not found, return an empty string. + inline StringRef getFuncName(uint64_t FuncMD5Hash); +}; + +std::error_code InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { + Data = D; + Address = BaseAddr; + return std::error_code(); +} + +template <typename NameIterRange> +void InstrProfSymtab::create(const NameIterRange &IterRange) { + for (auto Name : IterRange) + HashNameMap.push_back( + std::make_pair(IndexedInstrProf::ComputeHash(Name), Name.str())); + finalizeSymtab(); +} + +void InstrProfSymtab::finalizeSymtab() { + std::sort(HashNameMap.begin(), HashNameMap.end(), less_first()); + HashNameMap.erase(std::unique(HashNameMap.begin(), HashNameMap.end()), + HashNameMap.end()); + std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(), less_first()); + AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()), + AddrToMD5Map.end()); +} + +StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) { + auto Result = + std::lower_bound(HashNameMap.begin(), HashNameMap.end(), FuncMD5Hash, + [](const std::pair<uint64_t, std::string> &LHS, + uint64_t RHS) { return LHS.first < RHS; }); + if (Result != HashNameMap.end()) + return Result->second; + return StringRef(); +} + +struct InstrProfValueSiteRecord { + /// Value profiling data pairs at a given value site. + std::list<InstrProfValueData> ValueData; + + InstrProfValueSiteRecord() { ValueData.clear(); } + template <class InputIterator> + InstrProfValueSiteRecord(InputIterator F, InputIterator L) + : ValueData(F, L) {} + + /// Sort ValueData ascending by Value + void sortByTargetValues() { + ValueData.sort( + [](const InstrProfValueData &left, const InstrProfValueData &right) { + return left.Value < right.Value; + }); + } + + /// Merge data from another InstrProfValueSiteRecord + /// Optionally scale merged counts by \p Weight. + instrprof_error mergeValueData(InstrProfValueSiteRecord &Input, + uint64_t Weight = 1); +}; + /// Profiling information for a single function. struct InstrProfRecord { InstrProfRecord() {} @@ -52,8 +328,258 @@ struct InstrProfRecord { StringRef Name; uint64_t Hash; std::vector<uint64_t> Counts; + + typedef std::vector<std::pair<uint64_t, uint64_t>> ValueMapType; + + /// Return the number of value profile kinds with non-zero number + /// of profile sites. + inline uint32_t getNumValueKinds() const; + /// Return the number of instrumented sites for ValueKind. + inline uint32_t getNumValueSites(uint32_t ValueKind) const; + /// Return the total number of ValueData for ValueKind. + inline uint32_t getNumValueData(uint32_t ValueKind) const; + /// Return the number of value data collected for ValueKind at profiling + /// site: Site. + inline uint32_t getNumValueDataForSite(uint32_t ValueKind, + uint32_t Site) const; + /// Return the array of profiled values at \p Site. + inline std::unique_ptr<InstrProfValueData[]> + getValueForSite(uint32_t ValueKind, uint32_t Site, + uint64_t (*ValueMapper)(uint32_t, uint64_t) = 0) const; + inline void + getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind, uint32_t Site, + uint64_t (*ValueMapper)(uint32_t, uint64_t) = 0) const; + /// Reserve space for NumValueSites sites. + inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); + /// Add ValueData for ValueKind at value Site. + void addValueData(uint32_t ValueKind, uint32_t Site, + InstrProfValueData *VData, uint32_t N, + ValueMapType *ValueMap); + + /// Merge the counts in \p Other into this one. + /// Optionally scale merged counts by \p Weight. + instrprof_error merge(InstrProfRecord &Other, uint64_t Weight = 1); + + /// Clear value data entries + void clearValueData() { + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) + getValueSitesForKind(Kind).clear(); + } + +private: + std::vector<InstrProfValueSiteRecord> IndirectCallSites; + const std::vector<InstrProfValueSiteRecord> & + getValueSitesForKind(uint32_t ValueKind) const { + switch (ValueKind) { + case IPVK_IndirectCallTarget: + return IndirectCallSites; + default: + llvm_unreachable("Unknown value kind!"); + } + return IndirectCallSites; + } + + std::vector<InstrProfValueSiteRecord> & + getValueSitesForKind(uint32_t ValueKind) { + return const_cast<std::vector<InstrProfValueSiteRecord> &>( + const_cast<const InstrProfRecord *>(this) + ->getValueSitesForKind(ValueKind)); + } + + // Map indirect call target name hash to name string. + uint64_t remapValue(uint64_t Value, uint32_t ValueKind, + ValueMapType *HashKeys); + + // Merge Value Profile data from Src record to this record for ValueKind. + // Scale merged value counts by \p Weight. + instrprof_error mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src, + uint64_t Weight); +}; + +uint32_t InstrProfRecord::getNumValueKinds() const { + uint32_t NumValueKinds = 0; + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) + NumValueKinds += !(getValueSitesForKind(Kind).empty()); + return NumValueKinds; +} + +uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const { + uint32_t N = 0; + const std::vector<InstrProfValueSiteRecord> &SiteRecords = + getValueSitesForKind(ValueKind); + for (auto &SR : SiteRecords) { + N += SR.ValueData.size(); + } + return N; +} + +uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const { + return getValueSitesForKind(ValueKind).size(); +} + +uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, + uint32_t Site) const { + return getValueSitesForKind(ValueKind)[Site].ValueData.size(); +} + +std::unique_ptr<InstrProfValueData[]> InstrProfRecord::getValueForSite( + uint32_t ValueKind, uint32_t Site, + uint64_t (*ValueMapper)(uint32_t, uint64_t)) const { + uint32_t N = getNumValueDataForSite(ValueKind, Site); + if (N == 0) + return std::unique_ptr<InstrProfValueData[]>(nullptr); + + auto VD = llvm::make_unique<InstrProfValueData[]>(N); + getValueForSite(VD.get(), ValueKind, Site, ValueMapper); + + return VD; +} + +void InstrProfRecord::getValueForSite(InstrProfValueData Dest[], + uint32_t ValueKind, uint32_t Site, + uint64_t (*ValueMapper)(uint32_t, + uint64_t)) const { + uint32_t I = 0; + for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) { + Dest[I].Value = ValueMapper ? ValueMapper(ValueKind, V.Value) : V.Value; + Dest[I].Count = V.Count; + I++; + } +} + +void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { + std::vector<InstrProfValueSiteRecord> &ValueSites = + getValueSitesForKind(ValueKind); + ValueSites.reserve(NumValueSites); +} + +inline support::endianness getHostEndianness() { + return sys::IsLittleEndianHost ? support::little : support::big; +} + +// Include definitions for value profile data +#define INSTR_PROF_VALUE_PROF_DATA +#include "llvm/ProfileData/InstrProfData.inc" + + /* + * Initialize the record for runtime value profile data. + * Return 0 if the initialization is successful, otherwise + * return 1. + */ +int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, + const uint16_t *NumValueSites, + ValueProfNode **Nodes); + +/* Release memory allocated for the runtime record. */ +void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord); + +/* Return the size of ValueProfData structure that can be used to store + the value profile data collected at runtime. */ +uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record); + +/* Return a ValueProfData instance that stores the data collected at runtime. */ +ValueProfData * +serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, + ValueProfData *Dst); + +namespace IndexedInstrProf { + +enum class HashT : uint32_t { + MD5, + + Last = MD5 +}; + +static inline uint64_t MD5Hash(StringRef Str) { + MD5 Hash; + Hash.update(Str); + llvm::MD5::MD5Result Result; + Hash.final(Result); + // Return the least significant 8 bytes. Our MD5 implementation returns the + // result in little endian, so we may need to swap bytes. + using namespace llvm::support; + return endian::read<uint64_t, little, unaligned>(Result); +} + +inline uint64_t ComputeHash(HashT Type, StringRef K) { + switch (Type) { + case HashT::MD5: + return IndexedInstrProf::MD5Hash(K); + } + llvm_unreachable("Unhandled hash type"); +} + +const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81" +const uint64_t Version = INSTR_PROF_INDEX_VERSION; +const HashT HashType = HashT::MD5; + +inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); } + +// This structure defines the file header of the LLVM profile +// data file in indexed-format. +struct Header { + uint64_t Magic; + uint64_t Version; + uint64_t MaxFunctionCount; + uint64_t HashType; + uint64_t HashOffset; +}; + +} // end namespace IndexedInstrProf + +namespace RawInstrProf { + +const uint64_t Version = INSTR_PROF_RAW_VERSION; + +template <class IntPtrT> inline uint64_t getMagic(); +template <> inline uint64_t getMagic<uint64_t>() { + return INSTR_PROF_RAW_MAGIC_64; +} + +template <> inline uint64_t getMagic<uint32_t>() { + return INSTR_PROF_RAW_MAGIC_32; +} + +// Per-function profile data header/control structure. +// The definition should match the structure defined in +// compiler-rt/lib/profile/InstrProfiling.h. +// It should also match the synthesized type in +// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters. +template <class IntPtrT> struct LLVM_ALIGNAS(8) ProfileData { + #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; + #include "llvm/ProfileData/InstrProfData.inc" }; +// File header structure of the LLVM profile data in raw format. +// The definition should match the header referenced in +// compiler-rt/lib/profile/InstrProfilingFile.c and +// InstrProfilingBuffer.c. +struct Header { +#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name; +#include "llvm/ProfileData/InstrProfData.inc" +}; + +} // end namespace RawInstrProf + +namespace coverage { + +// Profile coverage map has the following layout: +// [CoverageMapFileHeader] +// [ArrayStart] +// [CovMapFunctionRecord] +// [CovMapFunctionRecord] +// ... +// [ArrayEnd] +// [Encoded Region Mapping Data] +LLVM_PACKED_START +template <class IntPtrT> struct CovMapFunctionRecord { + #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name; + #include "llvm/ProfileData/InstrProfData.inc" +}; +LLVM_PACKED_END + +} + } // end namespace llvm namespace std { diff --git a/include/llvm/ProfileData/InstrProfData.inc b/include/llvm/ProfileData/InstrProfData.inc new file mode 100644 index 000000000000..48dae506cabb --- /dev/null +++ b/include/llvm/ProfileData/InstrProfData.inc @@ -0,0 +1,735 @@ +/*===-- InstrProfData.inc - instr profiling runtime structures -----------=== *\ +|* +|* The LLVM Compiler Infrastructure +|* +|* This file is distributed under the University of Illinois Open Source +|* License. See LICENSE.TXT for details. +|* +\*===----------------------------------------------------------------------===*/ +/* + * This is the master file that defines all the data structure, signature, + * constant literals that are shared across profiling runtime library, + * compiler (instrumentation), and host tools (reader/writer). The entities + * defined in this file affect the profile runtime ABI, the raw profile format, + * or both. + * + * The file has two identical copies. The master copy lives in LLVM and + * the other one sits in compiler-rt/lib/profile directory. To make changes + * in this file, first modify the master copy and copy it over to compiler-rt. + * Testing of any change in this file can start only after the two copies are + * synced up. + * + * The first part of the file includes macros that defines types, names, and + * initializers for the member fields of the core data structures. The field + * declarations for one structure is enabled by defining the field activation + * macro associated with that structure. Only one field activation record + * can be defined at one time and the rest definitions will be filtered out by + * the preprocessor. + * + * Examples of how the template is used to instantiate structure definition: + * 1. To declare a structure: + * + * struct ProfData { + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * Type Name; + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * 2. To construct LLVM type arrays for the struct type: + * + * Type *DataTypes[] = { + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * LLVMType, + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * 4. To construct constant array for the initializers: + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * Initializer, + * Constant *ConstantVals[] = { + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * + * The second part of the file includes definitions all other entities that + * are related to runtime ABI and format. When no field activation macro is + * defined, this file can be included to introduce the definitions. + * +\*===----------------------------------------------------------------------===*/ + +/* INSTR_PROF_DATA start. */ +/* Definition of member fields of the per-function control structure. */ +#ifndef INSTR_PROF_DATA +#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif + +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ + NamePtr->getType()->getPointerElementType()->getArrayNumElements())) +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) +INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ + ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + Inc->getHash()->getZExtValue())) +INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), NamePtr, \ + ConstantExpr::getBitCast(NamePtr, llvm::Type::getInt8PtrTy(Ctx))) +INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ + ConstantExpr::getBitCast(CounterPtr, \ + llvm::Type::getInt64PtrTy(Ctx))) +INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ + FunctionAddr) +INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ + ConstantPointerNull::get(Int8PtrTy)) +INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) +#undef INSTR_PROF_DATA +/* INSTR_PROF_DATA end. */ + +/* INSTR_PROF_RAW_HEADER start */ +/* Definition of member fields of the raw profile header data structure. */ +#ifndef INSTR_PROF_RAW_HEADER +#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) +INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) +INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) +INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) +INSTR_PROF_RAW_HEADER(uint64_t, ValueDataSize, ValueDataSize) +INSTR_PROF_RAW_HEADER(uint64_t, ValueDataDelta, (uintptr_t)ValueDataBegin) +#undef INSTR_PROF_RAW_HEADER +/* INSTR_PROF_RAW_HEADER end */ + +/* VALUE_PROF_FUNC_PARAM start */ +/* Definition of parameter types of the runtime API used to do value profiling + * for a given value site. + */ +#ifndef VALUE_PROF_FUNC_PARAM +#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) +#define INSTR_PROF_COMMA +#else +#define INSTR_PROF_DATA_DEFINED +#define INSTR_PROF_COMMA , +#endif +VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) +#undef VALUE_PROF_FUNC_PARAM +#undef INSTR_PROF_COMMA +/* VALUE_PROF_FUNC_PARAM end */ + +/* VALUE_PROF_KIND start */ +#ifndef VALUE_PROF_KIND +#define VALUE_PROF_KIND(Enumerator, Value) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +/* These two kinds must be the last to be + * declared. This is to make sure the string + * array created with the template can be + * indexed with the kind value. + */ +VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) +VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget) + +#undef VALUE_PROF_KIND +/* VALUE_PROF_KIND end */ + +/* COVMAP_FUNC_RECORD start */ +/* Definition of member fields of the function record structure in coverage + * map. + */ +#ifndef COVMAP_FUNC_RECORD +#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ + NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ + llvm::Type::getInt8PtrTy(Ctx))) +COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ + llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ + NameValue.size())) +COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ + llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ + CoverageMapping.size())) +COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ + llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) +#undef COVMAP_FUNC_RECORD +/* COVMAP_FUNC_RECORD end. */ + + +#ifdef INSTR_PROF_VALUE_PROF_DATA +#define INSTR_PROF_DATA_DEFINED + +/*! + * This is the header of the data structure that defines the on-disk + * layout of the value profile data of a particular kind for one function. + */ +typedef struct ValueProfRecord { + /* The kind of the value profile record. */ + uint32_t Kind; + /* + * The number of value profile sites. It is guaranteed to be non-zero; + * otherwise the record for this kind won't be emitted. + */ + uint32_t NumValueSites; + /* + * The first element of the array that stores the number of profiled + * values for each value site. The size of the array is NumValueSites. + * Since NumValueSites is greater than zero, there is at least one + * element in the array. + */ + uint8_t SiteCountArray[1]; + + /* + * The fake declaration is for documentation purpose only. + * Align the start of next field to be on 8 byte boundaries. + uint8_t Padding[X]; + */ + + /* The array of value profile data. The size of the array is the sum + * of all elements in SiteCountArray[]. + InstrProfValueData ValueData[]; + */ + +#ifdef __cplusplus + /*! + * \brief Return the number of value sites. + */ + uint32_t getNumValueSites() const { return NumValueSites; } + /*! + * \brief Read data from this record and save it to Record. + */ + void deserializeTo(InstrProfRecord &Record, + InstrProfRecord::ValueMapType *VMap); + /* + * In-place byte swap: + * Do byte swap for this instance. \c Old is the original order before + * the swap, and \c New is the New byte order. + */ + void swapBytes(support::endianness Old, support::endianness New); +#endif +} ValueProfRecord; + +/*! + * Per-function header/control data structure for value profiling + * data in indexed format. + */ +typedef struct ValueProfData { + /* + * Total size in bytes including this field. It must be a multiple + * of sizeof(uint64_t). + */ + uint32_t TotalSize; + /* + *The number of value profile kinds that has value profile data. + * In this implementation, a value profile kind is considered to + * have profile data if the number of value profile sites for the + * kind is not zero. More aggressively, the implementation can + * choose to check the actual data value: if none of the value sites + * has any profiled values, the kind can be skipped. + */ + uint32_t NumValueKinds; + + /* + * Following are a sequence of variable length records. The prefix/header + * of each record is defined by ValueProfRecord type. The number of + * records is NumValueKinds. + * ValueProfRecord Record_1; + * ValueProfRecord Record_N; + */ + +#if __cplusplus + /*! + * Return the total size in bytes of the on-disk value profile data + * given the data stored in Record. + */ + static uint32_t getSize(const InstrProfRecord &Record); + /*! + * Return a pointer to \c ValueProfData instance ready to be streamed. + */ + static std::unique_ptr<ValueProfData> + serializeFrom(const InstrProfRecord &Record); + /*! + * Check the integrity of the record. Return the error code when + * an error is detected, otherwise return instrprof_error::success. + */ + instrprof_error checkIntegrity(); + /*! + * Return a pointer to \c ValueProfileData instance ready to be read. + * All data in the instance are properly byte swapped. The input + * data is assumed to be in little endian order. + */ + static ErrorOr<std::unique_ptr<ValueProfData>> + getValueProfData(const unsigned char *SrcBuffer, + const unsigned char *const SrcBufferEnd, + support::endianness SrcDataEndianness); + /*! + * Swap byte order from \c Endianness order to host byte order. + */ + void swapBytesToHost(support::endianness Endianness); + /*! + * Swap byte order from host byte order to \c Endianness order. + */ + void swapBytesFromHost(support::endianness Endianness); + /*! + * Return the total size of \c ValueProfileData. + */ + uint32_t getSize() const { return TotalSize; } + /*! + * Read data from this data and save it to \c Record. + */ + void deserializeTo(InstrProfRecord &Record, + InstrProfRecord::ValueMapType *VMap); + void operator delete(void *ptr) { ::operator delete(ptr); } +#endif +} ValueProfData; + +/* + * The closure is designed to abstact away two types of value profile data: + * - InstrProfRecord which is the primary data structure used to + * represent profile data in host tools (reader, writer, and profile-use) + * - value profile runtime data structure suitable to be used by C + * runtime library. + * + * Both sources of data need to serialize to disk/memory-buffer in common + * format: ValueProfData. The abstraction allows compiler-rt's raw profiler + * writer to share the same format and code with indexed profile writer. + * + * For documentation of the member methods below, refer to corresponding methods + * in class InstrProfRecord. + */ +typedef struct ValueProfRecordClosure { + const void *Record; + uint32_t (*GetNumValueKinds)(const void *Record); + uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); + uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); + uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); + + /* + * After extracting the value profile data from the value profile record, + * this method is used to map the in-memory value to on-disk value. If + * the method is null, value will be written out untranslated. + */ + uint64_t (*RemapValueData)(uint32_t, uint64_t Value); + void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, + uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)); + ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); +} ValueProfRecordClosure; + +/* + * A wrapper struct that represents value profile runtime data. + * Like InstrProfRecord class which is used by profiling host tools, + * ValueProfRuntimeRecord also implements the abstract intefaces defined in + * ValueProfRecordClosure so that the runtime data can be serialized using + * shared C implementation. In this structure, NumValueSites and Nodes + * members are the primary fields while other fields hold the derived + * information for fast implementation of closure interfaces. + */ +typedef struct ValueProfRuntimeRecord { + /* Number of sites for each value profile kind. */ + const uint16_t *NumValueSites; + /* An array of linked-list headers. The size of of the array is the + * total number of value profile sites : sum(NumValueSites[*])). Each + * linked-list stores the values profiled for a value profile site. */ + ValueProfNode **Nodes; + + /* Total number of value profile kinds which have at least one + * value profile sites. */ + uint32_t NumValueKinds; + /* An array recording the number of values tracked at each site. + * The size of the array is TotalNumValueSites. */ + uint8_t *SiteCountArray[IPVK_Last + 1]; + ValueProfNode **NodesKind[IPVK_Last + 1]; +} ValueProfRuntimeRecord; + +/* Forward declarations of C interfaces. */ +int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, + const uint16_t *NumValueSites, + ValueProfNode **Nodes); +void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord); +uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record); +ValueProfData * +serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, + ValueProfData *Dst); +uint32_t getNumValueKindsRT(const void *R); + +#undef INSTR_PROF_VALUE_PROF_DATA +#endif /* INSTR_PROF_VALUE_PROF_DATA */ + + +#ifdef INSTR_PROF_COMMON_API_IMPL +#define INSTR_PROF_DATA_DEFINED +#ifdef __cplusplus +#define INSTR_PROF_INLINE inline +#else +#define INSTR_PROF_INLINE +#endif + +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +/*! + * \brief Return the \c ValueProfRecord header size including the + * padding bytes. + */ +INSTR_PROF_INLINE +uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { + uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + + sizeof(uint8_t) * NumValueSites; + /* Round the size to multiple of 8 bytes. */ + Size = (Size + 7) & ~7; + return Size; +} + +/*! + * \brief Return the total size of the value profile record including the + * header and the value data. + */ +INSTR_PROF_INLINE +uint32_t getValueProfRecordSize(uint32_t NumValueSites, + uint32_t NumValueData) { + return getValueProfRecordHeaderSize(NumValueSites) + + sizeof(InstrProfValueData) * NumValueData; +} + +/*! + * \brief Return the pointer to the start of value data array. + */ +INSTR_PROF_INLINE +InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { + return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( + This->NumValueSites)); +} + +/*! + * \brief Return the total number of value data for \c This record. + */ +INSTR_PROF_INLINE +uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { + uint32_t NumValueData = 0; + uint32_t I; + for (I = 0; I < This->NumValueSites; I++) + NumValueData += This->SiteCountArray[I]; + return NumValueData; +} + +/*! + * \brief Use this method to advance to the next \c This \c ValueProfRecord. + */ +INSTR_PROF_INLINE +ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { + uint32_t NumValueData = getValueProfRecordNumValueData(This); + return (ValueProfRecord *)((char *)This + + getValueProfRecordSize(This->NumValueSites, + NumValueData)); +} + +/*! + * \brief Return the first \c ValueProfRecord instance. + */ +INSTR_PROF_INLINE +ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { + return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); +} + +/* Closure based interfaces. */ + +/*! + * Return the total size in bytes of the on-disk value profile data + * given the data stored in Record. + */ +uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) { + uint32_t Kind; + uint32_t TotalSize = sizeof(ValueProfData); + const void *Record = Closure->Record; + uint32_t NumValueKinds = Closure->GetNumValueKinds(Record); + if (NumValueKinds == 0) + return TotalSize; + + for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { + uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); + if (!NumValueSites) + continue; + TotalSize += getValueProfRecordSize(NumValueSites, + Closure->GetNumValueData(Record, Kind)); + } + return TotalSize; +} + +/*! + * Extract value profile data of a function for the profile kind \c ValueKind + * from the \c Closure and serialize the data into \c This record instance. + */ +void serializeValueProfRecordFrom(ValueProfRecord *This, + ValueProfRecordClosure *Closure, + uint32_t ValueKind, uint32_t NumValueSites) { + uint32_t S; + const void *Record = Closure->Record; + This->Kind = ValueKind; + This->NumValueSites = NumValueSites; + InstrProfValueData *DstVD = getValueProfRecordValueData(This); + + for (S = 0; S < NumValueSites; S++) { + uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); + This->SiteCountArray[S] = ND; + Closure->GetValueForSite(Record, DstVD, ValueKind, S, + Closure->RemapValueData); + DstVD += ND; + } +} + +/*! + * Extract value profile data of a function from the \c Closure + * and serialize the data into \c DstData if it is not NULL or heap + * memory allocated by the \c Closure's allocator method. + */ +ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure, + ValueProfData *DstData) { + uint32_t Kind; + uint32_t TotalSize = getValueProfDataSize(Closure); + + ValueProfData *VPD = + DstData ? DstData : Closure->AllocValueProfData(TotalSize); + + VPD->TotalSize = TotalSize; + VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); + ValueProfRecord *VR = getFirstValueProfRecord(VPD); + for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { + uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); + if (!NumValueSites) + continue; + serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); + VR = getValueProfRecordNext(VR); + } + return VPD; +} + +/* + * The value profiler runtime library stores the value profile data + * for a given function in \c NumValueSites and \c Nodes structures. + * \c ValueProfRuntimeRecord class is used to encapsulate the runtime + * profile data and provides fast interfaces to retrieve the profile + * information. This interface is used to initialize the runtime record + * and pre-compute the information needed for efficient implementation + * of callbacks required by ValueProfRecordClosure class. + */ +int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, + const uint16_t *NumValueSites, + ValueProfNode **Nodes) { + unsigned I, J, S = 0, NumValueKinds = 0; + RuntimeRecord->NumValueSites = NumValueSites; + RuntimeRecord->Nodes = Nodes; + for (I = 0; I <= IPVK_Last; I++) { + uint16_t N = NumValueSites[I]; + if (!N) { + RuntimeRecord->SiteCountArray[I] = 0; + continue; + } + NumValueKinds++; + RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1); + if (!RuntimeRecord->SiteCountArray[I]) + return 1; + RuntimeRecord->NodesKind[I] = Nodes ? &Nodes[S] : NULL; + for (J = 0; J < N; J++) { + /* Compute value count for each site. */ + uint32_t C = 0; + ValueProfNode *Site = Nodes ? RuntimeRecord->NodesKind[I][J] : NULL; + while (Site) { + C++; + Site = Site->Next; + } + if (C > UCHAR_MAX) + C = UCHAR_MAX; + RuntimeRecord->SiteCountArray[I][J] = C; + } + S += N; + } + RuntimeRecord->NumValueKinds = NumValueKinds; + return 0; +} + +void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) { + unsigned I; + for (I = 0; I <= IPVK_Last; I++) { + if (RuntimeRecord->SiteCountArray[I]) + free(RuntimeRecord->SiteCountArray[I]); + } +} + +/* ValueProfRecordClosure Interface implementation for + * ValueProfDataRuntimeRecord. */ +uint32_t getNumValueKindsRT(const void *R) { + return ((const ValueProfRuntimeRecord *)R)->NumValueKinds; +} + +uint32_t getNumValueSitesRT(const void *R, uint32_t VK) { + return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK]; +} + +uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) { + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + return Record->SiteCountArray[VK][S]; +} + +uint32_t getNumValueDataRT(const void *R, uint32_t VK) { + unsigned I, S = 0; + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + if (Record->SiteCountArray[VK] == 0) + return 0; + for (I = 0; I < Record->NumValueSites[VK]; I++) + S += Record->SiteCountArray[VK][I]; + return S; +} + +void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK, + uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) { + unsigned I, N = 0; + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + N = getNumValueDataForSiteRT(R, VK, S); + if (N == 0) + return; + ValueProfNode *VNode = Record->NodesKind[VK][S]; + for (I = 0; I < N; I++) { + Dst[I] = VNode->VData; + VNode = VNode->Next; + } +} + +ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) { + return (ValueProfData *)calloc(TotalSizeInBytes, 1); +} + +static ValueProfRecordClosure RTRecordClosure = {0, + getNumValueKindsRT, + getNumValueSitesRT, + getNumValueDataRT, + getNumValueDataForSiteRT, + 0, + getValueForSiteRT, + allocValueProfDataRT}; + +/* + * Return the size of ValueProfData structure to store data + * recorded in the runtime record. + */ +uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) { + RTRecordClosure.Record = Record; + return getValueProfDataSize(&RTRecordClosure); +} + +/* + * Return a ValueProfData instance that stores the data collected + * from runtime. If \c DstData is provided by the caller, the value + * profile data will be store in *DstData and DstData is returned, + * otherwise the method will allocate space for the value data and + * return pointer to the newly allocated space. + */ +ValueProfData * +serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, + ValueProfData *DstData) { + RTRecordClosure.Record = Record; + return serializeValueProfDataFrom(&RTRecordClosure, DstData); +} + + +#undef INSTR_PROF_COMMON_API_IMPL +#endif /* INSTR_PROF_COMMON_API_IMPL */ + +/*============================================================================*/ + + +#ifndef INSTR_PROF_DATA_DEFINED + +#ifndef INSTR_PROF_DATA_INC_ +#define INSTR_PROF_DATA_INC_ + +/* Helper macros. */ +#define INSTR_PROF_SIMPLE_QUOTE(x) #x +#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) +#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y +#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) + +/* Magic number to detect file format and endianness. + * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, + * so that utilities, like strings, don't grab it as a string. 129 is also + * invalid UTF-8, and high enough to be interesting. + * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" + * for 32-bit platforms. + */ +#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ + (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ + (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 +#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ + (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ + (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 + +/* Raw profile format version. */ +#define INSTR_PROF_RAW_VERSION 2 + +/* Runtime section names and name strings. */ +#define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data +#define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names +#define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts + +#define INSTR_PROF_DATA_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) +#define INSTR_PROF_NAME_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) +#define INSTR_PROF_CNTS_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) + +/* Macros to define start/stop section symbol for a given + * section on Linux. For instance + * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will + * expand to __start___llvm_prof_data + */ +#define INSTR_PROF_SECT_START(Sect) \ + INSTR_PROF_CONCAT(__start_,Sect) +#define INSTR_PROF_SECT_STOP(Sect) \ + INSTR_PROF_CONCAT(__stop_,Sect) + +/* Value Profiling API linkage name. */ +#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target +#define INSTR_PROF_VALUE_PROF_FUNC_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) + +/* InstrProfile per-function control data alignment. */ +#define INSTR_PROF_DATA_ALIGNMENT 8 + +/* The data structure that represents a tracked value by the + * value profiler. + */ +typedef struct InstrProfValueData { + /* Profiled value. */ + uint64_t Value; + /* Number of times the value appears in the training run. */ + uint64_t Count; +} InstrProfValueData; + +/* This is an internal data structure used by value profiler. It + * is defined here to allow serialization code sharing by LLVM + * to be used in unit test. + */ +typedef struct ValueProfNode { + InstrProfValueData VData; + struct ValueProfNode *Next; +} ValueProfNode; + +#endif /* INSTR_PROF_DATA_INC_ */ + +#else +#undef INSTR_PROF_DATA_DEFINED +#endif + diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h index f937e7d08d54..fed3e693e7a0 100644 --- a/include/llvm/ProfileData/InstrProfReader.h +++ b/include/llvm/ProfileData/InstrProfReader.h @@ -23,6 +23,7 @@ #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/OnDiskHashTable.h" +#include "llvm/Support/raw_ostream.h" #include <iterator> namespace llvm { @@ -53,7 +54,7 @@ class InstrProfReader { std::error_code LastError; public: - InstrProfReader() : LastError(instrprof_error::success) {} + InstrProfReader() : LastError(instrprof_error::success), Symtab() {} virtual ~InstrProfReader() {} /// Read the header. Required before reading first record. @@ -64,7 +65,20 @@ public: InstrProfIterator begin() { return InstrProfIterator(this); } InstrProfIterator end() { return InstrProfIterator(); } + /// Return the PGO symtab. There are three different readers: + /// Raw, Text, and Indexed profile readers. The first two types + /// of readers are used only by llvm-profdata tool, while the indexed + /// profile reader is also used by llvm-cov tool and the compiler ( + /// backend or frontend). Since creating PGO symtab can create + /// significant runtime and memory overhead (as it touches data + /// for the whole program), InstrProfSymtab for the indexed profile + /// reader should be created on demand and it is recommended to be + /// only used for dumping purpose with llvm-proftool, not with the + /// compiler. + virtual InstrProfSymtab &getSymtab() = 0; + protected: + std::unique_ptr<InstrProfSymtab> Symtab; /// Set the current std::error_code and return same. std::error_code error(std::error_code EC) { LastError = EC; @@ -107,14 +121,24 @@ private: TextInstrProfReader(const TextInstrProfReader &) = delete; TextInstrProfReader &operator=(const TextInstrProfReader &) = delete; + std::error_code readValueProfileData(InstrProfRecord &Record); + public: TextInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer_) : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {} + /// Return true if the given buffer is in text instrprof format. + static bool hasFormat(const MemoryBuffer &Buffer); + /// Read the header. - std::error_code readHeader() override { return success(); } + std::error_code readHeader() override; /// Read a single record. std::error_code readNextRecord(InstrProfRecord &Record) override; + + InstrProfSymtab &getSymtab() override { + assert(Symtab.get()); + return *Symtab.get(); + } }; /// Reader for the raw instrprof binary format from runtime. @@ -129,31 +153,19 @@ class RawInstrProfReader : public InstrProfReader { private: /// The profile data file contents. std::unique_ptr<MemoryBuffer> DataBuffer; - struct ProfileData { - const uint32_t NameSize; - const uint32_t NumCounters; - const uint64_t FuncHash; - const IntPtrT NamePtr; - const IntPtrT CounterPtr; - }; - struct RawHeader { - const uint64_t Magic; - const uint64_t Version; - const uint64_t DataSize; - const uint64_t CountersSize; - const uint64_t NamesSize; - const uint64_t CountersDelta; - const uint64_t NamesDelta; - }; - bool ShouldSwapBytes; uint64_t CountersDelta; uint64_t NamesDelta; - const ProfileData *Data; - const ProfileData *DataEnd; + const RawInstrProf::ProfileData<IntPtrT> *Data; + const RawInstrProf::ProfileData<IntPtrT> *DataEnd; const uint64_t *CountersStart; const char *NamesStart; + const uint8_t *ValueDataStart; const char *ProfileEnd; + uint32_t ValueKindLast; + uint32_t CurValueDataSize; + + InstrProfRecord::ValueMapType FunctionPtrToNameMap; RawInstrProfReader(const RawInstrProfReader &) = delete; RawInstrProfReader &operator=(const RawInstrProfReader &) = delete; @@ -165,13 +177,41 @@ public: std::error_code readHeader() override; std::error_code readNextRecord(InstrProfRecord &Record) override; + InstrProfSymtab &getSymtab() override { + assert(Symtab.get()); + return *Symtab.get(); + } + private: + void createSymtab(InstrProfSymtab &Symtab); std::error_code readNextHeader(const char *CurrentPos); - std::error_code readHeader(const RawHeader &Header); - template <class IntT> - IntT swap(IntT Int) const { + std::error_code readHeader(const RawInstrProf::Header &Header); + template <class IntT> IntT swap(IntT Int) const { return ShouldSwapBytes ? sys::getSwappedBytes(Int) : Int; } + support::endianness getDataEndianness() const { + support::endianness HostEndian = getHostEndianness(); + if (!ShouldSwapBytes) + return HostEndian; + if (HostEndian == support::little) + return support::big; + else + return support::little; + } + + inline uint8_t getNumPaddingBytes(uint64_t SizeInBytes) { + return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); + } + std::error_code readName(InstrProfRecord &Record); + std::error_code readFuncHash(InstrProfRecord &Record); + std::error_code readRawCounts(InstrProfRecord &Record); + std::error_code readValueProfilingData(InstrProfRecord &Record); + bool atEnd() const { return Data == DataEnd; } + void advanceData() { + Data++; + ValueDataStart += CurValueDataSize; + } + const uint64_t *getCounter(IntPtrT CounterPtr) const { ptrdiff_t Offset = (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t); return CountersStart + Offset; @@ -195,10 +235,15 @@ class InstrProfLookupTrait { std::vector<InstrProfRecord> DataBuffer; IndexedInstrProf::HashT HashType; unsigned FormatVersion; + // Endianness of the input value profile data. + // It should be LE by default, but can be changed + // for testing purpose. + support::endianness ValueProfDataEndianness; public: InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion) - : HashType(HashType), FormatVersion(FormatVersion) {} + : HashType(HashType), FormatVersion(FormatVersion), + ValueProfDataEndianness(support::little) {} typedef ArrayRef<InstrProfRecord> data_type; @@ -209,6 +254,7 @@ public: static bool EqualKey(StringRef A, StringRef B) { return A == B; } static StringRef GetInternalKey(StringRef K) { return K; } + static StringRef GetExternalKey(StringRef K) { return K; } hash_value_type ComputeHash(StringRef K); @@ -224,11 +270,64 @@ public: return StringRef((const char *)D, N); } + bool readValueProfilingData(const unsigned char *&D, + const unsigned char *const End); data_type ReadData(StringRef K, const unsigned char *D, offset_type N); + + // Used for testing purpose only. + void setValueProfDataEndianness(support::endianness Endianness) { + ValueProfDataEndianness = Endianness; + } +}; + +struct InstrProfReaderIndexBase { + // Read all the profile records with the same key pointed to the current + // iterator. + virtual std::error_code getRecords(ArrayRef<InstrProfRecord> &Data) = 0; + // Read all the profile records with the key equal to FuncName + virtual std::error_code getRecords(StringRef FuncName, + ArrayRef<InstrProfRecord> &Data) = 0; + virtual void advanceToNextKey() = 0; + virtual bool atEnd() const = 0; + virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; + virtual ~InstrProfReaderIndexBase() {} + virtual uint64_t getVersion() const = 0; + virtual void populateSymtab(InstrProfSymtab &) = 0; }; typedef OnDiskIterableChainedHashTable<InstrProfLookupTrait> - InstrProfReaderIndex; + OnDiskHashTableImplV3; + +template <typename HashTableImpl> +class InstrProfReaderIndex : public InstrProfReaderIndexBase { + +private: + std::unique_ptr<HashTableImpl> HashTable; + typename HashTableImpl::data_iterator RecordIterator; + uint64_t FormatVersion; + +public: + InstrProfReaderIndex(const unsigned char *Buckets, + const unsigned char *const Payload, + const unsigned char *const Base, + IndexedInstrProf::HashT HashType, uint64_t Version); + + std::error_code getRecords(ArrayRef<InstrProfRecord> &Data) override; + std::error_code getRecords(StringRef FuncName, + ArrayRef<InstrProfRecord> &Data) override; + void advanceToNextKey() override { RecordIterator++; } + bool atEnd() const override { + return RecordIterator == HashTable->data_end(); + } + void setValueProfDataEndianness(support::endianness Endianness) override { + HashTable->getInfoObj().setValueProfDataEndianness(Endianness); + } + ~InstrProfReaderIndex() override {} + uint64_t getVersion() const override { return FormatVersion; } + void populateSymtab(InstrProfSymtab &Symtab) override { + Symtab.create(HashTable->keys()); + } +}; /// Reader for the indexed binary instrprof format. class IndexedInstrProfReader : public InstrProfReader { @@ -236,17 +335,15 @@ private: /// The profile data file contents. std::unique_ptr<MemoryBuffer> DataBuffer; /// The index into the profile data. - std::unique_ptr<InstrProfReaderIndex> Index; - /// Iterator over the profile data. - InstrProfReaderIndex::data_iterator RecordIterator; - /// The file format version of the profile data. - uint64_t FormatVersion; + std::unique_ptr<InstrProfReaderIndexBase> Index; /// The maximal execution count among all functions. uint64_t MaxFunctionCount; IndexedInstrProfReader(const IndexedInstrProfReader &) = delete; IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; + public: + uint64_t getVersion() const { return Index->getVersion(); } IndexedInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer) : DataBuffer(std::move(DataBuffer)), Index(nullptr) {} @@ -258,9 +355,15 @@ public: /// Read a single record. std::error_code readNextRecord(InstrProfRecord &Record) override; + /// Return the pointer to InstrProfRecord associated with FuncName + /// and FuncHash + ErrorOr<InstrProfRecord> getInstrProfRecord(StringRef FuncName, + uint64_t FuncHash); + /// Fill Counts with the profile data for the given function name. std::error_code getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector<uint64_t> &Counts); + /// Return the maximum of all known function counts. uint64_t getMaximumFunctionCount() { return MaxFunctionCount; } @@ -270,6 +373,16 @@ public: static ErrorOr<std::unique_ptr<IndexedInstrProfReader>> create(std::unique_ptr<MemoryBuffer> Buffer); + + // Used for testing purpose only. + void setValueProfDataEndianness(support::endianness Endianness) { + Index->setValueProfDataEndianness(Endianness); + } + + // See description in the base class. This interface is designed + // to be used by llvm-profdata (for dumping). Avoid using this when + // the client is the compiler. + InstrProfSymtab &getSymtab() override; }; } // end namespace llvm diff --git a/include/llvm/ProfileData/InstrProfWriter.h b/include/llvm/ProfileData/InstrProfWriter.h index ce0bb5242498..e7f53de051c3 100644 --- a/include/llvm/ProfileData/InstrProfWriter.h +++ b/include/llvm/ProfileData/InstrProfWriter.h @@ -15,38 +15,43 @@ #ifndef LLVM_PROFILEDATA_INSTRPROFWRITER_H #define LLVM_PROFILEDATA_INSTRPROFWRITER_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" -#include <vector> namespace llvm { /// Writer for instrumentation based profile data. class InstrProfWriter { public: - typedef SmallDenseMap<uint64_t, std::vector<uint64_t>, 1> CounterData; + typedef SmallDenseMap<uint64_t, InstrProfRecord, 1> ProfilingData; + private: - StringMap<CounterData> FunctionData; + StringMap<ProfilingData> FunctionData; uint64_t MaxFunctionCount; + public: InstrProfWriter() : MaxFunctionCount(0) {} /// Add function counts for the given function. If there are already counts /// for this function and the hash and number of counts match, each counter is - /// summed. - std::error_code addFunctionCounts(StringRef FunctionName, - uint64_t FunctionHash, - ArrayRef<uint64_t> Counters); + /// summed. Optionally scale counts by \p Weight. + std::error_code addRecord(InstrProfRecord &&I, uint64_t Weight = 1); /// Write the profile to \c OS void write(raw_fd_ostream &OS); + /// Write the profile in text format to \c OS + void writeText(raw_fd_ostream &OS); + /// Write \c Record in text format to \c OS + static void writeRecordInText(const InstrProfRecord &Record, + InstrProfSymtab &Symtab, raw_fd_ostream &OS); /// Write the profile, returning the raw data. For testing. std::unique_ptr<MemoryBuffer> writeBuffer(); + // Internal interface for testing purpose only. + void setValueProfDataEndianness(support::endianness Endianness); + private: std::pair<uint64_t, uint64_t> writeImpl(raw_ostream &OS); }; diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 1b82e55aa77a..8df3fe803209 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -11,14 +11,17 @@ // sample profile data. // //===----------------------------------------------------------------------===// + #ifndef LLVM_PROFILEDATA_SAMPLEPROF_H_ #define LLVM_PROFILEDATA_SAMPLEPROF_H_ -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorOr.h" #include "llvm/Support/raw_ostream.h" + +#include <map> #include <system_error> namespace llvm { @@ -32,13 +35,27 @@ enum class sampleprof_error { too_large, truncated, malformed, - unrecognized_format + unrecognized_format, + unsupported_writing_format, + truncated_name_table, + not_implemented, + counter_overflow }; inline std::error_code make_error_code(sampleprof_error E) { return std::error_code(static_cast<int>(E), sampleprof_category()); } +inline sampleprof_error MergeResult(sampleprof_error &Accumulator, + sampleprof_error Result) { + // Prefer first error encountered as later errors may be secondary effects of + // the initial problem. + if (Accumulator == sampleprof_error::success && + Result != sampleprof_error::success) + Accumulator = Result; + return Accumulator; +} + } // end namespace llvm namespace std { @@ -57,7 +74,7 @@ static inline uint64_t SPMagic() { uint64_t('2') << (64 - 56) | uint64_t(0xff); } -static inline uint64_t SPVersion() { return 100; } +static inline uint64_t SPVersion() { return 102; } /// Represents the relative location of an instruction. /// @@ -69,36 +86,36 @@ static inline uint64_t SPVersion() { return 100; } /// that are on the same line but belong to different basic blocks /// (e.g., the two post-increment instructions in "if (p) x++; else y++;"). struct LineLocation { - LineLocation(int L, unsigned D) : LineOffset(L), Discriminator(D) {} - int LineOffset; - unsigned Discriminator; + LineLocation(uint32_t L, uint32_t D) : LineOffset(L), Discriminator(D) {} + void print(raw_ostream &OS) const; + void dump() const; + bool operator<(const LineLocation &O) const { + return LineOffset < O.LineOffset || + (LineOffset == O.LineOffset && Discriminator < O.Discriminator); + } + + uint32_t LineOffset; + uint32_t Discriminator; }; -} // End namespace sampleprof +raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); -template <> struct DenseMapInfo<sampleprof::LineLocation> { - typedef DenseMapInfo<int> OffsetInfo; - typedef DenseMapInfo<unsigned> DiscriminatorInfo; - static inline sampleprof::LineLocation getEmptyKey() { - return sampleprof::LineLocation(OffsetInfo::getEmptyKey(), - DiscriminatorInfo::getEmptyKey()); - } - static inline sampleprof::LineLocation getTombstoneKey() { - return sampleprof::LineLocation(OffsetInfo::getTombstoneKey(), - DiscriminatorInfo::getTombstoneKey()); - } - static inline unsigned getHashValue(sampleprof::LineLocation Val) { - return DenseMapInfo<std::pair<int, unsigned>>::getHashValue( - std::pair<int, unsigned>(Val.LineOffset, Val.Discriminator)); - } - static inline bool isEqual(sampleprof::LineLocation LHS, - sampleprof::LineLocation RHS) { - return LHS.LineOffset == RHS.LineOffset && - LHS.Discriminator == RHS.Discriminator; - } +/// Represents the relative location of a callsite. +/// +/// Callsite locations are specified by the line offset from the +/// beginning of the function (marked by the line where the function +/// head is), the discriminator value within that line, and the callee +/// function name. +struct CallsiteLocation : public LineLocation { + CallsiteLocation(uint32_t L, uint32_t D, StringRef N) + : LineLocation(L, D), CalleeName(N) {} + void print(raw_ostream &OS) const; + void dump() const; + + StringRef CalleeName; }; -namespace sampleprof { +raw_ostream &operator<<(raw_ostream &OS, const CallsiteLocation &Loc); /// Representation of a single sample record. /// @@ -112,52 +129,79 @@ namespace sampleprof { /// will be a list of one or more functions. class SampleRecord { public: - typedef StringMap<unsigned> CallTargetMap; + typedef StringMap<uint64_t> CallTargetMap; SampleRecord() : NumSamples(0), CallTargets() {} /// Increment the number of samples for this record by \p S. + /// Optionally scale sample count \p S by \p Weight. /// /// Sample counts accumulate using saturating arithmetic, to avoid wrapping /// around unsigned integers. - void addSamples(unsigned S) { - if (NumSamples <= std::numeric_limits<unsigned>::max() - S) - NumSamples += S; - else - NumSamples = std::numeric_limits<unsigned>::max(); + sampleprof_error addSamples(uint64_t S, uint64_t Weight = 1) { + bool Overflowed; + if (Weight > 1) { + S = SaturatingMultiply(S, Weight, &Overflowed); + if (Overflowed) + return sampleprof_error::counter_overflow; + } + NumSamples = SaturatingAdd(NumSamples, S, &Overflowed); + if (Overflowed) + return sampleprof_error::counter_overflow; + + return sampleprof_error::success; } /// Add called function \p F with samples \p S. + /// Optionally scale sample count \p S by \p Weight. /// /// Sample counts accumulate using saturating arithmetic, to avoid wrapping /// around unsigned integers. - void addCalledTarget(StringRef F, unsigned S) { - unsigned &TargetSamples = CallTargets[F]; - if (TargetSamples <= std::numeric_limits<unsigned>::max() - S) - TargetSamples += S; - else - TargetSamples = std::numeric_limits<unsigned>::max(); + sampleprof_error addCalledTarget(StringRef F, uint64_t S, + uint64_t Weight = 1) { + uint64_t &TargetSamples = CallTargets[F]; + bool Overflowed; + if (Weight > 1) { + S = SaturatingMultiply(S, Weight, &Overflowed); + if (Overflowed) + return sampleprof_error::counter_overflow; + } + TargetSamples = SaturatingAdd(TargetSamples, S, &Overflowed); + if (Overflowed) + return sampleprof_error::counter_overflow; + + return sampleprof_error::success; } /// Return true if this sample record contains function calls. bool hasCalls() const { return CallTargets.size() > 0; } - unsigned getSamples() const { return NumSamples; } + uint64_t getSamples() const { return NumSamples; } const CallTargetMap &getCallTargets() const { return CallTargets; } /// Merge the samples in \p Other into this record. - void merge(const SampleRecord &Other) { - addSamples(Other.getSamples()); - for (const auto &I : Other.getCallTargets()) - addCalledTarget(I.first(), I.second); + /// Optionally scale sample counts by \p Weight. + sampleprof_error merge(const SampleRecord &Other, uint64_t Weight = 1) { + sampleprof_error Result = addSamples(Other.getSamples(), Weight); + for (const auto &I : Other.getCallTargets()) { + MergeResult(Result, addCalledTarget(I.first(), I.second, Weight)); + } + return Result; } + void print(raw_ostream &OS, unsigned Indent) const; + void dump() const; + private: - unsigned NumSamples; + uint64_t NumSamples; CallTargetMap CallTargets; }; -typedef DenseMap<LineLocation, SampleRecord> BodySampleMap; +raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample); + +typedef std::map<LineLocation, SampleRecord> BodySampleMap; +class FunctionSamples; +typedef std::map<CallsiteLocation, FunctionSamples> CallsiteSampleMap; /// Representation of the samples collected for a function. /// @@ -167,59 +211,109 @@ typedef DenseMap<LineLocation, SampleRecord> BodySampleMap; class FunctionSamples { public: FunctionSamples() : TotalSamples(0), TotalHeadSamples(0) {} - void print(raw_ostream &OS = dbgs()); - void addTotalSamples(unsigned Num) { TotalSamples += Num; } - void addHeadSamples(unsigned Num) { TotalHeadSamples += Num; } - void addBodySamples(int LineOffset, unsigned Discriminator, unsigned Num) { - assert(LineOffset >= 0); - // When dealing with instruction weights, we use the value - // zero to indicate the absence of a sample. If we read an - // actual zero from the profile file, use the value 1 to - // avoid the confusion later on. - if (Num == 0) - Num = 1; - BodySamples[LineLocation(LineOffset, Discriminator)].addSamples(Num); - } - void addCalledTargetSamples(int LineOffset, unsigned Discriminator, - std::string FName, unsigned Num) { - assert(LineOffset >= 0); - BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(FName, - Num); + void print(raw_ostream &OS = dbgs(), unsigned Indent = 0) const; + void dump() const; + sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight = 1) { + bool Overflowed; + if (Weight > 1) { + Num = SaturatingMultiply(Num, Weight, &Overflowed); + if (Overflowed) + return sampleprof_error::counter_overflow; + } + TotalSamples = SaturatingAdd(TotalSamples, Num, &Overflowed); + if (Overflowed) + return sampleprof_error::counter_overflow; + + return sampleprof_error::success; } + sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) { + bool Overflowed; + if (Weight > 1) { + Num = SaturatingMultiply(Num, Weight, &Overflowed); + if (Overflowed) + return sampleprof_error::counter_overflow; + } + TotalHeadSamples = SaturatingAdd(TotalHeadSamples, Num, &Overflowed); + if (Overflowed) + return sampleprof_error::counter_overflow; - /// Return the sample record at the given location. - /// Each location is specified by \p LineOffset and \p Discriminator. - SampleRecord &sampleRecordAt(const LineLocation &Loc) { - return BodySamples[Loc]; + return sampleprof_error::success; + } + sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator, + uint64_t Num, uint64_t Weight = 1) { + return BodySamples[LineLocation(LineOffset, Discriminator)].addSamples( + Num, Weight); + } + sampleprof_error addCalledTargetSamples(uint32_t LineOffset, + uint32_t Discriminator, + std::string FName, uint64_t Num, + uint64_t Weight = 1) { + return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget( + FName, Num, Weight); } /// Return the number of samples collected at the given location. /// Each location is specified by \p LineOffset and \p Discriminator. - unsigned samplesAt(int LineOffset, unsigned Discriminator) { - return sampleRecordAt(LineLocation(LineOffset, Discriminator)).getSamples(); + /// If the location is not found in profile, return error. + ErrorOr<uint64_t> findSamplesAt(uint32_t LineOffset, + uint32_t Discriminator) const { + const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator)); + if (ret == BodySamples.end()) + return std::error_code(); + else + return ret->second.getSamples(); } - bool empty() const { return BodySamples.empty(); } + /// Return the function samples at the given callsite location. + FunctionSamples &functionSamplesAt(const CallsiteLocation &Loc) { + return CallsiteSamples[Loc]; + } + + /// Return a pointer to function samples at the given callsite location. + const FunctionSamples * + findFunctionSamplesAt(const CallsiteLocation &Loc) const { + auto iter = CallsiteSamples.find(Loc); + if (iter == CallsiteSamples.end()) { + return nullptr; + } else { + return &iter->second; + } + } + + bool empty() const { return TotalSamples == 0; } /// Return the total number of samples collected inside the function. - unsigned getTotalSamples() const { return TotalSamples; } + uint64_t getTotalSamples() const { return TotalSamples; } /// Return the total number of samples collected at the head of the /// function. - unsigned getHeadSamples() const { return TotalHeadSamples; } + uint64_t getHeadSamples() const { return TotalHeadSamples; } /// Return all the samples collected in the body of the function. const BodySampleMap &getBodySamples() const { return BodySamples; } + /// Return all the callsite samples collected in the body of the function. + const CallsiteSampleMap &getCallsiteSamples() const { + return CallsiteSamples; + } + /// Merge the samples in \p Other into this one. - void merge(const FunctionSamples &Other) { - addTotalSamples(Other.getTotalSamples()); - addHeadSamples(Other.getHeadSamples()); + /// Optionally scale samples by \p Weight. + sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) { + sampleprof_error Result = sampleprof_error::success; + MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight)); + MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight)); for (const auto &I : Other.getBodySamples()) { const LineLocation &Loc = I.first; const SampleRecord &Rec = I.second; - sampleRecordAt(Loc).merge(Rec); + MergeResult(Result, BodySamples[Loc].merge(Rec, Weight)); + } + for (const auto &I : Other.getCallsiteSamples()) { + const CallsiteLocation &Loc = I.first; + const FunctionSamples &Rec = I.second; + MergeResult(Result, functionSamplesAt(Loc).merge(Rec, Weight)); } + return Result; } private: @@ -227,12 +321,12 @@ private: /// /// Samples are cumulative, they include all the samples collected /// inside this function and all its inlined callees. - unsigned TotalSamples; + uint64_t TotalSamples; /// Total number of samples collected at the head of the function. /// This is an approximation of the number of calls made to this function /// at runtime. - unsigned TotalHeadSamples; + uint64_t TotalHeadSamples; /// Map instruction locations to collected samples. /// @@ -240,10 +334,53 @@ private: /// collected at the corresponding line offset. All line locations /// are an offset from the start of the function. BodySampleMap BodySamples; + + /// Map call sites to collected samples for the called function. + /// + /// Each entry in this map corresponds to all the samples + /// collected for the inlined function call at the given + /// location. For example, given: + /// + /// void foo() { + /// 1 bar(); + /// ... + /// 8 baz(); + /// } + /// + /// If the bar() and baz() calls were inlined inside foo(), this + /// map will contain two entries. One for all the samples collected + /// in the call to bar() at line offset 1, the other for all the samples + /// collected in the call to baz() at line offset 8. + CallsiteSampleMap CallsiteSamples; +}; + +raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS); + +/// Sort a LocationT->SampleT map by LocationT. +/// +/// It produces a sorted list of <LocationT, SampleT> records by ascending +/// order of LocationT. +template <class LocationT, class SampleT> class SampleSorter { +public: + typedef std::pair<const LocationT, SampleT> SamplesWithLoc; + typedef SmallVector<const SamplesWithLoc *, 20> SamplesWithLocList; + + SampleSorter(const std::map<LocationT, SampleT> &Samples) { + for (const auto &I : Samples) + V.push_back(&I); + std::stable_sort(V.begin(), V.end(), + [](const SamplesWithLoc *A, const SamplesWithLoc *B) { + return A->first < B->first; + }); + } + const SamplesWithLocList &get() const { return V; } + +private: + SamplesWithLocList V; }; -} // End namespace sampleprof +} // end namespace sampleprof -} // End namespace llvm +} // end namespace llvm #endif // LLVM_PROFILEDATA_SAMPLEPROF_H_ diff --git a/include/llvm/ProfileData/SampleProfReader.h b/include/llvm/ProfileData/SampleProfReader.h index c082a1abe951..6db0fbb0e7ab 100644 --- a/include/llvm/ProfileData/SampleProfReader.h +++ b/include/llvm/ProfileData/SampleProfReader.h @@ -9,11 +9,181 @@ // // This file contains definitions needed for reading sample profiles. // +// NOTE: If you are making changes to this file format, please remember +// to document them in the Clang documentation at +// tools/clang/docs/UsersManual.rst. +// +// Text format +// ----------- +// +// Sample profiles are written as ASCII text. The file is divided into +// sections, which correspond to each of the functions executed at runtime. +// Each section has the following format +// +// function1:total_samples:total_head_samples +// offset1[.discriminator]: number_of_samples [fn1:num fn2:num ... ] +// offset2[.discriminator]: number_of_samples [fn3:num fn4:num ... ] +// ... +// offsetN[.discriminator]: number_of_samples [fn5:num fn6:num ... ] +// offsetA[.discriminator]: fnA:num_of_total_samples +// offsetA1[.discriminator]: number_of_samples [fn7:num fn8:num ... ] +// ... +// +// This is a nested tree in which the identation represents the nesting level +// of the inline stack. There are no blank lines in the file. And the spacing +// within a single line is fixed. Additional spaces will result in an error +// while reading the file. +// +// Any line starting with the '#' character is completely ignored. +// +// Inlined calls are represented with indentation. The Inline stack is a +// stack of source locations in which the top of the stack represents the +// leaf function, and the bottom of the stack represents the actual +// symbol to which the instruction belongs. +// +// Function names must be mangled in order for the profile loader to +// match them in the current translation unit. The two numbers in the +// function header specify how many total samples were accumulated in the +// function (first number), and the total number of samples accumulated +// in the prologue of the function (second number). This head sample +// count provides an indicator of how frequently the function is invoked. +// +// There are two types of lines in the function body. +// +// * Sampled line represents the profile information of a source location. +// * Callsite line represents the profile information of a callsite. +// +// Each sampled line may contain several items. Some are optional (marked +// below): +// +// a. Source line offset. This number represents the line number +// in the function where the sample was collected. The line number is +// always relative to the line where symbol of the function is +// defined. So, if the function has its header at line 280, the offset +// 13 is at line 293 in the file. +// +// Note that this offset should never be a negative number. This could +// happen in cases like macros. The debug machinery will register the +// line number at the point of macro expansion. So, if the macro was +// expanded in a line before the start of the function, the profile +// converter should emit a 0 as the offset (this means that the optimizers +// will not be able to associate a meaningful weight to the instructions +// in the macro). +// +// b. [OPTIONAL] Discriminator. This is used if the sampled program +// was compiled with DWARF discriminator support +// (http://wiki.dwarfstd.org/index.php?title=Path_Discriminators). +// DWARF discriminators are unsigned integer values that allow the +// compiler to distinguish between multiple execution paths on the +// same source line location. +// +// For example, consider the line of code ``if (cond) foo(); else bar();``. +// If the predicate ``cond`` is true 80% of the time, then the edge +// into function ``foo`` should be considered to be taken most of the +// time. But both calls to ``foo`` and ``bar`` are at the same source +// line, so a sample count at that line is not sufficient. The +// compiler needs to know which part of that line is taken more +// frequently. +// +// This is what discriminators provide. In this case, the calls to +// ``foo`` and ``bar`` will be at the same line, but will have +// different discriminator values. This allows the compiler to correctly +// set edge weights into ``foo`` and ``bar``. +// +// c. Number of samples. This is an integer quantity representing the +// number of samples collected by the profiler at this source +// location. +// +// d. [OPTIONAL] Potential call targets and samples. If present, this +// line contains a call instruction. This models both direct and +// number of samples. For example, +// +// 130: 7 foo:3 bar:2 baz:7 +// +// The above means that at relative line offset 130 there is a call +// instruction that calls one of ``foo()``, ``bar()`` and ``baz()``, +// with ``baz()`` being the relatively more frequently called target. +// +// Each callsite line may contain several items. Some are optional. +// +// a. Source line offset. This number represents the line number of the +// callsite that is inlined in the profiled binary. +// +// b. [OPTIONAL] Discriminator. Same as the discriminator for sampled line. +// +// c. Number of samples. This is an integer quantity representing the +// total number of samples collected for the inlined instance at this +// callsite +// +// +// Binary format +// ------------- +// +// This is a more compact encoding. Numbers are encoded as ULEB128 values +// and all strings are encoded in a name table. The file is organized in +// the following sections: +// +// MAGIC (uint64_t) +// File identifier computed by function SPMagic() (0x5350524f463432ff) +// +// VERSION (uint32_t) +// File format version number computed by SPVersion() +// +// NAME TABLE +// SIZE (uint32_t) +// Number of entries in the name table. +// NAMES +// A NUL-separated list of SIZE strings. +// +// FUNCTION BODY (one for each uninlined function body present in the profile) +// HEAD_SAMPLES (uint64_t) [only for top-level functions] +// Total number of samples collected at the head (prologue) of the +// function. +// NOTE: This field should only be present for top-level functions +// (i.e., not inlined into any caller). Inlined function calls +// have no prologue, so they don't need this. +// NAME_IDX (uint32_t) +// Index into the name table indicating the function name. +// SAMPLES (uint64_t) +// Total number of samples collected in this function. +// NRECS (uint32_t) +// Total number of sampling records this function's profile. +// BODY RECORDS +// A list of NRECS entries. Each entry contains: +// OFFSET (uint32_t) +// Line offset from the start of the function. +// DISCRIMINATOR (uint32_t) +// Discriminator value (see description of discriminators +// in the text format documentation above). +// SAMPLES (uint64_t) +// Number of samples collected at this location. +// NUM_CALLS (uint32_t) +// Number of non-inlined function calls made at this location. In the +// case of direct calls, this number will always be 1. For indirect +// calls (virtual functions and function pointers) this will +// represent all the actual functions called at runtime. +// CALL_TARGETS +// A list of NUM_CALLS entries for each called function: +// NAME_IDX (uint32_t) +// Index into the name table with the callee name. +// SAMPLES (uint64_t) +// Number of samples collected at the call site. +// NUM_INLINED_FUNCTIONS (uint32_t) +// Number of callees inlined into this function. +// INLINED FUNCTION RECORDS +// A list of NUM_INLINED_FUNCTIONS entries describing each of the inlined +// callees. +// OFFSET (uint32_t) +// Line offset from the start of the function. +// DISCRIMINATOR (uint32_t) +// Discriminator value (see description of discriminators +// in the text format documentation above). +// FUNCTION BODY +// A FUNCTION BODY entry describing the inlined function. //===----------------------------------------------------------------------===// #ifndef LLVM_PROFILEDATA_SAMPLEPROFREADER_H #define LLVM_PROFILEDATA_SAMPLEPROFREADER_H -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" @@ -24,6 +194,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ErrorOr.h" +#include "llvm/Support/GCOV.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" @@ -57,7 +228,7 @@ namespace sampleprof { /// /// The reader supports two file formats: text and binary. The text format /// is useful for debugging and testing, while the binary format is more -/// compact. They can both be used interchangeably. +/// compact and I/O efficient. They can both be used interchangeably. class SampleProfileReader { public: SampleProfileReader(std::unique_ptr<MemoryBuffer> B, LLVMContext &C) @@ -86,7 +257,7 @@ public: StringMap<FunctionSamples> &getProfiles() { return Profiles; } /// \brief Report a parse error message. - void reportParseError(int64_t LineNumber, Twine Msg) const { + void reportError(int64_t LineNumber, Twine Msg) const { Ctx.diagnose(DiagnosticInfoSampleProfile(Buffer->getBufferIdentifier(), LineNumber, Msg)); } @@ -95,6 +266,10 @@ public: static ErrorOr<std::unique_ptr<SampleProfileReader>> create(StringRef Filename, LLVMContext &C); + /// \brief Create a sample profile reader from the supplied memory buffer. + static ErrorOr<std::unique_ptr<SampleProfileReader>> + create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C); + protected: /// \brief Map every function to its associated profile. /// @@ -120,6 +295,9 @@ public: /// \brief Read sample profiles from the associated file. std::error_code read() override; + + /// \brief Return true if \p Buffer is in the format supported by this class. + static bool hasFormat(const MemoryBuffer &Buffer); }; class SampleProfileReaderBinary : public SampleProfileReader { @@ -153,14 +331,75 @@ protected: /// \returns the read value. ErrorOr<StringRef> readString(); + /// Read a string indirectly via the name table. + ErrorOr<StringRef> readStringFromTable(); + /// \brief Return true if we've reached the end of file. bool at_eof() const { return Data >= End; } + /// Read the contents of the given profile instance. + std::error_code readProfile(FunctionSamples &FProfile); + /// \brief Points to the current location in the buffer. const uint8_t *Data; /// \brief Points to the end of the buffer. const uint8_t *End; + + /// Function name table. + std::vector<StringRef> NameTable; +}; + +typedef SmallVector<FunctionSamples *, 10> InlineCallStack; + +// Supported histogram types in GCC. Currently, we only need support for +// call target histograms. +enum HistType { + HIST_TYPE_INTERVAL, + HIST_TYPE_POW2, + HIST_TYPE_SINGLE_VALUE, + HIST_TYPE_CONST_DELTA, + HIST_TYPE_INDIR_CALL, + HIST_TYPE_AVERAGE, + HIST_TYPE_IOR, + HIST_TYPE_INDIR_CALL_TOPN +}; + +class SampleProfileReaderGCC : public SampleProfileReader { +public: + SampleProfileReaderGCC(std::unique_ptr<MemoryBuffer> B, LLVMContext &C) + : SampleProfileReader(std::move(B), C), GcovBuffer(Buffer.get()) {} + + /// \brief Read and validate the file header. + std::error_code readHeader() override; + + /// \brief Read sample profiles from the associated file. + std::error_code read() override; + + /// \brief Return true if \p Buffer is in the format supported by this class. + static bool hasFormat(const MemoryBuffer &Buffer); + +protected: + std::error_code readNameTable(); + std::error_code readOneFunctionProfile(const InlineCallStack &InlineStack, + bool Update, uint32_t Offset); + std::error_code readFunctionProfiles(); + std::error_code skipNextWord(); + template <typename T> ErrorOr<T> readNumber(); + ErrorOr<StringRef> readString(); + + /// \brief Read the section tag and check that it's the same as \p Expected. + std::error_code readSectionTag(uint32_t Expected); + + /// GCOV buffer containing the profile. + GCOVBuffer GcovBuffer; + + /// Function names in this profile. + std::vector<std::string> Names; + + /// GCOV tags used to separate sections in the profile file. + static const uint32_t GCOVTagAFDOFileNames = 0xaa000000; + static const uint32_t GCOVTagAFDOFunction = 0xac000000; }; } // End namespace sampleprof diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h index 302a82d32861..029dd2ebacb0 100644 --- a/include/llvm/ProfileData/SampleProfWriter.h +++ b/include/llvm/ProfileData/SampleProfWriter.h @@ -13,9 +13,8 @@ #ifndef LLVM_PROFILEDATA_SAMPLEPROFWRITER_H #define LLVM_PROFILEDATA_SAMPLEPROFWRITER_H +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringRef.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Module.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" @@ -30,77 +29,102 @@ enum SampleProfileFormat { SPF_None = 0, SPF_Text, SPF_Binary, SPF_GCC }; /// \brief Sample-based profile writer. Base class. class SampleProfileWriter { public: - SampleProfileWriter(StringRef Filename, std::error_code &EC, - sys::fs::OpenFlags Flags) - : OS(Filename, EC, Flags) {} virtual ~SampleProfileWriter() {} - /// \brief Write sample profiles in \p S for function \p FName. + /// Write sample profiles in \p S for function \p FName. /// - /// \returns true if the file was updated successfully. False, otherwise. - virtual bool write(StringRef FName, const FunctionSamples &S) = 0; + /// \returns status code of the file update operation. + virtual std::error_code write(StringRef FName, const FunctionSamples &S) = 0; - /// \brief Write sample profiles in \p S for function \p F. - bool write(const Function &F, const FunctionSamples &S) { - return write(F.getName(), S); - } - - /// \brief Write all the sample profiles for all the functions in \p M. + /// Write all the sample profiles in the given map of samples. /// - /// \returns true if the file was updated successfully. False, otherwise. - bool write(const Module &M, StringMap<FunctionSamples> &P) { - for (const auto &F : M) { - StringRef Name = F.getName(); - if (!write(Name, P[Name])) - return false; - } - return true; - } + /// \returns status code of the file update operation. + std::error_code write(const StringMap<FunctionSamples> &ProfileMap) { + if (std::error_code EC = writeHeader(ProfileMap)) + return EC; - /// \brief Write all the sample profiles in the given map of samples. - /// - /// \returns true if the file was updated successfully. False, otherwise. - bool write(StringMap<FunctionSamples> &ProfileMap) { - for (auto &I : ProfileMap) { + for (const auto &I : ProfileMap) { StringRef FName = I.first(); - FunctionSamples &Profile = I.second; - if (!write(FName, Profile)) - return false; + const FunctionSamples &Profile = I.second; + if (std::error_code EC = write(FName, Profile)) + return EC; } - return true; + return sampleprof_error::success; } - /// \brief Profile writer factory. Create a new writer based on the value of - /// \p Format. + raw_ostream &getOutputStream() { return *OutputStream; } + + /// Profile writer factory. + /// + /// Create a new file writer based on the value of \p Format. static ErrorOr<std::unique_ptr<SampleProfileWriter>> create(StringRef Filename, SampleProfileFormat Format); + /// Create a new stream writer based on the value of \p Format. + /// For testing. + static ErrorOr<std::unique_ptr<SampleProfileWriter>> + create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format); + protected: + SampleProfileWriter(std::unique_ptr<raw_ostream> &OS) + : OutputStream(std::move(OS)) {} + + /// \brief Write a file header for the profile file. + virtual std::error_code + writeHeader(const StringMap<FunctionSamples> &ProfileMap) = 0; + /// \brief Output stream where to emit the profile to. - raw_fd_ostream OS; + std::unique_ptr<raw_ostream> OutputStream; }; /// \brief Sample-based profile writer (text format). class SampleProfileWriterText : public SampleProfileWriter { public: - SampleProfileWriterText(StringRef F, std::error_code &EC) - : SampleProfileWriter(F, EC, sys::fs::F_Text) {} + std::error_code write(StringRef FName, const FunctionSamples &S) override; - bool write(StringRef FName, const FunctionSamples &S) override; - bool write(const Module &M, StringMap<FunctionSamples> &P) { - return SampleProfileWriter::write(M, P); +protected: + SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS) + : SampleProfileWriter(OS), Indent(0) {} + + std::error_code + writeHeader(const StringMap<FunctionSamples> &ProfileMap) override { + return sampleprof_error::success; } + +private: + /// Indent level to use when writing. + /// + /// This is used when printing inlined callees. + unsigned Indent; + + friend ErrorOr<std::unique_ptr<SampleProfileWriter>> + SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, + SampleProfileFormat Format); }; /// \brief Sample-based profile writer (binary format). class SampleProfileWriterBinary : public SampleProfileWriter { public: - SampleProfileWriterBinary(StringRef F, std::error_code &EC); + std::error_code write(StringRef F, const FunctionSamples &S) override; - bool write(StringRef F, const FunctionSamples &S) override; - bool write(const Module &M, StringMap<FunctionSamples> &P) { - return SampleProfileWriter::write(M, P); - } +protected: + SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS) + : SampleProfileWriter(OS), NameTable() {} + + std::error_code + writeHeader(const StringMap<FunctionSamples> &ProfileMap) override; + std::error_code writeNameIdx(StringRef FName); + std::error_code writeBody(StringRef FName, const FunctionSamples &S); + +private: + void addName(StringRef FName); + void addNames(const FunctionSamples &S); + + MapVector<StringRef, uint32_t> NameTable; + + friend ErrorOr<std::unique_ptr<SampleProfileWriter>> + SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, + SampleProfileFormat Format); }; } // End namespace sampleprof diff --git a/include/llvm/Support/ARMTargetParser.def b/include/llvm/Support/ARMTargetParser.def new file mode 100644 index 000000000000..2f99b0717adf --- /dev/null +++ b/include/llvm/Support/ARMTargetParser.def @@ -0,0 +1,223 @@ +//===- ARMTargetParser.def - ARM target parsing defines ---------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provides defines to build up the ARM target parser's logic. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef ARM_FPU +#define ARM_FPU(NAME, KIND, VERSION, NEON_SUPPORT, RESTRICTION) +#endif +ARM_FPU("invalid", FK_INVALID, FV_NONE, NS_None, FR_None) +ARM_FPU("none", FK_NONE, FV_NONE, NS_None, FR_None) +ARM_FPU("vfp", FK_VFP, FV_VFPV2, NS_None, FR_None) +ARM_FPU("vfpv2", FK_VFPV2, FV_VFPV2, NS_None, FR_None) +ARM_FPU("vfpv3", FK_VFPV3, FV_VFPV3, NS_None, FR_None) +ARM_FPU("vfpv3-fp16", FK_VFPV3_FP16, FV_VFPV3_FP16, NS_None, FR_None) +ARM_FPU("vfpv3-d16", FK_VFPV3_D16, FV_VFPV3, NS_None, FR_D16) +ARM_FPU("vfpv3-d16-fp16", FK_VFPV3_D16_FP16, FV_VFPV3_FP16, NS_None, FR_D16) +ARM_FPU("vfpv3xd", FK_VFPV3XD, FV_VFPV3, NS_None, FR_SP_D16) +ARM_FPU("vfpv3xd-fp16", FK_VFPV3XD_FP16, FV_VFPV3_FP16, NS_None, FR_SP_D16) +ARM_FPU("vfpv4", FK_VFPV4, FV_VFPV4, NS_None, FR_None) +ARM_FPU("vfpv4-d16", FK_VFPV4_D16, FV_VFPV4, NS_None, FR_D16) +ARM_FPU("fpv4-sp-d16", FK_FPV4_SP_D16, FV_VFPV4, NS_None, FR_SP_D16) +ARM_FPU("fpv5-d16", FK_FPV5_D16, FV_VFPV5, NS_None, FR_D16) +ARM_FPU("fpv5-sp-d16", FK_FPV5_SP_D16, FV_VFPV5, NS_None, FR_SP_D16) +ARM_FPU("fp-armv8", FK_FP_ARMV8, FV_VFPV5, NS_None, FR_None) +ARM_FPU("neon", FK_NEON, FV_VFPV3, NS_Neon, FR_None) +ARM_FPU("neon-fp16", FK_NEON_FP16, FV_VFPV3_FP16, NS_Neon, FR_None) +ARM_FPU("neon-vfpv4", FK_NEON_VFPV4, FV_VFPV4, NS_Neon, FR_None) +ARM_FPU("neon-fp-armv8", FK_NEON_FP_ARMV8, FV_VFPV5, NS_Neon, FR_None) +ARM_FPU("crypto-neon-fp-armv8", FK_CRYPTO_NEON_FP_ARMV8, FV_VFPV5, NS_Crypto, + FR_None) +ARM_FPU("softvfp", FK_SOFTVFP, FV_NONE, NS_None, FR_None) +#undef ARM_FPU + +#ifndef ARM_ARCH +#define ARM_ARCH(NAME, ID, CPU_ATTR, SUB_ARCH, ARCH_ATTR, ARCH_FPU, ARCH_BASE_EXT) +#endif +ARM_ARCH("invalid", AK_INVALID, nullptr, nullptr, + ARMBuildAttrs::CPUArch::Pre_v4, FK_NONE, AEK_NONE) +ARM_ARCH("armv2", AK_ARMV2, "2", "v2", ARMBuildAttrs::CPUArch::Pre_v4, + FK_NONE, AEK_NONE) +ARM_ARCH("armv2a", AK_ARMV2A, "2A", "v2a", ARMBuildAttrs::CPUArch::Pre_v4, + FK_NONE, AEK_NONE) +ARM_ARCH("armv3", AK_ARMV3, "3", "v3", ARMBuildAttrs::CPUArch::Pre_v4, + FK_NONE, AEK_NONE) +ARM_ARCH("armv3m", AK_ARMV3M, "3M", "v3m", ARMBuildAttrs::CPUArch::Pre_v4, + FK_NONE, AEK_NONE) +ARM_ARCH("armv4", AK_ARMV4, "4", "v4", ARMBuildAttrs::CPUArch::v4, + FK_NONE, AEK_NONE) +ARM_ARCH("armv4t", AK_ARMV4T, "4T", "v4t", ARMBuildAttrs::CPUArch::v4T, + FK_NONE, AEK_NONE) +ARM_ARCH("armv5t", AK_ARMV5T, "5T", "v5", ARMBuildAttrs::CPUArch::v5T, + FK_NONE, AEK_NONE) +ARM_ARCH("armv5te", AK_ARMV5TE, "5TE", "v5e", ARMBuildAttrs::CPUArch::v5TE, + FK_NONE, AEK_DSP) +ARM_ARCH("armv5tej", AK_ARMV5TEJ, "5TEJ", "v5e", ARMBuildAttrs::CPUArch::v5TEJ, + FK_NONE, AEK_DSP) +ARM_ARCH("armv6", AK_ARMV6, "6", "v6", ARMBuildAttrs::CPUArch::v6, + FK_VFPV2, AEK_DSP) +ARM_ARCH("armv6k", AK_ARMV6K, "6K", "v6k", ARMBuildAttrs::CPUArch::v6K, + FK_VFPV2, AEK_DSP) +ARM_ARCH("armv6t2", AK_ARMV6T2, "6T2", "v6t2", ARMBuildAttrs::CPUArch::v6T2, + FK_NONE, AEK_DSP) +ARM_ARCH("armv6kz", AK_ARMV6KZ, "6KZ", "v6kz", ARMBuildAttrs::CPUArch::v6KZ, + FK_VFPV2, (AEK_SEC | AEK_DSP)) +ARM_ARCH("armv6-m", AK_ARMV6M, "6-M", "v6m", ARMBuildAttrs::CPUArch::v6_M, + FK_NONE, AEK_NONE) +ARM_ARCH("armv7-a", AK_ARMV7A, "7-A", "v7", ARMBuildAttrs::CPUArch::v7, + FK_NEON, AEK_DSP) +ARM_ARCH("armv7-r", AK_ARMV7R, "7-R", "v7r", ARMBuildAttrs::CPUArch::v7, + FK_NONE, (AEK_HWDIV | AEK_DSP)) +ARM_ARCH("armv7-m", AK_ARMV7M, "7-M", "v7m", ARMBuildAttrs::CPUArch::v7, + FK_NONE, AEK_HWDIV) +ARM_ARCH("armv7e-m", AK_ARMV7EM, "7E-M", "v7em", ARMBuildAttrs::CPUArch::v7E_M, + FK_NONE, (AEK_HWDIV | AEK_DSP)) +ARM_ARCH("armv8-a", AK_ARMV8A, "8-A", "v8", ARMBuildAttrs::CPUArch::v8, + FK_CRYPTO_NEON_FP_ARMV8, (AEK_SEC | AEK_MP | AEK_VIRT | AEK_HWDIVARM | + AEK_HWDIV | AEK_DSP | AEK_CRC)) +ARM_ARCH("armv8.1-a", AK_ARMV8_1A, "8.1-A", "v8.1a", ARMBuildAttrs::CPUArch::v8, + FK_CRYPTO_NEON_FP_ARMV8, (AEK_SEC | AEK_MP | AEK_VIRT | AEK_HWDIVARM | + AEK_HWDIV | AEK_DSP | AEK_CRC)) +ARM_ARCH("armv8.2-a", AK_ARMV8_2A, "8.2-A", "v8.2a", ARMBuildAttrs::CPUArch::v8, + FK_CRYPTO_NEON_FP_ARMV8, (AEK_SEC | AEK_MP | AEK_VIRT | AEK_HWDIVARM | + AEK_HWDIV | AEK_DSP | AEK_CRC)) +// Non-standard Arch names. +ARM_ARCH("iwmmxt", AK_IWMMXT, "iwmmxt", "", ARMBuildAttrs::CPUArch::v5TE, + FK_NONE, AEK_NONE) +ARM_ARCH("iwmmxt2", AK_IWMMXT2, "iwmmxt2", "", ARMBuildAttrs::CPUArch::v5TE, + FK_NONE, AEK_NONE) +ARM_ARCH("xscale", AK_XSCALE, "xscale", "", ARMBuildAttrs::CPUArch::v5TE, + FK_NONE, AEK_NONE) +ARM_ARCH("armv7s", AK_ARMV7S, "7-S", "v7s", ARMBuildAttrs::CPUArch::v7, + FK_NEON_VFPV4, AEK_DSP) +ARM_ARCH("armv7k", AK_ARMV7K, "7-K", "v7k", ARMBuildAttrs::CPUArch::v7, + FK_NONE, AEK_DSP) +#undef ARM_ARCH + +#ifndef ARM_ARCH_EXT_NAME +#define ARM_ARCH_EXT_NAME(NAME, ID, FEATURE, NEGFEATURE) +#endif +// FIXME: This would be nicer were it tablegen +ARM_ARCH_EXT_NAME("invalid", AEK_INVALID, nullptr, nullptr) +ARM_ARCH_EXT_NAME("none", AEK_NONE, nullptr, nullptr) +ARM_ARCH_EXT_NAME("crc", AEK_CRC, "+crc", "-crc") +ARM_ARCH_EXT_NAME("crypto", AEK_CRYPTO, "+crypto","-crypto") +ARM_ARCH_EXT_NAME("fp", AEK_FP, nullptr, nullptr) +ARM_ARCH_EXT_NAME("idiv", (AEK_HWDIVARM | AEK_HWDIV), nullptr, nullptr) +ARM_ARCH_EXT_NAME("mp", AEK_MP, nullptr, nullptr) +ARM_ARCH_EXT_NAME("simd", AEK_SIMD, nullptr, nullptr) +ARM_ARCH_EXT_NAME("sec", AEK_SEC, nullptr, nullptr) +ARM_ARCH_EXT_NAME("virt", AEK_VIRT, nullptr, nullptr) +ARM_ARCH_EXT_NAME("fp16", AEK_FP16, "+fullfp16", "-fullfp16") +ARM_ARCH_EXT_NAME("os", AEK_OS, nullptr, nullptr) +ARM_ARCH_EXT_NAME("iwmmxt", AEK_IWMMXT, nullptr, nullptr) +ARM_ARCH_EXT_NAME("iwmmxt2", AEK_IWMMXT2, nullptr, nullptr) +ARM_ARCH_EXT_NAME("maverick", AEK_MAVERICK, nullptr, nullptr) +ARM_ARCH_EXT_NAME("xscale", AEK_XSCALE, nullptr, nullptr) +#undef ARM_ARCH_EXT_NAME + +#ifndef ARM_HW_DIV_NAME +#define ARM_HW_DIV_NAME(NAME, ID) +#endif +ARM_HW_DIV_NAME("invalid", AEK_INVALID) +ARM_HW_DIV_NAME("none", AEK_NONE) +ARM_HW_DIV_NAME("thumb", AEK_HWDIV) +ARM_HW_DIV_NAME("arm", AEK_HWDIVARM) +ARM_HW_DIV_NAME("arm,thumb", (AEK_HWDIVARM | AEK_HWDIV)) +#undef ARM_HW_DIV_NAME + +#ifndef ARM_CPU_NAME +#define ARM_CPU_NAME(NAME, ID, DEFAULT_FPU, IS_DEFAULT, DEFAULT_EXT) +#endif +ARM_CPU_NAME("arm2", AK_ARMV2, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm3", AK_ARMV2A, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm6", AK_ARMV3, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm7m", AK_ARMV3M, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm8", AK_ARMV4, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm810", AK_ARMV4, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("strongarm", AK_ARMV4, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("strongarm110", AK_ARMV4, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("strongarm1100", AK_ARMV4, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("strongarm1110", AK_ARMV4, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm7tdmi", AK_ARMV4T, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm7tdmi-s", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm710t", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm720t", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm9", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm9tdmi", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm920", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm920t", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm922t", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm9312", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm940t", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("ep9312", AK_ARMV4T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm10tdmi", AK_ARMV5T, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm1020t", AK_ARMV5T, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm9e", AK_ARMV5TE, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm946e-s", AK_ARMV5TE, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm966e-s", AK_ARMV5TE, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm968e-s", AK_ARMV5TE, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm10e", AK_ARMV5TE, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm1020e", AK_ARMV5TE, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm1022e", AK_ARMV5TE, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm926ej-s", AK_ARMV5TEJ, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm1136j-s", AK_ARMV6, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm1136jf-s", AK_ARMV6, FK_VFPV2, true, AEK_NONE) +ARM_CPU_NAME("arm1136jz-s", AK_ARMV6, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm1176j-s", AK_ARMV6K, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm1176jz-s", AK_ARMV6KZ, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("mpcore", AK_ARMV6K, FK_VFPV2, false, AEK_NONE) +ARM_CPU_NAME("mpcorenovfp", AK_ARMV6K, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("arm1176jzf-s", AK_ARMV6KZ, FK_VFPV2, true, AEK_NONE) +ARM_CPU_NAME("arm1156t2-s", AK_ARMV6T2, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("arm1156t2f-s", AK_ARMV6T2, FK_VFPV2, false, AEK_NONE) +ARM_CPU_NAME("cortex-m0", AK_ARMV6M, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("cortex-m0plus", AK_ARMV6M, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("cortex-m1", AK_ARMV6M, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("sc000", AK_ARMV6M, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("cortex-a5", AK_ARMV7A, FK_NEON_VFPV4, false, (AEK_SEC | AEK_MP)) +ARM_CPU_NAME("cortex-a7", AK_ARMV7A, FK_NEON_VFPV4, false, + (AEK_SEC | AEK_MP | AEK_VIRT | AEK_HWDIVARM | AEK_HWDIV)) +ARM_CPU_NAME("cortex-a8", AK_ARMV7A, FK_NEON, true, AEK_SEC) +ARM_CPU_NAME("cortex-a9", AK_ARMV7A, FK_NEON_FP16, false, (AEK_SEC | AEK_MP)) +ARM_CPU_NAME("cortex-a12", AK_ARMV7A, FK_NEON_VFPV4, false, + (AEK_SEC | AEK_MP | AEK_VIRT | AEK_HWDIVARM | AEK_HWDIV)) +ARM_CPU_NAME("cortex-a15", AK_ARMV7A, FK_NEON_VFPV4, false, + (AEK_SEC | AEK_MP | AEK_VIRT | AEK_HWDIVARM | AEK_HWDIV)) +ARM_CPU_NAME("cortex-a17", AK_ARMV7A, FK_NEON_VFPV4, false, + (AEK_SEC | AEK_MP | AEK_VIRT | AEK_HWDIVARM | AEK_HWDIV)) +ARM_CPU_NAME("krait", AK_ARMV7A, FK_NEON_VFPV4, false, + (AEK_HWDIVARM | AEK_HWDIV)) +ARM_CPU_NAME("cortex-r4", AK_ARMV7R, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("cortex-r4f", AK_ARMV7R, FK_VFPV3_D16, false, AEK_NONE) +ARM_CPU_NAME("cortex-r5", AK_ARMV7R, FK_VFPV3_D16, false, + (AEK_MP | AEK_HWDIVARM)) +ARM_CPU_NAME("cortex-r7", AK_ARMV7R, FK_VFPV3_D16_FP16, false, + (AEK_MP | AEK_HWDIVARM)) +ARM_CPU_NAME("sc300", AK_ARMV7M, FK_NONE, false, AEK_NONE) +ARM_CPU_NAME("cortex-m3", AK_ARMV7M, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("cortex-m4", AK_ARMV7EM, FK_FPV4_SP_D16, true, AEK_NONE) +ARM_CPU_NAME("cortex-m7", AK_ARMV7EM, FK_FPV5_D16, false, AEK_NONE) +ARM_CPU_NAME("cortex-a35", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, AEK_CRC) +ARM_CPU_NAME("cortex-a53", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, true, AEK_CRC) +ARM_CPU_NAME("cortex-a57", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, AEK_CRC) +ARM_CPU_NAME("cortex-a72", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, AEK_CRC) +ARM_CPU_NAME("cyclone", AK_ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, AEK_CRC) +// Non-standard Arch names. +ARM_CPU_NAME("iwmmxt", AK_IWMMXT, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("xscale", AK_XSCALE, FK_NONE, true, AEK_NONE) +ARM_CPU_NAME("swift", AK_ARMV7S, FK_NEON_VFPV4, true, + (AEK_HWDIVARM | AEK_HWDIV)) +// Invalid CPU +ARM_CPU_NAME("invalid", AK_INVALID, FK_INVALID, true, AEK_INVALID) +#undef ARM_CPU_NAME diff --git a/include/llvm/Support/AlignOf.h b/include/llvm/Support/AlignOf.h index 07da02d063c7..5268c8d16986 100644 --- a/include/llvm/Support/AlignOf.h +++ b/include/llvm/Support/AlignOf.h @@ -17,9 +17,15 @@ #include "llvm/Support/Compiler.h" #include <cstddef> +#include <type_traits> namespace llvm { -template <typename T> + +namespace detail { + +// For everything other than an abstract class we can calulate alignment by +// building a class with a single character and a member of the given type. +template <typename T, bool = std::is_abstract<T>::value> struct AlignmentCalcImpl { char x; #if defined(_MSC_VER) @@ -35,6 +41,25 @@ private: AlignmentCalcImpl() {} // Never instantiate. }; +// Abstract base class helper, this will have the minimal alignment and size +// for any abstract class. We don't even define its destructor because this +// type should never be used in a way that requires it. +struct AlignmentCalcImplBase { + virtual ~AlignmentCalcImplBase() = 0; +}; + +// When we have an abstract class type, specialize the alignment computation +// engine to create another abstract class that derives from both an empty +// abstract base class and the provided type. This has the same effect as the +// above except that it handles the fact that we can't actually create a member +// of type T. +template <typename T> +struct AlignmentCalcImpl<T, true> : AlignmentCalcImplBase, T { + virtual ~AlignmentCalcImpl() = 0; +}; + +} // End detail namespace. + /// AlignOf - A templated class that contains an enum value representing /// the alignment of the template argument. For example, /// AlignOf<int>::Alignment represents the alignment of type "int". The @@ -50,11 +75,13 @@ struct AlignOf { // llvm::AlignOf<Y>::<anonymous>' [-Wenum-compare] // by using constexpr instead of enum. // (except on MSVC, since it doesn't support constexpr yet). - static constexpr unsigned Alignment = - static_cast<unsigned int>(sizeof(AlignmentCalcImpl<T>) - sizeof(T)); + static constexpr unsigned Alignment = static_cast<unsigned int>( + sizeof(detail::AlignmentCalcImpl<T>) - sizeof(T)); #else - enum { Alignment = - static_cast<unsigned int>(sizeof(AlignmentCalcImpl<T>) - sizeof(T)) }; + enum { + Alignment = static_cast<unsigned int>(sizeof(detail::AlignmentCalcImpl<T>) - + sizeof(T)) + }; #endif enum { Alignment_GreaterEqual_2Bytes = Alignment >= 2 ? 1 : 0 }; enum { Alignment_GreaterEqual_4Bytes = Alignment >= 4 ? 1 : 0 }; diff --git a/include/llvm/Support/Allocator.h b/include/llvm/Support/Allocator.h index f9b5cf22f97d..c608736fa956 100644 --- a/include/llvm/Support/Allocator.h +++ b/include/llvm/Support/Allocator.h @@ -222,6 +222,8 @@ public: // Without this, MemorySanitizer messages for values originated from here // will point to the allocation of the entire slab. __msan_allocated_memory(AlignedPtr, Size); + // Similarly, tell ASan about this space. + __asan_unpoison_memory_region(AlignedPtr, Size); return AlignedPtr; } @@ -229,12 +231,16 @@ public: size_t PaddedSize = Size + Alignment - 1; if (PaddedSize > SizeThreshold) { void *NewSlab = Allocator.Allocate(PaddedSize, 0); + // We own the new slab and don't want anyone reading anyting other than + // pieces returned from this method. So poison the whole slab. + __asan_poison_memory_region(NewSlab, PaddedSize); CustomSizedSlabs.push_back(std::make_pair(NewSlab, PaddedSize)); uintptr_t AlignedAddr = alignAddr(NewSlab, Alignment); assert(AlignedAddr + Size <= (uintptr_t)NewSlab + PaddedSize); char *AlignedPtr = (char*)AlignedAddr; __msan_allocated_memory(AlignedPtr, Size); + __asan_unpoison_memory_region(AlignedPtr, Size); return AlignedPtr; } @@ -246,13 +252,16 @@ public: char *AlignedPtr = (char*)AlignedAddr; CurPtr = AlignedPtr + Size; __msan_allocated_memory(AlignedPtr, Size); + __asan_unpoison_memory_region(AlignedPtr, Size); return AlignedPtr; } // Pull in base class overloads. using AllocatorBase<BumpPtrAllocatorImpl>::Allocate; - void Deallocate(const void * /*Ptr*/, size_t /*Size*/) {} + void Deallocate(const void *Ptr, size_t Size) { + __asan_poison_memory_region(Ptr, Size); + } // Pull in base class overloads. using AllocatorBase<BumpPtrAllocatorImpl>::Deallocate; @@ -310,6 +319,10 @@ private: size_t AllocatedSlabSize = computeSlabSize(Slabs.size()); void *NewSlab = Allocator.Allocate(AllocatedSlabSize, 0); + // We own the new slab and don't want anyone reading anything other than + // pieces returned from this method. So poison the whole slab. + __asan_poison_memory_region(NewSlab, AllocatedSlabSize); + Slabs.push_back(NewSlab); CurPtr = (char *)(NewSlab); End = ((char *)NewSlab) + AllocatedSlabSize; diff --git a/include/llvm/Support/BlockFrequency.h b/include/llvm/Support/BlockFrequency.h index 4304a253b287..1b45cc52973f 100644 --- a/include/llvm/Support/BlockFrequency.h +++ b/include/llvm/Support/BlockFrequency.h @@ -14,12 +14,12 @@ #ifndef LLVM_SUPPORT_BLOCKFREQUENCY_H #define LLVM_SUPPORT_BLOCKFREQUENCY_H +#include "llvm/Support/BranchProbability.h" #include "llvm/Support/DataTypes.h" namespace llvm { class raw_ostream; -class BranchProbability; // This class represents Block Frequency as a 64-bit value. class BlockFrequency { @@ -37,34 +37,38 @@ public: /// \brief Multiplies with a branch probability. The computation will never /// overflow. - BlockFrequency &operator*=(const BranchProbability &Prob); - const BlockFrequency operator*(const BranchProbability &Prob) const; + BlockFrequency &operator*=(BranchProbability Prob); + BlockFrequency operator*(BranchProbability Prob) const; /// \brief Divide by a non-zero branch probability using saturating /// arithmetic. - BlockFrequency &operator/=(const BranchProbability &Prob); - BlockFrequency operator/(const BranchProbability &Prob) const; + BlockFrequency &operator/=(BranchProbability Prob); + BlockFrequency operator/(BranchProbability Prob) const; /// \brief Adds another block frequency using saturating arithmetic. - BlockFrequency &operator+=(const BlockFrequency &Freq); - const BlockFrequency operator+(const BlockFrequency &Freq) const; + BlockFrequency &operator+=(BlockFrequency Freq); + BlockFrequency operator+(BlockFrequency Freq) const; + + /// \brief Subtracts another block frequency using saturating arithmetic. + BlockFrequency &operator-=(BlockFrequency Freq); + BlockFrequency operator-(BlockFrequency Freq) const; /// \brief Shift block frequency to the right by count digits saturating to 1. BlockFrequency &operator>>=(const unsigned count); - bool operator<(const BlockFrequency &RHS) const { + bool operator<(BlockFrequency RHS) const { return Frequency < RHS.Frequency; } - bool operator<=(const BlockFrequency &RHS) const { + bool operator<=(BlockFrequency RHS) const { return Frequency <= RHS.Frequency; } - bool operator>(const BlockFrequency &RHS) const { + bool operator>(BlockFrequency RHS) const { return Frequency > RHS.Frequency; } - bool operator>=(const BlockFrequency &RHS) const { + bool operator>=(BlockFrequency RHS) const { return Frequency >= RHS.Frequency; } }; diff --git a/include/llvm/Support/BranchProbability.h b/include/llvm/Support/BranchProbability.h index a6429dd22a3b..26bc888d1cab 100644 --- a/include/llvm/Support/BranchProbability.h +++ b/include/llvm/Support/BranchProbability.h @@ -15,36 +15,59 @@ #define LLVM_SUPPORT_BRANCHPROBABILITY_H #include "llvm/Support/DataTypes.h" +#include <algorithm> #include <cassert> +#include <climits> +#include <numeric> namespace llvm { class raw_ostream; -// This class represents Branch Probability as a non-negative fraction. +// This class represents Branch Probability as a non-negative fraction that is +// no greater than 1. It uses a fixed-point-like implementation, in which the +// denominator is always a constant value (here we use 1<<31 for maximum +// precision). class BranchProbability { // Numerator uint32_t N; - // Denominator - uint32_t D; + // Denominator, which is a constant value. + static const uint32_t D = 1u << 31; + static const uint32_t UnknownN = UINT32_MAX; -public: - BranchProbability(uint32_t n, uint32_t d) : N(n), D(d) { - assert(d > 0 && "Denominator cannot be 0!"); - assert(n <= d && "Probability cannot be bigger than 1!"); - } + // Construct a BranchProbability with only numerator assuming the denominator + // is 1<<31. For internal use only. + explicit BranchProbability(uint32_t n) : N(n) {} - static BranchProbability getZero() { return BranchProbability(0, 1); } - static BranchProbability getOne() { return BranchProbability(1, 1); } +public: + BranchProbability() : N(UnknownN) {} + BranchProbability(uint32_t Numerator, uint32_t Denominator); + + bool isZero() const { return N == 0; } + bool isUnknown() const { return N == UnknownN; } + + static BranchProbability getZero() { return BranchProbability(0); } + static BranchProbability getOne() { return BranchProbability(D); } + static BranchProbability getUnknown() { return BranchProbability(UnknownN); } + // Create a BranchProbability object with the given numerator and 1<<31 + // as denominator. + static BranchProbability getRaw(uint32_t N) { return BranchProbability(N); } + // Create a BranchProbability object from 64-bit integers. + static BranchProbability getBranchProbability(uint64_t Numerator, + uint64_t Denominator); + + // Normalize given probabilties so that the sum of them becomes approximate + // one. + template <class ProbabilityIter> + static void normalizeProbabilities(ProbabilityIter Begin, + ProbabilityIter End); uint32_t getNumerator() const { return N; } - uint32_t getDenominator() const { return D; } + static uint32_t getDenominator() { return D; } // Return (1 - Probability). - BranchProbability getCompl() const { - return BranchProbability(D - N, D); - } + BranchProbability getCompl() const { return BranchProbability(D - N); } raw_ostream &print(raw_ostream &OS) const; @@ -66,24 +89,131 @@ public: /// \return \c Num divided by \c this. uint64_t scaleByInverse(uint64_t Num) const; - bool operator==(BranchProbability RHS) const { - return (uint64_t)N * RHS.D == (uint64_t)D * RHS.N; + BranchProbability &operator+=(BranchProbability RHS) { + assert(N != UnknownN && RHS.N != UnknownN && + "Unknown probability cannot participate in arithmetics."); + // Saturate the result in case of overflow. + N = (uint64_t(N) + RHS.N > D) ? D : N + RHS.N; + return *this; + } + + BranchProbability &operator-=(BranchProbability RHS) { + assert(N != UnknownN && RHS.N != UnknownN && + "Unknown probability cannot participate in arithmetics."); + // Saturate the result in case of underflow. + N = N < RHS.N ? 0 : N - RHS.N; + return *this; + } + + BranchProbability &operator*=(BranchProbability RHS) { + assert(N != UnknownN && RHS.N != UnknownN && + "Unknown probability cannot participate in arithmetics."); + N = (static_cast<uint64_t>(N) * RHS.N + D / 2) / D; + return *this; + } + + BranchProbability &operator/=(uint32_t RHS) { + assert(N != UnknownN && + "Unknown probability cannot participate in arithmetics."); + assert(RHS > 0 && "The divider cannot be zero."); + N /= RHS; + return *this; + } + + BranchProbability operator+(BranchProbability RHS) const { + BranchProbability Prob(*this); + return Prob += RHS; + } + + BranchProbability operator-(BranchProbability RHS) const { + BranchProbability Prob(*this); + return Prob -= RHS; + } + + BranchProbability operator*(BranchProbability RHS) const { + BranchProbability Prob(*this); + return Prob *= RHS; } - bool operator!=(BranchProbability RHS) const { - return !(*this == RHS); + + BranchProbability operator/(uint32_t RHS) const { + BranchProbability Prob(*this); + return Prob /= RHS; } + + bool operator==(BranchProbability RHS) const { return N == RHS.N; } + bool operator!=(BranchProbability RHS) const { return !(*this == RHS); } + bool operator<(BranchProbability RHS) const { - return (uint64_t)N * RHS.D < (uint64_t)D * RHS.N; + assert(N != UnknownN && RHS.N != UnknownN && + "Unknown probability cannot participate in comparisons."); + return N < RHS.N; + } + + bool operator>(BranchProbability RHS) const { + assert(N != UnknownN && RHS.N != UnknownN && + "Unknown probability cannot participate in comparisons."); + return RHS < *this; + } + + bool operator<=(BranchProbability RHS) const { + assert(N != UnknownN && RHS.N != UnknownN && + "Unknown probability cannot participate in comparisons."); + return !(RHS < *this); + } + + bool operator>=(BranchProbability RHS) const { + assert(N != UnknownN && RHS.N != UnknownN && + "Unknown probability cannot participate in comparisons."); + return !(*this < RHS); } - bool operator>(BranchProbability RHS) const { return RHS < *this; } - bool operator<=(BranchProbability RHS) const { return !(RHS < *this); } - bool operator>=(BranchProbability RHS) const { return !(*this < RHS); } }; -inline raw_ostream &operator<<(raw_ostream &OS, const BranchProbability &Prob) { +inline raw_ostream &operator<<(raw_ostream &OS, BranchProbability Prob) { return Prob.print(OS); } +template <class ProbabilityIter> +void BranchProbability::normalizeProbabilities(ProbabilityIter Begin, + ProbabilityIter End) { + if (Begin == End) + return; + + unsigned UnknownProbCount = 0; + uint64_t Sum = std::accumulate(Begin, End, uint64_t(0), + [&](uint64_t S, const BranchProbability &BP) { + if (!BP.isUnknown()) + return S + BP.N; + UnknownProbCount++; + return S; + }); + + if (UnknownProbCount > 0) { + BranchProbability ProbForUnknown = BranchProbability::getZero(); + // If the sum of all known probabilities is less than one, evenly distribute + // the complement of sum to unknown probabilities. Otherwise, set unknown + // probabilities to zeros and continue to normalize known probabilities. + if (Sum < BranchProbability::getDenominator()) + ProbForUnknown = BranchProbability::getRaw( + (BranchProbability::getDenominator() - Sum) / UnknownProbCount); + + std::replace_if(Begin, End, + [](const BranchProbability &BP) { return BP.isUnknown(); }, + ProbForUnknown); + + if (Sum <= BranchProbability::getDenominator()) + return; + } + + if (Sum == 0) { + BranchProbability BP(1, std::distance(Begin, End)); + std::fill(Begin, End, BP); + return; + } + + for (auto I = Begin; I != End; ++I) + I->N = (I->N * uint64_t(D) + Sum / 2) / Sum; +} + } #endif diff --git a/include/llvm/Support/CBindingWrapping.h b/include/llvm/Support/CBindingWrapping.h index 786ba183b3b0..d4633aa7d3c6 100644 --- a/include/llvm/Support/CBindingWrapping.h +++ b/include/llvm/Support/CBindingWrapping.h @@ -15,6 +15,7 @@ #define LLVM_SUPPORT_CBINDINGWRAPPING_H #include "llvm/Support/Casting.h" +#include "llvm-c/Types.h" #define DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ty, ref) \ inline ty *unwrap(ref P) { \ diff --git a/include/llvm/Support/COFF.h b/include/llvm/Support/COFF.h index 3c5ee06969d0..0162175efe3e 100644 --- a/include/llvm/Support/COFF.h +++ b/include/llvm/Support/COFF.h @@ -88,6 +88,7 @@ namespace COFF { IMAGE_FILE_MACHINE_AMD64 = 0x8664, IMAGE_FILE_MACHINE_ARM = 0x1C0, IMAGE_FILE_MACHINE_ARMNT = 0x1C4, + IMAGE_FILE_MACHINE_ARM64 = 0xAA64, IMAGE_FILE_MACHINE_EBC = 0xEBC, IMAGE_FILE_MACHINE_I386 = 0x14C, IMAGE_FILE_MACHINE_IA64 = 0x200, @@ -247,6 +248,7 @@ namespace COFF { enum SectionCharacteristics : uint32_t { SC_Invalid = 0xffffffff, + IMAGE_SCN_TYPE_NOLOAD = 0x00000002, IMAGE_SCN_TYPE_NO_PAD = 0x00000008, IMAGE_SCN_CNT_CODE = 0x00000020, IMAGE_SCN_CNT_INITIALIZED_DATA = 0x00000040, diff --git a/include/llvm/Support/CommandLine.h b/include/llvm/Support/CommandLine.h index 379d06a65741..943d2df37708 100644 --- a/include/llvm/Support/CommandLine.h +++ b/include/llvm/Support/CommandLine.h @@ -33,7 +33,6 @@ namespace llvm { -class BumpPtrStringSaver; class StringSaver; /// cl Namespace - This namespace contains all of the command line option @@ -206,9 +205,9 @@ class Option { unsigned AdditionalVals; // Greater than 0 for multi-valued option. public: - const char *ArgStr; // The argument string itself (ex: "help", "o") - const char *HelpStr; // The descriptive text message for -help - const char *ValueStr; // String describing what the value of this option is + StringRef ArgStr; // The argument string itself (ex: "help", "o") + StringRef HelpStr; // The descriptive text message for -help + StringRef ValueStr; // String describing what the value of this option is OptionCategory *Category; // The Category this option belongs to bool FullyInitialized; // Has addArguemnt been called? @@ -229,14 +228,14 @@ public: inline unsigned getNumAdditionalVals() const { return AdditionalVals; } // hasArgStr - Return true if the argstr != "" - bool hasArgStr() const { return ArgStr[0] != 0; } + bool hasArgStr() const { return !ArgStr.empty(); } //-------------------------------------------------------------------------=== // Accessor functions set by OptionModifiers // - void setArgStr(const char *S); - void setDescription(const char *S) { HelpStr = S; } - void setValueStr(const char *S) { ValueStr = S; } + void setArgStr(StringRef S); + void setDescription(StringRef S) { HelpStr = S; } + void setValueStr(StringRef S) { ValueStr = S; } void setNumOccurrencesFlag(enum NumOccurrencesFlag Val) { Occurrences = Val; } void setValueExpectedFlag(enum ValueExpected Val) { Value = Val; } void setHiddenFlag(enum OptionHidden Val) { HiddenFlag = Val; } @@ -276,7 +275,7 @@ public: virtual void printOptionValue(size_t GlobalWidth, bool Force) const = 0; - virtual void getExtraOptionNames(SmallVectorImpl<const char *> &) {} + virtual void getExtraOptionNames(SmallVectorImpl<StringRef> &) {} // addOccurrence - Wrapper around handleOccurrence that enforces Flags. // @@ -606,7 +605,7 @@ public: void initialize() {} - void getExtraOptionNames(SmallVectorImpl<const char *> &OptionNames) { + void getExtraOptionNames(SmallVectorImpl<StringRef> &OptionNames) { // If there has been no argstr specified, that means that we need to add an // argument for every possible option. This ensures that our options are // vectored to us. @@ -715,14 +714,14 @@ public: // class basic_parser_impl { // non-template implementation of basic_parser<t> public: - basic_parser_impl(Option &O) {} + basic_parser_impl(Option &) {} enum ValueExpected getValueExpectedFlagDefault() const { return ValueRequired; } - void getExtraOptionNames(SmallVectorImpl<const char *> &) {} + void getExtraOptionNames(SmallVectorImpl<StringRef> &) {} void initialize() {} @@ -1206,8 +1205,7 @@ class opt : public Option, enum ValueExpected getValueExpectedFlagDefault() const override { return Parser.getValueExpectedFlagDefault(); } - void - getExtraOptionNames(SmallVectorImpl<const char *> &OptionNames) override { + void getExtraOptionNames(SmallVectorImpl<StringRef> &OptionNames) override { return Parser.getExtraOptionNames(OptionNames); } @@ -1368,8 +1366,7 @@ class list : public Option, public list_storage<DataType, StorageClass> { enum ValueExpected getValueExpectedFlagDefault() const override { return Parser.getValueExpectedFlagDefault(); } - void - getExtraOptionNames(SmallVectorImpl<const char *> &OptionNames) override { + void getExtraOptionNames(SmallVectorImpl<StringRef> &OptionNames) override { return Parser.getExtraOptionNames(OptionNames); } @@ -1508,8 +1505,7 @@ class bits : public Option, public bits_storage<DataType, Storage> { enum ValueExpected getValueExpectedFlagDefault() const override { return Parser.getValueExpectedFlagDefault(); } - void - getExtraOptionNames(SmallVectorImpl<const char *> &OptionNames) override { + void getExtraOptionNames(SmallVectorImpl<StringRef> &OptionNames) override { return Parser.getExtraOptionNames(OptionNames); } diff --git a/include/llvm/Support/Compiler.h b/include/llvm/Support/Compiler.h index 141639839cc2..b3416bbfffb6 100644 --- a/include/llvm/Support/Compiler.h +++ b/include/llvm/Support/Compiler.h @@ -69,7 +69,7 @@ #if !defined(_MSC_VER) || defined(__clang__) || LLVM_MSC_PREREQ(1900) #define LLVM_NOEXCEPT noexcept #else -#define LLVM_NOEXCEPT +#define LLVM_NOEXCEPT throw() #endif /// \brief Does the compiler support ref-qualifiers for *this? @@ -189,7 +189,7 @@ /// 3.4 supported this but is buggy in various cases and produces unimplemented /// errors, just use it in GCC 4.0 and later. #if __has_attribute(always_inline) || LLVM_GNUC_PREREQ(4, 0, 0) -#define LLVM_ATTRIBUTE_ALWAYS_INLINE inline __attribute__((always_inline)) +#define LLVM_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline)) #elif defined(_MSC_VER) #define LLVM_ATTRIBUTE_ALWAYS_INLINE __forceinline #else @@ -293,6 +293,34 @@ # define LLVM_ALIGNAS(x) alignas(x) #endif +/// \macro LLVM_PACKED +/// \brief Used to specify a packed structure. +/// LLVM_PACKED( +/// struct A { +/// int i; +/// int j; +/// int k; +/// long long l; +/// }); +/// +/// LLVM_PACKED_START +/// struct B { +/// int i; +/// int j; +/// int k; +/// long long l; +/// }; +/// LLVM_PACKED_END +#ifdef _MSC_VER +# define LLVM_PACKED(d) __pragma(pack(push, 1)) d __pragma(pack(pop)) +# define LLVM_PACKED_START __pragma(pack(push, 1)) +# define LLVM_PACKED_END __pragma(pack(pop)) +#else +# define LLVM_PACKED(d) d __attribute__((packed)) +# define LLVM_PACKED_START _Pragma("pack(push, 1)") +# define LLVM_PACKED_END _Pragma("pack(pop)") +#endif + /// \macro LLVM_PTR_SIZE /// \brief A constant integer equivalent to the value of sizeof(void*). /// Generally used in combination with LLVM_ALIGNAS or when doing computation in @@ -333,8 +361,50 @@ /// \brief Whether LLVM itself is built with AddressSanitizer instrumentation. #if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) # define LLVM_ADDRESS_SANITIZER_BUILD 1 +# include <sanitizer/asan_interface.h> #else # define LLVM_ADDRESS_SANITIZER_BUILD 0 +# define __asan_poison_memory_region(p, size) +# define __asan_unpoison_memory_region(p, size) +#endif + +/// \macro LLVM_THREAD_SANITIZER_BUILD +/// \brief Whether LLVM itself is built with ThreadSanitizer instrumentation. +#if __has_feature(thread_sanitizer) || defined(__SANITIZE_THREAD__) +# define LLVM_THREAD_SANITIZER_BUILD 1 +#else +# define LLVM_THREAD_SANITIZER_BUILD 0 +#endif + +#if LLVM_THREAD_SANITIZER_BUILD +// Thread Sanitizer is a tool that finds races in code. +// See http://code.google.com/p/data-race-test/wiki/DynamicAnnotations . +// tsan detects these exact functions by name. +extern "C" { +void AnnotateHappensAfter(const char *file, int line, const volatile void *cv); +void AnnotateHappensBefore(const char *file, int line, const volatile void *cv); +void AnnotateIgnoreWritesBegin(const char *file, int line); +void AnnotateIgnoreWritesEnd(const char *file, int line); +} + +// This marker is used to define a happens-before arc. The race detector will +// infer an arc from the begin to the end when they share the same pointer +// argument. +# define TsanHappensBefore(cv) AnnotateHappensBefore(__FILE__, __LINE__, cv) + +// This marker defines the destination of a happens-before arc. +# define TsanHappensAfter(cv) AnnotateHappensAfter(__FILE__, __LINE__, cv) + +// Ignore any races on writes between here and the next TsanIgnoreWritesEnd. +# define TsanIgnoreWritesBegin() AnnotateIgnoreWritesBegin(__FILE__, __LINE__) + +// Resume checking for racy writes. +# define TsanIgnoreWritesEnd() AnnotateIgnoreWritesEnd(__FILE__, __LINE__) +#else +# define TsanHappensBefore(cv) +# define TsanHappensAfter(cv) +# define TsanIgnoreWritesBegin() +# define TsanIgnoreWritesEnd() #endif /// \brief Mark debug helper function definitions like dump() that should not be diff --git a/include/llvm/Support/CrashRecoveryContext.h b/include/llvm/Support/CrashRecoveryContext.h index c08c3c1f0d21..1a1c74368761 100644 --- a/include/llvm/Support/CrashRecoveryContext.h +++ b/include/llvm/Support/CrashRecoveryContext.h @@ -39,8 +39,6 @@ class CrashRecoveryContextCleanup; /// /// ... no crash was detected ... /// } -/// -/// Crash recovery contexts may not be nested. class CrashRecoveryContext { void *Impl; CrashRecoveryContextCleanup *head; @@ -109,10 +107,11 @@ class CrashRecoveryContextCleanup { protected: CrashRecoveryContext *context; CrashRecoveryContextCleanup(CrashRecoveryContext *context) - : context(context), cleanupFired(false) {} + : context(context), cleanupFired(false) {} + public: bool cleanupFired; - + virtual ~CrashRecoveryContextCleanup(); virtual void recoverResources() = 0; @@ -129,15 +128,16 @@ template<typename DERIVED, typename T> class CrashRecoveryContextCleanupBase : public CrashRecoveryContextCleanup { protected: T *resource; - CrashRecoveryContextCleanupBase(CrashRecoveryContext *context, T* resource) - : CrashRecoveryContextCleanup(context), resource(resource) {} + CrashRecoveryContextCleanupBase(CrashRecoveryContext *context, T *resource) + : CrashRecoveryContextCleanup(context), resource(resource) {} + public: static DERIVED *create(T *x) { if (x) { if (CrashRecoveryContext *context = CrashRecoveryContext::GetCurrent()) return new DERIVED(context, x); } - return 0; + return nullptr; } }; @@ -146,9 +146,9 @@ class CrashRecoveryContextDestructorCleanup : public CrashRecoveryContextCleanupBase<CrashRecoveryContextDestructorCleanup<T>, T> { public: CrashRecoveryContextDestructorCleanup(CrashRecoveryContext *context, - T *resource) - : CrashRecoveryContextCleanupBase< - CrashRecoveryContextDestructorCleanup<T>, T>(context, resource) {} + T *resource) + : CrashRecoveryContextCleanupBase< + CrashRecoveryContextDestructorCleanup<T>, T>(context, resource) {} virtual void recoverResources() { this->resource->~T(); @@ -171,7 +171,7 @@ class CrashRecoveryContextReleaseRefCleanup : public CrashRecoveryContextCleanupBase<CrashRecoveryContextReleaseRefCleanup<T>, T> { public: - CrashRecoveryContextReleaseRefCleanup(CrashRecoveryContext *context, + CrashRecoveryContextReleaseRefCleanup(CrashRecoveryContext *context, T *resource) : CrashRecoveryContextCleanupBase<CrashRecoveryContextReleaseRefCleanup<T>, T>(context, resource) {} @@ -182,6 +182,7 @@ public: template <typename T, typename Cleanup = CrashRecoveryContextDeleteCleanup<T> > class CrashRecoveryContextCleanupRegistrar { CrashRecoveryContextCleanup *cleanup; + public: CrashRecoveryContextCleanupRegistrar(T *x) : cleanup(Cleanup::create(x)) { @@ -189,16 +190,14 @@ public: cleanup->getContext()->registerCleanup(cleanup); } - ~CrashRecoveryContextCleanupRegistrar() { - unregister(); - } - + ~CrashRecoveryContextCleanupRegistrar() { unregister(); } + void unregister() { if (cleanup && !cleanup->cleanupFired) cleanup->getContext()->unregisterCleanup(cleanup); - cleanup = 0; + cleanup = nullptr; } }; -} +} // end namespace llvm -#endif +#endif // LLVM_SUPPORT_CRASHRECOVERYCONTEXT_H diff --git a/include/llvm/Support/DOTGraphTraits.h b/include/llvm/Support/DOTGraphTraits.h index 95e37c01d7d5..4381b5bf1633 100644 --- a/include/llvm/Support/DOTGraphTraits.h +++ b/include/llvm/Support/DOTGraphTraits.h @@ -72,11 +72,12 @@ public: return ""; } - /// hasNodeAddressLabel - If this method returns true, the address of the node - /// is added to the label of the node. - template<typename GraphType> - static bool hasNodeAddressLabel(const void *, const GraphType &) { - return false; + // getNodeIdentifierLabel - Returns a string representing the + // address or other unique identifier of the node. (Only used if + // non-empty.) + template <typename GraphType> + static std::string getNodeIdentifierLabel(const void *, const GraphType &) { + return ""; } template<typename GraphType> diff --git a/include/llvm/Support/Debug.h b/include/llvm/Support/Debug.h index fff4f986a6c0..6e213477d710 100644 --- a/include/llvm/Support/Debug.h +++ b/include/llvm/Support/Debug.h @@ -13,7 +13,7 @@ // // In particular, just wrap your code with the DEBUG() macro, and it will be // enabled automatically if you specify '-debug' on the command-line. -// Alternatively, you can also define the DEBUG_TYPE macro to "foo" specify +// DEBUG() requires the DEBUG_TYPE macro to be defined. Set it to "foo" specify // that your debug code belongs to class "foo". Be careful that you only do // this after including Debug.h and not around any #include of headers. Headers // should define and undef the macro acround the code that needs to use the diff --git a/include/llvm/Support/Dwarf.def b/include/llvm/Support/Dwarf.def index 4b923b897e6f..b15070b3e9b0 100644 --- a/include/llvm/Support/Dwarf.def +++ b/include/llvm/Support/Dwarf.def @@ -99,10 +99,6 @@ HANDLE_DW_TAG(0x0041, type_unit) HANDLE_DW_TAG(0x0042, rvalue_reference_type) HANDLE_DW_TAG(0x0043, template_alias) -// Mock tags we use as discriminators. -HANDLE_DW_TAG(0x0100, auto_variable) // Tag for local (auto) variables. -HANDLE_DW_TAG(0x0101, arg_variable) // Tag for argument variables. - // New in DWARF v5. HANDLE_DW_TAG(0x0044, coarray_type) HANDLE_DW_TAG(0x0045, generic_subrange) @@ -117,6 +113,11 @@ HANDLE_DW_TAG(0x4106, GNU_template_template_param) HANDLE_DW_TAG(0x4107, GNU_template_parameter_pack) HANDLE_DW_TAG(0x4108, GNU_formal_parameter_pack) HANDLE_DW_TAG(0x4200, APPLE_property) +HANDLE_DW_TAG(0xb000, BORLAND_property) +HANDLE_DW_TAG(0xb001, BORLAND_Delphi_string) +HANDLE_DW_TAG(0xb002, BORLAND_Delphi_dynamic_array) +HANDLE_DW_TAG(0xb003, BORLAND_Delphi_set) +HANDLE_DW_TAG(0xb004, BORLAND_Delphi_variant) HANDLE_DW_OP(0x03, addr) HANDLE_DW_OP(0x06, deref) @@ -319,6 +320,7 @@ HANDLE_DW_LANG(0x0021, C_plus_plus_14) HANDLE_DW_LANG(0x0022, Fortran03) HANDLE_DW_LANG(0x0023, Fortran08) HANDLE_DW_LANG(0x8001, Mips_Assembler) +HANDLE_DW_LANG(0xb000, BORLAND_Delphi) // DWARF attribute type encodings. HANDLE_DW_ATE(0x01, address) diff --git a/include/llvm/Support/Dwarf.h b/include/llvm/Support/Dwarf.h index 17e9c1540a41..cea61bd75833 100644 --- a/include/llvm/Support/Dwarf.h +++ b/include/llvm/Support/Dwarf.h @@ -40,6 +40,7 @@ enum LLVMConstants : uint32_t { // LLVM mock tags (see also llvm/Support/Dwarf.def). DW_TAG_invalid = ~0U, // Tag for invalid results. DW_VIRTUALITY_invalid = ~0U, // Virtuality for invalid results. + DW_MACINFO_invalid = ~0U, // Macinfo type for invalid results. // Other constants. DWARF_VERSION = 4, // Default dwarf version we output. @@ -195,6 +196,7 @@ enum Attribute : uint16_t { DW_AT_dwo_name = 0x76, DW_AT_reference = 0x77, DW_AT_rvalue_reference = 0x78, + DW_AT_macros = 0x79, DW_AT_lo_user = 0x2000, DW_AT_hi_user = 0x3fff, @@ -230,6 +232,7 @@ enum Attribute : uint16_t { DW_AT_GNU_template_name = 0x2110, DW_AT_GNU_odr_signature = 0x210f, + DW_AT_GNU_macros = 0x2119, // Extensions for Fission proposal. DW_AT_GNU_dwo_name = 0x2130, @@ -238,6 +241,26 @@ enum Attribute : uint16_t { DW_AT_GNU_addr_base = 0x2133, DW_AT_GNU_pubnames = 0x2134, DW_AT_GNU_pubtypes = 0x2135, + DW_AT_GNU_discriminator = 0x2136, + + // Borland extensions. + DW_AT_BORLAND_property_read = 0x3b11, + DW_AT_BORLAND_property_write = 0x3b12, + DW_AT_BORLAND_property_implements = 0x3b13, + DW_AT_BORLAND_property_index = 0x3b14, + DW_AT_BORLAND_property_default = 0x3b15, + DW_AT_BORLAND_Delphi_unit = 0x3b20, + DW_AT_BORLAND_Delphi_class = 0x3b21, + DW_AT_BORLAND_Delphi_record = 0x3b22, + DW_AT_BORLAND_Delphi_metaclass = 0x3b23, + DW_AT_BORLAND_Delphi_constructor = 0x3b24, + DW_AT_BORLAND_Delphi_destructor = 0x3b25, + DW_AT_BORLAND_Delphi_anonymous_method = 0x3b26, + DW_AT_BORLAND_Delphi_interface = 0x3b27, + DW_AT_BORLAND_Delphi_ABI = 0x3b28, + DW_AT_BORLAND_Delphi_return = 0x3b29, + DW_AT_BORLAND_Delphi_frameptr = 0x3b30, + DW_AT_BORLAND_closure = 0x3b31, // LLVM project extensions. DW_AT_LLVM_include_path = 0x3e00, @@ -370,6 +393,14 @@ enum CallingConvention { DW_CC_program = 0x02, DW_CC_nocall = 0x03, DW_CC_lo_user = 0x40, + DW_CC_GNU_borland_fastcall_i386 = 0x41, + DW_CC_BORLAND_safecall = 0xb0, + DW_CC_BORLAND_stdcall = 0xb1, + DW_CC_BORLAND_pascal = 0xb2, + DW_CC_BORLAND_msfastcall = 0xb3, + DW_CC_BORLAND_msreturn = 0xb4, + DW_CC_BORLAND_thiscall = 0xb5, + DW_CC_BORLAND_fastcall = 0xb6, DW_CC_hi_user = 0xff }; @@ -429,6 +460,24 @@ enum MacinfoRecordType { DW_MACINFO_vendor_ext = 0xff }; +enum MacroEntryType { + // Macro Information Entry Type Encodings + DW_MACRO_define = 0x01, + DW_MACRO_undef = 0x02, + DW_MACRO_start_file = 0x03, + DW_MACRO_end_file = 0x04, + DW_MACRO_define_indirect = 0x05, + DW_MACRO_undef_indirect = 0x06, + DW_MACRO_transparent_include = 0x07, + DW_MACRO_define_indirect_sup = 0x08, + DW_MACRO_undef_indirect_sup = 0x09, + DW_MACRO_transparent_include_sup = 0x0a, + DW_MACRO_define_indirectx = 0x0b, + DW_MACRO_undef_indirectx = 0x0c, + DW_MACRO_lo_user = 0xe0, + DW_MACRO_hi_user = 0xff +}; + enum CallFrameInfo { // Call frame instruction encodings DW_CFA_extended = 0x00, @@ -596,6 +645,7 @@ const char *GDBIndexEntryLinkageString(GDBIndexEntryLinkage Linkage); /// /// \li \a getTag() returns \a DW_TAG_invalid on invalid input. /// \li \a getVirtuality() returns \a DW_VIRTUALITY_invalid on invalid input. +/// \li \a getMacinfo() returns \a DW_MACINFO_invalid on invalid input. /// /// @{ unsigned getTag(StringRef TagString); @@ -603,6 +653,7 @@ unsigned getOperationEncoding(StringRef OperationEncodingString); unsigned getVirtuality(StringRef VirtualityString); unsigned getLanguage(StringRef LanguageString); unsigned getAttributeEncoding(StringRef EncodingString); +unsigned getMacinfo(StringRef MacinfoString); /// @} /// \brief Returns the symbolic string representing Val when used as a value @@ -610,7 +661,7 @@ unsigned getAttributeEncoding(StringRef EncodingString); const char *AttributeValueString(uint16_t Attr, unsigned Val); /// \brief Decsribes an entry of the various gnu_pub* debug sections. -/// +/// /// The gnu_pub* kind looks like: /// /// 0-3 reserved @@ -642,7 +693,6 @@ private: }; }; - } // End of namespace dwarf } // End of namespace llvm diff --git a/include/llvm/Support/ELF.h b/include/llvm/Support/ELF.h index 94a4bfb22025..97708a7cdd63 100644 --- a/include/llvm/Support/ELF.h +++ b/include/llvm/Support/ELF.h @@ -429,6 +429,33 @@ enum { #include "ELFRelocs/ARM.def" }; +// AVR specific e_flags +enum : unsigned { + EF_AVR_ARCH_AVR1 = 1, + EF_AVR_ARCH_AVR2 = 2, + EF_AVR_ARCH_AVR25 = 25, + EF_AVR_ARCH_AVR3 = 3, + EF_AVR_ARCH_AVR31 = 31, + EF_AVR_ARCH_AVR35 = 35, + EF_AVR_ARCH_AVR4 = 4, + EF_AVR_ARCH_AVR5 = 5, + EF_AVR_ARCH_AVR51 = 51, + EF_AVR_ARCH_AVR6 = 6, + EF_AVR_ARCH_AVRTINY = 100, + EF_AVR_ARCH_XMEGA1 = 101, + EF_AVR_ARCH_XMEGA2 = 102, + EF_AVR_ARCH_XMEGA3 = 103, + EF_AVR_ARCH_XMEGA4 = 104, + EF_AVR_ARCH_XMEGA5 = 105, + EF_AVR_ARCH_XMEGA6 = 106, + EF_AVR_ARCH_XMEGA7 = 107 +}; + +// ELF Relocation types for AVR +enum { +#include "ELFRelocs/AVR.def" +}; + // Mips Specific e_flags enum : unsigned { EF_MIPS_NOREORDER = 0x00000001, // Don't reorder instructions @@ -522,26 +549,28 @@ enum { ODK_PAGESIZE = 11 // Page size information }; -// Hexagon Specific e_flags -// Release 5 ABI +// Hexagon-specific e_flags enum { - // Object processor version flags, bits[3:0] + // Object processor version flags, bits[11:0] EF_HEXAGON_MACH_V2 = 0x00000001, // Hexagon V2 EF_HEXAGON_MACH_V3 = 0x00000002, // Hexagon V3 EF_HEXAGON_MACH_V4 = 0x00000003, // Hexagon V4 EF_HEXAGON_MACH_V5 = 0x00000004, // Hexagon V5 + EF_HEXAGON_MACH_V55 = 0x00000005, // Hexagon V55 + EF_HEXAGON_MACH_V60 = 0x00000060, // Hexagon V60 // Highest ISA version flags - EF_HEXAGON_ISA_MACH = 0x00000000, // Same as specified in bits[3:0] + EF_HEXAGON_ISA_MACH = 0x00000000, // Same as specified in bits[11:0] // of e_flags EF_HEXAGON_ISA_V2 = 0x00000010, // Hexagon V2 ISA EF_HEXAGON_ISA_V3 = 0x00000020, // Hexagon V3 ISA EF_HEXAGON_ISA_V4 = 0x00000030, // Hexagon V4 ISA - EF_HEXAGON_ISA_V5 = 0x00000040 // Hexagon V5 ISA + EF_HEXAGON_ISA_V5 = 0x00000040, // Hexagon V5 ISA + EF_HEXAGON_ISA_V55 = 0x00000050, // Hexagon V55 ISA + EF_HEXAGON_ISA_V60 = 0x00000060, // Hexagon V60 ISA }; -// Hexagon specific Section indexes for common small data -// Release 5 ABI +// Hexagon-specific section indexes for common small data enum { SHN_HEXAGON_SCOMMON = 0xff00, // Other access sizes SHN_HEXAGON_SCOMMON_1 = 0xff01, // Byte-sized access @@ -747,7 +776,12 @@ enum : unsigned { SHF_MIPS_ADDR = 0x40000000, // Section data is string data by default. - SHF_MIPS_STRING = 0x80000000 + SHF_MIPS_STRING = 0x80000000, + + SHF_AMDGPU_HSA_GLOBAL = 0x00100000, + SHF_AMDGPU_HSA_READONLY = 0x00200000, + SHF_AMDGPU_HSA_CODE = 0x00400000, + SHF_AMDGPU_HSA_AGENT = 0x00800000 }; // Section Group Flags @@ -828,7 +862,12 @@ enum { STT_LOOS = 10, // Lowest operating system-specific symbol type STT_HIOS = 12, // Highest operating system-specific symbol type STT_LOPROC = 13, // Lowest processor-specific symbol type - STT_HIPROC = 15 // Highest processor-specific symbol type + STT_HIPROC = 15, // Highest processor-specific symbol type + + // AMDGPU symbol types + STT_AMDGPU_HSA_KERNEL = 10, + STT_AMDGPU_HSA_INDIRECT_FUNCTION = 11, + STT_AMDGPU_HSA_METADATA = 12 }; enum { @@ -979,7 +1018,13 @@ enum { PT_MIPS_REGINFO = 0x70000000, // Register usage information. PT_MIPS_RTPROC = 0x70000001, // Runtime procedure table. PT_MIPS_OPTIONS = 0x70000002, // Options segment. - PT_MIPS_ABIFLAGS = 0x70000003 // Abiflags segment. + PT_MIPS_ABIFLAGS = 0x70000003, // Abiflags segment. + + // AMDGPU program header types. + PT_AMDGPU_HSA_LOAD_GLOBAL_PROGRAM = 0x60000000, + PT_AMDGPU_HSA_LOAD_GLOBAL_AGENT = 0x60000001, + PT_AMDGPU_HSA_LOAD_READONLY_AGENT = 0x60000002, + PT_AMDGPU_HSA_LOAD_CODE_AGENT = 0x60000003 }; // Segment flag bits. @@ -1139,8 +1184,10 @@ enum { DT_MIPS_GP_VALUE = 0x70000030, // GP value for auxiliary GOTs. DT_MIPS_AUX_DYNAMIC = 0x70000031, // Address of auxiliary .dynamic. DT_MIPS_PLTGOT = 0x70000032, // Address of the base of the PLTGOT. - DT_MIPS_RWPLT = 0x70000034 // Points to the base + DT_MIPS_RWPLT = 0x70000034, // Points to the base // of a writable PLT. + DT_MIPS_RLD_MAP_REL = 0x70000035 // Relative offset of run time loader + // map, used for debugging. }; // DT_FLAGS values. diff --git a/include/llvm/Support/ELFRelocs/AVR.def b/include/llvm/Support/ELFRelocs/AVR.def new file mode 100644 index 000000000000..5692d6cb9aa0 --- /dev/null +++ b/include/llvm/Support/ELFRelocs/AVR.def @@ -0,0 +1,40 @@ + +#ifndef ELF_RELOC +#error "ELF_RELOC must be defined" +#endif + +ELF_RELOC(R_AVR_NONE, 0) +ELF_RELOC(R_AVR_32, 1) +ELF_RELOC(R_AVR_7_PCREL, 2) +ELF_RELOC(R_AVR_13_PCREL, 3) +ELF_RELOC(R_AVR_16, 4) +ELF_RELOC(R_AVR_16_PM, 5) +ELF_RELOC(R_AVR_LO8_LDI, 6) +ELF_RELOC(R_AVR_HI8_LDI, 7) +ELF_RELOC(R_AVR_HH8_LDI, 8) +ELF_RELOC(R_AVR_LO8_LDI_NEG, 9) +ELF_RELOC(R_AVR_HI8_LDI_NEG, 10) +ELF_RELOC(R_AVR_HH8_LDI_NEG, 11) +ELF_RELOC(R_AVR_LO8_LDI_PM, 12) +ELF_RELOC(R_AVR_HI8_LDI_PM, 13) +ELF_RELOC(R_AVR_HH8_LDI_PM, 14) +ELF_RELOC(R_AVR_LO8_LDI_PM_NEG, 15) +ELF_RELOC(R_AVR_HI8_LDI_PM_NEG, 16) +ELF_RELOC(R_AVR_HH8_LDI_PM_NEG, 17) +ELF_RELOC(R_AVR_CALL, 18) +ELF_RELOC(R_AVR_LDI, 19) +ELF_RELOC(R_AVR_6, 20) +ELF_RELOC(R_AVR_6_ADIW, 21) +ELF_RELOC(R_AVR_MS8_LDI, 22) +ELF_RELOC(R_AVR_MS8_LDI_NEG, 23) +ELF_RELOC(R_AVR_LO8_LDI_GS, 24) +ELF_RELOC(R_AVR_HI8_LDI_GS, 25) +ELF_RELOC(R_AVR_8, 26) +ELF_RELOC(R_AVR_8_LO8, 27) +ELF_RELOC(R_AVR_8_HI8, 28) +ELF_RELOC(R_AVR_8_HLO8, 29) +ELF_RELOC(R_AVR_SYM_DIFF, 30) +ELF_RELOC(R_AVR_16_LDST, 31) +ELF_RELOC(R_AVR_LDS_STS_16, 33) +ELF_RELOC(R_AVR_PORT6, 34) +ELF_RELOC(R_AVR_PORT5, 35) diff --git a/include/llvm/Support/ELFRelocs/PowerPC.def b/include/llvm/Support/ELFRelocs/PowerPC.def index b6c39419b0f7..e4f8ee0ebe2b 100644 --- a/include/llvm/Support/ELFRelocs/PowerPC.def +++ b/include/llvm/Support/ELFRelocs/PowerPC.def @@ -3,6 +3,68 @@ #error "ELF_RELOC must be defined" #endif +// glibc's PowerPC asm/sigcontext.h, when compiling for PPC64, has the +// unfortunate behavior of including asm/elf.h, which defines R_PPC_NONE, etc. +// to their corresponding integer values. As a result, we need to undef them +// here before continuing. + +#undef R_PPC_NONE +#undef R_PPC_ADDR32 +#undef R_PPC_ADDR24 +#undef R_PPC_ADDR16 +#undef R_PPC_ADDR16_LO +#undef R_PPC_ADDR16_HI +#undef R_PPC_ADDR16_HA +#undef R_PPC_ADDR14 +#undef R_PPC_ADDR14_BRTAKEN +#undef R_PPC_ADDR14_BRNTAKEN +#undef R_PPC_REL24 +#undef R_PPC_REL14 +#undef R_PPC_REL14_BRTAKEN +#undef R_PPC_REL14_BRNTAKEN +#undef R_PPC_GOT16 +#undef R_PPC_GOT16_LO +#undef R_PPC_GOT16_HI +#undef R_PPC_GOT16_HA +#undef R_PPC_PLTREL24 +#undef R_PPC_JMP_SLOT +#undef R_PPC_LOCAL24PC +#undef R_PPC_REL32 +#undef R_PPC_TLS +#undef R_PPC_DTPMOD32 +#undef R_PPC_TPREL16 +#undef R_PPC_TPREL16_LO +#undef R_PPC_TPREL16_HI +#undef R_PPC_TPREL16_HA +#undef R_PPC_TPREL32 +#undef R_PPC_DTPREL16 +#undef R_PPC_DTPREL16_LO +#undef R_PPC_DTPREL16_HI +#undef R_PPC_DTPREL16_HA +#undef R_PPC_DTPREL32 +#undef R_PPC_GOT_TLSGD16 +#undef R_PPC_GOT_TLSGD16_LO +#undef R_PPC_GOT_TLSGD16_HI +#undef R_PPC_GOT_TLSGD16_HA +#undef R_PPC_GOT_TLSLD16 +#undef R_PPC_GOT_TLSLD16_LO +#undef R_PPC_GOT_TLSLD16_HI +#undef R_PPC_GOT_TLSLD16_HA +#undef R_PPC_GOT_TPREL16 +#undef R_PPC_GOT_TPREL16_LO +#undef R_PPC_GOT_TPREL16_HI +#undef R_PPC_GOT_TPREL16_HA +#undef R_PPC_GOT_DTPREL16 +#undef R_PPC_GOT_DTPREL16_LO +#undef R_PPC_GOT_DTPREL16_HI +#undef R_PPC_GOT_DTPREL16_HA +#undef R_PPC_TLSGD +#undef R_PPC_TLSLD +#undef R_PPC_REL16 +#undef R_PPC_REL16_LO +#undef R_PPC_REL16_HI +#undef R_PPC_REL16_HA + ELF_RELOC(R_PPC_NONE, 0) /* No relocation. */ ELF_RELOC(R_PPC_ADDR32, 1) ELF_RELOC(R_PPC_ADDR24, 2) diff --git a/include/llvm/Support/ELFRelocs/PowerPC64.def b/include/llvm/Support/ELFRelocs/PowerPC64.def index 7b2a3cb2235b..3a47c5a07574 100644 --- a/include/llvm/Support/ELFRelocs/PowerPC64.def +++ b/include/llvm/Support/ELFRelocs/PowerPC64.def @@ -3,6 +3,97 @@ #error "ELF_RELOC must be defined" #endif +// glibc's PowerPC asm/sigcontext.h, when compiling for PPC64, has the +// unfortunate behavior of including asm/elf.h, which defines R_PPC_NONE, etc. +// to their corresponding integer values. As a result, we need to undef them +// here before continuing. + +#undef R_PPC64_NONE +#undef R_PPC64_ADDR32 +#undef R_PPC64_ADDR24 +#undef R_PPC64_ADDR16 +#undef R_PPC64_ADDR16_LO +#undef R_PPC64_ADDR16_HI +#undef R_PPC64_ADDR16_HA +#undef R_PPC64_ADDR14 +#undef R_PPC64_ADDR14_BRTAKEN +#undef R_PPC64_ADDR14_BRNTAKEN +#undef R_PPC64_REL24 +#undef R_PPC64_REL14 +#undef R_PPC64_REL14_BRTAKEN +#undef R_PPC64_REL14_BRNTAKEN +#undef R_PPC64_GOT16 +#undef R_PPC64_GOT16_LO +#undef R_PPC64_GOT16_HI +#undef R_PPC64_GOT16_HA +#undef R_PPC64_GLOB_DAT +#undef R_PPC64_JMP_SLOT +#undef R_PPC64_RELATIVE +#undef R_PPC64_REL32 +#undef R_PPC64_ADDR64 +#undef R_PPC64_ADDR16_HIGHER +#undef R_PPC64_ADDR16_HIGHERA +#undef R_PPC64_ADDR16_HIGHEST +#undef R_PPC64_ADDR16_HIGHESTA +#undef R_PPC64_REL64 +#undef R_PPC64_TOC16 +#undef R_PPC64_TOC16_LO +#undef R_PPC64_TOC16_HI +#undef R_PPC64_TOC16_HA +#undef R_PPC64_TOC +#undef R_PPC64_ADDR16_DS +#undef R_PPC64_ADDR16_LO_DS +#undef R_PPC64_GOT16_DS +#undef R_PPC64_GOT16_LO_DS +#undef R_PPC64_TOC16_DS +#undef R_PPC64_TOC16_LO_DS +#undef R_PPC64_TLS +#undef R_PPC64_DTPMOD64 +#undef R_PPC64_TPREL16 +#undef R_PPC64_TPREL16_LO +#undef R_PPC64_TPREL16_HI +#undef R_PPC64_TPREL16_HA +#undef R_PPC64_TPREL64 +#undef R_PPC64_DTPREL16 +#undef R_PPC64_DTPREL16_LO +#undef R_PPC64_DTPREL16_HI +#undef R_PPC64_DTPREL16_HA +#undef R_PPC64_DTPREL64 +#undef R_PPC64_GOT_TLSGD16 +#undef R_PPC64_GOT_TLSGD16_LO +#undef R_PPC64_GOT_TLSGD16_HI +#undef R_PPC64_GOT_TLSGD16_HA +#undef R_PPC64_GOT_TLSLD16 +#undef R_PPC64_GOT_TLSLD16_LO +#undef R_PPC64_GOT_TLSLD16_HI +#undef R_PPC64_GOT_TLSLD16_HA +#undef R_PPC64_GOT_TPREL16_DS +#undef R_PPC64_GOT_TPREL16_LO_DS +#undef R_PPC64_GOT_TPREL16_HI +#undef R_PPC64_GOT_TPREL16_HA +#undef R_PPC64_GOT_DTPREL16_DS +#undef R_PPC64_GOT_DTPREL16_LO_DS +#undef R_PPC64_GOT_DTPREL16_HI +#undef R_PPC64_GOT_DTPREL16_HA +#undef R_PPC64_TPREL16_DS +#undef R_PPC64_TPREL16_LO_DS +#undef R_PPC64_TPREL16_HIGHER +#undef R_PPC64_TPREL16_HIGHERA +#undef R_PPC64_TPREL16_HIGHEST +#undef R_PPC64_TPREL16_HIGHESTA +#undef R_PPC64_DTPREL16_DS +#undef R_PPC64_DTPREL16_LO_DS +#undef R_PPC64_DTPREL16_HIGHER +#undef R_PPC64_DTPREL16_HIGHERA +#undef R_PPC64_DTPREL16_HIGHEST +#undef R_PPC64_DTPREL16_HIGHESTA +#undef R_PPC64_TLSGD +#undef R_PPC64_TLSLD +#undef R_PPC64_REL16 +#undef R_PPC64_REL16_LO +#undef R_PPC64_REL16_HI +#undef R_PPC64_REL16_HA + ELF_RELOC(R_PPC64_NONE, 0) ELF_RELOC(R_PPC64_ADDR32, 1) ELF_RELOC(R_PPC64_ADDR24, 2) @@ -21,7 +112,9 @@ ELF_RELOC(R_PPC64_GOT16, 14) ELF_RELOC(R_PPC64_GOT16_LO, 15) ELF_RELOC(R_PPC64_GOT16_HI, 16) ELF_RELOC(R_PPC64_GOT16_HA, 17) +ELF_RELOC(R_PPC64_GLOB_DAT, 20) ELF_RELOC(R_PPC64_JMP_SLOT, 21) +ELF_RELOC(R_PPC64_RELATIVE, 22) ELF_RELOC(R_PPC64_REL32, 26) ELF_RELOC(R_PPC64_ADDR64, 38) ELF_RELOC(R_PPC64_ADDR16_HIGHER, 39) diff --git a/include/llvm/Support/Endian.h b/include/llvm/Support/Endian.h index fd59009e0d3a..bc93c9a66eef 100644 --- a/include/llvm/Support/Endian.h +++ b/include/llvm/Support/Endian.h @@ -77,6 +77,95 @@ inline void write(void *memory, value_type value) { &value, sizeof(value_type)); } + +template <typename value_type> +using make_unsigned_t = typename std::make_unsigned<value_type>::type; + +/// Read a value of a particular endianness from memory, for a location +/// that starts at the given bit offset within the first byte. +template <typename value_type, endianness endian, std::size_t alignment> +inline value_type readAtBitAlignment(const void *memory, uint64_t startBit) { + assert(startBit < 8); + if (startBit == 0) + return read<value_type, endian, alignment>(memory); + else { + // Read two values and compose the result from them. + value_type val[2]; + memcpy(&val[0], + LLVM_ASSUME_ALIGNED( + memory, (detail::PickAlignment<value_type, alignment>::value)), + sizeof(value_type) * 2); + val[0] = byte_swap<value_type, endian>(val[0]); + val[1] = byte_swap<value_type, endian>(val[1]); + + // Shift bits from the lower value into place. + make_unsigned_t<value_type> lowerVal = val[0] >> startBit; + // Mask off upper bits after right shift in case of signed type. + make_unsigned_t<value_type> numBitsFirstVal = + (sizeof(value_type) * 8) - startBit; + lowerVal &= ((make_unsigned_t<value_type>)1 << numBitsFirstVal) - 1; + + // Get the bits from the upper value. + make_unsigned_t<value_type> upperVal = + val[1] & (((make_unsigned_t<value_type>)1 << startBit) - 1); + // Shift them in to place. + upperVal <<= numBitsFirstVal; + + return lowerVal | upperVal; + } +} + +/// Write a value to memory with a particular endianness, for a location +/// that starts at the given bit offset within the first byte. +template <typename value_type, endianness endian, std::size_t alignment> +inline void writeAtBitAlignment(void *memory, value_type value, + uint64_t startBit) { + assert(startBit < 8); + if (startBit == 0) + write<value_type, endian, alignment>(memory, value); + else { + // Read two values and shift the result into them. + value_type val[2]; + memcpy(&val[0], + LLVM_ASSUME_ALIGNED( + memory, (detail::PickAlignment<value_type, alignment>::value)), + sizeof(value_type) * 2); + val[0] = byte_swap<value_type, endian>(val[0]); + val[1] = byte_swap<value_type, endian>(val[1]); + + // Mask off any existing bits in the upper part of the lower value that + // we want to replace. + val[0] &= ((make_unsigned_t<value_type>)1 << startBit) - 1; + make_unsigned_t<value_type> numBitsFirstVal = + (sizeof(value_type) * 8) - startBit; + make_unsigned_t<value_type> lowerVal = value; + if (startBit > 0) { + // Mask off the upper bits in the new value that are not going to go into + // the lower value. This avoids a left shift of a negative value, which + // is undefined behavior. + lowerVal &= (((make_unsigned_t<value_type>)1 << numBitsFirstVal) - 1); + // Now shift the new bits into place + lowerVal <<= startBit; + } + val[0] |= lowerVal; + + // Mask off any existing bits in the lower part of the upper value that + // we want to replace. + val[1] &= ~(((make_unsigned_t<value_type>)1 << startBit) - 1); + // Next shift the bits that go into the upper value into position. + make_unsigned_t<value_type> upperVal = value >> numBitsFirstVal; + // Mask off upper bits after right shift in case of signed type. + upperVal &= ((make_unsigned_t<value_type>)1 << startBit) - 1; + val[1] |= upperVal; + + // Finally, rewrite values. + val[0] = byte_swap<value_type, endian>(val[0]); + val[1] = byte_swap<value_type, endian>(val[1]); + memcpy(LLVM_ASSUME_ALIGNED( + memory, (detail::PickAlignment<value_type, alignment>::value)), + &val[0], sizeof(value_type) * 2); + } +} } // end namespace endian namespace detail { @@ -208,19 +297,47 @@ typedef detail::packed_endian_specific_integral <int64_t, native, unaligned> unaligned_int64_t; namespace endian { -inline uint16_t read16le(const void *p) { return *(const ulittle16_t *)p; } -inline uint32_t read32le(const void *p) { return *(const ulittle32_t *)p; } -inline uint64_t read64le(const void *p) { return *(const ulittle64_t *)p; } -inline uint16_t read16be(const void *p) { return *(const ubig16_t *)p; } -inline uint32_t read32be(const void *p) { return *(const ubig32_t *)p; } -inline uint64_t read64be(const void *p) { return *(const ubig64_t *)p; } - -inline void write16le(void *p, uint16_t v) { *(ulittle16_t *)p = v; } -inline void write32le(void *p, uint32_t v) { *(ulittle32_t *)p = v; } -inline void write64le(void *p, uint64_t v) { *(ulittle64_t *)p = v; } -inline void write16be(void *p, uint16_t v) { *(ubig16_t *)p = v; } -inline void write32be(void *p, uint32_t v) { *(ubig32_t *)p = v; } -inline void write64be(void *p, uint64_t v) { *(ubig64_t *)p = v; } +template <typename T, endianness E> inline T read(const void *P) { + return *(const detail::packed_endian_specific_integral<T, E, unaligned> *)P; +} + +template <endianness E> inline uint16_t read16(const void *P) { + return read<uint16_t, E>(P); +} +template <endianness E> inline uint32_t read32(const void *P) { + return read<uint32_t, E>(P); +} +template <endianness E> inline uint64_t read64(const void *P) { + return read<uint64_t, E>(P); +} + +inline uint16_t read16le(const void *P) { return read16<little>(P); } +inline uint32_t read32le(const void *P) { return read32<little>(P); } +inline uint64_t read64le(const void *P) { return read64<little>(P); } +inline uint16_t read16be(const void *P) { return read16<big>(P); } +inline uint32_t read32be(const void *P) { return read32<big>(P); } +inline uint64_t read64be(const void *P) { return read64<big>(P); } + +template <typename T, endianness E> inline void write(void *P, T V) { + *(detail::packed_endian_specific_integral<T, E, unaligned> *)P = V; +} + +template <endianness E> inline void write16(void *P, uint16_t V) { + write<uint16_t, E>(P, V); +} +template <endianness E> inline void write32(void *P, uint32_t V) { + write<uint32_t, E>(P, V); +} +template <endianness E> inline void write64(void *P, uint64_t V) { + write<uint64_t, E>(P, V); +} + +inline void write16le(void *P, uint16_t V) { write16<little>(P, V); } +inline void write32le(void *P, uint32_t V) { write32<little>(P, V); } +inline void write64le(void *P, uint64_t V) { write64<little>(P, V); } +inline void write16be(void *P, uint16_t V) { write16<big>(P, V); } +inline void write32be(void *P, uint32_t V) { write32<big>(P, V); } +inline void write64be(void *P, uint64_t V) { write64<big>(P, V); } } // end namespace endian } // end namespace support } // end namespace llvm diff --git a/include/llvm/Support/ErrorHandling.h b/include/llvm/Support/ErrorHandling.h index 9afd52d1abc7..32f05e0e9610 100644 --- a/include/llvm/Support/ErrorHandling.h +++ b/include/llvm/Support/ErrorHandling.h @@ -61,22 +61,22 @@ namespace llvm { ~ScopedFatalErrorHandler() { remove_fatal_error_handler(); } }; - /// Reports a serious error, calling any installed error handler. These - /// functions are intended to be used for error conditions which are outside - /// the control of the compiler (I/O errors, invalid user input, etc.) - /// - /// If no error handler is installed the default is to print the message to - /// standard error, followed by a newline. - /// After the error handler is called this function will call exit(1), it - /// does not return. - LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, - bool gen_crash_diag = true); - LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const std::string &reason, - bool gen_crash_diag = true); - LLVM_ATTRIBUTE_NORETURN void report_fatal_error(StringRef reason, - bool gen_crash_diag = true); - LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const Twine &reason, - bool gen_crash_diag = true); +/// Reports a serious error, calling any installed error handler. These +/// functions are intended to be used for error conditions which are outside +/// the control of the compiler (I/O errors, invalid user input, etc.) +/// +/// If no error handler is installed the default is to print the message to +/// standard error, followed by a newline. +/// After the error handler is called this function will call exit(1), it +/// does not return. +LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, + bool gen_crash_diag = true); +LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const std::string &reason, + bool gen_crash_diag = true); +LLVM_ATTRIBUTE_NORETURN void report_fatal_error(StringRef reason, + bool gen_crash_diag = true); +LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const Twine &reason, + bool gen_crash_diag = true); /// This function calls abort(), and prints the optional message to stderr. /// Use the llvm_unreachable macro (that adds location info), instead of diff --git a/include/llvm/Support/ErrorOr.h b/include/llvm/Support/ErrorOr.h index 589404f9b4ee..ca6ede73e8df 100644 --- a/include/llvm/Support/ErrorOr.h +++ b/include/llvm/Support/ErrorOr.h @@ -1,4 +1,4 @@ -//===- llvm/Support/ErrorOr.h - Error Smart Pointer -----------------------===// +//===- llvm/Support/ErrorOr.h - Error Smart Pointer -------------*- C++ -*-===// // // The LLVM Linker // @@ -91,6 +91,7 @@ private: typedef typename std::remove_reference<T>::type &reference; typedef const typename std::remove_reference<T>::type &const_reference; typedef typename std::remove_reference<T>::type *pointer; + typedef const typename std::remove_reference<T>::type *const_pointer; public: template <class E> @@ -183,10 +184,14 @@ public: return toPointer(getStorage()); } + const_pointer operator->() const { return toPointer(getStorage()); } + reference operator *() { return *getStorage(); } + const_reference operator*() const { return *getStorage(); } + private: template <class OtherT> void copyConstruct(const ErrorOr<OtherT> &Other) { @@ -246,10 +251,14 @@ private: return Val; } + const_pointer toPointer(const_pointer Val) const { return Val; } + pointer toPointer(wrap *Val) { return &Val->get(); } + const_pointer toPointer(const wrap *Val) const { return &Val->get(); } + storage_type *getStorage() { assert(!HasError && "Cannot get value when an error exists!"); return reinterpret_cast<storage_type*>(TStorage.buffer); diff --git a/include/llvm/Support/FileOutputBuffer.h b/include/llvm/Support/FileOutputBuffer.h index fd8879c84622..3bcf64a8a08b 100644 --- a/include/llvm/Support/FileOutputBuffer.h +++ b/include/llvm/Support/FileOutputBuffer.h @@ -17,6 +17,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" namespace llvm { @@ -37,9 +38,8 @@ public: /// Factory method to create an OutputBuffer object which manages a read/write /// buffer of the specified size. When committed, the buffer will be written /// to the file at the specified path. - static std::error_code create(StringRef FilePath, size_t Size, - std::unique_ptr<FileOutputBuffer> &Result, - unsigned Flags = 0); + static ErrorOr<std::unique_ptr<FileOutputBuffer>> + create(StringRef FilePath, size_t Size, unsigned Flags = 0); /// Returns a pointer to the start of the buffer. uint8_t *getBufferStart() { diff --git a/include/llvm/Support/FileSystem.h b/include/llvm/Support/FileSystem.h index a736c324f8aa..4733ddb77575 100644 --- a/include/llvm/Support/FileSystem.h +++ b/include/llvm/Support/FileSystem.h @@ -95,21 +95,21 @@ enum perms { }; // Helper functions so that you can use & and | to manipulate perms bits: -inline perms operator|(perms l , perms r) { - return static_cast<perms>( - static_cast<unsigned short>(l) | static_cast<unsigned short>(r)); +inline perms operator|(perms l, perms r) { + return static_cast<perms>(static_cast<unsigned short>(l) | + static_cast<unsigned short>(r)); } -inline perms operator&(perms l , perms r) { - return static_cast<perms>( - static_cast<unsigned short>(l) & static_cast<unsigned short>(r)); +inline perms operator&(perms l, perms r) { + return static_cast<perms>(static_cast<unsigned short>(l) & + static_cast<unsigned short>(r)); } inline perms &operator|=(perms &l, perms r) { - l = l | r; - return l; + l = l | r; + return l; } inline perms &operator&=(perms &l, perms r) { - l = l & r; - return l; + l = l & r; + return l; } inline perms operator~(perms x) { return static_cast<perms>(~static_cast<unsigned short>(x)); @@ -156,6 +156,7 @@ class file_status friend bool equivalent(file_status A, file_status B); file_type Type; perms Perms; + public: #if defined(LLVM_ON_UNIX) file_status() : fs_st_dev(0), fs_st_ino(0), fs_st_mtime(0), @@ -267,6 +268,20 @@ private: /// @brief Make \a path an absolute path. /// +/// Makes \a path absolute using the \a current_directory if it is not already. +/// An empty \a path will result in the \a current_directory. +/// +/// /absolute/path => /absolute/path +/// relative/../path => <current-directory>/relative/../path +/// +/// @param path A path that is modified to be an absolute path. +/// @returns errc::success if \a path has been made absolute, otherwise a +/// platform-specific error_code. +std::error_code make_absolute(const Twine ¤t_directory, + SmallVectorImpl<char> &path); + +/// @brief Make \a path an absolute path. +/// /// Makes \a path absolute using the current directory if it is not already. An /// empty \a path will result in the current directory. /// @@ -285,7 +300,8 @@ std::error_code make_absolute(SmallVectorImpl<char> &path); /// specific error_code. If IgnoreExisting is false, also returns /// error if the directory already existed. std::error_code create_directories(const Twine &path, - bool IgnoreExisting = true); + bool IgnoreExisting = true, + perms Perms = owner_all | group_all); /// @brief Create the directory in path. /// @@ -293,7 +309,8 @@ std::error_code create_directories(const Twine &path, /// @returns errc::success if is_directory(path), otherwise a platform /// specific error_code. If IgnoreExisting is false, also returns /// error if the directory already existed. -std::error_code create_directory(const Twine &path, bool IgnoreExisting = true); +std::error_code create_directory(const Twine &path, bool IgnoreExisting = true, + perms Perms = owner_all | group_all); /// @brief Create a link from \a from to \a to. /// @@ -375,9 +392,7 @@ inline bool exists(const Twine &Path) { /// /// @param Path Input path. /// @returns True if we can execute it, false otherwise. -inline bool can_execute(const Twine &Path) { - return !access(Path, AccessMode::Execute); -} +bool can_execute(const Twine &Path); /// @brief Can we write this file? /// @@ -531,15 +546,15 @@ std::error_code status_known(const Twine &path, bool &result); /// /// Generates a unique path suitable for a temporary file and then opens it as a /// file. The name is based on \a model with '%' replaced by a random char in -/// [0-9a-f]. If \a model is not an absolute path, a suitable temporary -/// directory will be prepended. +/// [0-9a-f]. If \a model is not an absolute path, the temporary file will be +/// created in the current directory. /// /// Example: clang-%%-%%-%%-%%-%%.s => clang-a0-b1-c2-d3-e4.s /// /// This is an atomic operation. Either the file is created and opened, or the /// file system is left untouched. /// -/// The intendend use is for files that are to be kept, possibly after +/// The intended use is for files that are to be kept, possibly after /// renaming them. For example, when running 'clang -c foo.o', the file can /// be first created as foo-abc123.o and then renamed. /// diff --git a/include/llvm/Support/Format.h b/include/llvm/Support/Format.h index 4319a3ba2745..f0b437a0cbed 100644 --- a/include/llvm/Support/Format.h +++ b/include/llvm/Support/Format.h @@ -118,6 +118,7 @@ class FormattedString { unsigned Width; bool RightJustify; friend class raw_ostream; + public: FormattedString(StringRef S, unsigned W, bool R) : Str(S), Width(W), RightJustify(R) { } @@ -146,6 +147,7 @@ class FormattedNumber { bool Upper; bool HexPrefix; friend class raw_ostream; + public: FormattedNumber(uint64_t HV, int64_t DV, unsigned W, bool H, bool U, bool Prefix) @@ -178,7 +180,7 @@ inline FormattedNumber format_hex_no_prefix(uint64_t N, unsigned Width, return FormattedNumber(N, 0, Width, true, Upper, false); } -/// format_decimal - Output \p N as a right justified, fixed-width decimal. If +/// format_decimal - Output \p N as a right justified, fixed-width decimal. If /// number will not fit in width, full number is still printed. Examples: /// OS << format_decimal(0, 5) => " 0" /// OS << format_decimal(255, 5) => " 255" @@ -188,7 +190,6 @@ inline FormattedNumber format_decimal(int64_t N, unsigned Width) { return FormattedNumber(0, N, Width, false, false, false); } - } // end namespace llvm #endif diff --git a/include/llvm/Support/GCOV.h b/include/llvm/Support/GCOV.h index c2e34bd3eaeb..544434f036a4 100644 --- a/include/llvm/Support/GCOV.h +++ b/include/llvm/Support/GCOV.h @@ -30,12 +30,11 @@ class GCOVBlock; class FileInfo; namespace GCOV { -enum GCOVVersion { V402, V404 }; -} // end GCOV namespace +enum GCOVVersion { V402, V404, V704 }; -/// GCOVOptions - A struct for passing gcov options between functions. -struct GCOVOptions { - GCOVOptions(bool A, bool B, bool C, bool F, bool P, bool U, bool L, bool N) +/// \brief A struct for passing gcov options between functions. +struct Options { + Options(bool A, bool B, bool C, bool F, bool P, bool U, bool L, bool N) : AllBlocks(A), BranchInfo(B), BranchCount(C), FuncCoverage(F), PreservePaths(P), UncondBranch(U), LongFileNames(L), NoOutput(N) {} @@ -48,6 +47,7 @@ struct GCOVOptions { bool LongFileNames; bool NoOutput; }; +} // end GCOV namespace /// GCOVBuffer - A wrapper around MemoryBuffer to provide GCOV specific /// read operations. @@ -90,6 +90,11 @@ public: Version = GCOV::V404; return true; } + if (VersionStr == "*704") { + Cursor += 4; + Version = GCOV::V704; + return true; + } errs() << "Unexpected version: " << VersionStr << ".\n"; return false; } @@ -390,7 +395,7 @@ class FileInfo { }; public: - FileInfo(const GCOVOptions &Options) + FileInfo(const GCOV::Options &Options) : Options(Options), LineInfo(), RunCount(0), ProgramCount(0) {} void addBlockLine(StringRef Filename, uint32_t Line, const GCOVBlock *Block) { @@ -424,7 +429,7 @@ private: void printFuncCoverage(raw_ostream &OS) const; void printFileCoverage(raw_ostream &OS) const; - const GCOVOptions &Options; + const GCOV::Options &Options; StringMap<LineData> LineInfo; uint32_t RunCount; uint32_t ProgramCount; diff --git a/include/llvm/Support/GenericDomTree.h b/include/llvm/Support/GenericDomTree.h index 63678bb98bb1..8751f272cd29 100644 --- a/include/llvm/Support/GenericDomTree.h +++ b/include/llvm/Support/GenericDomTree.h @@ -371,8 +371,9 @@ public: void releaseMemory() { reset(); } /// getNode - return the (Post)DominatorTree node for the specified basic - /// block. This is the same as using operator[] on this class. - /// + /// block. This is the same as using operator[] on this class. The result + /// may (but is not required to) be null for a forward (backwards) + /// statically unreachable block. DomTreeNodeBase<NodeT> *getNode(NodeT *BB) const { auto I = DomTreeNodes.find(BB); if (I != DomTreeNodes.end()) @@ -380,6 +381,7 @@ public: return nullptr; } + /// See getNode. DomTreeNodeBase<NodeT> *operator[](NodeT *BB) const { return getNode(BB); } /// getRootNode - This returns the entry node for the CFG of the function. If @@ -732,13 +734,13 @@ public: for (typename TraitsTy::nodes_iterator I = TraitsTy::nodes_begin(&F), E = TraitsTy::nodes_end(&F); I != E; ++I) { - if (TraitsTy::child_begin(I) == TraitsTy::child_end(I)) - addRoot(I); + if (TraitsTy::child_begin(&*I) == TraitsTy::child_end(&*I)) + addRoot(&*I); // Prepopulate maps so that we don't get iterator invalidation issues // later. - this->IDoms[I] = nullptr; - this->DomTreeNodes[I] = nullptr; + this->IDoms[&*I] = nullptr; + this->DomTreeNodes[&*I] = nullptr; } Calculate<FT, Inverse<NodeT *>>(*this, F); diff --git a/include/llvm/Support/GenericDomTreeConstruction.h b/include/llvm/Support/GenericDomTreeConstruction.h index 7c065f939256..3e867dc6cbf1 100644 --- a/include/llvm/Support/GenericDomTreeConstruction.h +++ b/include/llvm/Support/GenericDomTreeConstruction.h @@ -21,7 +21,6 @@ /// //===----------------------------------------------------------------------===// - #ifndef LLVM_SUPPORT_GENERICDOMTREECONSTRUCTION_H #define LLVM_SUPPORT_GENERICDOMTREECONSTRUCTION_H @@ -88,7 +87,7 @@ unsigned DFSPass(DominatorTreeBase<typename GraphT::NodeType>& DT, // Increment the successor number for the next time we get to it. ++Worklist.back().second; - + // Visit the successor next, if it isn't already visited. typename GraphT::NodeType* Succ = *NextSucc; @@ -103,9 +102,9 @@ unsigned DFSPass(DominatorTreeBase<typename GraphT::NodeType>& DT, return N; } -template<class GraphT> -typename GraphT::NodeType* -Eval(DominatorTreeBase<typename GraphT::NodeType>& DT, +template <class GraphT> +typename GraphT::NodeType * +Eval(DominatorTreeBase<typename GraphT::NodeType> &DT, typename GraphT::NodeType *VIn, unsigned LastLinked) { typename DominatorTreeBase<typename GraphT::NodeType>::InfoRec &VInInfo = DT.Info[VIn]; @@ -117,7 +116,7 @@ Eval(DominatorTreeBase<typename GraphT::NodeType>& DT, if (VInInfo.Parent >= LastLinked) Work.push_back(VIn); - + while (!Work.empty()) { typename GraphT::NodeType* V = Work.back(); typename DominatorTreeBase<typename GraphT::NodeType>::InfoRec &VInfo = @@ -128,8 +127,8 @@ Eval(DominatorTreeBase<typename GraphT::NodeType>& DT, if (Visited.insert(VAncestor).second && VInfo.Parent >= LastLinked) { Work.push_back(VAncestor); continue; - } - Work.pop_back(); + } + Work.pop_back(); // Update VInfo based on Ancestor info if (VInfo.Parent < LastLinked) @@ -169,7 +168,7 @@ void Calculate(DominatorTreeBase<typename GraphTraits<NodeT>::NodeType>& DT, i != e; ++i) N = DFSPass<GraphT>(DT, DT.Roots[i], N); - // it might be that some blocks did not get a DFS number (e.g., blocks of + // it might be that some blocks did not get a DFS number (e.g., blocks of // infinite loops). In these cases an artificial exit node is required. MultipleRoots |= (DT.isPostDominator() && N != GraphTraits<FuncT*>::size(&F)); @@ -287,7 +286,6 @@ void Calculate(DominatorTreeBase<typename GraphTraits<NodeT>::NodeType>& DT, DT.updateDFSNumbers(); } - } #endif diff --git a/include/llvm/Support/GraphWriter.h b/include/llvm/Support/GraphWriter.h index b1af3d7c2632..86985c569464 100644 --- a/include/llvm/Support/GraphWriter.h +++ b/include/llvm/Support/GraphWriter.h @@ -175,8 +175,9 @@ public: O << DOT::EscapeString(DTraits.getNodeLabel(Node, G)); // If we should include the address of the node in the label, do so now. - if (DTraits.hasNodeAddressLabel(Node, G)) - O << "|" << static_cast<const void*>(Node); + std::string Id = DTraits.getNodeIdentifierLabel(Node, G); + if (!Id.empty()) + O << "|" << DOT::EscapeString(Id); std::string NodeDesc = DTraits.getNodeDescription(Node, G); if (!NodeDesc.empty()) @@ -199,8 +200,9 @@ public: O << DOT::EscapeString(DTraits.getNodeLabel(Node, G)); // If we should include the address of the node in the label, do so now. - if (DTraits.hasNodeAddressLabel(Node, G)) - O << "|" << static_cast<const void*>(Node); + std::string Id = DTraits.getNodeIdentifierLabel(Node, G); + if (!Id.empty()) + O << "|" << DOT::EscapeString(Id); std::string NodeDesc = DTraits.getNodeDescription(Node, G); if (!NodeDesc.empty()) diff --git a/include/llvm/Support/JamCRC.h b/include/llvm/Support/JamCRC.h new file mode 100644 index 000000000000..20c28a5f8e45 --- /dev/null +++ b/include/llvm/Support/JamCRC.h @@ -0,0 +1,48 @@ +//===-- llvm/Support/JamCRC.h - Cyclic Redundancy Check ---------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains an implementation of JamCRC. +// +// We will use the "Rocksoft^tm Model CRC Algorithm" to describe the properties +// of this CRC: +// Width : 32 +// Poly : 04C11DB7 +// Init : FFFFFFFF +// RefIn : True +// RefOut : True +// XorOut : 00000000 +// Check : 340BC6D9 (result of CRC for "123456789") +// +// N.B. We permit flexibility of the "Init" value. Some consumers of this need +// it to be zero. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_JAMCRC_H +#define LLVM_SUPPORT_JAMCRC_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/DataTypes.h" + +namespace llvm { +class JamCRC { +public: + JamCRC(uint32_t Init = 0xFFFFFFFFU) : CRC(Init) {} + + // \brief Update the CRC calculation with Data. + void update(ArrayRef<char> Data); + + uint32_t getCRC() const { return CRC; } + +private: + uint32_t CRC; +}; +} // End of namespace llvm + +#endif diff --git a/include/llvm/Support/MachO.h b/include/llvm/Support/MachO.h index 775127505923..54b8745de1c1 100644 --- a/include/llvm/Support/MachO.h +++ b/include/llvm/Support/MachO.h @@ -132,7 +132,9 @@ namespace llvm { LC_DYLIB_CODE_SIGN_DRS = 0x0000002Bu, LC_ENCRYPTION_INFO_64 = 0x0000002Cu, LC_LINKER_OPTION = 0x0000002Du, - LC_LINKER_OPTIMIZATION_HINT = 0x0000002Eu + LC_LINKER_OPTIMIZATION_HINT = 0x0000002Eu, + LC_VERSION_MIN_TVOS = 0x0000002Fu, + LC_VERSION_MIN_WATCHOS = 0x00000030u, }; enum : uint32_t { @@ -142,7 +144,6 @@ namespace llvm { SG_NORELOC = 0x4u, SG_PROTECTED_VERSION_1 = 0x8u, - // Constant masks for the "flags" field in llvm::MachO::section and // llvm::MachO::section_64 SECTION_TYPE = 0x000000ffu, // SECTION_TYPE @@ -334,7 +335,6 @@ namespace llvm { EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE = 0x02u }; - enum { // Constant masks for the "n_type" field in llvm::MachO::nlist and // llvm::MachO::nlist_64 @@ -385,7 +385,7 @@ namespace llvm { SELF_LIBRARY_ORDINAL = 0x0, MAX_LIBRARY_ORDINAL = 0xfd, DYNAMIC_LOOKUP_ORDINAL = 0xfe, - EXECUTABLE_ORDINAL = 0xff + EXECUTABLE_ORDINAL = 0xff }; enum StabType { @@ -506,7 +506,6 @@ namespace llvm { // Must be followed by ARM64_RELOC_PAGE21 or ARM64_RELOC_PAGEOFF12. ARM64_RELOC_ADDEND = 10, - // Constant values for the r_type field in an x86_64 architecture // llvm::MachO::relocation_info or llvm::MachO::scattered_relocation_info // structure @@ -530,7 +529,6 @@ namespace llvm { VM_PROT_EXECUTE = 0x4 }; - // Structs from <mach-o/loader.h> struct mach_header { @@ -784,7 +782,6 @@ namespace llvm { flags:8; }; - struct twolevel_hints_command { uint32_t cmd; uint32_t cmdsize; @@ -924,7 +921,6 @@ namespace llvm { uint64_t stacksize; }; - // Structs from <mach-o/fat.h> struct fat_header { uint32_t magic; @@ -995,7 +991,6 @@ namespace llvm { uint64_t n_value; }; - // Byte order swapping functions for MachO structs inline void swapStruct(mach_header &mh) { diff --git a/include/llvm/Support/ManagedStatic.h b/include/llvm/Support/ManagedStatic.h index addd34e704bc..2e131e47177d 100644 --- a/include/llvm/Support/ManagedStatic.h +++ b/include/llvm/Support/ManagedStatic.h @@ -15,8 +15,8 @@ #define LLVM_SUPPORT_MANAGEDSTATIC_H #include "llvm/Support/Atomic.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/Threading.h" -#include "llvm/Support/Valgrind.h" namespace llvm { diff --git a/include/llvm/Support/MathExtras.h b/include/llvm/Support/MathExtras.h index 2cf7e0e5d0b3..8111aeebe6ee 100644 --- a/include/llvm/Support/MathExtras.h +++ b/include/llvm/Support/MathExtras.h @@ -63,7 +63,7 @@ template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter { } }; -#if __GNUC__ >= 4 || _MSC_VER +#if __GNUC__ >= 4 || defined(_MSC_VER) template <typename T> struct TrailingZerosCounter<T, 4> { static std::size_t count(T Val, ZeroBehavior ZB) { if (ZB != ZB_Undefined && Val == 0) @@ -71,7 +71,7 @@ template <typename T> struct TrailingZerosCounter<T, 4> { #if __has_builtin(__builtin_ctz) || LLVM_GNUC_PREREQ(4, 0, 0) return __builtin_ctz(Val); -#elif _MSC_VER +#elif defined(_MSC_VER) unsigned long Index; _BitScanForward(&Index, Val); return Index; @@ -87,7 +87,7 @@ template <typename T> struct TrailingZerosCounter<T, 8> { #if __has_builtin(__builtin_ctzll) || LLVM_GNUC_PREREQ(4, 0, 0) return __builtin_ctzll(Val); -#elif _MSC_VER +#elif defined(_MSC_VER) unsigned long Index; _BitScanForward64(&Index, Val); return Index; @@ -132,7 +132,7 @@ template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter { } }; -#if __GNUC__ >= 4 || _MSC_VER +#if __GNUC__ >= 4 || defined(_MSC_VER) template <typename T> struct LeadingZerosCounter<T, 4> { static std::size_t count(T Val, ZeroBehavior ZB) { if (ZB != ZB_Undefined && Val == 0) @@ -140,7 +140,7 @@ template <typename T> struct LeadingZerosCounter<T, 4> { #if __has_builtin(__builtin_clz) || LLVM_GNUC_PREREQ(4, 0, 0) return __builtin_clz(Val); -#elif _MSC_VER +#elif defined(_MSC_VER) unsigned long Index; _BitScanReverse(&Index, Val); return Index ^ 31; @@ -156,7 +156,7 @@ template <typename T> struct LeadingZerosCounter<T, 8> { #if __has_builtin(__builtin_clzll) || LLVM_GNUC_PREREQ(4, 0, 0) return __builtin_clzll(Val); -#elif _MSC_VER +#elif defined(_MSC_VER) unsigned long Index; _BitScanReverse64(&Index, Val); return Index ^ 63; @@ -313,7 +313,7 @@ inline bool isShiftedUInt(uint64_t x) { /// isUIntN - Checks if an unsigned integer fits into the given (dynamic) /// bit width. inline bool isUIntN(unsigned N, uint64_t x) { - return x == (x & (~0ULL >> (64 - N))); + return N >= 64 || x < (UINT64_C(1)<<(N)); } /// isIntN - Checks if an signed integer fits into the given (dynamic) @@ -552,7 +552,7 @@ inline uint32_t FloatToBits(float Float) { inline uint64_t MinAlign(uint64_t A, uint64_t B) { // The largest power of 2 that divides both A and B. // - // Replace "-Value" by "1+~Value" in the following commented code to avoid + // Replace "-Value" by "1+~Value" in the following commented code to avoid // MSVC warning C4146 // return (A | B) & -(A | B); return (A | B) & (1 + ~(A | B)); @@ -599,15 +599,27 @@ inline uint64_t PowerOf2Floor(uint64_t A) { /// Returns the next integer (mod 2**64) that is greater than or equal to /// \p Value and is a multiple of \p Align. \p Align must be non-zero. /// +/// If non-zero \p Skew is specified, the return value will be a minimal +/// integer that is greater than or equal to \p Value and equal to +/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than +/// \p Align, its value is adjusted to '\p Skew mod \p Align'. +/// /// Examples: /// \code /// RoundUpToAlignment(5, 8) = 8 /// RoundUpToAlignment(17, 8) = 24 /// RoundUpToAlignment(~0LL, 8) = 0 /// RoundUpToAlignment(321, 255) = 510 +/// +/// RoundUpToAlignment(5, 8, 7) = 7 +/// RoundUpToAlignment(17, 8, 1) = 17 +/// RoundUpToAlignment(~0LL, 8, 3) = 3 +/// RoundUpToAlignment(321, 255, 42) = 552 /// \endcode -inline uint64_t RoundUpToAlignment(uint64_t Value, uint64_t Align) { - return (Value + Align - 1) / Align * Align; +inline uint64_t RoundUpToAlignment(uint64_t Value, uint64_t Align, + uint64_t Skew = 0) { + Skew %= Align; + return (Value + Align - 1 - Skew) / Align * Align + Skew; } /// Returns the offset to the next integer (mod 2**64) that is greater than @@ -641,6 +653,70 @@ inline int64_t SignExtend64(uint64_t X, unsigned B) { return int64_t(X << (64 - B)) >> (64 - B); } +/// \brief Add two unsigned integers, X and Y, of type T. +/// Clamp the result to the maximum representable value of T on overflow. +/// ResultOverflowed indicates if the result is larger than the maximum +/// representable value of type T. +template <typename T> +typename std::enable_if<std::is_unsigned<T>::value, T>::type +SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) { + bool Dummy; + bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; + // Hacker's Delight, p. 29 + T Z = X + Y; + Overflowed = (Z < X || Z < Y); + if (Overflowed) + return std::numeric_limits<T>::max(); + else + return Z; +} + +/// \brief Multiply two unsigned integers, X and Y, of type T. +/// Clamp the result to the maximum representable value of T on overflow. +/// ResultOverflowed indicates if the result is larger than the maximum +/// representable value of type T. +template <typename T> +typename std::enable_if<std::is_unsigned<T>::value, T>::type +SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) { + bool Dummy; + bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; + + // Hacker's Delight, p. 30 has a different algorithm, but we don't use that + // because it fails for uint16_t (where multiplication can have undefined + // behavior due to promotion to int), and requires a division in addition + // to the multiplication. + + Overflowed = false; + + // Log2(Z) would be either Log2Z or Log2Z + 1. + // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z + // will necessarily be less than Log2Max as desired. + int Log2Z = Log2_64(X) + Log2_64(Y); + const T Max = std::numeric_limits<T>::max(); + int Log2Max = Log2_64(Max); + if (Log2Z < Log2Max) { + return X * Y; + } + if (Log2Z > Log2Max) { + Overflowed = true; + return Max; + } + + // We're going to use the top bit, and maybe overflow one + // bit past it. Multiply all but the bottom bit then add + // that on at the end. + T Z = (X >> 1) * Y; + if (Z & ~(Max >> 1)) { + Overflowed = true; + return Max; + } + Z <<= 1; + if (X & 1) + return SaturatingAdd(Z, Y, ResultOverflowed); + + return Z; +} + extern const float huge_valf; } // End llvm namespace diff --git a/include/llvm/Support/Memory.h b/include/llvm/Support/Memory.h index b4305cb697d0..8103aea2fa25 100644 --- a/include/llvm/Support/Memory.h +++ b/include/llvm/Support/Memory.h @@ -1,4 +1,4 @@ -//===- llvm/Support/Memory.h - Memory Support --------------------*- C++ -*-===// +//===- llvm/Support/Memory.h - Memory Support -------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -32,6 +32,7 @@ namespace sys { MemoryBlock(void *addr, size_t size) : Address(addr), Size(size) { } void *base() const { return Address; } size_t size() const { return Size; } + private: void *Address; ///< Address of first byte of memory area size_t Size; ///< Size, in bytes of the memory area @@ -70,7 +71,7 @@ namespace sys { /// If the address following \p NearBlock is not so aligned, it will be /// rounded up to the next allocation granularity boundary. /// - /// \r a non-null MemoryBlock if the function was successful, + /// \r a non-null MemoryBlock if the function was successful, /// otherwise a null MemoryBlock is with \p EC describing the error. /// /// @brief Allocate mapped memory. @@ -86,7 +87,7 @@ namespace sys { /// /// \r error_success if the function was successful, or an error_code /// describing the failure if an error occurred. - /// + /// /// @brief Release mapped memory. static std::error_code releaseMappedMemory(MemoryBlock &Block); @@ -131,7 +132,6 @@ namespace sys { /// @brief Release Read/Write/Execute memory. static bool ReleaseRWX(MemoryBlock &block, std::string *ErrMsg = nullptr); - /// InvalidateInstructionCache - Before the JIT can run a block of code /// that has been emitted it must invalidate the instruction cache on some /// platforms. @@ -155,6 +155,31 @@ namespace sys { /// as writable. static bool setRangeWritable(const void *Addr, size_t Size); }; + + /// Owning version of MemoryBlock. + class OwningMemoryBlock { + public: + OwningMemoryBlock() = default; + explicit OwningMemoryBlock(MemoryBlock M) : M(M) {} + OwningMemoryBlock(OwningMemoryBlock &&Other) { + M = Other.M; + Other.M = MemoryBlock(); + } + OwningMemoryBlock& operator=(OwningMemoryBlock &&Other) { + M = Other.M; + Other.M = MemoryBlock(); + return *this; + } + ~OwningMemoryBlock() { + Memory::releaseMappedMemory(M); + } + void *base() const { return M.base(); } + size_t size() const { return M.size(); } + MemoryBlock getMemoryBlock() const { return M; } + private: + MemoryBlock M; + }; + } } diff --git a/include/llvm/Support/MemoryBuffer.h b/include/llvm/Support/MemoryBuffer.h index 81616d8ba3ac..73d643537a6f 100644 --- a/include/llvm/Support/MemoryBuffer.h +++ b/include/llvm/Support/MemoryBuffer.h @@ -14,7 +14,6 @@ #ifndef LLVM_SUPPORT_MEMORYBUFFER_H #define LLVM_SUPPORT_MEMORYBUFFER_H -#include "llvm-c/Support.h" #include "llvm/ADT/Twine.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/DataTypes.h" @@ -122,7 +121,8 @@ public: /// Open the specified file as a MemoryBuffer, or open stdin if the Filename /// is "-". static ErrorOr<std::unique_ptr<MemoryBuffer>> - getFileOrSTDIN(const Twine &Filename, int64_t FileSize = -1); + getFileOrSTDIN(const Twine &Filename, int64_t FileSize = -1, + bool RequiresNullTerminator = true); /// Map a subrange of the specified file as a MemoryBuffer. static ErrorOr<std::unique_ptr<MemoryBuffer>> @@ -151,6 +151,8 @@ class MemoryBufferRef { public: MemoryBufferRef() {} + MemoryBufferRef(MemoryBuffer& Buffer) + : Buffer(Buffer.getBuffer()), Identifier(Buffer.getBufferIdentifier()) {} MemoryBufferRef(StringRef Buffer, StringRef Identifier) : Buffer(Buffer), Identifier(Identifier) {} diff --git a/include/llvm/Support/OnDiskHashTable.h b/include/llvm/Support/OnDiskHashTable.h index 08e277ad5ce1..ac978d4c242c 100644 --- a/include/llvm/Support/OnDiskHashTable.h +++ b/include/llvm/Support/OnDiskHashTable.h @@ -53,6 +53,8 @@ namespace llvm { /// /// Write Data to Out. DataLen is the length from EmitKeyDataLength. /// static void EmitData(raw_ostream &Out, key_type_ref Key, /// data_type_ref Data, offset_type DataLen); +/// /// Determine if two keys are equal. Optional, only needed by contains. +/// static bool EqualKey(key_type_ref Key1, key_type_ref Key2); /// }; /// \endcode template <typename Info> class OnDiskChainedHashTableGenerator { @@ -122,13 +124,21 @@ public: /// Uses the provided Info instead of a stack allocated one. void insert(typename Info::key_type_ref Key, typename Info::data_type_ref Data, Info &InfoObj) { - ++NumEntries; if (4 * NumEntries >= 3 * NumBuckets) resize(NumBuckets * 2); insert(Buckets, NumBuckets, new (BA.Allocate()) Item(Key, Data, InfoObj)); } + /// \brief Determine whether an entry has been inserted. + bool contains(typename Info::key_type_ref Key, Info &InfoObj) { + unsigned Hash = InfoObj.ComputeHash(Key); + for (Item *I = Buckets[Hash & (NumBuckets - 1)].Head; I; I = I->Next) + if (I->Hash == Hash && InfoObj.EqualKey(I->Key, Key)) + return true; + return false; + } + /// \brief Emit the table to Out, which must not be at offset 0. offset_type Emit(raw_ostream &Out) { Info InfoObj; @@ -161,8 +171,22 @@ public: LE.write<typename Info::hash_value_type>(I->Hash); const std::pair<offset_type, offset_type> &Len = InfoObj.EmitKeyDataLength(Out, I->Key, I->Data); +#ifdef NDEBUG InfoObj.EmitKey(Out, I->Key, Len.first); InfoObj.EmitData(Out, I->Key, I->Data, Len.second); +#else + // In asserts mode, check that the users length matches the data they + // wrote. + uint64_t KeyStart = Out.tell(); + InfoObj.EmitKey(Out, I->Key, Len.first); + uint64_t DataStart = Out.tell(); + InfoObj.EmitData(Out, I->Key, I->Data, Len.second); + uint64_t End = Out.tell(); + assert(offset_type(DataStart - KeyStart) == Len.first && + "key length does not match bytes written"); + assert(offset_type(End - DataStart) == Len.second && + "data length does not match bytes written"); +#endif } } @@ -239,11 +263,12 @@ template <typename Info> class OnDiskChainedHashTable { Info InfoObj; public: + typedef Info InfoType; typedef typename Info::internal_key_type internal_key_type; typedef typename Info::external_key_type external_key_type; - typedef typename Info::data_type data_type; - typedef typename Info::hash_value_type hash_value_type; - typedef typename Info::offset_type offset_type; + typedef typename Info::data_type data_type; + typedef typename Info::hash_value_type hash_value_type; + typedef typename Info::offset_type offset_type; OnDiskChainedHashTable(offset_type NumBuckets, offset_type NumEntries, const unsigned char *Buckets, @@ -255,6 +280,21 @@ public: "'buckets' must have a 4-byte alignment"); } + /// Read the number of buckets and the number of entries from a hash table + /// produced by OnDiskHashTableGenerator::Emit, and advance the Buckets + /// pointer past them. + static std::pair<offset_type, offset_type> + readNumBucketsAndEntries(const unsigned char *&Buckets) { + assert((reinterpret_cast<uintptr_t>(Buckets) & 0x3) == 0 && + "buckets should be 4-byte aligned."); + using namespace llvm::support; + offset_type NumBuckets = + endian::readNext<offset_type, little, aligned>(Buckets); + offset_type NumEntries = + endian::readNext<offset_type, little, aligned>(Buckets); + return std::make_pair(NumBuckets, NumEntries); + } + offset_type getNumBuckets() const { return NumBuckets; } offset_type getNumEntries() const { return NumEntries; } const unsigned char *getBase() const { return Base; } @@ -275,6 +315,10 @@ public: : Key(K), Data(D), Len(L), InfoObj(InfoObj) {} data_type operator*() const { return InfoObj->ReadData(Key, Data, Len); } + + const unsigned char *getDataPtr() const { return Data; } + offset_type getDataLen() const { return Len; } + bool operator==(const iterator &X) const { return X.Data == Data; } bool operator!=(const iterator &X) const { return X.Data != Data; } }; @@ -356,17 +400,11 @@ public: static OnDiskChainedHashTable *Create(const unsigned char *Buckets, const unsigned char *const Base, const Info &InfoObj = Info()) { - using namespace llvm::support; assert(Buckets > Base); - assert((reinterpret_cast<uintptr_t>(Buckets) & 0x3) == 0 && - "buckets should be 4-byte aligned."); - - offset_type NumBuckets = - endian::readNext<offset_type, little, aligned>(Buckets); - offset_type NumEntries = - endian::readNext<offset_type, little, aligned>(Buckets); - return new OnDiskChainedHashTable<Info>(NumBuckets, NumEntries, Buckets, - Base, InfoObj); + auto NumBucketsAndEntries = readNumBucketsAndEntries(Buckets); + return new OnDiskChainedHashTable<Info>(NumBucketsAndEntries.first, + NumBucketsAndEntries.second, + Buckets, Base, InfoObj); } }; @@ -385,40 +423,30 @@ public: typedef typename base_type::hash_value_type hash_value_type; typedef typename base_type::offset_type offset_type; - OnDiskIterableChainedHashTable(offset_type NumBuckets, offset_type NumEntries, - const unsigned char *Buckets, - const unsigned char *Payload, - const unsigned char *Base, - const Info &InfoObj = Info()) - : base_type(NumBuckets, NumEntries, Buckets, Base, InfoObj), - Payload(Payload) {} - +private: /// \brief Iterates over all of the keys in the table. - class key_iterator { + class iterator_base { const unsigned char *Ptr; offset_type NumItemsInBucketLeft; offset_type NumEntriesLeft; - Info *InfoObj; public: typedef external_key_type value_type; - key_iterator(const unsigned char *const Ptr, offset_type NumEntries, - Info *InfoObj) - : Ptr(Ptr), NumItemsInBucketLeft(0), NumEntriesLeft(NumEntries), - InfoObj(InfoObj) {} - key_iterator() - : Ptr(nullptr), NumItemsInBucketLeft(0), NumEntriesLeft(0), - InfoObj(0) {} + iterator_base(const unsigned char *const Ptr, offset_type NumEntries) + : Ptr(Ptr), NumItemsInBucketLeft(0), NumEntriesLeft(NumEntries) {} + iterator_base() + : Ptr(nullptr), NumItemsInBucketLeft(0), NumEntriesLeft(0) {} - friend bool operator==(const key_iterator &X, const key_iterator &Y) { + friend bool operator==(const iterator_base &X, const iterator_base &Y) { return X.NumEntriesLeft == Y.NumEntriesLeft; } - friend bool operator!=(const key_iterator &X, const key_iterator &Y) { + friend bool operator!=(const iterator_base &X, const iterator_base &Y) { return X.NumEntriesLeft != Y.NumEntriesLeft; } - key_iterator &operator++() { // Preincrement + /// Move to the next item. + void advance() { using namespace llvm::support; if (!NumItemsInBucketLeft) { // 'Items' starts with a 16-bit unsigned integer representing the @@ -435,25 +463,58 @@ public: --NumItemsInBucketLeft; assert(NumEntriesLeft); --NumEntriesLeft; + } + + /// Get the start of the item as written by the trait (after the hash and + /// immediately before the key and value length). + const unsigned char *getItem() const { + return Ptr + (NumItemsInBucketLeft ? 0 : 2) + sizeof(hash_value_type); + } + }; + +public: + OnDiskIterableChainedHashTable(offset_type NumBuckets, offset_type NumEntries, + const unsigned char *Buckets, + const unsigned char *Payload, + const unsigned char *Base, + const Info &InfoObj = Info()) + : base_type(NumBuckets, NumEntries, Buckets, Base, InfoObj), + Payload(Payload) {} + + /// \brief Iterates over all of the keys in the table. + class key_iterator : public iterator_base { + Info *InfoObj; + + public: + typedef external_key_type value_type; + + key_iterator(const unsigned char *const Ptr, offset_type NumEntries, + Info *InfoObj) + : iterator_base(Ptr, NumEntries), InfoObj(InfoObj) {} + key_iterator() : iterator_base(), InfoObj() {} + + key_iterator &operator++() { + this->advance(); return *this; } key_iterator operator++(int) { // Postincrement - key_iterator tmp = *this; ++*this; return tmp; + key_iterator tmp = *this; + ++*this; + return tmp; } - value_type operator*() const { - const unsigned char *LocalPtr = Ptr; - if (!NumItemsInBucketLeft) - LocalPtr += 2; // number of items in bucket - LocalPtr += sizeof(hash_value_type); // Skip the hash. + internal_key_type getInternalKey() const { + auto *LocalPtr = this->getItem(); // Determine the length of the key and the data. - const std::pair<offset_type, offset_type> &L = - Info::ReadKeyDataLength(LocalPtr); + auto L = Info::ReadKeyDataLength(LocalPtr); // Read the key. - const internal_key_type &Key = InfoObj->ReadKey(LocalPtr, L.first); - return InfoObj->GetExternalKey(Key); + return InfoObj->ReadKey(LocalPtr, L.first); + } + + value_type operator*() const { + return InfoObj->GetExternalKey(getInternalKey()); } }; @@ -467,10 +528,7 @@ public: } /// \brief Iterates over all the entries in the table, returning the data. - class data_iterator { - const unsigned char *Ptr; - offset_type NumItemsInBucketLeft; - offset_type NumEntriesLeft; + class data_iterator : public iterator_base { Info *InfoObj; public: @@ -478,51 +536,24 @@ public: data_iterator(const unsigned char *const Ptr, offset_type NumEntries, Info *InfoObj) - : Ptr(Ptr), NumItemsInBucketLeft(0), NumEntriesLeft(NumEntries), - InfoObj(InfoObj) {} - data_iterator() - : Ptr(nullptr), NumItemsInBucketLeft(0), NumEntriesLeft(0), - InfoObj(nullptr) {} - - bool operator==(const data_iterator &X) const { - return X.NumEntriesLeft == NumEntriesLeft; - } - bool operator!=(const data_iterator &X) const { - return X.NumEntriesLeft != NumEntriesLeft; - } + : iterator_base(Ptr, NumEntries), InfoObj(InfoObj) {} + data_iterator() : iterator_base(), InfoObj() {} data_iterator &operator++() { // Preincrement - using namespace llvm::support; - if (!NumItemsInBucketLeft) { - // 'Items' starts with a 16-bit unsigned integer representing the - // number of items in this bucket. - NumItemsInBucketLeft = - endian::readNext<uint16_t, little, unaligned>(Ptr); - } - Ptr += sizeof(hash_value_type); // Skip the hash. - // Determine the length of the key and the data. - const std::pair<offset_type, offset_type> &L = - Info::ReadKeyDataLength(Ptr); - Ptr += L.first + L.second; - assert(NumItemsInBucketLeft); - --NumItemsInBucketLeft; - assert(NumEntriesLeft); - --NumEntriesLeft; + this->advance(); return *this; } data_iterator operator++(int) { // Postincrement - data_iterator tmp = *this; ++*this; return tmp; + data_iterator tmp = *this; + ++*this; + return tmp; } value_type operator*() const { - const unsigned char *LocalPtr = Ptr; - if (!NumItemsInBucketLeft) - LocalPtr += 2; // number of items in bucket - LocalPtr += sizeof(hash_value_type); // Skip the hash. + auto *LocalPtr = this->getItem(); // Determine the length of the key and the data. - const std::pair<offset_type, offset_type> &L = - Info::ReadKeyDataLength(LocalPtr); + auto L = Info::ReadKeyDataLength(LocalPtr); // Read the key. const internal_key_type &Key = InfoObj->ReadKey(LocalPtr, L.first); @@ -555,17 +586,12 @@ public: static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj = Info()) { - using namespace llvm::support; assert(Buckets > Base); - assert((reinterpret_cast<uintptr_t>(Buckets) & 0x3) == 0 && - "buckets should be 4-byte aligned."); - - offset_type NumBuckets = - endian::readNext<offset_type, little, aligned>(Buckets); - offset_type NumEntries = - endian::readNext<offset_type, little, aligned>(Buckets); + auto NumBucketsAndEntries = + OnDiskIterableChainedHashTable<Info>::readNumBucketsAndEntries(Buckets); return new OnDiskIterableChainedHashTable<Info>( - NumBuckets, NumEntries, Buckets, Payload, Base, InfoObj); + NumBucketsAndEntries.first, NumBucketsAndEntries.second, + Buckets, Payload, Base, InfoObj); } }; diff --git a/include/llvm/Support/Options.h b/include/llvm/Support/Options.h index 2742d3907c62..7b61b2308f57 100644 --- a/include/llvm/Support/Options.h +++ b/include/llvm/Support/Options.h @@ -71,7 +71,7 @@ private: /// \param Key unique key for option /// \param O option to map to \p Key /// - /// Allocated cl::Options are owened by the OptionRegistry and are deallocated + /// Allocated cl::Options are owned by the OptionRegistry and are deallocated /// on destruction or removal void addOption(void *Key, cl::Option *O); @@ -91,7 +91,7 @@ public: /// Options are keyed off the template parameters to generate unique static /// characters. The template parameters are (1) the type of the data the /// option stores (\p ValT), the class that will read the option (\p Base), - /// and the memeber that the class will store the data into (\p Mem). + /// and the member that the class will store the data into (\p Mem). template <typename ValT, typename Base, ValT(Base::*Mem)> static void registerOption(const char *ArgStr, const char *Desc, const ValT &InitValue) { diff --git a/include/llvm/Support/OutputBuffer.h b/include/llvm/Support/OutputBuffer.h deleted file mode 100644 index 6b98e99e28e0..000000000000 --- a/include/llvm/Support/OutputBuffer.h +++ /dev/null @@ -1,166 +0,0 @@ -//=== OutputBuffer.h - Output Buffer ----------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Methods to output values to a data buffer. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_SUPPORT_OUTPUTBUFFER_H -#define LLVM_SUPPORT_OUTPUTBUFFER_H - -#include <cassert> -#include <string> -#include <vector> - -namespace llvm { - - class OutputBuffer { - /// Output buffer. - std::vector<unsigned char> &Output; - - /// is64Bit/isLittleEndian - This information is inferred from the target - /// machine directly, indicating what header values and flags to set. - bool is64Bit, isLittleEndian; - public: - OutputBuffer(std::vector<unsigned char> &Out, - bool is64bit, bool le) - : Output(Out), is64Bit(is64bit), isLittleEndian(le) {} - - // align - Emit padding into the file until the current output position is - // aligned to the specified power of two boundary. - void align(unsigned Boundary) { - assert(Boundary && (Boundary & (Boundary - 1)) == 0 && - "Must align to 2^k boundary"); - size_t Size = Output.size(); - - if (Size & (Boundary - 1)) { - // Add padding to get alignment to the correct place. - size_t Pad = Boundary - (Size & (Boundary - 1)); - Output.resize(Size + Pad); - } - } - - //===------------------------------------------------------------------===// - // Out Functions - Output the specified value to the data buffer. - - void outbyte(unsigned char X) { - Output.push_back(X); - } - void outhalf(unsigned short X) { - if (isLittleEndian) { - Output.push_back(X & 255); - Output.push_back(X >> 8); - } else { - Output.push_back(X >> 8); - Output.push_back(X & 255); - } - } - void outword(unsigned X) { - if (isLittleEndian) { - Output.push_back((X >> 0) & 255); - Output.push_back((X >> 8) & 255); - Output.push_back((X >> 16) & 255); - Output.push_back((X >> 24) & 255); - } else { - Output.push_back((X >> 24) & 255); - Output.push_back((X >> 16) & 255); - Output.push_back((X >> 8) & 255); - Output.push_back((X >> 0) & 255); - } - } - void outxword(uint64_t X) { - if (isLittleEndian) { - Output.push_back(unsigned(X >> 0) & 255); - Output.push_back(unsigned(X >> 8) & 255); - Output.push_back(unsigned(X >> 16) & 255); - Output.push_back(unsigned(X >> 24) & 255); - Output.push_back(unsigned(X >> 32) & 255); - Output.push_back(unsigned(X >> 40) & 255); - Output.push_back(unsigned(X >> 48) & 255); - Output.push_back(unsigned(X >> 56) & 255); - } else { - Output.push_back(unsigned(X >> 56) & 255); - Output.push_back(unsigned(X >> 48) & 255); - Output.push_back(unsigned(X >> 40) & 255); - Output.push_back(unsigned(X >> 32) & 255); - Output.push_back(unsigned(X >> 24) & 255); - Output.push_back(unsigned(X >> 16) & 255); - Output.push_back(unsigned(X >> 8) & 255); - Output.push_back(unsigned(X >> 0) & 255); - } - } - void outaddr32(unsigned X) { - outword(X); - } - void outaddr64(uint64_t X) { - outxword(X); - } - void outaddr(uint64_t X) { - if (!is64Bit) - outword((unsigned)X); - else - outxword(X); - } - void outstring(const std::string &S, unsigned Length) { - unsigned len_to_copy = static_cast<unsigned>(S.length()) < Length - ? static_cast<unsigned>(S.length()) : Length; - unsigned len_to_fill = static_cast<unsigned>(S.length()) < Length - ? Length - static_cast<unsigned>(S.length()) : 0; - - for (unsigned i = 0; i < len_to_copy; ++i) - outbyte(S[i]); - - for (unsigned i = 0; i < len_to_fill; ++i) - outbyte(0); - } - - //===------------------------------------------------------------------===// - // Fix Functions - Replace an existing entry at an offset. - - void fixhalf(unsigned short X, unsigned Offset) { - unsigned char *P = &Output[Offset]; - P[0] = (X >> (isLittleEndian ? 0 : 8)) & 255; - P[1] = (X >> (isLittleEndian ? 8 : 0)) & 255; - } - void fixword(unsigned X, unsigned Offset) { - unsigned char *P = &Output[Offset]; - P[0] = (X >> (isLittleEndian ? 0 : 24)) & 255; - P[1] = (X >> (isLittleEndian ? 8 : 16)) & 255; - P[2] = (X >> (isLittleEndian ? 16 : 8)) & 255; - P[3] = (X >> (isLittleEndian ? 24 : 0)) & 255; - } - void fixxword(uint64_t X, unsigned Offset) { - unsigned char *P = &Output[Offset]; - P[0] = (X >> (isLittleEndian ? 0 : 56)) & 255; - P[1] = (X >> (isLittleEndian ? 8 : 48)) & 255; - P[2] = (X >> (isLittleEndian ? 16 : 40)) & 255; - P[3] = (X >> (isLittleEndian ? 24 : 32)) & 255; - P[4] = (X >> (isLittleEndian ? 32 : 24)) & 255; - P[5] = (X >> (isLittleEndian ? 40 : 16)) & 255; - P[6] = (X >> (isLittleEndian ? 48 : 8)) & 255; - P[7] = (X >> (isLittleEndian ? 56 : 0)) & 255; - } - void fixaddr(uint64_t X, unsigned Offset) { - if (!is64Bit) - fixword((unsigned)X, Offset); - else - fixxword(X, Offset); - } - - unsigned char &operator[](unsigned Index) { - return Output[Index]; - } - const unsigned char &operator[](unsigned Index) const { - return Output[Index]; - } - }; - -} // end llvm namespace - -#endif // LLVM_SUPPORT_OUTPUTBUFFER_H diff --git a/include/llvm/Support/Path.h b/include/llvm/Support/Path.h index 8fae853e2cf4..955cc991d9b7 100644 --- a/include/llvm/Support/Path.h +++ b/include/llvm/Support/Path.h @@ -61,7 +61,6 @@ public: reference operator*() const { return Component; } pointer operator->() const { return &Component; } const_iterator &operator++(); // preincrement - const_iterator &operator++(int); // postincrement bool operator==(const const_iterator &RHS) const; bool operator!=(const const_iterator &RHS) const { return !(*this == RHS); } @@ -87,7 +86,6 @@ public: reference operator*() const { return Component; } pointer operator->() const { return &Component; } reverse_iterator &operator++(); // preincrement - reverse_iterator &operator++(int); // postincrement bool operator==(const reverse_iterator &RHS) const; bool operator!=(const reverse_iterator &RHS) const { return !(*this == RHS); } }; @@ -218,7 +216,7 @@ StringRef root_name(StringRef path); /// @result The root directory of \a path if it has one, otherwise /// "". StringRef root_directory(StringRef path); - + /// @brief Get root path. /// /// Equivalent to root_name + root_directory. @@ -310,7 +308,7 @@ bool is_separator(char value); /// @result StringRef of the preferred separator, null-terminated. StringRef get_separator(); -/// @brief Get the typical temporary directory for the system, e.g., +/// @brief Get the typical temporary directory for the system, e.g., /// "/var/tmp" or "C:/TEMP" /// /// @param erasedOnReboot Whether to favor a path that is erased on reboot @@ -327,6 +325,22 @@ void system_temp_directory(bool erasedOnReboot, SmallVectorImpl<char> &result); /// @result True if a home directory is set, false otherwise. bool home_directory(SmallVectorImpl<char> &result); +/// @brief Get the user's cache directory. +/// +/// Expect the resulting path to be a directory shared with other +/// applications/services used by the user. Params \p Path1 to \p Path3 can be +/// used to append additional directory names to the resulting path. Recommended +/// pattern is <user_cache_directory>/<vendor>/<application>. +/// +/// @param Result Holds the resulting path. +/// @param Path1 Additional path to be appended to the user's cache directory +/// path. "" can be used to append nothing. +/// @param Path2 Second additional path to be appended. +/// @param Path3 Third additional path to be appended. +/// @result True if a cache directory path is set, false otherwise. +bool user_cache_directory(SmallVectorImpl<char> &Result, const Twine &Path1, + const Twine &Path2 = "", const Twine &Path3 = ""); + /// @brief Has root name? /// /// root_name != "" @@ -403,6 +417,19 @@ bool is_absolute(const Twine &path); /// @result True if the path is relative, false if it is not. bool is_relative(const Twine &path); +/// @brief Remove redundant leading "./" pieces and consecutive separators. +/// +/// @param path Input path. +/// @result The cleaned-up \a path. +StringRef remove_leading_dotslash(StringRef path); + +/// @brief In-place remove any './' and optionally '../' components from a path. +/// +/// @param path processed path +/// @param remove_dot_dot specify if '../' should be removed +/// @result True if path was changed +bool remove_dots(SmallVectorImpl<char> &path, bool remove_dot_dot = false); + } // end namespace path } // end namespace sys } // end namespace llvm diff --git a/include/llvm/Support/PointerLikeTypeTraits.h b/include/llvm/Support/PointerLikeTypeTraits.h index 837082139214..c12d237b2796 100644 --- a/include/llvm/Support/PointerLikeTypeTraits.h +++ b/include/llvm/Support/PointerLikeTypeTraits.h @@ -15,59 +15,70 @@ #ifndef LLVM_SUPPORT_POINTERLIKETYPETRAITS_H #define LLVM_SUPPORT_POINTERLIKETYPETRAITS_H +#include "llvm/Support/AlignOf.h" #include "llvm/Support/DataTypes.h" namespace llvm { - -/// PointerLikeTypeTraits - This is a traits object that is used to handle -/// pointer types and things that are just wrappers for pointers as a uniform -/// entity. -template <typename T> -class PointerLikeTypeTraits { + +/// A traits type that is used to handle pointer types and things that are just +/// wrappers for pointers as a uniform entity. +template <typename T> class PointerLikeTypeTraits { // getAsVoidPointer // getFromVoidPointer // getNumLowBitsAvailable }; +namespace detail { +/// A tiny meta function to compute the log2 of a compile time constant. +template <size_t N> +struct ConstantLog2 + : std::integral_constant<size_t, ConstantLog2<N / 2>::value + 1> {}; +template <> struct ConstantLog2<1> : std::integral_constant<size_t, 0> {}; +} + // Provide PointerLikeTypeTraits for non-cvr pointers. -template<typename T> -class PointerLikeTypeTraits<T*> { -public: - static inline void *getAsVoidPointer(T* P) { return P; } - static inline T *getFromVoidPointer(void *P) { - return static_cast<T*>(P); - } - - /// Note, we assume here that malloc returns objects at least 4-byte aligned. - /// However, this may be wrong, or pointers may be from something other than - /// malloc. In this case, you should specialize this template to reduce this. +template <typename T> struct PointerLikeTypeTraits<T *> { + static inline void *getAsVoidPointer(T *P) { return P; } + static inline T *getFromVoidPointer(void *P) { return static_cast<T *>(P); } + + enum { + NumLowBitsAvailable = detail::ConstantLog2<AlignOf<T>::Alignment>::value + }; +}; + +template <> struct PointerLikeTypeTraits<void *> { + static inline void *getAsVoidPointer(void *P) { return P; } + static inline void *getFromVoidPointer(void *P) { return P; } + + /// Note, we assume here that void* is related to raw malloc'ed memory and + /// that malloc returns objects at least 4-byte aligned. However, this may be + /// wrong, or pointers may be from something other than malloc. In this case, + /// you should specify a real typed pointer or avoid this template. /// /// All clients should use assertions to do a run-time check to ensure that /// this is actually true. enum { NumLowBitsAvailable = 2 }; }; - + // Provide PointerLikeTypeTraits for const pointers. -template<typename T> -class PointerLikeTypeTraits<const T*> { - typedef PointerLikeTypeTraits<T*> NonConst; +template <typename T> class PointerLikeTypeTraits<const T *> { + typedef PointerLikeTypeTraits<T *> NonConst; public: - static inline const void *getAsVoidPointer(const T* P) { - return NonConst::getAsVoidPointer(const_cast<T*>(P)); + static inline const void *getAsVoidPointer(const T *P) { + return NonConst::getAsVoidPointer(const_cast<T *>(P)); } static inline const T *getFromVoidPointer(const void *P) { - return NonConst::getFromVoidPointer(const_cast<void*>(P)); + return NonConst::getFromVoidPointer(const_cast<void *>(P)); } enum { NumLowBitsAvailable = NonConst::NumLowBitsAvailable }; }; // Provide PointerLikeTypeTraits for uintptr_t. -template<> -class PointerLikeTypeTraits<uintptr_t> { +template <> class PointerLikeTypeTraits<uintptr_t> { public: static inline void *getAsVoidPointer(uintptr_t P) { - return reinterpret_cast<void*>(P); + return reinterpret_cast<void *>(P); } static inline uintptr_t getFromVoidPointer(void *P) { return reinterpret_cast<uintptr_t>(P); @@ -75,7 +86,7 @@ public: // No bits are available! enum { NumLowBitsAvailable = 0 }; }; - + } // end namespace llvm #endif diff --git a/include/llvm/Support/PrettyStackTrace.h b/include/llvm/Support/PrettyStackTrace.h index 96afb60d8e51..027f9433969d 100644 --- a/include/llvm/Support/PrettyStackTrace.h +++ b/include/llvm/Support/PrettyStackTrace.h @@ -66,6 +66,18 @@ namespace llvm { void print(raw_ostream &OS) const override; }; + /// Returns the topmost element of the "pretty" stack state. + const void* SavePrettyStackState(); + + /// Restores the topmost element of the "pretty" stack state to State, which + /// should come from a previous call to SavePrettyStackState(). This is + /// useful when using a CrashRecoveryContext in code that also uses + /// PrettyStackTraceEntries, to make sure the stack that's printed if a crash + /// happens after a crash that's been recovered by CrashRecoveryContext + /// doesn't have frames on it that were added in code unwound by the + /// CrashRecoveryContext. + void RestorePrettyStackState(const void* State); + } // end namespace llvm #endif diff --git a/include/llvm/Support/Printable.h b/include/llvm/Support/Printable.h new file mode 100644 index 000000000000..5c1b8d5070d4 --- /dev/null +++ b/include/llvm/Support/Printable.h @@ -0,0 +1,52 @@ +//===--- Printable.h - Print function helpers -------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the Printable struct. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_PRINTABLE_H +#define LLVM_SUPPORT_PRINTABLE_H + +#include <functional> + +namespace llvm { + +class raw_ostream; + +/// Simple wrapper around std::function<void(raw_ostream&)>. +/// This class is usefull to construct print helpers for raw_ostream. +/// +/// Example: +/// Printable PrintRegister(unsigned Register) { +/// return Printable([Register](raw_ostream &OS) { +/// OS << getRegisterName(Register); +/// } +/// } +/// ... OS << PrintRegister(Register); ... +/// +/// Implementation note: Ideally this would just be a typedef, but doing so +/// leads to operator << being ambiguous as function has matching constructors +/// in some STL versions. I have seen the problem on gcc 4.6 libstdc++ and +/// microsoft STL. +class Printable { +public: + std::function<void(raw_ostream &OS)> Print; + Printable(const std::function<void(raw_ostream &OS)> Print) + : Print(Print) {} +}; + +static inline raw_ostream &operator<<(raw_ostream &OS, const Printable &P) { + P.Print(OS); + return OS; +} + +} + +#endif diff --git a/include/llvm/Support/Program.h b/include/llvm/Support/Program.h index b89a0f73ec68..43302101e3e0 100644 --- a/include/llvm/Support/Program.h +++ b/include/llvm/Support/Program.h @@ -67,8 +67,7 @@ struct ProcessInfo { /// \returns The fully qualified path to the first \p Name in \p Paths if it /// exists. \p Name if \p Name has slashes in it. Otherwise an error. ErrorOr<std::string> - findProgramByName(StringRef Name, - ArrayRef<StringRef> Paths = ArrayRef<StringRef>()); + findProgramByName(StringRef Name, ArrayRef<StringRef> Paths = None); // These functions change the specified standard stream (stdin or stdout) to // binary mode. They return errc::success if the specified stream diff --git a/include/llvm/Support/Recycler.h b/include/llvm/Support/Recycler.h index e97f36a735fd..a38050d81903 100644 --- a/include/llvm/Support/Recycler.h +++ b/include/llvm/Support/Recycler.h @@ -28,53 +28,36 @@ namespace llvm { /// void PrintRecyclerStats(size_t Size, size_t Align, size_t FreeListSize); -/// RecyclerStruct - Implementation detail for Recycler. This is a -/// class that the recycler imposes on free'd memory to carve out -/// next/prev pointers. -struct RecyclerStruct { - RecyclerStruct *Prev, *Next; -}; - -template<> -struct ilist_traits<RecyclerStruct> : - public ilist_default_traits<RecyclerStruct> { - static RecyclerStruct *getPrev(const RecyclerStruct *t) { return t->Prev; } - static RecyclerStruct *getNext(const RecyclerStruct *t) { return t->Next; } - static void setPrev(RecyclerStruct *t, RecyclerStruct *p) { t->Prev = p; } - static void setNext(RecyclerStruct *t, RecyclerStruct *n) { t->Next = n; } - - mutable RecyclerStruct Sentinel; - RecyclerStruct *createSentinel() const { - return &Sentinel; - } - static void destroySentinel(RecyclerStruct *) {} - - RecyclerStruct *provideInitialHead() const { return createSentinel(); } - RecyclerStruct *ensureHead(RecyclerStruct*) const { return createSentinel(); } - static void noteHead(RecyclerStruct*, RecyclerStruct*) {} - - static void deleteNode(RecyclerStruct *) { - llvm_unreachable("Recycler's ilist_traits shouldn't see a deleteNode call!"); - } -}; - /// Recycler - This class manages a linked-list of deallocated nodes /// and facilitates reusing deallocated memory in place of allocating /// new memory. /// template<class T, size_t Size = sizeof(T), size_t Align = AlignOf<T>::Alignment> class Recycler { - /// FreeList - Doubly-linked list of nodes that have deleted contents and - /// are not in active use. - /// - iplist<RecyclerStruct> FreeList; + struct FreeNode { + FreeNode *Next; + }; + + /// List of nodes that have deleted contents and are not in active use. + FreeNode *FreeList = nullptr; + + FreeNode *pop_val() { + auto *Val = FreeList; + FreeList = FreeList->Next; + return Val; + } + + void push(FreeNode *N) { + N->Next = FreeList; + FreeList = N; + } public: ~Recycler() { // If this fails, either the callee has lost track of some allocation, // or the callee isn't tracking allocations and should just call // clear() before deleting the Recycler. - assert(FreeList.empty() && "Non-empty recycler deleted!"); + assert(!FreeList && "Non-empty recycler deleted!"); } /// clear - Release all the tracked allocations to the allocator. The @@ -82,8 +65,8 @@ public: /// deleted; calling clear is one way to ensure this. template<class AllocatorType> void clear(AllocatorType &Allocator) { - while (!FreeList.empty()) { - T *t = reinterpret_cast<T *>(FreeList.remove(FreeList.begin())); + while (FreeList) { + T *t = reinterpret_cast<T *>(pop_val()); Allocator.Deallocate(t); } } @@ -93,9 +76,7 @@ public: /// /// There is no need to traverse the free list, pulling all the objects into /// cache. - void clear(BumpPtrAllocator&) { - FreeList.clearAndLeakNodesUnsafely(); - } + void clear(BumpPtrAllocator &) { FreeList = nullptr; } template<class SubClass, class AllocatorType> SubClass *Allocate(AllocatorType &Allocator) { @@ -103,9 +84,8 @@ public: "Recycler allocation alignment is less than object align!"); static_assert(sizeof(SubClass) <= Size, "Recycler allocation size is less than object size!"); - return !FreeList.empty() ? - reinterpret_cast<SubClass *>(FreeList.remove(FreeList.begin())) : - static_cast<SubClass *>(Allocator.Allocate(Size, Align)); + return FreeList ? reinterpret_cast<SubClass *>(pop_val()) + : static_cast<SubClass *>(Allocator.Allocate(Size, Align)); } template<class AllocatorType> @@ -115,14 +95,20 @@ public: template<class SubClass, class AllocatorType> void Deallocate(AllocatorType & /*Allocator*/, SubClass* Element) { - FreeList.push_front(reinterpret_cast<RecyclerStruct *>(Element)); + push(reinterpret_cast<FreeNode *>(Element)); } - void PrintStats() { - PrintRecyclerStats(Size, Align, FreeList.size()); - } + void PrintStats(); }; +template <class T, size_t Size, size_t Align> +void Recycler<T, Size, Align>::PrintStats() { + size_t S = 0; + for (auto *I = FreeList; I; I = I->Next) + ++S; + PrintRecyclerStats(Size, Align, S); +} + } #endif diff --git a/include/llvm/Support/Registry.h b/include/llvm/Support/Registry.h index 95c4e96f7f29..bbea97b289a6 100644 --- a/include/llvm/Support/Registry.h +++ b/include/llvm/Support/Registry.h @@ -37,7 +37,6 @@ namespace llvm { std::unique_ptr<T> instantiate() const { return Ctor(); } }; - /// Traits for registry entries. If using other than SimpleRegistryEntry, it /// is necessary to define an alternate traits class. template <typename T> @@ -53,7 +52,6 @@ namespace llvm { static const char *descof(const entry &Entry) { return Entry.getDesc(); } }; - /// A global registry used in conjunction with static constructors to make /// pluggable components (like targets or garbage collectors) "just work" when /// linked with an executable. @@ -102,7 +100,6 @@ namespace llvm { } }; - /// Iterators for registry entries. /// class iterator { @@ -122,10 +119,9 @@ namespace llvm { static iterator end() { return iterator(nullptr); } static iterator_range<iterator> entries() { - return iterator_range<iterator>(begin(), end()); + return make_range(begin(), end()); } - /// Abstract base class for registry listeners, which are informed when new /// entries are added to the registry. Simply subclass and instantiate: /// @@ -160,7 +156,7 @@ namespace llvm { } public: - listener() : Prev(ListenerTail), Next(0) { + listener() : Prev(ListenerTail), Next(nullptr) { if (Prev) Prev->Next = this; else @@ -180,7 +176,6 @@ namespace llvm { } }; - /// A static registration template. Use like such: /// /// Registry<Collector>::Add<FancyGC> @@ -210,7 +205,6 @@ namespace llvm { }; /// Registry::Parser now lives in llvm/Support/RegistryParser.h. - }; // Since these are defined in a header file, plugins must be sure to export @@ -228,6 +222,6 @@ namespace llvm { template <typename T, typename U> typename Registry<T,U>::listener *Registry<T,U>::ListenerTail; -} +} // end namespace llvm -#endif +#endif // LLVM_SUPPORT_REGISTRY_H diff --git a/include/llvm/Support/SMLoc.h b/include/llvm/Support/SMLoc.h index d5b4c57a8fd6..c6e9a14e82ac 100644 --- a/include/llvm/Support/SMLoc.h +++ b/include/llvm/Support/SMLoc.h @@ -22,6 +22,7 @@ namespace llvm { /// Represents a location in source code. class SMLoc { const char *Ptr; + public: SMLoc() : Ptr(nullptr) {} @@ -53,11 +54,10 @@ public: assert(Start.isValid() == End.isValid() && "Start and end should either both be valid or both be invalid!"); } - + bool isValid() const { return Start.isValid(); } }; - + } // end namespace llvm #endif - diff --git a/include/llvm/Support/ScaledNumber.h b/include/llvm/Support/ScaledNumber.h index 0a4262b7eec5..c6421efc8b49 100644 --- a/include/llvm/Support/ScaledNumber.h +++ b/include/llvm/Support/ScaledNumber.h @@ -282,7 +282,7 @@ int compare(DigitsT LDigits, int16_t LScale, DigitsT RDigits, int16_t RScale) { /// /// As a convenience, returns the matching scale. If the output value of one /// number is zero, returns the scale of the other. If both are zero, which -/// scale is returned is unspecifed. +/// scale is returned is unspecified. template <class DigitsT> int16_t matchScales(DigitsT &LDigits, int16_t &LScale, DigitsT &RDigits, int16_t &RScale) { @@ -334,7 +334,7 @@ std::pair<DigitsT, int16_t> getSum(DigitsT LDigits, int16_t LScale, DigitsT RDigits, int16_t RScale) { static_assert(!std::numeric_limits<DigitsT>::is_signed, "expected unsigned"); - // Check inputs up front. This is only relevent if addition overflows, but + // Check inputs up front. This is only relevant if addition overflows, but // testing here should catch more bugs. assert(LScale < INT16_MAX && "scale too large"); assert(RScale < INT16_MAX && "scale too large"); diff --git a/include/llvm/Support/Signals.h b/include/llvm/Support/Signals.h index 7e165d7f3a42..2a4d84bd891a 100644 --- a/include/llvm/Support/Signals.h +++ b/include/llvm/Support/Signals.h @@ -47,6 +47,9 @@ namespace sys { /// \brief Print the stack trace using the given \c raw_ostream object. void PrintStackTrace(raw_ostream &OS); + // Run all registered signal handlers. + void RunSignalHandlers(); + /// AddSignalHandler - Add a function to be called when an abort/kill signal /// is delivered to the process. The handler can have a cookie passed to it /// to identify what instance of the handler it is. diff --git a/include/llvm/Support/StreamingMemoryObject.h b/include/llvm/Support/StreamingMemoryObject.h index 7cb6438d1342..a5980c235946 100644 --- a/include/llvm/Support/StreamingMemoryObject.h +++ b/include/llvm/Support/StreamingMemoryObject.h @@ -50,8 +50,10 @@ public: /// starts (although it can be called anytime). void setKnownObjectSize(size_t size); + /// The number of bytes read at a time from the data streamer. + static const uint32_t kChunkSize = 4096 * 4; + private: - const static uint32_t kChunkSize = 4096 * 4; mutable std::vector<unsigned char> Bytes; std::unique_ptr<DataStreamer> Streamer; mutable size_t BytesRead; // Bytes read from stream diff --git a/include/llvm/Support/StringSaver.h b/include/llvm/Support/StringSaver.h index f3853ee91570..38fb7bb38339 100644 --- a/include/llvm/Support/StringSaver.h +++ b/include/llvm/Support/StringSaver.h @@ -18,25 +18,15 @@ namespace llvm { /// \brief Saves strings in the inheritor's stable storage and returns a stable /// raw character pointer. -class StringSaver { -protected: - ~StringSaver() {} - virtual const char *saveImpl(StringRef S); +class StringSaver final { + BumpPtrAllocator &Alloc; public: StringSaver(BumpPtrAllocator &Alloc) : Alloc(Alloc) {} const char *save(const char *S) { return save(StringRef(S)); } - const char *save(StringRef S) { return saveImpl(S); } + const char *save(StringRef S); const char *save(const Twine &S) { return save(StringRef(S.str())); } const char *save(std::string &S) { return save(StringRef(S)); } - -private: - BumpPtrAllocator &Alloc; -}; - -class BumpPtrStringSaver final : public StringSaver { -public: - BumpPtrStringSaver(BumpPtrAllocator &Alloc) : StringSaver(Alloc) {} }; } #endif diff --git a/include/llvm/Support/TargetParser.h b/include/llvm/Support/TargetParser.h index dab724895e86..c21019d0c5b8 100644 --- a/include/llvm/Support/TargetParser.h +++ b/include/llvm/Support/TargetParser.h @@ -20,7 +20,7 @@ #include <vector> namespace llvm { - class StringRef; +class StringRef; // Target specific information into their own namespaces. These should be // generated from TableGen because the information is already there, and there @@ -29,177 +29,117 @@ namespace llvm { // even if the back-end is not compiled with LLVM, plus we need to create a new // back-end to TableGen to create these clean tables. namespace ARM { - // FPU names. - enum FPUKind { - FK_INVALID = 0, - FK_NONE, - FK_VFP, - FK_VFPV2, - FK_VFPV3, - FK_VFPV3_FP16, - FK_VFPV3_D16, - FK_VFPV3_D16_FP16, - FK_VFPV3XD, - FK_VFPV3XD_FP16, - FK_VFPV4, - FK_VFPV4_D16, - FK_FPV4_SP_D16, - FK_FPV5_D16, - FK_FPV5_SP_D16, - FK_FP_ARMV8, - FK_NEON, - FK_NEON_FP16, - FK_NEON_VFPV4, - FK_NEON_FP_ARMV8, - FK_CRYPTO_NEON_FP_ARMV8, - FK_SOFTVFP, - FK_LAST - }; - - // FPU Version - enum FPUVersion { - FV_NONE = 0, - FV_VFPV2, - FV_VFPV3, - FV_VFPV3_FP16, - FV_VFPV4, - FV_VFPV5 - }; - - // An FPU name implies one of three levels of Neon support: - enum NeonSupportLevel { - NS_None = 0, ///< No Neon - NS_Neon, ///< Neon - NS_Crypto ///< Neon with Crypto - }; - - // An FPU name restricts the FPU in one of three ways: - enum FPURestriction { - FR_None = 0, ///< No restriction - FR_D16, ///< Only 16 D registers - FR_SP_D16 ///< Only single-precision instructions, with 16 D registers - }; - - // Arch names. - enum ArchKind { - AK_INVALID = 0, - AK_ARMV2, - AK_ARMV2A, - AK_ARMV3, - AK_ARMV3M, - AK_ARMV4, - AK_ARMV4T, - AK_ARMV5T, - AK_ARMV5TE, - AK_ARMV5TEJ, - AK_ARMV6, - AK_ARMV6K, - AK_ARMV6T2, - AK_ARMV6Z, - AK_ARMV6ZK, - AK_ARMV6M, - AK_ARMV6SM, - AK_ARMV7A, - AK_ARMV7R, - AK_ARMV7M, - AK_ARMV7EM, - AK_ARMV8A, - AK_ARMV8_1A, - // Non-standard Arch names. - AK_IWMMXT, - AK_IWMMXT2, - AK_XSCALE, - AK_ARMV5, - AK_ARMV5E, - AK_ARMV6J, - AK_ARMV6HL, - AK_ARMV7, - AK_ARMV7L, - AK_ARMV7HL, - AK_ARMV7S, - AK_LAST - }; - - // Arch extension modifiers for CPUs. - enum ArchExtKind { - AEK_INVALID = 0, - AEK_CRC, - AEK_CRYPTO, - AEK_FP, - AEK_HWDIV, - AEK_MP, - AEK_SIMD, - AEK_SEC, - AEK_VIRT, - // Unsupported extensions. - AEK_OS, - AEK_IWMMXT, - AEK_IWMMXT2, - AEK_MAVERICK, - AEK_XSCALE, - AEK_LAST - }; - - // ISA kinds. - enum ISAKind { - IK_INVALID = 0, - IK_ARM, - IK_THUMB, - IK_AARCH64 - }; - - // Endianness - // FIXME: BE8 vs. BE32? - enum EndianKind { - EK_INVALID = 0, - EK_LITTLE, - EK_BIG - }; - - // v6/v7/v8 Profile - enum ProfileKind { - PK_INVALID = 0, - PK_A, - PK_R, - PK_M - }; -} // namespace ARM -// Target Parsers, one per architecture. -class ARMTargetParser { - static StringRef getFPUSynonym(StringRef FPU); - static StringRef getArchSynonym(StringRef Arch); - -public: - static StringRef getCanonicalArchName(StringRef Arch); - - // Information by ID - static const char * getFPUName(unsigned FPUKind); - static unsigned getFPUVersion(unsigned FPUKind); - static unsigned getFPUNeonSupportLevel(unsigned FPUKind); - static unsigned getFPURestriction(unsigned FPUKind); - // FIXME: This should be moved to TargetTuple once it exists - static bool getFPUFeatures(unsigned FPUKind, - std::vector<const char*> &Features); - static const char * getArchName(unsigned ArchKind); - static unsigned getArchAttr(unsigned ArchKind); - static const char * getCPUAttr(unsigned ArchKind); - static const char * getSubArch(unsigned ArchKind); - static const char * getArchExtName(unsigned ArchExtKind); - static const char * getDefaultCPU(StringRef Arch); - - // Parser - static unsigned parseFPU(StringRef FPU); - static unsigned parseArch(StringRef Arch); - static unsigned parseArchExt(StringRef ArchExt); - static unsigned parseCPUArch(StringRef CPU); - static unsigned parseArchISA(StringRef Arch); - static unsigned parseArchEndian(StringRef Arch); - static unsigned parseArchProfile(StringRef Arch); - static unsigned parseArchVersion(StringRef Arch); +// FPU names. +enum FPUKind { +#define ARM_FPU(NAME, KIND, VERSION, NEON_SUPPORT, RESTRICTION) KIND, +#include "ARMTargetParser.def" + FK_LAST +}; + +// FPU Version +enum FPUVersion { + FV_NONE = 0, + FV_VFPV2, + FV_VFPV3, + FV_VFPV3_FP16, + FV_VFPV4, + FV_VFPV5 +}; + +// An FPU name implies one of three levels of Neon support: +enum NeonSupportLevel { + NS_None = 0, ///< No Neon + NS_Neon, ///< Neon + NS_Crypto ///< Neon with Crypto +}; + +// An FPU name restricts the FPU in one of three ways: +enum FPURestriction { + FR_None = 0, ///< No restriction + FR_D16, ///< Only 16 D registers + FR_SP_D16 ///< Only single-precision instructions, with 16 D registers +}; + +// Arch names. +enum ArchKind { +#define ARM_ARCH(NAME, ID, CPU_ATTR, SUB_ARCH, ARCH_ATTR, ARCH_FPU, ARCH_BASE_EXT) ID, +#include "ARMTargetParser.def" + AK_LAST +}; +// Arch extension modifiers for CPUs. +enum ArchExtKind : unsigned { + AEK_INVALID = 0x0, + AEK_NONE = 0x1, + AEK_CRC = 0x2, + AEK_CRYPTO = 0x4, + AEK_FP = 0x8, + AEK_HWDIV = 0x10, + AEK_HWDIVARM = 0x20, + AEK_MP = 0x40, + AEK_SIMD = 0x80, + AEK_SEC = 0x100, + AEK_VIRT = 0x200, + AEK_DSP = 0x400, + AEK_FP16 = 0x800, + // Unsupported extensions. + AEK_OS = 0x8000000, + AEK_IWMMXT = 0x10000000, + AEK_IWMMXT2 = 0x20000000, + AEK_MAVERICK = 0x40000000, + AEK_XSCALE = 0x80000000, }; +// ISA kinds. +enum ISAKind { IK_INVALID = 0, IK_ARM, IK_THUMB, IK_AARCH64 }; + +// Endianness +// FIXME: BE8 vs. BE32? +enum EndianKind { EK_INVALID = 0, EK_LITTLE, EK_BIG }; + +// v6/v7/v8 Profile +enum ProfileKind { PK_INVALID = 0, PK_A, PK_R, PK_M }; + +StringRef getCanonicalArchName(StringRef Arch); + +// Information by ID +StringRef getFPUName(unsigned FPUKind); +unsigned getFPUVersion(unsigned FPUKind); +unsigned getFPUNeonSupportLevel(unsigned FPUKind); +unsigned getFPURestriction(unsigned FPUKind); + +// FIXME: These should be moved to TargetTuple once it exists +bool getFPUFeatures(unsigned FPUKind, std::vector<const char *> &Features); +bool getHWDivFeatures(unsigned HWDivKind, std::vector<const char *> &Features); +bool getExtensionFeatures(unsigned Extensions, + std::vector<const char*> &Features); + +StringRef getArchName(unsigned ArchKind); +unsigned getArchAttr(unsigned ArchKind); +StringRef getCPUAttr(unsigned ArchKind); +StringRef getSubArch(unsigned ArchKind); +StringRef getArchExtName(unsigned ArchExtKind); +const char *getArchExtFeature(StringRef ArchExt); +StringRef getHWDivName(unsigned HWDivKind); + +// Information by Name +unsigned getDefaultFPU(StringRef CPU, unsigned ArchKind); +unsigned getDefaultExtensions(StringRef CPU, unsigned ArchKind); +StringRef getDefaultCPU(StringRef Arch); + +// Parser +unsigned parseHWDiv(StringRef HWDiv); +unsigned parseFPU(StringRef FPU); +unsigned parseArch(StringRef Arch); +unsigned parseArchExt(StringRef ArchExt); +unsigned parseCPUArch(StringRef CPU); +unsigned parseArchISA(StringRef Arch); +unsigned parseArchEndian(StringRef Arch); +unsigned parseArchProfile(StringRef Arch); +unsigned parseArchVersion(StringRef Arch); + +} // namespace ARM } // namespace llvm #endif diff --git a/include/llvm/Support/TargetRegistry.h b/include/llvm/Support/TargetRegistry.h index 40bf6fb20c9f..aec181b1d266 100644 --- a/include/llvm/Support/TargetRegistry.h +++ b/include/llvm/Support/TargetRegistry.h @@ -115,7 +115,7 @@ public: const MCRegisterInfo &MRI, const Triple &TT, StringRef CPU); typedef MCTargetAsmParser *(*MCAsmParserCtorTy)( - MCSubtargetInfo &STI, MCAsmParser &P, const MCInstrInfo &MII, + const MCSubtargetInfo &STI, MCAsmParser &P, const MCInstrInfo &MII, const MCTargetOptions &Options); typedef MCDisassembler *(*MCDisassemblerCtorTy)(const Target &T, const MCSubtargetInfo &STI, @@ -141,7 +141,8 @@ public: typedef MCStreamer *(*COFFStreamerCtorTy)(MCContext &Ctx, MCAsmBackend &TAB, raw_pwrite_stream &OS, MCCodeEmitter *Emitter, - bool RelaxAll); + bool RelaxAll, + bool IncrementalLinkerCompatible); typedef MCTargetStreamer *(*NullTargetStreamerCtorTy)(MCStreamer &S); typedef MCTargetStreamer *(*AsmTargetStreamerCtorTy)( MCStreamer &S, formatted_raw_ostream &OS, MCInstPrinter *InstPrint, @@ -382,7 +383,7 @@ public: /// /// \param Parser The target independent parser implementation to use for /// parsing and lexing. - MCTargetAsmParser *createMCAsmParser(MCSubtargetInfo &STI, + MCTargetAsmParser *createMCAsmParser(const MCSubtargetInfo &STI, MCAsmParser &Parser, const MCInstrInfo &MII, const MCTargetOptions &Options) const { @@ -437,6 +438,7 @@ public: MCAsmBackend &TAB, raw_pwrite_stream &OS, MCCodeEmitter *Emitter, const MCSubtargetInfo &STI, bool RelaxAll, + bool IncrementalLinkerCompatible, bool DWARFMustBeAtTheEnd) const { MCStreamer *S; switch (T.getObjectFormat()) { @@ -444,7 +446,8 @@ public: llvm_unreachable("Unknown object format"); case Triple::COFF: assert(T.isOSWindows() && "only Windows COFF is supported"); - S = COFFStreamerCtorFn(Ctx, TAB, OS, Emitter, RelaxAll); + S = COFFStreamerCtorFn(Ctx, TAB, OS, Emitter, RelaxAll, + IncrementalLinkerCompatible); break; case Triple::MachO: if (MachOStreamerCtorFn) @@ -1133,8 +1136,8 @@ template <class MCAsmParserImpl> struct RegisterMCAsmParser { } private: - static MCTargetAsmParser *Allocator(MCSubtargetInfo &STI, MCAsmParser &P, - const MCInstrInfo &MII, + static MCTargetAsmParser *Allocator(const MCSubtargetInfo &STI, + MCAsmParser &P, const MCInstrInfo &MII, const MCTargetOptions &Options) { return new MCAsmParserImpl(STI, P, MII, Options); } diff --git a/include/llvm/Support/TargetSelect.h b/include/llvm/Support/TargetSelect.h index a86e953f00ea..582785cb69a5 100644 --- a/include/llvm/Support/TargetSelect.h +++ b/include/llvm/Support/TargetSelect.h @@ -25,11 +25,11 @@ extern "C" { #define LLVM_TARGET(TargetName) void LLVMInitialize##TargetName##Target(); #include "llvm/Config/Targets.def" - + // Declare all of the target-MC-initialization functions that are available. #define LLVM_TARGET(TargetName) void LLVMInitialize##TargetName##TargetMC(); #include "llvm/Config/Targets.def" - + // Declare all of the available assembly printer initialization functions. #define LLVM_ASM_PRINTER(TargetName) void LLVMInitialize##TargetName##AsmPrinter(); #include "llvm/Config/AsmPrinters.def" @@ -54,7 +54,7 @@ namespace llvm { #define LLVM_TARGET(TargetName) LLVMInitialize##TargetName##TargetInfo(); #include "llvm/Config/Targets.def" } - + /// InitializeAllTargets - The main program should call this function if it /// wants access to all available target machines that LLVM is configured to /// support, to make them available via the TargetRegistry. @@ -67,7 +67,7 @@ namespace llvm { #define LLVM_TARGET(TargetName) LLVMInitialize##TargetName##Target(); #include "llvm/Config/Targets.def" } - + /// InitializeAllTargetMCs - The main program should call this function if it /// wants access to all available target MC that LLVM is configured to /// support, to make them available via the TargetRegistry. @@ -77,7 +77,7 @@ namespace llvm { #define LLVM_TARGET(TargetName) LLVMInitialize##TargetName##TargetMC(); #include "llvm/Config/Targets.def" } - + /// InitializeAllAsmPrinters - The main program should call this function if /// it wants all asm printers that LLVM is configured to support, to make them /// available via the TargetRegistry. @@ -87,7 +87,7 @@ namespace llvm { #define LLVM_ASM_PRINTER(TargetName) LLVMInitialize##TargetName##AsmPrinter(); #include "llvm/Config/AsmPrinters.def" } - + /// InitializeAllAsmParsers - The main program should call this function if it /// wants all asm parsers that LLVM is configured to support, to make them /// available via the TargetRegistry. @@ -97,7 +97,7 @@ namespace llvm { #define LLVM_ASM_PARSER(TargetName) LLVMInitialize##TargetName##AsmParser(); #include "llvm/Config/AsmParsers.def" } - + /// InitializeAllDisassemblers - The main program should call this function if /// it wants all disassemblers that LLVM is configured to support, to make /// them available via the TargetRegistry. @@ -107,9 +107,9 @@ namespace llvm { #define LLVM_DISASSEMBLER(TargetName) LLVMInitialize##TargetName##Disassembler(); #include "llvm/Config/Disassemblers.def" } - + /// InitializeNativeTarget - The main program should call this function to - /// initialize the native target corresponding to the host. This is useful + /// initialize the native target corresponding to the host. This is useful /// for JIT applications to ensure that the target gets linked in correctly. /// /// It is legal for a client to make multiple calls to this function. @@ -123,7 +123,7 @@ namespace llvm { #else return true; #endif - } + } /// InitializeNativeTargetAsmPrinter - The main program should call /// this function to initialize the native target asm printer. @@ -135,7 +135,7 @@ namespace llvm { #else return true; #endif - } + } /// InitializeNativeTargetAsmParser - The main program should call /// this function to initialize the native target asm parser. @@ -147,7 +147,7 @@ namespace llvm { #else return true; #endif - } + } /// InitializeNativeTargetDisassembler - The main program should call /// this function to initialize the native target disassembler. @@ -159,8 +159,7 @@ namespace llvm { #else return true; #endif - } - + } } #endif diff --git a/include/llvm/Support/ThreadPool.h b/include/llvm/Support/ThreadPool.h new file mode 100644 index 000000000000..745334db4450 --- /dev/null +++ b/include/llvm/Support/ThreadPool.h @@ -0,0 +1,136 @@ +//===-- llvm/Support/ThreadPool.h - A ThreadPool implementation -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines a crude C++11 based thread pool. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_THREAD_POOL_H +#define LLVM_SUPPORT_THREAD_POOL_H + +#include "llvm/Support/thread.h" + +#ifdef _MSC_VER +// concrt.h depends on eh.h for __uncaught_exception declaration +// even if we disable exceptions. +#include <eh.h> + +// Disable warnings from ppltasks.h transitively included by <future>. +#pragma warning(push) +#pragma warning(disable:4530) +#pragma warning(disable:4062) +#endif + +#include <future> + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#include <condition_variable> +#include <functional> +#include <memory> +#include <mutex> +#include <queue> +#include <utility> + +namespace llvm { + +/// A ThreadPool for asynchronous parallel execution on a defined number of +/// threads. +/// +/// The pool keeps a vector of threads alive, waiting on a condition variable +/// for some work to become available. +class ThreadPool { +public: +#ifndef _MSC_VER + using VoidTy = void; + using TaskTy = std::function<void()>; + using PackagedTaskTy = std::packaged_task<void()>; +#else + // MSVC 2013 has a bug and can't use std::packaged_task<void()>; + // We force it to use bool(bool) instead. + using VoidTy = bool; + using TaskTy = std::function<bool(bool)>; + using PackagedTaskTy = std::packaged_task<bool(bool)>; +#endif + + /// Construct a pool with the number of core available on the system (or + /// whatever the value returned by std::thread::hardware_concurrency() is). + ThreadPool(); + + /// Construct a pool of \p ThreadCount threads + ThreadPool(unsigned ThreadCount); + + /// Blocking destructor: the pool will wait for all the threads to complete. + ~ThreadPool(); + + /// Asynchronous submission of a task to the pool. The returned future can be + /// used to wait for the task to finish and is *non-blocking* on destruction. + template <typename Function, typename... Args> + inline std::shared_future<VoidTy> async(Function &&F, Args &&... ArgList) { + auto Task = + std::bind(std::forward<Function>(F), std::forward<Args>(ArgList)...); +#ifndef _MSC_VER + return asyncImpl(std::move(Task)); +#else + // This lambda has to be marked mutable because MSVC 2013's std::bind call + // operator isn't const qualified. + return asyncImpl([Task](VoidTy) mutable -> VoidTy { + Task(); + return VoidTy(); + }); +#endif + } + + /// Asynchronous submission of a task to the pool. The returned future can be + /// used to wait for the task to finish and is *non-blocking* on destruction. + template <typename Function> + inline std::shared_future<VoidTy> async(Function &&F) { +#ifndef _MSC_VER + return asyncImpl(std::forward<Function>(F)); +#else + return asyncImpl([F] (VoidTy) -> VoidTy { F(); return VoidTy(); }); +#endif + } + + /// Blocking wait for all the threads to complete and the queue to be empty. + /// It is an error to try to add new tasks while blocking on this call. + void wait(); + +private: + /// Asynchronous submission of a task to the pool. The returned future can be + /// used to wait for the task to finish and is *non-blocking* on destruction. + std::shared_future<VoidTy> asyncImpl(TaskTy F); + + /// Threads in flight + std::vector<llvm::thread> Threads; + + /// Tasks waiting for execution in the pool. + std::queue<PackagedTaskTy> Tasks; + + /// Locking and signaling for accessing the Tasks queue. + std::mutex QueueLock; + std::condition_variable QueueCondition; + + /// Locking and signaling for job completion + std::mutex CompletionLock; + std::condition_variable CompletionCondition; + + /// Keep track of the number of thread actually busy + std::atomic<unsigned> ActiveThreads; + +#if LLVM_ENABLE_THREADS // avoids warning for unused variable + /// Signal for the destruction of the pool, asking thread to exit. + bool EnableFlag; +#endif +}; +} + +#endif // LLVM_SUPPORT_THREAD_POOL_H diff --git a/include/llvm/Support/Threading.h b/include/llvm/Support/Threading.h index 3cca1d6a9913..9007c132a99a 100644 --- a/include/llvm/Support/Threading.h +++ b/include/llvm/Support/Threading.h @@ -21,7 +21,7 @@ namespace llvm { bool llvm_is_multithreaded(); /// llvm_execute_on_thread - Execute the given \p UserFn on a separate - /// thread, passing it the provided \p UserData and waits for thread + /// thread, passing it the provided \p UserData and waits for thread /// completion. /// /// This function does not guarantee that the code will actually be executed diff --git a/include/llvm/Support/Timer.h b/include/llvm/Support/Timer.h index 2cd30e2aaf32..499fe7b7e70c 100644 --- a/include/llvm/Support/Timer.h +++ b/include/llvm/Support/Timer.h @@ -30,26 +30,25 @@ class TimeRecord { ssize_t MemUsed; // Memory allocated (in bytes) public: TimeRecord() : WallTime(0), UserTime(0), SystemTime(0), MemUsed(0) {} - + /// getCurrentTime - Get the current time and memory usage. If Start is true /// we get the memory usage before the time, otherwise we get time before /// memory usage. This matters if the time to get the memory usage is /// significant and shouldn't be counted as part of a duration. static TimeRecord getCurrentTime(bool Start = true); - - double getProcessTime() const { return UserTime+SystemTime; } + + double getProcessTime() const { return UserTime + SystemTime; } double getUserTime() const { return UserTime; } double getSystemTime() const { return SystemTime; } double getWallTime() const { return WallTime; } ssize_t getMemUsed() const { return MemUsed; } - - + // operator< - Allow sorting. bool operator<(const TimeRecord &T) const { // Sort by Wall Time elapsed, as it is the only thing really accurate return WallTime < T.WallTime; } - + void operator+=(const TimeRecord &RHS) { WallTime += RHS.WallTime; UserTime += RHS.UserTime; @@ -62,12 +61,12 @@ public: SystemTime -= RHS.SystemTime; MemUsed -= RHS.MemUsed; } - - /// print - Print the current timer to standard error, and reset the "Started" - /// flag. + + /// Print the current time record to \p OS, with a breakdown showing + /// contributions to the \p Total time record. void print(const TimeRecord &Total, raw_ostream &OS) const; }; - + /// Timer - This class is used to track the amount of time spent between /// invocations of its startTimer()/stopTimer() methods. Given appropriate OS /// support it can also keep track of the RSS of the program at various points. @@ -77,11 +76,13 @@ public: /// if they are never started. /// class Timer { - TimeRecord Time; + TimeRecord Time; // The total time captured + TimeRecord StartTime; // The time startTimer() was last called std::string Name; // The name of this time variable. - bool Started; // Has this time variable ever been started? + bool Running; // Is the timer currently running? + bool Triggered; // Has the timer ever been triggered? TimerGroup *TG; // The TimerGroup this Timer is in. - + Timer **Prev, *Next; // Doubly linked list of timers in the group. public: explicit Timer(StringRef N) : TG(nullptr) { init(N); } @@ -99,25 +100,31 @@ public: explicit Timer() : TG(nullptr) {} void init(StringRef N); void init(StringRef N, TimerGroup &tg); - + const std::string &getName() const { return Name; } bool isInitialized() const { return TG != nullptr; } - - /// startTimer - Start the timer running. Time between calls to - /// startTimer/stopTimer is counted by the Timer class. Note that these calls - /// must be correctly paired. - /// + + /// Check if startTimer() has ever been called on this timer. + bool hasTriggered() const { return Triggered; } + + /// Start the timer running. Time between calls to startTimer/stopTimer is + /// counted by the Timer class. Note that these calls must be correctly + /// paired. void startTimer(); - /// stopTimer - Stop the timer. - /// + /// Stop the timer. void stopTimer(); + /// Clear the timer state. + void clear(); + + /// Return the duration for which this timer has been running. + TimeRecord getTotalTime() const { return Time; } + private: friend class TimerGroup; }; - /// The TimeRegion class is used as a helper class to call the startTimer() and /// stopTimer() methods of the Timer class. When the object is constructed, it /// starts the timer specified as its argument. When it is destroyed, it stops @@ -126,6 +133,7 @@ private: class TimeRegion { Timer *T; TimeRegion(const TimeRegion &) = delete; + public: explicit TimeRegion(Timer &t) : T(&t) { T->startTimer(); @@ -138,7 +146,6 @@ public: } }; - /// NamedRegionTimer - This class is basically a combination of TimeRegion and /// Timer. It allows you to declare a new timer, AND specify the region to /// time, all in one statement. All timers with the same name are merged. This @@ -151,7 +158,6 @@ struct NamedRegionTimer : public TimeRegion { bool Enabled = true); }; - /// The TimerGroup class is used to group together related timers into a single /// report that is printed when the TimerGroup is destroyed. It is illegal to /// destroy a TimerGroup object before all of the Timers in it are gone. A @@ -160,11 +166,12 @@ struct NamedRegionTimer : public TimeRegion { class TimerGroup { std::string Name; Timer *FirstTimer; // First timer in the group. - std::vector<std::pair<TimeRecord, std::string> > TimersToPrint; - + std::vector<std::pair<TimeRecord, std::string>> TimersToPrint; + TimerGroup **Prev, *Next; // Doubly linked list of TimerGroup's. TimerGroup(const TimerGroup &TG) = delete; void operator=(const TimerGroup &TG) = delete; + public: explicit TimerGroup(StringRef name); ~TimerGroup(); @@ -173,10 +180,10 @@ public: /// print - Print any started timers in this group and zero them. void print(raw_ostream &OS); - + /// printAll - This static method prints all timers and clears them all out. static void printAll(raw_ostream &OS); - + private: friend class Timer; void addTimer(Timer &T); diff --git a/include/llvm/Support/TrailingObjects.h b/include/llvm/Support/TrailingObjects.h new file mode 100644 index 000000000000..8529746eeccc --- /dev/null +++ b/include/llvm/Support/TrailingObjects.h @@ -0,0 +1,349 @@ +//===--- TrailingObjects.h - Variable-length classes ------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This header defines support for implementing classes that have +/// some trailing object (or arrays of objects) appended to them. The +/// main purpose is to make it obvious where this idiom is being used, +/// and to make the usage more idiomatic and more difficult to get +/// wrong. +/// +/// The TrailingObject template abstracts away the reinterpret_cast, +/// pointer arithmetic, and size calculations used for the allocation +/// and access of appended arrays of objects, and takes care that they +/// are all allocated at their required alignment. Additionally, it +/// ensures that the base type is final -- deriving from a class that +/// expects data appended immediately after it is typically not safe. +/// +/// Users are expected to derive from this template, and provide +/// numTrailingObjects implementations for each trailing type except +/// the last, e.g. like this sample: +/// +/// \code +/// class VarLengthObj : private TrailingObjects<VarLengthObj, int, double> { +/// friend TrailingObjects; +/// +/// unsigned NumInts, NumDoubles; +/// size_t numTrailingObjects(OverloadToken<int>) const { return NumInts; } +/// }; +/// \endcode +/// +/// You can access the appended arrays via 'getTrailingObjects', and +/// determine the size needed for allocation via +/// 'additionalSizeToAlloc' and 'totalSizeToAlloc'. +/// +/// All the methods implemented by this class are are intended for use +/// by the implementation of the class, not as part of its interface +/// (thus, private inheritance is suggested). +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_TRAILINGOBJECTS_H +#define LLVM_SUPPORT_TRAILINGOBJECTS_H + +#include "llvm/Support/AlignOf.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/type_traits.h" +#include <new> +#include <type_traits> + +namespace llvm { + +namespace trailing_objects_internal { +/// Helper template to calculate the max alignment requirement for a set of +/// objects. +template <typename First, typename... Rest> class AlignmentCalcHelper { +private: + enum { + FirstAlignment = AlignOf<First>::Alignment, + RestAlignment = AlignmentCalcHelper<Rest...>::Alignment, + }; + +public: + enum { + Alignment = FirstAlignment > RestAlignment ? FirstAlignment : RestAlignment + }; +}; + +template <typename First> class AlignmentCalcHelper<First> { +public: + enum { Alignment = AlignOf<First>::Alignment }; +}; + +/// The base class for TrailingObjects* classes. +class TrailingObjectsBase { +protected: + /// OverloadToken's purpose is to allow specifying function overloads + /// for different types, without actually taking the types as + /// parameters. (Necessary because member function templates cannot + /// be specialized, so overloads must be used instead of + /// specialization.) + template <typename T> struct OverloadToken {}; +}; + +/// This helper template works-around MSVC 2013's lack of useful +/// alignas() support. The argument to LLVM_ALIGNAS(), in MSVC, is +/// required to be a literal integer. But, you *can* use template +/// specialization to select between a bunch of different LLVM_ALIGNAS +/// expressions... +template <int Align> +class TrailingObjectsAligner : public TrailingObjectsBase {}; +template <> +class LLVM_ALIGNAS(1) TrailingObjectsAligner<1> : public TrailingObjectsBase {}; +template <> +class LLVM_ALIGNAS(2) TrailingObjectsAligner<2> : public TrailingObjectsBase {}; +template <> +class LLVM_ALIGNAS(4) TrailingObjectsAligner<4> : public TrailingObjectsBase {}; +template <> +class LLVM_ALIGNAS(8) TrailingObjectsAligner<8> : public TrailingObjectsBase {}; +template <> +class LLVM_ALIGNAS(16) TrailingObjectsAligner<16> : public TrailingObjectsBase { +}; +template <> +class LLVM_ALIGNAS(32) TrailingObjectsAligner<32> : public TrailingObjectsBase { +}; + +// Just a little helper for transforming a type pack into the same +// number of a different type. e.g.: +// ExtractSecondType<Foo..., int>::type +template <typename Ty1, typename Ty2> struct ExtractSecondType { + typedef Ty2 type; +}; + +// TrailingObjectsImpl is somewhat complicated, because it is a +// recursively inheriting template, in order to handle the template +// varargs. Each level of inheritance picks off a single trailing type +// then recurses on the rest. The "Align", "BaseTy", and +// "TopTrailingObj" arguments are passed through unchanged through the +// recursion. "PrevTy" is, at each level, the type handled by the +// level right above it. + +template <int Align, typename BaseTy, typename TopTrailingObj, typename PrevTy, + typename... MoreTys> +struct TrailingObjectsImpl { + // The main template definition is never used -- the two + // specializations cover all possibilities. +}; + +template <int Align, typename BaseTy, typename TopTrailingObj, typename PrevTy, + typename NextTy, typename... MoreTys> +struct TrailingObjectsImpl<Align, BaseTy, TopTrailingObj, PrevTy, NextTy, + MoreTys...> + : public TrailingObjectsImpl<Align, BaseTy, TopTrailingObj, NextTy, + MoreTys...> { + + typedef TrailingObjectsImpl<Align, BaseTy, TopTrailingObj, NextTy, MoreTys...> + ParentType; + + // Ensure the methods we inherit are not hidden. + using ParentType::getTrailingObjectsImpl; + using ParentType::additionalSizeToAllocImpl; + + static LLVM_CONSTEXPR bool requiresRealignment() { + return llvm::AlignOf<PrevTy>::Alignment < llvm::AlignOf<NextTy>::Alignment; + } + + // These two functions are helper functions for + // TrailingObjects::getTrailingObjects. They recurse to the left -- + // the result for each type in the list of trailing types depends on + // the result of calling the function on the type to the + // left. However, the function for the type to the left is + // implemented by a *subclass* of this class, so we invoke it via + // the TopTrailingObj, which is, via the + // curiously-recurring-template-pattern, the most-derived type in + // this recursion, and thus, contains all the overloads. + static const NextTy * + getTrailingObjectsImpl(const BaseTy *Obj, + TrailingObjectsBase::OverloadToken<NextTy>) { + auto *Ptr = TopTrailingObj::getTrailingObjectsImpl( + Obj, TrailingObjectsBase::OverloadToken<PrevTy>()) + + TopTrailingObj::callNumTrailingObjects( + Obj, TrailingObjectsBase::OverloadToken<PrevTy>()); + + if (requiresRealignment()) + return reinterpret_cast<const NextTy *>( + llvm::alignAddr(Ptr, llvm::alignOf<NextTy>())); + else + return reinterpret_cast<const NextTy *>(Ptr); + } + + static NextTy * + getTrailingObjectsImpl(BaseTy *Obj, + TrailingObjectsBase::OverloadToken<NextTy>) { + auto *Ptr = TopTrailingObj::getTrailingObjectsImpl( + Obj, TrailingObjectsBase::OverloadToken<PrevTy>()) + + TopTrailingObj::callNumTrailingObjects( + Obj, TrailingObjectsBase::OverloadToken<PrevTy>()); + + if (requiresRealignment()) + return reinterpret_cast<NextTy *>( + llvm::alignAddr(Ptr, llvm::alignOf<NextTy>())); + else + return reinterpret_cast<NextTy *>(Ptr); + } + + // Helper function for TrailingObjects::additionalSizeToAlloc: this + // function recurses to superclasses, each of which requires one + // fewer size_t argument, and adds its own size. + static LLVM_CONSTEXPR size_t additionalSizeToAllocImpl( + size_t SizeSoFar, size_t Count1, + typename ExtractSecondType<MoreTys, size_t>::type... MoreCounts) { + return additionalSizeToAllocImpl( + (requiresRealignment() + ? llvm::RoundUpToAlignment(SizeSoFar, llvm::alignOf<NextTy>()) + : SizeSoFar) + + sizeof(NextTy) * Count1, + MoreCounts...); + } +}; + +// The base case of the TrailingObjectsImpl inheritance recursion, +// when there's no more trailing types. +template <int Align, typename BaseTy, typename TopTrailingObj, typename PrevTy> +struct TrailingObjectsImpl<Align, BaseTy, TopTrailingObj, PrevTy> + : public TrailingObjectsAligner<Align> { + // This is a dummy method, only here so the "using" doesn't fail -- + // it will never be called, because this function recurses backwards + // up the inheritance chain to subclasses. + static void getTrailingObjectsImpl(); + + static LLVM_CONSTEXPR size_t additionalSizeToAllocImpl(size_t SizeSoFar) { + return SizeSoFar; + } + + template <bool CheckAlignment> static void verifyTrailingObjectsAlignment() {} +}; + +} // end namespace trailing_objects_internal + +// Finally, the main type defined in this file, the one intended for users... + +/// See the file comment for details on the usage of the +/// TrailingObjects type. +template <typename BaseTy, typename... TrailingTys> +class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl< + trailing_objects_internal::AlignmentCalcHelper< + TrailingTys...>::Alignment, + BaseTy, TrailingObjects<BaseTy, TrailingTys...>, + BaseTy, TrailingTys...> { + + template <int A, typename B, typename T, typename P, typename... M> + friend struct trailing_objects_internal::TrailingObjectsImpl; + + template <typename... Tys> class Foo {}; + + typedef trailing_objects_internal::TrailingObjectsImpl< + trailing_objects_internal::AlignmentCalcHelper<TrailingTys...>::Alignment, + BaseTy, TrailingObjects<BaseTy, TrailingTys...>, BaseTy, TrailingTys...> + ParentType; + using TrailingObjectsBase = trailing_objects_internal::TrailingObjectsBase; + + using ParentType::getTrailingObjectsImpl; + + // This function contains only a static_assert BaseTy is final. The + // static_assert must be in a function, and not at class-level + // because BaseTy isn't complete at class instantiation time, but + // will be by the time this function is instantiated. + static void verifyTrailingObjectsAssertions() { +#ifdef LLVM_IS_FINAL + static_assert(LLVM_IS_FINAL(BaseTy), "BaseTy must be final."); +#endif + } + + // These two methods are the base of the recursion for this method. + static const BaseTy * + getTrailingObjectsImpl(const BaseTy *Obj, + TrailingObjectsBase::OverloadToken<BaseTy>) { + return Obj; + } + + static BaseTy * + getTrailingObjectsImpl(BaseTy *Obj, + TrailingObjectsBase::OverloadToken<BaseTy>) { + return Obj; + } + + // callNumTrailingObjects simply calls numTrailingObjects on the + // provided Obj -- except when the type being queried is BaseTy + // itself. There is always only one of the base object, so that case + // is handled here. (An additional benefit of indirecting through + // this function is that consumers only say "friend + // TrailingObjects", and thus, only this class itself can call the + // numTrailingObjects function.) + static size_t + callNumTrailingObjects(const BaseTy *Obj, + TrailingObjectsBase::OverloadToken<BaseTy>) { + return 1; + } + + template <typename T> + static size_t callNumTrailingObjects(const BaseTy *Obj, + TrailingObjectsBase::OverloadToken<T>) { + return Obj->numTrailingObjects(TrailingObjectsBase::OverloadToken<T>()); + } + +public: + // make this (privately inherited) class public. + using ParentType::OverloadToken; + + /// Returns a pointer to the trailing object array of the given type + /// (which must be one of those specified in the class template). The + /// array may have zero or more elements in it. + template <typename T> const T *getTrailingObjects() const { + verifyTrailingObjectsAssertions(); + // Forwards to an impl function with overloads, since member + // function templates can't be specialized. + return this->getTrailingObjectsImpl( + static_cast<const BaseTy *>(this), + TrailingObjectsBase::OverloadToken<T>()); + } + + /// Returns a pointer to the trailing object array of the given type + /// (which must be one of those specified in the class template). The + /// array may have zero or more elements in it. + template <typename T> T *getTrailingObjects() { + verifyTrailingObjectsAssertions(); + // Forwards to an impl function with overloads, since member + // function templates can't be specialized. + return this->getTrailingObjectsImpl( + static_cast<BaseTy *>(this), TrailingObjectsBase::OverloadToken<T>()); + } + + /// Returns the size of the trailing data, if an object were + /// allocated with the given counts (The counts are in the same order + /// as the template arguments). This does not include the size of the + /// base object. The template arguments must be the same as those + /// used in the class; they are supplied here redundantly only so + /// that it's clear what the counts are counting in callers. + template <typename... Tys> + static LLVM_CONSTEXPR typename std::enable_if< + std::is_same<Foo<TrailingTys...>, Foo<Tys...>>::value, size_t>::type + additionalSizeToAlloc( + typename trailing_objects_internal::ExtractSecondType< + TrailingTys, size_t>::type... Counts) { + return ParentType::additionalSizeToAllocImpl(0, Counts...); + } + + /// Returns the total size of an object if it were allocated with the + /// given trailing object counts. This is the same as + /// additionalSizeToAlloc, except it *does* include the size of the base + /// object. + template <typename... Tys> + static LLVM_CONSTEXPR typename std::enable_if< + std::is_same<Foo<TrailingTys...>, Foo<Tys...>>::value, size_t>::type + totalSizeToAlloc(typename trailing_objects_internal::ExtractSecondType< + TrailingTys, size_t>::type... Counts) { + return sizeof(BaseTy) + ParentType::additionalSizeToAllocImpl(0, Counts...); + } +}; + +} // end namespace llvm + +#endif diff --git a/include/llvm/Support/UnicodeCharRanges.h b/include/llvm/Support/UnicodeCharRanges.h index 9f738dff1107..134698c3ec6b 100644 --- a/include/llvm/Support/UnicodeCharRanges.h +++ b/include/llvm/Support/UnicodeCharRanges.h @@ -51,6 +51,11 @@ public: /// the constructor, so it makes sense to create as few UnicodeCharSet /// instances per each array of ranges, as possible. #ifdef NDEBUG + + // FIXME: This could use constexpr + static_assert. This way we + // may get rid of NDEBUG in this header. Unfortunately there are some + // problems to get this working with MSVC 2013. Change this when + // the support for MSVC 2013 is dropped. LLVM_CONSTEXPR UnicodeCharSet(CharRanges Ranges) : Ranges(Ranges) {} #else UnicodeCharSet(CharRanges Ranges) : Ranges(Ranges) { diff --git a/include/llvm/Support/Valgrind.h b/include/llvm/Support/Valgrind.h index cebf75c49c19..12b0dc961daa 100644 --- a/include/llvm/Support/Valgrind.h +++ b/include/llvm/Support/Valgrind.h @@ -20,17 +20,6 @@ #include "llvm/Support/Compiler.h" #include <stddef.h> -#if LLVM_ENABLE_THREADS != 0 && !defined(NDEBUG) -// tsan (Thread Sanitizer) is a valgrind-based tool that detects these exact -// functions by name. -extern "C" { -void AnnotateHappensAfter(const char *file, int line, const volatile void *cv); -void AnnotateHappensBefore(const char *file, int line, const volatile void *cv); -void AnnotateIgnoreWritesBegin(const char *file, int line); -void AnnotateIgnoreWritesEnd(const char *file, int line); -} -#endif - namespace llvm { namespace sys { // True if Valgrind is controlling this process. @@ -39,34 +28,6 @@ namespace sys { // Discard valgrind's translation of code in the range [Addr .. Addr + Len). // Otherwise valgrind may continue to execute the old version of the code. void ValgrindDiscardTranslations(const void *Addr, size_t Len); - -#if LLVM_ENABLE_THREADS != 0 && !defined(NDEBUG) - // Thread Sanitizer is a valgrind tool that finds races in code. - // See http://code.google.com/p/data-race-test/wiki/DynamicAnnotations . - - // This marker is used to define a happens-before arc. The race detector will - // infer an arc from the begin to the end when they share the same pointer - // argument. - #define TsanHappensBefore(cv) \ - AnnotateHappensBefore(__FILE__, __LINE__, cv) - - // This marker defines the destination of a happens-before arc. - #define TsanHappensAfter(cv) \ - AnnotateHappensAfter(__FILE__, __LINE__, cv) - - // Ignore any races on writes between here and the next TsanIgnoreWritesEnd. - #define TsanIgnoreWritesBegin() \ - AnnotateIgnoreWritesBegin(__FILE__, __LINE__) - - // Resume checking for racy writes. - #define TsanIgnoreWritesEnd() \ - AnnotateIgnoreWritesEnd(__FILE__, __LINE__) -#else - #define TsanHappensBefore(cv) - #define TsanHappensAfter(cv) - #define TsanIgnoreWritesBegin() - #define TsanIgnoreWritesEnd() -#endif } } diff --git a/include/llvm/Support/YAMLParser.h b/include/llvm/Support/YAMLParser.h index 0fbb7d2e6c7e..b056ab6c1ce2 100644 --- a/include/llvm/Support/YAMLParser.h +++ b/include/llvm/Support/YAMLParser.h @@ -145,11 +145,12 @@ public: unsigned int getType() const { return TypeID; } void *operator new(size_t Size, BumpPtrAllocator &Alloc, - size_t Alignment = 16) throw() { + size_t Alignment = 16) LLVM_NOEXCEPT { return Alloc.Allocate(Size, Alignment); } - void operator delete(void *Ptr, BumpPtrAllocator &Alloc, size_t Size) throw() { + void operator delete(void *Ptr, BumpPtrAllocator &Alloc, + size_t Size) LLVM_NOEXCEPT { Alloc.Deallocate(Ptr, Size); } @@ -157,7 +158,7 @@ protected: std::unique_ptr<Document> &Doc; SMRange SourceRange; - void operator delete(void *) throw() {} + void operator delete(void *) LLVM_NOEXCEPT = delete; ~Node() = default; diff --git a/include/llvm/Support/YAMLTraits.h b/include/llvm/Support/YAMLTraits.h index c04294a5e87a..fb2badfd93ba 100644 --- a/include/llvm/Support/YAMLTraits.h +++ b/include/llvm/Support/YAMLTraits.h @@ -10,7 +10,6 @@ #ifndef LLVM_SUPPORT_YAMLTRAITS_H #define LLVM_SUPPORT_YAMLTRAITS_H - #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/Optional.h" @@ -29,7 +28,6 @@ namespace llvm { namespace yaml { - /// This class should be specialized by any type that needs to be converted /// to/from a YAML mapping. For example: /// @@ -52,7 +50,6 @@ struct MappingTraits { // static const bool flow = true; }; - /// This class should be specialized by any integral type that converts /// to/from a YAML scalar where there is a one-to-one mapping between /// in-memory values and a string in YAML. For example: @@ -70,7 +67,6 @@ struct ScalarEnumerationTraits { // static void enumeration(IO &io, T &value); }; - /// This class should be specialized by any integer type that is a union /// of bit values and the YAML representation is a flow sequence of /// strings. For example: @@ -88,7 +84,6 @@ struct ScalarBitSetTraits { // static void bitset(IO &io, T &value); }; - /// This class should be specialized by type that requires custom conversion /// to/from a yaml scalar. For example: /// @@ -149,7 +144,6 @@ struct BlockScalarTraits { // static StringRef input(StringRef Scalar, void *ctxt, T &Value); }; - /// This class should be specialized by any type that needs to be converted /// to/from a YAML sequence. For example: /// @@ -175,7 +169,6 @@ struct SequenceTraits { // static const bool flow = true; }; - /// This class should be specialized by any type that needs to be converted /// to/from a list of YAML documents. template<typename T> @@ -185,7 +178,6 @@ struct DocumentListTraits { // static T::value_type& element(IO &io, T &seq, size_t index); }; - // Only used by compiler if both template types are the same template <typename T, T> struct SameType; @@ -194,8 +186,6 @@ struct SameType; template <typename T> struct MissingTrait; - - // Test if ScalarEnumerationTraits<T> is defined on type T. template <class T> struct has_ScalarEnumerationTraits @@ -213,7 +203,6 @@ public: (sizeof(test<ScalarEnumerationTraits<T> >(nullptr)) == 1); }; - // Test if ScalarBitSetTraits<T> is defined on type T. template <class T> struct has_ScalarBitSetTraits @@ -230,7 +219,6 @@ public: static bool const value = (sizeof(test<ScalarBitSetTraits<T> >(nullptr)) == 1); }; - // Test if ScalarTraits<T> is defined on type T. template <class T> struct has_ScalarTraits @@ -252,7 +240,6 @@ public: (sizeof(test<ScalarTraits<T>>(nullptr, nullptr, nullptr)) == 1); }; - // Test if BlockScalarTraits<T> is defined on type T. template <class T> struct has_BlockScalarTraits @@ -272,7 +259,6 @@ public: (sizeof(test<BlockScalarTraits<T>>(nullptr, nullptr)) == 1); }; - // Test if MappingTraits<T> is defined on type T. template <class T> struct has_MappingTraits @@ -305,8 +291,6 @@ public: static bool const value = (sizeof(test<MappingTraits<T> >(nullptr)) == 1); }; - - // Test if SequenceTraits<T> is defined on type T. template <class T> struct has_SequenceMethodTraits @@ -323,7 +307,6 @@ public: static bool const value = (sizeof(test<SequenceTraits<T> >(nullptr)) == 1); }; - // has_FlowTraits<int> will cause an error with some compilers because // it subclasses int. Using this wrapper only instantiates the // real has_FlowTraits only if the template type is a class. @@ -353,14 +336,11 @@ public: static bool const value = sizeof(f<Derived>(nullptr)) == 2; }; - - // Test if SequenceTraits<T> is defined on type T template<typename T> struct has_SequenceTraits : public std::integral_constant<bool, has_SequenceMethodTraits<T>::value > { }; - // Test if DocumentListTraits<T> is defined on type T template <class T> struct has_DocumentListTraits @@ -453,7 +433,6 @@ inline bool needsQuotes(StringRef S) { return false; } - template<typename T> struct missingTraits : public std::integral_constant<bool, !has_ScalarEnumerationTraits<T>::value @@ -654,8 +633,6 @@ private: void *Ctxt; }; - - template<typename T> typename std::enable_if<has_ScalarEnumerationTraits<T>::value,void>::type yamlize(IO &io, T &Val, bool) { @@ -676,7 +653,6 @@ yamlize(IO &io, T &Val, bool) { } } - template<typename T> typename std::enable_if<has_ScalarTraits<T>::value,void>::type yamlize(IO &io, T &Val, bool) { @@ -791,7 +767,6 @@ yamlize(IO &io, T &Seq, bool) { } } - template<> struct ScalarTraits<bool> { static void output(const bool &, void*, llvm::raw_ostream &); @@ -883,8 +858,6 @@ struct ScalarTraits<double> { static bool mustQuote(StringRef) { return false; } }; - - // Utility for use within MappingTraits<>::mapping() method // to [de]normalize an object for use with YAML conversion. template <typename TNorm, typename TFinal> @@ -917,14 +890,12 @@ private: TFinal &Result; }; - - // Utility for use within MappingTraits<>::mapping() method // to [de]normalize an object for use with YAML conversion. template <typename TNorm, typename TFinal> struct MappingNormalizationHeap { MappingNormalizationHeap(IO &i_o, TFinal &Obj) - : io(i_o), BufPtr(NULL), Result(Obj) { + : io(i_o), BufPtr(nullptr), Result(Obj) { if ( io.outputting() ) { BufPtr = new (&Buffer) TNorm(io, Obj); } @@ -953,8 +924,6 @@ private: TFinal &Result; }; - - /// /// The Input class is used to parse a yaml document into in-memory structs /// and vectors. @@ -1083,7 +1052,6 @@ private: void setError(HNode *hnode, const Twine &message); void setError(Node *node, const Twine &message); - public: // These are only used by operator>>. They could be private // if those templated things could be made friends. @@ -1105,9 +1073,6 @@ private: bool ScalarMatchFound; }; - - - /// /// The Output class is used to generate a yaml document from in-memory structs /// and vectors. @@ -1181,9 +1146,6 @@ private: bool NeedsNewLine; }; - - - /// YAML I/O does conversion based on types. But often native data types /// are just a typedef of built in intergral types (e.g. int). But the C++ /// type matching system sees through the typedef and all the typedefed types @@ -1206,8 +1168,6 @@ private: _base value; \ }; - - /// /// Use these types instead of uintXX_t in any mapping to have /// its yaml output formatted as hexadecimal. @@ -1217,7 +1177,6 @@ LLVM_YAML_STRONG_TYPEDEF(uint16_t, Hex16) LLVM_YAML_STRONG_TYPEDEF(uint32_t, Hex32) LLVM_YAML_STRONG_TYPEDEF(uint64_t, Hex64) - template<> struct ScalarTraits<Hex8> { static void output(const Hex8 &, void*, llvm::raw_ostream &); @@ -1246,7 +1205,6 @@ struct ScalarTraits<Hex64> { static bool mustQuote(StringRef) { return false; } }; - // Define non-member operator>> so that Input can stream in a document list. template <typename T> inline @@ -1303,7 +1261,6 @@ operator>>(Input &yin, T &docSeq) { return yin; } - // Define non-member operator<< so that Output can stream out document list. template <typename T> inline @@ -1372,11 +1329,9 @@ operator<<(Output &yout, T &seq) { return yout; } - } // namespace yaml } // namespace llvm - /// Utility for declaring that a std::vector of a particular type /// should be considered a YAML sequence. #define LLVM_YAML_IS_SEQUENCE_VECTOR(_type) \ @@ -1436,6 +1391,4 @@ operator<<(Output &yout, T &seq) { } \ } - - #endif // LLVM_SUPPORT_YAMLTRAITS_H diff --git a/include/llvm/Support/circular_raw_ostream.h b/include/llvm/Support/circular_raw_ostream.h index 19f9c2c4b155..b46fd7f730c9 100644 --- a/include/llvm/Support/circular_raw_ostream.h +++ b/include/llvm/Support/circular_raw_ostream.h @@ -17,8 +17,7 @@ #include "llvm/Support/raw_ostream.h" -namespace llvm -{ +namespace llvm { /// circular_raw_ostream - A raw_ostream which *can* save its data /// to a circular buffer, or can pass it through directly to an /// underlying stream if specified with a buffer of zero. @@ -154,5 +153,4 @@ namespace llvm }; } // end llvm namespace - #endif diff --git a/include/llvm/Support/raw_ostream.h b/include/llvm/Support/raw_ostream.h index 28e512c86941..d1e96f892a4b 100644 --- a/include/llvm/Support/raw_ostream.h +++ b/include/llvm/Support/raw_ostream.h @@ -218,14 +218,13 @@ public: // Formatted output, see the leftJustify() function in Support/Format.h. raw_ostream &operator<<(const FormattedString &); - + // Formatted output, see the formatHex() function in Support/Format.h. raw_ostream &operator<<(const FormattedNumber &); - + /// indent - Insert 'NumSpaces' spaces. raw_ostream &indent(unsigned NumSpaces); - /// Changes the foreground color of text that will be output from this point /// forward. /// @param Color ANSI color to use, the special SAVEDCOLOR can be used to @@ -246,7 +245,7 @@ public: /// outputting colored text, or before program exit. virtual raw_ostream &resetColor() { return *this; } - /// Reverses the forground and background colors. + /// Reverses the foreground and background colors. virtual raw_ostream &reverseColor() { return *this; } /// This function determines if this stream is connected to a "tty" or @@ -316,7 +315,7 @@ private: }; /// An abstract base class for streams implementations that also support a -/// pwrite operation. This is usefull for code that can mostly stream out data, +/// pwrite operation. This is useful for code that can mostly stream out data, /// but needs to patch in a header that needs to know the output size. class raw_pwrite_stream : public raw_ostream { virtual void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) = 0; @@ -350,10 +349,6 @@ class raw_fd_ostream : public raw_pwrite_stream { /// bool Error; - /// Controls whether the stream should attempt to use atomic writes, when - /// possible. - bool UseAtomicWrites; - uint64_t pos; bool SupportsSeeking; @@ -403,16 +398,6 @@ public: /// to the offset specified from the beginning of the file. uint64_t seek(uint64_t off); - /// Set the stream to attempt to use atomic writes for individual output - /// routines where possible. - /// - /// Note that because raw_ostream's are typically buffered, this flag is only - /// sensible when used on unbuffered streams which will flush their output - /// immediately. - void SetUseAtomicWrites(bool Value) { - UseAtomicWrites = Value; - } - raw_ostream &changeColor(enum Colors colors, bool bold=false, bool bg=false) override; raw_ostream &resetColor() override; @@ -471,6 +456,7 @@ class raw_string_ostream : public raw_ostream { /// Return the current position within the stream, not counting the bytes /// currently in the buffer. uint64_t current_pos() const override { return OS.size(); } + public: explicit raw_string_ostream(std::string &O) : OS(O) {} ~raw_string_ostream() override; @@ -485,6 +471,9 @@ public: /// A raw_ostream that writes to an SmallVector or SmallString. This is a /// simple adaptor class. This class does not encounter output errors. +/// raw_svector_ostream operates without a buffer, delegating all memory +/// management to the SmallString. Thus the SmallString is always up-to-date, +/// may be used directly and there is no need to call flush(). class raw_svector_ostream : public raw_pwrite_stream { SmallVectorImpl<char> &OS; @@ -493,32 +482,23 @@ class raw_svector_ostream : public raw_pwrite_stream { void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override; - /// Return the current position within the stream, not counting the bytes - /// currently in the buffer. + /// Return the current position within the stream. uint64_t current_pos() const override; -protected: - // Like the regular constructor, but doesn't call init. - explicit raw_svector_ostream(SmallVectorImpl<char> &O, unsigned); - void init(); - public: /// Construct a new raw_svector_ostream. /// /// \param O The vector to write to; this should generally have at least 128 /// bytes free to avoid any extraneous memory overhead. - explicit raw_svector_ostream(SmallVectorImpl<char> &O); - ~raw_svector_ostream() override; - + explicit raw_svector_ostream(SmallVectorImpl<char> &O) : OS(O) { + SetUnbuffered(); + } + ~raw_svector_ostream() override {} - /// This is called when the SmallVector we're appending to is changed outside - /// of the raw_svector_ostream's control. It is only safe to do this if the - /// raw_svector_ostream has previously been flushed. - void resync(); + void flush() = delete; - /// Flushes the stream contents to the target vector and return a StringRef - /// for the vector contents. - StringRef str(); + /// Return a StringRef for the vector contents. + StringRef str() { return StringRef(OS.data(), OS.size()); } }; /// A raw_ostream that discards all output. @@ -541,12 +521,10 @@ class buffer_ostream : public raw_svector_ostream { SmallVector<char, 0> Buffer; public: - buffer_ostream(raw_ostream &OS) : raw_svector_ostream(Buffer, 0), OS(OS) { - init(); - } - ~buffer_ostream() { OS << str(); } + buffer_ostream(raw_ostream &OS) : raw_svector_ostream(Buffer), OS(OS) {} + ~buffer_ostream() override { OS << str(); } }; } // end llvm namespace -#endif +#endif // LLVM_SUPPORT_RAW_OSTREAM_H diff --git a/include/llvm/Support/thread.h b/include/llvm/Support/thread.h new file mode 100644 index 000000000000..2d130418a57f --- /dev/null +++ b/include/llvm/Support/thread.h @@ -0,0 +1,66 @@ +//===-- llvm/Support/thread.h - Wrapper for <thread> ------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This header is a wrapper for <thread> that works around problems with the +// MSVC headers when exceptions are disabled. It also provides llvm::thread, +// which is either a typedef of std::thread or a replacement that calls the +// function synchronously depending on the value of LLVM_ENABLE_THREADS. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_THREAD_H +#define LLVM_SUPPORT_THREAD_H + +#include "llvm/Config/llvm-config.h" + +#if LLVM_ENABLE_THREADS + +#ifdef _MSC_VER +// concrt.h depends on eh.h for __uncaught_exception declaration +// even if we disable exceptions. +#include <eh.h> + +// Suppress 'C++ exception handler used, but unwind semantics are not enabled.' +#pragma warning(push) +#pragma warning(disable:4530) +#endif + +#include <thread> + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +namespace llvm { +typedef std::thread thread; +} + +#else // !LLVM_ENABLE_THREADS + +#include <utility> + +namespace llvm { + +struct thread { + thread() {} + thread(thread &&other) {} + template <class Function, class... Args> + explicit thread(Function &&f, Args &&... args) { + f(std::forward<Args>(args)...); + } + thread(const thread &) = delete; + + void join() {} +}; + +} + +#endif // LLVM_ENABLE_THREADS + +#endif diff --git a/include/llvm/Support/type_traits.h b/include/llvm/Support/type_traits.h index 45465aea004b..88385c3fae1e 100644 --- a/include/llvm/Support/type_traits.h +++ b/include/llvm/Support/type_traits.h @@ -93,6 +93,15 @@ struct add_const_past_pointer< } +// If the compiler supports detecting whether a class is final, define +// an LLVM_IS_FINAL macro. If it cannot be defined properly, this +// macro will be left undefined. +#if __cplusplus >= 201402L +#define LLVM_IS_FINAL(Ty) std::is_final<Ty>() +#elif __has_feature(is_final) || LLVM_GNUC_PREREQ(4, 7, 0) +#define LLVM_IS_FINAL(Ty) __is_final(Ty) +#endif + #ifdef LLVM_DEFINED_HAS_FEATURE #undef __has_feature #endif diff --git a/include/llvm/TableGen/Record.h b/include/llvm/TableGen/Record.h index b4642c991192..eb1c5c78b9c0 100644 --- a/include/llvm/TableGen/Record.h +++ b/include/llvm/TableGen/Record.h @@ -366,7 +366,7 @@ class TypedInit : public Init { protected: explicit TypedInit(InitKind K, RecTy *T) : Init(K), Ty(T) {} - ~TypedInit() { + ~TypedInit() override { // If this is a DefInit we need to delete the RecordRecTy. if (getKind() == IK_DefInit) delete Ty; @@ -547,7 +547,7 @@ public: class StringInit : public TypedInit { std::string Value; - explicit StringInit(const std::string &V) + explicit StringInit(StringRef V) : TypedInit(IK_StringInit, StringRecTy::get()), Value(V) {} StringInit(const StringInit &Other) = delete; @@ -836,8 +836,6 @@ public: class VarInit : public TypedInit { Init *VarName; - explicit VarInit(const std::string &VN, RecTy *T) - : TypedInit(IK_VarInit, T), VarName(StringInit::get(VN)) {} explicit VarInit(Init *VN, RecTy *T) : TypedInit(IK_VarInit, T), VarName(VN) {} @@ -1589,6 +1587,6 @@ Init *QualifyName(Record &CurRec, MultiClass *CurMultiClass, Init *QualifyName(Record &CurRec, MultiClass *CurMultiClass, const std::string &Name, const std::string &Scoper); -} // End llvm namespace +} // end llvm namespace -#endif +#endif // LLVM_TABLEGEN_RECORD_H diff --git a/include/llvm/Target/CostTable.h b/include/llvm/Target/CostTable.h index 34f6041137c1..2499f5c3189c 100644 --- a/include/llvm/Target/CostTable.h +++ b/include/llvm/Target/CostTable.h @@ -15,64 +15,54 @@ #ifndef LLVM_TARGET_COSTTABLE_H_ #define LLVM_TARGET_COSTTABLE_H_ +#include "llvm/ADT/ArrayRef.h" +#include "llvm/CodeGen/MachineValueType.h" + namespace llvm { /// Cost Table Entry -template <class TypeTy> struct CostTblEntry { int ISD; - TypeTy Type; + MVT::SimpleValueType Type; unsigned Cost; }; /// Find in cost table, TypeTy must be comparable to CompareTy by == -template <class TypeTy, class CompareTy> -int CostTableLookup(const CostTblEntry<TypeTy> *Tbl, unsigned len, int ISD, - CompareTy Ty) { - for (unsigned int i = 0; i < len; ++i) - if (ISD == Tbl[i].ISD && Ty == Tbl[i].Type) - return i; +inline const CostTblEntry *CostTableLookup(ArrayRef<CostTblEntry> Tbl, + int ISD, MVT Ty) { + auto I = std::find_if(Tbl.begin(), Tbl.end(), + [=](const CostTblEntry &Entry) { + return ISD == Entry.ISD && Ty == Entry.Type; }); + if (I != Tbl.end()) + return I; // Could not find an entry. - return -1; -} - -/// Find in cost table, TypeTy must be comparable to CompareTy by == -template <class TypeTy, class CompareTy, unsigned N> -int CostTableLookup(const CostTblEntry<TypeTy>(&Tbl)[N], int ISD, - CompareTy Ty) { - return CostTableLookup(Tbl, N, ISD, Ty); + return nullptr; } /// Type Conversion Cost Table -template <class TypeTy> struct TypeConversionCostTblEntry { int ISD; - TypeTy Dst; - TypeTy Src; + MVT::SimpleValueType Dst; + MVT::SimpleValueType Src; unsigned Cost; }; /// Find in type conversion cost table, TypeTy must be comparable to CompareTy /// by == -template <class TypeTy, class CompareTy> -int ConvertCostTableLookup(const TypeConversionCostTblEntry<TypeTy> *Tbl, - unsigned len, int ISD, CompareTy Dst, - CompareTy Src) { - for (unsigned int i = 0; i < len; ++i) - if (ISD == Tbl[i].ISD && Src == Tbl[i].Src && Dst == Tbl[i].Dst) - return i; +inline const TypeConversionCostTblEntry * +ConvertCostTableLookup(ArrayRef<TypeConversionCostTblEntry> Tbl, + int ISD, MVT Dst, MVT Src) { + auto I = std::find_if(Tbl.begin(), Tbl.end(), + [=](const TypeConversionCostTblEntry &Entry) { + return ISD == Entry.ISD && Src == Entry.Src && + Dst == Entry.Dst; + }); + if (I != Tbl.end()) + return I; // Could not find an entry. - return -1; -} - -/// Find in type conversion cost table, TypeTy must be comparable to CompareTy -/// by == -template <class TypeTy, class CompareTy, unsigned N> -int ConvertCostTableLookup(const TypeConversionCostTblEntry<TypeTy>(&Tbl)[N], - int ISD, CompareTy Dst, CompareTy Src) { - return ConvertCostTableLookup(Tbl, N, ISD, Dst, Src); + return nullptr; } } // namespace llvm diff --git a/include/llvm/Target/Target.td b/include/llvm/Target/Target.td index e0aea181a639..79046b2b7352 100644 --- a/include/llvm/Target/Target.td +++ b/include/llvm/Target/Target.td @@ -441,6 +441,30 @@ class Instruction { string PostEncoderMethod = ""; string DecoderMethod = ""; + // Is the instruction decoder method able to completely determine if the + // given instruction is valid or not. If the TableGen definition of the + // instruction specifies bitpattern A??B where A and B are static bits, the + // hasCompleteDecoder flag says whether the decoder method fully handles the + // ?? space, i.e. if it is a final arbiter for the instruction validity. + // If not then the decoder attempts to continue decoding when the decoder + // method fails. + // + // This allows to handle situations where the encoding is not fully + // orthogonal. Example: + // * InstA with bitpattern 0b0000????, + // * InstB with bitpattern 0b000000?? but the associated decoder method + // DecodeInstB() returns Fail when ?? is 0b00 or 0b11. + // + // The decoder tries to decode a bitpattern that matches both InstA and + // InstB bitpatterns first as InstB (because it is the most specific + // encoding). In the default case (hasCompleteDecoder = 1), when + // DecodeInstB() returns Fail the bitpattern gets rejected. By setting + // hasCompleteDecoder = 0 in InstB, the decoder is informed that + // DecodeInstB() is not able to determine if all possible values of ?? are + // valid or not. If DecodeInstB() returns Fail the decoder will attempt to + // decode the bitpattern as InstA too. + bit hasCompleteDecoder = 1; + /// Target-specific flags. This becomes the TSFlags field in TargetInstrDesc. bits<64> TSFlags = 0; @@ -595,6 +619,8 @@ class Operand<ValueType ty> : DAGOperand { string PrintMethod = "printOperand"; string EncoderMethod = ""; string DecoderMethod = ""; + bit hasCompleteDecoder = 1; + string OperandNamespace = "MCOI"; string OperandType = "OPERAND_UNKNOWN"; dag MIOperandInfo = (ops); @@ -910,9 +936,6 @@ class AsmParser { // ShouldEmitMatchRegisterName - Set to false if the target needs a hand // written register name matcher bit ShouldEmitMatchRegisterName = 1; - - /// Does the instruction mnemonic allow '.' - bit MnemonicContainsDot = 0; } def DefaultAsmParser : AsmParser; @@ -940,6 +963,15 @@ class AsmParserVariant { // register tokens as constrained registers, instead of tokens, for the // purposes of matching. string RegisterPrefix = ""; + + // TokenizingCharacters - Characters that are standalone tokens + string TokenizingCharacters = "[]*!"; + + // SeparatorCharacters - Characters that are not tokens + string SeparatorCharacters = " \t,"; + + // BreakCharacters - Characters that start new identifiers + string BreakCharacters = ""; } def DefaultAsmParserVariant : AsmParserVariant; diff --git a/include/llvm/Target/TargetCallingConv.h b/include/llvm/Target/TargetCallingConv.h index 9d4e7a04d905..0c6c1f1468c4 100644 --- a/include/llvm/Target/TargetCallingConv.h +++ b/include/llvm/Target/TargetCallingConv.h @@ -46,6 +46,8 @@ namespace ISD { static const uint64_t SplitOffs = 11; static const uint64_t InAlloca = 1ULL<<12; ///< Passed with inalloca static const uint64_t InAllocaOffs = 12; + static const uint64_t SplitEnd = 1ULL<<13; ///< Last part of a split + static const uint64_t SplitEndOffs = 13; static const uint64_t OrigAlign = 0x1FULL<<27; static const uint64_t OrigAlignOffs = 27; static const uint64_t ByValSize = 0x3fffffffULL<<32; ///< Struct size @@ -103,6 +105,9 @@ namespace ISD { bool isSplit() const { return Flags & Split; } void setSplit() { Flags |= One << SplitOffs; } + bool isSplitEnd() const { return Flags & SplitEnd; } + void setSplitEnd() { Flags |= One << SplitEndOffs; } + unsigned getOrigAlign() const { return (unsigned) ((One << ((Flags & OrigAlign) >> OrigAlignOffs)) / 2); diff --git a/include/llvm/Target/TargetFrameLowering.h b/include/llvm/Target/TargetFrameLowering.h index 3af2227410f7..cadd07d71f12 100644 --- a/include/llvm/Target/TargetFrameLowering.h +++ b/include/llvm/Target/TargetFrameLowering.h @@ -70,6 +70,18 @@ public: /// unsigned getStackAlignment() const { return StackAlignment; } + /// alignSPAdjust - This method aligns the stack adjustment to the correct + /// alignment. + /// + int alignSPAdjust(int SPAdj) const { + if (SPAdj < 0) { + SPAdj = -RoundUpToAlignment(-SPAdj, StackAlignment); + } else { + SPAdj = RoundUpToAlignment(SPAdj, StackAlignment); + } + return SPAdj; + } + /// getTransientStackAlignment - This method returns the number of bytes to /// which the stack pointer must be aligned at all times, even between /// calls. @@ -84,6 +96,11 @@ public: return StackRealignable; } + /// Return the skew that has to be applied to stack alignment under + /// certain conditions (e.g. stack was adjusted before function \p MF + /// was called). + virtual unsigned getStackAlignmentSkew(const MachineFunction &MF) const; + /// getOffsetOfLocalArea - This method returns the offset of the local area /// from the stack pointer on entrance to a function. /// @@ -129,6 +146,11 @@ public: return false; } + /// Returns true if the target will correctly handle shrink wrapping. + virtual bool enableShrinkWrapping(const MachineFunction &MF) const { + return false; + } + /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. virtual void emitPrologue(MachineFunction &MF, @@ -136,6 +158,10 @@ public: virtual void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const = 0; + /// Replace a StackProbe stub (if any) with the actual probe code inline + virtual void inlineStackProbe(MachineFunction &MF, + MachineBasicBlock &PrologueMBB) const {} + /// Adjust the prologue to have the function use segmented stacks. This works /// by adding a check even before the "normal" function prologue. virtual void adjustForSegmentedStacks(MachineFunction &MF, @@ -207,10 +233,6 @@ public: // has any stack objects. However, targets may want to override this. virtual bool needsFrameIndexResolution(const MachineFunction &MF) const; - /// getFrameIndexOffset - Returns the displacement from the frame register to - /// the stack frame of the specified index. - virtual int getFrameIndexOffset(const MachineFunction &MF, int FI) const; - /// getFrameIndexReference - This method should return the base register /// and offset used to reference a frame index location. The offset is /// returned directly, and the base register is returned via FrameReg. @@ -218,10 +240,11 @@ public: unsigned &FrameReg) const; /// Same as above, except that the 'base register' will always be RSP, not - /// RBP on x86. This is used exclusively for lowering STATEPOINT nodes. + /// RBP on x86. This is generally used for emitting statepoint or EH tables + /// that use offsets from RSP. /// TODO: This should really be a parameterizable choice. virtual int getFrameIndexReferenceFromSP(const MachineFunction &MF, int FI, - unsigned &FrameReg) const { + unsigned &FrameReg) const { // default to calling normal version, we override this on x86 only llvm_unreachable("unimplemented for non-x86"); return 0; @@ -246,6 +269,10 @@ public: RegScavenger *RS = nullptr) const { } + virtual unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const { + report_fatal_error("WinEH not implemented for this target"); + } + /// eliminateCallFramePseudoInstr - This method is called during prolog/epilog /// code insertion to eliminate call frame setup and destroy pseudo /// instructions (but only if the Target is using them). It is responsible diff --git a/include/llvm/Target/TargetInstrInfo.h b/include/llvm/Target/TargetInstrInfo.h index 8b314f454b18..0cebcf1c6b5d 100644 --- a/include/llvm/Target/TargetInstrInfo.h +++ b/include/llvm/Target/TargetInstrInfo.h @@ -19,6 +19,7 @@ #include "llvm/CodeGen/MachineCombinerPattern.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/MC/MCInstrInfo.h" +#include "llvm/Support/BranchProbability.h" #include "llvm/Target/TargetRegisterInfo.h" namespace llvm { @@ -38,7 +39,6 @@ class SelectionDAG; class ScheduleDAG; class TargetRegisterClass; class TargetRegisterInfo; -class BranchProbability; class TargetSubtargetInfo; class TargetSchedModel; class DFAPacketizer; @@ -54,13 +54,18 @@ class TargetInstrInfo : public MCInstrInfo { TargetInstrInfo(const TargetInstrInfo &) = delete; void operator=(const TargetInstrInfo &) = delete; public: - TargetInstrInfo(unsigned CFSetupOpcode = ~0u, unsigned CFDestroyOpcode = ~0u) - : CallFrameSetupOpcode(CFSetupOpcode), - CallFrameDestroyOpcode(CFDestroyOpcode) { - } + TargetInstrInfo(unsigned CFSetupOpcode = ~0u, unsigned CFDestroyOpcode = ~0u, + unsigned CatchRetOpcode = ~0u) + : CallFrameSetupOpcode(CFSetupOpcode), + CallFrameDestroyOpcode(CFDestroyOpcode), + CatchRetOpcode(CatchRetOpcode) {} virtual ~TargetInstrInfo(); + static bool isGenericOpcode(unsigned Opc) { + return Opc <= TargetOpcode::GENERIC_OP_END; + } + /// Given a machine instruction descriptor, returns the register /// class constraint for OpNum, or NULL. const TargetRegisterClass *getRegClass(const MCInstrDesc &TID, @@ -94,6 +99,41 @@ protected: return false; } + /// This method commutes the operands of the given machine instruction MI. + /// The operands to be commuted are specified by their indices OpIdx1 and + /// OpIdx2. + /// + /// If a target has any instructions that are commutable but require + /// converting to different instructions or making non-trivial changes + /// to commute them, this method can be overloaded to do that. + /// The default implementation simply swaps the commutable operands. + /// + /// If NewMI is false, MI is modified in place and returned; otherwise, a + /// new machine instruction is created and returned. + /// + /// Do not call this method for a non-commutable instruction. + /// Even though the instruction is commutable, the method may still + /// fail to commute the operands, null pointer is returned in such cases. + virtual MachineInstr *commuteInstructionImpl(MachineInstr *MI, + bool NewMI, + unsigned OpIdx1, + unsigned OpIdx2) const; + + /// Assigns the (CommutableOpIdx1, CommutableOpIdx2) pair of commutable + /// operand indices to (ResultIdx1, ResultIdx2). + /// One or both input values of the pair: (ResultIdx1, ResultIdx2) may be + /// predefined to some indices or be undefined (designated by the special + /// value 'CommuteAnyOperandIndex'). + /// The predefined result indices cannot be re-defined. + /// The function returns true iff after the result pair redefinition + /// the fixed result pair is equal to or equivalent to the source pair of + /// indices: (CommutableOpIdx1, CommutableOpIdx2). It is assumed here that + /// the pairs (x,y) and (y,x) are equivalent. + static bool fixCommutedOpIndices(unsigned &ResultIdx1, + unsigned &ResultIdx2, + unsigned CommutableOpIdx1, + unsigned CommutableOpIdx2); + private: /// For instructions with opcodes for which the M_REMATERIALIZABLE flag is /// set and the target hook isReallyTriviallyReMaterializable returns false, @@ -111,6 +151,8 @@ public: unsigned getCallFrameSetupOpcode() const { return CallFrameSetupOpcode; } unsigned getCallFrameDestroyOpcode() const { return CallFrameDestroyOpcode; } + unsigned getCatchReturnOpcode() const { return CatchRetOpcode; } + /// Returns the actual stack pointer adjustment made by an instruction /// as part of a call sequence. By default, only call frame setup/destroy /// instructions adjust the stack, but targets may want to override this @@ -250,20 +292,51 @@ public: return nullptr; } - /// If a target has any instructions that are commutable but require - /// converting to different instructions or making non-trivial changes to - /// commute them, this method can overloaded to do that. - /// The default implementation simply swaps the commutable operands. + // This constant can be used as an input value of operand index passed to + // the method findCommutedOpIndices() to tell the method that the + // corresponding operand index is not pre-defined and that the method + // can pick any commutable operand. + static const unsigned CommuteAnyOperandIndex = ~0U; + + /// This method commutes the operands of the given machine instruction MI. + /// + /// The operands to be commuted are specified by their indices OpIdx1 and + /// OpIdx2. OpIdx1 and OpIdx2 arguments may be set to a special value + /// 'CommuteAnyOperandIndex', which means that the method is free to choose + /// any arbitrarily chosen commutable operand. If both arguments are set to + /// 'CommuteAnyOperandIndex' then the method looks for 2 different commutable + /// operands; then commutes them if such operands could be found. + /// /// If NewMI is false, MI is modified in place and returned; otherwise, a - /// new machine instruction is created and returned. Do not call this - /// method for a non-commutable instruction, but there may be some cases - /// where this method fails and returns null. - virtual MachineInstr *commuteInstruction(MachineInstr *MI, - bool NewMI = false) const; - - /// If specified MI is commutable, return the two operand indices that would - /// swap value. Return false if the instruction - /// is not in a form which this routine understands. + /// new machine instruction is created and returned. + /// + /// Do not call this method for a non-commutable instruction or + /// for non-commuable operands. + /// Even though the instruction is commutable, the method may still + /// fail to commute the operands, null pointer is returned in such cases. + MachineInstr * + commuteInstruction(MachineInstr *MI, + bool NewMI = false, + unsigned OpIdx1 = CommuteAnyOperandIndex, + unsigned OpIdx2 = CommuteAnyOperandIndex) const; + + /// Returns true iff the routine could find two commutable operands in the + /// given machine instruction. + /// The 'SrcOpIdx1' and 'SrcOpIdx2' are INPUT and OUTPUT arguments. + /// If any of the INPUT values is set to the special value + /// 'CommuteAnyOperandIndex' then the method arbitrarily picks a commutable + /// operand, then returns its index in the corresponding argument. + /// If both of INPUT values are set to 'CommuteAnyOperandIndex' then method + /// looks for 2 commutable operands. + /// If INPUT values refer to some operands of MI, then the method simply + /// returns true if the corresponding operands are commutable and returns + /// false otherwise. + /// + /// For example, calling this method this way: + /// unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex; + /// findCommutedOpIndices(MI, Op1, Op2); + /// can be interpreted as a query asking to find an operand that would be + /// commutable with the operand#1. virtual bool findCommutedOpIndices(MachineInstr *MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const; @@ -511,7 +584,7 @@ public: virtual bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, unsigned ExtraPredCycles, - const BranchProbability &Probability) const { + BranchProbability Probability) const { return false; } @@ -526,7 +599,7 @@ public: unsigned NumTCycles, unsigned ExtraTCycles, MachineBasicBlock &FMBB, unsigned NumFCycles, unsigned ExtraFCycles, - const BranchProbability &Probability) const { + BranchProbability Probability) const { return false; } @@ -538,7 +611,7 @@ public: /// will be properly predicted. virtual bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, - const BranchProbability &Probability) const { + BranchProbability Probability) const { return false; } @@ -724,13 +797,30 @@ public: /// order since the pattern evaluator stops checking as soon as it finds a /// faster sequence. /// \param Root - Instruction that could be combined with one of its operands - /// \param Pattern - Vector of possible combination patterns + /// \param Patterns - Vector of possible combination patterns virtual bool getMachineCombinerPatterns( MachineInstr &Root, - SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Pattern) const { + SmallVectorImpl<MachineCombinerPattern> &Patterns) const; + + /// Return true if the input \P Inst is part of a chain of dependent ops + /// that are suitable for reassociation, otherwise return false. + /// If the instruction's operands must be commuted to have a previous + /// instruction of the same type define the first source operand, \P Commuted + /// will be set to true. + bool isReassociationCandidate(const MachineInstr &Inst, bool &Commuted) const; + + /// Return true when \P Inst is both associative and commutative. + virtual bool isAssociativeAndCommutative(const MachineInstr &Inst) const { return false; } + /// Return true when \P Inst has reassociable operands in the same \P MBB. + virtual bool hasReassociableOperands(const MachineInstr &Inst, + const MachineBasicBlock *MBB) const; + + /// Return true when \P Inst has reassociable sibling. + bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const; + /// When getMachineCombinerPatterns() finds patterns, this function generates /// the instructions that could replace the original code sequence. The client /// has to decide whether the actual replacement is beneficial or not. @@ -742,12 +832,26 @@ public: /// \param InstrIdxForVirtReg - map of virtual register to instruction in /// InsInstr that defines it virtual void genAlternativeCodeSequence( - MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern, + MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl<MachineInstr *> &InsInstrs, SmallVectorImpl<MachineInstr *> &DelInstrs, - DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const { + DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const; + + /// Attempt to reassociate \P Root and \P Prev according to \P Pattern to + /// reduce critical path length. + void reassociateOps(MachineInstr &Root, MachineInstr &Prev, + MachineCombinerPattern Pattern, + SmallVectorImpl<MachineInstr *> &InsInstrs, + SmallVectorImpl<MachineInstr *> &DelInstrs, + DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const; + + /// This is an architecture-specific helper function of reassociateOps. + /// Set special operand attributes for new instructions after reassociation. + virtual void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, + MachineInstr &NewMI1, + MachineInstr &NewMI2) const { return; - } + }; /// Return true when a target supports MachineCombiner. virtual bool useMachineCombiner() const { return false; } @@ -819,10 +923,6 @@ protected: } public: - /// Returns true for the specified load / store if folding is possible. - virtual bool canFoldMemoryOperand(const MachineInstr *MI, - ArrayRef<unsigned> Ops) const; - /// unfoldMemoryOperand - Separate a single instruction which folded a load or /// a store or a load and a store into two or more instruction. If this is /// possible, returns true as well as the new instructions by reference. @@ -1266,8 +1366,73 @@ public: return 5; } + /// Return an array that contains the ids of the target indices (used for the + /// TargetIndex machine operand) and their names. + /// + /// MIR Serialization is able to serialize only the target indices that are + /// defined by this method. + virtual ArrayRef<std::pair<int, const char *>> + getSerializableTargetIndices() const { + return None; + } + + /// Decompose the machine operand's target flags into two values - the direct + /// target flag value and any of bit flags that are applied. + virtual std::pair<unsigned, unsigned> + decomposeMachineOperandsTargetFlags(unsigned /*TF*/) const { + return std::make_pair(0u, 0u); + } + + /// Return an array that contains the direct target flag values and their + /// names. + /// + /// MIR Serialization is able to serialize only the target flags that are + /// defined by this method. + virtual ArrayRef<std::pair<unsigned, const char *>> + getSerializableDirectMachineOperandTargetFlags() const { + return None; + } + + /// Return an array that contains the bitmask target flag values and their + /// names. + /// + /// MIR Serialization is able to serialize only the target flags that are + /// defined by this method. + virtual ArrayRef<std::pair<unsigned, const char *>> + getSerializableBitmaskMachineOperandTargetFlags() const { + return None; + } + private: unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode; + unsigned CatchRetOpcode; +}; + +/// \brief Provide DenseMapInfo for TargetInstrInfo::RegSubRegPair. +template<> +struct DenseMapInfo<TargetInstrInfo::RegSubRegPair> { + typedef DenseMapInfo<unsigned> RegInfo; + + static inline TargetInstrInfo::RegSubRegPair getEmptyKey() { + return TargetInstrInfo::RegSubRegPair(RegInfo::getEmptyKey(), + RegInfo::getEmptyKey()); + } + static inline TargetInstrInfo::RegSubRegPair getTombstoneKey() { + return TargetInstrInfo::RegSubRegPair(RegInfo::getTombstoneKey(), + RegInfo::getTombstoneKey()); + } + /// \brief Reuse getHashValue implementation from + /// std::pair<unsigned, unsigned>. + static unsigned getHashValue(const TargetInstrInfo::RegSubRegPair &Val) { + std::pair<unsigned, unsigned> PairVal = + std::make_pair(Val.Reg, Val.SubReg); + return DenseMapInfo<std::pair<unsigned, unsigned>>::getHashValue(PairVal); + } + static bool isEqual(const TargetInstrInfo::RegSubRegPair &LHS, + const TargetInstrInfo::RegSubRegPair &RHS) { + return RegInfo::isEqual(LHS.Reg, RHS.Reg) && + RegInfo::isEqual(LHS.SubReg, RHS.SubReg); + } }; } // End llvm namespace diff --git a/include/llvm/Target/TargetItinerary.td b/include/llvm/Target/TargetItinerary.td index cc74006dc9fe..a37bbf2474c5 100644 --- a/include/llvm/Target/TargetItinerary.td +++ b/include/llvm/Target/TargetItinerary.td @@ -134,3 +134,19 @@ class ProcessorItineraries<list<FuncUnit> fu, list<Bypass> bp, // info. Subtargets using NoItineraries can bypass the scheduler's // expensive HazardRecognizer because no reservation table is needed. def NoItineraries : ProcessorItineraries<[], [], []>; + +//===----------------------------------------------------------------------===// +// Combo Function Unit data - This is a map of combo function unit names to +// the list of functional units that are included in the combination. +// +class ComboFuncData<FuncUnit ComboFunc, list<FuncUnit> funclist> { + FuncUnit TheComboFunc = ComboFunc; + list<FuncUnit> FuncList = funclist; +} + +//===----------------------------------------------------------------------===// +// Combo Function Units - This is a list of all combo function unit data. +class ComboFuncUnits<list<ComboFuncData> cfd> { + list<ComboFuncData> CFD = cfd; +} + diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h index 4412d9b3c68e..140c36591acc 100644 --- a/include/llvm/Target/TargetLowering.h +++ b/include/llvm/Target/TargetLowering.h @@ -83,20 +83,22 @@ class TargetLoweringBase { public: /// This enum indicates whether operations are valid for a target, and if not, /// what action should be used to make them valid. - enum LegalizeAction { + enum LegalizeAction : uint8_t { Legal, // The target natively supports this operation. Promote, // This operation should be executed in a larger type. Expand, // Try to expand this to other ops, otherwise use a libcall. + LibCall, // Don't try to expand this to other ops, always use a libcall. Custom // Use the LowerOperation hook to implement custom lowering. }; /// This enum indicates whether a types are legal for a target, and if not, /// what action should be used to make them valid. - enum LegalizeTypeAction { + enum LegalizeTypeAction : uint8_t { TypeLegal, // The target natively supports this type. TypePromoteInteger, // Replace this integer with a larger one. TypeExpandInteger, // Split this integer into two of half the size. - TypeSoftenFloat, // Convert this float to a same size integer type. + TypeSoftenFloat, // Convert this float to a same size integer type, + // if an operation is not supported in target HW. TypeExpandFloat, // Split this float into two of half the size. TypeScalarizeVector, // Replace this one-element vector with its element. TypeSplitVector, // Split this vector into two of half the size. @@ -124,16 +126,17 @@ public: // mask (ex: x86 blends). }; - /// Enum that specifies what a AtomicRMWInst is expanded to, if at all. Exists - /// because different targets have different levels of support for these - /// atomic RMW instructions, and also have different options w.r.t. what they - /// should expand to. - enum class AtomicRMWExpansionKind { - None, // Don't expand the instruction. - LLSC, // Expand the instruction into loadlinked/storeconditional; used - // by ARM/AArch64. Implies `hasLoadLinkedStoreConditional` - // returns true. - CmpXChg, // Expand the instruction into cmpxchg; used by at least X86. + /// Enum that specifies what an atomic load/AtomicRMWInst is expanded + /// to, if at all. Exists because different targets have different levels of + /// support for these atomic instructions, and also have different options + /// w.r.t. what they should expand to. + enum class AtomicExpansionKind { + None, // Don't expand the instruction. + LLSC, // Expand the instruction into loadlinked/storeconditional; used + // by ARM/AArch64. + LLOnly, // Expand the (load) instruction into just a load-linked, which has + // greater atomic guarantees than a normal load. + CmpXChg, // Expand the instruction into cmpxchg; used by at least X86. }; static ISD::NodeType getExtendForContent(BooleanContent Content) { @@ -226,7 +229,11 @@ public: /// Return true if integer divide is usually cheaper than a sequence of /// several shifts, adds, and multiplies for this target. - bool isIntDivCheap() const { return IntDivIsCheap; } + /// The definition of "cheaper" may depend on whether we're optimizing + /// for speed or for size. + virtual bool isIntDivCheap(EVT VT, AttributeSet Attr) const { + return false; + } /// Return true if sqrt(x) is as cheap or cheaper than 1 / rsqrt(x) bool isFsqrtCheap() const { @@ -242,9 +249,6 @@ public: return BypassSlowDivWidths; } - /// Return true if pow2 sdiv is cheaper than a chain of sra/srl/add/sra. - bool isPow2SDivCheap() const { return Pow2SDivIsCheap; } - /// Return true if Flow Control is an expensive operation that should be /// avoided. bool isJumpExpensive() const { return JumpIsExpensive; } @@ -409,20 +413,20 @@ public: class ValueTypeActionImpl { /// ValueTypeActions - For each value type, keep a LegalizeTypeAction enum /// that indicates how instruction selection should deal with the type. - uint8_t ValueTypeActions[MVT::LAST_VALUETYPE]; + LegalizeTypeAction ValueTypeActions[MVT::LAST_VALUETYPE]; public: ValueTypeActionImpl() { - std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions), 0); + std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions), + TypeLegal); } LegalizeTypeAction getTypeAction(MVT VT) const { - return (LegalizeTypeAction)ValueTypeActions[VT.SimpleTy]; + return ValueTypeActions[VT.SimpleTy]; } void setTypeAction(MVT VT, LegalizeTypeAction Action) { - unsigned I = VT.SimpleTy; - ValueTypeActions[I] = Action; + ValueTypeActions[VT.SimpleTy] = Action; } }; @@ -546,8 +550,7 @@ public: // If a target-specific SDNode requires legalization, require the target // to provide custom legalization for it. if (Op > array_lengthof(OpActions[0])) return Custom; - unsigned I = (unsigned) VT.getSimpleVT().SimpleTy; - return (LegalizeAction)OpActions[I][Op]; + return OpActions[(unsigned)VT.getSimpleVT().SimpleTy][Op]; } /// Return true if the specified operation is legal on this target or can be @@ -591,7 +594,7 @@ public: unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy; assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE && "Table isn't big enough!"); - return (LegalizeAction)LoadExtActions[ValI][MemI][ExtType]; + return LoadExtActions[ValI][MemI][ExtType]; } /// Return true if the specified load with extension is legal on this target. @@ -617,7 +620,7 @@ public: unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy; assert(ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE && "Table isn't big enough!"); - return (LegalizeAction)TruncStoreActions[ValI][MemI]; + return TruncStoreActions[ValI][MemI]; } /// Return true if the specified store with truncation is legal on this @@ -672,9 +675,9 @@ public: ((unsigned)VT.SimpleTy >> 4) < array_lengthof(CondCodeActions[0]) && "Table isn't big enough!"); // See setCondCodeAction for how this is encoded. - uint32_t Shift = 2 * (VT.SimpleTy & 0xF); - uint32_t Value = CondCodeActions[CC][VT.SimpleTy >> 4]; - LegalizeAction Action = (LegalizeAction) ((Value >> Shift) & 0x3); + uint32_t Shift = 4 * (VT.SimpleTy & 0x7); + uint32_t Value = CondCodeActions[CC][VT.SimpleTy >> 3]; + LegalizeAction Action = (LegalizeAction) ((Value >> Shift) & 0xF); assert(Action != Promote && "Can't promote condition code!"); return Action; } @@ -832,6 +835,10 @@ public: return TargetDAGCombineArray[NT >> 3] & (1 << (NT&7)); } + unsigned getGatherAllAliasesMaxDepth() const { + return GatherAllAliasesMaxDepth; + } + /// \brief Get maximum # of store operations permitted for llvm.memset /// /// This function returns the maximum number of store operations permitted @@ -878,6 +885,14 @@ public: return false; } + /// Return true if the target supports a memory access of this type for the + /// given address space and alignment. If the access is allowed, the optional + /// final parameter returns if the access is also fast (as defined by the + /// target). + bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, + unsigned AddrSpace = 0, unsigned Alignment = 1, + bool *Fast = nullptr) const; + /// Returns the target specific optimal type for load and store operations as /// a result of memset, memcpy, and memmove lowering. /// @@ -930,15 +945,19 @@ public: } /// If a physical register, this returns the register that receives the - /// exception address on entry to a landing pad. - unsigned getExceptionPointerRegister() const { - return ExceptionPointerRegister; + /// exception address on entry to an EH pad. + virtual unsigned + getExceptionPointerRegister(const Constant *PersonalityFn) const { + // 0 is guaranteed to be the NoRegister value on all targets + return 0; } /// If a physical register, this returns the register that receives the /// exception typeid on entry to a landing pad. - unsigned getExceptionSelectorRegister() const { - return ExceptionSelectorRegister; + virtual unsigned + getExceptionSelectorRegister(const Constant *PersonalityFn) const { + // 0 is guaranteed to be the NoRegister value on all targets + return 0; } /// Returns the target's jmp_buf size in bytes (if never set, the default is @@ -987,6 +1006,10 @@ public: return false; } + /// If the target has a standard location for the unsafe stack pointer, + /// returns the address of that location. Otherwise, returns nullptr. + virtual Value *getSafeStackPointerLocation(IRBuilder<> &IRB) const; + /// Returns true if a cast between SrcAS and DestAS is a noop. virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const { return false; @@ -1009,8 +1032,8 @@ public: int InstructionOpcodeToISD(unsigned Opcode) const; /// Estimate the cost of type-legalization and the legalized type. - std::pair<unsigned, MVT> getTypeLegalizationCost(const DataLayout &DL, - Type *Ty) const; + std::pair<int, MVT> getTypeLegalizationCost(const DataLayout &DL, + Type *Ty) const; /// @} @@ -1018,10 +1041,6 @@ public: /// \name Helpers for atomic expansion. /// @{ - /// True if AtomicExpandPass should use emitLoadLinked/emitStoreConditional - /// and expand AtomicCmpXchgInst. - virtual bool hasLoadLinkedStoreConditional() const { return false; } - /// Perform a load-linked operation on Addr, returning a "Value *" with the /// corresponding pointee type. This may entail some non-trivial operations to /// truncate or reconstruct types that will be illegal in the backend. See @@ -1093,6 +1112,14 @@ public: } /// @} + // Emits code that executes when the comparison result in the ll/sc + // expansion of a cmpxchg instruction is such that the store-conditional will + // not execute. This makes it possible to balance out the load-linked with + // a dedicated instruction, if desired. + // E.g., on ARM, if ldrex isn't followed by strex, the exclusive monitor would + // be unnecessarily held, except if clrex, inserted by this hook, is executed. + virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const {} + /// Returns true if the given (atomic) store should be expanded by the /// IR-level AtomicExpand pass into an "atomic xchg" which ignores its input. virtual bool shouldExpandAtomicStoreInIR(StoreInst *SI) const { @@ -1102,18 +1129,25 @@ public: /// Returns true if arguments should be sign-extended in lib calls. virtual bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const { return IsSigned; - } + } + + /// Returns how the given (atomic) load should be expanded by the + /// IR-level AtomicExpand pass. + virtual AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const { + return AtomicExpansionKind::None; + } - /// Returns true if the given (atomic) load should be expanded by the - /// IR-level AtomicExpand pass into a load-linked instruction - /// (through emitLoadLinked()). - virtual bool shouldExpandAtomicLoadInIR(LoadInst *LI) const { return false; } + /// Returns true if the given atomic cmpxchg should be expanded by the + /// IR-level AtomicExpand pass into a load-linked/store-conditional sequence + /// (through emitLoadLinked() and emitStoreConditional()). + virtual bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { + return false; + } /// Returns how the IR-level AtomicExpand pass should expand the given /// AtomicRMW, if at all. Default is to never expand. - virtual AtomicRMWExpansionKind - shouldExpandAtomicRMWInIR(AtomicRMWInst *) const { - return AtomicRMWExpansionKind::None; + virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const { + return AtomicExpansionKind::None; } /// On some platforms, an AtomicRMW that never actually modifies the value @@ -1204,18 +1238,6 @@ protected: StackPointerRegisterToSaveRestore = R; } - /// If set to a physical register, this sets the register that receives the - /// exception address on entry to a landing pad. - void setExceptionPointerRegister(unsigned R) { - ExceptionPointerRegister = R; - } - - /// If set to a physical register, this sets the register that receives the - /// exception typeid on entry to a landing pad. - void setExceptionSelectorRegister(unsigned R) { - ExceptionSelectorRegister = R; - } - /// Tells the code generator not to expand operations into sequences that use /// the select operations if possible. void setSelectIsExpensive(bool isExpensive = true) { @@ -1244,11 +1266,6 @@ protected: /// control. void setJumpIsExpensive(bool isExpensive = true); - /// Tells the code generator that integer divide is expensive, and if - /// possible, should be replaced by an alternate sequence of instructions not - /// containing an integer divide. - void setIntDivIsCheap(bool isCheap = true) { IntDivIsCheap = isCheap; } - /// Tells the code generator that fsqrt is cheap, and should not be replaced /// with an alternative sequence of instructions. void setFsqrtIsCheap(bool isCheap = true) { FsqrtIsCheap = isCheap; } @@ -1264,10 +1281,6 @@ protected: BypassSlowDivWidths[SlowBitWidth] = FastBitWidth; } - /// Tells the code generator that it shouldn't generate sra/srl/add/sra for a - /// signed divide by power of two; let the target handle it. - void setPow2SDivIsCheap(bool isCheap = true) { Pow2SDivIsCheap = isCheap; } - /// Add the specified register class as an available regclass for the /// specified value type. This indicates the selector can handle values of /// that class natively. @@ -1279,7 +1292,7 @@ protected: /// Remove all register classes. void clearRegisterClasses() { - memset(RegClassForVT, 0,MVT::LAST_VALUETYPE * sizeof(TargetRegisterClass*)); + std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr); AvailableRegClasses.clear(); } @@ -1302,7 +1315,7 @@ protected: void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action) { assert(Op < array_lengthof(OpActions[0]) && "Table isn't big enough!"); - OpActions[(unsigned)VT.SimpleTy][Op] = (uint8_t)Action; + OpActions[(unsigned)VT.SimpleTy][Op] = Action; } /// Indicate that the specified load with extension does not work with the @@ -1311,7 +1324,7 @@ protected: LegalizeAction Action) { assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid() && MemVT.isValid() && "Table isn't big enough!"); - LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy][ExtType] = (uint8_t)Action; + LoadExtActions[(unsigned)ValVT.SimpleTy][MemVT.SimpleTy][ExtType] = Action; } /// Indicate that the specified truncating store does not work with the @@ -1319,7 +1332,7 @@ protected: void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action) { assert(ValVT.isValid() && MemVT.isValid() && "Table isn't big enough!"); - TruncStoreActions[ValVT.SimpleTy][MemVT.SimpleTy] = (uint8_t)Action; + TruncStoreActions[(unsigned)ValVT.SimpleTy][MemVT.SimpleTy] = Action; } /// Indicate that the specified indexed load does or does not work with the @@ -1356,12 +1369,13 @@ protected: LegalizeAction Action) { assert(VT.isValid() && (unsigned)CC < array_lengthof(CondCodeActions) && "Table isn't big enough!"); - /// The lower 5 bits of the SimpleTy index into Nth 2bit set from the 32-bit - /// value and the upper 27 bits index into the second dimension of the array + assert((unsigned)Action < 0x10 && "too many bits for bitfield array"); + /// The lower 3 bits of the SimpleTy index into Nth 4bit set from the 32-bit + /// value and the upper 29 bits index into the second dimension of the array /// to select what 32-bit value to use. - uint32_t Shift = 2 * (VT.SimpleTy & 0xF); - CondCodeActions[CC][VT.SimpleTy >> 4] &= ~((uint32_t)0x3 << Shift); - CondCodeActions[CC][VT.SimpleTy >> 4] |= (uint32_t)Action << Shift; + uint32_t Shift = 4 * (VT.SimpleTy & 0x7); + CondCodeActions[CC][VT.SimpleTy >> 3] &= ~((uint32_t)0xF << Shift); + CondCodeActions[CC][VT.SimpleTy >> 3] |= (uint32_t)Action << Shift; } /// If Opc/OrigVT is specified as being promoted, the promotion code defaults @@ -1504,23 +1518,24 @@ public: return false; } - /// Return true if it's free to truncate a value of type Ty1 to type - /// Ty2. e.g. On x86 it's free to truncate a i32 value in register EAX to i16 + /// Return true if it's free to truncate a value of type FromTy to type + /// ToTy. e.g. On x86 it's free to truncate a i32 value in register EAX to i16 /// by referencing its sub-register AX. - virtual bool isTruncateFree(Type * /*Ty1*/, Type * /*Ty2*/) const { + /// Targets must return false when FromTy <= ToTy. + virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const { return false; } - /// Return true if a truncation from Ty1 to Ty2 is permitted when deciding + /// Return true if a truncation from FromTy to ToTy is permitted when deciding /// whether a call is in tail position. Typically this means that both results /// would be assigned to the same register or stack slot, but it could mean /// the target performs adequate checks of its own before proceeding with the - /// tail call. - virtual bool allowTruncateForTailCall(Type * /*Ty1*/, Type * /*Ty2*/) const { + /// tail call. Targets must return false when FromTy <= ToTy. + virtual bool allowTruncateForTailCall(Type *FromTy, Type *ToTy) const { return false; } - virtual bool isTruncateFree(EVT /*VT1*/, EVT /*VT2*/) const { + virtual bool isTruncateFree(EVT FromVT, EVT ToVT) const { return false; } @@ -1553,19 +1568,21 @@ public: return isExtFreeImpl(I); } - /// Return true if any actual instruction that defines a value of type Ty1 - /// implicitly zero-extends the value to Ty2 in the result register. + /// Return true if any actual instruction that defines a value of type FromTy + /// implicitly zero-extends the value to ToTy in the result register. /// - /// This does not necessarily include registers defined in unknown ways, such - /// as incoming arguments, or copies from unknown virtual registers. Also, if - /// isTruncateFree(Ty2, Ty1) is true, this does not necessarily apply to - /// truncate instructions. e.g. on x86-64, all instructions that define 32-bit - /// values implicit zero-extend the result out to 64 bits. - virtual bool isZExtFree(Type * /*Ty1*/, Type * /*Ty2*/) const { + /// The function should return true when it is likely that the truncate can + /// be freely folded with an instruction defining a value of FromTy. If + /// the defining instruction is unknown (because you're looking at a + /// function argument, PHI, etc.) then the target may require an + /// explicit truncate, which is not necessarily free, but this function + /// does not deal with those cases. + /// Targets must return false when FromTy >= ToTy. + virtual bool isZExtFree(Type *FromTy, Type *ToTy) const { return false; } - virtual bool isZExtFree(EVT /*VT1*/, EVT /*VT2*/) const { + virtual bool isZExtFree(EVT FromTy, EVT ToTy) const { return false; } @@ -1699,6 +1716,12 @@ public: return false; } + // Return true if it is profitable to use a scalar input to a BUILD_VECTOR + // even if the vector itself has multiple uses. + virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const { + return false; + } + //===--------------------------------------------------------------------===// // Runtime Library hooks // @@ -1755,12 +1778,6 @@ private: /// combined with "shift" to BitExtract instructions. bool HasExtractBitsInsn; - /// Tells the code generator not to expand integer divides by constants into a - /// sequence of muls, adds, and shifts. This is a hack until a real cost - /// model is in place. If we ever optimize for size, this will be set to true - /// unconditionally. - bool IntDivIsCheap; - // Don't expand fsqrt with an approximation based on the inverse sqrt. bool FsqrtIsCheap; @@ -1770,10 +1787,6 @@ private: /// div/rem when the operands are positive and less than 256. DenseMap <unsigned int, unsigned int> BypassSlowDivWidths; - /// Tells the code generator that it shouldn't generate sra/srl/add/sra for a - /// signed divide by power of two; let the target handle it. - bool Pow2SDivIsCheap; - /// Tells the code generator that it shouldn't generate extra flow control /// instructions and should attempt to combine flow control instructions via /// predication. @@ -1841,14 +1854,6 @@ private: /// llvm.savestack/llvm.restorestack should save and restore. unsigned StackPointerRegisterToSaveRestore; - /// If set to a physical register, this specifies the register that receives - /// the exception address on entry to a landing pad. - unsigned ExceptionPointerRegister; - - /// If set to a physical register, this specifies the register that receives - /// the exception typeid on entry to a landing pad. - unsigned ExceptionSelectorRegister; - /// This indicates the default register class to use for each ValueType the /// target supports natively. const TargetRegisterClass *RegClassForVT[MVT::LAST_VALUETYPE]; @@ -1880,17 +1885,17 @@ private: /// operations are Legal (aka, supported natively by the target), but /// operations that are not should be described. Note that operations on /// non-legal value types are not described here. - uint8_t OpActions[MVT::LAST_VALUETYPE][ISD::BUILTIN_OP_END]; + LegalizeAction OpActions[MVT::LAST_VALUETYPE][ISD::BUILTIN_OP_END]; /// For each load extension type and each value type, keep a LegalizeAction /// that indicates how instruction selection should deal with a load of a /// specific value type and extension type. - uint8_t LoadExtActions[MVT::LAST_VALUETYPE][MVT::LAST_VALUETYPE] - [ISD::LAST_LOADEXT_TYPE]; + LegalizeAction LoadExtActions[MVT::LAST_VALUETYPE][MVT::LAST_VALUETYPE] + [ISD::LAST_LOADEXT_TYPE]; /// For each value type pair keep a LegalizeAction that indicates whether a /// truncating store of a specific value type and truncating type is legal. - uint8_t TruncStoreActions[MVT::LAST_VALUETYPE][MVT::LAST_VALUETYPE]; + LegalizeAction TruncStoreActions[MVT::LAST_VALUETYPE][MVT::LAST_VALUETYPE]; /// For each indexed mode and each value type, keep a pair of LegalizeAction /// that indicates how instruction selection should deal with the load / @@ -1903,11 +1908,12 @@ private: /// For each condition code (ISD::CondCode) keep a LegalizeAction that /// indicates how instruction selection should deal with the condition code. /// - /// Because each CC action takes up 2 bits, we need to have the array size be + /// Because each CC action takes up 4 bits, we need to have the array size be /// large enough to fit all of the value types. This can be done by rounding - /// up the MVT::LAST_VALUETYPE value to the next multiple of 16. - uint32_t CondCodeActions[ISD::SETCC_INVALID][(MVT::LAST_VALUETYPE + 15) / 16]; + /// up the MVT::LAST_VALUETYPE value to the next multiple of 8. + uint32_t CondCodeActions[ISD::SETCC_INVALID][(MVT::LAST_VALUETYPE + 7) / 8]; +protected: ValueTypeActionImpl ValueTypeActions; private: @@ -1947,6 +1953,12 @@ protected: /// is[Z|FP]ExtFree of the related types is not true. virtual bool isExtFreeImpl(const Instruction *I) const { return false; } + /// Depth that GatherAllAliases should should continue looking for chain + /// dependencies when trying to find a more preferrable chain. As an + /// approximation, this should be more than the number of consecutive stores + /// expected to be merged. + unsigned GatherAllAliasesMaxDepth; + /// \brief Specify maximum number of store instructions per memset call. /// /// When lowering \@llvm.memset this field specifies the maximum number of @@ -1993,7 +2005,7 @@ protected: unsigned MaxStoresPerMemmove; /// Maximum number of store instructions that may be substituted for a call to - /// memmove, used for functions with OpSize attribute. + /// memmove, used for functions with OptSize attribute. unsigned MaxStoresPerMemmoveOptSize; /// Tells the code generator that select is more expensive than a branch if @@ -2087,9 +2099,9 @@ public: /// Returns a pair of (return value, chain). /// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC. std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, - EVT RetVT, const SDValue *Ops, - unsigned NumOps, bool isSigned, - SDLoc dl, bool doesNotReturn = false, + EVT RetVT, ArrayRef<SDValue> Ops, + bool isSigned, SDLoc dl, + bool doesNotReturn = false, bool isReturnValueUsed = true) const; //===--------------------------------------------------------------------===// @@ -2251,6 +2263,29 @@ public: return false; } + /// Return true if the target supports that a subset of CSRs for the given + /// machine function is handled explicitly via copies. + virtual bool supportSplitCSR(MachineFunction *MF) const { + return false; + } + + /// Perform necessary initialization to handle a subset of CSRs explicitly + /// via copies. This function is called at the beginning of instruction + /// selection. + virtual void initializeSplitCSR(MachineBasicBlock *Entry) const { + llvm_unreachable("Not Implemented"); + } + + /// Insert explicit copies in entry and exit blocks. We copy a subset of + /// CSRs to virtual registers in the entry block, and copy them back to + /// physical registers in the exit blocks. This function is called at the end + /// of instruction selection. + virtual void insertCopiesSplitCSR( + MachineBasicBlock *Entry, + const SmallVectorImpl<MachineBasicBlock *> &Exits) const { + llvm_unreachable("Not Implemented"); + } + //===--------------------------------------------------------------------===// // Lowering methods - These methods must be implemented by targets so that // the SelectionDAGBuilder code knows how to lower these. @@ -2726,16 +2761,21 @@ public: SDValue BuildUDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, bool IsAfterLegalization, std::vector<SDNode *> *Created) const; + + /// Targets may override this function to provide custom SDIV lowering for + /// power-of-2 denominators. If the target returns an empty SDValue, LLVM + /// assumes SDIV is expensive and replaces it with a series of other integer + /// operations. virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, - std::vector<SDNode *> *Created) const { - return SDValue(); - } + std::vector<SDNode *> *Created) const; - /// Indicate whether this target prefers to combine the given number of FDIVs - /// with the same divisor. - virtual bool combineRepeatedFPDivisors(unsigned NumUsers) const { - return false; + /// Indicate whether this target prefers to combine FDIVs with the same + /// divisor. If the transform should never be done, return zero. If the + /// transform should be done, return the minimum number of divisor uses + /// that must exist. + virtual unsigned combineRepeatedFPDivisors() const { + return 0; } /// Hooks for building estimates in place of slower divisions and square @@ -2821,6 +2861,10 @@ public: virtual bool useLoadStackGuardNode() const { return false; } + + /// Lower TLS global address SDNode for target independent emulated TLS model. + virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, + SelectionDAG &DAG) const; }; /// Given an LLVM IR type and return type attributes, compute the return value diff --git a/include/llvm/Target/TargetLoweringObjectFile.h b/include/llvm/Target/TargetLoweringObjectFile.h index 5b626c244ba0..cb52698c58b9 100644 --- a/include/llvm/Target/TargetLoweringObjectFile.h +++ b/include/llvm/Target/TargetLoweringObjectFile.h @@ -42,16 +42,15 @@ class TargetLoweringObjectFile : public MCObjectFileInfo { void operator=(const TargetLoweringObjectFile&) = delete; protected: - const DataLayout *DL; bool SupportIndirectSymViaGOTPCRel; bool SupportGOTPCRelWithOffset; public: MCContext &getContext() const { return *Ctx; } - TargetLoweringObjectFile() : MCObjectFileInfo(), Ctx(nullptr), DL(nullptr), - SupportIndirectSymViaGOTPCRel(false), - SupportGOTPCRelWithOffset(true) {} + TargetLoweringObjectFile() + : MCObjectFileInfo(), Ctx(nullptr), SupportIndirectSymViaGOTPCRel(false), + SupportGOTPCRelWithOffset(true) {} virtual ~TargetLoweringObjectFile(); @@ -60,8 +59,7 @@ public: /// implementations a chance to set up their default sections. virtual void Initialize(MCContext &ctx, const TargetMachine &TM); - virtual void emitPersonalityValue(MCStreamer &Streamer, - const TargetMachine &TM, + virtual void emitPersonalityValue(MCStreamer &Streamer, const DataLayout &TM, const MCSymbol *Sym) const; /// Emit the module flags that the platform cares about. @@ -71,7 +69,8 @@ public: /// Given a constant with the SectionKind, return a section that it should be /// placed in. - virtual MCSection *getSectionForConstant(SectionKind Kind, + virtual MCSection *getSectionForConstant(const DataLayout &DL, + SectionKind Kind, const Constant *C) const; /// Classify the specified global variable into a set of target independent @@ -94,8 +93,7 @@ public: } virtual void getNameWithPrefix(SmallVectorImpl<char> &OutName, - const GlobalValue *GV, - bool CannotUsePrivateLabel, Mangler &Mang, + const GlobalValue *GV, Mangler &Mang, const TargetMachine &TM) const; virtual MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang, diff --git a/include/llvm/Target/TargetMachine.h b/include/llvm/Target/TargetMachine.h index f1e9d1718f5a..74e91b5790cb 100644 --- a/include/llvm/Target/TargetMachine.h +++ b/include/llvm/Target/TargetMachine.h @@ -76,7 +76,12 @@ protected: // Can only create subclasses. /// The Target that this machine was created for. const Target &TheTarget; - /// For ABI type size and alignment. + /// DataLayout for the target: keep ABI type size and alignment. + /// + /// The DataLayout is created based on the string representation provided + /// during construction. It is kept here only to avoid reparsing the string + /// but should not really be used during compilation, because it has an + /// internal cache that is context specific. const DataLayout DL; /// Triple string, CPU name, and target feature strings the TargetMachine @@ -97,6 +102,12 @@ protected: // Can only create subclasses. const MCSubtargetInfo *STI; unsigned RequireStructuredCFG : 1; + unsigned O0WantsFastISel : 1; + + /// This API is here to support the C API, deprecated in 3.7 release. + /// This should never be used outside of legacy existing client. + const DataLayout &getDataLayout() const { return DL; } + friend struct C_API_PRIVATE_ACCESS; public: mutable TargetOptions Options; @@ -125,15 +136,23 @@ public: return *static_cast<const STC*>(getSubtargetImpl(F)); } - /// Deprecated in 3.7, will be removed in 3.8. Use createDataLayout() instead. - /// - /// This method returns a pointer to the DataLayout for the target. It should - /// be unchanging for every subtarget. - const DataLayout *getDataLayout() const { return &DL; } - /// Create a DataLayout. const DataLayout createDataLayout() const { return DL; } + /// Test if a DataLayout if compatible with the CodeGen for this target. + /// + /// The LLVM Module owns a DataLayout that is used for the target independent + /// optimizations and code generation. This hook provides a target specific + /// check on the validity of this DataLayout. + bool isCompatibleDataLayout(const DataLayout &Candidate) const { + return DL == Candidate; + } + + /// Get the pointer size for this target. + /// + /// This is the only time the DataLayout in the TargetMachine is used. + unsigned getPointerSize() const { return DL.getPointerSize(); } + /// \brief Reset the target options based on the function's attributes. // FIXME: Remove TargetOptions that affect per-function code generation // from TargetMachine. @@ -172,6 +191,8 @@ public: void setOptLevel(CodeGenOpt::Level Level) const; void setFastISel(bool Enable) { Options.EnableFastISel = Enable; } + bool getO0WantsFastISel() { return O0WantsFastISel; } + void setO0WantsFastISel(bool Enable) { O0WantsFastISel = Enable; } bool shouldPrintMachineCode() const { return Options.PrintMachineCode; } @@ -234,6 +255,13 @@ public: return true; } + /// True if subtarget inserts the final scheduling pass on its own. + /// + /// Branch relaxation, which must happen after block placement, can + /// on some targets (e.g. SystemZ) expose additional post-RA + /// scheduling opportunities. + virtual bool targetSchedulesPostRAScheduling() const { return false; }; + void getNameWithPrefix(SmallVectorImpl<char> &Name, const GlobalValue *GV, Mangler &Mang, bool MayAlwaysUsePrivate = false) const; MCSymbol *getSymbol(const GlobalValue *GV, Mangler &Mang) const; diff --git a/include/llvm/Target/TargetOpcodes.h b/include/llvm/Target/TargetOpcodes.h index 50197191109d..db37bdb62582 100644 --- a/include/llvm/Target/TargetOpcodes.h +++ b/include/llvm/Target/TargetOpcodes.h @@ -126,8 +126,12 @@ enum { /// Loading instruction that may page fault, bundled with associated /// information on how to handle such a page fault. It is intended to support /// "zero cost" null checks in managed languages by allowing LLVM to fold - /// comparisions into existing memory operations. + /// comparisons into existing memory operations. FAULTING_LOAD_OP = 22, + + /// BUILTIN_OP_END - This must be the last enum value in this list. + /// The target-specific post-isel opcode values start here. + GENERIC_OP_END = FAULTING_LOAD_OP, }; } // end namespace TargetOpcode } // end namespace llvm diff --git a/include/llvm/Target/TargetOptions.h b/include/llvm/Target/TargetOptions.h index d52cb60cf108..d98d0fa0ed5f 100644 --- a/include/llvm/Target/TargetOptions.h +++ b/include/llvm/Target/TargetOptions.h @@ -58,24 +58,53 @@ namespace llvm { }; } + enum class EABI { + Unknown, + Default, // Default means not specified + EABI4, // Target-specific (either 4, 5 or gnu depending on triple). + EABI5, + GNU + }; + + /// Identify a debugger for "tuning" the debug info. + /// + /// The "debugger tuning" concept allows us to present a more intuitive + /// interface that unpacks into different sets of defaults for the various + /// individual feature-flag settings, that suit the preferences of the + /// various debuggers. However, it's worth remembering that debuggers are + /// not the only consumers of debug info, and some variations in DWARF might + /// better be treated as target/platform issues. Fundamentally, + /// o if the feature is useful (or not) to a particular debugger, regardless + /// of the target, that's a tuning decision; + /// o if the feature is useful (or not) on a particular platform, regardless + /// of the debugger, that's a target decision. + /// It's not impossible to see both factors in some specific case. + /// + /// The "tuning" should be used to set defaults for individual feature flags + /// in DwarfDebug; if a given feature has a more specific command-line option, + /// that option should take precedence over the tuning. + enum class DebuggerKind { + Default, // No specific tuning requested. + GDB, // Tune debug info for gdb. + LLDB, // Tune debug info for lldb. + SCE // Tune debug info for SCE targets (e.g. PS4). + }; + class TargetOptions { public: TargetOptions() - : PrintMachineCode(false), - LessPreciseFPMADOption(false), UnsafeFPMath(false), - NoInfsFPMath(false), NoNaNsFPMath(false), - HonorSignDependentRoundingFPMathOption(false), - NoZerosInBSS(false), - GuaranteedTailCallOpt(false), - StackAlignmentOverride(0), + : PrintMachineCode(false), LessPreciseFPMADOption(false), + UnsafeFPMath(false), NoInfsFPMath(false), NoNaNsFPMath(false), + HonorSignDependentRoundingFPMathOption(false), NoZerosInBSS(false), + GuaranteedTailCallOpt(false), StackAlignmentOverride(0), EnableFastISel(false), PositionIndependentExecutable(false), UseInitArray(false), DisableIntegratedAS(false), CompressDebugSections(false), FunctionSections(false), DataSections(false), UniqueSectionNames(true), TrapUnreachable(false), - FloatABIType(FloatABI::Default), + EmulatedTLS(false), FloatABIType(FloatABI::Default), AllowFPOpFusion(FPOpFusion::Standard), Reciprocals(TargetRecip()), - JTType(JumpTable::Single), - ThreadModel(ThreadModel::POSIX) {} + JTType(JumpTable::Single), ThreadModel(ThreadModel::POSIX), + EABIVersion(EABI::Default), DebuggerTuning(DebuggerKind::Default) {} /// PrintMachineCode - This flag is enabled when the -print-machineinstrs /// option is specified on the command line, and should enable debugging @@ -172,6 +201,10 @@ namespace llvm { /// Emit target-specific trap instruction for 'unreachable' IR instructions. unsigned TrapUnreachable : 1; + /// EmulatedTLS - This flag enables emulated TLS model, using emutls + /// function in the runtime library.. + unsigned EmulatedTLS : 1; + /// FloatABIType - This setting is set by -float-abi=xxx option is specfied /// on the command line. This setting may either be Default, Soft, or Hard. /// Default selects the target's default behavior. Soft selects the ABI for @@ -200,7 +233,7 @@ namespace llvm { /// This class encapsulates options for reciprocal-estimate code generation. TargetRecip Reciprocals; - + /// JTType - This flag specifies the type of jump-instruction table to /// create for functions that have the jumptable attribute. JumpTable::JumpTableType JTType; @@ -209,6 +242,12 @@ namespace llvm { /// for things like atomics ThreadModel::Model ThreadModel; + /// EABIVersion - This flag specifies the EABI version + EABI EABIVersion; + + /// Which debugger to tune for. + DebuggerKind DebuggerTuning; + /// Machine level options. MCTargetOptions MCOptions; }; @@ -231,11 +270,14 @@ inline bool operator==(const TargetOptions &LHS, ARE_EQUAL(PositionIndependentExecutable) && ARE_EQUAL(UseInitArray) && ARE_EQUAL(TrapUnreachable) && + ARE_EQUAL(EmulatedTLS) && ARE_EQUAL(FloatABIType) && ARE_EQUAL(AllowFPOpFusion) && ARE_EQUAL(Reciprocals) && ARE_EQUAL(JTType) && ARE_EQUAL(ThreadModel) && + ARE_EQUAL(EABIVersion) && + ARE_EQUAL(DebuggerTuning) && ARE_EQUAL(MCOptions); #undef ARE_EQUAL } diff --git a/include/llvm/Target/TargetRecip.h b/include/llvm/Target/TargetRecip.h index 4cc3672d758d..210d49324848 100644 --- a/include/llvm/Target/TargetRecip.h +++ b/include/llvm/Target/TargetRecip.h @@ -31,20 +31,20 @@ public: /// Initialize all or part of the operations from command-line options or /// a front end. TargetRecip(const std::vector<std::string> &Args); - + /// Set whether a particular reciprocal operation is enabled and how many /// refinement steps are needed when using it. Use "all" to set enablement /// and refinement steps for all operations. - void setDefaults(const StringRef &Key, bool Enable, unsigned RefSteps); + void setDefaults(StringRef Key, bool Enable, unsigned RefSteps); /// Return true if the reciprocal operation has been enabled by default or /// from the command-line. Return false if the operation has been disabled /// by default or from the command-line. - bool isEnabled(const StringRef &Key) const; + bool isEnabled(StringRef Key) const; /// Return the number of iterations necessary to refine the /// the result of a machine instruction for the given reciprocal operation. - unsigned getRefinementSteps(const StringRef &Key) const; + unsigned getRefinementSteps(StringRef Key) const; bool operator==(const TargetRecip &Other) const; @@ -52,14 +52,14 @@ private: enum { Uninitialized = -1 }; - + struct RecipParams { int8_t Enabled; int8_t RefinementSteps; - + RecipParams() : Enabled(Uninitialized), RefinementSteps(Uninitialized) {} }; - + std::map<StringRef, RecipParams> RecipMap; typedef std::map<StringRef, RecipParams>::iterator RecipIter; typedef std::map<StringRef, RecipParams>::const_iterator ConstRecipIter; diff --git a/include/llvm/Target/TargetRegisterInfo.h b/include/llvm/Target/TargetRegisterInfo.h index 0ee936a76211..fccaad4705d5 100644 --- a/include/llvm/Target/TargetRegisterInfo.h +++ b/include/llvm/Target/TargetRegisterInfo.h @@ -21,6 +21,8 @@ #include "llvm/CodeGen/MachineValueType.h" #include "llvm/IR/CallingConv.h" #include "llvm/MC/MCRegisterInfo.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Printable.h" #include <cassert> #include <functional> @@ -32,6 +34,24 @@ class RegScavenger; template<class T> class SmallVectorImpl; class VirtRegMap; class raw_ostream; +class LiveRegMatrix; + +/// A bitmask representing the covering of a register with sub-registers. +/// +/// This is typically used to track liveness at sub-register granularity. +/// Lane masks for sub-register indices are similar to register units for +/// physical registers. The individual bits in a lane mask can't be assigned +/// any specific meaning. They can be used to check if two sub-register +/// indices overlap. +/// +/// Iff the target has a register such that: +/// +/// getSubReg(Reg, A) overlaps getSubReg(Reg, B) +/// +/// then: +/// +/// (getSubRegIndexLaneMask(A) & getSubRegIndexLaneMask(B)) != 0 +typedef unsigned LaneBitmask; class TargetRegisterClass { public: @@ -45,7 +65,7 @@ public: const vt_iterator VTs; const uint32_t *SubClassMask; const uint16_t *SuperRegIndices; - const unsigned LaneMask; + const LaneBitmask LaneMask; /// Classes with a higher priority value are assigned first by register /// allocators using a greedy heuristic. The value is in the range [0,63]. const uint8_t AllocationPriority; @@ -54,8 +74,7 @@ public: const sc_iterator SuperClasses; ArrayRef<MCPhysReg> (*OrderFunc)(const MachineFunction&); - /// getID() - Return the register class ID number. - /// + /// Return the register class ID number. unsigned getID() const { return MC->getID(); } /// begin/end - Return all of the registers in this class. @@ -63,46 +82,42 @@ public: iterator begin() const { return MC->begin(); } iterator end() const { return MC->end(); } - /// getNumRegs - Return the number of registers in this class. - /// + /// Return the number of registers in this class. unsigned getNumRegs() const { return MC->getNumRegs(); } - /// getRegister - Return the specified register in the class. - /// + /// Return the specified register in the class. unsigned getRegister(unsigned i) const { return MC->getRegister(i); } - /// contains - Return true if the specified register is included in this - /// register class. This does not include virtual registers. + /// Return true if the specified register is included in this register class. + /// This does not include virtual registers. bool contains(unsigned Reg) const { return MC->contains(Reg); } - /// contains - Return true if both registers are in this class. + /// Return true if both registers are in this class. bool contains(unsigned Reg1, unsigned Reg2) const { return MC->contains(Reg1, Reg2); } - /// getSize - Return the size of the register in bytes, which is also the size + /// Return the size of the register in bytes, which is also the size /// of a stack slot allocated to hold a spilled copy of this register. unsigned getSize() const { return MC->getSize(); } - /// getAlignment - Return the minimum required alignment for a register of - /// this class. + /// Return the minimum required alignment for a register of this class. unsigned getAlignment() const { return MC->getAlignment(); } - /// getCopyCost - Return the cost of copying a value between two registers in - /// this class. A negative number means the register class is very expensive + /// Return the cost of copying a value between two registers in this class. + /// A negative number means the register class is very expensive /// to copy e.g. status flag register classes. int getCopyCost() const { return MC->getCopyCost(); } - /// isAllocatable - Return true if this register class may be used to create - /// virtual registers. + /// Return true if this register class may be used to create virtual + /// registers. bool isAllocatable() const { return MC->isAllocatable(); } - /// hasType - return true if this TargetRegisterClass has the ValueType vt. - /// + /// Return true if this TargetRegisterClass has the ValueType vt. bool hasType(MVT vt) const { for(int i = 0; VTs[i] != MVT::Other; ++i) if (MVT(VTs[i]) == vt) @@ -122,41 +137,39 @@ public: return I; } - /// hasSubClass - return true if the specified TargetRegisterClass + /// Return true if the specified TargetRegisterClass /// is a proper sub-class of this TargetRegisterClass. bool hasSubClass(const TargetRegisterClass *RC) const { return RC != this && hasSubClassEq(RC); } - /// hasSubClassEq - Returns true if RC is a sub-class of or equal to this - /// class. + /// Returns true if RC is a sub-class of or equal to this class. bool hasSubClassEq(const TargetRegisterClass *RC) const { unsigned ID = RC->getID(); return (SubClassMask[ID / 32] >> (ID % 32)) & 1; } - /// hasSuperClass - return true if the specified TargetRegisterClass is a + /// Return true if the specified TargetRegisterClass is a /// proper super-class of this TargetRegisterClass. bool hasSuperClass(const TargetRegisterClass *RC) const { return RC->hasSubClass(this); } - /// hasSuperClassEq - Returns true if RC is a super-class of or equal to this - /// class. + /// Returns true if RC is a super-class of or equal to this class. bool hasSuperClassEq(const TargetRegisterClass *RC) const { return RC->hasSubClassEq(this); } - /// getSubClassMask - Returns a bit vector of subclasses, including this one. + /// Returns a bit vector of subclasses, including this one. /// The vector is indexed by class IDs, see hasSubClassEq() above for how to /// use it. const uint32_t *getSubClassMask() const { return SubClassMask; } - /// getSuperRegIndices - Returns a 0-terminated list of sub-register indices - /// that project some super-register class into this register class. The list - /// has an entry for each Idx such that: + /// Returns a 0-terminated list of sub-register indices that project some + /// super-register class into this register class. The list has an entry for + /// each Idx such that: /// /// There exists SuperRC where: /// For all Reg in SuperRC: @@ -166,23 +179,23 @@ public: return SuperRegIndices; } - /// getSuperClasses - Returns a NULL terminated list of super-classes. The + /// Returns a NULL-terminated list of super-classes. The /// classes are ordered by ID which is also a topological ordering from large /// to small classes. The list does NOT include the current class. sc_iterator getSuperClasses() const { return SuperClasses; } - /// isASubClass - return true if this TargetRegisterClass is a subset + /// Return true if this TargetRegisterClass is a subset /// class of at least one other TargetRegisterClass. bool isASubClass() const { return SuperClasses[0] != nullptr; } - /// getRawAllocationOrder - Returns the preferred order for allocating - /// registers from this register class in MF. The raw order comes directly - /// from the .td file and may include reserved registers that are not - /// allocatable. Register allocators should also make sure to allocate + /// Returns the preferred order for allocating registers from this register + /// class in MF. The raw order comes directly from the .td file and may + /// include reserved registers that are not allocatable. + /// Register allocators should also make sure to allocate /// callee-saved registers only after all the volatiles are used. The /// RegisterClassInfo class provides filtered allocation orders with /// callee-saved registers moved to the end. @@ -200,13 +213,13 @@ public: /// Returns the combination of all lane masks of register in this class. /// The lane masks of the registers are the combination of all lane masks /// of their subregisters. - unsigned getLaneMask() const { + LaneBitmask getLaneMask() const { return LaneMask; } }; -/// TargetRegisterInfoDesc - Extra information, not in MCRegisterDesc, about -/// registers. These are used by codegen, not by MC. +/// Extra information, not in MCRegisterDesc, about registers. +/// These are used by codegen, not by MC. struct TargetRegisterInfoDesc { unsigned CostPerUse; // Extra cost of instructions using register. bool inAllocatableClass; // Register belongs to an allocatable regclass. @@ -232,7 +245,7 @@ private: const TargetRegisterInfoDesc *InfoDesc; // Extra desc array for codegen const char *const *SubRegIndexNames; // Names of subreg indexes. // Pointer to array of lane masks, one per sub-reg index. - const unsigned *SubRegIndexLaneMasks; + const LaneBitmask *SubRegIndexLaneMasks; regclass_iterator RegClassBegin, RegClassEnd; // List of regclasses unsigned CoveringLanes; @@ -242,7 +255,7 @@ protected: regclass_iterator RegClassBegin, regclass_iterator RegClassEnd, const char *const *SRINames, - const unsigned *SRILaneMasks, + const LaneBitmask *SRILaneMasks, unsigned CoveringLanes); virtual ~TargetRegisterInfo(); public: @@ -270,77 +283,74 @@ public: return int(Reg) >= (1 << 30); } - /// stackSlot2Index - Compute the frame index from a register value - /// representing a stack slot. + /// Compute the frame index from a register value representing a stack slot. static int stackSlot2Index(unsigned Reg) { assert(isStackSlot(Reg) && "Not a stack slot"); return int(Reg - (1u << 30)); } - /// index2StackSlot - Convert a non-negative frame index to a stack slot - /// register value. + /// Convert a non-negative frame index to a stack slot register value. static unsigned index2StackSlot(int FI) { assert(FI >= 0 && "Cannot hold a negative frame index."); return FI + (1u << 30); } - /// isPhysicalRegister - Return true if the specified register number is in + /// Return true if the specified register number is in /// the physical register namespace. static bool isPhysicalRegister(unsigned Reg) { assert(!isStackSlot(Reg) && "Not a register! Check isStackSlot() first."); return int(Reg) > 0; } - /// isVirtualRegister - Return true if the specified register number is in + /// Return true if the specified register number is in /// the virtual register namespace. static bool isVirtualRegister(unsigned Reg) { assert(!isStackSlot(Reg) && "Not a register! Check isStackSlot() first."); return int(Reg) < 0; } - /// virtReg2Index - Convert a virtual register number to a 0-based index. + /// Convert a virtual register number to a 0-based index. /// The first virtual register in a function will get the index 0. static unsigned virtReg2Index(unsigned Reg) { assert(isVirtualRegister(Reg) && "Not a virtual register"); return Reg & ~(1u << 31); } - /// index2VirtReg - Convert a 0-based index to a virtual register number. + /// Convert a 0-based index to a virtual register number. /// This is the inverse operation of VirtReg2IndexFunctor below. static unsigned index2VirtReg(unsigned Index) { return Index | (1u << 31); } - /// getMinimalPhysRegClass - Returns the Register Class of a physical - /// register of the given type, picking the most sub register class of - /// the right type that contains this physreg. + /// Returns the Register Class of a physical register of the given type, + /// picking the most sub register class of the right type that contains this + /// physreg. const TargetRegisterClass * getMinimalPhysRegClass(unsigned Reg, MVT VT = MVT::Other) const; - /// getAllocatableClass - Return the maximal subclass of the given register - /// class that is alloctable, or NULL. + /// Return the maximal subclass of the given register class that is + /// allocatable or NULL. const TargetRegisterClass * getAllocatableClass(const TargetRegisterClass *RC) const; - /// getAllocatableSet - Returns a bitset indexed by register number - /// indicating if a register is allocatable or not. If a register class is - /// specified, returns the subset for the class. + /// Returns a bitset indexed by register number indicating if a register is + /// allocatable or not. If a register class is specified, returns the subset + /// for the class. BitVector getAllocatableSet(const MachineFunction &MF, const TargetRegisterClass *RC = nullptr) const; - /// getCostPerUse - Return the additional cost of using this register instead + /// Return the additional cost of using this register instead /// of other registers in its class. unsigned getCostPerUse(unsigned RegNo) const { return InfoDesc[RegNo].CostPerUse; } - /// isInAllocatableClass - Return true if the register is in the allocation - /// of any register class. + /// Return true if the register is in the allocation of any register class. bool isInAllocatableClass(unsigned RegNo) const { return InfoDesc[RegNo].inAllocatableClass; } - /// getSubRegIndexName - Return the human-readable symbolic target-specific + /// Return the human-readable symbolic target-specific /// name for the specified SubRegIndex. const char *getSubRegIndexName(unsigned SubIdx) const { assert(SubIdx && SubIdx < getNumSubRegIndices() && @@ -348,44 +358,15 @@ public: return SubRegIndexNames[SubIdx-1]; } - /// getSubRegIndexLaneMask - Return a bitmask representing the parts of a - /// register that are covered by SubIdx. + /// Return a bitmask representing the parts of a register that are covered by + /// SubIdx \see LaneBitmask. /// - /// Lane masks for sub-register indices are similar to register units for - /// physical registers. The individual bits in a lane mask can't be assigned - /// any specific meaning. They can be used to check if two sub-register - /// indices overlap. - /// - /// If the target has a register such that: - /// - /// getSubReg(Reg, A) overlaps getSubReg(Reg, B) - /// - /// then: - /// - /// (getSubRegIndexLaneMask(A) & getSubRegIndexLaneMask(B)) != 0 - /// - /// The converse is not necessarily true. If two lane masks have a common - /// bit, the corresponding sub-registers may not overlap, but it can be - /// assumed that they usually will. /// SubIdx == 0 is allowed, it has the lane mask ~0u. - unsigned getSubRegIndexLaneMask(unsigned SubIdx) const { + LaneBitmask getSubRegIndexLaneMask(unsigned SubIdx) const { assert(SubIdx < getNumSubRegIndices() && "This is not a subregister index"); return SubRegIndexLaneMasks[SubIdx]; } - /// Returns true if the given lane mask is imprecise. - /// - /// LaneMasks as given by getSubRegIndexLaneMask() have a limited number of - /// bits, so for targets with more than 31 disjunct subregister indices there - /// may be cases where: - /// getSubReg(Reg,A) does not overlap getSubReg(Reg,B) - /// but we still have - /// (getSubRegIndexLaneMask(A) & getSubRegIndexLaneMask(B)) != 0. - /// This function returns true in those cases. - static bool isImpreciseLaneMask(unsigned LaneMask) { - return LaneMask & 0x80000000u; - } - /// The lane masks returned by getSubRegIndexLaneMask() above can only be /// used to determine if sub-registers overlap - they can't be used to /// determine if a set of sub-registers completely cover another @@ -409,10 +390,10 @@ public: /// /// If (MaskA & ~(MaskB & Covering)) == 0, then SubA is completely covered by /// SubB. - unsigned getCoveringLanes() const { return CoveringLanes; } + LaneBitmask getCoveringLanes() const { return CoveringLanes; } - /// regsOverlap - Returns true if the two registers are equal or alias each - /// other. The registers may be virtual register. + /// Returns true if the two registers are equal or alias each other. + /// The registers may be virtual registers. bool regsOverlap(unsigned regA, unsigned regB) const { if (regA == regB) return true; if (isVirtualRegister(regA) || isVirtualRegister(regB)) @@ -429,7 +410,7 @@ public: return false; } - /// hasRegUnit - Returns true if Reg contains RegUnit. + /// Returns true if Reg contains RegUnit. bool hasRegUnit(unsigned Reg, unsigned RegUnit) const { for (MCRegUnitIterator Units(Reg, this); Units.isValid(); ++Units) if (*Units == RegUnit) @@ -437,18 +418,23 @@ public: return false; } - /// getCalleeSavedRegs - Return a null-terminated list of all of the - /// callee saved registers on this target. The register should be in the - /// order of desired callee-save stack frame offset. The first register is - /// closest to the incoming stack pointer if stack grows down, and vice versa. + /// Return a null-terminated list of all of the callee-saved registers on + /// this target. The register should be in the order of desired callee-save + /// stack frame offset. The first register is closest to the incoming stack + /// pointer if stack grows down, and vice versa. /// virtual const MCPhysReg* getCalleeSavedRegs(const MachineFunction *MF) const = 0; - /// getCallPreservedMask - Return a mask of call-preserved registers for the - /// given calling convention on the current function. The mask should - /// include all call-preserved aliases. This is used by the register - /// allocator to determine which registers can be live across a call. + virtual const MCPhysReg* + getCalleeSavedRegsViaCopy(const MachineFunction *MF) const { + return nullptr; + } + + /// Return a mask of call-preserved registers for the given calling convention + /// on the current function. The mask should include all call-preserved + /// aliases. This is used by the register allocator to determine which + /// registers can be live across a call. /// /// The mask is an array containing (TRI::getNumRegs()+31)/32 entries. /// A set bit indicates that all bits of the corresponding register are @@ -469,13 +455,18 @@ public: return nullptr; } + /// Return a register mask that clobbers everything. + virtual const uint32_t *getNoPreservedMask() const { + llvm_unreachable("target does not provide no presered mask"); + } + /// Return all the call-preserved register masks defined for this target. virtual ArrayRef<const uint32_t *> getRegMasks() const = 0; virtual ArrayRef<const char *> getRegMaskNames() const = 0; - /// getReservedRegs - Returns a bitset indexed by physical register number - /// indicating if a register is a special register that has particular uses - /// and should be considered unavailable at all times, e.g. SP, RA. This is + /// Returns a bitset indexed by physical register number indicating if a + /// register is a special register that has particular uses and should be + /// considered unavailable at all times, e.g. SP, RA. This is /// used by register scavenger to determine what registers are free. virtual BitVector getReservedRegs(const MachineFunction &MF) const = 0; @@ -484,14 +475,14 @@ public: /// remove pseudo-registers that should be ignored). virtual void adjustStackMapLiveOutMask(uint32_t *Mask) const { } - /// getMatchingSuperReg - Return a super-register of the specified register + /// Return a super-register of the specified register /// Reg so its sub-register of index SubIdx is Reg. unsigned getMatchingSuperReg(unsigned Reg, unsigned SubIdx, const TargetRegisterClass *RC) const { return MCRegisterInfo::getMatchingSuperReg(Reg, SubIdx, RC->MC); } - /// getMatchingSuperRegClass - Return a subclass of the specified register + /// Return a subclass of the specified register /// class A so that each register in it has a sub-register of the /// specified sub-register index which is in the specified register class B. /// @@ -500,7 +491,16 @@ public: getMatchingSuperRegClass(const TargetRegisterClass *A, const TargetRegisterClass *B, unsigned Idx) const; - /// getSubClassWithSubReg - Returns the largest legal sub-class of RC that + // For a copy-like instruction that defines a register of class DefRC with + // subreg index DefSubReg, reading from another source with class SrcRC and + // subregister SrcSubReg return true if this is a preferrable copy + // instruction or an earlier use should be used. + virtual bool shouldRewriteCopySrc(const TargetRegisterClass *DefRC, + unsigned DefSubReg, + const TargetRegisterClass *SrcRC, + unsigned SrcSubReg) const; + + /// Returns the largest legal sub-class of RC that /// supports the sub-register index Idx. /// If no such sub-class exists, return NULL. /// If all registers in RC already have an Idx sub-register, return RC. @@ -518,7 +518,7 @@ public: return RC; } - /// composeSubRegIndices - Return the subregister index you get from composing + /// Return the subregister index you get from composing /// two subregister indices. /// /// The special null sub-register index composes as the identity. @@ -541,10 +541,11 @@ public: /// Transforms a LaneMask computed for one subregister to the lanemask that /// would have been computed when composing the subsubregisters with IdxA /// first. @sa composeSubRegIndices() - unsigned composeSubRegIndexLaneMask(unsigned IdxA, unsigned LaneMask) const { + LaneBitmask composeSubRegIndexLaneMask(unsigned IdxA, + LaneBitmask Mask) const { if (!IdxA) - return LaneMask; - return composeSubRegIndexLaneMaskImpl(IdxA, LaneMask); + return Mask; + return composeSubRegIndexLaneMaskImpl(IdxA, Mask); } /// Debugging helper: dump register in human readable form to dbgs() stream. @@ -558,13 +559,13 @@ protected: } /// Overridden by TableGen in targets that have sub-registers. - virtual unsigned - composeSubRegIndexLaneMaskImpl(unsigned, unsigned) const { + virtual LaneBitmask + composeSubRegIndexLaneMaskImpl(unsigned, LaneBitmask) const { llvm_unreachable("Target has no sub-registers"); } public: - /// getCommonSuperRegClass - Find a common super-register class if it exists. + /// Find a common super-register class if it exists. /// /// Find a register class, SuperRC and two sub-register indices, PreA and /// PreB, such that: @@ -605,44 +606,47 @@ public: return (unsigned)(regclass_end()-regclass_begin()); } - /// getRegClass - Returns the register class associated with the enumeration - /// value. See class MCOperandInfo. + /// Returns the register class associated with the enumeration value. + /// See class MCOperandInfo. const TargetRegisterClass *getRegClass(unsigned i) const { assert(i < getNumRegClasses() && "Register Class ID out of range"); return RegClassBegin[i]; } - /// getRegClassName - Returns the name of the register class. + /// Returns the name of the register class. const char *getRegClassName(const TargetRegisterClass *Class) const { return MCRegisterInfo::getRegClassName(Class->MC); } - /// getCommonSubClass - find the largest common subclass of A and B. Return - /// NULL if there is no common subclass. + /// Find the largest common subclass of A and B. + /// Return NULL if there is no common subclass. + /// The common subclass should contain + /// simple value type SVT if it is not the Any type. const TargetRegisterClass * getCommonSubClass(const TargetRegisterClass *A, - const TargetRegisterClass *B) const; + const TargetRegisterClass *B, + const MVT::SimpleValueType SVT = + MVT::SimpleValueType::Any) const; - /// getPointerRegClass - Returns a TargetRegisterClass used for pointer - /// values. If a target supports multiple different pointer register classes, + /// Returns a TargetRegisterClass used for pointer values. + /// If a target supports multiple different pointer register classes, /// kind specifies which one is indicated. virtual const TargetRegisterClass * getPointerRegClass(const MachineFunction &MF, unsigned Kind=0) const { llvm_unreachable("Target didn't implement getPointerRegClass!"); } - /// getCrossCopyRegClass - Returns a legal register class to copy a register - /// in the specified class to or from. If it is possible to copy the register - /// directly without using a cross register class copy, return the specified - /// RC. Returns NULL if it is not possible to copy between a two registers of - /// the specified class. + /// Returns a legal register class to copy a register in the specified class + /// to or from. If it is possible to copy the register directly without using + /// a cross register class copy, return the specified RC. Returns NULL if it + /// is not possible to copy between two registers of the specified class. virtual const TargetRegisterClass * getCrossCopyRegClass(const TargetRegisterClass *RC) const { return RC; } - /// getLargestLegalSuperClass - Returns the largest super class of RC that is - /// legal to use in the current sub-target and has the same spill size. + /// Returns the largest super class of RC that is legal to use in the current + /// sub-target and has the same spill size. /// The returned register class can be used to create virtual registers which /// means that all its registers can be copied and spilled. virtual const TargetRegisterClass * @@ -653,9 +657,9 @@ public: return RC; } - /// getRegPressureLimit - Return the register pressure "high water mark" for - /// the specific register class. The scheduler is in high register pressure - /// mode (for the specific register class) if it goes over the limit. + /// Return the register pressure "high water mark" for the specific register + /// class. The scheduler is in high register pressure mode (for the specific + /// register class) if it goes over the limit. /// /// Note: this is the old register pressure model that relies on a manually /// specified representative register class per value type. @@ -664,6 +668,15 @@ public: return 0; } + /// Return a heuristic for the machine scheduler to compare the profitability + /// of increasing one register pressure set versus another. The scheduler + /// will prefer increasing the register pressure of the set which returns + /// the largest value for this function. + virtual unsigned getRegPressureSetScore(const MachineFunction &MF, + unsigned PSetID) const { + return PSetID; + } + /// Get the weight in units of pressure for this register class. virtual const RegClassWeight &getRegClassWeight( const TargetRegisterClass *RC) const = 0; @@ -709,14 +722,15 @@ public: ArrayRef<MCPhysReg> Order, SmallVectorImpl<MCPhysReg> &Hints, const MachineFunction &MF, - const VirtRegMap *VRM = nullptr) const; - - /// updateRegAllocHint - A callback to allow target a chance to update - /// register allocation hints when a register is "changed" (e.g. coalesced) - /// to another register. e.g. On ARM, some virtual registers should target - /// register pairs, if one of pair is coalesced to another register, the - /// allocation hint of the other half of the pair should be changed to point - /// to the new register. + const VirtRegMap *VRM = nullptr, + const LiveRegMatrix *Matrix = nullptr) + const; + + /// A callback to allow target a chance to update register allocation hints + /// when a register is "changed" (e.g. coalesced) to another register. + /// e.g. On ARM, some virtual registers should target register pairs, + /// if one of pair is coalesced to another register, the allocation hint of + /// the other half of the pair should be changed to point to the new register. virtual void updateRegAllocHint(unsigned Reg, unsigned NewReg, MachineFunction &MF) const { // Do nothing. @@ -738,73 +752,72 @@ public: /// register if it is available. virtual unsigned getCSRFirstUseCost() const { return 0; } - /// requiresRegisterScavenging - returns true if the target requires (and can - /// make use of) the register scavenger. + /// Returns true if the target requires (and can make use of) the register + /// scavenger. virtual bool requiresRegisterScavenging(const MachineFunction &MF) const { return false; } - /// useFPForScavengingIndex - returns true if the target wants to use - /// frame pointer based accesses to spill to the scavenger emergency spill - /// slot. + /// Returns true if the target wants to use frame pointer based accesses to + /// spill to the scavenger emergency spill slot. virtual bool useFPForScavengingIndex(const MachineFunction &MF) const { return true; } - /// requiresFrameIndexScavenging - returns true if the target requires post - /// PEI scavenging of registers for materializing frame index constants. + /// Returns true if the target requires post PEI scavenging of registers for + /// materializing frame index constants. virtual bool requiresFrameIndexScavenging(const MachineFunction &MF) const { return false; } - /// requiresVirtualBaseRegisters - Returns true if the target wants the - /// LocalStackAllocation pass to be run and virtual base registers - /// used for more efficient stack access. + /// Returns true if the target wants the LocalStackAllocation pass to be run + /// and virtual base registers used for more efficient stack access. virtual bool requiresVirtualBaseRegisters(const MachineFunction &MF) const { return false; } - /// hasReservedSpillSlot - Return true if target has reserved a spill slot in - /// the stack frame of the given function for the specified register. e.g. On - /// x86, if the frame register is required, the first fixed stack object is - /// reserved as its spill slot. This tells PEI not to create a new stack frame + /// Return true if target has reserved a spill slot in the stack frame of + /// the given function for the specified register. e.g. On x86, if the frame + /// register is required, the first fixed stack object is reserved as its + /// spill slot. This tells PEI not to create a new stack frame /// object for the given register. It should be called only after - /// processFunctionBeforeCalleeSavedScan(). + /// determineCalleeSaves(). virtual bool hasReservedSpillSlot(const MachineFunction &MF, unsigned Reg, int &FrameIdx) const { return false; } - /// trackLivenessAfterRegAlloc - returns true if the live-ins should be tracked - /// after register allocation. + /// Returns true if the live-ins should be tracked after register allocation. virtual bool trackLivenessAfterRegAlloc(const MachineFunction &MF) const { return false; } - /// needsStackRealignment - true if storage within the function requires the - /// stack pointer to be aligned more than the normal calling convention calls - /// for. - virtual bool needsStackRealignment(const MachineFunction &MF) const { - return false; - } + /// True if the stack can be realigned for the target. + virtual bool canRealignStack(const MachineFunction &MF) const; + + /// True if storage within the function requires the stack pointer to be + /// aligned more than the normal calling convention calls for. + /// This cannot be overriden by the target, but canRealignStack can be + /// overridden. + bool needsStackRealignment(const MachineFunction &MF) const; - /// getFrameIndexInstrOffset - Get the offset from the referenced frame - /// index in the instruction, if there is one. + /// Get the offset from the referenced frame index in the instruction, + /// if there is one. virtual int64_t getFrameIndexInstrOffset(const MachineInstr *MI, int Idx) const { return 0; } - /// needsFrameBaseReg - Returns true if the instruction's frame index - /// reference would be better served by a base register other than FP - /// or SP. Used by LocalStackFrameAllocation to determine which frame index + /// Returns true if the instruction's frame index reference would be better + /// served by a base register other than FP or SP. + /// Used by LocalStackFrameAllocation to determine which frame index /// references it should create new base registers for. virtual bool needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { return false; } - /// materializeFrameBaseRegister - Insert defining instruction(s) for - /// BaseReg to be a pointer to FrameIdx before insertion point I. + /// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx + /// before insertion point I. virtual void materializeFrameBaseRegister(MachineBasicBlock *MBB, unsigned BaseReg, int FrameIdx, int64_t Offset) const { @@ -812,24 +825,23 @@ public: "target"); } - /// resolveFrameIndex - Resolve a frame index operand of an instruction + /// Resolve a frame index operand of an instruction /// to reference the indicated base register plus offset instead. virtual void resolveFrameIndex(MachineInstr &MI, unsigned BaseReg, int64_t Offset) const { llvm_unreachable("resolveFrameIndex does not exist on this target"); } - /// isFrameOffsetLegal - Determine whether a given base register plus offset - /// immediate is encodable to resolve a frame index. + /// Determine whether a given base register plus offset immediate is + /// encodable to resolve a frame index. virtual bool isFrameOffsetLegal(const MachineInstr *MI, unsigned BaseReg, int64_t Offset) const { llvm_unreachable("isFrameOffsetLegal does not exist on this target"); } - - /// saveScavengerRegister - Spill the register so it can be used by the - /// register scavenger. Return true if the register was spilled, false - /// otherwise. If this function does not spill the register, the scavenger + /// Spill the register so it can be used by the register scavenger. + /// Return true if the register was spilled, false otherwise. + /// If this function does not spill the register, the scavenger /// will instead spill it to the emergency spill slot. /// virtual bool saveScavengerRegister(MachineBasicBlock &MBB, @@ -840,13 +852,13 @@ public: return false; } - /// eliminateFrameIndex - This method must be overriden to eliminate abstract - /// frame indices from instructions which may use them. The instruction - /// referenced by the iterator contains an MO_FrameIndex operand which must be - /// eliminated by this method. This method may modify or replace the - /// specified instruction, as long as it keeps the iterator pointing at the - /// finished product. SPAdj is the SP adjustment due to call frame setup - /// instruction. FIOperandNum is the FI operand number. + /// This method must be overriden to eliminate abstract frame indices from + /// instructions which may use them. The instruction referenced by the + /// iterator contains an MO_FrameIndex operand which must be eliminated by + /// this method. This method may modify or replace the specified instruction, + /// as long as it keeps the iterator pointing at the finished product. + /// SPAdj is the SP adjustment due to call frame setup instruction. + /// FIOperandNum is the FI operand number. virtual void eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj, unsigned FIOperandNum, RegScavenger *RS = nullptr) const = 0; @@ -935,7 +947,6 @@ struct VirtReg2IndexFunctor : public std::unary_function<unsigned, unsigned> { } }; -/// PrintReg - Helper class for printing registers on a raw_ostream. /// Prints virtual and physical registers with or without a TRI instance. /// /// The format is: @@ -946,24 +957,10 @@ struct VirtReg2IndexFunctor : public std::unary_function<unsigned, unsigned> { /// %physreg17 - a physical register when no TRI instance given. /// /// Usage: OS << PrintReg(Reg, TRI) << '\n'; -/// -class PrintReg { - const TargetRegisterInfo *TRI; - unsigned Reg; - unsigned SubIdx; -public: - explicit PrintReg(unsigned reg, const TargetRegisterInfo *tri = nullptr, - unsigned subidx = 0) - : TRI(tri), Reg(reg), SubIdx(subidx) {} - void print(raw_ostream&) const; -}; - -static inline raw_ostream &operator<<(raw_ostream &OS, const PrintReg &PR) { - PR.print(OS); - return OS; -} +Printable PrintReg(unsigned Reg, const TargetRegisterInfo *TRI = nullptr, + unsigned SubRegIdx = 0); -/// PrintRegUnit - Helper class for printing register units on a raw_ostream. +/// Create Printable object to print register units on a \ref raw_ostream. /// /// Register units are named after their root registers: /// @@ -971,36 +968,14 @@ static inline raw_ostream &operator<<(raw_ostream &OS, const PrintReg &PR) { /// FP0~ST7 - Dual roots. /// /// Usage: OS << PrintRegUnit(Unit, TRI) << '\n'; -/// -class PrintRegUnit { -protected: - const TargetRegisterInfo *TRI; - unsigned Unit; -public: - PrintRegUnit(unsigned unit, const TargetRegisterInfo *tri) - : TRI(tri), Unit(unit) {} - void print(raw_ostream&) const; -}; - -static inline raw_ostream &operator<<(raw_ostream &OS, const PrintRegUnit &PR) { - PR.print(OS); - return OS; -} +Printable PrintRegUnit(unsigned Unit, const TargetRegisterInfo *TRI); -/// PrintVRegOrUnit - It is often convenient to track virtual registers and -/// physical register units in the same list. -class PrintVRegOrUnit : protected PrintRegUnit { -public: - PrintVRegOrUnit(unsigned VRegOrUnit, const TargetRegisterInfo *tri) - : PrintRegUnit(VRegOrUnit, tri) {} - void print(raw_ostream&) const; -}; +/// \brief Create Printable object to print virtual registers and physical +/// registers on a \ref raw_ostream. +Printable PrintVRegOrUnit(unsigned VRegOrUnit, const TargetRegisterInfo *TRI); -static inline raw_ostream &operator<<(raw_ostream &OS, - const PrintVRegOrUnit &PR) { - PR.print(OS); - return OS; -} +/// Create Printable object to print LaneBitmasks on a \ref raw_ostream. +Printable PrintLaneMask(LaneBitmask LaneMask); } // End llvm namespace diff --git a/include/llvm/Target/TargetSelectionDAG.td b/include/llvm/Target/TargetSelectionDAG.td index 4abbe3793995..565473658404 100644 --- a/include/llvm/Target/TargetSelectionDAG.td +++ b/include/llvm/Target/TargetSelectionDAG.td @@ -80,6 +80,11 @@ class SDTCisSameNumEltsAs<int OpNum, int OtherOp> : SDTypeConstraint<OpNum> { int OtherOperandNum = OtherOp; } +// SDTCisSameSizeAs - The two specified operands have identical size. +class SDTCisSameSizeAs<int OpNum, int OtherOp> : SDTypeConstraint<OpNum> { + int OtherOperandNum = OtherOp; +} + //===----------------------------------------------------------------------===// // Selection DAG Type Profile definitions. // @@ -186,6 +191,10 @@ def SDTBrind : SDTypeProfile<0, 1, [ // brind SDTCisPtrTy<0> ]>; +def SDTCatchret : SDTypeProfile<0, 2, [ // catchret + SDTCisVT<0, OtherVT>, SDTCisVT<1, OtherVT> +]>; + def SDTNone : SDTypeProfile<0, 0, []>; // ret, trap def SDTLoad : SDTypeProfile<1, 1, [ // load @@ -201,11 +210,12 @@ def SDTIStore : SDTypeProfile<1, 3, [ // indexed store ]>; def SDTMaskedStore: SDTypeProfile<0, 3, [ // masked store - SDTCisPtrTy<0>, SDTCisVec<1>, SDTCisVec<2> + SDTCisPtrTy<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisSameNumEltsAs<1, 2> ]>; def SDTMaskedLoad: SDTypeProfile<1, 3, [ // masked load - SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisVec<2>, SDTCisSameAs<0, 3> + SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisVec<2>, SDTCisSameAs<0, 3>, + SDTCisSameNumEltsAs<0, 2> ]>; def SDTMaskedGather: SDTypeProfile<2, 3, [ // masked gather @@ -387,6 +397,7 @@ def umin : SDNode<"ISD::UMIN" , SDTIntBinOp>; def umax : SDNode<"ISD::UMAX" , SDTIntBinOp>; def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>; +def bitreverse : SDNode<"ISD::BITREVERSE" , SDTIntUnaryOp>; def bswap : SDNode<"ISD::BSWAP" , SDTIntUnaryOp>; def ctlz : SDNode<"ISD::CTLZ" , SDTIntUnaryOp>; def cttz : SDNode<"ISD::CTTZ" , SDTIntUnaryOp>; @@ -412,6 +423,8 @@ def fmad : SDNode<"ISD::FMAD" , SDTFPTernaryOp>; def fabs : SDNode<"ISD::FABS" , SDTFPUnaryOp>; def fminnum : SDNode<"ISD::FMINNUM" , SDTFPBinOp>; def fmaxnum : SDNode<"ISD::FMAXNUM" , SDTFPBinOp>; +def fminnan : SDNode<"ISD::FMINNAN" , SDTFPBinOp>; +def fmaxnan : SDNode<"ISD::FMAXNAN" , SDTFPBinOp>; def fgetsign : SDNode<"ISD::FGETSIGN" , SDTFPToIntOp>; def fneg : SDNode<"ISD::FNEG" , SDTFPUnaryOp>; def fsqrt : SDNode<"ISD::FSQRT" , SDTFPUnaryOp>; @@ -447,6 +460,12 @@ def brcc : SDNode<"ISD::BR_CC" , SDTBrCC, [SDNPHasChain]>; def brcond : SDNode<"ISD::BRCOND" , SDTBrcond, [SDNPHasChain]>; def brind : SDNode<"ISD::BRIND" , SDTBrind, [SDNPHasChain]>; def br : SDNode<"ISD::BR" , SDTBr, [SDNPHasChain]>; +def catchret : SDNode<"ISD::CATCHRET" , SDTCatchret, + [SDNPHasChain, SDNPSideEffect]>; +def cleanupret : SDNode<"ISD::CLEANUPRET" , SDTNone, [SDNPHasChain]>; +def catchpad : SDNode<"ISD::CATCHPAD" , SDTNone, + [SDNPHasChain, SDNPSideEffect]>; + def trap : SDNode<"ISD::TRAP" , SDTNone, [SDNPHasChain, SDNPSideEffect]>; def debugtrap : SDNode<"ISD::DEBUGTRAP" , SDTNone, @@ -513,6 +532,9 @@ def vector_shuffle : SDNode<"ISD::VECTOR_SHUFFLE", SDTVecShuffle, []>; def build_vector : SDNode<"ISD::BUILD_VECTOR", SDTypeProfile<1, -1, []>, []>; def scalar_to_vector : SDNode<"ISD::SCALAR_TO_VECTOR", SDTypeProfile<1, 1, []>, []>; + +// vector_extract/vector_insert are deprecated. extractelt/insertelt +// are preferred. def vector_extract : SDNode<"ISD::EXTRACT_VECTOR_ELT", SDTypeProfile<1, 2, [SDTCisPtrTy<2>]>, []>; def vector_insert : SDNode<"ISD::INSERT_VECTOR_ELT", @@ -523,7 +545,7 @@ def concat_vectors : SDNode<"ISD::CONCAT_VECTORS", // This operator does not do subvector type checking. The ARM // backend, at least, needs it. def vector_extract_subvec : SDNode<"ISD::EXTRACT_SUBVECTOR", - SDTypeProfile<1, 2, [SDTCisInt<2>, SDTCisVec<1>, SDTCisVec<0>]>, + SDTypeProfile<1, 2, [SDTCisInt<2>, SDTCisVec<1>, SDTCisVec<0>]>, []>; // This operator does subvector type checking. @@ -815,6 +837,21 @@ def truncstoref64 : PatFrag<(ops node:$val, node:$ptr), return cast<StoreSDNode>(N)->getMemoryVT() == MVT::f64; }]>; +def truncstorevi8 : PatFrag<(ops node:$val, node:$ptr), + (truncstore node:$val, node:$ptr), [{ + return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8; +}]>; + +def truncstorevi16 : PatFrag<(ops node:$val, node:$ptr), + (truncstore node:$val, node:$ptr), [{ + return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16; +}]>; + +def truncstorevi32 : PatFrag<(ops node:$val, node:$ptr), + (truncstore node:$val, node:$ptr), [{ + return cast<StoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32; +}]>; + // indexed store fragments. def istore : PatFrag<(ops node:$val, node:$base, node:$offset), (ist node:$val, node:$base, node:$offset), [{ @@ -889,6 +926,24 @@ def post_truncstf32 : PatFrag<(ops node:$val, node:$base, node:$offset), return cast<StoreSDNode>(N)->getMemoryVT() == MVT::f32; }]>; +// nontemporal store fragments. +def nontemporalstore : PatFrag<(ops node:$val, node:$ptr), + (store node:$val, node:$ptr), [{ + return cast<StoreSDNode>(N)->isNonTemporal(); +}]>; + +def alignednontemporalstore : PatFrag<(ops node:$val, node:$ptr), + (nontemporalstore node:$val, node:$ptr), [{ + StoreSDNode *St = cast<StoreSDNode>(N); + return St->getAlignment() >= St->getMemoryVT().getStoreSize(); +}]>; + +def unalignednontemporalstore : PatFrag<(ops node:$val, node:$ptr), + (nontemporalstore node:$val, node:$ptr), [{ + StoreSDNode *St = cast<StoreSDNode>(N); + return St->getAlignment() < St->getMemoryVT().getStoreSize(); +}]>; + // setcc convenience fragments. def setoeq : PatFrag<(ops node:$lhs, node:$rhs), (setcc node:$lhs, node:$rhs, SETOEQ)>; diff --git a/include/llvm/Target/TargetSelectionDAGInfo.h b/include/llvm/Target/TargetSelectionDAGInfo.h index 53db5aa84292..a7143ac3fa66 100644 --- a/include/llvm/Target/TargetSelectionDAGInfo.h +++ b/include/llvm/Target/TargetSelectionDAGInfo.h @@ -21,7 +21,7 @@ namespace llvm { //===----------------------------------------------------------------------===// -/// TargetSelectionDAGInfo - Targets can subclass this to parameterize the +/// Targets can subclass this to parameterize the /// SelectionDAG lowering and instruction selection process. /// class TargetSelectionDAGInfo { @@ -32,8 +32,8 @@ public: explicit TargetSelectionDAGInfo() = default; virtual ~TargetSelectionDAGInfo(); - /// EmitTargetCodeForMemcpy - Emit target-specific code that performs a - /// memcpy. This can be used by targets to provide code sequences for cases + /// Emit target-specific code that performs a memcpy. + /// This can be used by targets to provide code sequences for cases /// that don't fit the target's parameters for simple loads/stores and can be /// more efficient than using a library call. This function can return a null /// SDValue if the target declines to use custom code and a different @@ -56,8 +56,8 @@ public: return SDValue(); } - /// EmitTargetCodeForMemmove - Emit target-specific code that performs a - /// memmove. This can be used by targets to provide code sequences for cases + /// Emit target-specific code that performs a memmove. + /// This can be used by targets to provide code sequences for cases /// that don't fit the target's parameters for simple loads/stores and can be /// more efficient than using a library call. This function can return a null /// SDValue if the target declines to use custom code and a different @@ -72,8 +72,8 @@ public: return SDValue(); } - /// EmitTargetCodeForMemset - Emit target-specific code that performs a - /// memset. This can be used by targets to provide code sequences for cases + /// Emit target-specific code that performs a memset. + /// This can be used by targets to provide code sequences for cases /// that don't fit the target's parameters for simple stores and can be more /// efficient than using a library call. This function can return a null /// SDValue if the target declines to use custom code and a different @@ -87,11 +87,10 @@ public: return SDValue(); } - /// EmitTargetCodeForMemcmp - Emit target-specific code that performs a - /// memcmp, in cases where that is faster than a libcall. The first - /// returned SDValue is the result of the memcmp and the second is - /// the chain. Both SDValues can be null if a normal libcall should - /// be used. + /// Emit target-specific code that performs a memcmp, in cases where that is + /// faster than a libcall. The first returned SDValue is the result of the + /// memcmp and the second is the chain. Both SDValues can be null if a normal + /// libcall should be used. virtual std::pair<SDValue, SDValue> EmitTargetCodeForMemcmp(SelectionDAG &DAG, SDLoc dl, SDValue Chain, @@ -101,11 +100,10 @@ public: return std::make_pair(SDValue(), SDValue()); } - /// EmitTargetCodeForMemchr - Emit target-specific code that performs a - /// memchr, in cases where that is faster than a libcall. The first - /// returned SDValue is the result of the memchr and the second is - /// the chain. Both SDValues can be null if a normal libcall should - /// be used. + /// Emit target-specific code that performs a memchr, in cases where that is + /// faster than a libcall. The first returned SDValue is the result of the + /// memchr and the second is the chain. Both SDValues can be null if a normal + /// libcall should be used. virtual std::pair<SDValue, SDValue> EmitTargetCodeForMemchr(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, @@ -113,8 +111,8 @@ public: return std::make_pair(SDValue(), SDValue()); } - /// EmitTargetCodeForStrcpy - Emit target-specific code that performs a - /// strcpy or stpcpy, in cases where that is faster than a libcall. + /// Emit target-specific code that performs a strcpy or stpcpy, in cases + /// where that is faster than a libcall. /// The first returned SDValue is the result of the copy (the start /// of the destination string for strcpy, a pointer to the null terminator /// for stpcpy) and the second is the chain. Both SDValues can be null @@ -128,11 +126,10 @@ public: return std::make_pair(SDValue(), SDValue()); } - /// EmitTargetCodeForStrcmp - Emit target-specific code that performs a - /// strcmp, in cases where that is faster than a libcall. The first - /// returned SDValue is the result of the strcmp and the second is - /// the chain. Both SDValues can be null if a normal libcall should - /// be used. + /// Emit target-specific code that performs a strcmp, in cases where that is + /// faster than a libcall. + /// The first returned SDValue is the result of the strcmp and the second is + /// the chain. Both SDValues can be null if a normal libcall should be used. virtual std::pair<SDValue, SDValue> EmitTargetCodeForStrcmp(SelectionDAG &DAG, SDLoc dl, SDValue Chain, diff --git a/include/llvm/Target/TargetSubtargetInfo.h b/include/llvm/Target/TargetSubtargetInfo.h index 07c0c66bfa18..d50aa4932f8f 100644 --- a/include/llvm/Target/TargetSubtargetInfo.h +++ b/include/llvm/Target/TargetSubtargetInfo.h @@ -15,6 +15,7 @@ #define LLVM_TARGET_TARGETSUBTARGETINFO_H #include "llvm/CodeGen/PBQPRAConstraint.h" +#include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/Support/CodeGen.h" @@ -81,6 +82,11 @@ public: virtual const TargetSelectionDAGInfo *getSelectionDAGInfo() const { return nullptr; } + /// Target can subclass this hook to select a different DAG scheduler. + virtual RegisterScheduler::FunctionPassCtor + getDAGScheduler(CodeGenOpt::Level) const { + return nullptr; + } /// getRegisterInfo - If register information is available, return it. If /// not, return null. This is kept separate from RegInfo until RegInfo has diff --git a/include/llvm/Transforms/IPO.h b/include/llvm/Transforms/IPO.h index 2ea47301bb4c..0c374a070ce8 100644 --- a/include/llvm/Transforms/IPO.h +++ b/include/llvm/Transforms/IPO.h @@ -16,9 +16,11 @@ #define LLVM_TRANSFORMS_IPO_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" namespace llvm { +class FunctionInfoIndex; class ModulePass; class Pass; class Function; @@ -85,6 +87,10 @@ ModulePass *createGVExtractionPass(std::vector<GlobalValue*>& GVs, bool deleteFn = false); //===----------------------------------------------------------------------===// +/// This pass performs iterative function importing from other modules. +Pass *createFunctionImportPass(const FunctionInfoIndex *Index = nullptr); + +//===----------------------------------------------------------------------===// /// createFunctionInliningPass - Return a new pass object that uses a heuristic /// to inline direct function calls to small functions. /// @@ -209,6 +215,15 @@ ModulePass *createBarrierNoopPass(); /// to bitsets. ModulePass *createLowerBitSetsPass(); +/// \brief This pass export CFI checks for use by external modules. +ModulePass *createCrossDSOCFIPass(); + +//===----------------------------------------------------------------------===// +// SampleProfilePass - Loads sample profile data from disk and generates +// IR metadata to reflect the profile. +ModulePass *createSampleProfileLoaderPass(); +ModulePass *createSampleProfileLoaderPass(StringRef Name); + } // End llvm namespace #endif diff --git a/include/llvm/Transforms/IPO/ForceFunctionAttrs.h b/include/llvm/Transforms/IPO/ForceFunctionAttrs.h new file mode 100644 index 000000000000..0ff4afe79b0c --- /dev/null +++ b/include/llvm/Transforms/IPO/ForceFunctionAttrs.h @@ -0,0 +1,35 @@ +//===-- ForceFunctionAttrs.h - Force function attrs for debugging ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// Super simple passes to force specific function attrs from the commandline +/// into the IR for debugging purposes. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_FORCEFUNCTIONATTRS_H +#define LLVM_TRANSFORMS_IPO_FORCEFUNCTIONATTRS_H + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// Pass which forces specific function attributes into the IR, primarily as +/// a debugging tool. +class ForceFunctionAttrsPass { +public: + static StringRef name() { return "ForceFunctionAttrsPass"; } + PreservedAnalyses run(Module &M); +}; + +/// Create a legacy pass manager instance of a pass to force function attrs. +Pass *createForceFunctionAttrsLegacyPass(); + +} + +#endif // LLVM_TRANSFORMS_IPO_FORCEFUNCTIONATTRS_H diff --git a/include/llvm/Transforms/IPO/FunctionImport.h b/include/llvm/Transforms/IPO/FunctionImport.h new file mode 100644 index 000000000000..d7707790a017 --- /dev/null +++ b/include/llvm/Transforms/IPO/FunctionImport.h @@ -0,0 +1,43 @@ +//===- llvm/Transforms/IPO/FunctionImport.h - ThinLTO importing -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUNCTIONIMPORT_H +#define LLVM_FUNCTIONIMPORT_H + +#include "llvm/ADT/StringMap.h" +#include <functional> + +namespace llvm { +class LLVMContext; +class Module; +class FunctionInfoIndex; + +/// The function importer is automatically importing function from other modules +/// based on the provided summary informations. +class FunctionImporter { + + /// The summaries index used to trigger importing. + const FunctionInfoIndex &Index; + + /// Factory function to load a Module for a given identifier + std::function<std::unique_ptr<Module>(StringRef Identifier)> ModuleLoader; + +public: + /// Create a Function Importer. + FunctionImporter( + const FunctionInfoIndex &Index, + std::function<std::unique_ptr<Module>(StringRef Identifier)> ModuleLoader) + : Index(Index), ModuleLoader(ModuleLoader) {} + + /// Import functions in Module \p M based on the summary informations. + bool importFunctions(Module &M); +}; +} + +#endif // LLVM_FUNCTIONIMPORT_H diff --git a/include/llvm/Transforms/IPO/InferFunctionAttrs.h b/include/llvm/Transforms/IPO/InferFunctionAttrs.h new file mode 100644 index 000000000000..80afc02c62ae --- /dev/null +++ b/include/llvm/Transforms/IPO/InferFunctionAttrs.h @@ -0,0 +1,38 @@ +//===-- InferFunctionAttrs.h - Infer implicit function attributes ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Interfaces for passes which infer implicit function attributes from the +/// name and signature of function declarations. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_INFERFUNCTIONATTRS_H +#define LLVM_TRANSFORMS_IPO_INFERFUNCTIONATTRS_H + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// A pass which infers function attributes from the names and signatures of +/// function declarations in a module. +class InferFunctionAttrsPass { +public: + static StringRef name() { return "InferFunctionAttrsPass"; } + PreservedAnalyses run(Module &M, AnalysisManager<Module> *AM); +}; + +/// Create a legacy pass manager instance of a pass to infer function +/// attributes. +Pass *createInferFunctionAttrsLegacyPass(); + +} + +#endif // LLVM_TRANSFORMS_IPO_INFERFUNCTIONATTRS_H diff --git a/include/llvm/Transforms/IPO/InlinerPass.h b/include/llvm/Transforms/IPO/InlinerPass.h index 6a644ad4a63b..58ef0cbbfb5d 100644 --- a/include/llvm/Transforms/IPO/InlinerPass.h +++ b/include/llvm/Transforms/IPO/InlinerPass.h @@ -20,11 +20,11 @@ #include "llvm/Analysis/CallGraphSCCPass.h" namespace llvm { - class CallSite; - class DataLayout; - class InlineCost; - template<class PtrType, unsigned SmallSize> - class SmallPtrSet; +class AssumptionCacheTracker; +class CallSite; +class DataLayout; +class InlineCost; +template <class PtrType, unsigned SmallSize> class SmallPtrSet; /// Inliner - This class contains all of the helper code which is used to /// perform the inlining operations that do not depend on the policy. @@ -84,6 +84,9 @@ private: /// shouldInline - Return true if the inliner should attempt to /// inline at the given CallSite. bool shouldInline(CallSite CS); + +protected: + AssumptionCacheTracker *ACT; }; } // End llvm namespace diff --git a/include/llvm/Transforms/IPO/LowerBitSets.h b/include/llvm/Transforms/IPO/LowerBitSets.h index 55d7d84560a0..e5fb7b98fcb3 100644 --- a/include/llvm/Transforms/IPO/LowerBitSets.h +++ b/include/llvm/Transforms/IPO/LowerBitSets.h @@ -26,8 +26,9 @@ namespace llvm { class DataLayout; -class GlobalVariable; +class GlobalObject; class Value; +class raw_ostream; struct BitSetInfo { // The indices of the set bits in the bitset. @@ -55,8 +56,10 @@ struct BitSetInfo { bool containsGlobalOffset(uint64_t Offset) const; bool containsValue(const DataLayout &DL, - const DenseMap<GlobalVariable *, uint64_t> &GlobalLayout, + const DenseMap<GlobalObject *, uint64_t> &GlobalLayout, Value *V, uint64_t COffset = 0) const; + + void print(raw_ostream &OS) const; }; struct BitSetBuilder { diff --git a/include/llvm/Transforms/IPO/PassManagerBuilder.h b/include/llvm/Transforms/IPO/PassManagerBuilder.h index 1334dd0da23c..a4e7bce8ef4a 100644 --- a/include/llvm/Transforms/IPO/PassManagerBuilder.h +++ b/include/llvm/Transforms/IPO/PassManagerBuilder.h @@ -15,9 +15,11 @@ #ifndef LLVM_TRANSFORMS_IPO_PASSMANAGERBUILDER_H #define LLVM_TRANSFORMS_IPO_PASSMANAGERBUILDER_H +#include <memory> #include <vector> namespace llvm { +class FunctionInfoIndex; class Pass; class TargetLibraryInfoImpl; class TargetMachine; @@ -81,6 +83,11 @@ public: /// run after everything else. EP_OptimizerLast, + /// EP_VectorizerStart - This extension point allows adding optimization + /// passes before the vectorizer and other highly target specific + /// optimization passes are executed. + EP_VectorizerStart, + /// EP_EnabledOnOptLevel0 - This extension point allows adding passes that /// should not be disabled by O0 optimization level. The passes will be /// inserted after the inlining pass. @@ -109,6 +116,9 @@ public: /// added to the per-module passes. Pass *Inliner; + /// The function summary index to use for function importing. + const FunctionInfoIndex *FunctionIndex; + bool DisableTailCalls; bool DisableUnitAtATime; bool DisableUnrollLoops; diff --git a/include/llvm/Transforms/IPO/StripDeadPrototypes.h b/include/llvm/Transforms/IPO/StripDeadPrototypes.h new file mode 100644 index 000000000000..9dddd12871c4 --- /dev/null +++ b/include/llvm/Transforms/IPO/StripDeadPrototypes.h @@ -0,0 +1,34 @@ +//===-- StripDeadPrototypes.h - Remove unused function declarations -------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass loops over all of the functions in the input module, looking for +// dead declarations and removes them. Dead declarations are declarations of +// functions for which no implementation is available (i.e., declarations for +// unused library functions). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_STRIPDEADPROTOTYPES_H +#define LLVM_TRANSFORMS_IPO_STRIPDEADPROTOTYPES_H + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// Pass to remove unused function declarations. +class StripDeadPrototypesPass { +public: + static StringRef name() { return "StripDeadPrototypesPass"; } + PreservedAnalyses run(Module &M); +}; + +} + +#endif // LLVM_TRANSFORMS_IPO_STRIPDEADPROTOTYPES_H diff --git a/include/llvm/Transforms/InstCombine/InstCombineWorklist.h b/include/llvm/Transforms/InstCombine/InstCombineWorklist.h index a6bad343db43..5d2b2d000009 100644 --- a/include/llvm/Transforms/InstCombine/InstCombineWorklist.h +++ b/include/llvm/Transforms/InstCombine/InstCombineWorklist.h @@ -60,13 +60,13 @@ public: /// AddInitialGroup - Add the specified batch of stuff in reverse order. /// which should only be done when the worklist is empty and when the group /// has no duplicates. - void AddInitialGroup(Instruction *const *List, unsigned NumEntries) { + void AddInitialGroup(ArrayRef<Instruction *> List) { assert(Worklist.empty() && "Worklist must be empty to add initial group"); - Worklist.reserve(NumEntries+16); - WorklistMap.resize(NumEntries); - DEBUG(dbgs() << "IC: ADDING: " << NumEntries << " instrs to worklist\n"); - for (unsigned Idx = 0; NumEntries; --NumEntries) { - Instruction *I = List[NumEntries-1]; + Worklist.reserve(List.size()+16); + WorklistMap.resize(List.size()); + DEBUG(dbgs() << "IC: ADDING: " << List.size() << " instrs to worklist\n"); + unsigned Idx = 0; + for (Instruction *I : reverse(List)) { WorklistMap.insert(std::make_pair(I, Idx++)); Worklist.push_back(I); } diff --git a/include/llvm/Transforms/Instrumentation.h b/include/llvm/Transforms/Instrumentation.h index 250e3893cb15..38dfeb04ace3 100644 --- a/include/llvm/Transforms/Instrumentation.h +++ b/include/llvm/Transforms/Instrumentation.h @@ -15,6 +15,7 @@ #define LLVM_TRANSFORMS_INSTRUMENTATION_H #include "llvm/ADT/StringRef.h" +#include "llvm/IR/BasicBlock.h" #include <vector> #if defined(__GNUC__) && defined(__linux__) && !defined(ANDROID) @@ -33,6 +34,16 @@ inline void *getDFSanRetValTLSPtrForJIT() { namespace llvm { +class TargetMachine; + +/// Instrumentation passes often insert conditional checks into entry blocks. +/// Call this function before splitting the entry block to move instructions +/// that must remain in the entry block up before the split point. Static +/// allocas and llvm.localescape calls, for example, must remain in the entry +/// block. +BasicBlock::iterator PrepareToSplitEntryBlock(BasicBlock &BB, + BasicBlock::iterator IP); + class ModulePass; class FunctionPass; @@ -68,6 +79,11 @@ struct GCOVOptions { ModulePass *createGCOVProfilerPass(const GCOVOptions &Options = GCOVOptions::getDefault()); +// PGO Instrumention +ModulePass *createPGOInstrumentationGenPass(); +ModulePass * +createPGOInstrumentationUsePass(StringRef Filename = StringRef("")); + /// Options for the frontend instrumentation based profiling pass. struct InstrProfOptions { InstrProfOptions() : NoRedZone(false) {} @@ -84,8 +100,10 @@ ModulePass *createInstrProfilingPass( const InstrProfOptions &Options = InstrProfOptions()); // Insert AddressSanitizer (address sanity checking) instrumentation -FunctionPass *createAddressSanitizerFunctionPass(bool CompileKernel = false); -ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false); +FunctionPass *createAddressSanitizerFunctionPass(bool CompileKernel = false, + bool Recover = false); +ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false, + bool Recover = false); // Insert MemorySanitizer instrumentation (detection of uninitialized reads) FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0); @@ -134,7 +152,25 @@ FunctionPass *createBoundsCheckingPass(); /// \brief This pass splits the stack into a safe stack and an unsafe stack to /// protect against stack-based overflow vulnerabilities. -FunctionPass *createSafeStackPass(); +FunctionPass *createSafeStackPass(const TargetMachine *TM = nullptr); + +/// \brief Calculate what to divide by to scale counts. +/// +/// Given the maximum count, calculate a divisor that will scale all the +/// weights to strictly less than UINT32_MAX. +static inline uint64_t calculateCountScale(uint64_t MaxCount) { + return MaxCount < UINT32_MAX ? 1 : MaxCount / UINT32_MAX + 1; +} + +/// \brief Scale an individual branch count. +/// +/// Scale a 64-bit weight down to 32-bits using \c Scale. +/// +static inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) { + uint64_t Scaled = Count / Scale; + assert(Scaled <= UINT32_MAX && "overflow 32-bits"); + return Scaled; +} } // End llvm namespace diff --git a/include/llvm/Transforms/Scalar.h b/include/llvm/Transforms/Scalar.h index 4676c95d7cd4..9173de1112f3 100644 --- a/include/llvm/Transforms/Scalar.h +++ b/include/llvm/Transforms/Scalar.h @@ -93,7 +93,7 @@ FunctionPass *createBitTrackingDCEPass(); // // SROA - Replace aggregates or pieces of aggregates with scalar SSA values. // -FunctionPass *createSROAPass(bool RequiresDomTree = true); +FunctionPass *createSROAPass(); //===----------------------------------------------------------------------===// // @@ -161,7 +161,8 @@ Pass *createLoopStrengthReducePass(); // It can also be configured to focus on size optimizations only. // Pass *createGlobalMergePass(const TargetMachine *TM, unsigned MaximalOffset, - bool OnlyOptimizeForSize = false); + bool OnlyOptimizeForSize = false, + bool MergeExternalByDefault = false); //===----------------------------------------------------------------------===// // @@ -407,13 +408,6 @@ FunctionPass *createPartiallyInlineLibCallsPass(); //===----------------------------------------------------------------------===// // -// SampleProfilePass - Loads sample profile data from disk and generates -// IR metadata to reflect the profile. -FunctionPass *createSampleProfileLoaderPass(); -FunctionPass *createSampleProfileLoaderPass(StringRef Name); - -//===----------------------------------------------------------------------===// -// // ScalarizerPass - Converts vector operations into scalar operations // FunctionPass *createScalarizerPass(); @@ -486,6 +480,12 @@ FunctionPass *createNaryReassociatePass(); // FunctionPass *createLoopDistributePass(); +//===----------------------------------------------------------------------===// +// +// LoopLoadElimination - Perform loop-aware load elimination. +// +FunctionPass *createLoopLoadEliminationPass(); + } // End llvm namespace #endif diff --git a/include/llvm/Transforms/Scalar/ADCE.h b/include/llvm/Transforms/Scalar/ADCE.h new file mode 100644 index 000000000000..f9bc7b77c14a --- /dev/null +++ b/include/llvm/Transforms/Scalar/ADCE.h @@ -0,0 +1,38 @@ +//===- ADCE.h - Aggressive dead code elimination --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provides the interface for the Aggressive Dead Code Elimination +// pass. This pass optimistically assumes that all instructions are dead until +// proven otherwise, allowing it to eliminate dead computations that other DCE +// passes do not catch, particularly involving loop computations. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_ADCE_H +#define LLVM_TRANSFORMS_SCALAR_ADCE_H + +#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// A DCE pass that assumes instructions are dead until proven otherwise. +/// +/// This pass eliminates dead code by optimistically assuming that all +/// instructions are dead until proven otherwise. This allows it to eliminate +/// dead computations that other DCE passes do not catch, particularly involving +/// loop computations. +class ADCEPass { +public: + static StringRef name() { return "ADCEPass"; } + PreservedAnalyses run(Function &F); +}; +} + +#endif // LLVM_TRANSFORMS_SCALAR_ADCE_H diff --git a/include/llvm/Transforms/Scalar/SROA.h b/include/llvm/Transforms/Scalar/SROA.h new file mode 100644 index 000000000000..f90cc7b686ba --- /dev/null +++ b/include/llvm/Transforms/Scalar/SROA.h @@ -0,0 +1,129 @@ +//===- SROA.h - Scalar Replacement Of Aggregates ----------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file provides the interface for LLVM's Scalar Replacement of +/// Aggregates pass. This pass provides both aggregate splitting and the +/// primary SSA formation used in the compiler. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_SROA_H +#define LLVM_TRANSFORMS_SCALAR_SROA_H + +#include "llvm/ADT/SetVector.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// A private "module" namespace for types and utilities used by SROA. These +/// are implementation details and should not be used by clients. +namespace sroa { +class AllocaSliceRewriter; +class AllocaSlices; +class Partition; +class SROALegacyPass; +} + +/// \brief An optimization pass providing Scalar Replacement of Aggregates. +/// +/// This pass takes allocations which can be completely analyzed (that is, they +/// don't escape) and tries to turn them into scalar SSA values. There are +/// a few steps to this process. +/// +/// 1) It takes allocations of aggregates and analyzes the ways in which they +/// are used to try to split them into smaller allocations, ideally of +/// a single scalar data type. It will split up memcpy and memset accesses +/// as necessary and try to isolate individual scalar accesses. +/// 2) It will transform accesses into forms which are suitable for SSA value +/// promotion. This can be replacing a memset with a scalar store of an +/// integer value, or it can involve speculating operations on a PHI or +/// select to be a PHI or select of the results. +/// 3) Finally, this will try to detect a pattern of accesses which map cleanly +/// onto insert and extract operations on a vector value, and convert them to +/// this form. By doing so, it will enable promotion of vector aggregates to +/// SSA vector values. +class SROA { + LLVMContext *C; + DominatorTree *DT; + AssumptionCache *AC; + + /// \brief Worklist of alloca instructions to simplify. + /// + /// Each alloca in the function is added to this. Each new alloca formed gets + /// added to it as well to recursively simplify unless that alloca can be + /// directly promoted. Finally, each time we rewrite a use of an alloca other + /// the one being actively rewritten, we add it back onto the list if not + /// already present to ensure it is re-visited. + SetVector<AllocaInst *, SmallVector<AllocaInst *, 16>> Worklist; + + /// \brief A collection of instructions to delete. + /// We try to batch deletions to simplify code and make things a bit more + /// efficient. + SetVector<Instruction *, SmallVector<Instruction *, 8>> DeadInsts; + + /// \brief Post-promotion worklist. + /// + /// Sometimes we discover an alloca which has a high probability of becoming + /// viable for SROA after a round of promotion takes place. In those cases, + /// the alloca is enqueued here for re-processing. + /// + /// Note that we have to be very careful to clear allocas out of this list in + /// the event they are deleted. + SetVector<AllocaInst *, SmallVector<AllocaInst *, 16>> PostPromotionWorklist; + + /// \brief A collection of alloca instructions we can directly promote. + std::vector<AllocaInst *> PromotableAllocas; + + /// \brief A worklist of PHIs to speculate prior to promoting allocas. + /// + /// All of these PHIs have been checked for the safety of speculation and by + /// being speculated will allow promoting allocas currently in the promotable + /// queue. + SetVector<PHINode *, SmallVector<PHINode *, 2>> SpeculatablePHIs; + + /// \brief A worklist of select instructions to speculate prior to promoting + /// allocas. + /// + /// All of these select instructions have been checked for the safety of + /// speculation and by being speculated will allow promoting allocas + /// currently in the promotable queue. + SetVector<SelectInst *, SmallVector<SelectInst *, 2>> SpeculatableSelects; + +public: + SROA() : C(nullptr), DT(nullptr), AC(nullptr) {} + + static StringRef name() { return "SROA"; } + + /// \brief Run the pass over the function. + PreservedAnalyses run(Function &F, AnalysisManager<Function> *AM); + +private: + friend class sroa::AllocaSliceRewriter; + friend class sroa::SROALegacyPass; + + /// Helper used by both the public run method and by the legacy pass. + PreservedAnalyses runImpl(Function &F, DominatorTree &RunDT, + AssumptionCache &RunAC); + + bool presplitLoadsAndStores(AllocaInst &AI, sroa::AllocaSlices &AS); + AllocaInst *rewritePartition(AllocaInst &AI, sroa::AllocaSlices &AS, + sroa::Partition &P); + bool splitAlloca(AllocaInst &AI, sroa::AllocaSlices &AS); + bool runOnAlloca(AllocaInst &AI); + void clobberUse(Use &U); + void deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas); + bool promoteAllocas(Function &F); +}; + +} + +#endif diff --git a/include/llvm/Transforms/Utils/BasicBlockUtils.h b/include/llvm/Transforms/Utils/BasicBlockUtils.h index 9b919b62ee41..13c856dfdc9a 100644 --- a/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -22,7 +22,6 @@ namespace llvm { -class AliasAnalysis; class MemoryDependenceAnalysis; class DominatorTree; class LoopInfo; @@ -40,7 +39,7 @@ void DeleteDeadBlock(BasicBlock *BB); /// any single-entry PHI nodes in it, fold them away. This handles the case /// when all entries to the PHI nodes in a block are guaranteed equal, such as /// when the block has exactly one predecessor. -void FoldSingleEntryPHINodes(BasicBlock *BB, AliasAnalysis *AA = nullptr, +void FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceAnalysis *MemDep = nullptr); /// DeleteDeadPHIs - Examine each PHI in the given block and delete it if it @@ -54,7 +53,6 @@ bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI = nullptr); /// if possible. The return value indicates success or failure. bool MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT = nullptr, LoopInfo *LI = nullptr, - AliasAnalysis *AA = nullptr, MemoryDependenceAnalysis *MemDep = nullptr); // ReplaceInstWithValue - Replace all uses of an instruction (specified by BI) @@ -82,27 +80,15 @@ void ReplaceInstWithInst(Instruction *From, Instruction *To); /// This provides a builder interface for overriding the default options used /// during critical edge splitting. struct CriticalEdgeSplittingOptions { - AliasAnalysis *AA; DominatorTree *DT; LoopInfo *LI; bool MergeIdenticalEdges; bool DontDeleteUselessPHIs; bool PreserveLCSSA; - CriticalEdgeSplittingOptions() - : AA(nullptr), DT(nullptr), LI(nullptr), MergeIdenticalEdges(false), - DontDeleteUselessPHIs(false), PreserveLCSSA(false) {} - - /// \brief Basic case of setting up all the analysis. - CriticalEdgeSplittingOptions(AliasAnalysis *AA, DominatorTree *DT = nullptr, + CriticalEdgeSplittingOptions(DominatorTree *DT = nullptr, LoopInfo *LI = nullptr) - : AA(AA), DT(DT), LI(LI), MergeIdenticalEdges(false), - DontDeleteUselessPHIs(false), PreserveLCSSA(false) {} - - /// \brief A common pattern is to preserve the dominator tree and loop - /// info but not care about AA. - CriticalEdgeSplittingOptions(DominatorTree *DT, LoopInfo *LI) - : AA(nullptr), DT(DT), LI(LI), MergeIdenticalEdges(false), + : DT(DT), LI(LI), MergeIdenticalEdges(false), DontDeleteUselessPHIs(false), PreserveLCSSA(false) {} CriticalEdgeSplittingOptions &setMergeIdenticalEdges() { @@ -214,15 +200,13 @@ BasicBlock *SplitBlock(BasicBlock *Old, Instruction *SplitPt, /// It will have Suffix+".split_lp". See SplitLandingPadPredecessors for more /// details on this case. /// -/// This currently updates the LLVM IR, AliasAnalysis, DominatorTree, -/// DominanceFrontier, LoopInfo, and LCCSA but no other analyses. -/// In particular, it does not preserve LoopSimplify (because it's -/// complicated to handle the case where one of the edges being split -/// is an exit of a loop with other exits). +/// This currently updates the LLVM IR, DominatorTree, LoopInfo, and LCCSA but +/// no other analyses. In particular, it does not preserve LoopSimplify +/// (because it's complicated to handle the case where one of the edges being +/// split is an exit of a loop with other exits). /// BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds, const char *Suffix, - AliasAnalysis *AA = nullptr, DominatorTree *DT = nullptr, LoopInfo *LI = nullptr, bool PreserveLCSSA = false); @@ -234,17 +218,15 @@ BasicBlock *SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds, /// OrigBB is clone into both of the new basic blocks. The new blocks are given /// the suffixes 'Suffix1' and 'Suffix2', and are returned in the NewBBs vector. /// -/// This currently updates the LLVM IR, AliasAnalysis, DominatorTree, -/// DominanceFrontier, LoopInfo, and LCCSA but no other analyses. In particular, -/// it does not preserve LoopSimplify (because it's complicated to handle the -/// case where one of the edges being split is an exit of a loop with other -/// exits). +/// This currently updates the LLVM IR, DominatorTree, LoopInfo, and LCCSA but +/// no other analyses. In particular, it does not preserve LoopSimplify +/// (because it's complicated to handle the case where one of the edges being +/// split is an exit of a loop with other exits). /// void SplitLandingPadPredecessors(BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix, const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs, - AliasAnalysis *AA = nullptr, DominatorTree *DT = nullptr, LoopInfo *LI = nullptr, bool PreserveLCSSA = false); diff --git a/include/llvm/Transforms/Utils/Cloning.h b/include/llvm/Transforms/Utils/Cloning.h index 2caa9a2462df..92a1d52f1011 100644 --- a/include/llvm/Transforms/Utils/Cloning.h +++ b/include/llvm/Transforms/Utils/Cloning.h @@ -20,9 +20,11 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/ValueHandle.h" #include "llvm/IR/ValueMap.h" #include "llvm/Transforms/Utils/ValueMapper.h" +#include <functional> namespace llvm { @@ -43,14 +45,21 @@ class DataLayout; class Loop; class LoopInfo; class AllocaInst; -class AliasAnalysis; class AssumptionCacheTracker; class DominatorTree; -/// CloneModule - Return an exact copy of the specified module +/// Return an exact copy of the specified module /// -Module *CloneModule(const Module *M); -Module *CloneModule(const Module *M, ValueToValueMapTy &VMap); +std::unique_ptr<Module> CloneModule(const Module *M); +std::unique_ptr<Module> CloneModule(const Module *M, ValueToValueMapTy &VMap); + +/// Return a copy of the specified module. The ShouldCloneDefinition function +/// controls whether a specific GlobalValue's definition is cloned. If the +/// function returns false, the module copy will contain an external reference +/// in place of the global definition. +std::unique_ptr<Module> +CloneModule(const Module *M, ValueToValueMapTy &VMap, + std::function<bool(const GlobalValue *)> ShouldCloneDefinition); /// ClonedCodeInfo - This struct can be used to capture information about code /// being cloned, while it is being cloned. @@ -65,6 +74,11 @@ struct ClonedCodeInfo { /// size. bool ContainsDynamicAllocas; + /// All cloned call sites that have operand bundles attached are appended to + /// this vector. This vector may contain nulls or undefs if some of the + /// originally inserted callsites were DCE'ed after they were cloned. + std::vector<WeakVH> OperandBundleCallSites; + ClonedCodeInfo() : ContainsCalls(false), ContainsDynamicAllocas(false) {} }; @@ -193,14 +207,12 @@ void CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc, class InlineFunctionInfo { public: explicit InlineFunctionInfo(CallGraph *cg = nullptr, - AliasAnalysis *AA = nullptr, AssumptionCacheTracker *ACT = nullptr) - : CG(cg), AA(AA), ACT(ACT) {} + : CG(cg), ACT(ACT) {} /// CG - If non-null, InlineFunction will update the callgraph to reflect the /// changes it makes. CallGraph *CG; - AliasAnalysis *AA; AssumptionCacheTracker *ACT; /// StaticAllocas - InlineFunction fills this in with all static allocas that @@ -228,11 +240,11 @@ public: /// function by one level. /// bool InlineFunction(CallInst *C, InlineFunctionInfo &IFI, - bool InsertLifetime = true); + AAResults *CalleeAAR = nullptr, bool InsertLifetime = true); bool InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI, - bool InsertLifetime = true); + AAResults *CalleeAAR = nullptr, bool InsertLifetime = true); bool InlineFunction(CallSite CS, InlineFunctionInfo &IFI, - bool InsertLifetime = true); + AAResults *CalleeAAR = nullptr, bool InsertLifetime = true); /// \brief Clones a loop \p OrigLoop. Returns the loop and the blocks in \p /// Blocks. diff --git a/include/llvm/Transforms/Utils/Local.h b/include/llvm/Transforms/Utils/Local.h index a1bb367ac7b6..81b376f0c212 100644 --- a/include/llvm/Transforms/Utils/Local.h +++ b/include/llvm/Transforms/Utils/Local.h @@ -15,6 +15,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_LOCAL_H #define LLVM_TRANSFORMS_UTILS_LOCAL_H +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/GetElementPtrTypeIterator.h" @@ -40,7 +41,6 @@ class DataLayout; class TargetLibraryInfo; class TargetTransformInfo; class DIBuilder; -class AliasAnalysis; class DominatorTree; template<typename T> class SmallVectorImpl; @@ -271,11 +271,34 @@ bool LowerDbgDeclare(Function &F); /// an alloca, if any. DbgDeclareInst *FindAllocaDbgDeclare(Value *V); -/// \brief Replaces llvm.dbg.declare instruction when an alloca is replaced with -/// a new value. If Deref is true, tan additional DW_OP_deref is prepended to -/// the expression. +/// \brief Replaces llvm.dbg.declare instruction when the address it describes +/// is replaced with a new value. If Deref is true, an additional DW_OP_deref is +/// prepended to the expression. If Offset is non-zero, a constant displacement +/// is added to the expression (after the optional Deref). Offset can be +/// negative. +bool replaceDbgDeclare(Value *Address, Value *NewAddress, + Instruction *InsertBefore, DIBuilder &Builder, + bool Deref, int Offset); + +/// \brief Replaces llvm.dbg.declare instruction when the alloca it describes +/// is replaced with a new value. If Deref is true, an additional DW_OP_deref is +/// prepended to the expression. If Offset is non-zero, a constant displacement +/// is added to the expression (after the optional Deref). Offset can be +/// negative. New llvm.dbg.declare is inserted immediately before AI. bool replaceDbgDeclareForAlloca(AllocaInst *AI, Value *NewAllocaAddress, - DIBuilder &Builder, bool Deref); + DIBuilder &Builder, bool Deref, int Offset = 0); + +/// \brief Insert an unreachable instruction before the specified +/// instruction, making it and the rest of the code in the block dead. +void changeToUnreachable(Instruction *I, bool UseLLVMTrap); + +/// Replace 'BB's terminator with one that does not have an unwind successor +/// block. Rewrites `invoke` to `call`, etc. Updates any PHIs in unwind +/// successor. +/// +/// \param BB Block whose terminator will be replaced. Its terminator must +/// have an unwind successor. +void removeUnwindEdge(BasicBlock *BB); /// \brief Remove all blocks that can not be reached from the function's entry. /// @@ -291,6 +314,22 @@ void combineMetadata(Instruction *K, const Instruction *J, ArrayRef<unsigned> Kn /// the given edge. Returns the number of replacements made. unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, const BasicBlockEdge &Edge); +/// \brief Replace each use of 'From' with 'To' if that use is dominated by +/// the given BasicBlock. Returns the number of replacements made. +unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, + const BasicBlock *BB); + + +/// \brief Return true if the CallSite CS calls a gc leaf function. +/// +/// A leaf function is a function that does not safepoint the thread during its +/// execution. During a call or invoke to such a function, the callers stack +/// does not have to be made parseable. +/// +/// Most passes can and should ignore this information, and it is only used +/// during lowering by the GC infrastructure. +bool callsGCLeafFunction(ImmutableCallSite CS); + } // End llvm namespace #endif diff --git a/include/llvm/Transforms/Utils/LoopUtils.h b/include/llvm/Transforms/Utils/LoopUtils.h index 15747bc7f1ac..17aaee03e4a8 100644 --- a/include/llvm/Transforms/Utils/LoopUtils.h +++ b/include/llvm/Transforms/Utils/LoopUtils.h @@ -15,11 +15,11 @@ #define LLVM_TRANSFORMS_UTILS_LOOPUTILS_H #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" namespace llvm { -class AliasAnalysis; class AliasSet; class AliasSetTracker; class AssumptionCache; @@ -85,24 +85,35 @@ public: RecurrenceDescriptor() : StartValue(nullptr), LoopExitInstr(nullptr), Kind(RK_NoRecurrence), - MinMaxKind(MRK_Invalid) {} + MinMaxKind(MRK_Invalid), UnsafeAlgebraInst(nullptr), + RecurrenceType(nullptr), IsSigned(false) {} RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurrenceKind K, - MinMaxRecurrenceKind MK) - : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK) {} + MinMaxRecurrenceKind MK, Instruction *UAI, Type *RT, + bool Signed, SmallPtrSetImpl<Instruction *> &CI) + : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK), + UnsafeAlgebraInst(UAI), RecurrenceType(RT), IsSigned(Signed) { + CastInsts.insert(CI.begin(), CI.end()); + } /// This POD struct holds information about a potential recurrence operation. class InstDesc { public: - InstDesc(bool IsRecur, Instruction *I) - : IsRecurrence(IsRecur), PatternLastInst(I), MinMaxKind(MRK_Invalid) {} + InstDesc(bool IsRecur, Instruction *I, Instruction *UAI = nullptr) + : IsRecurrence(IsRecur), PatternLastInst(I), MinMaxKind(MRK_Invalid), + UnsafeAlgebraInst(UAI) {} - InstDesc(Instruction *I, MinMaxRecurrenceKind K) - : IsRecurrence(true), PatternLastInst(I), MinMaxKind(K) {} + InstDesc(Instruction *I, MinMaxRecurrenceKind K, Instruction *UAI = nullptr) + : IsRecurrence(true), PatternLastInst(I), MinMaxKind(K), + UnsafeAlgebraInst(UAI) {} bool isRecurrence() { return IsRecurrence; } + bool hasUnsafeAlgebra() { return UnsafeAlgebraInst != nullptr; } + + Instruction *getUnsafeAlgebraInst() { return UnsafeAlgebraInst; } + MinMaxRecurrenceKind getMinMaxKind() { return MinMaxKind; } Instruction *getPatternInst() { return PatternLastInst; } @@ -115,6 +126,8 @@ public: Instruction *PatternLastInst; // If this is a min/max pattern the comparison predicate. MinMaxRecurrenceKind MinMaxKind; + // Recurrence has unsafe algebra. + Instruction *UnsafeAlgebraInst; }; /// Returns a struct describing if the instruction 'I' can be a recurrence @@ -125,7 +138,7 @@ public: static InstDesc isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, InstDesc &Prev, bool HasFunNoNaNAttr); - /// Returns true if instuction I has multiple uses in Insts + /// Returns true if instruction I has multiple uses in Insts static bool hasMultipleUsesOf(Instruction *I, SmallPtrSetImpl<Instruction *> &Insts); @@ -167,6 +180,51 @@ public: Instruction *getLoopExitInstr() { return LoopExitInstr; } + /// Returns true if the recurrence has unsafe algebra which requires a relaxed + /// floating-point model. + bool hasUnsafeAlgebra() { return UnsafeAlgebraInst != nullptr; } + + /// Returns first unsafe algebra instruction in the PHI node's use-chain. + Instruction *getUnsafeAlgebraInst() { return UnsafeAlgebraInst; } + + /// Returns true if the recurrence kind is an integer kind. + static bool isIntegerRecurrenceKind(RecurrenceKind Kind); + + /// Returns true if the recurrence kind is a floating point kind. + static bool isFloatingPointRecurrenceKind(RecurrenceKind Kind); + + /// Returns true if the recurrence kind is an arithmetic kind. + static bool isArithmeticRecurrenceKind(RecurrenceKind Kind); + + /// Determines if Phi may have been type-promoted. If Phi has a single user + /// that ANDs the Phi with a type mask, return the user. RT is updated to + /// account for the narrower bit width represented by the mask, and the AND + /// instruction is added to CI. + static Instruction *lookThroughAnd(PHINode *Phi, Type *&RT, + SmallPtrSetImpl<Instruction *> &Visited, + SmallPtrSetImpl<Instruction *> &CI); + + /// Returns true if all the source operands of a recurrence are either + /// SExtInsts or ZExtInsts. This function is intended to be used with + /// lookThroughAnd to determine if the recurrence has been type-promoted. The + /// source operands are added to CI, and IsSigned is updated to indicate if + /// all source operands are SExtInsts. + static bool getSourceExtensionKind(Instruction *Start, Instruction *Exit, + Type *RT, bool &IsSigned, + SmallPtrSetImpl<Instruction *> &Visited, + SmallPtrSetImpl<Instruction *> &CI); + + /// Returns the type of the recurrence. This type can be narrower than the + /// actual type of the Phi if the recurrence has been type-promoted. + Type *getRecurrenceType() { return RecurrenceType; } + + /// Returns a reference to the instructions used for type-promoting the + /// recurrence. + SmallPtrSet<Instruction *, 8> &getCastInsts() { return CastInsts; } + + /// Returns true if all source operands of the recurrence are SExtInsts. + bool isSigned() { return IsSigned; } + private: // The starting value of the recurrence. // It does not have to be zero! @@ -177,19 +235,74 @@ private: RecurrenceKind Kind; // If this a min/max recurrence the kind of recurrence. MinMaxRecurrenceKind MinMaxKind; + // First occurance of unasfe algebra in the PHI's use-chain. + Instruction *UnsafeAlgebraInst; + // The type of the recurrence. + Type *RecurrenceType; + // True if all source operands of the recurrence are SExtInsts. + bool IsSigned; + // Instructions used for type-promoting the recurrence. + SmallPtrSet<Instruction *, 8> CastInsts; +}; + +/// A struct for saving information about induction variables. +class InductionDescriptor { +public: + /// This enum represents the kinds of inductions that we support. + enum InductionKind { + IK_NoInduction, ///< Not an induction variable. + IK_IntInduction, ///< Integer induction variable. Step = C. + IK_PtrInduction ///< Pointer induction var. Step = C / sizeof(elem). + }; + +public: + /// Default constructor - creates an invalid induction. + InductionDescriptor() + : StartValue(nullptr), IK(IK_NoInduction), StepValue(nullptr) {} + + /// Get the consecutive direction. Returns: + /// 0 - unknown or non-consecutive. + /// 1 - consecutive and increasing. + /// -1 - consecutive and decreasing. + int getConsecutiveDirection() const; + + /// Compute the transformed value of Index at offset StartValue using step + /// StepValue. + /// For integer induction, returns StartValue + Index * StepValue. + /// For pointer induction, returns StartValue[Index * StepValue]. + /// FIXME: The newly created binary instructions should contain nsw/nuw + /// flags, which can be found from the original scalar operations. + Value *transform(IRBuilder<> &B, Value *Index) const; + + Value *getStartValue() const { return StartValue; } + InductionKind getKind() const { return IK; } + ConstantInt *getStepValue() const { return StepValue; } + + static bool isInductionPHI(PHINode *Phi, ScalarEvolution *SE, + InductionDescriptor &D); + +private: + /// Private constructor - used by \c isInductionPHI. + InductionDescriptor(Value *Start, InductionKind K, ConstantInt *Step); + + /// Start value. + TrackingVH<Value> StartValue; + /// Induction kind. + InductionKind IK; + /// Step value. + ConstantInt *StepValue; }; -BasicBlock *InsertPreheaderForLoop(Loop *L, Pass *P); +BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, + bool PreserveLCSSA); /// \brief Simplify each loop in a loop nest recursively. /// /// This takes a potentially un-simplified loop L (and its children) and turns -/// it into a simplified loop nest with preheaders and single backedges. It -/// will optionally update \c AliasAnalysis and \c ScalarEvolution analyses if -/// passed into it. -bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, Pass *PP, - AliasAnalysis *AA = nullptr, ScalarEvolution *SE = nullptr, - AssumptionCache *AC = nullptr); +/// it into a simplified loop nest with preheaders and single backedges. It will +/// update \c AliasAnalysis and \c ScalarEvolution analyses if they're non-null. +bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, + AssumptionCache *AC, bool PreserveLCSSA); /// \brief Put loop into LCSSA form. /// @@ -203,7 +316,7 @@ bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, Pass *PP, /// /// Returns true if any modifications are made to the loop. bool formLCSSA(Loop &L, DominatorTree &DT, LoopInfo *LI, - ScalarEvolution *SE = nullptr); + ScalarEvolution *SE); /// \brief Put a loop nest into LCSSA form. /// @@ -215,7 +328,7 @@ bool formLCSSA(Loop &L, DominatorTree &DT, LoopInfo *LI, /// /// Returns true if any modifications are made to the loop. bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI, - ScalarEvolution *SE = nullptr); + ScalarEvolution *SE); /// \brief Walk the specified region of the CFG (defined by all blocks /// dominated by the specified block, and that are in the current loop) in @@ -242,10 +355,10 @@ bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, /// \brief Try to promote memory values to scalars by sinking stores out of /// the loop and moving loads to before the loop. We do this by looping over -/// the stores in the loop, looking for stores to Must pointers which are +/// the stores in the loop, looking for stores to Must pointers which are /// loop invariant. It takes AliasSet, Loop exit blocks vector, loop exit blocks /// insertion point vector, PredIteratorCache, LoopInfo, DominatorTree, Loop, -/// AliasSet information for all instructions of the loop and loop safety +/// AliasSet information for all instructions of the loop and loop safety /// information as arguments. It returns changed status. bool promoteLoopAccessesToScalars(AliasSet &, SmallVectorImpl<BasicBlock*> &, SmallVectorImpl<Instruction*> &, @@ -254,15 +367,13 @@ bool promoteLoopAccessesToScalars(AliasSet &, SmallVectorImpl<BasicBlock*> &, LICMSafetyInfo *); /// \brief Computes safety information for a loop -/// checks loop body & header for the possiblity of may throw +/// checks loop body & header for the possibility of may throw /// exception, it takes LICMSafetyInfo and loop as argument. /// Updates safety information in LICMSafetyInfo argument. void computeLICMSafetyInfo(LICMSafetyInfo *, Loop *); -/// \brief Checks if the given PHINode in a loop header is an induction -/// variable. Returns true if this is an induction PHI along with the step -/// value. -bool isInductionPHI(PHINode *, ScalarEvolution *, ConstantInt *&); +/// \brief Returns the instructions that use values defined in the loop. +SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L); } #endif diff --git a/include/llvm/Transforms/Utils/LoopVersioning.h b/include/llvm/Transforms/Utils/LoopVersioning.h index 009fba48c6a3..3b70594e0b63 100644 --- a/include/llvm/Transforms/Utils/LoopVersioning.h +++ b/include/llvm/Transforms/Utils/LoopVersioning.h @@ -16,13 +16,17 @@ #ifndef LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H #define LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H +#include "llvm/Analysis/LoopAccessAnalysis.h" +#include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Transforms/Utils/ValueMapper.h" +#include "llvm/Transforms/Utils/LoopUtils.h" namespace llvm { class Loop; class LoopAccessInfo; class LoopInfo; +class ScalarEvolution; /// \brief This class emits a version of the loop where run-time checks ensure /// that may-alias pointers can't overlap. @@ -31,13 +35,13 @@ class LoopInfo; /// already has a preheader. class LoopVersioning { public: + /// \brief Expects LoopAccessInfo, Loop, LoopInfo, DominatorTree as input. + /// It uses runtime check provided by the user. If \p UseLAIChecks is true, + /// we will retain the default checks made by LAI. Otherwise, construct an + /// object having no checks and we expect the user to add them. LoopVersioning(const LoopAccessInfo &LAI, Loop *L, LoopInfo *LI, - DominatorTree *DT, - const SmallVector<int, 8> *PtrToPartition = nullptr); - - /// \brief Returns true if we need memchecks to disambiguate may-aliasing - /// accesses. - bool needsRuntimeChecks() const; + DominatorTree *DT, ScalarEvolution *SE, + bool UseLAIChecks = true); /// \brief Performs the CFG manipulation part of versioning the loop including /// the DominatorTree and LoopInfo updates. @@ -52,15 +56,11 @@ public: /// analyze L /// if versioning is necessary version L /// transform L - void versionLoop(Pass *P); + void versionLoop() { versionLoop(findDefsUsedOutsideOfLoop(VersionedLoop)); } - /// \brief Adds the necessary PHI nodes for the versioned loops based on the - /// loop-defined values used outside of the loop. - /// - /// This needs to be called after versionLoop if there are defs in the loop - /// that are used outside the loop. FIXME: this should be invoked internally - /// by versionLoop and made private. - void addPHINodes(const SmallVectorImpl<Instruction *> &DefsUsedOutside); + /// \brief Same but if the client has already precomputed the set of values + /// used outside the loop, this API will allows passing that. + void versionLoop(const SmallVectorImpl<Instruction *> &DefsUsedOutside); /// \brief Returns the versioned loop. Control flows here if pointers in the /// loop don't alias (i.e. all memchecks passed). (This loop is actually the @@ -71,7 +71,21 @@ public: /// loop may alias (i.e. one of the memchecks failed). Loop *getNonVersionedLoop() { return NonVersionedLoop; } + /// \brief Sets the runtime alias checks for versioning the loop. + void setAliasChecks( + const SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks); + + /// \brief Sets the runtime SCEV checks for versioning the loop. + void setSCEVChecks(SCEVUnionPredicate Check); + private: + /// \brief Adds the necessary PHI nodes for the versioned loops based on the + /// loop-defined values used outside of the loop. + /// + /// This needs to be called after versionLoop if there are defs in the loop + /// that are used outside the loop. + void addPHINodes(const SmallVectorImpl<Instruction *> &DefsUsedOutside); + /// \brief The original loop. This becomes the "versioned" one. I.e., /// control flows here if pointers in the loop don't alias. Loop *VersionedLoop; @@ -79,21 +93,21 @@ private: /// loop may alias (memchecks failed). Loop *NonVersionedLoop; - /// \brief For each memory pointer it contains the partitionId it is used in. - /// If nullptr, no partitioning is used. - /// - /// The I-th entry corresponds to I-th entry in LAI.getRuntimePointerCheck(). - /// If the pointer is used in multiple partitions the entry is set to -1. - const SmallVector<int, 8> *PtrToPartition; - /// \brief This maps the instructions from VersionedLoop to their counterpart /// in NonVersionedLoop. ValueToValueMapTy VMap; + /// \brief The set of alias checks that we are versioning for. + SmallVector<RuntimePointerChecking::PointerCheck, 4> AliasChecks; + + /// \brief The set of SCEV checks that we are versioning for. + SCEVUnionPredicate Preds; + /// \brief Analyses used. const LoopAccessInfo &LAI; LoopInfo *LI; DominatorTree *DT; + ScalarEvolution *SE; }; } diff --git a/include/llvm/Transforms/Utils/ModuleUtils.h b/include/llvm/Transforms/Utils/ModuleUtils.h index 622265bae143..0f23d34de5db 100644 --- a/include/llvm/Transforms/Utils/ModuleUtils.h +++ b/include/llvm/Transforms/Utils/ModuleUtils.h @@ -15,6 +15,7 @@ #define LLVM_TRANSFORMS_UTILS_MODULEUTILS_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" #include <utility> // for std::pair namespace llvm { @@ -56,7 +57,8 @@ Function *checkSanitizerInterfaceFunction(Constant *FuncOrBitcast); /// respectively. std::pair<Function *, Function *> createSanitizerCtorAndInitFunctions( Module &M, StringRef CtorName, StringRef InitName, - ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs); + ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs, + StringRef VersionCheckName = StringRef()); } // End llvm namespace #endif // LLVM_TRANSFORMS_UTILS_MODULEUTILS_H diff --git a/include/llvm/Transforms/Utils/SSAUpdaterImpl.h b/include/llvm/Transforms/Utils/SSAUpdaterImpl.h index ed0841c46c27..425ecd3cfb5e 100644 --- a/include/llvm/Transforms/Utils/SSAUpdaterImpl.h +++ b/include/llvm/Transforms/Utils/SSAUpdaterImpl.h @@ -378,7 +378,7 @@ public: void FindExistingPHI(BlkT *BB, BlockListTy *BlockList) { for (typename BlkT::iterator BBI = BB->begin(), BBE = BB->end(); BBI != BBE; ++BBI) { - PhiT *SomePHI = Traits::InstrIsPHI(BBI); + PhiT *SomePHI = Traits::InstrIsPHI(&*BBI); if (!SomePHI) break; if (CheckIfPHIMatches(SomePHI)) { diff --git a/include/llvm/Transforms/Utils/SimplifyIndVar.h b/include/llvm/Transforms/Utils/SimplifyIndVar.h index dcb1d67cbf75..3c55e64537c7 100644 --- a/include/llvm/Transforms/Utils/SimplifyIndVar.h +++ b/include/llvm/Transforms/Utils/SimplifyIndVar.h @@ -25,7 +25,7 @@ class CastInst; class DominatorTree; class IVUsers; class Loop; -class LPPassManager; +class LoopInfo; class PHINode; class ScalarEvolution; @@ -57,13 +57,14 @@ public: /// simplifyUsersOfIV - Simplify instructions that use this induction variable /// by using ScalarEvolution to analyze the IV's recurrence. -bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, LPPassManager *LPM, - SmallVectorImpl<WeakVH> &Dead, IVVisitor *V = nullptr); +bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, DominatorTree *DT, + LoopInfo *LI, SmallVectorImpl<WeakVH> &Dead, + IVVisitor *V = nullptr); /// SimplifyLoopIVs - Simplify users of induction variables within this /// loop. This does not actually change or add IVs. -bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, LPPassManager *LPM, - SmallVectorImpl<WeakVH> &Dead); +bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, DominatorTree *DT, + LoopInfo *LI, SmallVectorImpl<WeakVH> &Dead); } // namespace llvm diff --git a/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/include/llvm/Transforms/Utils/SimplifyLibCalls.h index 41159603aae5..410a075aeb98 100644 --- a/include/llvm/Transforms/Utils/SimplifyLibCalls.h +++ b/include/llvm/Transforms/Utils/SimplifyLibCalls.h @@ -131,8 +131,11 @@ private: Value *optimizePow(CallInst *CI, IRBuilder<> &B); Value *optimizeExp2(CallInst *CI, IRBuilder<> &B); Value *optimizeFabs(CallInst *CI, IRBuilder<> &B); + Value *optimizeFMinFMax(CallInst *CI, IRBuilder<> &B); + Value *optimizeLog(CallInst *CI, IRBuilder<> &B); Value *optimizeSqrt(CallInst *CI, IRBuilder<> &B); Value *optimizeSinCosPi(CallInst *CI, IRBuilder<> &B); + Value *optimizeTan(CallInst *CI, IRBuilder<> &B); // Integer Library Call Optimizations Value *optimizeFFS(CallInst *CI, IRBuilder<> &B); diff --git a/include/llvm/Transforms/Utils/SplitModule.h b/include/llvm/Transforms/Utils/SplitModule.h new file mode 100644 index 000000000000..7d896d1993d6 --- /dev/null +++ b/include/llvm/Transforms/Utils/SplitModule.h @@ -0,0 +1,43 @@ +//===- SplitModule.h - Split a module into partitions -----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the function llvm::SplitModule, which splits a module +// into multiple linkable partitions. It can be used to implement parallel code +// generation for link-time optimization. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_SPLITMODULE_H +#define LLVM_TRANSFORMS_UTILS_SPLITMODULE_H + +#include <functional> +#include <memory> + +namespace llvm { + +class Module; +class StringRef; + +/// Splits the module M into N linkable partitions. The function ModuleCallback +/// is called N times passing each individual partition as the MPart argument. +/// +/// FIXME: This function does not deal with the somewhat subtle symbol +/// visibility issues around module splitting, including (but not limited to): +/// +/// - Internal symbols should not collide with symbols defined outside the +/// module. +/// - Internal symbols defined in module-level inline asm should be visible to +/// each partition. +void SplitModule( + std::unique_ptr<Module> M, unsigned N, + std::function<void(std::unique_ptr<Module> MPart)> ModuleCallback); + +} // End llvm namespace + +#endif diff --git a/include/llvm/Transforms/Utils/UnrollLoop.h b/include/llvm/Transforms/Utils/UnrollLoop.h index 7f2cf8d7f59e..710817cddf6a 100644 --- a/include/llvm/Transforms/Utils/UnrollLoop.h +++ b/include/llvm/Transforms/Utils/UnrollLoop.h @@ -21,20 +21,23 @@ namespace llvm { class AssumptionCache; +class DominatorTree; class Loop; class LoopInfo; class LPPassManager; class MDNode; class Pass; +class ScalarEvolution; bool UnrollLoop(Loop *L, unsigned Count, unsigned TripCount, bool AllowRuntime, bool AllowExpensiveTripCount, unsigned TripMultiple, - LoopInfo *LI, Pass *PP, LPPassManager *LPM, - AssumptionCache *AC); + LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, + AssumptionCache *AC, bool PreserveLCSSA); bool UnrollRuntimeLoopProlog(Loop *L, unsigned Count, bool AllowExpensiveTripCount, LoopInfo *LI, - LPPassManager *LPM); + ScalarEvolution *SE, DominatorTree *DT, + bool PreserveLCSSA); MDNode *GetUnrollMetadata(MDNode *LoopID, StringRef Name); } diff --git a/include/llvm/Transforms/Utils/ValueMapper.h b/include/llvm/Transforms/Utils/ValueMapper.h index 047ab818711b..469022f34c56 100644 --- a/include/llvm/Transforms/Utils/ValueMapper.h +++ b/include/llvm/Transforms/Utils/ValueMapper.h @@ -38,13 +38,34 @@ namespace llvm { /// to materialize Values on demand. class ValueMaterializer { virtual void anchor(); // Out of line method. - public: - virtual ~ValueMaterializer() {} - /// materializeValueFor - The client should implement this method if they - /// want to generate a mapped Value on demand. For example, if linking - /// lazily. - virtual Value *materializeValueFor(Value *V) = 0; + protected: + ~ValueMaterializer() = default; + ValueMaterializer() = default; + ValueMaterializer(const ValueMaterializer&) = default; + ValueMaterializer &operator=(const ValueMaterializer&) = default; + + public: + /// The client should implement this method if they want to generate a + /// mapped Value on demand. For example, if linking lazily. + virtual Value *materializeDeclFor(Value *V) = 0; + + /// If the data being mapped is recursive, the above function can map + /// just the declaration and this is called to compute the initializer. + /// It is called after the mapping is recorded, so it doesn't need to worry + /// about recursion. + virtual void materializeInitFor(GlobalValue *New, GlobalValue *Old); + + /// If the client needs to handle temporary metadata it must implement + /// these methods. + virtual Metadata *mapTemporaryMetadata(Metadata *MD) { return nullptr; } + virtual void replaceTemporaryMetadata(const Metadata *OrigMD, + Metadata *NewMD) {} + + /// The client should implement this method if some metadata need + /// not be mapped, for example DISubprogram metadata for functions not + /// linked into the destination module. + virtual bool isMetadataNeeded(Metadata *MD) { return true; } }; /// RemapFlags - These are flags that the value mapping APIs allow. @@ -59,7 +80,20 @@ namespace llvm { /// RF_IgnoreMissingEntries - If this flag is set, the remapper ignores /// entries that are not in the value map. If it is unset, it aborts if an /// operand is asked to be remapped which doesn't exist in the mapping. - RF_IgnoreMissingEntries = 2 + RF_IgnoreMissingEntries = 2, + + /// Instruct the remapper to move distinct metadata instead of duplicating + /// it when there are module-level changes. + RF_MoveDistinctMDs = 4, + + /// Any global values not in value map are mapped to null instead of + /// mapping to self. Illegal if RF_IgnoreMissingEntries is also set. + RF_NullMapMissingGlobalValues = 8, + + /// Set when there is still temporary metadata that must be handled, + /// such as when we are doing function importing and will materialize + /// and link metadata as a postpass. + RF_HaveUnmaterializedMetadata = 16, }; static inline RemapFlags operator|(RemapFlags LHS, RemapFlags RHS) { diff --git a/include/llvm/module.modulemap b/include/llvm/module.modulemap index dcc5ce1059ff..0adce0c9602d 100644 --- a/include/llvm/module.modulemap +++ b/include/llvm/module.modulemap @@ -190,17 +190,15 @@ module LLVM_Utils { // Exclude this; it's fundamentally non-modular. exclude header "Support/PluginLoader.h" - // Exclude this; it's a weirdly-factored part of llvm-gcov and conflicts - // with the Analysis module (which also defines an llvm::GCOVOptions). - exclude header "Support/GCOV.h" - // FIXME: Mislayered? exclude header "Support/TargetRegistry.h" // These are intended for textual inclusion. + textual header "Support/ARMTargetParser.def" textual header "Support/Dwarf.def" textual header "Support/ELFRelocs/AArch64.def" textual header "Support/ELFRelocs/ARM.def" + textual header "Support/ELFRelocs/AVR.def" textual header "Support/ELFRelocs/Hexagon.def" textual header "Support/ELFRelocs/i386.def" textual header "Support/ELFRelocs/Mips.def" @@ -210,6 +208,12 @@ module LLVM_Utils { textual header "Support/ELFRelocs/SystemZ.def" textual header "Support/ELFRelocs/x86_64.def" } + + // This part of the module is usable from both C and C++ code. + module ConvertUTF { + header "Support/ConvertUTF.h" + export * + } } module LLVM_CodeGen_MachineValueType { |