aboutsummaryrefslogtreecommitdiff
path: root/lib/profile
diff options
context:
space:
mode:
Diffstat (limited to 'lib/profile')
-rw-r--r--lib/profile/CMakeLists.txt62
-rw-r--r--lib/profile/InstrProfData.inc735
-rw-r--r--lib/profile/InstrProfiling.c76
-rw-r--r--lib/profile/InstrProfiling.h77
-rw-r--r--lib/profile/InstrProfilingBuffer.c82
-rw-r--r--lib/profile/InstrProfilingFile.c98
-rw-r--r--lib/profile/InstrProfilingInternal.h78
-rw-r--r--lib/profile/InstrProfilingPlatformDarwin.c45
-rw-r--r--lib/profile/InstrProfilingPlatformLinux.c59
-rw-r--r--lib/profile/InstrProfilingPlatformOther.c48
-rw-r--r--lib/profile/InstrProfilingPort.h76
-rw-r--r--lib/profile/InstrProfilingRuntime.cc3
-rw-r--r--lib/profile/InstrProfilingUtil.c3
-rw-r--r--lib/profile/InstrProfilingValue.c180
-rw-r--r--lib/profile/InstrProfilingWriter.c175
15 files changed, 1565 insertions, 232 deletions
diff --git a/lib/profile/CMakeLists.txt b/lib/profile/CMakeLists.txt
index d03409fc45b7..1b10ade0eee6 100644
--- a/lib/profile/CMakeLists.txt
+++ b/lib/profile/CMakeLists.txt
@@ -1,27 +1,71 @@
+
+CHECK_CXX_SOURCE_COMPILES("
+#ifdef _MSC_VER
+#include <Intrin.h> /* Workaround for PR19898. */
+#include <windows.h>
+#endif
+int main() {
+#ifdef _MSC_VER
+ volatile LONG val = 1;
+ MemoryBarrier();
+ InterlockedCompareExchange(&val, 0, 1);
+ InterlockedIncrement(&val);
+ InterlockedDecrement(&val);
+#else
+ volatile unsigned long val = 1;
+ __sync_synchronize();
+ __sync_val_compare_and_swap(&val, 1, 0);
+ __sync_add_and_fetch(&val, 1);
+ __sync_sub_and_fetch(&val, 1);
+#endif
+ return 0;
+ }
+" COMPILER_RT_TARGET_HAS_ATOMICS)
+
add_custom_target(profile)
set(PROFILE_SOURCES
GCDAProfiling.c
InstrProfiling.c
+ InstrProfilingValue.c
InstrProfilingBuffer.c
InstrProfilingFile.c
+ InstrProfilingWriter.c
InstrProfilingPlatformDarwin.c
+ InstrProfilingPlatformLinux.c
InstrProfilingPlatformOther.c
InstrProfilingRuntime.cc
InstrProfilingUtil.c)
+if(UNIX)
+ set(EXTRA_FLAGS
+ -fPIC
+ -Wno-pedantic)
+else()
+ set(EXTRA_FLAGS
+ -fPIC)
+endif()
+
+if(COMPILER_RT_TARGET_HAS_ATOMICS)
+ set(EXTRA_FLAGS
+ ${EXTRA_FLAGS}
+ -DCOMPILER_RT_HAS_ATOMICS=1)
+endif()
+
if(APPLE)
- add_compiler_rt_osx_static_runtime(clang_rt.profile_osx
+ add_compiler_rt_runtime(clang_rt.profile
+ STATIC
+ OS ${PROFILE_SUPPORTED_OS}
ARCHS ${PROFILE_SUPPORTED_ARCH}
- SOURCES ${PROFILE_SOURCES})
- add_dependencies(profile clang_rt.profile_osx)
+ SOURCES ${PROFILE_SOURCES}
+ PARENT_TARGET profile)
else()
- foreach(arch ${PROFILE_SUPPORTED_ARCH})
- add_compiler_rt_runtime(clang_rt.profile-${arch} ${arch} STATIC
- CFLAGS -fPIC
- SOURCES ${PROFILE_SOURCES})
- add_dependencies(profile clang_rt.profile-${arch})
- endforeach()
+ add_compiler_rt_runtime(clang_rt.profile
+ STATIC
+ ARCHS ${PROFILE_SUPPORTED_ARCH}
+ CFLAGS ${EXTRA_FLAGS}
+ SOURCES ${PROFILE_SOURCES}
+ PARENT_TARGET profile)
endif()
add_dependencies(compiler-rt profile)
diff --git a/lib/profile/InstrProfData.inc b/lib/profile/InstrProfData.inc
new file mode 100644
index 000000000000..48dae506cabb
--- /dev/null
+++ b/lib/profile/InstrProfData.inc
@@ -0,0 +1,735 @@
+/*===-- InstrProfData.inc - instr profiling runtime structures -----------=== *\
+|*
+|* The LLVM Compiler Infrastructure
+|*
+|* This file is distributed under the University of Illinois Open Source
+|* License. See LICENSE.TXT for details.
+|*
+\*===----------------------------------------------------------------------===*/
+/*
+ * This is the master file that defines all the data structure, signature,
+ * constant literals that are shared across profiling runtime library,
+ * compiler (instrumentation), and host tools (reader/writer). The entities
+ * defined in this file affect the profile runtime ABI, the raw profile format,
+ * or both.
+ *
+ * The file has two identical copies. The master copy lives in LLVM and
+ * the other one sits in compiler-rt/lib/profile directory. To make changes
+ * in this file, first modify the master copy and copy it over to compiler-rt.
+ * Testing of any change in this file can start only after the two copies are
+ * synced up.
+ *
+ * The first part of the file includes macros that defines types, names, and
+ * initializers for the member fields of the core data structures. The field
+ * declarations for one structure is enabled by defining the field activation
+ * macro associated with that structure. Only one field activation record
+ * can be defined at one time and the rest definitions will be filtered out by
+ * the preprocessor.
+ *
+ * Examples of how the template is used to instantiate structure definition:
+ * 1. To declare a structure:
+ *
+ * struct ProfData {
+ * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
+ * Type Name;
+ * #include "llvm/ProfileData/InstrProfData.inc"
+ * };
+ *
+ * 2. To construct LLVM type arrays for the struct type:
+ *
+ * Type *DataTypes[] = {
+ * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
+ * LLVMType,
+ * #include "llvm/ProfileData/InstrProfData.inc"
+ * };
+ *
+ * 4. To construct constant array for the initializers:
+ * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
+ * Initializer,
+ * Constant *ConstantVals[] = {
+ * #include "llvm/ProfileData/InstrProfData.inc"
+ * };
+ *
+ *
+ * The second part of the file includes definitions all other entities that
+ * are related to runtime ABI and format. When no field activation macro is
+ * defined, this file can be included to introduce the definitions.
+ *
+\*===----------------------------------------------------------------------===*/
+
+/* INSTR_PROF_DATA start. */
+/* Definition of member fields of the per-function control structure. */
+#ifndef INSTR_PROF_DATA
+#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+
+INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
+ NamePtr->getType()->getPointerElementType()->getArrayNumElements()))
+INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
+ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
+INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
+ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
+ Inc->getHash()->getZExtValue()))
+INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), NamePtr, \
+ ConstantExpr::getBitCast(NamePtr, llvm::Type::getInt8PtrTy(Ctx)))
+INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
+ ConstantExpr::getBitCast(CounterPtr, \
+ llvm::Type::getInt64PtrTy(Ctx)))
+INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
+ FunctionAddr)
+INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
+ ConstantPointerNull::get(Int8PtrTy))
+INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
+ ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
+#undef INSTR_PROF_DATA
+/* INSTR_PROF_DATA end. */
+
+/* INSTR_PROF_RAW_HEADER start */
+/* Definition of member fields of the raw profile header data structure. */
+#ifndef INSTR_PROF_RAW_HEADER
+#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
+INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
+INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
+INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
+INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
+INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
+INSTR_PROF_RAW_HEADER(uint64_t, ValueDataSize, ValueDataSize)
+INSTR_PROF_RAW_HEADER(uint64_t, ValueDataDelta, (uintptr_t)ValueDataBegin)
+#undef INSTR_PROF_RAW_HEADER
+/* INSTR_PROF_RAW_HEADER end */
+
+/* VALUE_PROF_FUNC_PARAM start */
+/* Definition of parameter types of the runtime API used to do value profiling
+ * for a given value site.
+ */
+#ifndef VALUE_PROF_FUNC_PARAM
+#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
+#define INSTR_PROF_COMMA
+#else
+#define INSTR_PROF_DATA_DEFINED
+#define INSTR_PROF_COMMA ,
+#endif
+VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
+ INSTR_PROF_COMMA
+VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
+VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
+#undef VALUE_PROF_FUNC_PARAM
+#undef INSTR_PROF_COMMA
+/* VALUE_PROF_FUNC_PARAM end */
+
+/* VALUE_PROF_KIND start */
+#ifndef VALUE_PROF_KIND
+#define VALUE_PROF_KIND(Enumerator, Value)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0)
+/* These two kinds must be the last to be
+ * declared. This is to make sure the string
+ * array created with the template can be
+ * indexed with the kind value.
+ */
+VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget)
+VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget)
+
+#undef VALUE_PROF_KIND
+/* VALUE_PROF_KIND end */
+
+/* COVMAP_FUNC_RECORD start */
+/* Definition of member fields of the function record structure in coverage
+ * map.
+ */
+#ifndef COVMAP_FUNC_RECORD
+#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
+#else
+#define INSTR_PROF_DATA_DEFINED
+#endif
+COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
+ NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
+ llvm::Type::getInt8PtrTy(Ctx)))
+COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
+ llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\
+ NameValue.size()))
+COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
+ llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\
+ CoverageMapping.size()))
+COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
+ llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash))
+#undef COVMAP_FUNC_RECORD
+/* COVMAP_FUNC_RECORD end. */
+
+
+#ifdef INSTR_PROF_VALUE_PROF_DATA
+#define INSTR_PROF_DATA_DEFINED
+
+/*!
+ * This is the header of the data structure that defines the on-disk
+ * layout of the value profile data of a particular kind for one function.
+ */
+typedef struct ValueProfRecord {
+ /* The kind of the value profile record. */
+ uint32_t Kind;
+ /*
+ * The number of value profile sites. It is guaranteed to be non-zero;
+ * otherwise the record for this kind won't be emitted.
+ */
+ uint32_t NumValueSites;
+ /*
+ * The first element of the array that stores the number of profiled
+ * values for each value site. The size of the array is NumValueSites.
+ * Since NumValueSites is greater than zero, there is at least one
+ * element in the array.
+ */
+ uint8_t SiteCountArray[1];
+
+ /*
+ * The fake declaration is for documentation purpose only.
+ * Align the start of next field to be on 8 byte boundaries.
+ uint8_t Padding[X];
+ */
+
+ /* The array of value profile data. The size of the array is the sum
+ * of all elements in SiteCountArray[].
+ InstrProfValueData ValueData[];
+ */
+
+#ifdef __cplusplus
+ /*!
+ * \brief Return the number of value sites.
+ */
+ uint32_t getNumValueSites() const { return NumValueSites; }
+ /*!
+ * \brief Read data from this record and save it to Record.
+ */
+ void deserializeTo(InstrProfRecord &Record,
+ InstrProfRecord::ValueMapType *VMap);
+ /*
+ * In-place byte swap:
+ * Do byte swap for this instance. \c Old is the original order before
+ * the swap, and \c New is the New byte order.
+ */
+ void swapBytes(support::endianness Old, support::endianness New);
+#endif
+} ValueProfRecord;
+
+/*!
+ * Per-function header/control data structure for value profiling
+ * data in indexed format.
+ */
+typedef struct ValueProfData {
+ /*
+ * Total size in bytes including this field. It must be a multiple
+ * of sizeof(uint64_t).
+ */
+ uint32_t TotalSize;
+ /*
+ *The number of value profile kinds that has value profile data.
+ * In this implementation, a value profile kind is considered to
+ * have profile data if the number of value profile sites for the
+ * kind is not zero. More aggressively, the implementation can
+ * choose to check the actual data value: if none of the value sites
+ * has any profiled values, the kind can be skipped.
+ */
+ uint32_t NumValueKinds;
+
+ /*
+ * Following are a sequence of variable length records. The prefix/header
+ * of each record is defined by ValueProfRecord type. The number of
+ * records is NumValueKinds.
+ * ValueProfRecord Record_1;
+ * ValueProfRecord Record_N;
+ */
+
+#if __cplusplus
+ /*!
+ * Return the total size in bytes of the on-disk value profile data
+ * given the data stored in Record.
+ */
+ static uint32_t getSize(const InstrProfRecord &Record);
+ /*!
+ * Return a pointer to \c ValueProfData instance ready to be streamed.
+ */
+ static std::unique_ptr<ValueProfData>
+ serializeFrom(const InstrProfRecord &Record);
+ /*!
+ * Check the integrity of the record. Return the error code when
+ * an error is detected, otherwise return instrprof_error::success.
+ */
+ instrprof_error checkIntegrity();
+ /*!
+ * Return a pointer to \c ValueProfileData instance ready to be read.
+ * All data in the instance are properly byte swapped. The input
+ * data is assumed to be in little endian order.
+ */
+ static ErrorOr<std::unique_ptr<ValueProfData>>
+ getValueProfData(const unsigned char *SrcBuffer,
+ const unsigned char *const SrcBufferEnd,
+ support::endianness SrcDataEndianness);
+ /*!
+ * Swap byte order from \c Endianness order to host byte order.
+ */
+ void swapBytesToHost(support::endianness Endianness);
+ /*!
+ * Swap byte order from host byte order to \c Endianness order.
+ */
+ void swapBytesFromHost(support::endianness Endianness);
+ /*!
+ * Return the total size of \c ValueProfileData.
+ */
+ uint32_t getSize() const { return TotalSize; }
+ /*!
+ * Read data from this data and save it to \c Record.
+ */
+ void deserializeTo(InstrProfRecord &Record,
+ InstrProfRecord::ValueMapType *VMap);
+ void operator delete(void *ptr) { ::operator delete(ptr); }
+#endif
+} ValueProfData;
+
+/*
+ * The closure is designed to abstact away two types of value profile data:
+ * - InstrProfRecord which is the primary data structure used to
+ * represent profile data in host tools (reader, writer, and profile-use)
+ * - value profile runtime data structure suitable to be used by C
+ * runtime library.
+ *
+ * Both sources of data need to serialize to disk/memory-buffer in common
+ * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
+ * writer to share the same format and code with indexed profile writer.
+ *
+ * For documentation of the member methods below, refer to corresponding methods
+ * in class InstrProfRecord.
+ */
+typedef struct ValueProfRecordClosure {
+ const void *Record;
+ uint32_t (*GetNumValueKinds)(const void *Record);
+ uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
+ uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
+ uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
+
+ /*
+ * After extracting the value profile data from the value profile record,
+ * this method is used to map the in-memory value to on-disk value. If
+ * the method is null, value will be written out untranslated.
+ */
+ uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
+ void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
+ uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t));
+ ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
+} ValueProfRecordClosure;
+
+/*
+ * A wrapper struct that represents value profile runtime data.
+ * Like InstrProfRecord class which is used by profiling host tools,
+ * ValueProfRuntimeRecord also implements the abstract intefaces defined in
+ * ValueProfRecordClosure so that the runtime data can be serialized using
+ * shared C implementation. In this structure, NumValueSites and Nodes
+ * members are the primary fields while other fields hold the derived
+ * information for fast implementation of closure interfaces.
+ */
+typedef struct ValueProfRuntimeRecord {
+ /* Number of sites for each value profile kind. */
+ const uint16_t *NumValueSites;
+ /* An array of linked-list headers. The size of of the array is the
+ * total number of value profile sites : sum(NumValueSites[*])). Each
+ * linked-list stores the values profiled for a value profile site. */
+ ValueProfNode **Nodes;
+
+ /* Total number of value profile kinds which have at least one
+ * value profile sites. */
+ uint32_t NumValueKinds;
+ /* An array recording the number of values tracked at each site.
+ * The size of the array is TotalNumValueSites. */
+ uint8_t *SiteCountArray[IPVK_Last + 1];
+ ValueProfNode **NodesKind[IPVK_Last + 1];
+} ValueProfRuntimeRecord;
+
+/* Forward declarations of C interfaces. */
+int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
+ const uint16_t *NumValueSites,
+ ValueProfNode **Nodes);
+void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord);
+uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record);
+ValueProfData *
+serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
+ ValueProfData *Dst);
+uint32_t getNumValueKindsRT(const void *R);
+
+#undef INSTR_PROF_VALUE_PROF_DATA
+#endif /* INSTR_PROF_VALUE_PROF_DATA */
+
+
+#ifdef INSTR_PROF_COMMON_API_IMPL
+#define INSTR_PROF_DATA_DEFINED
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#else
+#define INSTR_PROF_INLINE
+#endif
+
+#ifndef offsetof
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif
+
+/*!
+ * \brief Return the \c ValueProfRecord header size including the
+ * padding bytes.
+ */
+INSTR_PROF_INLINE
+uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
+ uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
+ sizeof(uint8_t) * NumValueSites;
+ /* Round the size to multiple of 8 bytes. */
+ Size = (Size + 7) & ~7;
+ return Size;
+}
+
+/*!
+ * \brief Return the total size of the value profile record including the
+ * header and the value data.
+ */
+INSTR_PROF_INLINE
+uint32_t getValueProfRecordSize(uint32_t NumValueSites,
+ uint32_t NumValueData) {
+ return getValueProfRecordHeaderSize(NumValueSites) +
+ sizeof(InstrProfValueData) * NumValueData;
+}
+
+/*!
+ * \brief Return the pointer to the start of value data array.
+ */
+INSTR_PROF_INLINE
+InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
+ return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
+ This->NumValueSites));
+}
+
+/*!
+ * \brief Return the total number of value data for \c This record.
+ */
+INSTR_PROF_INLINE
+uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
+ uint32_t NumValueData = 0;
+ uint32_t I;
+ for (I = 0; I < This->NumValueSites; I++)
+ NumValueData += This->SiteCountArray[I];
+ return NumValueData;
+}
+
+/*!
+ * \brief Use this method to advance to the next \c This \c ValueProfRecord.
+ */
+INSTR_PROF_INLINE
+ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
+ uint32_t NumValueData = getValueProfRecordNumValueData(This);
+ return (ValueProfRecord *)((char *)This +
+ getValueProfRecordSize(This->NumValueSites,
+ NumValueData));
+}
+
+/*!
+ * \brief Return the first \c ValueProfRecord instance.
+ */
+INSTR_PROF_INLINE
+ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
+ return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
+}
+
+/* Closure based interfaces. */
+
+/*!
+ * Return the total size in bytes of the on-disk value profile data
+ * given the data stored in Record.
+ */
+uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) {
+ uint32_t Kind;
+ uint32_t TotalSize = sizeof(ValueProfData);
+ const void *Record = Closure->Record;
+ uint32_t NumValueKinds = Closure->GetNumValueKinds(Record);
+ if (NumValueKinds == 0)
+ return TotalSize;
+
+ for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
+ uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
+ if (!NumValueSites)
+ continue;
+ TotalSize += getValueProfRecordSize(NumValueSites,
+ Closure->GetNumValueData(Record, Kind));
+ }
+ return TotalSize;
+}
+
+/*!
+ * Extract value profile data of a function for the profile kind \c ValueKind
+ * from the \c Closure and serialize the data into \c This record instance.
+ */
+void serializeValueProfRecordFrom(ValueProfRecord *This,
+ ValueProfRecordClosure *Closure,
+ uint32_t ValueKind, uint32_t NumValueSites) {
+ uint32_t S;
+ const void *Record = Closure->Record;
+ This->Kind = ValueKind;
+ This->NumValueSites = NumValueSites;
+ InstrProfValueData *DstVD = getValueProfRecordValueData(This);
+
+ for (S = 0; S < NumValueSites; S++) {
+ uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
+ This->SiteCountArray[S] = ND;
+ Closure->GetValueForSite(Record, DstVD, ValueKind, S,
+ Closure->RemapValueData);
+ DstVD += ND;
+ }
+}
+
+/*!
+ * Extract value profile data of a function from the \c Closure
+ * and serialize the data into \c DstData if it is not NULL or heap
+ * memory allocated by the \c Closure's allocator method.
+ */
+ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
+ ValueProfData *DstData) {
+ uint32_t Kind;
+ uint32_t TotalSize = getValueProfDataSize(Closure);
+
+ ValueProfData *VPD =
+ DstData ? DstData : Closure->AllocValueProfData(TotalSize);
+
+ VPD->TotalSize = TotalSize;
+ VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
+ ValueProfRecord *VR = getFirstValueProfRecord(VPD);
+ for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
+ uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
+ if (!NumValueSites)
+ continue;
+ serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
+ VR = getValueProfRecordNext(VR);
+ }
+ return VPD;
+}
+
+/*
+ * The value profiler runtime library stores the value profile data
+ * for a given function in \c NumValueSites and \c Nodes structures.
+ * \c ValueProfRuntimeRecord class is used to encapsulate the runtime
+ * profile data and provides fast interfaces to retrieve the profile
+ * information. This interface is used to initialize the runtime record
+ * and pre-compute the information needed for efficient implementation
+ * of callbacks required by ValueProfRecordClosure class.
+ */
+int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
+ const uint16_t *NumValueSites,
+ ValueProfNode **Nodes) {
+ unsigned I, J, S = 0, NumValueKinds = 0;
+ RuntimeRecord->NumValueSites = NumValueSites;
+ RuntimeRecord->Nodes = Nodes;
+ for (I = 0; I <= IPVK_Last; I++) {
+ uint16_t N = NumValueSites[I];
+ if (!N) {
+ RuntimeRecord->SiteCountArray[I] = 0;
+ continue;
+ }
+ NumValueKinds++;
+ RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1);
+ if (!RuntimeRecord->SiteCountArray[I])
+ return 1;
+ RuntimeRecord->NodesKind[I] = Nodes ? &Nodes[S] : NULL;
+ for (J = 0; J < N; J++) {
+ /* Compute value count for each site. */
+ uint32_t C = 0;
+ ValueProfNode *Site = Nodes ? RuntimeRecord->NodesKind[I][J] : NULL;
+ while (Site) {
+ C++;
+ Site = Site->Next;
+ }
+ if (C > UCHAR_MAX)
+ C = UCHAR_MAX;
+ RuntimeRecord->SiteCountArray[I][J] = C;
+ }
+ S += N;
+ }
+ RuntimeRecord->NumValueKinds = NumValueKinds;
+ return 0;
+}
+
+void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) {
+ unsigned I;
+ for (I = 0; I <= IPVK_Last; I++) {
+ if (RuntimeRecord->SiteCountArray[I])
+ free(RuntimeRecord->SiteCountArray[I]);
+ }
+}
+
+/* ValueProfRecordClosure Interface implementation for
+ * ValueProfDataRuntimeRecord. */
+uint32_t getNumValueKindsRT(const void *R) {
+ return ((const ValueProfRuntimeRecord *)R)->NumValueKinds;
+}
+
+uint32_t getNumValueSitesRT(const void *R, uint32_t VK) {
+ return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK];
+}
+
+uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) {
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ return Record->SiteCountArray[VK][S];
+}
+
+uint32_t getNumValueDataRT(const void *R, uint32_t VK) {
+ unsigned I, S = 0;
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ if (Record->SiteCountArray[VK] == 0)
+ return 0;
+ for (I = 0; I < Record->NumValueSites[VK]; I++)
+ S += Record->SiteCountArray[VK][I];
+ return S;
+}
+
+void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK,
+ uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) {
+ unsigned I, N = 0;
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ N = getNumValueDataForSiteRT(R, VK, S);
+ if (N == 0)
+ return;
+ ValueProfNode *VNode = Record->NodesKind[VK][S];
+ for (I = 0; I < N; I++) {
+ Dst[I] = VNode->VData;
+ VNode = VNode->Next;
+ }
+}
+
+ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) {
+ return (ValueProfData *)calloc(TotalSizeInBytes, 1);
+}
+
+static ValueProfRecordClosure RTRecordClosure = {0,
+ getNumValueKindsRT,
+ getNumValueSitesRT,
+ getNumValueDataRT,
+ getNumValueDataForSiteRT,
+ 0,
+ getValueForSiteRT,
+ allocValueProfDataRT};
+
+/*
+ * Return the size of ValueProfData structure to store data
+ * recorded in the runtime record.
+ */
+uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) {
+ RTRecordClosure.Record = Record;
+ return getValueProfDataSize(&RTRecordClosure);
+}
+
+/*
+ * Return a ValueProfData instance that stores the data collected
+ * from runtime. If \c DstData is provided by the caller, the value
+ * profile data will be store in *DstData and DstData is returned,
+ * otherwise the method will allocate space for the value data and
+ * return pointer to the newly allocated space.
+ */
+ValueProfData *
+serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
+ ValueProfData *DstData) {
+ RTRecordClosure.Record = Record;
+ return serializeValueProfDataFrom(&RTRecordClosure, DstData);
+}
+
+
+#undef INSTR_PROF_COMMON_API_IMPL
+#endif /* INSTR_PROF_COMMON_API_IMPL */
+
+/*============================================================================*/
+
+
+#ifndef INSTR_PROF_DATA_DEFINED
+
+#ifndef INSTR_PROF_DATA_INC_
+#define INSTR_PROF_DATA_INC_
+
+/* Helper macros. */
+#define INSTR_PROF_SIMPLE_QUOTE(x) #x
+#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
+#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
+#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
+
+/* Magic number to detect file format and endianness.
+ * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
+ * so that utilities, like strings, don't grab it as a string. 129 is also
+ * invalid UTF-8, and high enough to be interesting.
+ * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
+ * for 32-bit platforms.
+ */
+#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
+ (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
+ (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
+#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
+ (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
+ (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
+
+/* Raw profile format version. */
+#define INSTR_PROF_RAW_VERSION 2
+
+/* Runtime section names and name strings. */
+#define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data
+#define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names
+#define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts
+
+#define INSTR_PROF_DATA_SECT_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME)
+#define INSTR_PROF_NAME_SECT_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME)
+#define INSTR_PROF_CNTS_SECT_NAME_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME)
+
+/* Macros to define start/stop section symbol for a given
+ * section on Linux. For instance
+ * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
+ * expand to __start___llvm_prof_data
+ */
+#define INSTR_PROF_SECT_START(Sect) \
+ INSTR_PROF_CONCAT(__start_,Sect)
+#define INSTR_PROF_SECT_STOP(Sect) \
+ INSTR_PROF_CONCAT(__stop_,Sect)
+
+/* Value Profiling API linkage name. */
+#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
+#define INSTR_PROF_VALUE_PROF_FUNC_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
+
+/* InstrProfile per-function control data alignment. */
+#define INSTR_PROF_DATA_ALIGNMENT 8
+
+/* The data structure that represents a tracked value by the
+ * value profiler.
+ */
+typedef struct InstrProfValueData {
+ /* Profiled value. */
+ uint64_t Value;
+ /* Number of times the value appears in the training run. */
+ uint64_t Count;
+} InstrProfValueData;
+
+/* This is an internal data structure used by value profiler. It
+ * is defined here to allow serialization code sharing by LLVM
+ * to be used in unit test.
+ */
+typedef struct ValueProfNode {
+ InstrProfValueData VData;
+ struct ValueProfNode *Next;
+} ValueProfNode;
+
+#endif /* INSTR_PROF_DATA_INC_ */
+
+#else
+#undef INSTR_PROF_DATA_DEFINED
+#endif
+
diff --git a/lib/profile/InstrProfiling.c b/lib/profile/InstrProfiling.c
index 8d010df28f18..58778aeec16a 100644
--- a/lib/profile/InstrProfiling.c
+++ b/lib/profile/InstrProfiling.c
@@ -8,41 +8,61 @@
\*===----------------------------------------------------------------------===*/
#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
#include <string.h>
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "InstrProfData.inc"
-__attribute__((visibility("hidden")))
-uint64_t __llvm_profile_get_magic(void) {
- /* Magic number to detect file format and endianness.
- *
- * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
- * so that utilities, like strings, don't grab it as a string. 129 is also
- * invalid UTF-8, and high enough to be interesting.
- *
- * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
- * for 32-bit platforms.
- */
- unsigned char R = sizeof(void *) == sizeof(uint64_t) ? 'r' : 'R';
- return
- (uint64_t)255 << 56 |
- (uint64_t)'l' << 48 |
- (uint64_t)'p' << 40 |
- (uint64_t)'r' << 32 |
- (uint64_t)'o' << 24 |
- (uint64_t)'f' << 16 |
- (uint64_t) R << 8 |
- (uint64_t)129;
+char *(*GetEnvHook)(const char *) = 0;
+
+COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) {
+ return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64)
+ : (INSTR_PROF_RAW_MAGIC_32);
+}
+
+/* Return the number of bytes needed to add to SizeInBytes to make it
+ * the result a multiple of 8.
+ */
+COMPILER_RT_VISIBILITY uint8_t
+__llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) {
+ return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t));
}
-__attribute__((visibility("hidden")))
-uint64_t __llvm_profile_get_version(void) {
- /* This should be bumped any time the output format changes. */
- return 1;
+COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) {
+ return INSTR_PROF_RAW_VERSION;
}
-__attribute__((visibility("hidden")))
-void __llvm_profile_reset_counters(void) {
+COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) {
uint64_t *I = __llvm_profile_begin_counters();
uint64_t *E = __llvm_profile_end_counters();
- memset(I, 0, sizeof(uint64_t)*(E - I));
+ memset(I, 0, sizeof(uint64_t) * (E - I));
+
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const __llvm_profile_data *DI;
+ for (DI = DataBegin; DI != DataEnd; ++DI) {
+ uint64_t CurrentVSiteCount = 0;
+ uint32_t VKI, i;
+ if (!DI->Values)
+ continue;
+
+ ValueProfNode **ValueCounters = (ValueProfNode **)DI->Values;
+
+ for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
+ CurrentVSiteCount += DI->NumValueSites[VKI];
+
+ for (i = 0; i < CurrentVSiteCount; ++i) {
+ ValueProfNode *CurrentVNode = ValueCounters[i];
+
+ while (CurrentVNode) {
+ CurrentVNode->VData.Count = 0;
+ CurrentVNode = CurrentVNode->Next;
+ }
+ }
+ }
}
+
diff --git a/lib/profile/InstrProfiling.h b/lib/profile/InstrProfiling.h
index 3778a88893e6..d27ca569d535 100644
--- a/lib/profile/InstrProfiling.h
+++ b/lib/profile/InstrProfiling.h
@@ -10,32 +10,31 @@
#ifndef PROFILE_INSTRPROFILING_H_
#define PROFILE_INSTRPROFILING_H_
-#if defined(__FreeBSD__) && defined(__i386__)
-
-/* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to
- * FreeBSD 10, r232261) when compiled in 32-bit mode.
- */
-#define PRIu64 "llu"
-typedef unsigned int uint32_t;
-typedef unsigned long long uint64_t;
-typedef uint32_t uintptr_t;
-
-#else /* defined(__FreeBSD__) && defined(__i386__) */
-
-#include <inttypes.h>
-#include <stdint.h>
-
-#endif /* defined(__FreeBSD__) && defined(__i386__) */
+#include "InstrProfilingPort.h"
+#include "InstrProfData.inc"
+
+enum ValueKind {
+#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value,
+#include "InstrProfData.inc"
+};
+
+typedef void *IntPtrT;
+typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT)
+ __llvm_profile_data {
+#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) Type Name;
+#include "InstrProfData.inc"
+} __llvm_profile_data;
-#define PROFILE_HEADER_SIZE 7
+typedef struct __llvm_profile_header {
+#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) Type Name;
+#include "InstrProfData.inc"
+} __llvm_profile_header;
-typedef struct __llvm_profile_data {
- const uint32_t NameSize;
- const uint32_t NumCounters;
- const uint64_t FuncHash;
- const char *const Name;
- uint64_t *const Counters;
-} __llvm_profile_data;
+/*!
+ * \brief Get number of bytes necessary to pad the argument to eight
+ * byte boundary.
+ */
+uint8_t __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes);
/*!
* \brief Get required size for profile buffer.
@@ -58,9 +57,37 @@ uint64_t *__llvm_profile_begin_counters(void);
uint64_t *__llvm_profile_end_counters(void);
/*!
+ * \brief Clear profile counters to zero.
+ *
+ */
+void __llvm_profile_reset_counters(void);
+
+/*!
+ * \brief Counts the number of times a target value is seen.
+ *
+ * Records the target value for the CounterIndex if not seen before. Otherwise,
+ * increments the counter associated w/ the target value.
+ * void __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
+ * uint32_t CounterIndex);
+ */
+void INSTR_PROF_VALUE_PROF_FUNC(
+#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName
+#include "InstrProfData.inc"
+);
+
+/*!
+ * \brief Prepares the value profiling data for output.
+ *
+ * Returns an array of pointers to value profile data.
+ */
+struct ValueProfData;
+struct ValueProfData **__llvm_profile_gather_value_data(uint64_t *Size);
+
+/*!
* \brief Write instrumentation data to the current file.
*
- * Writes to the file with the last name given to \a __llvm_profile_set_filename(),
+ * Writes to the file with the last name given to \a *
+ * __llvm_profile_set_filename(),
* or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable,
* or if that's not set, the last name given to
* \a __llvm_profile_override_default_filename(), or if that's not set,
diff --git a/lib/profile/InstrProfilingBuffer.c b/lib/profile/InstrProfilingBuffer.c
index 3c429c8a85ea..4227ca6b66ea 100644
--- a/lib/profile/InstrProfilingBuffer.c
+++ b/lib/profile/InstrProfilingBuffer.c
@@ -10,9 +10,7 @@
#include "InstrProfiling.h"
#include "InstrProfilingInternal.h"
-#include <string.h>
-
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_size_for_buffer(void) {
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
@@ -27,78 +25,28 @@ uint64_t __llvm_profile_get_size_for_buffer(void) {
#define PROFILE_RANGE_SIZE(Range) (Range##End - Range##Begin)
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_size_for_buffer_internal(
- const __llvm_profile_data *DataBegin,
- const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin,
- const uint64_t *CountersEnd, const char *NamesBegin,
- const char *NamesEnd) {
+ const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
+ const uint64_t *CountersBegin, const uint64_t *CountersEnd,
+ const char *NamesBegin, const char *NamesEnd) {
/* Match logic in __llvm_profile_write_buffer(). */
const uint64_t NamesSize = PROFILE_RANGE_SIZE(Names) * sizeof(char);
- const uint64_t Padding = sizeof(uint64_t) - NamesSize % sizeof(uint64_t);
- return sizeof(uint64_t) * PROFILE_HEADER_SIZE +
- PROFILE_RANGE_SIZE(Data) * sizeof(__llvm_profile_data) +
- PROFILE_RANGE_SIZE(Counters) * sizeof(uint64_t) +
- NamesSize + Padding;
+ const uint8_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize);
+ return sizeof(__llvm_profile_header) +
+ PROFILE_RANGE_SIZE(Data) * sizeof(__llvm_profile_data) +
+ PROFILE_RANGE_SIZE(Counters) * sizeof(uint64_t) + NamesSize + Padding;
}
-__attribute__((visibility("hidden")))
-int __llvm_profile_write_buffer(char *Buffer) {
- /* Match logic in __llvm_profile_get_size_for_buffer().
- * Match logic in __llvm_profile_write_file().
- */
- const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
- const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- const uint64_t *CountersBegin = __llvm_profile_begin_counters();
- const uint64_t *CountersEnd = __llvm_profile_end_counters();
- const char *NamesBegin = __llvm_profile_begin_names();
- const char *NamesEnd = __llvm_profile_end_names();
-
- return __llvm_profile_write_buffer_internal(Buffer, DataBegin, DataEnd,
- CountersBegin, CountersEnd,
- NamesBegin, NamesEnd);
+COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) {
+ return llvmWriteProfData(llvmBufferWriter, Buffer, 0, 0);
}
-__attribute__((visibility("hidden")))
-int __llvm_profile_write_buffer_internal(
+COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
char *Buffer, const __llvm_profile_data *DataBegin,
const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin,
const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd) {
- /* Match logic in __llvm_profile_get_size_for_buffer().
- * Match logic in __llvm_profile_write_file().
- */
-
- /* Calculate size of sections. */
- const uint64_t DataSize = DataEnd - DataBegin;
- const uint64_t CountersSize = CountersEnd - CountersBegin;
- const uint64_t NamesSize = NamesEnd - NamesBegin;
- const uint64_t Padding = sizeof(uint64_t) - NamesSize % sizeof(uint64_t);
-
- /* Enough zeroes for padding. */
- const char Zeroes[sizeof(uint64_t)] = {0};
-
- /* Create the header. */
- uint64_t Header[PROFILE_HEADER_SIZE];
- Header[0] = __llvm_profile_get_magic();
- Header[1] = __llvm_profile_get_version();
- Header[2] = DataSize;
- Header[3] = CountersSize;
- Header[4] = NamesSize;
- Header[5] = (uintptr_t)CountersBegin;
- Header[6] = (uintptr_t)NamesBegin;
-
- /* Write the data. */
-#define UPDATE_memcpy(Data, Size) \
- do { \
- memcpy(Buffer, Data, Size); \
- Buffer += Size; \
- } while (0)
- UPDATE_memcpy(Header, PROFILE_HEADER_SIZE * sizeof(uint64_t));
- UPDATE_memcpy(DataBegin, DataSize * sizeof(__llvm_profile_data));
- UPDATE_memcpy(CountersBegin, CountersSize * sizeof(uint64_t));
- UPDATE_memcpy(NamesBegin, NamesSize * sizeof(char));
- UPDATE_memcpy(Zeroes, Padding * sizeof(char));
-#undef UPDATE_memcpy
-
- return 0;
+ return llvmWriteProfDataImpl(llvmBufferWriter, Buffer, DataBegin, DataEnd,
+ CountersBegin, CountersEnd, 0, 0, NamesBegin,
+ NamesEnd);
}
diff --git a/lib/profile/InstrProfilingFile.c b/lib/profile/InstrProfilingFile.c
index 68e8c7b07871..4ea7fbf9738a 100644
--- a/lib/profile/InstrProfilingFile.c
+++ b/lib/profile/InstrProfilingFile.c
@@ -8,6 +8,7 @@
\*===----------------------------------------------------------------------===*/
#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
#include "InstrProfilingUtil.h"
#include <errno.h>
#include <stdio.h>
@@ -16,47 +17,39 @@
#define UNCONST(ptr) ((void *)(uintptr_t)(ptr))
-static int writeFile(FILE *File) {
- /* Match logic in __llvm_profile_write_buffer(). */
- const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
- const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- const uint64_t *CountersBegin = __llvm_profile_begin_counters();
- const uint64_t *CountersEnd = __llvm_profile_end_counters();
- const char *NamesBegin = __llvm_profile_begin_names();
- const char *NamesEnd = __llvm_profile_end_names();
-
- /* Calculate size of sections. */
- const uint64_t DataSize = DataEnd - DataBegin;
- const uint64_t CountersSize = CountersEnd - CountersBegin;
- const uint64_t NamesSize = NamesEnd - NamesBegin;
- const uint64_t Padding = sizeof(uint64_t) - NamesSize % sizeof(uint64_t);
-
- /* Enough zeroes for padding. */
- const char Zeroes[sizeof(uint64_t)] = {0};
-
- /* Create the header. */
- uint64_t Header[PROFILE_HEADER_SIZE];
- Header[0] = __llvm_profile_get_magic();
- Header[1] = __llvm_profile_get_version();
- Header[2] = DataSize;
- Header[3] = CountersSize;
- Header[4] = NamesSize;
- Header[5] = (uintptr_t)CountersBegin;
- Header[6] = (uintptr_t)NamesBegin;
-
- /* Write the data. */
-#define CHECK_fwrite(Data, Size, Length, File) \
- do { if (fwrite(Data, Size, Length, File) != Length) return -1; } while (0)
- CHECK_fwrite(Header, sizeof(uint64_t), PROFILE_HEADER_SIZE, File);
- CHECK_fwrite(DataBegin, sizeof(__llvm_profile_data), DataSize, File);
- CHECK_fwrite(CountersBegin, sizeof(uint64_t), CountersSize, File);
- CHECK_fwrite(NamesBegin, sizeof(char), NamesSize, File);
- CHECK_fwrite(Zeroes, sizeof(char), Padding, File);
-#undef CHECK_fwrite
-
+/* Return 1 if there is an error, otherwise return 0. */
+static uint32_t fileWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs,
+ void **WriterCtx) {
+ uint32_t I;
+ FILE *File = (FILE *)*WriterCtx;
+ for (I = 0; I < NumIOVecs; I++) {
+ if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) !=
+ IOVecs[I].NumElm)
+ return 1;
+ }
return 0;
}
+COMPILER_RT_VISIBILITY ProfBufferIO *
+llvmCreateBufferIOInternal(void *File, uint32_t BufferSz) {
+ CallocHook = calloc;
+ FreeHook = free;
+ return llvmCreateBufferIO(fileWriter, File, BufferSz);
+}
+
+static int writeFile(FILE *File) {
+ const char *BufferSzStr = 0;
+ uint64_t ValueDataSize = 0;
+ struct ValueProfData **ValueDataArray =
+ __llvm_profile_gather_value_data(&ValueDataSize);
+ FreeHook = &free;
+ CallocHook = &calloc;
+ BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
+ if (BufferSzStr && BufferSzStr[0])
+ VPBufferSize = atoi(BufferSzStr);
+ return llvmWriteProfData(fileWriter, File, ValueDataArray, ValueDataSize);
+}
+
static int writeFileWithName(const char *OutputName) {
int RetVal;
FILE *OutputFile;
@@ -64,7 +57,7 @@ static int writeFileWithName(const char *OutputName) {
return -1;
/* Append to the file to support profiling multiple shared objects. */
- OutputFile = fopen(OutputName, "a");
+ OutputFile = fopen(OutputName, "ab");
if (!OutputFile)
return -1;
@@ -74,8 +67,8 @@ static int writeFileWithName(const char *OutputName) {
return RetVal;
}
-__attribute__((weak)) int __llvm_profile_OwnsFilename = 0;
-__attribute__((weak)) const char *__llvm_profile_CurrentFilename = NULL;
+COMPILER_RT_WEAK int __llvm_profile_OwnsFilename = 0;
+COMPILER_RT_WEAK const char *__llvm_profile_CurrentFilename = NULL;
static void truncateCurrentFile(void) {
const char *Filename;
@@ -182,7 +175,7 @@ static void setFilenameAutomatically(void) {
resetFilenameToDefault();
}
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
void __llvm_profile_initialize_file(void) {
/* Check if the filename has been initialized. */
if (__llvm_profile_CurrentFilename)
@@ -192,12 +185,12 @@ void __llvm_profile_initialize_file(void) {
setFilenameAutomatically();
}
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
void __llvm_profile_set_filename(const char *Filename) {
setFilenamePossiblyWithPid(Filename);
}
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
void __llvm_profile_override_default_filename(const char *Filename) {
/* If the env var is set, skip setting filename from argument. */
const char *Env_Filename = getenv("LLVM_PROFILE_FILE");
@@ -206,27 +199,28 @@ void __llvm_profile_override_default_filename(const char *Filename) {
setFilenamePossiblyWithPid(Filename);
}
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
int __llvm_profile_write_file(void) {
int rc;
+ GetEnvHook = &getenv;
/* Check the filename. */
- if (!__llvm_profile_CurrentFilename)
+ if (!__llvm_profile_CurrentFilename) {
+ PROF_ERR("LLVM Profile: Failed to write file : %s\n", "Filename not set");
return -1;
+ }
/* Write the file. */
rc = writeFileWithName(__llvm_profile_CurrentFilename);
- if (rc && getenv("LLVM_PROFILE_VERBOSE_ERRORS"))
- fprintf(stderr, "LLVM Profile: Failed to write file \"%s\": %s\n",
+ if (rc)
+ PROF_ERR("LLVM Profile: Failed to write file \"%s\": %s\n",
__llvm_profile_CurrentFilename, strerror(errno));
return rc;
}
-static void writeFileWithoutReturn(void) {
- __llvm_profile_write_file();
-}
+static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
int __llvm_profile_register_write_file_atexit(void) {
static int HasBeenRegistered = 0;
diff --git a/lib/profile/InstrProfilingInternal.h b/lib/profile/InstrProfilingInternal.h
index ede39cd9d713..4aab78ea509c 100644
--- a/lib/profile/InstrProfilingInternal.h
+++ b/lib/profile/InstrProfilingInternal.h
@@ -11,6 +11,7 @@
#define PROFILE_INSTRPROFILING_INTERNALH_
#include "InstrProfiling.h"
+#include "stddef.h"
/*!
* \brief Write instrumentation data to the given buffer, given explicit
@@ -37,4 +38,81 @@ int __llvm_profile_write_buffer_internal(
const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin,
const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd);
+/*!
+ * The data structure describing the data to be written by the
+ * low level writer callback function.
+ */
+typedef struct ProfDataIOVec {
+ const void *Data;
+ size_t ElmSize;
+ size_t NumElm;
+} ProfDataIOVec;
+
+typedef uint32_t (*WriterCallback)(ProfDataIOVec *, uint32_t NumIOVecs,
+ void **WriterCtx);
+
+/*!
+ * The data structure for buffered IO of profile data.
+ */
+typedef struct ProfBufferIO {
+ /* File handle. */
+ void *File;
+ /* Low level IO callback. */
+ WriterCallback FileWriter;
+ /* The start of the buffer. */
+ uint8_t *BufferStart;
+ /* Total size of the buffer. */
+ uint32_t BufferSz;
+ /* Current byte offset from the start of the buffer. */
+ uint32_t CurOffset;
+} ProfBufferIO;
+
+/* The creator interface used by testing. */
+ProfBufferIO *llvmCreateBufferIOInternal(void *File, uint32_t DefaultBufferSz);
+/*!
+ * This is the interface to create a handle for buffered IO.
+ */
+ProfBufferIO *llvmCreateBufferIO(WriterCallback FileWriter, void *File,
+ uint32_t DefaultBufferSz);
+/*!
+ * The interface to destroy the bufferIO handle and reclaim
+ * the memory.
+ */
+void llvmDeleteBufferIO(ProfBufferIO *BufferIO);
+
+/*!
+ * This is the interface to write \c Data of \c Size bytes through
+ * \c BufferIO. Returns 0 if successful, otherwise return -1.
+ */
+int llvmBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data,
+ uint32_t Size);
+/*!
+ * The interface to flush the remaining data in the buffer.
+ * through the low level writer callback.
+ */
+int llvmBufferIOFlush(ProfBufferIO *BufferIO);
+
+/* The low level interface to write data into a buffer. It is used as the
+ * callback by other high level writer methods such as buffered IO writer
+ * and profile data writer. */
+uint32_t llvmBufferWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs,
+ void **WriterCtx);
+
+int llvmWriteProfData(WriterCallback Writer, void *WriterCtx,
+ struct ValueProfData **ValueDataArray,
+ const uint64_t ValueDataSize);
+int llvmWriteProfDataImpl(WriterCallback Writer, void *WriterCtx,
+ const __llvm_profile_data *DataBegin,
+ const __llvm_profile_data *DataEnd,
+ const uint64_t *CountersBegin,
+ const uint64_t *CountersEnd,
+ struct ValueProfData **ValueDataBeginArray,
+ const uint64_t ValueDataSize, const char *NamesBegin,
+ const char *NamesEnd);
+
+extern char *(*GetEnvHook)(const char *);
+extern void (*FreeHook)(void *);
+extern void* (*CallocHook)(size_t, size_t);
+extern uint32_t VPBufferSize;
+
#endif
diff --git a/lib/profile/InstrProfilingPlatformDarwin.c b/lib/profile/InstrProfilingPlatformDarwin.c
index 02299cc4630c..30ddbd2e4982 100644
--- a/lib/profile/InstrProfilingPlatformDarwin.c
+++ b/lib/profile/InstrProfilingPlatformDarwin.c
@@ -11,33 +11,36 @@
#if defined(__APPLE__)
/* Use linker magic to find the bounds of the Data section. */
-__attribute__((visibility("hidden")))
-extern __llvm_profile_data DataStart __asm("section$start$__DATA$__llvm_prf_data");
-__attribute__((visibility("hidden")))
-extern __llvm_profile_data DataEnd __asm("section$end$__DATA$__llvm_prf_data");
-__attribute__((visibility("hidden")))
-extern char NamesStart __asm("section$start$__DATA$__llvm_prf_names");
-__attribute__((visibility("hidden")))
-extern char NamesEnd __asm("section$end$__DATA$__llvm_prf_names");
-__attribute__((visibility("hidden")))
-extern uint64_t CountersStart __asm("section$start$__DATA$__llvm_prf_cnts");
-__attribute__((visibility("hidden")))
-extern uint64_t CountersEnd __asm("section$end$__DATA$__llvm_prf_cnts");
+COMPILER_RT_VISIBILITY
+extern __llvm_profile_data
+ DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR);
+COMPILER_RT_VISIBILITY
+extern __llvm_profile_data
+ DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR);
+COMPILER_RT_VISIBILITY
+extern char
+ NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR);
+COMPILER_RT_VISIBILITY
+extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR);
+COMPILER_RT_VISIBILITY
+extern uint64_t
+ CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR);
+COMPILER_RT_VISIBILITY
+extern uint64_t
+ CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR);
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
const __llvm_profile_data *__llvm_profile_begin_data(void) {
return &DataStart;
}
-__attribute__((visibility("hidden")))
-const __llvm_profile_data *__llvm_profile_end_data(void) {
- return &DataEnd;
-}
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
+const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; }
+COMPILER_RT_VISIBILITY
const char *__llvm_profile_begin_names(void) { return &NamesStart; }
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
const char *__llvm_profile_end_names(void) { return &NamesEnd; }
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; }
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; }
#endif
diff --git a/lib/profile/InstrProfilingPlatformLinux.c b/lib/profile/InstrProfilingPlatformLinux.c
new file mode 100644
index 000000000000..7843f47caa1b
--- /dev/null
+++ b/lib/profile/InstrProfilingPlatformLinux.c
@@ -0,0 +1,59 @@
+/*===- InstrProfilingPlatformLinux.c - Profile data Linux platform ------===*\
+|*
+|* The LLVM Compiler Infrastructure
+|*
+|* This file is distributed under the University of Illinois Open Source
+|* License. See LICENSE.TXT for details.
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+
+#if defined(__linux__) || defined(__FreeBSD__)
+#include <stdlib.h>
+
+#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME)
+#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_SECT_NAME)
+#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_SECT_NAME)
+#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_SECT_NAME)
+#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_SECT_NAME)
+#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_SECT_NAME)
+
+/* Declare section start and stop symbols for various sections
+ * generated by compiler instrumentation.
+ */
+extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY;
+extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY;
+extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY;
+extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY;
+extern char PROF_NAME_START COMPILER_RT_VISIBILITY;
+extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY;
+
+/* Add dummy data to ensure the section is always created. */
+__llvm_profile_data
+ __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME_STR);
+uint64_t
+ __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME_STR);
+char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME_STR);
+
+COMPILER_RT_VISIBILITY const __llvm_profile_data *
+__llvm_profile_begin_data(void) {
+ return &PROF_DATA_START;
+}
+COMPILER_RT_VISIBILITY const __llvm_profile_data *
+__llvm_profile_end_data(void) {
+ return &PROF_DATA_STOP;
+}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) {
+ return &PROF_NAME_START;
+}
+COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) {
+ return &PROF_NAME_STOP;
+}
+COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) {
+ return &PROF_CNTS_START;
+}
+COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) {
+ return &PROF_CNTS_STOP;
+}
+#endif
diff --git a/lib/profile/InstrProfilingPlatformOther.c b/lib/profile/InstrProfilingPlatformOther.c
index 548d6a396b76..58ceb3458a0a 100644
--- a/lib/profile/InstrProfilingPlatformOther.c
+++ b/lib/profile/InstrProfilingPlatformOther.c
@@ -9,7 +9,7 @@
#include "InstrProfiling.h"
-#if !defined(__APPLE__)
+#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__)
#include <stdlib.h>
static const __llvm_profile_data *DataFirst = NULL;
@@ -26,49 +26,43 @@ static uint64_t *CountersLast = NULL;
* calls are only required (and only emitted) on targets where we haven't
* implemented linker magic to find the bounds of the sections.
*/
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
void __llvm_profile_register_function(void *Data_) {
/* TODO: Only emit this function if we can't use linker magic. */
- const __llvm_profile_data *Data = (__llvm_profile_data*)Data_;
+ const __llvm_profile_data *Data = (__llvm_profile_data *)Data_;
if (!DataFirst) {
DataFirst = Data;
DataLast = Data + 1;
- NamesFirst = Data->Name;
- NamesLast = Data->Name + Data->NameSize;
- CountersFirst = Data->Counters;
- CountersLast = Data->Counters + Data->NumCounters;
+ NamesFirst = Data->NamePtr;
+ NamesLast = (const char *)Data->NamePtr + Data->NameSize;
+ CountersFirst = Data->CounterPtr;
+ CountersLast = (uint64_t *)Data->CounterPtr + Data->NumCounters;
return;
}
-#define UPDATE_FIRST(First, New) \
- First = New < First ? New : First
+#define UPDATE_FIRST(First, New) First = New < First ? New : First
UPDATE_FIRST(DataFirst, Data);
- UPDATE_FIRST(NamesFirst, Data->Name);
- UPDATE_FIRST(CountersFirst, Data->Counters);
+ UPDATE_FIRST(NamesFirst, (const char *)Data->NamePtr);
+ UPDATE_FIRST(CountersFirst, (uint64_t *)Data->CounterPtr);
#undef UPDATE_FIRST
-#define UPDATE_LAST(Last, New) \
- Last = New > Last ? New : Last
+#define UPDATE_LAST(Last, New) Last = New > Last ? New : Last
UPDATE_LAST(DataLast, Data + 1);
- UPDATE_LAST(NamesLast, Data->Name + Data->NameSize);
- UPDATE_LAST(CountersLast, Data->Counters + Data->NumCounters);
+ UPDATE_LAST(NamesLast, (const char *)Data->NamePtr + Data->NameSize);
+ UPDATE_LAST(CountersLast, (uint64_t *)Data->CounterPtr + Data->NumCounters);
#undef UPDATE_LAST
}
-__attribute__((visibility("hidden")))
-const __llvm_profile_data *__llvm_profile_begin_data(void) {
- return DataFirst;
-}
-__attribute__((visibility("hidden")))
-const __llvm_profile_data *__llvm_profile_end_data(void) {
- return DataLast;
-}
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
+const __llvm_profile_data *__llvm_profile_begin_data(void) { return DataFirst; }
+COMPILER_RT_VISIBILITY
+const __llvm_profile_data *__llvm_profile_end_data(void) { return DataLast; }
+COMPILER_RT_VISIBILITY
const char *__llvm_profile_begin_names(void) { return NamesFirst; }
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
const char *__llvm_profile_end_names(void) { return NamesLast; }
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; }
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
uint64_t *__llvm_profile_end_counters(void) { return CountersLast; }
#endif
diff --git a/lib/profile/InstrProfilingPort.h b/lib/profile/InstrProfilingPort.h
new file mode 100644
index 000000000000..da4f18fcbb46
--- /dev/null
+++ b/lib/profile/InstrProfilingPort.h
@@ -0,0 +1,76 @@
+/*===- InstrProfilingPort.h- Support library for PGO instrumentation ------===*\
+|*
+|* The LLVM Compiler Infrastructure
+|*
+|* This file is distributed under the University of Illinois Open Source
+|* License. See LICENSE.TXT for details.
+|*
+\*===----------------------------------------------------------------------===*/
+
+#ifndef PROFILE_INSTRPROFILING_PORT_H_
+#define PROFILE_INSTRPROFILING_PORT_H_
+
+#ifdef _MSC_VER
+#define COMPILER_RT_ALIGNAS(x) __declspec(align(x))
+#define COMPILER_RT_VISIBILITY
+#define COMPILER_RT_WEAK __declspec(selectany)
+#elif __GNUC__
+#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x)))
+#define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden")))
+#define COMPILER_RT_WEAK __attribute__((weak))
+#endif
+
+#define COMPILER_RT_SECTION(Sect) __attribute__((section(Sect)))
+
+#if COMPILER_RT_HAS_ATOMICS == 1
+#ifdef _MSC_VER
+#include <windows.h>
+#if defined(_WIN64)
+#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \
+ (InterlockedCompareExchange64((LONGLONG volatile *)Ptr, (LONGLONG)NewV, \
+ (LONGLONG)OldV) == (LONGLONG)OldV)
+#else
+#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \
+ (InterlockedCompareExchange((LONG volatile *)Ptr, (LONG)NewV, (LONG)OldV) == \
+ (LONG)OldV)
+#endif
+#else
+#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \
+ __sync_bool_compare_and_swap(Ptr, OldV, NewV)
+#endif
+#else
+#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \
+ BoolCmpXchg((void **)Ptr, OldV, NewV)
+#endif
+
+#define PROF_ERR(Format, ...) \
+ if (GetEnvHook && GetEnvHook("LLVM_PROFILE_VERBOSE_ERRORS")) \
+ fprintf(stderr, Format, __VA_ARGS__);
+
+#if defined(__FreeBSD__) && defined(__i386__)
+
+/* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to
+ * FreeBSD 10, r232261) when compiled in 32-bit mode.
+ */
+#define PRIu64 "llu"
+typedef unsigned char uint8_t;
+typedef unsigned short uint16_t;
+typedef unsigned int uint32_t;
+typedef unsigned long long uint64_t;
+typedef uint32_t uintptr_t;
+#elif defined(__FreeBSD__) && defined(__x86_64__)
+#define PRIu64 "lu"
+typedef unsigned char uint8_t;
+typedef unsigned short uint16_t;
+typedef unsigned int uint32_t;
+typedef unsigned long long uint64_t;
+typedef unsigned long int uintptr_t;
+
+#else /* defined(__FreeBSD__) && defined(__i386__) */
+
+#include <inttypes.h>
+#include <stdint.h>
+
+#endif /* defined(__FreeBSD__) && defined(__i386__) */
+
+#endif /* PROFILE_INSTRPROFILING_PORT_H_ */
diff --git a/lib/profile/InstrProfilingRuntime.cc b/lib/profile/InstrProfilingRuntime.cc
index 081ecb29e987..12ad9f1573f4 100644
--- a/lib/profile/InstrProfilingRuntime.cc
+++ b/lib/profile/InstrProfilingRuntime.cc
@@ -11,8 +11,7 @@ extern "C" {
#include "InstrProfiling.h"
-__attribute__((visibility("hidden"))) int __llvm_profile_runtime;
-
+COMPILER_RT_VISIBILITY int __llvm_profile_runtime;
}
namespace {
diff --git a/lib/profile/InstrProfilingUtil.c b/lib/profile/InstrProfilingUtil.c
index e146dfca83c8..6f0443d3bb5d 100644
--- a/lib/profile/InstrProfilingUtil.c
+++ b/lib/profile/InstrProfilingUtil.c
@@ -8,6 +8,7 @@
\*===----------------------------------------------------------------------===*/
#include "InstrProfilingUtil.h"
+#include "InstrProfiling.h"
#ifdef _WIN32
#include <direct.h>
@@ -18,7 +19,7 @@ int mkdir(const char*, unsigned short);
#include <sys/types.h>
#endif
-__attribute__((visibility("hidden")))
+COMPILER_RT_VISIBILITY
void __llvm_profile_recursive_mkdir(char *path) {
int i;
diff --git a/lib/profile/InstrProfilingValue.c b/lib/profile/InstrProfilingValue.c
new file mode 100644
index 000000000000..39b4da446a81
--- /dev/null
+++ b/lib/profile/InstrProfilingValue.c
@@ -0,0 +1,180 @@
+/*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\
+|*
+|* The LLVM Compiler Infrastructure
+|*
+|* This file is distributed under the University of Illinois Open Source
+|* License. See LICENSE.TXT for details.
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#define INSTR_PROF_VALUE_PROF_DATA
+#define INSTR_PROF_COMMON_API_IMPL
+#include "InstrProfData.inc"
+
+#define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
+#define PROF_OOM_RETURN(Msg) \
+ { \
+ PROF_OOM(Msg) \
+ free(ValueDataArray); \
+ return NULL; \
+ }
+
+#if COMPILER_RT_HAS_ATOMICS != 1
+COMPILER_RT_VISIBILITY
+uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
+ void *R = *Ptr;
+ if (R == OldV) {
+ *Ptr = NewV;
+ return 1;
+ }
+ return 0;
+}
+#endif
+
+/* This method is only used in value profiler mock testing. */
+COMPILER_RT_VISIBILITY void
+__llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
+ uint32_t ValueKind, uint16_t NumValueSites) {
+ *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
+}
+
+/* This method is only used in value profiler mock testing. */
+COMPILER_RT_VISIBILITY const __llvm_profile_data *
+__llvm_profile_iterate_data(const __llvm_profile_data *Data) {
+ return Data + 1;
+}
+
+/* This method is only used in value profiler mock testing. */
+COMPILER_RT_VISIBILITY void *
+__llvm_get_function_addr(const __llvm_profile_data *Data) {
+ return Data->FunctionPointer;
+}
+
+/* Allocate an array that holds the pointers to the linked lists of
+ * value profile counter nodes. The number of element of the array
+ * is the total number of value profile sites instrumented. Returns
+ * 0 if allocation fails.
+ */
+
+static int allocateValueProfileCounters(__llvm_profile_data *Data) {
+ uint64_t NumVSites = 0;
+ uint32_t VKI;
+ for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
+ NumVSites += Data->NumValueSites[VKI];
+
+ ValueProfNode **Mem =
+ (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
+ if (!Mem)
+ return 0;
+ if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
+ free(Mem);
+ return 0;
+ }
+ return 1;
+}
+
+COMPILER_RT_VISIBILITY void
+__llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
+ uint32_t CounterIndex) {
+
+ __llvm_profile_data *PData = (__llvm_profile_data *)Data;
+ if (!PData)
+ return;
+
+ if (!PData->Values) {
+ if (!allocateValueProfileCounters(PData))
+ return;
+ }
+
+ ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
+ ValueProfNode *PrevVNode = NULL;
+ ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
+
+ uint8_t VDataCount = 0;
+ while (CurrentVNode) {
+ if (TargetValue == CurrentVNode->VData.Value) {
+ CurrentVNode->VData.Count++;
+ return;
+ }
+ PrevVNode = CurrentVNode;
+ CurrentVNode = CurrentVNode->Next;
+ ++VDataCount;
+ }
+
+ if (VDataCount >= UCHAR_MAX)
+ return;
+
+ CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
+ if (!CurrentVNode)
+ return;
+
+ CurrentVNode->VData.Value = TargetValue;
+ CurrentVNode->VData.Count++;
+
+ uint32_t Success = 0;
+ if (!ValueCounters[CounterIndex])
+ Success =
+ COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
+ else if (PrevVNode && !PrevVNode->Next)
+ Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
+
+ if (!Success) {
+ free(CurrentVNode);
+ return;
+ }
+}
+
+COMPILER_RT_VISIBILITY ValueProfData **
+__llvm_profile_gather_value_data(uint64_t *ValueDataSize) {
+ size_t S = 0;
+ __llvm_profile_data *I;
+ ValueProfData **ValueDataArray;
+
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+
+ if (!ValueDataSize)
+ return NULL;
+
+ ValueDataArray =
+ (ValueProfData **)calloc(DataEnd - DataBegin, sizeof(void *));
+ if (!ValueDataArray)
+ PROF_OOM_RETURN("Failed to write value profile data ");
+
+ /*
+ * Compute the total Size of the buffer to hold ValueProfData
+ * structures for functions with value profile data.
+ */
+ for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
+ ValueProfRuntimeRecord R;
+ if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
+ PROF_OOM_RETURN("Failed to write value profile data ");
+
+ /* Compute the size of ValueProfData from this runtime record. */
+ if (getNumValueKindsRT(&R) != 0) {
+ ValueProfData *VD = NULL;
+ uint32_t VS = getValueProfDataSizeRT(&R);
+ VD = (ValueProfData *)calloc(VS, sizeof(uint8_t));
+ if (!VD)
+ PROF_OOM_RETURN("Failed to write value profile data ");
+ serializeValueProfDataFromRT(&R, VD);
+ ValueDataArray[I - DataBegin] = VD;
+ S += VS;
+ }
+ finalizeValueProfRuntimeRecord(&R);
+ }
+
+ if (!S) {
+ free(ValueDataArray);
+ ValueDataArray = NULL;
+ }
+
+ *ValueDataSize = S;
+ return ValueDataArray;
+}
diff --git a/lib/profile/InstrProfilingWriter.c b/lib/profile/InstrProfilingWriter.c
new file mode 100644
index 000000000000..a07bc538ed4b
--- /dev/null
+++ b/lib/profile/InstrProfilingWriter.c
@@ -0,0 +1,175 @@
+/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
+|*
+|* The LLVM Compiler Infrastructure
+|*
+|* This file is distributed under the University of Illinois Open Source
+|* License. See LICENSE.TXT for details.
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include <string.h>
+
+#define INSTR_PROF_VALUE_PROF_DATA
+#include "InstrProfData.inc"
+void (*FreeHook)(void *) = NULL;
+void* (*CallocHook)(size_t, size_t) = NULL;
+uint32_t VPBufferSize = 0;
+
+/* The buffer writer is reponsponsible in keeping writer state
+ * across the call.
+ */
+COMPILER_RT_VISIBILITY uint32_t llvmBufferWriter(ProfDataIOVec *IOVecs,
+ uint32_t NumIOVecs,
+ void **WriterCtx) {
+ uint32_t I;
+ char **Buffer = (char **)WriterCtx;
+ for (I = 0; I < NumIOVecs; I++) {
+ size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
+ memcpy(*Buffer, IOVecs[I].Data, Length);
+ *Buffer += Length;
+ }
+ return 0;
+}
+
+static void llvmInitBufferIO(ProfBufferIO *BufferIO, WriterCallback FileWriter,
+ void *File, uint8_t *Buffer, uint32_t BufferSz) {
+ BufferIO->File = File;
+ BufferIO->FileWriter = FileWriter;
+ BufferIO->BufferStart = Buffer;
+ BufferIO->BufferSz = BufferSz;
+ BufferIO->CurOffset = 0;
+}
+
+COMPILER_RT_VISIBILITY ProfBufferIO *
+llvmCreateBufferIO(WriterCallback FileWriter, void *File, uint32_t BufferSz) {
+ ProfBufferIO *BufferIO = (ProfBufferIO *)CallocHook(1, sizeof(ProfBufferIO));
+ uint8_t *Buffer = (uint8_t *)CallocHook(1, BufferSz);
+ if (!Buffer) {
+ FreeHook(BufferIO);
+ return 0;
+ }
+ llvmInitBufferIO(BufferIO, FileWriter, File, Buffer, BufferSz);
+ return BufferIO;
+}
+
+COMPILER_RT_VISIBILITY void llvmDeleteBufferIO(ProfBufferIO *BufferIO) {
+ FreeHook(BufferIO->BufferStart);
+ FreeHook(BufferIO);
+}
+
+COMPILER_RT_VISIBILITY int
+llvmBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
+ /* Buffer is not large enough, it is time to flush. */
+ if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
+ if (llvmBufferIOFlush(BufferIO) != 0)
+ return -1;
+ }
+ /* Special case, bypass the buffer completely. */
+ ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size}};
+ if (Size > BufferIO->BufferSz) {
+ if (BufferIO->FileWriter(IO, 1, &BufferIO->File))
+ return -1;
+ } else {
+ /* Write the data to buffer */
+ uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
+ llvmBufferWriter(IO, 1, (void **)&Buffer);
+ BufferIO->CurOffset = Buffer - BufferIO->BufferStart;
+ }
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY int llvmBufferIOFlush(ProfBufferIO *BufferIO) {
+ if (BufferIO->CurOffset) {
+ ProfDataIOVec IO[] = {
+ {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset}};
+ if (BufferIO->FileWriter(IO, 1, &BufferIO->File))
+ return -1;
+ BufferIO->CurOffset = 0;
+ }
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY int llvmWriteProfData(WriterCallback Writer,
+ void *WriterCtx,
+ ValueProfData **ValueDataArray,
+ const uint64_t ValueDataSize) {
+ /* Match logic in __llvm_profile_write_buffer(). */
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const uint64_t *CountersBegin = __llvm_profile_begin_counters();
+ const uint64_t *CountersEnd = __llvm_profile_end_counters();
+ const char *NamesBegin = __llvm_profile_begin_names();
+ const char *NamesEnd = __llvm_profile_end_names();
+ return llvmWriteProfDataImpl(Writer, WriterCtx, DataBegin, DataEnd,
+ CountersBegin, CountersEnd, ValueDataArray,
+ ValueDataSize, NamesBegin, NamesEnd);
+}
+
+#define VP_BUFFER_SIZE 8 * 1024
+static int writeValueProfData(WriterCallback Writer, void *WriterCtx,
+ ValueProfData **ValueDataBegin,
+ uint64_t NumVData) {
+ ProfBufferIO *BufferIO;
+ uint32_t I = 0, BufferSz;
+
+ if (!ValueDataBegin)
+ return 0;
+
+ BufferSz = VPBufferSize ? VPBufferSize : VP_BUFFER_SIZE;
+ BufferIO = llvmCreateBufferIO(Writer, WriterCtx, BufferSz);
+
+ for (I = 0; I < NumVData; I++) {
+ ValueProfData *CurVData = ValueDataBegin[I];
+ if (!CurVData)
+ continue;
+ if (llvmBufferIOWrite(BufferIO, (const uint8_t *)CurVData,
+ CurVData->TotalSize) != 0)
+ return -1;
+ }
+
+ if (llvmBufferIOFlush(BufferIO) != 0)
+ return -1;
+ llvmDeleteBufferIO(BufferIO);
+
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY int llvmWriteProfDataImpl(
+ WriterCallback Writer, void *WriterCtx,
+ const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
+ const uint64_t *CountersBegin, const uint64_t *CountersEnd,
+ ValueProfData **ValueDataBegin, const uint64_t ValueDataSize,
+ const char *NamesBegin, const char *NamesEnd) {
+
+ /* Calculate size of sections. */
+ const uint64_t DataSize = DataEnd - DataBegin;
+ const uint64_t CountersSize = CountersEnd - CountersBegin;
+ const uint64_t NamesSize = NamesEnd - NamesBegin;
+ const uint64_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize);
+
+ /* Enough zeroes for padding. */
+ const char Zeroes[sizeof(uint64_t)] = {0};
+
+ /* Create the header. */
+ __llvm_profile_header Header;
+
+ if (!DataSize)
+ return 0;
+
+ /* Initialize header struture. */
+#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
+#include "InstrProfData.inc"
+
+ /* Write the data. */
+ ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1},
+ {DataBegin, sizeof(__llvm_profile_data), DataSize},
+ {CountersBegin, sizeof(uint64_t), CountersSize},
+ {NamesBegin, sizeof(uint8_t), NamesSize},
+ {Zeroes, sizeof(uint8_t), Padding}};
+ if (Writer(IOVec, sizeof(IOVec) / sizeof(*IOVec), &WriterCtx))
+ return -1;
+
+ return writeValueProfData(Writer, WriterCtx, ValueDataBegin, DataSize);
+}