aboutsummaryrefslogtreecommitdiff
path: root/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
diff options
context:
space:
mode:
Diffstat (limited to 'lib/sanitizer_common/sanitizer_coverage_libcdep.cc')
-rw-r--r--lib/sanitizer_common/sanitizer_coverage_libcdep.cc498
1 files changed, 352 insertions, 146 deletions
diff --git a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
index 49887b1e91a9..f511c996d8c2 100644
--- a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
+++ b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
@@ -24,8 +24,12 @@
// and atomically set Guard to -Guard.
// - __sanitizer_cov_dump: dump the coverage data to disk.
// For every module of the current process that has coverage data
-// this will create a file module_name.PID.sancov. The file format is simple:
-// it's just a sorted sequence of 4-byte offsets in the module.
+// this will create a file module_name.PID.sancov.
+//
+// The file format is simple: the first 8 bytes is the magic,
+// one of 0xC0BFFFFFFFFFFF64 and 0xC0BFFFFFFFFFFF32. The last byte of the
+// magic defines the size of the following offsets.
+// The rest of the data is the offsets in the module.
//
// Eventually, this coverage implementation should be obsoleted by a more
// powerful general purpose Clang/LLVM coverage instrumentation.
@@ -43,6 +47,9 @@
#include "sanitizer_symbolizer.h"
#include "sanitizer_flags.h"
+static const u64 kMagic64 = 0xC0BFFFFFFFFFFF64ULL;
+static const u64 kMagic32 = 0xC0BFFFFFFFFFFF32ULL;
+
static atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once.
static atomic_uintptr_t coverage_counter;
@@ -56,7 +63,7 @@ static atomic_uintptr_t coverage_counter;
// dump current memory layout to another file.
static bool cov_sandboxed = false;
-static int cov_fd = kInvalidFd;
+static fd_t cov_fd = kInvalidFd;
static unsigned int cov_max_block_size = 0;
static bool coverage_enabled = false;
static const char *coverage_dir;
@@ -77,21 +84,34 @@ class CoverageData {
uptr cache_size);
void DumpCallerCalleePairs();
void DumpTrace();
+ void DumpAsBitSet();
+ void DumpCounters();
+ void DumpOffsets();
+ void DumpAll();
ALWAYS_INLINE
void TraceBasicBlock(s32 *id);
void InitializeGuardArray(s32 *guards);
- void InitializeGuards(s32 *guards, uptr n, const char *module_name);
+ void InitializeGuards(s32 *guards, uptr n, const char *module_name,
+ uptr caller_pc);
+ void InitializeCounters(u8 *counters, uptr n);
void ReinitializeGuards();
+ uptr GetNumberOf8bitCounters();
+ uptr Update8bitCounterBitsetAndClearCounters(u8 *bitset);
uptr *data();
uptr size();
private:
+ void DirectOpen();
+ void UpdateModuleNameVec(uptr caller_pc, uptr range_beg, uptr range_end);
+
// Maximal size pc array may ever grow.
// We MmapNoReserve this space to ensure that the array is contiguous.
- static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 24, 1 << 27);
+ static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(
+ 1 << (SANITIZER_ANDROID ? 24 : (SANITIZER_WINDOWS ? 27 : 26)),
+ 1 << 27);
// The amount file mapping for the pc array is grown by.
static const uptr kPcArrayMmapSize = 64 * 1024;
@@ -105,13 +125,27 @@ class CoverageData {
// Current file mapped size of the pc array.
uptr pc_array_mapped_size;
// Descriptor of the file mapped pc array.
- int pc_fd;
+ fd_t pc_fd;
// Vector of coverage guard arrays, protected by mu.
InternalMmapVectorNoCtor<s32*> guard_array_vec;
- // Vector of module (compilation unit) names.
- InternalMmapVectorNoCtor<const char*> comp_unit_name_vec;
+ struct NamedPcRange {
+ const char *copied_module_name;
+ uptr beg, end; // elements [beg,end) in pc_array.
+ };
+
+ // Vector of module and compilation unit pc ranges.
+ InternalMmapVectorNoCtor<NamedPcRange> comp_unit_name_vec;
+ InternalMmapVectorNoCtor<NamedPcRange> module_name_vec;
+
+ struct CounterAndSize {
+ u8 *counters;
+ uptr n;
+ };
+
+ InternalMmapVectorNoCtor<CounterAndSize> counters_vec;
+ uptr num_8bit_counters;
// Caller-Callee (cc) array, size and current index.
static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24);
@@ -133,8 +167,6 @@ class CoverageData {
static const uptr kTrPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27);
StaticSpinMutex mu;
-
- void DirectOpen();
};
static CoverageData coverage_data;
@@ -145,9 +177,9 @@ void CoverageData::DirectOpen() {
InternalScopedString path(kMaxPathLength);
internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw",
coverage_dir, internal_getpid());
- pc_fd = OpenFile(path.data(), true);
- if (internal_iserror(pc_fd)) {
- Report(" Coverage: failed to open %s for writing\n", path.data());
+ pc_fd = OpenFile(path.data(), RdWr);
+ if (pc_fd == kInvalidFd) {
+ Report("Coverage: failed to open %s for reading/writing\n", path.data());
Die();
}
@@ -180,10 +212,13 @@ void CoverageData::Enable() {
tr_event_array = reinterpret_cast<u32 *>(MmapNoReserveOrDie(
sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(),
"CovInit::tr_event_array"));
- Mprotect(reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]),
- GetMmapGranularity());
+ MprotectNoAccess(
+ reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]),
+ GetMmapGranularity());
tr_event_array_size = kTrEventArrayMaxSize;
tr_event_pointer = tr_event_array;
+
+ num_8bit_counters = 0;
}
void CoverageData::InitializeGuardArray(s32 *guards) {
@@ -197,22 +232,22 @@ void CoverageData::InitializeGuardArray(s32 *guards) {
void CoverageData::Disable() {
if (pc_array) {
- internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize);
+ UnmapOrDie(pc_array, sizeof(uptr) * kPcArrayMaxSize);
pc_array = nullptr;
}
if (cc_array) {
- internal_munmap(cc_array, sizeof(uptr *) * kCcArrayMaxSize);
+ UnmapOrDie(cc_array, sizeof(uptr *) * kCcArrayMaxSize);
cc_array = nullptr;
}
if (tr_event_array) {
- internal_munmap(tr_event_array,
- sizeof(tr_event_array[0]) * kTrEventArrayMaxSize +
- GetMmapGranularity());
+ UnmapOrDie(tr_event_array,
+ sizeof(tr_event_array[0]) * kTrEventArrayMaxSize +
+ GetMmapGranularity());
tr_event_array = nullptr;
tr_event_pointer = nullptr;
}
if (pc_fd != kInvalidFd) {
- internal_close(pc_fd);
+ CloseFile(pc_fd);
pc_fd = kInvalidFd;
}
}
@@ -231,8 +266,9 @@ void CoverageData::ReInit() {
// In memory-mapped mode we must extend the new file to the known array
// size.
uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
+ uptr npcs = size / sizeof(uptr);
Enable();
- if (size) Extend(size);
+ if (size) Extend(npcs);
if (coverage_enabled) CovUpdateMapping(coverage_dir);
} else {
Enable();
@@ -289,16 +325,69 @@ void CoverageData::Extend(uptr npcs) {
atomic_store(&pc_array_size, size, memory_order_release);
}
+void CoverageData::InitializeCounters(u8 *counters, uptr n) {
+ if (!counters) return;
+ CHECK_EQ(reinterpret_cast<uptr>(counters) % 16, 0);
+ n = RoundUpTo(n, 16); // The compiler must ensure that counters is 16-aligned.
+ SpinMutexLock l(&mu);
+ counters_vec.push_back({counters, n});
+ num_8bit_counters += n;
+}
+
+void CoverageData::UpdateModuleNameVec(uptr caller_pc, uptr range_beg,
+ uptr range_end) {
+ auto sym = Symbolizer::GetOrInit();
+ if (!sym)
+ return;
+ const char *module_name = sym->GetModuleNameForPc(caller_pc);
+ if (!module_name) return;
+ if (module_name_vec.empty() ||
+ module_name_vec.back().copied_module_name != module_name)
+ module_name_vec.push_back({module_name, range_beg, range_end});
+ else
+ module_name_vec.back().end = range_end;
+}
+
void CoverageData::InitializeGuards(s32 *guards, uptr n,
- const char *module_name) {
+ const char *comp_unit_name,
+ uptr caller_pc) {
// The array 'guards' has n+1 elements, we use the element zero
// to store 'n'.
CHECK_LT(n, 1 << 30);
guards[0] = static_cast<s32>(n);
InitializeGuardArray(guards);
SpinMutexLock l(&mu);
- comp_unit_name_vec.push_back(module_name);
+ uptr range_end = atomic_load(&pc_array_index, memory_order_relaxed);
+ uptr range_beg = range_end - n;
+ comp_unit_name_vec.push_back({comp_unit_name, range_beg, range_end});
guard_array_vec.push_back(guards);
+ UpdateModuleNameVec(caller_pc, range_beg, range_end);
+}
+
+static const uptr kBundleCounterBits = 16;
+
+// When coverage_order_pcs==true and SANITIZER_WORDSIZE==64
+// we insert the global counter into the first 16 bits of the PC.
+uptr BundlePcAndCounter(uptr pc, uptr counter) {
+ if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
+ return pc;
+ static const uptr kMaxCounter = (1 << kBundleCounterBits) - 1;
+ if (counter > kMaxCounter)
+ counter = kMaxCounter;
+ CHECK_EQ(0, pc >> (SANITIZER_WORDSIZE - kBundleCounterBits));
+ return pc | (counter << (SANITIZER_WORDSIZE - kBundleCounterBits));
+}
+
+uptr UnbundlePc(uptr bundle) {
+ if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
+ return bundle;
+ return (bundle << kBundleCounterBits) >> kBundleCounterBits;
+}
+
+uptr UnbundleCounter(uptr bundle) {
+ if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs)
+ return 0;
+ return bundle >> (SANITIZER_WORDSIZE - kBundleCounterBits);
}
// If guard is negative, atomically set it to -guard and store the PC in
@@ -316,8 +405,8 @@ void CoverageData::Add(uptr pc, u32 *guard) {
return; // May happen after fork when pc_array_index becomes 0.
CHECK_LT(idx * sizeof(uptr),
atomic_load(&pc_array_size, memory_order_acquire));
- pc_array[idx] = pc;
- atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed);
+ uptr counter = atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed);
+ pc_array[idx] = BundlePcAndCounter(pc, counter);
}
// Registers a pair caller=>callee.
@@ -354,6 +443,64 @@ void CoverageData::IndirCall(uptr caller, uptr callee, uptr callee_cache[],
}
}
+uptr CoverageData::GetNumberOf8bitCounters() {
+ return num_8bit_counters;
+}
+
+// Map every 8bit counter to a 8-bit bitset and clear the counter.
+uptr CoverageData::Update8bitCounterBitsetAndClearCounters(u8 *bitset) {
+ uptr num_new_bits = 0;
+ uptr cur = 0;
+ // For better speed we map 8 counters to 8 bytes of bitset at once.
+ static const uptr kBatchSize = 8;
+ CHECK_EQ(reinterpret_cast<uptr>(bitset) % kBatchSize, 0);
+ for (uptr i = 0, len = counters_vec.size(); i < len; i++) {
+ u8 *c = counters_vec[i].counters;
+ uptr n = counters_vec[i].n;
+ CHECK_EQ(n % 16, 0);
+ CHECK_EQ(cur % kBatchSize, 0);
+ CHECK_EQ(reinterpret_cast<uptr>(c) % kBatchSize, 0);
+ if (!bitset) {
+ internal_bzero_aligned16(c, n);
+ cur += n;
+ continue;
+ }
+ for (uptr j = 0; j < n; j += kBatchSize, cur += kBatchSize) {
+ CHECK_LT(cur, num_8bit_counters);
+ u64 *pc64 = reinterpret_cast<u64*>(c + j);
+ u64 *pb64 = reinterpret_cast<u64*>(bitset + cur);
+ u64 c64 = *pc64;
+ u64 old_bits_64 = *pb64;
+ u64 new_bits_64 = old_bits_64;
+ if (c64) {
+ *pc64 = 0;
+ for (uptr k = 0; k < kBatchSize; k++) {
+ u64 x = (c64 >> (8 * k)) & 0xff;
+ if (x) {
+ u64 bit = 0;
+ /**/ if (x >= 128) bit = 128;
+ else if (x >= 32) bit = 64;
+ else if (x >= 16) bit = 32;
+ else if (x >= 8) bit = 16;
+ else if (x >= 4) bit = 8;
+ else if (x >= 3) bit = 4;
+ else if (x >= 2) bit = 2;
+ else if (x >= 1) bit = 1;
+ u64 mask = bit << (8 * k);
+ if (!(new_bits_64 & mask)) {
+ num_new_bits++;
+ new_bits_64 |= mask;
+ }
+ }
+ }
+ *pb64 = new_bits_64;
+ }
+ }
+ }
+ CHECK_EQ(cur, num_8bit_counters);
+ return num_new_bits;
+}
+
uptr *CoverageData::data() {
return pc_array;
}
@@ -372,15 +519,15 @@ struct CovHeader {
static void CovWritePacked(int pid, const char *module, const void *blob,
unsigned int blob_size) {
- if (cov_fd < 0) return;
+ if (cov_fd == kInvalidFd) return;
unsigned module_name_length = internal_strlen(module);
CovHeader header = {pid, module_name_length, blob_size};
if (cov_max_block_size == 0) {
// Writing to a file. Just go ahead.
- internal_write(cov_fd, &header, sizeof(header));
- internal_write(cov_fd, module, module_name_length);
- internal_write(cov_fd, blob, blob_size);
+ WriteToFile(cov_fd, &header, sizeof(header));
+ WriteToFile(cov_fd, module, module_name_length);
+ WriteToFile(cov_fd, blob, blob_size);
} else {
// Writing to a socket. We want to split the data into appropriately sized
// blocks.
@@ -403,8 +550,7 @@ static void CovWritePacked(int pid, const char *module, const void *blob,
internal_memcpy(block_data_begin, blob_pos, payload_size);
blob_pos += payload_size;
((CovHeader *)block.data())->data_length = payload_size;
- internal_write(cov_fd, block.data(),
- header_size_with_module + payload_size);
+ WriteToFile(cov_fd, block.data(), header_size_with_module + payload_size);
}
}
}
@@ -413,25 +559,25 @@ static void CovWritePacked(int pid, const char *module, const void *blob,
// If packed = true and name == 0: <pid>.<sancov>.<packed>.
// If packed = true and name != 0: <name>.<sancov>.<packed> (name is
// user-supplied).
-static int CovOpenFile(bool packed, const char *name,
- const char *extension = "sancov") {
- InternalScopedString path(kMaxPathLength);
+static fd_t CovOpenFile(InternalScopedString *path, bool packed,
+ const char *name, const char *extension = "sancov") {
+ path->clear();
if (!packed) {
CHECK(name);
- path.append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(),
+ path->append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(),
extension);
} else {
if (!name)
- path.append("%s/%zd.%s.packed", coverage_dir, internal_getpid(),
+ path->append("%s/%zd.%s.packed", coverage_dir, internal_getpid(),
extension);
else
- path.append("%s/%s.%s.packed", coverage_dir, name, extension);
- }
- uptr fd = OpenFile(path.data(), true);
- if (internal_iserror(fd)) {
- Report(" SanitizerCoverage: failed to open %s for writing\n", path.data());
- return -1;
+ path->append("%s/%s.%s.packed", coverage_dir, name, extension);
}
+ error_t err;
+ fd_t fd = OpenFile(path->data(), WrOnly, &err);
+ if (fd == kInvalidFd)
+ Report("SanitizerCoverage: failed to open %s for writing (reason: %d)\n",
+ path->data(), err);
return fd;
}
@@ -446,38 +592,40 @@ void CoverageData::DumpTrace() {
for (uptr i = 0, n = size(); i < n; i++) {
const char *module_name = "<unknown>";
uptr module_address = 0;
- sym->GetModuleNameAndOffsetForPC(pc_array[i], &module_name,
+ sym->GetModuleNameAndOffsetForPC(UnbundlePc(pc_array[i]), &module_name,
&module_address);
out.append("%s 0x%zx\n", module_name, module_address);
}
- int fd = CovOpenFile(false, "trace-points");
- if (fd < 0) return;
- internal_write(fd, out.data(), out.length());
- internal_close(fd);
+ InternalScopedString path(kMaxPathLength);
+ fd_t fd = CovOpenFile(&path, false, "trace-points");
+ if (fd == kInvalidFd) return;
+ WriteToFile(fd, out.data(), out.length());
+ CloseFile(fd);
- fd = CovOpenFile(false, "trace-compunits");
- if (fd < 0) return;
+ fd = CovOpenFile(&path, false, "trace-compunits");
+ if (fd == kInvalidFd) return;
out.clear();
for (uptr i = 0; i < comp_unit_name_vec.size(); i++)
- out.append("%s\n", comp_unit_name_vec[i]);
- internal_write(fd, out.data(), out.length());
- internal_close(fd);
+ out.append("%s\n", comp_unit_name_vec[i].copied_module_name);
+ WriteToFile(fd, out.data(), out.length());
+ CloseFile(fd);
- fd = CovOpenFile(false, "trace-events");
- if (fd < 0) return;
+ fd = CovOpenFile(&path, false, "trace-events");
+ if (fd == kInvalidFd) return;
uptr bytes_to_write = max_idx * sizeof(tr_event_array[0]);
u8 *event_bytes = reinterpret_cast<u8*>(tr_event_array);
// The trace file could be huge, and may not be written with a single syscall.
while (bytes_to_write) {
- uptr actually_written = internal_write(fd, event_bytes, bytes_to_write);
- if (actually_written <= bytes_to_write) {
+ uptr actually_written;
+ if (WriteToFile(fd, event_bytes, bytes_to_write, &actually_written) &&
+ actually_written <= bytes_to_write) {
bytes_to_write -= actually_written;
event_bytes += actually_written;
} else {
break;
}
}
- internal_close(fd);
+ CloseFile(fd);
VReport(1, " CovDump: Trace: %zd PCs written\n", size());
VReport(1, " CovDump: Trace: %zd Events written\n", max_idx);
}
@@ -514,10 +662,11 @@ void CoverageData::DumpCallerCalleePairs() {
callee_module_address);
}
}
- int fd = CovOpenFile(false, "caller-callee");
- if (fd < 0) return;
- internal_write(fd, out.data(), out.length());
- internal_close(fd);
+ InternalScopedString path(kMaxPathLength);
+ fd_t fd = CovOpenFile(&path, false, "caller-callee");
+ if (fd == kInvalidFd) return;
+ WriteToFile(fd, out.data(), out.length());
+ CloseFile(fd);
VReport(1, " CovDump: %zd caller-callee pairs written\n", total);
}
@@ -534,86 +683,123 @@ void CoverageData::TraceBasicBlock(s32 *id) {
tr_event_pointer++;
}
-static void CovDumpAsBitSet() {
+void CoverageData::DumpCounters() {
+ if (!common_flags()->coverage_counters) return;
+ uptr n = coverage_data.GetNumberOf8bitCounters();
+ if (!n) return;
+ InternalScopedBuffer<u8> bitset(n);
+ coverage_data.Update8bitCounterBitsetAndClearCounters(bitset.data());
+ InternalScopedString path(kMaxPathLength);
+
+ for (uptr m = 0; m < module_name_vec.size(); m++) {
+ auto r = module_name_vec[m];
+ CHECK(r.copied_module_name);
+ CHECK_LE(r.beg, r.end);
+ CHECK_LE(r.end, size());
+ const char *base_name = StripModuleName(r.copied_module_name);
+ fd_t fd =
+ CovOpenFile(&path, /* packed */ false, base_name, "counters-sancov");
+ if (fd == kInvalidFd) return;
+ WriteToFile(fd, bitset.data() + r.beg, r.end - r.beg);
+ CloseFile(fd);
+ VReport(1, " CovDump: %zd counters written for '%s'\n", r.end - r.beg,
+ base_name);
+ }
+}
+
+void CoverageData::DumpAsBitSet() {
if (!common_flags()->coverage_bitset) return;
- if (!coverage_data.size()) return;
- int fd = CovOpenFile(/* packed */false, "combined", "bitset-sancov");
- if (fd < 0) return;
- uptr n = coverage_data.size();
- uptr n_set_bits = 0;
- InternalScopedBuffer<char> out(n);
- for (uptr i = 0; i < n; i++) {
- uptr pc = coverage_data.data()[i];
- out[i] = pc ? '1' : '0';
- if (pc)
- n_set_bits++;
+ if (!size()) return;
+ InternalScopedBuffer<char> out(size());
+ InternalScopedString path(kMaxPathLength);
+ for (uptr m = 0; m < module_name_vec.size(); m++) {
+ uptr n_set_bits = 0;
+ auto r = module_name_vec[m];
+ CHECK(r.copied_module_name);
+ CHECK_LE(r.beg, r.end);
+ CHECK_LE(r.end, size());
+ for (uptr i = r.beg; i < r.end; i++) {
+ uptr pc = UnbundlePc(pc_array[i]);
+ out[i] = pc ? '1' : '0';
+ if (pc)
+ n_set_bits++;
+ }
+ const char *base_name = StripModuleName(r.copied_module_name);
+ fd_t fd = CovOpenFile(&path, /* packed */false, base_name, "bitset-sancov");
+ if (fd == kInvalidFd) return;
+ WriteToFile(fd, out.data() + r.beg, r.end - r.beg);
+ CloseFile(fd);
+ VReport(1,
+ " CovDump: bitset of %zd bits written for '%s', %zd bits are set\n",
+ r.end - r.beg, base_name, n_set_bits);
}
- internal_write(fd, out.data(), n);
- internal_close(fd);
- VReport(1, " CovDump: bitset of %zd bits written, %zd bits are set\n", n,
- n_set_bits);
}
-// Dump the coverage on disk.
-static void CovDump() {
- if (!coverage_enabled || common_flags()->coverage_direct) return;
-#if !SANITIZER_WINDOWS
- if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
- return;
- CovDumpAsBitSet();
- coverage_data.DumpTrace();
+void CoverageData::DumpOffsets() {
+ auto sym = Symbolizer::GetOrInit();
if (!common_flags()->coverage_pcs) return;
- uptr size = coverage_data.size();
- InternalMmapVector<u32> offsets(size);
- uptr *vb = coverage_data.data();
- uptr *ve = vb + size;
- SortArray(vb, size);
- MemoryMappingLayout proc_maps(/*cache_enabled*/true);
- uptr mb, me, off, prot;
- InternalScopedString module(kMaxPathLength);
+ CHECK_NE(sym, nullptr);
+ InternalMmapVector<uptr> offsets(0);
InternalScopedString path(kMaxPathLength);
- for (int i = 0;
- proc_maps.Next(&mb, &me, &off, module.data(), module.size(), &prot);
- i++) {
- if ((prot & MemoryMappingLayout::kProtectionExecute) == 0)
- continue;
- while (vb < ve && *vb < mb) vb++;
- if (vb >= ve) break;
- if (*vb < me) {
- offsets.clear();
- const uptr *old_vb = vb;
- CHECK_LE(off, *vb);
- for (; vb < ve && *vb < me; vb++) {
- uptr diff = *vb - (i ? mb : 0) + off;
- CHECK_LE(diff, 0xffffffffU);
- offsets.push_back(static_cast<u32>(diff));
- }
- const char *module_name = StripModuleName(module.data());
- if (cov_sandboxed) {
- if (cov_fd >= 0) {
- CovWritePacked(internal_getpid(), module_name, offsets.data(),
- offsets.size() * sizeof(u32));
- VReport(1, " CovDump: %zd PCs written to packed file\n", vb - old_vb);
- }
- } else {
- // One file per module per process.
- path.clear();
- path.append("%s/%s.%zd.sancov", coverage_dir, module_name,
- internal_getpid());
- int fd = CovOpenFile(false /* packed */, module_name);
- if (fd > 0) {
- internal_write(fd, offsets.data(), offsets.size() * sizeof(u32));
- internal_close(fd);
- VReport(1, " CovDump: %s: %zd PCs written\n", path.data(),
- vb - old_vb);
- }
+ for (uptr m = 0; m < module_name_vec.size(); m++) {
+ offsets.clear();
+ uptr num_words_for_magic = SANITIZER_WORDSIZE == 64 ? 1 : 2;
+ for (uptr i = 0; i < num_words_for_magic; i++)
+ offsets.push_back(0);
+ auto r = module_name_vec[m];
+ CHECK(r.copied_module_name);
+ CHECK_LE(r.beg, r.end);
+ CHECK_LE(r.end, size());
+ for (uptr i = r.beg; i < r.end; i++) {
+ uptr pc = UnbundlePc(pc_array[i]);
+ uptr counter = UnbundleCounter(pc_array[i]);
+ if (!pc) continue; // Not visited.
+ uptr offset = 0;
+ sym->GetModuleNameAndOffsetForPC(pc, nullptr, &offset);
+ offsets.push_back(BundlePcAndCounter(offset, counter));
+ }
+
+ CHECK_GE(offsets.size(), num_words_for_magic);
+ SortArray(offsets.data(), offsets.size());
+ for (uptr i = 0; i < offsets.size(); i++)
+ offsets[i] = UnbundlePc(offsets[i]);
+
+ uptr num_offsets = offsets.size() - num_words_for_magic;
+ u64 *magic_p = reinterpret_cast<u64*>(offsets.data());
+ CHECK_EQ(*magic_p, 0ULL);
+ // FIXME: we may want to write 32-bit offsets even in 64-mode
+ // if all the offsets are small enough.
+ *magic_p = SANITIZER_WORDSIZE == 64 ? kMagic64 : kMagic32;
+
+ const char *module_name = StripModuleName(r.copied_module_name);
+ if (cov_sandboxed) {
+ if (cov_fd != kInvalidFd) {
+ CovWritePacked(internal_getpid(), module_name, offsets.data(),
+ offsets.size() * sizeof(offsets[0]));
+ VReport(1, " CovDump: %zd PCs written to packed file\n", num_offsets);
}
+ } else {
+ // One file per module per process.
+ fd_t fd = CovOpenFile(&path, false /* packed */, module_name);
+ if (fd == kInvalidFd) continue;
+ WriteToFile(fd, offsets.data(), offsets.size() * sizeof(offsets[0]));
+ CloseFile(fd);
+ VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), num_offsets);
}
}
- if (cov_fd >= 0)
- internal_close(cov_fd);
- coverage_data.DumpCallerCalleePairs();
-#endif // !SANITIZER_WINDOWS
+ if (cov_fd != kInvalidFd)
+ CloseFile(cov_fd);
+}
+
+void CoverageData::DumpAll() {
+ if (!coverage_enabled || common_flags()->coverage_direct) return;
+ if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
+ return;
+ DumpAsBitSet();
+ DumpCounters();
+ DumpTrace();
+ DumpOffsets();
+ DumpCallerCalleePairs();
}
void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
@@ -621,17 +807,21 @@ void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
if (!coverage_enabled) return;
cov_sandboxed = args->coverage_sandboxed;
if (!cov_sandboxed) return;
- cov_fd = args->coverage_fd;
cov_max_block_size = args->coverage_max_block_size;
- if (cov_fd < 0)
+ if (args->coverage_fd >= 0) {
+ cov_fd = (fd_t)args->coverage_fd;
+ } else {
+ InternalScopedString path(kMaxPathLength);
// Pre-open the file now. The sandbox won't allow us to do it later.
- cov_fd = CovOpenFile(true /* packed */, 0);
+ cov_fd = CovOpenFile(&path, true /* packed */, 0);
+ }
}
-int MaybeOpenCovFile(const char *name) {
+fd_t MaybeOpenCovFile(const char *name) {
CHECK(name);
- if (!coverage_enabled) return -1;
- return CovOpenFile(true /* packed */, name);
+ if (!coverage_enabled) return kInvalidFd;
+ InternalScopedString path(kMaxPathLength);
+ return CovOpenFile(&path, true /* packed */, name);
}
void CovBeforeFork() {
@@ -649,9 +839,7 @@ void InitializeCoverage(bool enabled, const char *dir) {
coverage_dir = dir;
coverage_data.Init();
if (enabled) coverage_data.Enable();
-#if !SANITIZER_WINDOWS
if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump);
-#endif
}
void ReInitializeCoverage(bool enabled, const char *dir) {
@@ -674,7 +862,8 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) {
}
SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_with_check(u32 *guard) {
atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard);
- if (__sanitizer::atomic_load(atomic_guard, memory_order_relaxed))
+ if (static_cast<s32>(
+ __sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) < 0)
__sanitizer_cov(guard);
}
SANITIZER_INTERFACE_ATTRIBUTE void
@@ -687,10 +876,14 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() {
coverage_dir = common_flags()->coverage_dir;
coverage_data.Init();
}
-SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); }
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() {
+ coverage_data.DumpAll();
+}
SANITIZER_INTERFACE_ATTRIBUTE void
-__sanitizer_cov_module_init(s32 *guards, uptr npcs, const char *module_name) {
- coverage_data.InitializeGuards(guards, npcs, module_name);
+__sanitizer_cov_module_init(s32 *guards, uptr npcs, u8 *counters,
+ const char *comp_unit_name) {
+ coverage_data.InitializeGuards(guards, npcs, comp_unit_name, GET_CALLER_PC());
+ coverage_data.InitializeCounters(counters, npcs);
if (!common_flags()->coverage_direct) return;
if (SANITIZER_ANDROID && coverage_enabled) {
// dlopen/dlclose interceptors do not work on Android, so we rely on
@@ -701,7 +894,7 @@ __sanitizer_cov_module_init(s32 *guards, uptr npcs, const char *module_name) {
}
SANITIZER_INTERFACE_ATTRIBUTE
sptr __sanitizer_maybe_open_cov_file(const char *name) {
- return MaybeOpenCovFile(name);
+ return (sptr)MaybeOpenCovFile(name);
}
SANITIZER_INTERFACE_ATTRIBUTE
uptr __sanitizer_get_total_unique_coverage() {
@@ -728,4 +921,17 @@ uptr __sanitizer_get_coverage_guards(uptr **data) {
*data = coverage_data.data();
return coverage_data.size();
}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+uptr __sanitizer_get_number_of_counters() {
+ return coverage_data.GetNumberOf8bitCounters();
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+uptr __sanitizer_update_counter_bitset_and_clear_counters(u8 *bitset) {
+ return coverage_data.Update8bitCounterBitsetAndClearCounters(bitset);
+}
+// Default empty implementation (weak). Users should redefine it.
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void __sanitizer_cov_trace_cmp() {}
} // extern "C"