aboutsummaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2018-07-28 11:06:48 +0000
committerDimitry Andric <dim@FreeBSD.org>2018-07-28 11:06:48 +0000
commit93c1b73a09a52d4a265f683bf1954b08bb430049 (patch)
tree5543464d74945196cc890e9d9099e5d0660df7eb /lib
parent0d8e7490d6e8a13a8f0977d9b7771803b9f64ea0 (diff)
downloadsrc-93c1b73a09a52d4a265f683bf1954b08bb430049.tar.gz
src-93c1b73a09a52d4a265f683bf1954b08bb430049.zip
Vendor import of compiler-rt trunk r338150:vendor/compiler-rt/compiler-rt-trunk-r338150
Notes
Notes: svn path=/vendor/compiler-rt/dist/; revision=336817 svn path=/vendor/compiler-rt/compiler-rt-trunk-r338150/; revision=336818; tag=vendor/compiler-rt/compiler-rt-trunk-r338150
Diffstat (limited to 'lib')
-rw-r--r--lib/asan/.clang-format1
-rw-r--r--lib/asan/CMakeLists.txt62
-rw-r--r--lib/asan/asan_allocator.cc94
-rw-r--r--lib/asan/asan_allocator.h6
-rw-r--r--lib/asan/asan_debugging.cc3
-rw-r--r--lib/asan/asan_descriptions.cc70
-rw-r--r--lib/asan/asan_descriptions.h17
-rw-r--r--lib/asan/asan_errors.cc163
-rw-r--r--lib/asan/asan_errors.h271
-rw-r--r--lib/asan/asan_flags.cc9
-rw-r--r--lib/asan/asan_flags.inc9
-rw-r--r--lib/asan/asan_globals.cc11
-rw-r--r--lib/asan/asan_globals_win.cc4
-rw-r--r--lib/asan/asan_interceptors.cc67
-rw-r--r--lib/asan/asan_interceptors.h21
-rw-r--r--lib/asan/asan_interceptors_memintrinsics.cc16
-rw-r--r--lib/asan/asan_interceptors_memintrinsics.h25
-rw-r--r--lib/asan/asan_internal.h7
-rw-r--r--lib/asan/asan_linux.cc3
-rw-r--r--lib/asan/asan_mac.cc28
-rw-r--r--lib/asan/asan_malloc_linux.cc79
-rw-r--r--lib/asan/asan_malloc_local.h44
-rw-r--r--lib/asan/asan_malloc_mac.cc3
-rw-r--r--lib/asan/asan_mapping.h123
-rw-r--r--lib/asan/asan_mapping_myriad.h86
-rw-r--r--lib/asan/asan_memory_profile.cc12
-rw-r--r--lib/asan/asan_new_delete.cc24
-rw-r--r--lib/asan/asan_poisoning.cc14
-rw-r--r--lib/asan/asan_poisoning.h5
-rw-r--r--lib/asan/asan_report.cc72
-rw-r--r--lib/asan/asan_report.h12
-rw-r--r--lib/asan/asan_rtems.cc253
-rw-r--r--lib/asan/asan_rtl.cc42
-rw-r--r--lib/asan/asan_shadow_setup.cc10
-rw-r--r--lib/asan/asan_thread.cc30
-rw-r--r--lib/asan/asan_win.cc9
-rw-r--r--lib/asan/asan_win_dll_thunk.cc2
-rwxr-xr-xlib/asan/scripts/asan_device_setup9
-rw-r--r--lib/asan/tests/CMakeLists.txt9
-rw-r--r--lib/asan/tests/asan_test.cc5
-rw-r--r--lib/builtins/CMakeLists.txt52
-rw-r--r--lib/builtins/arm/chkstk.S34
-rw-r--r--lib/builtins/clear_cache.c17
-rw-r--r--lib/builtins/clzdi2.c6
-rw-r--r--lib/builtins/cpu_model.c4
-rw-r--r--lib/builtins/ctzdi2.c6
-rw-r--r--lib/builtins/emutls.c65
-rw-r--r--lib/builtins/hexagon/common_entry_exit_abi1.S103
-rw-r--r--lib/builtins/hexagon/common_entry_exit_abi2.S268
-rw-r--r--lib/builtins/hexagon/common_entry_exit_legacy.S157
-rw-r--r--lib/builtins/hexagon/dfaddsub.S398
-rw-r--r--lib/builtins/hexagon/dfdiv.S492
-rw-r--r--lib/builtins/hexagon/dffma.S705
-rw-r--r--lib/builtins/hexagon/dfminmax.S79
-rw-r--r--lib/builtins/hexagon/dfmul.S418
-rw-r--r--lib/builtins/hexagon/dfsqrt.S406
-rw-r--r--lib/builtins/hexagon/divdi3.S85
-rw-r--r--lib/builtins/hexagon/divsi3.S84
-rw-r--r--lib/builtins/hexagon/fabs_opt.S37
-rw-r--r--lib/builtins/hexagon/fastmath2_dlib_asm.S491
-rw-r--r--lib/builtins/hexagon/fastmath2_ldlib_asm.S345
-rw-r--r--lib/builtins/hexagon/fastmath_dlib_asm.S400
-rw-r--r--lib/builtins/hexagon/fma_opt.S31
-rw-r--r--lib/builtins/hexagon/fmax_opt.S30
-rw-r--r--lib/builtins/hexagon/fmin_opt.S30
-rw-r--r--lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S125
-rw-r--r--lib/builtins/hexagon/memcpy_likely_aligned.S64
-rw-r--r--lib/builtins/hexagon/moddi3.S83
-rw-r--r--lib/builtins/hexagon/modsi3.S66
-rw-r--r--lib/builtins/hexagon/sfdiv_opt.S66
-rw-r--r--lib/builtins/hexagon/sfsqrt_opt.S82
-rw-r--r--lib/builtins/hexagon/udivdi3.S71
-rw-r--r--lib/builtins/hexagon/udivmoddi4.S71
-rw-r--r--lib/builtins/hexagon/udivmodsi4.S60
-rw-r--r--lib/builtins/hexagon/udivsi3.S56
-rw-r--r--lib/builtins/hexagon/umoddi3.S74
-rw-r--r--lib/builtins/hexagon/umodsi3.S55
-rw-r--r--lib/builtins/int_types.h2
-rw-r--r--lib/builtins/os_version_check.c77
-rw-r--r--lib/builtins/riscv/mulsi3.S28
-rw-r--r--lib/cfi/CMakeLists.txt2
-rw-r--r--lib/cfi/cfi.cc27
-rw-r--r--lib/cfi/cfi_blacklist.txt4
-rw-r--r--lib/dfsan/.clang-format1
-rw-r--r--lib/dfsan/CMakeLists.txt14
-rw-r--r--lib/dfsan/dfsan.cc3
-rw-r--r--lib/dfsan/dfsan_custom.cc24
-rw-r--r--lib/dfsan/done_abilist.txt17
-rwxr-xr-xlib/dfsan/scripts/check_custom_wrappers.sh2
-rw-r--r--lib/esan/CMakeLists.txt14
-rw-r--r--lib/esan/esan.cpp6
-rw-r--r--lib/esan/esan_interceptors.cpp61
-rw-r--r--lib/esan/esan_sideline_linux.cpp2
-rw-r--r--lib/fuzzer/CMakeLists.txt81
-rw-r--r--lib/fuzzer/FuzzerClangCounters.cpp49
-rw-r--r--lib/fuzzer/FuzzerCommand.h3
-rw-r--r--lib/fuzzer/FuzzerCorpus.h59
-rw-r--r--lib/fuzzer/FuzzerDataFlowTrace.cpp91
-rw-r--r--lib/fuzzer/FuzzerDataFlowTrace.h56
-rw-r--r--lib/fuzzer/FuzzerDefs.h45
-rw-r--r--lib/fuzzer/FuzzerDictionary.h12
-rw-r--r--lib/fuzzer/FuzzerDriver.cpp19
-rw-r--r--lib/fuzzer/FuzzerExtFunctions.def4
-rw-r--r--lib/fuzzer/FuzzerExtFunctionsWeak.cpp5
-rw-r--r--lib/fuzzer/FuzzerExtraCounters.cpp3
-rw-r--r--lib/fuzzer/FuzzerFlags.def27
-rw-r--r--lib/fuzzer/FuzzerIO.cpp8
-rw-r--r--lib/fuzzer/FuzzerIO.h2
-rw-r--r--lib/fuzzer/FuzzerIOPosix.cpp2
-rw-r--r--lib/fuzzer/FuzzerInterface.h22
-rw-r--r--lib/fuzzer/FuzzerInternal.h26
-rw-r--r--lib/fuzzer/FuzzerLoop.cpp153
-rw-r--r--lib/fuzzer/FuzzerMain.cpp2
-rw-r--r--lib/fuzzer/FuzzerMutate.cpp93
-rw-r--r--lib/fuzzer/FuzzerMutate.h21
-rw-r--r--lib/fuzzer/FuzzerOptions.h11
-rw-r--r--lib/fuzzer/FuzzerShmemPosix.cpp9
-rw-r--r--lib/fuzzer/FuzzerTracePC.cpp240
-rw-r--r--lib/fuzzer/FuzzerTracePC.h53
-rw-r--r--lib/fuzzer/FuzzerUtil.cpp19
-rw-r--r--lib/fuzzer/FuzzerUtil.h4
-rw-r--r--lib/fuzzer/FuzzerUtilFuchsia.cpp395
-rw-r--r--lib/fuzzer/FuzzerUtilLinux.cpp5
-rw-r--r--lib/fuzzer/FuzzerUtilPosix.cpp3
-rw-r--r--lib/fuzzer/afl/afl_driver.cpp39
-rwxr-xr-xlib/fuzzer/build.sh2
-rw-r--r--lib/fuzzer/dataflow/DataFlow.cpp217
-rwxr-xr-xlib/fuzzer/scripts/collect_data_flow.py79
-rwxr-xr-xlib/fuzzer/scripts/merge_data_flow.py36
-rwxr-xr-xlib/fuzzer/scripts/unbalanced_allocs.py10
-rw-r--r--lib/fuzzer/tests/CMakeLists.txt32
-rw-r--r--lib/fuzzer/tests/FuzzerUnittest.cpp29
-rw-r--r--lib/hwasan/.clang-format1
-rw-r--r--lib/hwasan/CMakeLists.txt52
-rw-r--r--lib/hwasan/hwasan.cc89
-rw-r--r--lib/hwasan/hwasan.h30
-rw-r--r--lib/hwasan/hwasan_allocator.cc84
-rw-r--r--lib/hwasan/hwasan_dynamic_shadow.cc132
-rw-r--r--lib/hwasan/hwasan_dynamic_shadow.h27
-rw-r--r--lib/hwasan/hwasan_flags.inc4
-rw-r--r--lib/hwasan/hwasan_interceptors.cc55
-rw-r--r--lib/hwasan/hwasan_interface_internal.h18
-rw-r--r--lib/hwasan/hwasan_linux.cc294
-rw-r--r--lib/hwasan/hwasan_mapping.h85
-rw-r--r--lib/hwasan/hwasan_new_delete.cc5
-rw-r--r--lib/hwasan/hwasan_poisoning.cc3
-rw-r--r--lib/hwasan/hwasan_report.cc10
-rw-r--r--lib/hwasan/hwasan_report.h36
-rw-r--r--lib/hwasan/hwasan_thread.cc39
-rw-r--r--lib/hwasan/hwasan_thread.h5
-rw-r--r--lib/interception/.clang-format1
-rw-r--r--lib/interception/CMakeLists.txt10
-rw-r--r--lib/interception/interception.h19
-rw-r--r--lib/interception/interception_linux.cc6
-rw-r--r--lib/interception/interception_linux.h8
-rw-r--r--lib/interception/interception_win.cc1
-rw-r--r--lib/lsan/.clang-format1
-rw-r--r--lib/lsan/CMakeLists.txt14
-rw-r--r--lib/lsan/lsan.cc2
-rw-r--r--lib/lsan/lsan_allocator.cc74
-rw-r--r--lib/lsan/lsan_allocator.h4
-rw-r--r--lib/lsan/lsan_common.cc52
-rw-r--r--lib/lsan/lsan_common.h7
-rw-r--r--lib/lsan/lsan_common_mac.cc12
-rw-r--r--lib/lsan/lsan_interceptors.cc57
-rw-r--r--lib/lsan/lsan_malloc_mac.cc3
-rw-r--r--lib/lsan/lsan_thread.cc5
-rw-r--r--lib/msan/.clang-format1
-rw-r--r--lib/msan/CMakeLists.txt24
-rw-r--r--lib/msan/msan.cc15
-rw-r--r--lib/msan/msan.h21
-rw-r--r--lib/msan/msan_allocator.cc48
-rw-r--r--lib/msan/msan_interceptors.cc136
-rw-r--r--lib/msan/msan_interface_internal.h6
-rw-r--r--lib/msan/msan_linux.cc3
-rw-r--r--lib/msan/msan_new_delete.cc5
-rw-r--r--lib/msan/msan_poisoning.cc3
-rw-r--r--lib/msan/msan_report.cc5
-rw-r--r--lib/msan/msan_report.h34
-rw-r--r--lib/msan/tests/CMakeLists.txt8
-rw-r--r--lib/msan/tests/msan_test.cc145
-rw-r--r--lib/profile/CMakeLists.txt13
-rw-r--r--lib/profile/GCDAProfiling.c187
-rw-r--r--lib/profile/InstrProfData.inc2
-rw-r--r--lib/profile/InstrProfiling.h12
-rw-r--r--lib/profile/InstrProfilingFile.c73
-rw-r--r--lib/profile/InstrProfilingMerge.c5
-rw-r--r--lib/profile/InstrProfilingMergeFile.c16
-rw-r--r--lib/profile/InstrProfilingPlatformFuchsia.c183
-rw-r--r--lib/profile/InstrProfilingPlatformLinux.c2
-rw-r--r--lib/profile/InstrProfilingPort.h2
-rw-r--r--lib/profile/InstrProfilingUtil.c18
-rw-r--r--lib/profile/InstrProfilingValue.c33
-rw-r--r--lib/safestack/.clang-format1
-rw-r--r--lib/safestack/safestack.cc30
-rw-r--r--lib/sanitizer_common/.clang-format1
-rw-r--r--lib/sanitizer_common/CMakeLists.txt103
-rw-r--r--lib/sanitizer_common/sanitizer_allocator.cc64
-rw-r--r--lib/sanitizer_common/sanitizer_allocator.h27
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_bytemap.h4
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_checks.h8
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_combined.h12
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_internal.h16
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_local_cache.h94
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_primary32.h72
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_primary64.h27
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_report.cc125
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_report.h38
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_secondary.h85
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_size_class_map.h31
-rw-r--r--lib/sanitizer_common/sanitizer_allocator_stats.h2
-rw-r--r--lib/sanitizer_common/sanitizer_atomic_clang_other.h2
-rw-r--r--lib/sanitizer_common/sanitizer_common.cc41
-rw-r--r--lib/sanitizer_common/sanitizer_common.h214
-rw-r--r--lib/sanitizer_common/sanitizer_common_interceptors.inc915
-rwxr-xr-xlib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc6
-rw-r--r--lib/sanitizer_common/sanitizer_common_interface.inc1
-rw-r--r--lib/sanitizer_common/sanitizer_common_libcdep.cc258
-rw-r--r--lib/sanitizer_common/sanitizer_common_nolibc.cc1
-rw-r--r--lib/sanitizer_common/sanitizer_coverage_fuchsia.cc22
-rw-r--r--lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc3
-rw-r--r--lib/sanitizer_common/sanitizer_deadlock_detector2.cc8
-rw-r--r--lib/sanitizer_common/sanitizer_errno.h3
-rw-r--r--lib/sanitizer_common/sanitizer_file.cc59
-rw-r--r--lib/sanitizer_common/sanitizer_flag_parser.h4
-rw-r--r--lib/sanitizer_common/sanitizer_flags.cc8
-rw-r--r--lib/sanitizer_common/sanitizer_flags.inc12
-rw-r--r--lib/sanitizer_common/sanitizer_fuchsia.cc163
-rw-r--r--lib/sanitizer_common/sanitizer_getauxval.h7
-rw-r--r--lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc1489
-rw-r--r--lib/sanitizer_common/sanitizer_interface_internal.h6
-rw-r--r--lib/sanitizer_common/sanitizer_internal_defs.h49
-rw-r--r--lib/sanitizer_common/sanitizer_libc.cc27
-rw-r--r--lib/sanitizer_common/sanitizer_libc.h7
-rw-r--r--lib/sanitizer_common/sanitizer_libignore.cc7
-rw-r--r--lib/sanitizer_common/sanitizer_libignore.h8
-rw-r--r--lib/sanitizer_common/sanitizer_linux.cc395
-rw-r--r--lib/sanitizer_common/sanitizer_linux.h34
-rw-r--r--lib/sanitizer_common/sanitizer_linux_libcdep.cc232
-rw-r--r--lib/sanitizer_common/sanitizer_linux_s390.cc2
-rw-r--r--lib/sanitizer_common/sanitizer_mac.cc69
-rw-r--r--lib/sanitizer_common/sanitizer_malloc_mac.inc35
-rw-r--r--lib/sanitizer_common/sanitizer_openbsd.cc101
-rw-r--r--lib/sanitizer_common/sanitizer_platform.h41
-rw-r--r--lib/sanitizer_common/sanitizer_platform_interceptors.h197
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc1942
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_netbsd.h1970
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_openbsd.cc279
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_openbsd.h382
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_posix.cc21
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_posix.h27
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_solaris.h12
-rw-r--r--lib/sanitizer_common/sanitizer_posix.cc7
-rw-r--r--lib/sanitizer_common/sanitizer_posix.h1
-rw-r--r--lib/sanitizer_common/sanitizer_posix_libcdep.cc62
-rw-r--r--lib/sanitizer_common/sanitizer_printf.cc38
-rw-r--r--lib/sanitizer_common/sanitizer_procmaps.h7
-rw-r--r--lib/sanitizer_common/sanitizer_procmaps_bsd.cc (renamed from lib/sanitizer_common/sanitizer_procmaps_freebsd.cc)64
-rw-r--r--lib/sanitizer_common/sanitizer_procmaps_common.cc7
-rw-r--r--lib/sanitizer_common/sanitizer_quarantine.h7
-rw-r--r--lib/sanitizer_common/sanitizer_report_decorator.h5
-rw-r--r--lib/sanitizer_common/sanitizer_rtems.cc282
-rw-r--r--lib/sanitizer_common/sanitizer_rtems.h21
-rw-r--r--lib/sanitizer_common/sanitizer_stackdepot.cc6
-rw-r--r--lib/sanitizer_common/sanitizer_stacktrace.cc8
-rw-r--r--lib/sanitizer_common/sanitizer_stacktrace.h13
-rw-r--r--lib/sanitizer_common/sanitizer_stacktrace_printer.cc6
-rw-r--r--lib/sanitizer_common/sanitizer_stacktrace_sparc.cc58
-rw-r--r--lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc42
-rw-r--r--lib/sanitizer_common/sanitizer_suppressions.cc2
-rw-r--r--lib/sanitizer_common/sanitizer_symbolizer.h7
-rw-r--r--lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h40
-rw-r--r--lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc11
-rw-r--r--lib/sanitizer_common/sanitizer_symbolizer_markup.cc (renamed from lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc)78
-rw-r--r--lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc7
-rw-r--r--lib/sanitizer_common/sanitizer_symbolizer_report.cc282
-rw-r--r--lib/sanitizer_common/sanitizer_symbolizer_rtems.h41
-rw-r--r--lib/sanitizer_common/sanitizer_symbolizer_win.cc4
-rw-r--r--lib/sanitizer_common/sanitizer_syscall_generic.inc3
-rw-r--r--lib/sanitizer_common/sanitizer_syscalls_netbsd.inc3786
-rw-r--r--lib/sanitizer_common/sanitizer_tls_get_addr.cc3
-rw-r--r--lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc2
-rw-r--r--lib/sanitizer_common/sanitizer_unwind_win.cc75
-rw-r--r--lib/sanitizer_common/sanitizer_vector.h4
-rw-r--r--lib/sanitizer_common/sanitizer_win.cc121
-rwxr-xr-xlib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh7
-rw-r--r--lib/sanitizer_common/symbolizer/scripts/global_symbols.txt9
-rw-r--r--lib/sanitizer_common/tests/CMakeLists.txt14
-rw-r--r--lib/sanitizer_common/tests/sanitizer_allocator_test.cc27
-rw-r--r--lib/sanitizer_common/tests/sanitizer_common_test.cc97
-rw-r--r--lib/sanitizer_common/tests/sanitizer_libc_test.cc89
-rw-r--r--lib/sanitizer_common/tests/sanitizer_linux_test.cc56
-rw-r--r--lib/sanitizer_common/tests/sanitizer_printf_test.cc8
-rw-r--r--lib/sanitizer_common/tests/sanitizer_procmaps_test.cc6
-rw-r--r--lib/sanitizer_common/tests/sanitizer_test_utils.h8
-rw-r--r--lib/sanitizer_common/tests/sanitizer_vector_test.cc17
-rw-r--r--lib/scudo/CMakeLists.txt96
-rw-r--r--lib/scudo/scudo_allocator.cpp495
-rw-r--r--lib/scudo/scudo_allocator.h36
-rw-r--r--lib/scudo/scudo_allocator_combined.h15
-rw-r--r--lib/scudo/scudo_allocator_secondary.h173
-rw-r--r--lib/scudo/scudo_errors.cpp77
-rw-r--r--lib/scudo/scudo_errors.h35
-rw-r--r--lib/scudo/scudo_flags.cpp20
-rw-r--r--lib/scudo/scudo_interceptors.cpp75
-rw-r--r--lib/scudo/scudo_interface_internal.h13
-rw-r--r--lib/scudo/scudo_malloc.cpp85
-rw-r--r--lib/scudo/scudo_new_delete.cpp108
-rw-r--r--lib/scudo/scudo_platform.h16
-rw-r--r--lib/scudo/scudo_termination.cpp2
-rw-r--r--lib/scudo/scudo_tsd.h24
-rw-r--r--lib/scudo/scudo_tsd_exclusive.cpp4
-rw-r--r--lib/scudo/scudo_tsd_exclusive.inc4
-rw-r--r--lib/scudo/scudo_tsd_shared.cpp67
-rw-r--r--lib/scudo/scudo_tsd_shared.inc16
-rw-r--r--lib/scudo/scudo_utils.cpp21
-rw-r--r--lib/stats/CMakeLists.txt6
-rw-r--r--lib/stats/stats.cc2
-rw-r--r--lib/tsan/.clang-format1
-rw-r--r--lib/tsan/CMakeLists.txt34
-rw-r--r--lib/tsan/dd/CMakeLists.txt3
-rwxr-xr-xlib/tsan/go/buildgo.sh25
-rw-r--r--lib/tsan/go/test.c19
-rw-r--r--lib/tsan/go/tsan_go.cc8
-rw-r--r--lib/tsan/rtl/tsan_debugging.cc7
-rw-r--r--lib/tsan/rtl/tsan_interceptors.cc125
-rw-r--r--lib/tsan/rtl/tsan_interceptors_mac.cc37
-rw-r--r--lib/tsan/rtl/tsan_interface.h13
-rw-r--r--lib/tsan/rtl/tsan_malloc_mac.cc10
-rw-r--r--lib/tsan/rtl/tsan_mman.cc50
-rw-r--r--lib/tsan/rtl/tsan_new_delete.cc12
-rw-r--r--lib/tsan/rtl/tsan_platform.h105
-rw-r--r--lib/tsan/rtl/tsan_platform_linux.cc21
-rw-r--r--lib/tsan/rtl/tsan_platform_posix.cc59
-rw-r--r--lib/tsan/rtl/tsan_rtl.cc28
-rw-r--r--lib/tsan/rtl/tsan_rtl.h6
-rw-r--r--lib/tsan/rtl/tsan_rtl_mutex.cc6
-rw-r--r--lib/tsan/rtl/tsan_rtl_report.cc4
-rw-r--r--lib/tsan/rtl/tsan_stack_trace.cc5
-rw-r--r--lib/tsan/rtl/tsan_stack_trace.h4
-rw-r--r--lib/tsan/rtl/tsan_suppressions.cc2
-rw-r--r--lib/tsan/rtl/tsan_symbolize.cc41
-rw-r--r--lib/tsan/rtl/tsan_sync.cc3
-rw-r--r--lib/tsan/tests/CMakeLists.txt3
-rw-r--r--lib/tsan/tests/unit/tsan_mman_test.cc45
-rw-r--r--lib/ubsan/CMakeLists.txt49
-rw-r--r--lib/ubsan/ubsan_diag.cc8
-rw-r--r--lib/ubsan/ubsan_diag.h20
-rw-r--r--lib/ubsan/ubsan_flags.cc13
-rw-r--r--lib/ubsan/ubsan_flags.inc3
-rw-r--r--lib/ubsan/ubsan_handlers.cc122
-rw-r--r--lib/ubsan/ubsan_handlers.h2
-rw-r--r--lib/ubsan/ubsan_handlers_cxx.cc47
-rw-r--r--lib/ubsan/ubsan_init_standalone_preinit.cc36
-rw-r--r--lib/ubsan/ubsan_interface.inc2
-rw-r--r--lib/ubsan/ubsan_monitor.cc76
-rw-r--r--lib/ubsan/ubsan_monitor.h49
-rw-r--r--lib/ubsan/ubsan_platform.h11
-rw-r--r--lib/ubsan/ubsan_signals_standalone.cc23
-rw-r--r--lib/ubsan/ubsan_win_weak_interception.cc1
-rw-r--r--lib/ubsan_minimal/CMakeLists.txt2
-rw-r--r--lib/ubsan_minimal/ubsan_minimal_handlers.cc13
-rw-r--r--lib/xray/CMakeLists.txt225
-rw-r--r--lib/xray/tests/CMakeLists.txt74
-rw-r--r--lib/xray/tests/unit/CMakeLists.txt8
-rw-r--r--lib/xray/tests/unit/allocator_test.cc42
-rw-r--r--lib/xray/tests/unit/buffer_queue_test.cc8
-rw-r--r--lib/xray/tests/unit/fdr_logging_test.cc17
-rw-r--r--lib/xray/tests/unit/function_call_trie_test.cc286
-rw-r--r--lib/xray/tests/unit/profile_collector_test.cc179
-rw-r--r--lib/xray/tests/unit/segmented_array_test.cc200
-rw-r--r--lib/xray/xray_AArch64.cc6
-rw-r--r--lib/xray/xray_allocator.h129
-rw-r--r--lib/xray/xray_arm.cc6
-rw-r--r--lib/xray/xray_basic_flags.cc50
-rw-r--r--lib/xray/xray_basic_flags.h38
-rw-r--r--lib/xray/xray_basic_flags.inc24
-rw-r--r--lib/xray/xray_basic_logging.cc (renamed from lib/xray/xray_inmemory_log.cc)292
-rw-r--r--lib/xray/xray_basic_logging.h (renamed from lib/xray/xray_inmemory_log.h)3
-rw-r--r--lib/xray/xray_buffer_queue.cc52
-rw-r--r--lib/xray/xray_buffer_queue.h127
-rw-r--r--lib/xray/xray_fdr_flags.cc48
-rw-r--r--lib/xray/xray_fdr_flags.h38
-rw-r--r--lib/xray/xray_fdr_flags.inc29
-rw-r--r--lib/xray/xray_fdr_log_records.h2
-rw-r--r--lib/xray/xray_fdr_logging.cc1108
-rw-r--r--lib/xray/xray_fdr_logging_impl.h705
-rw-r--r--lib/xray/xray_flags.cc17
-rw-r--r--lib/xray/xray_flags.h2
-rw-r--r--lib/xray/xray_flags.inc23
-rw-r--r--lib/xray/xray_function_call_trie.h455
-rw-r--r--lib/xray/xray_init.cc32
-rw-r--r--lib/xray/xray_interface.cc131
-rw-r--r--lib/xray/xray_interface_internal.h10
-rw-r--r--lib/xray/xray_log_interface.cc128
-rw-r--r--lib/xray/xray_mips.cc6
-rw-r--r--lib/xray/xray_mips64.cc6
-rw-r--r--lib/xray/xray_powerpc64.cc6
-rw-r--r--lib/xray/xray_profile_collector.cc318
-rw-r--r--lib/xray/xray_profile_collector.h88
-rw-r--r--lib/xray/xray_profiling.cc372
-rw-r--r--lib/xray/xray_profiling_flags.cc40
-rw-r--r--lib/xray/xray_profiling_flags.h39
-rw-r--r--lib/xray/xray_profiling_flags.inc29
-rw-r--r--lib/xray/xray_recursion_guard.h57
-rw-r--r--lib/xray/xray_segmented_array.h375
-rw-r--r--lib/xray/xray_trampoline_x86_64.S127
-rw-r--r--lib/xray/xray_utils.cc16
-rw-r--r--lib/xray/xray_utils.h30
-rw-r--r--lib/xray/xray_x86_64.cc75
-rw-r--r--lib/xray/xray_x86_64.inc5
411 files changed, 30548 insertions, 5156 deletions
diff --git a/lib/asan/.clang-format b/lib/asan/.clang-format
index f6cb8ad931f5..560308c91dee 100644
--- a/lib/asan/.clang-format
+++ b/lib/asan/.clang-format
@@ -1 +1,2 @@
BasedOnStyle: Google
+AllowShortIfStatementsOnASingleLine: false
diff --git a/lib/asan/CMakeLists.txt b/lib/asan/CMakeLists.txt
index fbd72f69298f..2ae5c85ecefb 100644
--- a/lib/asan/CMakeLists.txt
+++ b/lib/asan/CMakeLists.txt
@@ -23,6 +23,7 @@ set(ASAN_SOURCES
asan_posix.cc
asan_premap_shadow.cc
asan_report.cc
+ asan_rtems.cc
asan_rtl.cc
asan_shadow_setup.cc
asan_stack.cc
@@ -37,6 +38,34 @@ set(ASAN_CXX_SOURCES
set(ASAN_PREINIT_SOURCES
asan_preinit.cc)
+SET(ASAN_HEADERS
+ asan_activation.h
+ asan_activation_flags.inc
+ asan_allocator.h
+ asan_descriptions.h
+ asan_errors.h
+ asan_fake_stack.h
+ asan_flags.h
+ asan_flags.inc
+ asan_init_version.h
+ asan_interceptors.h
+ asan_interceptors_memintrinsics.h
+ asan_interface.inc
+ asan_interface_internal.h
+ asan_internal.h
+ asan_lock.h
+ asan_malloc_local.h
+ asan_mapping.h
+ asan_mapping_myriad.h
+ asan_poisoning.h
+ asan_premap_shadow.h
+ asan_report.h
+ asan_scariness_score.h
+ asan_stack.h
+ asan_stats.h
+ asan_suppressions.h
+ asan_thread.h)
+
include_directories(..)
set(ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS})
@@ -46,20 +75,6 @@ append_rtti_flag(OFF ASAN_CFLAGS)
set(ASAN_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
-if(ANDROID)
-# On Android, -z global does not do what it is documented to do.
-# On Android, -z global moves the library ahead in the lookup order,
-# placing it right after the LD_PRELOADs. This is used to compensate for the fact
-# that Android linker does not look at the dependencies of the main executable
-# that aren't dependencies of the current DSO when resolving symbols from said DSO.
-# As a net result, this allows running ASan executables without LD_PRELOAD-ing the
-# ASan runtime library.
-# The above is applicable to L MR1 or newer.
- if (COMPILER_RT_HAS_Z_GLOBAL)
- list(APPEND ASAN_DYNAMIC_LINK_FLAGS -Wl,-z,global)
- endif()
-endif()
-
set(ASAN_DYNAMIC_DEFINITIONS
${ASAN_COMMON_DEFINITIONS} ASAN_DYNAMIC=1)
append_list_if(WIN32 INTERCEPTION_DYNAMIC_CRT ASAN_DYNAMIC_DEFINITIONS)
@@ -83,21 +98,28 @@ add_compiler_rt_object_libraries(RTAsan_dynamic
OS ${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${ASAN_SUPPORTED_ARCH}
SOURCES ${ASAN_SOURCES} ${ASAN_CXX_SOURCES}
+ ADDITIONAL_HEADERS ${ASAN_HEADERS}
CFLAGS ${ASAN_DYNAMIC_CFLAGS}
DEFS ${ASAN_DYNAMIC_DEFINITIONS})
if(NOT APPLE)
add_compiler_rt_object_libraries(RTAsan
ARCHS ${ASAN_SUPPORTED_ARCH}
- SOURCES ${ASAN_SOURCES} CFLAGS ${ASAN_CFLAGS}
+ SOURCES ${ASAN_SOURCES}
+ ADDITIONAL_HEADERS ${ASAN_HEADERS}
+ CFLAGS ${ASAN_CFLAGS}
DEFS ${ASAN_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(RTAsan_cxx
ARCHS ${ASAN_SUPPORTED_ARCH}
- SOURCES ${ASAN_CXX_SOURCES} CFLAGS ${ASAN_CFLAGS}
+ SOURCES ${ASAN_CXX_SOURCES}
+ ADDITIONAL_HEADERS ${ASAN_HEADERS}
+ CFLAGS ${ASAN_CFLAGS}
DEFS ${ASAN_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(RTAsan_preinit
ARCHS ${ASAN_SUPPORTED_ARCH}
- SOURCES ${ASAN_PREINIT_SOURCES} CFLAGS ${ASAN_CFLAGS}
+ SOURCES ${ASAN_PREINIT_SOURCES}
+ ADDITIONAL_HEADERS ${ASAN_HEADERS}
+ CFLAGS ${ASAN_CFLAGS}
DEFS ${ASAN_COMMON_DEFINITIONS})
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc "")
@@ -125,6 +147,8 @@ if(APPLE)
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
+ RTSanitizerCommonCoverage
+ RTSanitizerCommonSymbolizer
RTLSanCommon
RTUbsan
CFLAGS ${ASAN_DYNAMIC_CFLAGS}
@@ -138,6 +162,8 @@ else()
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
+ RTSanitizerCommonCoverage
+ RTSanitizerCommonSymbolizer
RTLSanCommon
RTUbsan)
@@ -223,7 +249,7 @@ else()
DEFS ${ASAN_DYNAMIC_DEFINITIONS}
PARENT_TARGET asan)
- if (UNIX AND NOT ${arch} STREQUAL "i386")
+ if (SANITIZER_USE_SYMBOLS AND NOT ${arch} STREQUAL "i386")
add_sanitizer_rt_symbols(clang_rt.asan_cxx
ARCHS ${arch})
add_dependencies(asan clang_rt.asan_cxx-${arch}-symbols)
diff --git a/lib/asan/asan_allocator.cc b/lib/asan/asan_allocator.cc
index a437ae1cd3b1..c0fad4fa042b 100644
--- a/lib/asan/asan_allocator.cc
+++ b/lib/asan/asan_allocator.cc
@@ -134,8 +134,9 @@ struct AsanChunk: ChunkBase {
};
struct QuarantineCallback {
- explicit QuarantineCallback(AllocatorCache *cache)
- : cache_(cache) {
+ QuarantineCallback(AllocatorCache *cache, BufferedStackTrace *stack)
+ : cache_(cache),
+ stack_(stack) {
}
void Recycle(AsanChunk *m) {
@@ -168,7 +169,7 @@ struct QuarantineCallback {
void *res = get_allocator().Allocate(cache_, size, 1);
// TODO(alekseys): Consider making quarantine OOM-friendly.
if (UNLIKELY(!res))
- return DieOnFailure::OnOOM();
+ ReportOutOfMemory(size, stack_);
return res;
}
@@ -176,7 +177,9 @@ struct QuarantineCallback {
get_allocator().Deallocate(cache_, p);
}
- AllocatorCache *cache_;
+ private:
+ AllocatorCache* const cache_;
+ BufferedStackTrace* const stack_;
};
typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine;
@@ -397,8 +400,11 @@ struct Allocator {
AllocType alloc_type, bool can_fill) {
if (UNLIKELY(!asan_inited))
AsanInitFromRtl();
- if (RssLimitExceeded())
- return AsanAllocator::FailureHandler::OnOOM();
+ if (RssLimitExceeded()) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportRssLimitExceeded(stack);
+ }
Flags &fl = *flags();
CHECK(stack);
const uptr min_alignment = SHADOW_GRANULARITY;
@@ -431,9 +437,13 @@ struct Allocator {
}
CHECK(IsAligned(needed_size, min_alignment));
if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize) {
- Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
- (void*)size);
- return AsanAllocator::FailureHandler::OnBadRequest();
+ if (AllocatorMayReturnNull()) {
+ Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
+ (void*)size);
+ return nullptr;
+ }
+ ReportAllocationSizeTooBig(size, needed_size, kMaxAllowedMallocSize,
+ stack);
}
AsanThread *t = GetCurrentThread();
@@ -446,8 +456,12 @@ struct Allocator {
AllocatorCache *cache = &fallback_allocator_cache;
allocated = allocator.Allocate(cache, needed_size, 8);
}
- if (!allocated)
- return nullptr;
+ if (UNLIKELY(!allocated)) {
+ SetAllocatorOutOfMemory();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportOutOfMemory(size, stack);
+ }
if (*(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0 && CanPoisonMemory()) {
// Heap poisoning is enabled, but the allocator provides an unpoisoned
@@ -583,13 +597,13 @@ struct Allocator {
if (t) {
AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
AllocatorCache *ac = GetAllocatorCache(ms);
- quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), m,
- m->UsedSize());
+ quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac, stack), m,
+ m->UsedSize());
} else {
SpinMutexLock l(&fallback_mutex);
AllocatorCache *ac = &fallback_allocator_cache;
- quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), m,
- m->UsedSize());
+ quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac, stack),
+ m, m->UsedSize());
}
}
@@ -660,8 +674,11 @@ struct Allocator {
}
void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
- if (CheckForCallocOverflow(size, nmemb))
- return AsanAllocator::FailureHandler::OnBadRequest();
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportCallocOverflow(nmemb, size, stack);
+ }
void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC, false);
// If the memory comes from the secondary allocator no need to clear it
// as it comes directly from mmap.
@@ -677,9 +694,9 @@ struct Allocator {
ReportFreeNotMalloced((uptr)ptr, stack);
}
- void CommitBack(AsanThreadLocalMallocStorage *ms) {
+ void CommitBack(AsanThreadLocalMallocStorage *ms, BufferedStackTrace *stack) {
AllocatorCache *ac = GetAllocatorCache(ms);
- quarantine.Drain(GetQuarantineCache(ms), QuarantineCallback(ac));
+ quarantine.Drain(GetQuarantineCache(ms), QuarantineCallback(ac, stack));
allocator.SwallowCache(ac);
}
@@ -739,17 +756,19 @@ struct Allocator {
return AsanChunkView(m1);
}
- void Purge() {
+ void Purge(BufferedStackTrace *stack) {
AsanThread *t = GetCurrentThread();
if (t) {
AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
quarantine.DrainAndRecycle(GetQuarantineCache(ms),
- QuarantineCallback(GetAllocatorCache(ms)));
+ QuarantineCallback(GetAllocatorCache(ms),
+ stack));
}
{
SpinMutexLock l(&fallback_mutex);
quarantine.DrainAndRecycle(&fallback_quarantine_cache,
- QuarantineCallback(&fallback_allocator_cache));
+ QuarantineCallback(&fallback_allocator_cache,
+ stack));
}
allocator.ForceReleaseToOS();
@@ -836,7 +855,8 @@ AsanChunkView FindHeapChunkByAllocBeg(uptr addr) {
}
void AsanThreadLocalMallocStorage::CommitBack() {
- instance.CommitBack(this);
+ GET_STACK_TRACE_MALLOC;
+ instance.CommitBack(this, &stack);
}
void PrintInternalAllocatorStats() {
@@ -883,7 +903,9 @@ void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
uptr PageSize = GetPageSizeCached();
if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
errno = errno_ENOMEM;
- return AsanAllocator::FailureHandler::OnBadRequest();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportPvallocOverflow(size, stack);
}
// pvalloc(0) should allocate one page.
size = size ? RoundUpTo(size, PageSize) : PageSize;
@@ -895,20 +917,35 @@ void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
AllocType alloc_type) {
if (UNLIKELY(!IsPowerOfTwo(alignment))) {
errno = errno_EINVAL;
- return AsanAllocator::FailureHandler::OnBadRequest();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportInvalidAllocationAlignment(alignment, stack);
}
return SetErrnoOnNull(
instance.Allocate(size, alignment, stack, alloc_type, true));
}
+void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
+ if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+ errno = errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportInvalidAlignedAllocAlignment(size, alignment, stack);
+ }
+ return SetErrnoOnNull(
+ instance.Allocate(size, alignment, stack, FROM_MALLOC, true));
+}
+
int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
BufferedStackTrace *stack) {
if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
- AsanAllocator::FailureHandler::OnBadRequest();
- return errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return errno_EINVAL;
+ ReportInvalidPosixMemalignAlignment(alignment, stack);
}
void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true);
if (UNLIKELY(!ptr))
+ // OOM error is already taken care of by Allocate.
return errno_ENOMEM;
CHECK(IsAligned((uptr)ptr, alignment));
*memptr = ptr;
@@ -1054,7 +1091,8 @@ uptr __sanitizer_get_allocated_size(const void *p) {
}
void __sanitizer_purge_allocator() {
- instance.Purge();
+ GET_STACK_TRACE_MALLOC;
+ instance.Purge(&stack);
}
#if !SANITIZER_SUPPORTS_WEAK_HOOKS
diff --git a/lib/asan/asan_allocator.h b/lib/asan/asan_allocator.h
index 26483db4c60e..93d6f29c5bf5 100644
--- a/lib/asan/asan_allocator.h
+++ b/lib/asan/asan_allocator.h
@@ -125,11 +125,12 @@ const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
typedef DefaultSizeClassMap SizeClassMap;
# elif defined(__powerpc64__)
-const uptr kAllocatorSpace = 0xa0000000000ULL;
+const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x20000000000ULL; // 2T.
typedef DefaultSizeClassMap SizeClassMap;
# elif defined(__aarch64__) && SANITIZER_ANDROID
-const uptr kAllocatorSpace = 0x3000000000ULL;
+// Android needs to support 39, 42 and 48 bit VMA.
+const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
typedef VeryCompactSizeClassMap SizeClassMap;
# elif defined(__aarch64__)
@@ -207,6 +208,7 @@ void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack);
void *asan_valloc(uptr size, BufferedStackTrace *stack);
void *asan_pvalloc(uptr size, BufferedStackTrace *stack);
+void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack);
int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
BufferedStackTrace *stack);
uptr asan_malloc_usable_size(const void *ptr, uptr pc, uptr bp);
diff --git a/lib/asan/asan_debugging.cc b/lib/asan/asan_debugging.cc
index 37c5922dc0fc..7c877ded30cf 100644
--- a/lib/asan/asan_debugging.cc
+++ b/lib/asan/asan_debugging.cc
@@ -27,7 +27,8 @@ using namespace __asan;
static void FindInfoForStackVar(uptr addr, const char *frame_descr, uptr offset,
char *name, uptr name_size,
uptr &region_address, uptr &region_size) {
- InternalMmapVector<StackVarDescr> vars(16);
+ InternalMmapVector<StackVarDescr> vars;
+ vars.reserve(16);
if (!ParseFrameDescription(frame_descr, &vars)) {
return;
}
diff --git a/lib/asan/asan_descriptions.cc b/lib/asan/asan_descriptions.cc
index 86c6af7d9f75..cdb562d975ff 100644
--- a/lib/asan/asan_descriptions.cc
+++ b/lib/asan/asan_descriptions.cc
@@ -20,23 +20,25 @@
namespace __asan {
-// Return " (thread_name) " or an empty string if the name is empty.
-const char *ThreadNameWithParenthesis(AsanThreadContext *t, char buff[],
- uptr buff_len) {
- const char *name = t->name;
- if (name[0] == '\0') return "";
- buff[0] = 0;
- internal_strncat(buff, " (", 3);
- internal_strncat(buff, name, buff_len - 4);
- internal_strncat(buff, ")", 2);
- return buff;
+AsanThreadIdAndName::AsanThreadIdAndName(AsanThreadContext *t) {
+ Init(t->tid, t->name);
}
-const char *ThreadNameWithParenthesis(u32 tid, char buff[], uptr buff_len) {
- if (tid == kInvalidTid) return "";
- asanThreadRegistry().CheckLocked();
- AsanThreadContext *t = GetThreadContextByTidLocked(tid);
- return ThreadNameWithParenthesis(t, buff, buff_len);
+AsanThreadIdAndName::AsanThreadIdAndName(u32 tid) {
+ if (tid == kInvalidTid) {
+ Init(tid, "");
+ } else {
+ asanThreadRegistry().CheckLocked();
+ AsanThreadContext *t = GetThreadContextByTidLocked(tid);
+ Init(tid, t->name);
+ }
+}
+
+void AsanThreadIdAndName::Init(u32 tid, const char *tname) {
+ int len = internal_snprintf(name, sizeof(name), "T%d", tid);
+ CHECK(((unsigned int)len) < sizeof(name));
+ if (tname[0] != '\0')
+ internal_snprintf(&name[len], sizeof(name) - len, " (%s)", tname);
}
void DescribeThread(AsanThreadContext *context) {
@@ -47,18 +49,15 @@ void DescribeThread(AsanThreadContext *context) {
return;
}
context->announced = true;
- char tname[128];
InternalScopedString str(1024);
- str.append("Thread T%d%s", context->tid,
- ThreadNameWithParenthesis(context->tid, tname, sizeof(tname)));
+ str.append("Thread %s", AsanThreadIdAndName(context).c_str());
if (context->parent_tid == kInvalidTid) {
str.append(" created by unknown thread\n");
Printf("%s", str.data());
return;
}
- str.append(
- " created by T%d%s here:\n", context->parent_tid,
- ThreadNameWithParenthesis(context->parent_tid, tname, sizeof(tname)));
+ str.append(" created by %s here:\n",
+ AsanThreadIdAndName(context->parent_tid).c_str());
Printf("%s", str.data());
StackDepotGet(context->stack_id).Print();
// Recursively described parent thread if needed.
@@ -358,10 +357,9 @@ bool GlobalAddressDescription::PointsInsideTheSameVariable(
void StackAddressDescription::Print() const {
Decorator d;
- char tname[128];
Printf("%s", d.Location());
- Printf("Address %p is located in stack of thread T%d%s", addr, tid,
- ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
+ Printf("Address %p is located in stack of thread %s", addr,
+ AsanThreadIdAndName(tid).c_str());
if (!frame_descr) {
Printf("%s\n", d.Default());
@@ -380,7 +378,8 @@ void StackAddressDescription::Print() const {
StackTrace alloca_stack(&frame_pc, 1);
alloca_stack.Print();
- InternalMmapVector<StackVarDescr> vars(16);
+ InternalMmapVector<StackVarDescr> vars;
+ vars.reserve(16);
if (!ParseFrameDescription(frame_descr, &vars)) {
Printf(
"AddressSanitizer can't parse the stack frame "
@@ -402,7 +401,7 @@ void StackAddressDescription::Print() const {
}
Printf(
"HINT: this may be a false positive if your program uses "
- "some custom stack unwind mechanism or swapcontext\n");
+ "some custom stack unwind mechanism, swapcontext or vfork\n");
if (SANITIZER_WINDOWS)
Printf(" (longjmp, SEH and C++ exceptions *are* supported)\n");
else
@@ -418,26 +417,19 @@ void HeapAddressDescription::Print() const {
AsanThreadContext *alloc_thread = GetThreadContextByTidLocked(alloc_tid);
StackTrace alloc_stack = GetStackTraceFromId(alloc_stack_id);
- char tname[128];
Decorator d;
AsanThreadContext *free_thread = nullptr;
if (free_tid != kInvalidTid) {
free_thread = GetThreadContextByTidLocked(free_tid);
- Printf("%sfreed by thread T%d%s here:%s\n", d.Allocation(),
- free_thread->tid,
- ThreadNameWithParenthesis(free_thread, tname, sizeof(tname)),
- d.Default());
+ Printf("%sfreed by thread %s here:%s\n", d.Allocation(),
+ AsanThreadIdAndName(free_thread).c_str(), d.Default());
StackTrace free_stack = GetStackTraceFromId(free_stack_id);
free_stack.Print();
- Printf("%spreviously allocated by thread T%d%s here:%s\n", d.Allocation(),
- alloc_thread->tid,
- ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)),
- d.Default());
+ Printf("%spreviously allocated by thread %s here:%s\n", d.Allocation(),
+ AsanThreadIdAndName(alloc_thread).c_str(), d.Default());
} else {
- Printf("%sallocated by thread T%d%s here:%s\n", d.Allocation(),
- alloc_thread->tid,
- ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)),
- d.Default());
+ Printf("%sallocated by thread %s here:%s\n", d.Allocation(),
+ AsanThreadIdAndName(alloc_thread).c_str(), d.Default());
}
alloc_stack.Print();
DescribeThread(GetCurrentThread());
diff --git a/lib/asan/asan_descriptions.h b/lib/asan/asan_descriptions.h
index 1a1b01cf20cd..5c2d7662a35a 100644
--- a/lib/asan/asan_descriptions.h
+++ b/lib/asan/asan_descriptions.h
@@ -26,9 +26,20 @@ void DescribeThread(AsanThreadContext *context);
static inline void DescribeThread(AsanThread *t) {
if (t) DescribeThread(t->context());
}
-const char *ThreadNameWithParenthesis(AsanThreadContext *t, char buff[],
- uptr buff_len);
-const char *ThreadNameWithParenthesis(u32 tid, char buff[], uptr buff_len);
+
+class AsanThreadIdAndName {
+ public:
+ explicit AsanThreadIdAndName(AsanThreadContext *t);
+ explicit AsanThreadIdAndName(u32 tid);
+
+ // Contains "T%tid (%name)" or "T%tid" if the name is empty.
+ const char *c_str() const { return &name[0]; }
+
+ private:
+ void Init(u32 tid, const char *tname);
+
+ char name[128];
+};
class Decorator : public __sanitizer::SanitizerCommonDecorator {
public:
diff --git a/lib/asan/asan_errors.cc b/lib/asan/asan_errors.cc
index 0f4a3abff9a7..33d0613f79f4 100644
--- a/lib/asan/asan_errors.cc
+++ b/lib/asan/asan_errors.cc
@@ -45,13 +45,11 @@ void ErrorDeadlySignal::Print() {
void ErrorDoubleFree::Print() {
Decorator d;
- Printf("%s", d.Warning());
- char tname[128];
+ Printf("%s", d.Error());
Report(
- "ERROR: AddressSanitizer: attempting %s on %p in "
- "thread T%d%s:\n",
- scariness.GetDescription(), addr_description.addr, tid,
- ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
+ "ERROR: AddressSanitizer: attempting %s on %p in thread %s:\n",
+ scariness.GetDescription(), addr_description.addr,
+ AsanThreadIdAndName(tid).c_str());
Printf("%s", d.Default());
scariness.Print();
GET_STACK_TRACE_FATAL(second_free_stack->trace[0],
@@ -63,13 +61,11 @@ void ErrorDoubleFree::Print() {
void ErrorNewDeleteTypeMismatch::Print() {
Decorator d;
- Printf("%s", d.Warning());
- char tname[128];
+ Printf("%s", d.Error());
Report(
- "ERROR: AddressSanitizer: %s on %p in thread "
- "T%d%s:\n",
- scariness.GetDescription(), addr_description.addr, tid,
- ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
+ "ERROR: AddressSanitizer: %s on %p in thread %s:\n",
+ scariness.GetDescription(), addr_description.addr,
+ AsanThreadIdAndName(tid).c_str());
Printf("%s object passed to delete has wrong type:\n", d.Default());
if (delete_size != 0) {
Printf(
@@ -106,13 +102,11 @@ void ErrorNewDeleteTypeMismatch::Print() {
void ErrorFreeNotMalloced::Print() {
Decorator d;
- Printf("%s", d.Warning());
- char tname[128];
+ Printf("%s", d.Error());
Report(
"ERROR: AddressSanitizer: attempting free on address "
- "which was not malloc()-ed: %p in thread T%d%s\n",
- addr_description.Address(), tid,
- ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
+ "which was not malloc()-ed: %p in thread %s\n",
+ addr_description.Address(), AsanThreadIdAndName(tid).c_str());
Printf("%s", d.Default());
CHECK_GT(free_stack->size, 0);
scariness.Print();
@@ -129,7 +123,7 @@ void ErrorAllocTypeMismatch::Print() {
"operator delete []"};
CHECK_NE(alloc_type, dealloc_type);
Decorator d;
- Printf("%s", d.Warning());
+ Printf("%s", d.Error());
Report("ERROR: AddressSanitizer: %s (%s vs %s) on %p\n",
scariness.GetDescription(),
alloc_names[alloc_type], dealloc_names[dealloc_type],
@@ -148,7 +142,7 @@ void ErrorAllocTypeMismatch::Print() {
void ErrorMallocUsableSizeNotOwned::Print() {
Decorator d;
- Printf("%s", d.Warning());
+ Printf("%s", d.Error());
Report(
"ERROR: AddressSanitizer: attempting to call malloc_usable_size() for "
"pointer which is not owned: %p\n",
@@ -161,7 +155,7 @@ void ErrorMallocUsableSizeNotOwned::Print() {
void ErrorSanitizerGetAllocatedSizeNotOwned::Print() {
Decorator d;
- Printf("%s", d.Warning());
+ Printf("%s", d.Error());
Report(
"ERROR: AddressSanitizer: attempting to call "
"__sanitizer_get_allocated_size() for pointer which is not owned: %p\n",
@@ -172,11 +166,123 @@ void ErrorSanitizerGetAllocatedSizeNotOwned::Print() {
ReportErrorSummary(scariness.GetDescription(), stack);
}
+void ErrorCallocOverflow::Print() {
+ Decorator d;
+ Printf("%s", d.Error());
+ Report(
+ "ERROR: AddressSanitizer: calloc parameters overflow: count * size "
+ "(%zd * %zd) cannot be represented in type size_t (thread %s)\n",
+ count, size, AsanThreadIdAndName(tid).c_str());
+ Printf("%s", d.Default());
+ stack->Print();
+ PrintHintAllocatorCannotReturnNull();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
+void ErrorPvallocOverflow::Print() {
+ Decorator d;
+ Printf("%s", d.Error());
+ Report(
+ "ERROR: AddressSanitizer: pvalloc parameters overflow: size 0x%zx "
+ "rounded up to system page size 0x%zx cannot be represented in type "
+ "size_t (thread %s)\n",
+ size, GetPageSizeCached(), AsanThreadIdAndName(tid).c_str());
+ Printf("%s", d.Default());
+ stack->Print();
+ PrintHintAllocatorCannotReturnNull();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
+void ErrorInvalidAllocationAlignment::Print() {
+ Decorator d;
+ Printf("%s", d.Error());
+ Report(
+ "ERROR: AddressSanitizer: invalid allocation alignment: %zd, "
+ "alignment must be a power of two (thread %s)\n",
+ alignment, AsanThreadIdAndName(tid).c_str());
+ Printf("%s", d.Default());
+ stack->Print();
+ PrintHintAllocatorCannotReturnNull();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
+void ErrorInvalidAlignedAllocAlignment::Print() {
+ Decorator d;
+ Printf("%s", d.Error());
+#if SANITIZER_POSIX
+ Report("ERROR: AddressSanitizer: invalid alignment requested in "
+ "aligned_alloc: %zd, alignment must be a power of two and the "
+ "requested size 0x%zx must be a multiple of alignment "
+ "(thread %s)\n", alignment, size, AsanThreadIdAndName(tid).c_str());
+#else
+ Report("ERROR: AddressSanitizer: invalid alignment requested in "
+ "aligned_alloc: %zd, the requested size 0x%zx must be a multiple of "
+ "alignment (thread %s)\n", alignment, size,
+ AsanThreadIdAndName(tid).c_str());
+#endif
+ Printf("%s", d.Default());
+ stack->Print();
+ PrintHintAllocatorCannotReturnNull();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
+void ErrorInvalidPosixMemalignAlignment::Print() {
+ Decorator d;
+ Printf("%s", d.Error());
+ Report(
+ "ERROR: AddressSanitizer: invalid alignment requested in posix_memalign: "
+ "%zd, alignment must be a power of two and a multiple of sizeof(void*) "
+ "== %zd (thread %s)\n",
+ alignment, sizeof(void*), AsanThreadIdAndName(tid).c_str()); // NOLINT
+ Printf("%s", d.Default());
+ stack->Print();
+ PrintHintAllocatorCannotReturnNull();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
+void ErrorAllocationSizeTooBig::Print() {
+ Decorator d;
+ Printf("%s", d.Error());
+ Report(
+ "ERROR: AddressSanitizer: requested allocation size 0x%zx (0x%zx after "
+ "adjustments for alignment, red zones etc.) exceeds maximum supported "
+ "size of 0x%zx (thread %s)\n",
+ user_size, total_size, max_size, AsanThreadIdAndName(tid).c_str());
+ Printf("%s", d.Default());
+ stack->Print();
+ PrintHintAllocatorCannotReturnNull();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
+void ErrorRssLimitExceeded::Print() {
+ Decorator d;
+ Printf("%s", d.Error());
+ Report(
+ "ERROR: AddressSanitizer: specified RSS limit exceeded, currently set to "
+ "soft_rss_limit_mb=%zd\n", common_flags()->soft_rss_limit_mb);
+ Printf("%s", d.Default());
+ stack->Print();
+ PrintHintAllocatorCannotReturnNull();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
+void ErrorOutOfMemory::Print() {
+ Decorator d;
+ Printf("%s", d.Error());
+ Report(
+ "ERROR: AddressSanitizer: allocator is out of memory trying to allocate "
+ "0x%zx bytes\n", requested_size);
+ Printf("%s", d.Default());
+ stack->Print();
+ PrintHintAllocatorCannotReturnNull();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
void ErrorStringFunctionMemoryRangesOverlap::Print() {
Decorator d;
char bug_type[100];
internal_snprintf(bug_type, sizeof(bug_type), "%s-param-overlap", function);
- Printf("%s", d.Warning());
+ Printf("%s", d.Error());
Report(
"ERROR: AddressSanitizer: %s: memory ranges [%p,%p) and [%p, %p) "
"overlap\n",
@@ -193,7 +299,7 @@ void ErrorStringFunctionMemoryRangesOverlap::Print() {
void ErrorStringFunctionSizeOverflow::Print() {
Decorator d;
- Printf("%s", d.Warning());
+ Printf("%s", d.Error());
Report("ERROR: AddressSanitizer: %s: (size=%zd)\n",
scariness.GetDescription(), size);
Printf("%s", d.Default());
@@ -221,7 +327,7 @@ void ErrorBadParamsToAnnotateContiguousContainer::Print() {
void ErrorODRViolation::Print() {
Decorator d;
- Printf("%s", d.Warning());
+ Printf("%s", d.Error());
Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(),
global1.beg);
Printf("%s", d.Default());
@@ -250,7 +356,7 @@ void ErrorODRViolation::Print() {
void ErrorInvalidPointerPair::Print() {
Decorator d;
- Printf("%s", d.Warning());
+ Printf("%s", d.Error());
Report("ERROR: AddressSanitizer: %s: %p %p\n", scariness.GetDescription(),
addr1_description.Address(), addr2_description.Address());
Printf("%s", d.Default());
@@ -414,6 +520,7 @@ static void PrintLegend(InternalScopedString *str) {
PrintShadowByte(str, " ASan internal: ", kAsanInternalHeapMagic);
PrintShadowByte(str, " Left alloca redzone: ", kAsanAllocaLeftMagic);
PrintShadowByte(str, " Right alloca redzone: ", kAsanAllocaRightMagic);
+ PrintShadowByte(str, " Shadow gap: ", kAsanShadowGap);
}
static void PrintShadowBytes(InternalScopedString *str, const char *before,
@@ -453,17 +560,15 @@ static void PrintShadowMemoryForAddress(uptr addr) {
void ErrorGeneric::Print() {
Decorator d;
- Printf("%s", d.Warning());
+ Printf("%s", d.Error());
uptr addr = addr_description.Address();
Report("ERROR: AddressSanitizer: %s on address %p at pc %p bp %p sp %p\n",
bug_descr, (void *)addr, pc, bp, sp);
Printf("%s", d.Default());
- char tname[128];
- Printf("%s%s of size %zu at %p thread T%d%s%s\n", d.Access(),
+ Printf("%s%s of size %zu at %p thread %s%s\n", d.Access(),
access_size ? (is_write ? "WRITE" : "READ") : "ACCESS", access_size,
- (void *)addr, tid,
- ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.Default());
+ (void *)addr, AsanThreadIdAndName(tid).c_str(), d.Default());
scariness.Print();
GET_STACK_TRACE_FATAL(pc, bp);
diff --git a/lib/asan/asan_errors.h b/lib/asan/asan_errors.h
index 518ba0c6945e..574197ebff86 100644
--- a/lib/asan/asan_errors.h
+++ b/lib/asan/asan_errors.h
@@ -20,20 +20,30 @@
namespace __asan {
+// (*) VS2013 does not implement unrestricted unions, so we need a trivial
+// default constructor explicitly defined for each particular error.
+
+// None of the error classes own the stack traces mentioned in them.
+
struct ErrorBase {
- ErrorBase() = default;
- explicit ErrorBase(u32 tid_) : tid(tid_) {}
ScarinessScoreBase scariness;
u32 tid;
+
+ ErrorBase() = default; // (*)
+ explicit ErrorBase(u32 tid_) : tid(tid_) {}
+ ErrorBase(u32 tid_, int initial_score, const char *reason) : tid(tid_) {
+ scariness.Clear();
+ scariness.Scare(initial_score, reason);
+ }
};
struct ErrorDeadlySignal : ErrorBase {
SignalContext signal;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorDeadlySignal() = default;
+
+ ErrorDeadlySignal() = default; // (*)
ErrorDeadlySignal(u32 tid, const SignalContext &sig)
- : ErrorBase(tid), signal(sig) {
+ : ErrorBase(tid),
+ signal(sig) {
scariness.Clear();
if (signal.IsStackOverflow()) {
scariness.Scare(10, "stack-overflow");
@@ -55,125 +65,206 @@ struct ErrorDeadlySignal : ErrorBase {
};
struct ErrorDoubleFree : ErrorBase {
- // ErrorDoubleFree doesn't own the stack trace.
const BufferedStackTrace *second_free_stack;
HeapAddressDescription addr_description;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorDoubleFree() = default;
+
+ ErrorDoubleFree() = default; // (*)
ErrorDoubleFree(u32 tid, BufferedStackTrace *stack, uptr addr)
- : ErrorBase(tid), second_free_stack(stack) {
+ : ErrorBase(tid, 42, "double-free"),
+ second_free_stack(stack) {
CHECK_GT(second_free_stack->size, 0);
GetHeapAddressInformation(addr, 1, &addr_description);
- scariness.Clear();
- scariness.Scare(42, "double-free");
}
void Print();
};
struct ErrorNewDeleteTypeMismatch : ErrorBase {
- // ErrorNewDeleteTypeMismatch doesn't own the stack trace.
const BufferedStackTrace *free_stack;
HeapAddressDescription addr_description;
uptr delete_size;
uptr delete_alignment;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorNewDeleteTypeMismatch() = default;
+
+ ErrorNewDeleteTypeMismatch() = default; // (*)
ErrorNewDeleteTypeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr,
uptr delete_size_, uptr delete_alignment_)
- : ErrorBase(tid), free_stack(stack), delete_size(delete_size_),
+ : ErrorBase(tid, 10, "new-delete-type-mismatch"),
+ free_stack(stack),
+ delete_size(delete_size_),
delete_alignment(delete_alignment_) {
GetHeapAddressInformation(addr, 1, &addr_description);
- scariness.Clear();
- scariness.Scare(10, "new-delete-type-mismatch");
}
void Print();
};
struct ErrorFreeNotMalloced : ErrorBase {
- // ErrorFreeNotMalloced doesn't own the stack trace.
const BufferedStackTrace *free_stack;
AddressDescription addr_description;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorFreeNotMalloced() = default;
+
+ ErrorFreeNotMalloced() = default; // (*)
ErrorFreeNotMalloced(u32 tid, BufferedStackTrace *stack, uptr addr)
- : ErrorBase(tid),
+ : ErrorBase(tid, 40, "bad-free"),
free_stack(stack),
- addr_description(addr, /*shouldLockThreadRegistry=*/false) {
- scariness.Clear();
- scariness.Scare(40, "bad-free");
- }
+ addr_description(addr, /*shouldLockThreadRegistry=*/false) {}
void Print();
};
struct ErrorAllocTypeMismatch : ErrorBase {
- // ErrorAllocTypeMismatch doesn't own the stack trace.
const BufferedStackTrace *dealloc_stack;
HeapAddressDescription addr_description;
AllocType alloc_type, dealloc_type;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorAllocTypeMismatch() = default;
+
+ ErrorAllocTypeMismatch() = default; // (*)
ErrorAllocTypeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr,
AllocType alloc_type_, AllocType dealloc_type_)
- : ErrorBase(tid),
+ : ErrorBase(tid, 10, "alloc-dealloc-mismatch"),
dealloc_stack(stack),
alloc_type(alloc_type_),
dealloc_type(dealloc_type_) {
GetHeapAddressInformation(addr, 1, &addr_description);
- scariness.Clear();
- scariness.Scare(10, "alloc-dealloc-mismatch");
};
void Print();
};
struct ErrorMallocUsableSizeNotOwned : ErrorBase {
- // ErrorMallocUsableSizeNotOwned doesn't own the stack trace.
const BufferedStackTrace *stack;
AddressDescription addr_description;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorMallocUsableSizeNotOwned() = default;
+
+ ErrorMallocUsableSizeNotOwned() = default; // (*)
ErrorMallocUsableSizeNotOwned(u32 tid, BufferedStackTrace *stack_, uptr addr)
- : ErrorBase(tid),
+ : ErrorBase(tid, 10, "bad-malloc_usable_size"),
stack(stack_),
- addr_description(addr, /*shouldLockThreadRegistry=*/false) {
- scariness.Clear();
- scariness.Scare(10, "bad-malloc_usable_size");
- }
+ addr_description(addr, /*shouldLockThreadRegistry=*/false) {}
void Print();
};
struct ErrorSanitizerGetAllocatedSizeNotOwned : ErrorBase {
- // ErrorSanitizerGetAllocatedSizeNotOwned doesn't own the stack trace.
const BufferedStackTrace *stack;
AddressDescription addr_description;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorSanitizerGetAllocatedSizeNotOwned() = default;
+
+ ErrorSanitizerGetAllocatedSizeNotOwned() = default; // (*)
ErrorSanitizerGetAllocatedSizeNotOwned(u32 tid, BufferedStackTrace *stack_,
uptr addr)
- : ErrorBase(tid),
+ : ErrorBase(tid, 10, "bad-__sanitizer_get_allocated_size"),
stack(stack_),
- addr_description(addr, /*shouldLockThreadRegistry=*/false) {
- scariness.Clear();
- scariness.Scare(10, "bad-__sanitizer_get_allocated_size");
- }
+ addr_description(addr, /*shouldLockThreadRegistry=*/false) {}
+ void Print();
+};
+
+struct ErrorCallocOverflow : ErrorBase {
+ const BufferedStackTrace *stack;
+ uptr count;
+ uptr size;
+
+ ErrorCallocOverflow() = default; // (*)
+ ErrorCallocOverflow(u32 tid, BufferedStackTrace *stack_, uptr count_,
+ uptr size_)
+ : ErrorBase(tid, 10, "calloc-overflow"),
+ stack(stack_),
+ count(count_),
+ size(size_) {}
+ void Print();
+};
+
+struct ErrorPvallocOverflow : ErrorBase {
+ const BufferedStackTrace *stack;
+ uptr size;
+
+ ErrorPvallocOverflow() = default; // (*)
+ ErrorPvallocOverflow(u32 tid, BufferedStackTrace *stack_, uptr size_)
+ : ErrorBase(tid, 10, "pvalloc-overflow"),
+ stack(stack_),
+ size(size_) {}
+ void Print();
+};
+
+struct ErrorInvalidAllocationAlignment : ErrorBase {
+ const BufferedStackTrace *stack;
+ uptr alignment;
+
+ ErrorInvalidAllocationAlignment() = default; // (*)
+ ErrorInvalidAllocationAlignment(u32 tid, BufferedStackTrace *stack_,
+ uptr alignment_)
+ : ErrorBase(tid, 10, "invalid-allocation-alignment"),
+ stack(stack_),
+ alignment(alignment_) {}
+ void Print();
+};
+
+struct ErrorInvalidAlignedAllocAlignment : ErrorBase {
+ const BufferedStackTrace *stack;
+ uptr size;
+ uptr alignment;
+
+ ErrorInvalidAlignedAllocAlignment() = default; // (*)
+ ErrorInvalidAlignedAllocAlignment(u32 tid, BufferedStackTrace *stack_,
+ uptr size_, uptr alignment_)
+ : ErrorBase(tid, 10, "invalid-aligned-alloc-alignment"),
+ stack(stack_),
+ size(size_),
+ alignment(alignment_) {}
+ void Print();
+};
+
+struct ErrorInvalidPosixMemalignAlignment : ErrorBase {
+ const BufferedStackTrace *stack;
+ uptr alignment;
+
+ ErrorInvalidPosixMemalignAlignment() = default; // (*)
+ ErrorInvalidPosixMemalignAlignment(u32 tid, BufferedStackTrace *stack_,
+ uptr alignment_)
+ : ErrorBase(tid, 10, "invalid-posix-memalign-alignment"),
+ stack(stack_),
+ alignment(alignment_) {}
+ void Print();
+};
+
+struct ErrorAllocationSizeTooBig : ErrorBase {
+ const BufferedStackTrace *stack;
+ uptr user_size;
+ uptr total_size;
+ uptr max_size;
+
+ ErrorAllocationSizeTooBig() = default; // (*)
+ ErrorAllocationSizeTooBig(u32 tid, BufferedStackTrace *stack_,
+ uptr user_size_, uptr total_size_, uptr max_size_)
+ : ErrorBase(tid, 10, "allocation-size-too-big"),
+ stack(stack_),
+ user_size(user_size_),
+ total_size(total_size_),
+ max_size(max_size_) {}
+ void Print();
+};
+
+struct ErrorRssLimitExceeded : ErrorBase {
+ const BufferedStackTrace *stack;
+
+ ErrorRssLimitExceeded() = default; // (*)
+ ErrorRssLimitExceeded(u32 tid, BufferedStackTrace *stack_)
+ : ErrorBase(tid, 10, "rss-limit-exceeded"),
+ stack(stack_) {}
+ void Print();
+};
+
+struct ErrorOutOfMemory : ErrorBase {
+ const BufferedStackTrace *stack;
+ uptr requested_size;
+
+ ErrorOutOfMemory() = default; // (*)
+ ErrorOutOfMemory(u32 tid, BufferedStackTrace *stack_, uptr requested_size_)
+ : ErrorBase(tid, 10, "out-of-memory"),
+ stack(stack_),
+ requested_size(requested_size_) {}
void Print();
};
struct ErrorStringFunctionMemoryRangesOverlap : ErrorBase {
- // ErrorStringFunctionMemoryRangesOverlap doesn't own the stack trace.
const BufferedStackTrace *stack;
uptr length1, length2;
AddressDescription addr1_description;
AddressDescription addr2_description;
const char *function;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorStringFunctionMemoryRangesOverlap() = default;
+
+ ErrorStringFunctionMemoryRangesOverlap() = default; // (*)
ErrorStringFunctionMemoryRangesOverlap(u32 tid, BufferedStackTrace *stack_,
uptr addr1, uptr length1_, uptr addr2,
uptr length2_, const char *function_)
@@ -193,65 +284,51 @@ struct ErrorStringFunctionMemoryRangesOverlap : ErrorBase {
};
struct ErrorStringFunctionSizeOverflow : ErrorBase {
- // ErrorStringFunctionSizeOverflow doesn't own the stack trace.
const BufferedStackTrace *stack;
AddressDescription addr_description;
uptr size;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorStringFunctionSizeOverflow() = default;
+
+ ErrorStringFunctionSizeOverflow() = default; // (*)
ErrorStringFunctionSizeOverflow(u32 tid, BufferedStackTrace *stack_,
uptr addr, uptr size_)
- : ErrorBase(tid),
+ : ErrorBase(tid, 10, "negative-size-param"),
stack(stack_),
addr_description(addr, /*shouldLockThreadRegistry=*/false),
- size(size_) {
- scariness.Clear();
- scariness.Scare(10, "negative-size-param");
- }
+ size(size_) {}
void Print();
};
struct ErrorBadParamsToAnnotateContiguousContainer : ErrorBase {
- // ErrorBadParamsToAnnotateContiguousContainer doesn't own the stack trace.
const BufferedStackTrace *stack;
uptr beg, end, old_mid, new_mid;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorBadParamsToAnnotateContiguousContainer() = default;
+
+ ErrorBadParamsToAnnotateContiguousContainer() = default; // (*)
// PS4: Do we want an AddressDescription for beg?
ErrorBadParamsToAnnotateContiguousContainer(u32 tid,
BufferedStackTrace *stack_,
uptr beg_, uptr end_,
uptr old_mid_, uptr new_mid_)
- : ErrorBase(tid),
+ : ErrorBase(tid, 10, "bad-__sanitizer_annotate_contiguous_container"),
stack(stack_),
beg(beg_),
end(end_),
old_mid(old_mid_),
- new_mid(new_mid_) {
- scariness.Clear();
- scariness.Scare(10, "bad-__sanitizer_annotate_contiguous_container");
- }
+ new_mid(new_mid_) {}
void Print();
};
struct ErrorODRViolation : ErrorBase {
__asan_global global1, global2;
u32 stack_id1, stack_id2;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorODRViolation() = default;
+
+ ErrorODRViolation() = default; // (*)
ErrorODRViolation(u32 tid, const __asan_global *g1, u32 stack_id1_,
const __asan_global *g2, u32 stack_id2_)
- : ErrorBase(tid),
+ : ErrorBase(tid, 10, "odr-violation"),
global1(*g1),
global2(*g2),
stack_id1(stack_id1_),
- stack_id2(stack_id2_) {
- scariness.Clear();
- scariness.Scare(10, "odr-violation");
- }
+ stack_id2(stack_id2_) {}
void Print();
};
@@ -259,20 +336,16 @@ struct ErrorInvalidPointerPair : ErrorBase {
uptr pc, bp, sp;
AddressDescription addr1_description;
AddressDescription addr2_description;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorInvalidPointerPair() = default;
+
+ ErrorInvalidPointerPair() = default; // (*)
ErrorInvalidPointerPair(u32 tid, uptr pc_, uptr bp_, uptr sp_, uptr p1,
uptr p2)
- : ErrorBase(tid),
+ : ErrorBase(tid, 10, "invalid-pointer-pair"),
pc(pc_),
bp(bp_),
sp(sp_),
addr1_description(p1, 1, /*shouldLockThreadRegistry=*/false),
- addr2_description(p2, 1, /*shouldLockThreadRegistry=*/false) {
- scariness.Clear();
- scariness.Scare(10, "invalid-pointer-pair");
- }
+ addr2_description(p2, 1, /*shouldLockThreadRegistry=*/false) {}
void Print();
};
@@ -283,9 +356,8 @@ struct ErrorGeneric : ErrorBase {
const char *bug_descr;
bool is_write;
u8 shadow_val;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorGeneric() = default;
+
+ ErrorGeneric() = default; // (*)
ErrorGeneric(u32 tid, uptr addr, uptr pc_, uptr bp_, uptr sp_, bool is_write_,
uptr access_size_);
void Print();
@@ -300,6 +372,14 @@ struct ErrorGeneric : ErrorBase {
macro(AllocTypeMismatch) \
macro(MallocUsableSizeNotOwned) \
macro(SanitizerGetAllocatedSizeNotOwned) \
+ macro(CallocOverflow) \
+ macro(PvallocOverflow) \
+ macro(InvalidAllocationAlignment) \
+ macro(InvalidAlignedAllocAlignment) \
+ macro(InvalidPosixMemalignAlignment) \
+ macro(AllocationSizeTooBig) \
+ macro(RssLimitExceeded) \
+ macro(OutOfMemory) \
macro(StringFunctionMemoryRangesOverlap) \
macro(StringFunctionSizeOverflow) \
macro(BadParamsToAnnotateContiguousContainer) \
@@ -334,6 +414,7 @@ struct ErrorDescription {
};
ErrorDescription() { internal_memset(this, 0, sizeof(*this)); }
+ explicit ErrorDescription(LinkerInitialized) {}
ASAN_FOR_EACH_ERROR_KIND(ASAN_ERROR_DESCRIPTION_CONSTRUCTOR)
bool IsValid() { return kind != kErrorKindInvalid; }
diff --git a/lib/asan/asan_flags.cc b/lib/asan/asan_flags.cc
index 562168e6d428..5682ab4eb476 100644
--- a/lib/asan/asan_flags.cc
+++ b/lib/asan/asan_flags.cc
@@ -33,10 +33,7 @@ static const char *MaybeCallAsanDefaultOptions() {
static const char *MaybeUseAsanDefaultOptionsCompileDefinition() {
#ifdef ASAN_DEFAULT_OPTIONS
-// Stringize the macro value.
-# define ASAN_STRINGIZE(x) #x
-# define ASAN_STRINGIZE_OPTIONS(options) ASAN_STRINGIZE(options)
- return ASAN_STRINGIZE_OPTIONS(ASAN_DEFAULT_OPTIONS);
+ return SANITIZER_STRINGIFY(ASAN_DEFAULT_OPTIONS);
#else
return "";
#endif
@@ -163,6 +160,10 @@ void InitializeFlags() {
CHECK_LE(f->max_redzone, 2048);
CHECK(IsPowerOfTwo(f->redzone));
CHECK(IsPowerOfTwo(f->max_redzone));
+ if (SANITIZER_RTEMS) {
+ CHECK(!f->unmap_shadow_on_exit);
+ CHECK(!f->protect_shadow_gap);
+ }
// quarantine_size is deprecated but we still honor it.
// quarantine_size can not be used together with quarantine_size_mb.
diff --git a/lib/asan/asan_flags.inc b/lib/asan/asan_flags.inc
index 00071d39f041..4af94c56fca0 100644
--- a/lib/asan/asan_flags.inc
+++ b/lib/asan/asan_flags.inc
@@ -88,7 +88,8 @@ ASAN_FLAG(bool, check_malloc_usable_size, true,
"295.*.")
ASAN_FLAG(bool, unmap_shadow_on_exit, false,
"If set, explicitly unmaps the (huge) shadow at exit.")
-ASAN_FLAG(bool, protect_shadow_gap, true, "If set, mprotect the shadow gap")
+ASAN_FLAG(bool, protect_shadow_gap, !SANITIZER_RTEMS,
+ "If set, mprotect the shadow gap")
ASAN_FLAG(bool, print_stats, false,
"Print various statistics after printing an error message or if "
"atexit=1.")
@@ -136,9 +137,9 @@ ASAN_FLAG(
"Android. ")
ASAN_FLAG(
int, detect_invalid_pointer_pairs, 0,
- "If non-zero, try to detect operations like <, <=, >, >= and - on "
- "invalid pointer pairs (e.g. when pointers belong to different objects). "
- "The bigger the value the harder we try.")
+ "If >= 2, detect operations like <, <=, >, >= and - on invalid pointer "
+ "pairs (e.g. when pointers belong to different objects); "
+ "If == 1, detect invalid operations only when both pointers are non-null.")
ASAN_FLAG(
bool, detect_container_overflow, true,
"If true, honor the container overflow annotations. See "
diff --git a/lib/asan/asan_globals.cc b/lib/asan/asan_globals.cc
index 0db65d010349..898f7f40d31b 100644
--- a/lib/asan/asan_globals.cc
+++ b/lib/asan/asan_globals.cc
@@ -224,8 +224,9 @@ static void RegisterGlobal(const Global *g) {
list_of_all_globals = l;
if (g->has_dynamic_init) {
if (!dynamic_init_globals) {
- dynamic_init_globals = new(allocator_for_globals)
- VectorOfGlobals(kDynamicInitGlobalsInitialCapacity);
+ dynamic_init_globals =
+ new (allocator_for_globals) VectorOfGlobals; // NOLINT
+ dynamic_init_globals->reserve(kDynamicInitGlobalsInitialCapacity);
}
DynInitGlobal dyn_global = { *g, false };
dynamic_init_globals->push_back(dyn_global);
@@ -358,9 +359,11 @@ void __asan_register_globals(__asan_global *globals, uptr n) {
GET_STACK_TRACE_MALLOC;
u32 stack_id = StackDepotPut(stack);
BlockingMutexLock lock(&mu_for_globals);
- if (!global_registration_site_vector)
+ if (!global_registration_site_vector) {
global_registration_site_vector =
- new(allocator_for_globals) GlobalRegistrationSiteVector(128);
+ new (allocator_for_globals) GlobalRegistrationSiteVector; // NOLINT
+ global_registration_site_vector->reserve(128);
+ }
GlobalRegistrationSite site = {stack_id, &globals[0], &globals[n - 1]};
global_registration_site_vector->push_back(site);
if (flags()->report_globals >= 2) {
diff --git a/lib/asan/asan_globals_win.cc b/lib/asan/asan_globals_win.cc
index 261762b63e2c..29ab5ebf16d4 100644
--- a/lib/asan/asan_globals_win.cc
+++ b/lib/asan/asan_globals_win.cc
@@ -19,9 +19,9 @@ namespace __asan {
#pragma section(".ASAN$GA", read, write) // NOLINT
#pragma section(".ASAN$GZ", read, write) // NOLINT
extern "C" __declspec(allocate(".ASAN$GA"))
-__asan_global __asan_globals_start = {};
+ ALIGNED(sizeof(__asan_global)) __asan_global __asan_globals_start = {};
extern "C" __declspec(allocate(".ASAN$GZ"))
-__asan_global __asan_globals_end = {};
+ ALIGNED(sizeof(__asan_global)) __asan_global __asan_globals_end = {};
#pragma comment(linker, "/merge:.ASAN=.data")
static void call_on_globals(void (*hook)(__asan_global *, uptr)) {
diff --git a/lib/asan/asan_interceptors.cc b/lib/asan/asan_interceptors.cc
index cb7dcb3b1ca8..aac2bb8a6bbf 100644
--- a/lib/asan/asan_interceptors.cc
+++ b/lib/asan/asan_interceptors.cc
@@ -24,15 +24,20 @@
#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_libc.h"
-// There is no general interception at all on Fuchsia.
+// There is no general interception at all on Fuchsia and RTEMS.
// Only the functions in asan_interceptors_memintrinsics.cc are
// really defined to replace libc functions.
-#if !SANITIZER_FUCHSIA
+#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
#if SANITIZER_POSIX
#include "sanitizer_common/sanitizer_posix.h"
#endif
+#if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION || \
+ ASAN_INTERCEPT__SJLJ_UNWIND_RAISEEXCEPTION
+#include <unwind.h>
+#endif
+
#if defined(__i386) && SANITIZER_LINUX
#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.1"
#elif defined(__mips__) && SANITIZER_LINUX
@@ -178,6 +183,7 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
(void)(s); \
} while (false)
#include "sanitizer_common/sanitizer_common_syscalls.inc"
+#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
struct ThreadStartParam {
atomic_uintptr_t t;
@@ -269,7 +275,15 @@ INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp,
uptr stack, ssize;
ReadContextStack(ucp, &stack, &ssize);
ClearShadowMemoryForContextStack(stack, ssize);
+#if __has_attribute(__indirect_return__) && \
+ (defined(__x86_64__) || defined(__i386__))
+ int (*real_swapcontext)(struct ucontext_t *, struct ucontext_t *)
+ __attribute__((__indirect_return__))
+ = REAL(swapcontext);
+ int res = real_swapcontext(oucp, ucp);
+#else
int res = REAL(swapcontext)(oucp, ucp);
+#endif
// swapcontext technically does not return, but program may swap context to
// "oucp" later, that would look as if swapcontext() returned 0.
// We need to clear shadow for ucp once again, as it may be in arbitrary
@@ -318,6 +332,32 @@ INTERCEPTOR(void, __cxa_throw, void *a, void *b, void *c) {
}
#endif
+#if ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION
+INTERCEPTOR(void, __cxa_rethrow_primary_exception, void *a) {
+ CHECK(REAL(__cxa_rethrow_primary_exception));
+ __asan_handle_no_return();
+ REAL(__cxa_rethrow_primary_exception)(a);
+}
+#endif
+
+#if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION
+INTERCEPTOR(_Unwind_Reason_Code, _Unwind_RaiseException,
+ _Unwind_Exception *object) {
+ CHECK(REAL(_Unwind_RaiseException));
+ __asan_handle_no_return();
+ return REAL(_Unwind_RaiseException)(object);
+}
+#endif
+
+#if ASAN_INTERCEPT__SJLJ_UNWIND_RAISEEXCEPTION
+INTERCEPTOR(_Unwind_Reason_Code, _Unwind_SjLj_RaiseException,
+ _Unwind_Exception *object) {
+ CHECK(REAL(_Unwind_SjLj_RaiseException));
+ __asan_handle_no_return();
+ return REAL(_Unwind_SjLj_RaiseException)(object);
+}
+#endif
+
#if ASAN_INTERCEPT_INDEX
# if ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX
INTERCEPTOR(char*, index, const char *string, int c)
@@ -540,14 +580,6 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
}
#endif // ASAN_INTERCEPT___CXA_ATEXIT
-#if ASAN_INTERCEPT_FORK
-INTERCEPTOR(int, fork, void) {
- ENSURE_ASAN_INITED();
- int pid = REAL(fork)();
- return pid;
-}
-#endif // ASAN_INTERCEPT_FORK
-
// ---------------------- InitializeAsanInterceptors ---------------- {{{1
namespace __asan {
void InitializeAsanInterceptors() {
@@ -598,6 +630,17 @@ void InitializeAsanInterceptors() {
#if ASAN_INTERCEPT___CXA_THROW
ASAN_INTERCEPT_FUNC(__cxa_throw);
#endif
+#if ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION
+ ASAN_INTERCEPT_FUNC(__cxa_rethrow_primary_exception);
+#endif
+ // Indirectly intercept std::rethrow_exception.
+#if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION
+ INTERCEPT_FUNCTION(_Unwind_RaiseException);
+#endif
+ // Indirectly intercept std::rethrow_exception.
+#if ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION
+ INTERCEPT_FUNCTION(_Unwind_SjLj_RaiseException);
+#endif
// Intercept threading-related functions
#if ASAN_INTERCEPT_PTHREAD_CREATE
@@ -614,10 +657,6 @@ void InitializeAsanInterceptors() {
ASAN_INTERCEPT_FUNC(__cxa_atexit);
#endif
-#if ASAN_INTERCEPT_FORK
- ASAN_INTERCEPT_FUNC(fork);
-#endif
-
InitializePlatformInterceptors();
VReport(1, "AddressSanitizer: libc interceptors initialized\n");
diff --git a/lib/asan/asan_interceptors.h b/lib/asan/asan_interceptors.h
index e13bdecfdede..50895b167990 100644
--- a/lib/asan/asan_interceptors.h
+++ b/lib/asan/asan_interceptors.h
@@ -34,10 +34,10 @@ void InitializePlatformInterceptors();
} // namespace __asan
-// There is no general interception at all on Fuchsia.
+// There is no general interception at all on Fuchsia and RTEMS.
// Only the functions in asan_interceptors_memintrinsics.h are
// really defined to replace libc functions.
-#if !SANITIZER_FUCHSIA
+#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
// Use macro to describe if specific function should be
// intercepted on a given platform.
@@ -46,13 +46,11 @@ void InitializePlatformInterceptors();
# define ASAN_INTERCEPT__LONGJMP 1
# define ASAN_INTERCEPT_INDEX 1
# define ASAN_INTERCEPT_PTHREAD_CREATE 1
-# define ASAN_INTERCEPT_FORK 1
#else
# define ASAN_INTERCEPT_ATOLL_AND_STRTOLL 0
# define ASAN_INTERCEPT__LONGJMP 0
# define ASAN_INTERCEPT_INDEX 0
# define ASAN_INTERCEPT_PTHREAD_CREATE 0
-# define ASAN_INTERCEPT_FORK 0
#endif
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
@@ -80,13 +78,20 @@ void InitializePlatformInterceptors();
# define ASAN_INTERCEPT___LONGJMP_CHK 0
#endif
-// Android bug: https://code.google.com/p/android/issues/detail?id=61799
-#if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && \
- !(SANITIZER_ANDROID && defined(__i386)) && \
- !SANITIZER_SOLARIS
+#if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && !SANITIZER_SOLARIS && \
+ !SANITIZER_NETBSD
# define ASAN_INTERCEPT___CXA_THROW 1
+# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1
+# if defined(_GLIBCXX_SJLJ_EXCEPTIONS) || (SANITIZER_IOS && defined(__arm__))
+# define ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION 1
+# else
+# define ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION 1
+# endif
#else
# define ASAN_INTERCEPT___CXA_THROW 0
+# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 0
+# define ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION 0
+# define ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION 0
#endif
#if !SANITIZER_WINDOWS
diff --git a/lib/asan/asan_interceptors_memintrinsics.cc b/lib/asan/asan_interceptors_memintrinsics.cc
index c89cb011492e..39e32cdad12e 100644
--- a/lib/asan/asan_interceptors_memintrinsics.cc
+++ b/lib/asan/asan_interceptors_memintrinsics.cc
@@ -31,14 +31,14 @@ void *__asan_memmove(void *to, const void *from, uptr size) {
ASAN_MEMMOVE_IMPL(nullptr, to, from, size);
}
-#if SANITIZER_FUCHSIA
+#if SANITIZER_FUCHSIA || SANITIZER_RTEMS
-// Fuchsia doesn't use sanitizer_common_interceptors.inc, but the only
-// things there it wants are these three. Just define them as aliases
-// here rather than repeating the contents.
+// Fuchsia and RTEMS don't use sanitizer_common_interceptors.inc, but
+// the only things there it wants are these three. Just define them
+// as aliases here rather than repeating the contents.
-decltype(memcpy) memcpy[[gnu::alias("__asan_memcpy")]];
-decltype(memmove) memmove[[gnu::alias("__asan_memmove")]];
-decltype(memset) memset[[gnu::alias("__asan_memset")]];
+extern "C" decltype(__asan_memcpy) memcpy[[gnu::alias("__asan_memcpy")]];
+extern "C" decltype(__asan_memmove) memmove[[gnu::alias("__asan_memmove")]];
+extern "C" decltype(__asan_memset) memset[[gnu::alias("__asan_memset")]];
-#endif // SANITIZER_FUCHSIA
+#endif // SANITIZER_FUCHSIA || SANITIZER_RTEMS
diff --git a/lib/asan/asan_interceptors_memintrinsics.h b/lib/asan/asan_interceptors_memintrinsics.h
index 5a8339a23e97..a071e8f684a0 100644
--- a/lib/asan/asan_interceptors_memintrinsics.h
+++ b/lib/asan/asan_interceptors_memintrinsics.h
@@ -133,15 +133,22 @@ static inline bool RangesOverlap(const char *offset1, uptr length1,
const char *offset2, uptr length2) {
return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1));
}
-#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \
- const char *offset1 = (const char*)_offset1; \
- const char *offset2 = (const char*)_offset2; \
- if (RangesOverlap(offset1, length1, offset2, length2)) { \
- GET_STACK_TRACE_FATAL_HERE; \
- ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \
- offset2, length2, &stack); \
- } \
-} while (0)
+#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) \
+ do { \
+ const char *offset1 = (const char *)_offset1; \
+ const char *offset2 = (const char *)_offset2; \
+ if (RangesOverlap(offset1, length1, offset2, length2)) { \
+ GET_STACK_TRACE_FATAL_HERE; \
+ bool suppressed = IsInterceptorSuppressed(name); \
+ if (!suppressed && HaveStackTraceBasedSuppressions()) { \
+ suppressed = IsStackTraceSuppressed(&stack); \
+ } \
+ if (!suppressed) { \
+ ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \
+ offset2, length2, &stack); \
+ } \
+ } \
+ } while (0)
} // namespace __asan
diff --git a/lib/asan/asan_internal.h b/lib/asan/asan_internal.h
index 19133e5291a9..654878cd15f0 100644
--- a/lib/asan/asan_internal.h
+++ b/lib/asan/asan_internal.h
@@ -36,7 +36,7 @@
// If set, values like allocator chunk size, as well as defaults for some flags
// will be changed towards less memory overhead.
#ifndef ASAN_LOW_MEMORY
-# if SANITIZER_IOS || SANITIZER_ANDROID
+# if SANITIZER_IOS || SANITIZER_ANDROID || SANITIZER_RTEMS
# define ASAN_LOW_MEMORY 1
# else
# define ASAN_LOW_MEMORY 0
@@ -78,7 +78,7 @@ void InitializeShadowMemory();
// asan_malloc_linux.cc / asan_malloc_mac.cc
void ReplaceSystemMalloc();
-// asan_linux.cc / asan_mac.cc / asan_win.cc
+// asan_linux.cc / asan_mac.cc / asan_rtems.cc / asan_win.cc
uptr FindDynamicShadowStart();
void *AsanDoesNotSupportStaticLinkage();
void AsanCheckDynamicRTPrereqs();
@@ -147,6 +147,9 @@ const int kAsanArrayCookieMagic = 0xac;
const int kAsanIntraObjectRedzone = 0xbb;
const int kAsanAllocaLeftMagic = 0xca;
const int kAsanAllocaRightMagic = 0xcb;
+// Used to populate the shadow gap for systems without memory
+// protection there (i.e. Myriad).
+const int kAsanShadowGap = 0xcc;
static const uptr kCurrentStackFrameMagic = 0x41B58AB3;
static const uptr kRetiredStackFrameMagic = 0x45E0360E;
diff --git a/lib/asan/asan_linux.cc b/lib/asan/asan_linux.cc
index 047e1dbb72fa..625f32d408df 100644
--- a/lib/asan/asan_linux.cc
+++ b/lib/asan/asan_linux.cc
@@ -32,6 +32,7 @@
#include <sys/types.h>
#include <dlfcn.h>
#include <fcntl.h>
+#include <limits.h>
#include <pthread.h>
#include <stdio.h>
#include <unistd.h>
@@ -214,7 +215,7 @@ void AsanCheckIncompatibleRT() {
// the functions in dynamic ASan runtime instead of the functions in
// system libraries, causing crashes later in ASan initialization.
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
- char filename[128];
+ char filename[PATH_MAX];
MemoryMappedSegment segment(filename, sizeof(filename));
while (proc_maps.Next(&segment)) {
if (IsDynamicRTName(segment.filename)) {
diff --git a/lib/asan/asan_mac.cc b/lib/asan/asan_mac.cc
index b7af1a58664c..17a0ec57701d 100644
--- a/lib/asan/asan_mac.cc
+++ b/lib/asan/asan_mac.cc
@@ -62,16 +62,36 @@ uptr FindDynamicShadowStart() {
uptr space_size = kHighShadowEnd + left_padding;
uptr largest_gap_found = 0;
- uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
- granularity, &largest_gap_found);
+ uptr max_occupied_addr = 0;
+ VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+ uptr shadow_start =
+ FindAvailableMemoryRange(space_size, alignment, granularity,
+ &largest_gap_found, &max_occupied_addr);
// If the shadow doesn't fit, restrict the address space to make it fit.
if (shadow_start == 0) {
+ VReport(
+ 2,
+ "Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
+ largest_gap_found, max_occupied_addr);
uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment);
+ if (new_max_vm < max_occupied_addr) {
+ Report("Unable to find a memory range for dynamic shadow.\n");
+ Report(
+ "space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
+ "new_max_vm = %p\n",
+ space_size, largest_gap_found, max_occupied_addr, new_max_vm);
+ CHECK(0 && "cannot place shadow");
+ }
RestrictMemoryToMaxAddress(new_max_vm);
kHighMemEnd = new_max_vm - 1;
space_size = kHighShadowEnd + left_padding;
- shadow_start =
- FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+ VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+ shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
+ nullptr, nullptr);
+ if (shadow_start == 0) {
+ Report("Unable to find a memory range after restricting VM.\n");
+ CHECK(0 && "cannot place shadow after restricting vm");
+ }
}
CHECK_NE((uptr)0, shadow_start);
CHECK(IsAligned(shadow_start, alignment));
diff --git a/lib/asan/asan_malloc_linux.cc b/lib/asan/asan_malloc_linux.cc
index 6697ff8764ed..76bdff999d62 100644
--- a/lib/asan/asan_malloc_linux.cc
+++ b/lib/asan/asan_malloc_linux.cc
@@ -16,19 +16,23 @@
#include "sanitizer_common/sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \
- SANITIZER_NETBSD || SANITIZER_SOLARIS
+ SANITIZER_NETBSD || SANITIZER_RTEMS || SANITIZER_SOLARIS
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
#include "asan_allocator.h"
#include "asan_interceptors.h"
#include "asan_internal.h"
+#include "asan_malloc_local.h"
#include "asan_stack.h"
// ---------------------- Replacement functions ---------------- {{{1
using namespace __asan; // NOLINT
static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
+static uptr last_dlsym_alloc_size_in_words;
+static const uptr kDlsymAllocPoolSize = SANITIZER_RTEMS ? 4096 : 1024;
static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
static INLINE bool IsInDlsymAllocPool(const void *ptr) {
@@ -39,21 +43,73 @@ static INLINE bool IsInDlsymAllocPool(const void *ptr) {
static void *AllocateFromLocalPool(uptr size_in_bytes) {
uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
void *mem = (void*)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ last_dlsym_alloc_size_in_words = size_in_words;
allocated_for_dlsym += size_in_words;
CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
return mem;
}
+static void DeallocateFromLocalPool(const void *ptr) {
+ // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
+ // error messages and instead uses malloc followed by free. To avoid pool
+ // exhaustion due to long object filenames, handle that special case here.
+ uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
+ void *prev_mem = (void*)&alloc_memory_for_dlsym[prev_offset];
+ if (prev_mem == ptr) {
+ REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
+ allocated_for_dlsym = prev_offset;
+ last_dlsym_alloc_size_in_words = 0;
+ }
+}
+
+static int PosixMemalignFromLocalPool(void **memptr, uptr alignment,
+ uptr size_in_bytes) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(alignment)))
+ return errno_EINVAL;
+
+ CHECK(alignment >= kWordSize);
+
+ uptr addr = (uptr)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ uptr aligned_addr = RoundUpTo(addr, alignment);
+ uptr aligned_size = RoundUpTo(size_in_bytes, kWordSize);
+
+ uptr *end_mem = (uptr*)(aligned_addr + aligned_size);
+ uptr allocated = end_mem - alloc_memory_for_dlsym;
+ if (allocated >= kDlsymAllocPoolSize)
+ return errno_ENOMEM;
+
+ allocated_for_dlsym = allocated;
+ *memptr = (void*)aligned_addr;
+ return 0;
+}
+
+#if SANITIZER_RTEMS
+void* MemalignFromLocalPool(uptr alignment, uptr size) {
+ void *ptr = nullptr;
+ alignment = Max(alignment, kWordSize);
+ PosixMemalignFromLocalPool(&ptr, alignment, size);
+ return ptr;
+}
+
+bool IsFromLocalPool(const void *ptr) {
+ return IsInDlsymAllocPool(ptr);
+}
+#endif
+
static INLINE bool MaybeInDlsym() {
// Fuchsia doesn't use dlsym-based interceptors.
return !SANITIZER_FUCHSIA && asan_init_is_running;
}
+static INLINE bool UseLocalPool() {
+ return EarlyMalloc() || MaybeInDlsym();
+}
+
static void *ReallocFromLocalPool(void *ptr, uptr size) {
const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
void *new_ptr;
- if (UNLIKELY(MaybeInDlsym())) {
+ if (UNLIKELY(UseLocalPool())) {
new_ptr = AllocateFromLocalPool(size);
} else {
ENSURE_ASAN_INITED();
@@ -66,8 +122,10 @@ static void *ReallocFromLocalPool(void *ptr, uptr size) {
INTERCEPTOR(void, free, void *ptr) {
GET_STACK_TRACE_FREE;
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+ DeallocateFromLocalPool(ptr);
return;
+ }
asan_free(ptr, &stack, FROM_MALLOC);
}
@@ -81,7 +139,7 @@ INTERCEPTOR(void, cfree, void *ptr) {
#endif // SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void*, malloc, uptr size) {
- if (UNLIKELY(MaybeInDlsym()))
+ if (UNLIKELY(UseLocalPool()))
// Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
return AllocateFromLocalPool(size);
ENSURE_ASAN_INITED();
@@ -90,7 +148,7 @@ INTERCEPTOR(void*, malloc, uptr size) {
}
INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
- if (UNLIKELY(MaybeInDlsym()))
+ if (UNLIKELY(UseLocalPool()))
// Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
return AllocateFromLocalPool(nmemb * size);
ENSURE_ASAN_INITED();
@@ -101,7 +159,7 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
if (UNLIKELY(IsInDlsymAllocPool(ptr)))
return ReallocFromLocalPool(ptr, size);
- if (UNLIKELY(MaybeInDlsym()))
+ if (UNLIKELY(UseLocalPool()))
return AllocateFromLocalPool(size);
ENSURE_ASAN_INITED();
GET_STACK_TRACE_MALLOC;
@@ -122,10 +180,12 @@ INTERCEPTOR(void*, __libc_memalign, uptr boundary, uptr size) {
}
#endif // SANITIZER_INTERCEPT_MEMALIGN
+#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
INTERCEPTOR(void*, aligned_alloc, uptr boundary, uptr size) {
GET_STACK_TRACE_MALLOC;
- return asan_memalign(boundary, size, &stack, FROM_MALLOC);
+ return asan_aligned_alloc(boundary, size, &stack);
}
+#endif // SANITIZER_INTERCEPT_ALIGNED_ALLOC
INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
GET_CURRENT_PC_BP_SP;
@@ -154,8 +214,9 @@ INTERCEPTOR(int, mallopt, int cmd, int value) {
#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
+ if (UNLIKELY(UseLocalPool()))
+ return PosixMemalignFromLocalPool(memptr, alignment, size);
GET_STACK_TRACE_MALLOC;
- // Printf("posix_memalign: %zx %zu\n", alignment, size);
return asan_posix_memalign(memptr, alignment, size, &stack);
}
diff --git a/lib/asan/asan_malloc_local.h b/lib/asan/asan_malloc_local.h
new file mode 100644
index 000000000000..0e8de207d98d
--- /dev/null
+++ b/lib/asan/asan_malloc_local.h
@@ -0,0 +1,44 @@
+//===-- asan_malloc_local.h -------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Provide interfaces to check for and handle local pool memory allocation.
+//===----------------------------------------------------------------------===//
+
+#ifndef ASAN_MALLOC_LOCAL_H
+#define ASAN_MALLOC_LOCAL_H
+
+#include "sanitizer_common/sanitizer_platform.h"
+#include "asan_internal.h"
+
+// On RTEMS, we use the local pool to handle memory allocation when the ASan
+// run-time is not up.
+static INLINE bool EarlyMalloc() {
+ return SANITIZER_RTEMS && (!__asan::asan_inited ||
+ __asan::asan_init_is_running);
+}
+
+void* MemalignFromLocalPool(uptr alignment, uptr size);
+
+#if SANITIZER_RTEMS
+
+bool IsFromLocalPool(const void *ptr);
+
+#define ALLOCATE_FROM_LOCAL_POOL UNLIKELY(EarlyMalloc())
+#define IS_FROM_LOCAL_POOL(ptr) UNLIKELY(IsFromLocalPool(ptr))
+
+#else // SANITIZER_RTEMS
+
+#define ALLOCATE_FROM_LOCAL_POOL 0
+#define IS_FROM_LOCAL_POOL(ptr) 0
+
+#endif // SANITIZER_RTEMS
+
+#endif // ASAN_MALLOC_LOCAL_H
diff --git a/lib/asan/asan_malloc_mac.cc b/lib/asan/asan_malloc_mac.cc
index 744728d40df5..733ba2d86e13 100644
--- a/lib/asan/asan_malloc_mac.cc
+++ b/lib/asan/asan_malloc_mac.cc
@@ -38,6 +38,9 @@ using namespace __asan;
#define COMMON_MALLOC_CALLOC(count, size) \
GET_STACK_TRACE_MALLOC; \
void *p = asan_calloc(count, size, &stack);
+#define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \
+ GET_STACK_TRACE_MALLOC; \
+ int res = asan_posix_memalign(memptr, alignment, size, &stack);
#define COMMON_MALLOC_VALLOC(size) \
GET_STACK_TRACE_MALLOC; \
void *p = asan_memalign(GetPageSizeCached(), size, &stack, FROM_MALLOC);
diff --git a/lib/asan/asan_mapping.h b/lib/asan/asan_mapping.h
index d3f360ffc969..3f8cc004b95f 100644
--- a/lib/asan/asan_mapping.h
+++ b/lib/asan/asan_mapping.h
@@ -122,6 +122,13 @@
// || `[0x400000000000, 0x47ffffffffff]` || LowShadow ||
// || `[0x000000000000, 0x3fffffffffff]` || LowMem ||
//
+// Shadow mapping on NerBSD/i386 with SHADOW_OFFSET == 0x40000000:
+// || `[0x60000000, 0xfffff000]` || HighMem ||
+// || `[0x4c000000, 0x5fffffff]` || HighShadow ||
+// || `[0x48000000, 0x4bffffff]` || ShadowGap ||
+// || `[0x40000000, 0x47ffffff]` || LowShadow ||
+// || `[0x00000000, 0x3fffffff]` || LowMem ||
+//
// Default Windows/i386 mapping:
// (the exact location of HighShadow/HighMem may vary depending
// on WoW64, /LARGEADDRESSAWARE, etc).
@@ -130,11 +137,17 @@
// || `[0x36000000, 0x39ffffff]` || ShadowGap ||
// || `[0x30000000, 0x35ffffff]` || LowShadow ||
// || `[0x00000000, 0x2fffffff]` || LowMem ||
+//
+// Shadow mapping on Myriad2 (for shadow scale 5):
+// || `[0x9ff80000, 0x9fffffff]` || ShadowGap ||
+// || `[0x9f000000, 0x9ff7ffff]` || LowShadow ||
+// || `[0x80000000, 0x9effffff]` || LowMem ||
+// || `[0x00000000, 0x7fffffff]` || Ignored ||
#if defined(ASAN_SHADOW_SCALE)
static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE;
#else
-static const u64 kDefaultShadowScale = 3;
+static const u64 kDefaultShadowScale = SANITIZER_MYRIAD2 ? 5 : 3;
#endif
static const u64 kDefaultShadowSentinel = ~(uptr)0;
static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000
@@ -152,9 +165,19 @@ static const u64 kPPC64_ShadowOffset64 = 1ULL << 44;
static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52;
static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000
static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000
+static const u64 kNetBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000
static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000
static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
+static const u64 kMyriadMemoryOffset32 = 0x80000000ULL;
+static const u64 kMyriadMemorySize32 = 0x20000000ULL;
+static const u64 kMyriadMemoryEnd32 =
+ kMyriadMemoryOffset32 + kMyriadMemorySize32 - 1;
+static const u64 kMyriadShadowOffset32 =
+ (kMyriadMemoryOffset32 + kMyriadMemorySize32 -
+ (kMyriadMemorySize32 >> kDefaultShadowScale));
+static const u64 kMyriadCacheBitMask32 = 0x40000000ULL;
+
#define SHADOW_SCALE kDefaultShadowScale
#if SANITIZER_FUCHSIA
@@ -166,6 +189,8 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
# define SHADOW_OFFSET kMIPS32_ShadowOffset32
# elif SANITIZER_FREEBSD
# define SHADOW_OFFSET kFreeBSD_ShadowOffset32
+# elif SANITIZER_NETBSD
+# define SHADOW_OFFSET kNetBSD_ShadowOffset32
# elif SANITIZER_WINDOWS
# define SHADOW_OFFSET kWindowsShadowOffset32
# elif SANITIZER_IOS
@@ -174,6 +199,8 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
# else
# define SHADOW_OFFSET kIosShadowOffset32
# endif
+# elif SANITIZER_MYRIAD2
+# define SHADOW_OFFSET kMyriadShadowOffset32
# else
# define SHADOW_OFFSET kDefaultShadowOffset32
# endif
@@ -212,6 +239,39 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
#endif
#define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE)
+
+#define DO_ASAN_MAPPING_PROFILE 0 // Set to 1 to profile the functions below.
+
+#if DO_ASAN_MAPPING_PROFILE
+# define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++;
+#else
+# define PROFILE_ASAN_MAPPING()
+#endif
+
+// If 1, all shadow boundaries are constants.
+// Don't set to 1 other than for testing.
+#define ASAN_FIXED_MAPPING 0
+
+namespace __asan {
+
+extern uptr AsanMappingProfile[];
+
+#if ASAN_FIXED_MAPPING
+// Fixed mapping for 64-bit Linux. Mostly used for performance comparison
+// with non-fixed mapping. As of r175253 (Feb 2013) the performance
+// difference between fixed and non-fixed mapping is below the noise level.
+static uptr kHighMemEnd = 0x7fffffffffffULL;
+static uptr kMidMemBeg = 0x3000000000ULL;
+static uptr kMidMemEnd = 0x4fffffffffULL;
+#else
+extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; // Initialized in __asan_init.
+#endif
+
+} // namespace __asan
+
+#if SANITIZER_MYRIAD2
+#include "asan_mapping_myriad.h"
+#else
#define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET))
#define kLowMemBeg 0
@@ -243,36 +303,11 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
#define kShadowGap3Beg (kMidMemBeg ? kMidMemEnd + 1 : 0)
#define kShadowGap3End (kMidMemBeg ? kHighShadowBeg - 1 : 0)
-#define DO_ASAN_MAPPING_PROFILE 0 // Set to 1 to profile the functions below.
-
-#if DO_ASAN_MAPPING_PROFILE
-# define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++;
-#else
-# define PROFILE_ASAN_MAPPING()
-#endif
-
-// If 1, all shadow boundaries are constants.
-// Don't set to 1 other than for testing.
-#define ASAN_FIXED_MAPPING 0
-
namespace __asan {
-extern uptr AsanMappingProfile[];
-
-#if ASAN_FIXED_MAPPING
-// Fixed mapping for 64-bit Linux. Mostly used for performance comparison
-// with non-fixed mapping. As of r175253 (Feb 2013) the performance
-// difference between fixed and non-fixed mapping is below the noise level.
-static uptr kHighMemEnd = 0x7fffffffffffULL;
-static uptr kMidMemBeg = 0x3000000000ULL;
-static uptr kMidMemEnd = 0x4fffffffffULL;
-#else
-extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; // Initialized in __asan_init.
-#endif
-
static inline bool AddrIsInLowMem(uptr a) {
PROFILE_ASAN_MAPPING();
- return a < kLowMemEnd;
+ return a <= kLowMemEnd;
}
static inline bool AddrIsInLowShadow(uptr a) {
@@ -280,14 +315,24 @@ static inline bool AddrIsInLowShadow(uptr a) {
return a >= kLowShadowBeg && a <= kLowShadowEnd;
}
+static inline bool AddrIsInMidMem(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ return kMidMemBeg && a >= kMidMemBeg && a <= kMidMemEnd;
+}
+
+static inline bool AddrIsInMidShadow(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ return kMidMemBeg && a >= kMidShadowBeg && a <= kMidShadowEnd;
+}
+
static inline bool AddrIsInHighMem(uptr a) {
PROFILE_ASAN_MAPPING();
- return a >= kHighMemBeg && a <= kHighMemEnd;
+ return kHighMemBeg && a >= kHighMemBeg && a <= kHighMemEnd;
}
-static inline bool AddrIsInMidMem(uptr a) {
+static inline bool AddrIsInHighShadow(uptr a) {
PROFILE_ASAN_MAPPING();
- return kMidMemBeg && a >= kMidMemBeg && a <= kMidMemEnd;
+ return kHighMemBeg && a >= kHighShadowBeg && a <= kHighShadowEnd;
}
static inline bool AddrIsInShadowGap(uptr a) {
@@ -305,6 +350,12 @@ static inline bool AddrIsInShadowGap(uptr a) {
return a >= kShadowGapBeg && a <= kShadowGapEnd;
}
+} // namespace __asan
+
+#endif // SANITIZER_MYRIAD2
+
+namespace __asan {
+
static inline bool AddrIsInMem(uptr a) {
PROFILE_ASAN_MAPPING();
return AddrIsInLowMem(a) || AddrIsInMidMem(a) || AddrIsInHighMem(a) ||
@@ -317,16 +368,6 @@ static inline uptr MemToShadow(uptr p) {
return MEM_TO_SHADOW(p);
}
-static inline bool AddrIsInHighShadow(uptr a) {
- PROFILE_ASAN_MAPPING();
- return a >= kHighShadowBeg && a <= kHighMemEnd;
-}
-
-static inline bool AddrIsInMidShadow(uptr a) {
- PROFILE_ASAN_MAPPING();
- return kMidMemBeg && a >= kMidShadowBeg && a <= kMidMemEnd;
-}
-
static inline bool AddrIsInShadow(uptr a) {
PROFILE_ASAN_MAPPING();
return AddrIsInLowShadow(a) || AddrIsInMidShadow(a) || AddrIsInHighShadow(a);
@@ -339,6 +380,8 @@ static inline bool AddrIsAlignedByGranularity(uptr a) {
static inline bool AddressIsPoisoned(uptr a) {
PROFILE_ASAN_MAPPING();
+ if (SANITIZER_MYRIAD2 && !AddrIsInMem(a) && !AddrIsInShadow(a))
+ return false;
const uptr kAccessSize = 1;
u8 *shadow_address = (u8*)MEM_TO_SHADOW(a);
s8 shadow_value = *shadow_address;
diff --git a/lib/asan/asan_mapping_myriad.h b/lib/asan/asan_mapping_myriad.h
new file mode 100644
index 000000000000..baa3247bd190
--- /dev/null
+++ b/lib/asan/asan_mapping_myriad.h
@@ -0,0 +1,86 @@
+//===-- asan_mapping_myriad.h -----------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Myriad-specific definitions for ASan memory mapping.
+//===----------------------------------------------------------------------===//
+#ifndef ASAN_MAPPING_MYRIAD_H
+#define ASAN_MAPPING_MYRIAD_H
+
+#define RAW_ADDR(mem) ((mem) & ~kMyriadCacheBitMask32)
+#define MEM_TO_SHADOW(mem) \
+ (((RAW_ADDR(mem) - kLowMemBeg) >> SHADOW_SCALE) + (SHADOW_OFFSET))
+
+#define kLowMemBeg kMyriadMemoryOffset32
+#define kLowMemEnd (SHADOW_OFFSET - 1)
+
+#define kLowShadowBeg SHADOW_OFFSET
+#define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd)
+
+#define kHighMemBeg 0
+
+#define kHighShadowBeg 0
+#define kHighShadowEnd 0
+
+#define kMidShadowBeg 0
+#define kMidShadowEnd 0
+
+#define kShadowGapBeg (kLowShadowEnd + 1)
+#define kShadowGapEnd kMyriadMemoryEnd32
+
+#define kShadowGap2Beg 0
+#define kShadowGap2End 0
+
+#define kShadowGap3Beg 0
+#define kShadowGap3End 0
+
+namespace __asan {
+
+static inline bool AddrIsInLowMem(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ a = RAW_ADDR(a);
+ return a >= kLowMemBeg && a <= kLowMemEnd;
+}
+
+static inline bool AddrIsInLowShadow(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ a = RAW_ADDR(a);
+ return a >= kLowShadowBeg && a <= kLowShadowEnd;
+}
+
+static inline bool AddrIsInMidMem(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ return false;
+}
+
+static inline bool AddrIsInMidShadow(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ return false;
+}
+
+static inline bool AddrIsInHighMem(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ return false;
+}
+
+static inline bool AddrIsInHighShadow(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ return false;
+}
+
+static inline bool AddrIsInShadowGap(uptr a) {
+ PROFILE_ASAN_MAPPING();
+ a = RAW_ADDR(a);
+ return a >= kShadowGapBeg && a <= kShadowGapEnd;
+}
+
+} // namespace __asan
+
+#endif // ASAN_MAPPING_MYRIAD_H
diff --git a/lib/asan/asan_memory_profile.cc b/lib/asan/asan_memory_profile.cc
index 603284c8c688..8c86d9fe49f2 100644
--- a/lib/asan/asan_memory_profile.cc
+++ b/lib/asan/asan_memory_profile.cc
@@ -31,9 +31,9 @@ struct AllocationSite {
class HeapProfile {
public:
- HeapProfile() : allocations_(1024) {}
+ HeapProfile() { allocations_.reserve(1024); }
- void ProcessChunk(const AsanChunkView& cv) {
+ void ProcessChunk(const AsanChunkView &cv) {
if (cv.IsAllocated()) {
total_allocated_user_size_ += cv.UsedSize();
total_allocated_count_++;
@@ -49,10 +49,10 @@ class HeapProfile {
}
void Print(uptr top_percent, uptr max_number_of_contexts) {
- InternalSort(&allocations_, allocations_.size(),
- [](const AllocationSite &a, const AllocationSite &b) {
- return a.total_size > b.total_size;
- });
+ Sort(allocations_.data(), allocations_.size(),
+ [](const AllocationSite &a, const AllocationSite &b) {
+ return a.total_size > b.total_size;
+ });
CHECK(total_allocated_user_size_);
uptr total_shown = 0;
Printf("Live Heap Allocations: %zd bytes in %zd chunks; quarantined: "
diff --git a/lib/asan/asan_new_delete.cc b/lib/asan/asan_new_delete.cc
index 072f027addd6..30efd61a9680 100644
--- a/lib/asan/asan_new_delete.cc
+++ b/lib/asan/asan_new_delete.cc
@@ -14,6 +14,8 @@
#include "asan_allocator.h"
#include "asan_internal.h"
+#include "asan_malloc_local.h"
+#include "asan_report.h"
#include "asan_stack.h"
#include "interception/interception.h"
@@ -67,16 +69,28 @@ struct nothrow_t {};
enum class align_val_t: size_t {};
} // namespace std
-// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+// TODO(alekseyshl): throw std::bad_alloc instead of dying on OOM.
+// For local pool allocation, align to SHADOW_GRANULARITY to match asan
+// allocator behavior.
#define OPERATOR_NEW_BODY(type, nothrow) \
+ if (ALLOCATE_FROM_LOCAL_POOL) {\
+ void *res = MemalignFromLocalPool(SHADOW_GRANULARITY, size);\
+ if (!nothrow) CHECK(res);\
+ return res;\
+ }\
GET_STACK_TRACE_MALLOC;\
void *res = asan_memalign(0, size, &stack, type);\
- if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+ if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\
return res;
#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \
+ if (ALLOCATE_FROM_LOCAL_POOL) {\
+ void *res = MemalignFromLocalPool((uptr)align, size);\
+ if (!nothrow) CHECK(res);\
+ return res;\
+ }\
GET_STACK_TRACE_MALLOC;\
void *res = asan_memalign((uptr)align, size, &stack, type);\
- if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+ if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\
return res;
// On OS X it's not enough to just provide our own 'operator new' and
@@ -128,18 +142,22 @@ INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
#endif // !SANITIZER_MAC
#define OPERATOR_DELETE_BODY(type) \
+ if (IS_FROM_LOCAL_POOL(ptr)) return;\
GET_STACK_TRACE_FREE;\
asan_delete(ptr, 0, 0, &stack, type);
#define OPERATOR_DELETE_BODY_SIZE(type) \
+ if (IS_FROM_LOCAL_POOL(ptr)) return;\
GET_STACK_TRACE_FREE;\
asan_delete(ptr, size, 0, &stack, type);
#define OPERATOR_DELETE_BODY_ALIGN(type) \
+ if (IS_FROM_LOCAL_POOL(ptr)) return;\
GET_STACK_TRACE_FREE;\
asan_delete(ptr, 0, static_cast<uptr>(align), &stack, type);
#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \
+ if (IS_FROM_LOCAL_POOL(ptr)) return;\
GET_STACK_TRACE_FREE;\
asan_delete(ptr, size, static_cast<uptr>(align), &stack, type);
diff --git a/lib/asan/asan_poisoning.cc b/lib/asan/asan_poisoning.cc
index c3a82aa0049e..1e9c37a13a16 100644
--- a/lib/asan/asan_poisoning.cc
+++ b/lib/asan/asan_poisoning.cc
@@ -32,7 +32,7 @@ bool CanPoisonMemory() {
}
void PoisonShadow(uptr addr, uptr size, u8 value) {
- if (!CanPoisonMemory()) return;
+ if (value && !CanPoisonMemory()) return;
CHECK(AddrIsAlignedByGranularity(addr));
CHECK(AddrIsInMem(addr));
CHECK(AddrIsAlignedByGranularity(addr + size));
@@ -182,8 +182,15 @@ int __asan_address_is_poisoned(void const volatile *addr) {
uptr __asan_region_is_poisoned(uptr beg, uptr size) {
if (!size) return 0;
uptr end = beg + size;
- if (!AddrIsInMem(beg)) return beg;
- if (!AddrIsInMem(end)) return end;
+ if (SANITIZER_MYRIAD2) {
+ // On Myriad, address not in DRAM range need to be treated as
+ // unpoisoned.
+ if (!AddrIsInMem(beg) && !AddrIsInShadow(beg)) return 0;
+ if (!AddrIsInMem(end) && !AddrIsInShadow(end)) return 0;
+ } else {
+ if (!AddrIsInMem(beg)) return beg;
+ if (!AddrIsInMem(end)) return end;
+ }
CHECK_LT(beg, end);
uptr aligned_b = RoundUpTo(beg, SHADOW_GRANULARITY);
uptr aligned_e = RoundDownTo(end, SHADOW_GRANULARITY);
@@ -452,4 +459,3 @@ bool WordIsPoisoned(uptr addr) {
return (__asan_region_is_poisoned(addr, sizeof(uptr)) != 0);
}
}
-
diff --git a/lib/asan/asan_poisoning.h b/lib/asan/asan_poisoning.h
index 1e00070bcf63..c94794cead89 100644
--- a/lib/asan/asan_poisoning.h
+++ b/lib/asan/asan_poisoning.h
@@ -38,7 +38,7 @@ void PoisonShadowPartialRightRedzone(uptr addr,
// performance-critical code with care.
ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size,
u8 value) {
- DCHECK(CanPoisonMemory());
+ DCHECK(!value || CanPoisonMemory());
uptr shadow_beg = MEM_TO_SHADOW(aligned_beg);
uptr shadow_end = MEM_TO_SHADOW(
aligned_beg + aligned_size - SHADOW_GRANULARITY) + 1;
@@ -51,6 +51,9 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size,
// changed at all. It doesn't currently have an efficient means
// to zero a bunch of pages, but maybe we should add one.
SANITIZER_FUCHSIA == 1 ||
+ // RTEMS doesn't have have pages, let alone a fast way to zero
+ // them, so default to memset.
+ SANITIZER_RTEMS == 1 ||
shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg);
} else {
diff --git a/lib/asan/asan_report.cc b/lib/asan/asan_report.cc
index e3bc02994efd..439712350e46 100644
--- a/lib/asan/asan_report.cc
+++ b/lib/asan/asan_report.cc
@@ -84,7 +84,7 @@ static void PrintZoneForPointer(uptr ptr, uptr zone_ptr,
bool ParseFrameDescription(const char *frame_descr,
InternalMmapVector<StackVarDescr> *vars) {
CHECK(frame_descr);
- char *p;
+ const char *p;
// This string is created by the compiler and has the following form:
// "n alloc_1 alloc_2 ... alloc_n"
// where alloc_i looks like "offset size len ObjectName"
@@ -134,6 +134,10 @@ class ScopedInErrorReport {
}
~ScopedInErrorReport() {
+ if (halt_on_error_ && !__sanitizer_acquire_crash_state()) {
+ asanThreadRegistry().Unlock();
+ return;
+ }
ASAN_ON_ERROR();
if (current_error_.IsValid()) current_error_.Print();
@@ -152,7 +156,7 @@ class ScopedInErrorReport {
// Copy the message buffer so that we could start logging without holding a
// lock that gets aquired during printing.
- InternalScopedBuffer<char> buffer_copy(kErrorMessageBufferSize);
+ InternalMmapVector<char> buffer_copy(kErrorMessageBufferSize);
{
BlockingMutexLock l(&error_message_buf_mutex);
internal_memcpy(buffer_copy.data(),
@@ -202,7 +206,7 @@ class ScopedInErrorReport {
bool halt_on_error_;
};
-ErrorDescription ScopedInErrorReport::current_error_;
+ErrorDescription ScopedInErrorReport::current_error_(LINKER_INITIALIZED);
void ReportDeadlySignal(const SignalContext &sig) {
ScopedInErrorReport in_report(/*fatal*/ true);
@@ -254,6 +258,62 @@ void ReportSanitizerGetAllocatedSizeNotOwned(uptr addr,
in_report.ReportError(error);
}
+void ReportCallocOverflow(uptr count, uptr size, BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report(/*fatal*/ true);
+ ErrorCallocOverflow error(GetCurrentTidOrInvalid(), stack, count, size);
+ in_report.ReportError(error);
+}
+
+void ReportPvallocOverflow(uptr size, BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report(/*fatal*/ true);
+ ErrorPvallocOverflow error(GetCurrentTidOrInvalid(), stack, size);
+ in_report.ReportError(error);
+}
+
+void ReportInvalidAllocationAlignment(uptr alignment,
+ BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report(/*fatal*/ true);
+ ErrorInvalidAllocationAlignment error(GetCurrentTidOrInvalid(), stack,
+ alignment);
+ in_report.ReportError(error);
+}
+
+void ReportInvalidAlignedAllocAlignment(uptr size, uptr alignment,
+ BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report(/*fatal*/ true);
+ ErrorInvalidAlignedAllocAlignment error(GetCurrentTidOrInvalid(), stack,
+ size, alignment);
+ in_report.ReportError(error);
+}
+
+void ReportInvalidPosixMemalignAlignment(uptr alignment,
+ BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report(/*fatal*/ true);
+ ErrorInvalidPosixMemalignAlignment error(GetCurrentTidOrInvalid(), stack,
+ alignment);
+ in_report.ReportError(error);
+}
+
+void ReportAllocationSizeTooBig(uptr user_size, uptr total_size, uptr max_size,
+ BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report(/*fatal*/ true);
+ ErrorAllocationSizeTooBig error(GetCurrentTidOrInvalid(), stack, user_size,
+ total_size, max_size);
+ in_report.ReportError(error);
+}
+
+void ReportRssLimitExceeded(BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report(/*fatal*/ true);
+ ErrorRssLimitExceeded error(GetCurrentTidOrInvalid(), stack);
+ in_report.ReportError(error);
+}
+
+void ReportOutOfMemory(uptr requested_size, BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report(/*fatal*/ true);
+ ErrorOutOfMemory error(GetCurrentTidOrInvalid(), stack, requested_size);
+ in_report.ReportError(error);
+}
+
void ReportStringFunctionMemoryRangesOverlap(const char *function,
const char *offset1, uptr length1,
const char *offset2, uptr length2,
@@ -343,7 +403,11 @@ static bool IsInvalidPointerPair(uptr a1, uptr a2) {
}
static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) {
- if (!flags()->detect_invalid_pointer_pairs) return;
+ switch (flags()->detect_invalid_pointer_pairs) {
+ case 0 : return;
+ case 1 : if (p1 == nullptr || p2 == nullptr) return; break;
+ }
+
uptr a1 = reinterpret_cast<uptr>(p1);
uptr a2 = reinterpret_cast<uptr>(p2);
diff --git a/lib/asan/asan_report.h b/lib/asan/asan_report.h
index f2cdad47ee81..f7153d4810d0 100644
--- a/lib/asan/asan_report.h
+++ b/lib/asan/asan_report.h
@@ -58,6 +58,18 @@ void ReportAllocTypeMismatch(uptr addr, BufferedStackTrace *free_stack,
void ReportMallocUsableSizeNotOwned(uptr addr, BufferedStackTrace *stack);
void ReportSanitizerGetAllocatedSizeNotOwned(uptr addr,
BufferedStackTrace *stack);
+void ReportCallocOverflow(uptr count, uptr size, BufferedStackTrace *stack);
+void ReportPvallocOverflow(uptr size, BufferedStackTrace *stack);
+void ReportInvalidAllocationAlignment(uptr alignment,
+ BufferedStackTrace *stack);
+void ReportInvalidAlignedAllocAlignment(uptr size, uptr alignment,
+ BufferedStackTrace *stack);
+void ReportInvalidPosixMemalignAlignment(uptr alignment,
+ BufferedStackTrace *stack);
+void ReportAllocationSizeTooBig(uptr user_size, uptr total_size, uptr max_size,
+ BufferedStackTrace *stack);
+void ReportRssLimitExceeded(BufferedStackTrace *stack);
+void ReportOutOfMemory(uptr requested_size, BufferedStackTrace *stack);
void ReportStringFunctionMemoryRangesOverlap(const char *function,
const char *offset1, uptr length1,
const char *offset2, uptr length2,
diff --git a/lib/asan/asan_rtems.cc b/lib/asan/asan_rtems.cc
new file mode 100644
index 000000000000..a4af940057eb
--- /dev/null
+++ b/lib/asan/asan_rtems.cc
@@ -0,0 +1,253 @@
+//===-- asan_rtems.cc -----------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// RTEMS-specific details.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_rtems.h"
+#if SANITIZER_RTEMS
+
+#include "asan_internal.h"
+#include "asan_interceptors.h"
+#include "asan_mapping.h"
+#include "asan_poisoning.h"
+#include "asan_report.h"
+#include "asan_stack.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+
+#include <pthread.h>
+#include <stdlib.h>
+
+namespace __asan {
+
+static void ResetShadowMemory() {
+ uptr shadow_start = SHADOW_OFFSET;
+ uptr shadow_end = MEM_TO_SHADOW(kMyriadMemoryEnd32);
+ uptr gap_start = MEM_TO_SHADOW(shadow_start);
+ uptr gap_end = MEM_TO_SHADOW(shadow_end);
+
+ REAL(memset)((void *)shadow_start, 0, shadow_end - shadow_start);
+ REAL(memset)((void *)gap_start, kAsanShadowGap, gap_end - gap_start);
+}
+
+void InitializeShadowMemory() {
+ kHighMemEnd = 0;
+ kMidMemBeg = 0;
+ kMidMemEnd = 0;
+
+ ResetShadowMemory();
+}
+
+void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
+ UNIMPLEMENTED();
+}
+
+void AsanCheckDynamicRTPrereqs() {}
+void AsanCheckIncompatibleRT() {}
+void InitializeAsanInterceptors() {}
+void InitializePlatformInterceptors() {}
+void InitializePlatformExceptionHandlers() {}
+
+// RTEMS only support static linking; it sufficies to return with no
+// error.
+void *AsanDoesNotSupportStaticLinkage() { return nullptr; }
+
+void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+ UNIMPLEMENTED();
+}
+
+void EarlyInit() {
+ // Provide early initialization of shadow memory so that
+ // instrumented code running before full initialzation will not
+ // report spurious errors.
+ ResetShadowMemory();
+}
+
+// We can use a plain thread_local variable for TSD.
+static thread_local void *per_thread;
+
+void *AsanTSDGet() { return per_thread; }
+
+void AsanTSDSet(void *tsd) { per_thread = tsd; }
+
+// There's no initialization needed, and the passed-in destructor
+// will never be called. Instead, our own thread destruction hook
+// (below) will call AsanThread::TSDDtor directly.
+void AsanTSDInit(void (*destructor)(void *tsd)) {
+ DCHECK(destructor == &PlatformTSDDtor);
+}
+
+void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); }
+
+//
+// Thread registration. We provide an API similar to the Fushia port.
+//
+
+struct AsanThread::InitOptions {
+ uptr stack_bottom, stack_size, tls_bottom, tls_size;
+};
+
+// Shared setup between thread creation and startup for the initial thread.
+static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
+ uptr user_id, bool detached,
+ uptr stack_bottom, uptr stack_size,
+ uptr tls_bottom, uptr tls_size) {
+ // In lieu of AsanThread::Create.
+ AsanThread *thread = (AsanThread *)MmapOrDie(sizeof(AsanThread), __func__);
+ AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
+ asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args);
+
+ // On other systems, AsanThread::Init() is called from the new
+ // thread itself. But on RTEMS we already know the stack address
+ // range beforehand, so we can do most of the setup right now.
+ const AsanThread::InitOptions options = {stack_bottom, stack_size,
+ tls_bottom, tls_size};
+ thread->Init(&options);
+ return thread;
+}
+
+// This gets the same arguments passed to Init by CreateAsanThread, above.
+// We're in the creator thread before the new thread is actually started, but
+// its stack and tls address range are already known.
+void AsanThread::SetThreadStackAndTls(const AsanThread::InitOptions *options) {
+ DCHECK_NE(GetCurrentThread(), this);
+ DCHECK_NE(GetCurrentThread(), nullptr);
+ CHECK_NE(options->stack_bottom, 0);
+ CHECK_NE(options->stack_size, 0);
+ stack_bottom_ = options->stack_bottom;
+ stack_top_ = options->stack_bottom + options->stack_size;
+ tls_begin_ = options->tls_bottom;
+ tls_end_ = options->tls_bottom + options->tls_size;
+}
+
+// Called by __asan::AsanInitInternal (asan_rtl.c). Unlike other ports, the
+// main thread on RTEMS does not require special treatment; its AsanThread is
+// already created by the provided hooks. This function simply looks up and
+// returns the created thread.
+AsanThread *CreateMainThread() {
+ return GetThreadContextByTidLocked(0)->thread;
+}
+
+// This is called before each thread creation is attempted. So, in
+// its first call, the calling thread is the initial and sole thread.
+static void *BeforeThreadCreateHook(uptr user_id, bool detached,
+ uptr stack_bottom, uptr stack_size,
+ uptr tls_bottom, uptr tls_size) {
+ EnsureMainThreadIDIsCorrect();
+ // Strict init-order checking is thread-hostile.
+ if (flags()->strict_init_order) StopInitOrderChecking();
+
+ GET_STACK_TRACE_THREAD;
+ u32 parent_tid = GetCurrentTidOrInvalid();
+
+ return CreateAsanThread(&stack, parent_tid, user_id, detached,
+ stack_bottom, stack_size, tls_bottom, tls_size);
+}
+
+// This is called after creating a new thread (in the creating thread),
+// with the pointer returned by BeforeThreadCreateHook (above).
+static void ThreadCreateHook(void *hook, bool aborted) {
+ AsanThread *thread = static_cast<AsanThread *>(hook);
+ if (!aborted) {
+ // The thread was created successfully.
+ // ThreadStartHook is already running in the new thread.
+ } else {
+ // The thread wasn't created after all.
+ // Clean up everything we set up in BeforeThreadCreateHook.
+ asanThreadRegistry().FinishThread(thread->tid());
+ UnmapOrDie(thread, sizeof(AsanThread));
+ }
+}
+
+// This is called (1) in the newly-created thread before it runs anything else,
+// with the pointer returned by BeforeThreadCreateHook (above). (2) before a
+// thread restart.
+static void ThreadStartHook(void *hook, uptr os_id) {
+ if (!hook)
+ return;
+
+ AsanThread *thread = static_cast<AsanThread *>(hook);
+ SetCurrentThread(thread);
+
+ ThreadStatus status =
+ asanThreadRegistry().GetThreadLocked(thread->tid())->status;
+ DCHECK(status == ThreadStatusCreated || status == ThreadStatusRunning);
+ // Determine whether we are starting or restarting the thread.
+ if (status == ThreadStatusCreated)
+ // In lieu of AsanThread::ThreadStart.
+ asanThreadRegistry().StartThread(thread->tid(), os_id,
+ /*workerthread*/ false, nullptr);
+ else {
+ // In a thread restart, a thread may resume execution at an
+ // arbitrary function entry point, with its stack and TLS state
+ // reset. We unpoison the stack in that case.
+ PoisonShadow(thread->stack_bottom(), thread->stack_size(), 0);
+ }
+}
+
+// Each thread runs this just before it exits,
+// with the pointer returned by BeforeThreadCreateHook (above).
+// All per-thread destructors have already been called.
+static void ThreadExitHook(void *hook, uptr os_id) {
+ AsanThread *thread = static_cast<AsanThread *>(hook);
+ if (thread)
+ AsanThread::TSDDtor(thread->context());
+}
+
+static void HandleExit() {
+ // Disable ASan by setting it to uninitialized. Also reset the
+ // shadow memory to avoid reporting errors after the run-time has
+ // been desroyed.
+ if (asan_inited) {
+ asan_inited = false;
+ ResetShadowMemory();
+ }
+}
+
+} // namespace __asan
+
+// These are declared (in extern "C") by <some_path/sanitizer.h>.
+// The system runtime will call our definitions directly.
+
+extern "C" {
+void __sanitizer_early_init() {
+ __asan::EarlyInit();
+}
+
+void *__sanitizer_before_thread_create_hook(uptr thread, bool detached,
+ const char *name,
+ void *stack_base, size_t stack_size,
+ void *tls_base, size_t tls_size) {
+ return __asan::BeforeThreadCreateHook(
+ thread, detached,
+ reinterpret_cast<uptr>(stack_base), stack_size,
+ reinterpret_cast<uptr>(tls_base), tls_size);
+}
+
+void __sanitizer_thread_create_hook(void *handle, uptr thread, int status) {
+ __asan::ThreadCreateHook(handle, status != 0);
+}
+
+void __sanitizer_thread_start_hook(void *handle, uptr self) {
+ __asan::ThreadStartHook(handle, self);
+}
+
+void __sanitizer_thread_exit_hook(void *handle, uptr self) {
+ __asan::ThreadExitHook(handle, self);
+}
+
+void __sanitizer_exit() {
+ __asan::HandleExit();
+}
+} // "C"
+
+#endif // SANITIZER_RTEMS
diff --git a/lib/asan/asan_rtl.cc b/lib/asan/asan_rtl.cc
index 21fd0e240708..4cff736f213a 100644
--- a/lib/asan/asan_rtl.cc
+++ b/lib/asan/asan_rtl.cc
@@ -56,7 +56,8 @@ static void AsanDie() {
UnmapOrDie((void*)kLowShadowBeg, kMidMemBeg - kLowShadowBeg);
UnmapOrDie((void*)kMidMemEnd, kHighShadowEnd - kMidMemEnd);
} else {
- UnmapOrDie((void*)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
+ if (kHighShadowEnd)
+ UnmapOrDie((void*)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
}
}
}
@@ -65,8 +66,14 @@ static void AsanCheckFailed(const char *file, int line, const char *cond,
u64 v1, u64 v2) {
Report("AddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
line, cond, (uptr)v1, (uptr)v2);
- // FIXME: check for infinite recursion without a thread-local counter here.
- PRINT_CURRENT_STACK_CHECK();
+
+ // Print a stack trace the first time we come here. Otherwise, we probably
+ // failed a CHECK during symbolization.
+ static atomic_uint32_t num_calls;
+ if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) == 0) {
+ PRINT_CURRENT_STACK_CHECK();
+ }
+
Die();
}
@@ -140,6 +147,8 @@ ASAN_REPORT_ERROR_N(load, false)
ASAN_REPORT_ERROR_N(store, true)
#define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg, fatal) \
+ if (SANITIZER_MYRIAD2 && !AddrIsInMem(addr) && !AddrIsInShadow(addr)) \
+ return; \
uptr sp = MEM_TO_SHADOW(addr); \
uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \
: *reinterpret_cast<u16 *>(sp); \
@@ -306,6 +315,7 @@ static void asan_atexit() {
}
static void InitializeHighMemEnd() {
+#if !SANITIZER_MYRIAD2
#if !ASAN_FIXED_MAPPING
kHighMemEnd = GetMaxUserVirtualAddress();
// Increase kHighMemEnd to make sure it's properly
@@ -313,13 +323,16 @@ static void InitializeHighMemEnd() {
kHighMemEnd |= SHADOW_GRANULARITY * GetMmapGranularity() - 1;
#endif // !ASAN_FIXED_MAPPING
CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0);
+#endif // !SANITIZER_MYRIAD2
}
void PrintAddressSpaceLayout() {
- Printf("|| `[%p, %p]` || HighMem ||\n",
- (void*)kHighMemBeg, (void*)kHighMemEnd);
- Printf("|| `[%p, %p]` || HighShadow ||\n",
- (void*)kHighShadowBeg, (void*)kHighShadowEnd);
+ if (kHighMemBeg) {
+ Printf("|| `[%p, %p]` || HighMem ||\n",
+ (void*)kHighMemBeg, (void*)kHighMemEnd);
+ Printf("|| `[%p, %p]` || HighShadow ||\n",
+ (void*)kHighShadowBeg, (void*)kHighShadowEnd);
+ }
if (kMidMemBeg) {
Printf("|| `[%p, %p]` || ShadowGap3 ||\n",
(void*)kShadowGap3Beg, (void*)kShadowGap3End);
@@ -338,11 +351,14 @@ void PrintAddressSpaceLayout() {
Printf("|| `[%p, %p]` || LowMem ||\n",
(void*)kLowMemBeg, (void*)kLowMemEnd);
}
- Printf("MemToShadow(shadow): %p %p %p %p",
+ Printf("MemToShadow(shadow): %p %p",
(void*)MEM_TO_SHADOW(kLowShadowBeg),
- (void*)MEM_TO_SHADOW(kLowShadowEnd),
- (void*)MEM_TO_SHADOW(kHighShadowBeg),
- (void*)MEM_TO_SHADOW(kHighShadowEnd));
+ (void*)MEM_TO_SHADOW(kLowShadowEnd));
+ if (kHighMemBeg) {
+ Printf(" %p %p",
+ (void*)MEM_TO_SHADOW(kHighShadowBeg),
+ (void*)MEM_TO_SHADOW(kHighShadowEnd));
+ }
if (kMidMemBeg) {
Printf(" %p %p",
(void*)MEM_TO_SHADOW(kMidShadowBeg),
@@ -374,6 +390,7 @@ static void AsanInitInternal() {
asan_init_is_running = true;
CacheBinaryName();
+ CheckASLR();
// Initialize flags. This must be done early, because most of the
// initialization steps look at flags().
@@ -526,6 +543,9 @@ void NOINLINE __asan_handle_no_return() {
if (curr_thread) {
top = curr_thread->stack_top();
bottom = ((uptr)&local_stack - PageSize) & ~(PageSize - 1);
+ } else if (SANITIZER_RTEMS) {
+ // Give up On RTEMS.
+ return;
} else {
CHECK(!SANITIZER_FUCHSIA);
// If we haven't seen this thread, try asking the OS for stack bounds.
diff --git a/lib/asan/asan_shadow_setup.cc b/lib/asan/asan_shadow_setup.cc
index b3cf0b5c1ce2..083926e70aa2 100644
--- a/lib/asan/asan_shadow_setup.cc
+++ b/lib/asan/asan_shadow_setup.cc
@@ -14,8 +14,9 @@
#include "sanitizer_common/sanitizer_platform.h"
-// asan_fuchsia.cc has its own InitializeShadowMemory implementation.
-#if !SANITIZER_FUCHSIA
+// asan_fuchsia.cc and asan_rtems.cc have their own
+// InitializeShadowMemory implementation.
+#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
#include "asan_internal.h"
#include "asan_mapping.h"
@@ -30,8 +31,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
uptr size = end - beg + 1;
DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
- void *res = MmapFixedNoReserve(beg, size, name);
- if (res != (void *)beg) {
+ if (!MmapFixedNoReserve(beg, size, name)) {
Report(
"ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
"Perhaps you're using ulimit -v\n",
@@ -162,4 +162,4 @@ void InitializeShadowMemory() {
} // namespace __asan
-#endif // !SANITIZER_FUCHSIA
+#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
diff --git a/lib/asan/asan_thread.cc b/lib/asan/asan_thread.cc
index ad81512dff08..faf423d305b7 100644
--- a/lib/asan/asan_thread.cc
+++ b/lib/asan/asan_thread.cc
@@ -221,22 +221,25 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
void AsanThread::Init(const InitOptions *options) {
next_stack_top_ = next_stack_bottom_ = 0;
atomic_store(&stack_switching_, false, memory_order_release);
- fake_stack_ = nullptr; // Will be initialized lazily if needed.
CHECK_EQ(this->stack_size(), 0U);
SetThreadStackAndTls(options);
CHECK_GT(this->stack_size(), 0U);
CHECK(AddrIsInMem(stack_bottom_));
CHECK(AddrIsInMem(stack_top_ - 1));
ClearShadowForThreadStackAndTLS();
+ fake_stack_ = nullptr;
+ if (__asan_option_detect_stack_use_after_return)
+ AsyncSignalSafeLazyInitFakeStack();
int local = 0;
VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(),
(void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_,
&local);
}
-// Fuchsia doesn't use ThreadStart.
-// asan_fuchsia.c defines CreateMainThread and SetThreadStackAndTls.
-#if !SANITIZER_FUCHSIA
+// Fuchsia and RTEMS don't use ThreadStart.
+// asan_fuchsia.c/asan_rtems.c define CreateMainThread and
+// SetThreadStackAndTls.
+#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
thread_return_t AsanThread::ThreadStart(
tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) {
@@ -296,12 +299,17 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) {
CHECK(AddrIsInStack((uptr)&local));
}
-#endif // !SANITIZER_FUCHSIA
+#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
void AsanThread::ClearShadowForThreadStackAndTLS() {
PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0);
- if (tls_begin_ != tls_end_)
- PoisonShadow(tls_begin_, tls_end_ - tls_begin_, 0);
+ if (tls_begin_ != tls_end_) {
+ uptr tls_begin_aligned = RoundDownTo(tls_begin_, SHADOW_GRANULARITY);
+ uptr tls_end_aligned = RoundUpTo(tls_end_, SHADOW_GRANULARITY);
+ FastPoisonShadowPartialRightRedzone(tls_begin_aligned,
+ tls_end_ - tls_begin_aligned,
+ tls_end_aligned - tls_end_, 0);
+ }
}
bool AsanThread::GetStackFrameAccessByAddr(uptr addr,
@@ -386,6 +394,9 @@ static bool ThreadStackContainsAddress(ThreadContextBase *tctx_base,
}
AsanThread *GetCurrentThread() {
+ if (SANITIZER_RTEMS && !asan_inited)
+ return nullptr;
+
AsanThreadContext *context =
reinterpret_cast<AsanThreadContext *>(AsanTSDGet());
if (!context) {
@@ -477,6 +488,11 @@ void UnlockThreadRegistry() {
__asan::asanThreadRegistry().Unlock();
}
+ThreadRegistry *GetThreadRegistryLocked() {
+ __asan::asanThreadRegistry().CheckLocked();
+ return &__asan::asanThreadRegistry();
+}
+
void EnsureMainThreadIDIsCorrect() {
__asan::EnsureMainThreadIDIsCorrect();
}
diff --git a/lib/asan/asan_win.cc b/lib/asan/asan_win.cc
index 68eedd1863bc..67125d38ad4a 100644
--- a/lib/asan/asan_win.cc
+++ b/lib/asan/asan_win.cc
@@ -222,8 +222,8 @@ uptr FindDynamicShadowStart() {
uptr alignment = 8 * granularity;
uptr left_padding = granularity;
uptr space_size = kHighShadowEnd + left_padding;
- uptr shadow_start =
- FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+ uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
+ granularity, nullptr, nullptr);
CHECK_NE((uptr)0, shadow_start);
CHECK(IsAligned(shadow_start, alignment));
return shadow_start;
@@ -265,11 +265,6 @@ ShadowExceptionHandler(PEXCEPTION_POINTERS exception_pointers) {
// Determine the address of the page that is being accessed.
uptr page = RoundDownTo(addr, page_size);
- // Query the existing page.
- MEMORY_BASIC_INFORMATION mem_info = {};
- if (::VirtualQuery((LPVOID)page, &mem_info, sizeof(mem_info)) == 0)
- return EXCEPTION_CONTINUE_SEARCH;
-
// Commit the page.
uptr result =
(uptr)::VirtualAlloc((LPVOID)page, page_size, MEM_COMMIT, PAGE_READWRITE);
diff --git a/lib/asan/asan_win_dll_thunk.cc b/lib/asan/asan_win_dll_thunk.cc
index c67116c42ca2..c6a313d24026 100644
--- a/lib/asan/asan_win_dll_thunk.cc
+++ b/lib/asan/asan_win_dll_thunk.cc
@@ -99,7 +99,7 @@ INTERCEPTOR(int, _except_handler4, void *a, void *b, void *c, void *d) {
}
#endif
-// Window specific functions not included in asan_interface.inc.
+// Windows specific functions not included in asan_interface.inc.
INTERCEPT_WRAP_W_V(__asan_should_detect_stack_use_after_return)
INTERCEPT_WRAP_W_V(__asan_get_shadow_memory_dynamic_address)
INTERCEPT_WRAP_W_W(__asan_unhandled_exception_filter)
diff --git a/lib/asan/scripts/asan_device_setup b/lib/asan/scripts/asan_device_setup
index 92a109727267..5e679e366514 100755
--- a/lib/asan/scripts/asan_device_setup
+++ b/lib/asan/scripts/asan_device_setup
@@ -309,7 +309,7 @@ if [[ -n "$ASAN_RT64" ]]; then
cp "$ASAN_RT_PATH/$ASAN_RT64" "$TMPDIR/"
fi
-ASAN_OPTIONS=start_deactivated=1,malloc_context_size=0
+ASAN_OPTIONS=start_deactivated=1
# The name of a symlink to libclang_rt.asan-$ARCH-android.so used in LD_PRELOAD.
# The idea is to have the same name in lib and lib64 to keep it from falling
@@ -336,6 +336,13 @@ exec $_to \$@
EOF
}
+# On Android-L not allowing user segv handler breaks some applications.
+# Since ~May 2017 this is the default setting; included for compatibility with
+# older library versions.
+if [[ PRE_L -eq 0 ]]; then
+ ASAN_OPTIONS="$ASAN_OPTIONS,allow_user_segv_handler=1"
+fi
+
if [[ x$extra_options != x ]] ; then
ASAN_OPTIONS="$ASAN_OPTIONS,$extra_options"
fi
diff --git a/lib/asan/tests/CMakeLists.txt b/lib/asan/tests/CMakeLists.txt
index 67a8fafaba3c..1b7060591530 100644
--- a/lib/asan/tests/CMakeLists.txt
+++ b/lib/asan/tests/CMakeLists.txt
@@ -237,6 +237,9 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS AND NOT ANDROID)
if(APPLE)
darwin_filter_host_archs(ASAN_SUPPORTED_ARCH ASAN_TEST_ARCH)
endif()
+ if(OS_NAME MATCHES "SunOS")
+ list(REMOVE_ITEM ASAN_TEST_ARCH x86_64)
+ endif()
foreach(arch ${ASAN_TEST_ARCH})
@@ -248,6 +251,8 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS AND NOT ANDROID)
$<TARGET_OBJECTS:RTInterception.osx>
$<TARGET_OBJECTS:RTSanitizerCommon.osx>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.osx>
+ $<TARGET_OBJECTS:RTSanitizerCommonCoverage.osx>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.osx>
$<TARGET_OBJECTS:RTLSanCommon.osx>
$<TARGET_OBJECTS:RTUbsan.osx>)
else()
@@ -257,6 +262,8 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS AND NOT ANDROID)
$<TARGET_OBJECTS:RTInterception.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
$<TARGET_OBJECTS:RTLSanCommon.${arch}>
$<TARGET_OBJECTS:RTUbsan.${arch}>
$<TARGET_OBJECTS:RTUbsan_cxx.${arch}>)
@@ -280,6 +287,8 @@ if(ANDROID)
$<TARGET_OBJECTS:RTInterception.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
$<TARGET_OBJECTS:RTUbsan.${arch}>
$<TARGET_OBJECTS:RTUbsan_cxx.${arch}>
${COMPILER_RT_GTEST_SOURCE}
diff --git a/lib/asan/tests/asan_test.cc b/lib/asan/tests/asan_test.cc
index ed000327f126..11a3506a4853 100644
--- a/lib/asan/tests/asan_test.cc
+++ b/lib/asan/tests/asan_test.cc
@@ -25,6 +25,11 @@
#endif
#endif
+#if defined(__sun__) && defined(__svr4__)
+using std::_setjmp;
+using std::_longjmp;
+#endif
+
NOINLINE void *malloc_fff(size_t size) {
void *res = malloc/**/(size); break_optimization(0); return res;}
NOINLINE void *malloc_eee(size_t size) {
diff --git a/lib/builtins/CMakeLists.txt b/lib/builtins/CMakeLists.txt
index 0b50b5bb8d23..75ff66405649 100644
--- a/lib/builtins/CMakeLists.txt
+++ b/lib/builtins/CMakeLists.txt
@@ -173,8 +173,8 @@ set(GENERIC_TF_SOURCES
trunctfsf2.c)
option(COMPILER_RT_EXCLUDE_ATOMIC_BUILTIN
- "Skip the atomic builtin (this may be needed if system headers are unavailable)"
- Off)
+ "Skip the atomic builtin (these should normally be provided by a shared library)"
+ On)
if(NOT FUCHSIA AND NOT COMPILER_RT_BAREMETAL_BUILD)
set(GENERIC_SOURCES
@@ -406,6 +406,7 @@ if(MINGW)
arm/aeabi_ldivmod.S
arm/aeabi_uidivmod.S
arm/aeabi_uldivmod.S
+ arm/chkstk.S
divmoddi4.c
divmodsi4.c
divdi3.c
@@ -459,6 +460,41 @@ set(armv6m_SOURCES ${thumb1_SOURCES})
set(armv7m_SOURCES ${arm_SOURCES})
set(armv7em_SOURCES ${arm_SOURCES})
+# hexagon arch
+set(hexagon_SOURCES ${GENERIC_SOURCES} ${GENERIC_TF_SOURCES})
+set(hexagon_SOURCES
+ hexagon/common_entry_exit_abi1.S
+ hexagon/common_entry_exit_abi2.S
+ hexagon/common_entry_exit_legacy.S
+ hexagon/dfaddsub.S
+ hexagon/dfdiv.S
+ hexagon/dffma.S
+ hexagon/dfminmax.S
+ hexagon/dfmul.S
+ hexagon/dfsqrt.S
+ hexagon/divdi3.S
+ hexagon/divsi3.S
+ hexagon/fabs_opt.S
+ hexagon/fastmath2_dlib_asm.S
+ hexagon/fastmath2_ldlib_asm.S
+ hexagon/fastmath_dlib_asm.S
+ hexagon/fma_opt.S
+ hexagon/fmax_opt.S
+ hexagon/fmin_opt.S
+ hexagon/memcpy_forward_vp4cp4n2.S
+ hexagon/memcpy_likely_aligned.S
+ hexagon/moddi3.S
+ hexagon/modsi3.S
+ hexagon/sfdiv_opt.S
+ hexagon/sfsqrt_opt.S
+ hexagon/udivdi3.S
+ hexagon/udivmoddi4.S
+ hexagon/udivmodsi4.S
+ hexagon/udivsi3.S
+ hexagon/umoddi3.S
+ hexagon/umodsi3.S)
+
+
set(mips_SOURCES ${GENERIC_SOURCES})
set(mipsel_SOURCES ${mips_SOURCES})
set(mips64_SOURCES ${GENERIC_TF_SOURCES}
@@ -480,6 +516,12 @@ set(powerpc64_SOURCES
${GENERIC_SOURCES})
set(powerpc64le_SOURCES ${powerpc64_SOURCES})
+set(riscv_SOURCES ${GENERIC_SOURCES} ${GENERIC_TF_SOURCES})
+set(riscv32_SOURCES
+ riscv/mulsi3.S
+ ${riscv_SOURCES})
+set(riscv64_SOURCES ${riscv_SOURCES})
+
set(wasm32_SOURCES
${GENERIC_TF_SOURCES}
${GENERIC_SOURCES})
@@ -542,6 +584,12 @@ else ()
list(APPEND BUILTIN_CFLAGS -fomit-frame-pointer -DCOMPILER_RT_ARMHF_TARGET)
endif()
+ # For RISCV32, we must force enable int128 for compiling long
+ # double routines.
+ if("${arch}" STREQUAL "riscv32")
+ list(APPEND BUILTIN_CFLAGS -fforce-enable-int128)
+ endif()
+
add_compiler_rt_runtime(clang_rt.builtins
STATIC
ARCHS ${arch}
diff --git a/lib/builtins/arm/chkstk.S b/lib/builtins/arm/chkstk.S
new file mode 100644
index 000000000000..e3002105897e
--- /dev/null
+++ b/lib/builtins/arm/chkstk.S
@@ -0,0 +1,34 @@
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+
+#include "../assembly.h"
+
+// __chkstk routine
+// This routine is windows specific.
+// http://msdn.microsoft.com/en-us/library/ms648426.aspx
+
+// This clobbers the register r12, and the condition codes, and uses r5 and r6
+// as temporaries by backing them up and restoring them afterwards.
+// Does not modify any memory or the stack pointer.
+
+// movw r4, #256 // Number of bytes of stack, in units of 4 byte
+// bl __chkstk
+// sub.w sp, sp, r4
+
+#define PAGE_SIZE 4096
+
+ .p2align 2
+DEFINE_COMPILERRT_FUNCTION(__chkstk)
+ lsl r4, r4, #2
+ mov r12, sp
+ push {r5, r6}
+ mov r5, r4
+1:
+ sub r12, r12, #PAGE_SIZE
+ subs r5, r5, #PAGE_SIZE
+ ldr r6, [r12]
+ bgt 1b
+
+ pop {r5, r6}
+ bx lr
+END_COMPILERRT_FUNCTION(__chkstk)
diff --git a/lib/builtins/clear_cache.c b/lib/builtins/clear_cache.c
index 4a01cb46d4ac..9dcab344ad1b 100644
--- a/lib/builtins/clear_cache.c
+++ b/lib/builtins/clear_cache.c
@@ -33,6 +33,11 @@ uintptr_t GetCurrentProcess(void);
#include <machine/sysarch.h>
#endif
+#if defined(__OpenBSD__) && defined(__mips__)
+ #include <sys/types.h>
+ #include <machine/sysarch.h>
+#endif
+
#if defined(__linux__) && defined(__mips__)
#include <sys/cachectl.h>
#include <sys/syscall.h>
@@ -96,6 +101,8 @@ void __clear_cache(void *start, void *end) {
* Intel processors have a unified instruction and data cache
* so there is nothing to do
*/
+#elif defined(_WIN32) && (defined(__arm__) || defined(__aarch64__))
+ FlushInstructionCache(GetCurrentProcess(), start, end - start);
#elif defined(__arm__) && !defined(__APPLE__)
#if defined(__FreeBSD__) || defined(__NetBSD__)
struct arm_sync_icache_args arg;
@@ -123,8 +130,6 @@ void __clear_cache(void *start, void *end) {
: "r"(syscall_nr), "r"(start_reg), "r"(end_reg),
"r"(flags));
assert(start_reg == 0 && "Cache flush syscall failed.");
- #elif defined(_WIN32)
- FlushInstructionCache(GetCurrentProcess(), start, end - start);
#else
compilerrt_abort();
#endif
@@ -142,6 +147,8 @@ void __clear_cache(void *start, void *end) {
#else
syscall(__NR_cacheflush, start, (end_int - start_int), BCACHE);
#endif
+#elif defined(__mips__) && defined(__OpenBSD__)
+ cacheflush(start, (uintptr_t)end - (uintptr_t)start, BCACHE);
#elif defined(__aarch64__) && !defined(__APPLE__)
uint64_t xstart = (uint64_t)(uintptr_t) start;
uint64_t xend = (uint64_t)(uintptr_t) end;
@@ -156,12 +163,14 @@ void __clear_cache(void *start, void *end) {
* uintptr_t in case this runs in an IPL32 environment.
*/
const size_t dcache_line_size = 4 << ((ctr_el0 >> 16) & 15);
- for (addr = xstart; addr < xend; addr += dcache_line_size)
+ for (addr = xstart & ~(dcache_line_size - 1); addr < xend;
+ addr += dcache_line_size)
__asm __volatile("dc cvau, %0" :: "r"(addr));
__asm __volatile("dsb ish");
const size_t icache_line_size = 4 << ((ctr_el0 >> 0) & 15);
- for (addr = xstart; addr < xend; addr += icache_line_size)
+ for (addr = xstart & ~(icache_line_size - 1); addr < xend;
+ addr += icache_line_size)
__asm __volatile("ic ivau, %0" :: "r"(addr));
__asm __volatile("isb sy");
#elif defined (__powerpc64__)
diff --git a/lib/builtins/clzdi2.c b/lib/builtins/clzdi2.c
index b9e64da492bd..b56d98f5c01f 100644
--- a/lib/builtins/clzdi2.c
+++ b/lib/builtins/clzdi2.c
@@ -16,6 +16,12 @@
/* Returns: the number of leading 0-bits */
+#if !defined(__clang__) && (defined(__sparc64__) || defined(__mips64) || defined(__riscv__))
+/* gcc resolves __builtin_clz -> __clzdi2 leading to infinite recursion */
+#define __builtin_clz(a) __clzsi2(a)
+extern si_int __clzsi2(si_int);
+#endif
+
/* Precondition: a != 0 */
COMPILER_RT_ABI si_int
diff --git a/lib/builtins/cpu_model.c b/lib/builtins/cpu_model.c
index 4c96e9cd85d5..43b913390fc4 100644
--- a/lib/builtins/cpu_model.c
+++ b/lib/builtins/cpu_model.c
@@ -416,9 +416,9 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
*Subtype = AMDFAM15H_BDVER3;
break; // "bdver3"; 30h-3Fh: Steamroller
}
- if (Model >= 0x10 && Model <= 0x1f) {
+ if ((Model >= 0x10 && Model <= 0x1f) || Model == 0x02) {
*Subtype = AMDFAM15H_BDVER2;
- break; // "bdver2"; 10h-1Fh: Piledriver
+ break; // "bdver2"; 02h, 10h-1Fh: Piledriver
}
if (Model <= 0x0f) {
*Subtype = AMDFAM15H_BDVER1;
diff --git a/lib/builtins/ctzdi2.c b/lib/builtins/ctzdi2.c
index db3c6fdc08f1..eecde29718d7 100644
--- a/lib/builtins/ctzdi2.c
+++ b/lib/builtins/ctzdi2.c
@@ -16,6 +16,12 @@
/* Returns: the number of trailing 0-bits */
+#if !defined(__clang__) && (defined(__sparc64__) || defined(__mips64) || defined(__riscv__))
+/* gcc resolves __builtin_ctz -> __ctzdi2 leading to infinite recursion */
+#define __builtin_ctz(a) __ctzsi2(a)
+extern si_int __ctzsi2(si_int);
+#endif
+
/* Precondition: a != 0 */
COMPILER_RT_ABI si_int
diff --git a/lib/builtins/emutls.c b/lib/builtins/emutls.c
index 5dd8dd154771..07d436e267d8 100644
--- a/lib/builtins/emutls.c
+++ b/lib/builtins/emutls.c
@@ -14,7 +14,22 @@
#include "int_lib.h"
#include "int_util.h"
+#ifdef __BIONIC__
+/* There are 4 pthread key cleanup rounds on Bionic. Delay emutls deallocation
+ to round 2. We need to delay deallocation because:
+ - Android versions older than M lack __cxa_thread_atexit_impl, so apps
+ use a pthread key destructor to call C++ destructors.
+ - Apps might use __thread/thread_local variables in pthread destructors.
+ We can't wait until the final two rounds, because jemalloc needs two rounds
+ after the final malloc/free call to free its thread-specific data (see
+ https://reviews.llvm.org/D46978#1107507). */
+#define EMUTLS_SKIP_DESTRUCTOR_ROUNDS 1
+#else
+#define EMUTLS_SKIP_DESTRUCTOR_ROUNDS 0
+#endif
+
typedef struct emutls_address_array {
+ uintptr_t skip_destructor_rounds;
uintptr_t size; /* number of elements in the 'data' array */
void* data[];
} emutls_address_array;
@@ -65,9 +80,30 @@ static __inline void emutls_memalign_free(void *base) {
#endif
}
+static __inline void emutls_setspecific(emutls_address_array *value) {
+ pthread_setspecific(emutls_pthread_key, (void*) value);
+}
+
+static __inline emutls_address_array* emutls_getspecific() {
+ return (emutls_address_array*) pthread_getspecific(emutls_pthread_key);
+}
+
static void emutls_key_destructor(void* ptr) {
- emutls_shutdown((emutls_address_array*)ptr);
- free(ptr);
+ emutls_address_array *array = (emutls_address_array*)ptr;
+ if (array->skip_destructor_rounds > 0) {
+ /* emutls is deallocated using a pthread key destructor. These
+ * destructors are called in several rounds to accommodate destructor
+ * functions that (re)initialize key values with pthread_setspecific.
+ * Delay the emutls deallocation to accommodate other end-of-thread
+ * cleanup tasks like calling thread_local destructors (e.g. the
+ * __cxa_thread_atexit fallback in libc++abi).
+ */
+ array->skip_destructor_rounds--;
+ emutls_setspecific(array);
+ } else {
+ emutls_shutdown(array);
+ free(ptr);
+ }
}
static __inline void emutls_init(void) {
@@ -88,15 +124,7 @@ static __inline void emutls_unlock() {
pthread_mutex_unlock(&emutls_mutex);
}
-static __inline void emutls_setspecific(emutls_address_array *value) {
- pthread_setspecific(emutls_pthread_key, (void*) value);
-}
-
-static __inline emutls_address_array* emutls_getspecific() {
- return (emutls_address_array*) pthread_getspecific(emutls_pthread_key);
-}
-
-#else
+#else /* _WIN32 */
#include <windows.h>
#include <malloc.h>
@@ -222,11 +250,11 @@ static __inline void __atomic_store_n(void *ptr, uintptr_t val, unsigned type) {
InterlockedExchangePointer((void *volatile *)ptr, (void *)val);
}
-#endif
+#endif /* __ATOMIC_RELEASE */
#pragma warning (pop)
-#endif
+#endif /* _WIN32 */
static size_t emutls_num_object = 0; /* number of allocated TLS objects */
@@ -314,11 +342,12 @@ static __inline void emutls_check_array_set_size(emutls_address_array *array,
* which must be no smaller than the given index.
*/
static __inline uintptr_t emutls_new_data_array_size(uintptr_t index) {
- /* Need to allocate emutls_address_array with one extra slot
- * to store the data array size.
+ /* Need to allocate emutls_address_array with extra slots
+ * to store the header.
* Round up the emutls_address_array size to multiple of 16.
*/
- return ((index + 1 + 15) & ~((uintptr_t)15)) - 1;
+ uintptr_t header_words = sizeof(emutls_address_array) / sizeof(void *);
+ return ((index + header_words + 15) & ~((uintptr_t)15)) - header_words;
}
/* Returns the size in bytes required for an emutls_address_array with
@@ -337,8 +366,10 @@ emutls_get_address_array(uintptr_t index) {
if (array == NULL) {
uintptr_t new_size = emutls_new_data_array_size(index);
array = (emutls_address_array*) malloc(emutls_asize(new_size));
- if (array)
+ if (array) {
memset(array->data, 0, new_size * sizeof(void*));
+ array->skip_destructor_rounds = EMUTLS_SKIP_DESTRUCTOR_ROUNDS;
+ }
emutls_check_array_set_size(array, new_size);
} else if (index > array->size) {
uintptr_t orig_size = array->size;
diff --git a/lib/builtins/hexagon/common_entry_exit_abi1.S b/lib/builtins/hexagon/common_entry_exit_abi1.S
new file mode 100644
index 000000000000..d5479d2a52fa
--- /dev/null
+++ b/lib/builtins/hexagon/common_entry_exit_abi1.S
@@ -0,0 +1,103 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+/* Functions that implement common sequences in function prologues and epilogues
+ used to save code size */
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .globl \name
+ .type \name, @function
+ .falign
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+ .macro FALLTHROUGH_TAIL_CALL name0 name1
+ .size \name0, . - \name0
+ .globl \name1
+ .type \name1, @function
+ .falign
+\name1:
+ .endm
+
+
+
+
+/* Save r25:24 at fp+#-8 and r27:26 at fp+#-16. */
+
+
+
+
+/* The compiler knows that the __save_* functions clobber LR. No other
+ registers should be used without informing the compiler. */
+
+/* Since we can only issue one store per packet, we don't hurt performance by
+ simply jumping to the right point in this sequence of stores. */
+
+FUNCTION_BEGIN __save_r24_through_r27
+ memd(fp+#-16) = r27:26
+FALLTHROUGH_TAIL_CALL __save_r24_through_r27 __save_r24_through_r25
+ {
+ memd(fp+#-8) = r25:24
+ jumpr lr
+ }
+FUNCTION_END __save_r24_through_r25
+
+
+
+
+/* For each of the *_before_tailcall functions, jumpr lr is executed in parallel
+ with deallocframe. That way, the return gets the old value of lr, which is
+ where these functions need to return, and at the same time, lr gets the value
+ it needs going into the tail call. */
+
+FUNCTION_BEGIN __restore_r24_through_r27_and_deallocframe_before_tailcall
+ r27:26 = memd(fp+#-16)
+FALLTHROUGH_TAIL_CALL __restore_r24_through_r27_and_deallocframe_before_tailcall __restore_r24_through_r25_and_deallocframe_before_tailcall
+ {
+ r25:24 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r24_through_r25_and_deallocframe_before_tailcall
+
+
+
+
+/* Here we use the extra load bandwidth to restore LR early, allowing the return
+ to occur in parallel with the deallocframe. */
+
+FUNCTION_BEGIN __restore_r24_through_r27_and_deallocframe
+ {
+ lr = memw(fp+#4)
+ r27:26 = memd(fp+#-16)
+ }
+ {
+ r25:24 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r24_through_r27_and_deallocframe
+
+
+
+
+/* Here the load bandwidth is maximized. */
+
+FUNCTION_BEGIN __restore_r24_through_r25_and_deallocframe
+ {
+ r25:24 = memd(fp+#-8)
+ deallocframe
+ }
+ jumpr lr
+FUNCTION_END __restore_r24_through_r25_and_deallocframe
diff --git a/lib/builtins/hexagon/common_entry_exit_abi2.S b/lib/builtins/hexagon/common_entry_exit_abi2.S
new file mode 100644
index 000000000000..6f470343db49
--- /dev/null
+++ b/lib/builtins/hexagon/common_entry_exit_abi2.S
@@ -0,0 +1,268 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+/* Functions that implement common sequences in function prologues and epilogues
+ used to save code size */
+
+ .macro FUNCTION_BEGIN name
+ .p2align 2
+ .section .text.\name,"ax",@progbits
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+ .macro FALLTHROUGH_TAIL_CALL name0 name1
+ .p2align 2
+ .size \name0, . - \name0
+ .globl \name1
+ .type \name1, @function
+\name1:
+ .endm
+
+
+
+
+/* Save r17:16 at fp+#-8, r19:18 at fp+#-16, r21:20 at fp+#-24, r23:22 at
+ fp+#-32, r25:24 at fp+#-40, and r27:26 at fp+#-48.
+ The compiler knows that the __save_* functions clobber LR. No other
+ registers should be used without informing the compiler. */
+
+FUNCTION_BEGIN __save_r16_through_r27
+ {
+ memd(fp+#-48) = r27:26
+ memd(fp+#-40) = r25:24
+ }
+ {
+ memd(fp+#-32) = r23:22
+ memd(fp+#-24) = r21:20
+ }
+ {
+ memd(fp+#-16) = r19:18
+ memd(fp+#-8) = r17:16
+ jumpr lr
+ }
+FUNCTION_END __save_r16_through_r27
+
+FUNCTION_BEGIN __save_r16_through_r25
+ {
+ memd(fp+#-40) = r25:24
+ memd(fp+#-32) = r23:22
+ }
+ {
+ memd(fp+#-24) = r21:20
+ memd(fp+#-16) = r19:18
+ }
+ {
+ memd(fp+#-8) = r17:16
+ jumpr lr
+ }
+FUNCTION_END __save_r16_through_r25
+
+FUNCTION_BEGIN __save_r16_through_r23
+ {
+ memd(fp+#-32) = r23:22
+ memd(fp+#-24) = r21:20
+ }
+ {
+ memd(fp+#-16) = r19:18
+ memd(fp+#-8) = r17:16
+ jumpr lr
+ }
+FUNCTION_END __save_r16_through_r23
+
+FUNCTION_BEGIN __save_r16_through_r21
+ {
+ memd(fp+#-24) = r21:20
+ memd(fp+#-16) = r19:18
+ }
+ {
+ memd(fp+#-8) = r17:16
+ jumpr lr
+ }
+FUNCTION_END __save_r16_through_r21
+
+FUNCTION_BEGIN __save_r16_through_r19
+ {
+ memd(fp+#-16) = r19:18
+ memd(fp+#-8) = r17:16
+ jumpr lr
+ }
+FUNCTION_END __save_r16_through_r19
+
+FUNCTION_BEGIN __save_r16_through_r17
+ {
+ memd(fp+#-8) = r17:16
+ jumpr lr
+ }
+FUNCTION_END __save_r16_through_r17
+
+/* For each of the *_before_tailcall functions, jumpr lr is executed in parallel
+ with deallocframe. That way, the return gets the old value of lr, which is
+ where these functions need to return, and at the same time, lr gets the value
+ it needs going into the tail call. */
+
+
+FUNCTION_BEGIN __restore_r16_through_r27_and_deallocframe_before_tailcall
+ r27:26 = memd(fp+#-48)
+ {
+ r25:24 = memd(fp+#-40)
+ r23:22 = memd(fp+#-32)
+ }
+ {
+ r21:20 = memd(fp+#-24)
+ r19:18 = memd(fp+#-16)
+ }
+ {
+ r17:16 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r16_through_r27_and_deallocframe_before_tailcall
+
+FUNCTION_BEGIN __restore_r16_through_r25_and_deallocframe_before_tailcall
+ {
+ r25:24 = memd(fp+#-40)
+ r23:22 = memd(fp+#-32)
+ }
+ {
+ r21:20 = memd(fp+#-24)
+ r19:18 = memd(fp+#-16)
+ }
+ {
+ r17:16 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r16_through_r25_and_deallocframe_before_tailcall
+
+FUNCTION_BEGIN __restore_r16_through_r23_and_deallocframe_before_tailcall
+ {
+ r23:22 = memd(fp+#-32)
+ r21:20 = memd(fp+#-24)
+ }
+ r19:18 = memd(fp+#-16)
+ {
+ r17:16 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r16_through_r23_and_deallocframe_before_tailcall
+
+
+FUNCTION_BEGIN __restore_r16_through_r21_and_deallocframe_before_tailcall
+ {
+ r21:20 = memd(fp+#-24)
+ r19:18 = memd(fp+#-16)
+ }
+ {
+ r17:16 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r16_through_r19_and_deallocframe_before_tailcall
+
+FUNCTION_BEGIN __restore_r16_through_r19_and_deallocframe_before_tailcall
+ r19:18 = memd(fp+#-16)
+ {
+ r17:16 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r16_through_r19_and_deallocframe_before_tailcall
+
+FUNCTION_BEGIN __restore_r16_through_r17_and_deallocframe_before_tailcall
+ {
+ r17:16 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r16_through_r17_and_deallocframe_before_tailcall
+
+
+FUNCTION_BEGIN __restore_r16_through_r27_and_deallocframe
+ r27:26 = memd(fp+#-48)
+ {
+ r25:24 = memd(fp+#-40)
+ r23:22 = memd(fp+#-32)
+ }
+ {
+ r21:20 = memd(fp+#-24)
+ r19:18 = memd(fp+#-16)
+ }
+ {
+ r17:16 = memd(fp+#-8)
+ dealloc_return
+ }
+FUNCTION_END __restore_r16_through_r27_and_deallocframe
+
+FUNCTION_BEGIN __restore_r16_through_r25_and_deallocframe
+ {
+ r25:24 = memd(fp+#-40)
+ r23:22 = memd(fp+#-32)
+ }
+ {
+ r21:20 = memd(fp+#-24)
+ r19:18 = memd(fp+#-16)
+ }
+ {
+ r17:16 = memd(fp+#-8)
+ dealloc_return
+ }
+FUNCTION_END __restore_r16_through_r25_and_deallocframe
+
+FUNCTION_BEGIN __restore_r16_through_r23_and_deallocframe
+ {
+ r23:22 = memd(fp+#-32)
+ }
+ {
+ r21:20 = memd(fp+#-24)
+ r19:18 = memd(fp+#-16)
+ }
+ {
+ r17:16 = memd(fp+#-8)
+ dealloc_return
+ }
+FUNCTION_END __restore_r16_through_r23_and_deallocframe
+
+FUNCTION_BEGIN __restore_r16_through_r21_and_deallocframe
+ {
+ r21:20 = memd(fp+#-24)
+ r19:18 = memd(fp+#-16)
+ }
+ {
+ r17:16 = memd(fp+#-8)
+ dealloc_return
+ }
+FUNCTION_END __restore_r16_through_r21_and_deallocframe
+
+FUNCTION_BEGIN __restore_r16_through_r19_and_deallocframe
+ {
+ r19:18 = memd(fp+#-16)
+ r17:16 = memd(fp+#-8)
+ }
+ {
+ dealloc_return
+ }
+FUNCTION_END __restore_r16_through_r19_and_deallocframe
+
+FUNCTION_BEGIN __restore_r16_through_r17_and_deallocframe
+ {
+ r17:16 = memd(fp+#-8)
+ dealloc_return
+ }
+FUNCTION_END __restore_r16_through_r17_and_deallocframe
+
+FUNCTION_BEGIN __deallocframe
+ dealloc_return
+FUNCTION_END __deallocframe
diff --git a/lib/builtins/hexagon/common_entry_exit_legacy.S b/lib/builtins/hexagon/common_entry_exit_legacy.S
new file mode 100644
index 000000000000..3258f15a3267
--- /dev/null
+++ b/lib/builtins/hexagon/common_entry_exit_legacy.S
@@ -0,0 +1,157 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+/* Functions that implement common sequences in function prologues and epilogues
+ used to save code size */
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .globl \name
+ .type \name, @function
+ .falign
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+ .macro FALLTHROUGH_TAIL_CALL name0 name1
+ .size \name0, . - \name0
+ .globl \name1
+ .type \name1, @function
+ .falign
+\name1:
+ .endm
+
+
+
+
+/* Save r27:26 at fp+#-8, r25:24 at fp+#-16, r23:22 at fp+#-24, r21:20 at
+ fp+#-32, r19:18 at fp+#-40, and r17:16 at fp+#-48. */
+
+
+
+
+/* The compiler knows that the __save_* functions clobber LR. No other
+ registers should be used without informing the compiler. */
+
+/* Since we can only issue one store per packet, we don't hurt performance by
+ simply jumping to the right point in this sequence of stores. */
+
+FUNCTION_BEGIN __save_r27_through_r16
+ memd(fp+#-48) = r17:16
+FALLTHROUGH_TAIL_CALL __save_r27_through_r16 __save_r27_through_r18
+ memd(fp+#-40) = r19:18
+FALLTHROUGH_TAIL_CALL __save_r27_through_r18 __save_r27_through_r20
+ memd(fp+#-32) = r21:20
+FALLTHROUGH_TAIL_CALL __save_r27_through_r20 __save_r27_through_r22
+ memd(fp+#-24) = r23:22
+FALLTHROUGH_TAIL_CALL __save_r27_through_r22 __save_r27_through_r24
+ memd(fp+#-16) = r25:24
+ {
+ memd(fp+#-8) = r27:26
+ jumpr lr
+ }
+FUNCTION_END __save_r27_through_r24
+
+
+
+
+/* For each of the *_before_sibcall functions, jumpr lr is executed in parallel
+ with deallocframe. That way, the return gets the old value of lr, which is
+ where these functions need to return, and at the same time, lr gets the value
+ it needs going into the sibcall. */
+
+FUNCTION_BEGIN __restore_r27_through_r20_and_deallocframe_before_sibcall
+ {
+ r21:20 = memd(fp+#-32)
+ r23:22 = memd(fp+#-24)
+ }
+FALLTHROUGH_TAIL_CALL __restore_r27_through_r20_and_deallocframe_before_sibcall __restore_r27_through_r24_and_deallocframe_before_sibcall
+ {
+ r25:24 = memd(fp+#-16)
+ jump __restore_r27_through_r26_and_deallocframe_before_sibcall
+ }
+FUNCTION_END __restore_r27_through_r24_and_deallocframe_before_sibcall
+
+
+
+
+FUNCTION_BEGIN __restore_r27_through_r16_and_deallocframe_before_sibcall
+ r17:16 = memd(fp+#-48)
+FALLTHROUGH_TAIL_CALL __restore_r27_through_r16_and_deallocframe_before_sibcall __restore_r27_through_r18_and_deallocframe_before_sibcall
+ {
+ r19:18 = memd(fp+#-40)
+ r21:20 = memd(fp+#-32)
+ }
+FALLTHROUGH_TAIL_CALL __restore_r27_through_r18_and_deallocframe_before_sibcall __restore_r27_through_r22_and_deallocframe_before_sibcall
+ {
+ r23:22 = memd(fp+#-24)
+ r25:24 = memd(fp+#-16)
+ }
+FALLTHROUGH_TAIL_CALL __restore_r27_through_r22_and_deallocframe_before_sibcall __restore_r27_through_r26_and_deallocframe_before_sibcall
+ {
+ r27:26 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r27_through_r26_and_deallocframe_before_sibcall
+
+
+
+
+/* Here we use the extra load bandwidth to restore LR early, allowing the return
+ to occur in parallel with the deallocframe. */
+
+FUNCTION_BEGIN __restore_r27_through_r16_and_deallocframe
+ {
+ r17:16 = memd(fp+#-48)
+ r19:18 = memd(fp+#-40)
+ }
+FALLTHROUGH_TAIL_CALL __restore_r27_through_r16_and_deallocframe __restore_r27_through_r20_and_deallocframe
+ {
+ r21:20 = memd(fp+#-32)
+ r23:22 = memd(fp+#-24)
+ }
+FALLTHROUGH_TAIL_CALL __restore_r27_through_r20_and_deallocframe __restore_r27_through_r24_and_deallocframe
+ {
+ lr = memw(fp+#4)
+ r25:24 = memd(fp+#-16)
+ }
+ {
+ r27:26 = memd(fp+#-8)
+ deallocframe
+ jumpr lr
+ }
+FUNCTION_END __restore_r27_through_r24_and_deallocframe
+
+
+
+
+/* Here the load bandwidth is maximized for all three functions. */
+
+FUNCTION_BEGIN __restore_r27_through_r18_and_deallocframe
+ {
+ r19:18 = memd(fp+#-40)
+ r21:20 = memd(fp+#-32)
+ }
+FALLTHROUGH_TAIL_CALL __restore_r27_through_r18_and_deallocframe __restore_r27_through_r22_and_deallocframe
+ {
+ r23:22 = memd(fp+#-24)
+ r25:24 = memd(fp+#-16)
+ }
+FALLTHROUGH_TAIL_CALL __restore_r27_through_r22_and_deallocframe __restore_r27_through_r26_and_deallocframe
+ {
+ r27:26 = memd(fp+#-8)
+ deallocframe
+ }
+ jumpr lr
+FUNCTION_END __restore_r27_through_r26_and_deallocframe
diff --git a/lib/builtins/hexagon/dfaddsub.S b/lib/builtins/hexagon/dfaddsub.S
new file mode 100644
index 000000000000..4173f86a4f54
--- /dev/null
+++ b/lib/builtins/hexagon/dfaddsub.S
@@ -0,0 +1,398 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+/* Double Precision Multiply */
+
+#define A r1:0
+#define AH r1
+#define AL r0
+#define B r3:2
+#define BH r3
+#define BL r2
+
+#define EXPA r4
+#define EXPB r5
+#define EXPB_A r5:4
+
+#define ZTMP r7:6
+#define ZTMPH r7
+#define ZTMPL r6
+
+#define ATMP r13:12
+#define ATMPH r13
+#define ATMPL r12
+
+#define BTMP r9:8
+#define BTMPH r9
+#define BTMPL r8
+
+#define ATMP2 r11:10
+#define ATMP2H r11
+#define ATMP2L r10
+
+#define EXPDIFF r15
+#define EXTRACTOFF r14
+#define EXTRACTAMT r15:14
+
+#define TMP r28
+
+#define MANTBITS 52
+#define HI_MANTBITS 20
+#define EXPBITS 11
+#define BIAS 1024
+#define MANTISSA_TO_INT_BIAS 52
+#define SR_BIT_INEXACT 5
+
+#ifndef SR_ROUND_OFF
+#define SR_ROUND_OFF 22
+#endif
+
+#define NORMAL p3
+#define BIGB p2
+
+#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG
+#define FAST_ALIAS(TAG) .global __hexagon_fast_##TAG ; .set __hexagon_fast_##TAG, __hexagon_##TAG
+#define FAST2_ALIAS(TAG) .global __hexagon_fast2_##TAG ; .set __hexagon_fast2_##TAG, __hexagon_##TAG
+#define END(TAG) .size TAG,.-TAG
+
+ .text
+ .global __hexagon_adddf3
+ .global __hexagon_subdf3
+ .type __hexagon_adddf3, @function
+ .type __hexagon_subdf3, @function
+
+Q6_ALIAS(adddf3)
+FAST_ALIAS(adddf3)
+FAST2_ALIAS(adddf3)
+Q6_ALIAS(subdf3)
+FAST_ALIAS(subdf3)
+FAST2_ALIAS(subdf3)
+
+ .p2align 5
+__hexagon_adddf3:
+ {
+ EXPA = extractu(AH,#EXPBITS,#HI_MANTBITS)
+ EXPB = extractu(BH,#EXPBITS,#HI_MANTBITS)
+ ATMP = combine(##0x20000000,#0)
+ }
+ {
+ NORMAL = dfclass(A,#2)
+ NORMAL = dfclass(B,#2)
+ BTMP = ATMP
+ BIGB = cmp.gtu(EXPB,EXPA) // Is B substantially greater than A?
+ }
+ {
+ if (!NORMAL) jump .Ladd_abnormal // If abnormal, go to special code
+ if (BIGB) A = B // if B >> A, swap A and B
+ if (BIGB) B = A // If B >> A, swap A and B
+ if (BIGB) EXPB_A = combine(EXPA,EXPB) // swap exponents
+ }
+ {
+ ATMP = insert(A,#MANTBITS,#EXPBITS-2) // Q1.62
+ BTMP = insert(B,#MANTBITS,#EXPBITS-2) // Q1.62
+ EXPDIFF = sub(EXPA,EXPB)
+ ZTMP = combine(#62,#1)
+ }
+#undef BIGB
+#undef NORMAL
+#define B_POS p3
+#define A_POS p2
+#define NO_STICKIES p1
+.Ladd_continue:
+ {
+ EXPDIFF = min(EXPDIFF,ZTMPH) // If exponent difference >= ~60,
+ // will collapse to sticky bit
+ ATMP2 = neg(ATMP)
+ A_POS = cmp.gt(AH,#-1)
+ EXTRACTOFF = #0
+ }
+ {
+ if (!A_POS) ATMP = ATMP2
+ ATMP2 = extractu(BTMP,EXTRACTAMT)
+ BTMP = ASR(BTMP,EXPDIFF)
+#undef EXTRACTAMT
+#undef EXPDIFF
+#undef EXTRACTOFF
+#define ZERO r15:14
+ ZERO = #0
+ }
+ {
+ NO_STICKIES = cmp.eq(ATMP2,ZERO)
+ if (!NO_STICKIES.new) BTMPL = or(BTMPL,ZTMPL)
+ EXPB = add(EXPA,#-BIAS-60)
+ B_POS = cmp.gt(BH,#-1)
+ }
+ {
+ ATMP = add(ATMP,BTMP) // ADD!!!
+ ATMP2 = sub(ATMP,BTMP) // Negate and ADD --> SUB!!!
+ ZTMP = combine(#54,##2045)
+ }
+ {
+ p0 = cmp.gtu(EXPA,ZTMPH) // must be pretty high in case of large cancellation
+ p0 = !cmp.gtu(EXPA,ZTMPL)
+ if (!p0.new) jump:nt .Ladd_ovf_unf
+ if (!B_POS) ATMP = ATMP2 // if B neg, pick difference
+ }
+ {
+ A = convert_d2df(ATMP) // Convert to Double Precision, taking care of flags, etc. So nice!
+ p0 = cmp.eq(ATMPH,#0)
+ p0 = cmp.eq(ATMPL,#0)
+ if (p0.new) jump:nt .Ladd_zero // or maybe conversion handles zero case correctly?
+ }
+ {
+ AH += asl(EXPB,#HI_MANTBITS)
+ jumpr r31
+ }
+ .falign
+__hexagon_subdf3:
+ {
+ BH = togglebit(BH,#31)
+ jump __qdsp_adddf3
+ }
+
+
+ .falign
+.Ladd_zero:
+ // True zero, full cancellation
+ // +0 unless round towards negative infinity
+ {
+ TMP = USR
+ A = #0
+ BH = #1
+ }
+ {
+ TMP = extractu(TMP,#2,#22)
+ BH = asl(BH,#31)
+ }
+ {
+ p0 = cmp.eq(TMP,#2)
+ if (p0.new) AH = xor(AH,BH)
+ jumpr r31
+ }
+ .falign
+.Ladd_ovf_unf:
+ // Overflow or Denormal is possible
+ // Good news: Underflow flag is not possible!
+ /*
+ * ATMP has 2's complement value
+ *
+ * EXPA has A's exponent, EXPB has EXPA-BIAS-60
+ *
+ * Convert, extract exponent, add adjustment.
+ * If > 2046, overflow
+ * If <= 0, denormal
+ *
+ * Note that we've not done our zero check yet, so do that too
+ *
+ */
+ {
+ A = convert_d2df(ATMP)
+ p0 = cmp.eq(ATMPH,#0)
+ p0 = cmp.eq(ATMPL,#0)
+ if (p0.new) jump:nt .Ladd_zero
+ }
+ {
+ TMP = extractu(AH,#EXPBITS,#HI_MANTBITS)
+ AH += asl(EXPB,#HI_MANTBITS)
+ }
+ {
+ EXPB = add(EXPB,TMP)
+ B = combine(##0x00100000,#0)
+ }
+ {
+ p0 = cmp.gt(EXPB,##BIAS+BIAS-2)
+ if (p0.new) jump:nt .Ladd_ovf
+ }
+ {
+ p0 = cmp.gt(EXPB,#0)
+ if (p0.new) jumpr:t r31
+ TMP = sub(#1,EXPB)
+ }
+ {
+ B = insert(A,#MANTBITS,#0)
+ A = ATMP
+ }
+ {
+ B = lsr(B,TMP)
+ }
+ {
+ A = insert(B,#63,#0)
+ jumpr r31
+ }
+ .falign
+.Ladd_ovf:
+ // We get either max finite value or infinity. Either way, overflow+inexact
+ {
+ A = ATMP // 2's complement value
+ TMP = USR
+ ATMP = combine(##0x7fefffff,#-1) // positive max finite
+ }
+ {
+ EXPB = extractu(TMP,#2,#SR_ROUND_OFF) // rounding bits
+ TMP = or(TMP,#0x28) // inexact + overflow
+ BTMP = combine(##0x7ff00000,#0) // positive infinity
+ }
+ {
+ USR = TMP
+ EXPB ^= lsr(AH,#31) // Does sign match rounding?
+ TMP = EXPB // unmodified rounding mode
+ }
+ {
+ p0 = !cmp.eq(TMP,#1) // If not round-to-zero and
+ p0 = !cmp.eq(EXPB,#2) // Not rounding the other way,
+ if (p0.new) ATMP = BTMP // we should get infinity
+ }
+ {
+ A = insert(ATMP,#63,#0) // insert inf/maxfinite, leave sign
+ }
+ {
+ p0 = dfcmp.eq(A,A)
+ jumpr r31
+ }
+
+.Ladd_abnormal:
+ {
+ ATMP = extractu(A,#63,#0) // strip off sign
+ BTMP = extractu(B,#63,#0) // strip off sign
+ }
+ {
+ p3 = cmp.gtu(ATMP,BTMP)
+ if (!p3.new) A = B // sort values
+ if (!p3.new) B = A // sort values
+ }
+ {
+ // Any NaN --> NaN, possibly raise invalid if sNaN
+ p0 = dfclass(A,#0x0f) // A not NaN?
+ if (!p0.new) jump:nt .Linvalid_nan_add
+ if (!p3) ATMP = BTMP
+ if (!p3) BTMP = ATMP
+ }
+ {
+ // Infinity + non-infinity number is infinity
+ // Infinity + infinity --> inf or nan
+ p1 = dfclass(A,#0x08) // A is infinity
+ if (p1.new) jump:nt .Linf_add
+ }
+ {
+ p2 = dfclass(B,#0x01) // B is zero
+ if (p2.new) jump:nt .LB_zero // so return A or special 0+0
+ ATMP = #0
+ }
+ // We are left with adding one or more subnormals
+ {
+ p0 = dfclass(A,#4)
+ if (p0.new) jump:nt .Ladd_two_subnormal
+ ATMP = combine(##0x20000000,#0)
+ }
+ {
+ EXPA = extractu(AH,#EXPBITS,#HI_MANTBITS)
+ EXPB = #1
+ // BTMP already ABS(B)
+ BTMP = asl(BTMP,#EXPBITS-2)
+ }
+#undef ZERO
+#define EXTRACTOFF r14
+#define EXPDIFF r15
+ {
+ ATMP = insert(A,#MANTBITS,#EXPBITS-2)
+ EXPDIFF = sub(EXPA,EXPB)
+ ZTMP = combine(#62,#1)
+ jump .Ladd_continue
+ }
+
+.Ladd_two_subnormal:
+ {
+ ATMP = extractu(A,#63,#0)
+ BTMP = extractu(B,#63,#0)
+ }
+ {
+ ATMP = neg(ATMP)
+ BTMP = neg(BTMP)
+ p0 = cmp.gt(AH,#-1)
+ p1 = cmp.gt(BH,#-1)
+ }
+ {
+ if (p0) ATMP = A
+ if (p1) BTMP = B
+ }
+ {
+ ATMP = add(ATMP,BTMP)
+ }
+ {
+ BTMP = neg(ATMP)
+ p0 = cmp.gt(ATMPH,#-1)
+ B = #0
+ }
+ {
+ if (!p0) A = BTMP
+ if (p0) A = ATMP
+ BH = ##0x80000000
+ }
+ {
+ if (!p0) AH = or(AH,BH)
+ p0 = dfcmp.eq(A,B)
+ if (p0.new) jump:nt .Lzero_plus_zero
+ }
+ {
+ jumpr r31
+ }
+
+.Linvalid_nan_add:
+ {
+ TMP = convert_df2sf(A) // will generate invalid if sNaN
+ p0 = dfclass(B,#0x0f) // if B is not NaN
+ if (p0.new) B = A // make it whatever A is
+ }
+ {
+ BL = convert_df2sf(B) // will generate invalid if sNaN
+ A = #-1
+ jumpr r31
+ }
+ .falign
+.LB_zero:
+ {
+ p0 = dfcmp.eq(ATMP,A) // is A also zero?
+ if (!p0.new) jumpr:t r31 // If not, just return A
+ }
+ // 0 + 0 is special
+ // if equal integral values, they have the same sign, which is fine for all rounding
+ // modes.
+ // If unequal in sign, we get +0 for all rounding modes except round down
+.Lzero_plus_zero:
+ {
+ p0 = cmp.eq(A,B)
+ if (p0.new) jumpr:t r31
+ }
+ {
+ TMP = USR
+ }
+ {
+ TMP = extractu(TMP,#2,#SR_ROUND_OFF)
+ A = #0
+ }
+ {
+ p0 = cmp.eq(TMP,#2)
+ if (p0.new) AH = ##0x80000000
+ jumpr r31
+ }
+.Linf_add:
+ // adding infinities is only OK if they are equal
+ {
+ p0 = !cmp.eq(AH,BH) // Do they have different signs
+ p0 = dfclass(B,#8) // And is B also infinite?
+ if (!p0.new) jumpr:t r31 // If not, just a normal inf
+ }
+ {
+ BL = ##0x7f800001 // sNAN
+ }
+ {
+ A = convert_sf2df(BL) // trigger invalid, set NaN
+ jumpr r31
+ }
+END(__hexagon_adddf3)
diff --git a/lib/builtins/hexagon/dfdiv.S b/lib/builtins/hexagon/dfdiv.S
new file mode 100644
index 000000000000..0c5dbe272c89
--- /dev/null
+++ b/lib/builtins/hexagon/dfdiv.S
@@ -0,0 +1,492 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+/* Double Precision Divide */
+
+#define A r1:0
+#define AH r1
+#define AL r0
+
+#define B r3:2
+#define BH r3
+#define BL r2
+
+#define Q r5:4
+#define QH r5
+#define QL r4
+
+#define PROD r7:6
+#define PRODHI r7
+#define PRODLO r6
+
+#define SFONE r8
+#define SFDEN r9
+#define SFERROR r10
+#define SFRECIP r11
+
+#define EXPBA r13:12
+#define EXPB r13
+#define EXPA r12
+
+#define REMSUB2 r15:14
+
+
+
+#define SIGN r28
+
+#define Q_POSITIVE p3
+#define NORMAL p2
+#define NO_OVF_UNF p1
+#define P_TMP p0
+
+#define RECIPEST_SHIFT 3
+#define QADJ 61
+
+#define DFCLASS_NORMAL 0x02
+#define DFCLASS_NUMBER 0x0F
+#define DFCLASS_INFINITE 0x08
+#define DFCLASS_ZERO 0x01
+#define DFCLASS_NONZERO (DFCLASS_NUMBER ^ DFCLASS_ZERO)
+#define DFCLASS_NONINFINITE (DFCLASS_NUMBER ^ DFCLASS_INFINITE)
+
+#define DF_MANTBITS 52
+#define DF_EXPBITS 11
+#define SF_MANTBITS 23
+#define SF_EXPBITS 8
+#define DF_BIAS 0x3ff
+
+#define SR_ROUND_OFF 22
+
+#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG
+#define FAST_ALIAS(TAG) .global __hexagon_fast_##TAG ; .set __hexagon_fast_##TAG, __hexagon_##TAG
+#define FAST2_ALIAS(TAG) .global __hexagon_fast2_##TAG ; .set __hexagon_fast2_##TAG, __hexagon_##TAG
+#define END(TAG) .size TAG,.-TAG
+
+ .text
+ .global __hexagon_divdf3
+ .type __hexagon_divdf3,@function
+ Q6_ALIAS(divdf3)
+ FAST_ALIAS(divdf3)
+ FAST2_ALIAS(divdf3)
+ .p2align 5
+__hexagon_divdf3:
+ {
+ NORMAL = dfclass(A,#DFCLASS_NORMAL)
+ NORMAL = dfclass(B,#DFCLASS_NORMAL)
+ EXPBA = combine(BH,AH)
+ SIGN = xor(AH,BH)
+ }
+#undef A
+#undef AH
+#undef AL
+#undef B
+#undef BH
+#undef BL
+#define REM r1:0
+#define REMHI r1
+#define REMLO r0
+#define DENOM r3:2
+#define DENOMHI r3
+#define DENOMLO r2
+ {
+ if (!NORMAL) jump .Ldiv_abnormal
+ PROD = extractu(DENOM,#SF_MANTBITS,#DF_MANTBITS-SF_MANTBITS)
+ SFONE = ##0x3f800001
+ }
+ {
+ SFDEN = or(SFONE,PRODLO)
+ EXPB = extractu(EXPB,#DF_EXPBITS,#DF_MANTBITS-32)
+ EXPA = extractu(EXPA,#DF_EXPBITS,#DF_MANTBITS-32)
+ Q_POSITIVE = cmp.gt(SIGN,#-1)
+ }
+#undef SIGN
+#define ONE r28
+.Ldenorm_continue:
+ {
+ SFRECIP,P_TMP = sfrecipa(SFONE,SFDEN)
+ SFERROR = and(SFONE,#-2)
+ ONE = #1
+ EXPA = sub(EXPA,EXPB)
+ }
+#undef EXPB
+#define RECIPEST r13
+ {
+ SFERROR -= sfmpy(SFRECIP,SFDEN):lib
+ REMHI = insert(ONE,#DF_EXPBITS+1,#DF_MANTBITS-32)
+ RECIPEST = ##0x00800000 << RECIPEST_SHIFT
+ }
+ {
+ SFRECIP += sfmpy(SFRECIP,SFERROR):lib
+ DENOMHI = insert(ONE,#DF_EXPBITS+1,#DF_MANTBITS-32)
+ SFERROR = and(SFONE,#-2)
+ }
+ {
+ SFERROR -= sfmpy(SFRECIP,SFDEN):lib
+ QH = #-DF_BIAS+1
+ QL = #DF_BIAS-1
+ }
+ {
+ SFRECIP += sfmpy(SFRECIP,SFERROR):lib
+ NO_OVF_UNF = cmp.gt(EXPA,QH)
+ NO_OVF_UNF = !cmp.gt(EXPA,QL)
+ }
+ {
+ RECIPEST = insert(SFRECIP,#SF_MANTBITS,#RECIPEST_SHIFT)
+ Q = #0
+ EXPA = add(EXPA,#-QADJ)
+ }
+#undef SFERROR
+#undef SFRECIP
+#define TMP r10
+#define TMP1 r11
+ {
+ RECIPEST = add(RECIPEST,#((-3) << RECIPEST_SHIFT))
+ }
+
+#define DIV_ITER1B(QSHIFTINSN,QSHIFT,REMSHIFT,EXTRA) \
+ { \
+ PROD = mpyu(RECIPEST,REMHI); \
+ REM = asl(REM,# ## ( REMSHIFT )); \
+ }; \
+ { \
+ PRODLO = # ## 0; \
+ REM -= mpyu(PRODHI,DENOMLO); \
+ REMSUB2 = mpyu(PRODHI,DENOMHI); \
+ }; \
+ { \
+ Q += QSHIFTINSN(PROD, # ## ( QSHIFT )); \
+ REM -= asl(REMSUB2, # ## 32); \
+ EXTRA \
+ }
+
+
+ DIV_ITER1B(ASL,14,15,)
+ DIV_ITER1B(ASR,1,15,)
+ DIV_ITER1B(ASR,16,15,)
+ DIV_ITER1B(ASR,31,15,PROD=# ( 0 );)
+
+#undef REMSUB2
+#define TMPPAIR r15:14
+#define TMPPAIRHI r15
+#define TMPPAIRLO r14
+#undef RECIPEST
+#define EXPB r13
+ {
+ // compare or sub with carry
+ TMPPAIR = sub(REM,DENOM)
+ P_TMP = cmp.gtu(DENOM,REM)
+ // set up amt to add to q
+ if (!P_TMP.new) PRODLO = #2
+ }
+ {
+ Q = add(Q,PROD)
+ if (!P_TMP) REM = TMPPAIR
+ TMPPAIR = #0
+ }
+ {
+ P_TMP = cmp.eq(REM,TMPPAIR)
+ if (!P_TMP.new) QL = or(QL,ONE)
+ }
+ {
+ PROD = neg(Q)
+ }
+ {
+ if (!Q_POSITIVE) Q = PROD
+ }
+#undef REM
+#undef REMHI
+#undef REMLO
+#undef DENOM
+#undef DENOMLO
+#undef DENOMHI
+#define A r1:0
+#define AH r1
+#define AL r0
+#define B r3:2
+#define BH r3
+#define BL r2
+ {
+ A = convert_d2df(Q)
+ if (!NO_OVF_UNF) jump .Ldiv_ovf_unf
+ }
+ {
+ AH += asl(EXPA,#DF_MANTBITS-32)
+ jumpr r31
+ }
+
+.Ldiv_ovf_unf:
+ {
+ AH += asl(EXPA,#DF_MANTBITS-32)
+ EXPB = extractu(AH,#DF_EXPBITS,#DF_MANTBITS-32)
+ }
+ {
+ PROD = abs(Q)
+ EXPA = add(EXPA,EXPB)
+ }
+ {
+ P_TMP = cmp.gt(EXPA,##DF_BIAS+DF_BIAS) // overflow
+ if (P_TMP.new) jump:nt .Ldiv_ovf
+ }
+ {
+ P_TMP = cmp.gt(EXPA,#0)
+ if (P_TMP.new) jump:nt .Lpossible_unf // round up to normal possible...
+ }
+ /* Underflow */
+ /* We know what the infinite range exponent should be (EXPA) */
+ /* Q is 2's complement, PROD is abs(Q) */
+ /* Normalize Q, shift right, add a high bit, convert, change exponent */
+
+#define FUDGE1 7 // how much to shift right
+#define FUDGE2 4 // how many guard/round to keep at lsbs
+
+ {
+ EXPB = add(clb(PROD),#-1) // doesn't need to be added in since
+ EXPA = sub(#FUDGE1,EXPA) // we extract post-converted exponent
+ TMP = USR
+ TMP1 = #63
+ }
+ {
+ EXPB = min(EXPA,TMP1)
+ TMP1 = or(TMP,#0x030)
+ PROD = asl(PROD,EXPB)
+ EXPA = #0
+ }
+ {
+ TMPPAIR = extractu(PROD,EXPBA) // bits that will get shifted out
+ PROD = lsr(PROD,EXPB) // shift out bits
+ B = #1
+ }
+ {
+ P_TMP = cmp.gtu(B,TMPPAIR)
+ if (!P_TMP.new) PRODLO = or(BL,PRODLO)
+ PRODHI = setbit(PRODHI,#DF_MANTBITS-32+FUDGE2)
+ }
+ {
+ Q = neg(PROD)
+ P_TMP = bitsclr(PRODLO,#(1<<FUDGE2)-1)
+ if (!P_TMP.new) TMP = TMP1
+ }
+ {
+ USR = TMP
+ if (Q_POSITIVE) Q = PROD
+ TMP = #-DF_BIAS-(DF_MANTBITS+FUDGE2)
+ }
+ {
+ A = convert_d2df(Q)
+ }
+ {
+ AH += asl(TMP,#DF_MANTBITS-32)
+ jumpr r31
+ }
+
+
+.Lpossible_unf:
+ /* If upper parts of Q were all F's, but abs(A) == 0x00100000_00000000, we rounded up to min_normal */
+ /* The answer is correct, but we need to raise Underflow */
+ {
+ B = extractu(A,#63,#0)
+ TMPPAIR = combine(##0x00100000,#0) // min normal
+ TMP = #0x7FFF
+ }
+ {
+ P_TMP = dfcmp.eq(TMPPAIR,B) // Is everything zero in the rounded value...
+ P_TMP = bitsset(PRODHI,TMP) // but a bunch of bits set in the unrounded abs(quotient)?
+ }
+
+#if (__HEXAGON_ARCH__ == 60)
+ TMP = USR // If not, just return
+ if (!P_TMP) jumpr r31 // Else, we want to set Unf+Inexact
+ // Note that inexact is already set...
+#else
+ {
+ if (!P_TMP) jumpr r31 // If not, just return
+ TMP = USR // Else, we want to set Unf+Inexact
+ } // Note that inexact is already set...
+#endif
+ {
+ TMP = or(TMP,#0x30)
+ }
+ {
+ USR = TMP
+ }
+ {
+ p0 = dfcmp.eq(A,A)
+ jumpr r31
+ }
+
+.Ldiv_ovf:
+ /*
+ * Raise Overflow, and choose the correct overflow value (saturated normal or infinity)
+ */
+ {
+ TMP = USR
+ B = combine(##0x7fefffff,#-1)
+ AH = mux(Q_POSITIVE,#0,#-1)
+ }
+ {
+ PROD = combine(##0x7ff00000,#0)
+ QH = extractu(TMP,#2,#SR_ROUND_OFF)
+ TMP = or(TMP,#0x28)
+ }
+ {
+ USR = TMP
+ QH ^= lsr(AH,#31)
+ QL = QH
+ }
+ {
+ p0 = !cmp.eq(QL,#1) // if not round-to-zero
+ p0 = !cmp.eq(QH,#2) // and not rounding the other way
+ if (p0.new) B = PROD // go to inf
+ p0 = dfcmp.eq(B,B) // get exceptions
+ }
+ {
+ A = insert(B,#63,#0)
+ jumpr r31
+ }
+
+#undef ONE
+#define SIGN r28
+#undef NORMAL
+#undef NO_OVF_UNF
+#define P_INF p1
+#define P_ZERO p2
+.Ldiv_abnormal:
+ {
+ P_TMP = dfclass(A,#DFCLASS_NUMBER)
+ P_TMP = dfclass(B,#DFCLASS_NUMBER)
+ Q_POSITIVE = cmp.gt(SIGN,#-1)
+ }
+ {
+ P_INF = dfclass(A,#DFCLASS_INFINITE)
+ P_INF = dfclass(B,#DFCLASS_INFINITE)
+ }
+ {
+ P_ZERO = dfclass(A,#DFCLASS_ZERO)
+ P_ZERO = dfclass(B,#DFCLASS_ZERO)
+ }
+ {
+ if (!P_TMP) jump .Ldiv_nan
+ if (P_INF) jump .Ldiv_invalid
+ }
+ {
+ if (P_ZERO) jump .Ldiv_invalid
+ }
+ {
+ P_ZERO = dfclass(A,#DFCLASS_NONZERO) // nonzero
+ P_ZERO = dfclass(B,#DFCLASS_NONINFINITE) // non-infinite
+ }
+ {
+ P_INF = dfclass(A,#DFCLASS_NONINFINITE) // non-infinite
+ P_INF = dfclass(B,#DFCLASS_NONZERO) // nonzero
+ }
+ {
+ if (!P_ZERO) jump .Ldiv_zero_result
+ if (!P_INF) jump .Ldiv_inf_result
+ }
+ /* Now we've narrowed it down to (de)normal / (de)normal */
+ /* Set up A/EXPA B/EXPB and go back */
+#undef P_ZERO
+#undef P_INF
+#define P_TMP2 p1
+ {
+ P_TMP = dfclass(A,#DFCLASS_NORMAL)
+ P_TMP2 = dfclass(B,#DFCLASS_NORMAL)
+ TMP = ##0x00100000
+ }
+ {
+ EXPBA = combine(BH,AH)
+ AH = insert(TMP,#DF_EXPBITS+1,#DF_MANTBITS-32) // clear out hidden bit, sign bit
+ BH = insert(TMP,#DF_EXPBITS+1,#DF_MANTBITS-32) // clear out hidden bit, sign bit
+ }
+ {
+ if (P_TMP) AH = or(AH,TMP) // if normal, add back in hidden bit
+ if (P_TMP2) BH = or(BH,TMP) // if normal, add back in hidden bit
+ }
+ {
+ QH = add(clb(A),#-DF_EXPBITS)
+ QL = add(clb(B),#-DF_EXPBITS)
+ TMP = #1
+ }
+ {
+ EXPA = extractu(EXPA,#DF_EXPBITS,#DF_MANTBITS-32)
+ EXPB = extractu(EXPB,#DF_EXPBITS,#DF_MANTBITS-32)
+ }
+ {
+ A = asl(A,QH)
+ B = asl(B,QL)
+ if (!P_TMP) EXPA = sub(TMP,QH)
+ if (!P_TMP2) EXPB = sub(TMP,QL)
+ } // recreate values needed by resume coke
+ {
+ PROD = extractu(B,#SF_MANTBITS,#DF_MANTBITS-SF_MANTBITS)
+ }
+ {
+ SFDEN = or(SFONE,PRODLO)
+ jump .Ldenorm_continue
+ }
+
+.Ldiv_zero_result:
+ {
+ AH = xor(AH,BH)
+ B = #0
+ }
+ {
+ A = insert(B,#63,#0)
+ jumpr r31
+ }
+.Ldiv_inf_result:
+ {
+ p2 = dfclass(B,#DFCLASS_ZERO)
+ p2 = dfclass(A,#DFCLASS_NONINFINITE)
+ }
+ {
+ TMP = USR
+ if (!p2) jump 1f
+ AH = xor(AH,BH)
+ }
+ {
+ TMP = or(TMP,#0x04) // DBZ
+ }
+ {
+ USR = TMP
+ }
+1:
+ {
+ B = combine(##0x7ff00000,#0)
+ p0 = dfcmp.uo(B,B) // take possible exception
+ }
+ {
+ A = insert(B,#63,#0)
+ jumpr r31
+ }
+.Ldiv_nan:
+ {
+ p0 = dfclass(A,#0x10)
+ p1 = dfclass(B,#0x10)
+ if (!p0.new) A = B
+ if (!p1.new) B = A
+ }
+ {
+ QH = convert_df2sf(A) // get possible invalid exceptions
+ QL = convert_df2sf(B)
+ }
+ {
+ A = #-1
+ jumpr r31
+ }
+
+.Ldiv_invalid:
+ {
+ TMP = ##0x7f800001
+ }
+ {
+ A = convert_sf2df(TMP) // get invalid, get DF qNaN
+ jumpr r31
+ }
+END(__hexagon_divdf3)
diff --git a/lib/builtins/hexagon/dffma.S b/lib/builtins/hexagon/dffma.S
new file mode 100644
index 000000000000..97b885a3bf27
--- /dev/null
+++ b/lib/builtins/hexagon/dffma.S
@@ -0,0 +1,705 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG
+#define END(TAG) .size TAG,.-TAG
+
+/* Double Precision Multiply */
+
+
+#define A r1:0
+#define AH r1
+#define AL r0
+#define B r3:2
+#define BH r3
+#define BL r2
+#define C r5:4
+#define CH r5
+#define CL r4
+
+
+
+#define BTMP r15:14
+#define BTMPH r15
+#define BTMPL r14
+
+#define ATMP r13:12
+#define ATMPH r13
+#define ATMPL r12
+
+#define CTMP r11:10
+#define CTMPH r11
+#define CTMPL r10
+
+#define PP_LL r9:8
+#define PP_LL_H r9
+#define PP_LL_L r8
+
+#define PP_ODD r7:6
+#define PP_ODD_H r7
+#define PP_ODD_L r6
+
+
+#define PP_HH r17:16
+#define PP_HH_H r17
+#define PP_HH_L r16
+
+#define EXPA r18
+#define EXPB r19
+#define EXPBA r19:18
+
+#define TMP r28
+
+#define P_TMP p0
+#define PROD_NEG p3
+#define EXACT p2
+#define SWAP p1
+
+#define MANTBITS 52
+#define HI_MANTBITS 20
+#define EXPBITS 11
+#define BIAS 1023
+#define STACKSPACE 32
+
+#define ADJUST 4
+
+#define FUDGE 7
+#define FUDGE2 3
+
+#ifndef SR_ROUND_OFF
+#define SR_ROUND_OFF 22
+#endif
+
+ /*
+ * First, classify for normal values, and abort if abnormal
+ *
+ * Next, unpack mantissa into 0x1000_0000_0000_0000 + mant<<8
+ *
+ * Since we know that the 2 MSBs of the H registers is zero, we should never carry
+ * the partial products that involve the H registers
+ *
+ * Try to buy X slots, at the expense of latency if needed
+ *
+ * We will have PP_HH with the upper bits of the product, PP_LL with the lower
+ * PP_HH can have a maximum of 0x03FF_FFFF_FFFF_FFFF or thereabouts
+ * PP_HH can have a minimum of 0x0100_0000_0000_0000
+ *
+ * 0x0100_0000_0000_0000 has EXP of EXPA+EXPB-BIAS
+ *
+ * We need to align CTMP.
+ * If CTMP >> PP, convert PP to 64 bit with sticky, align CTMP, and follow normal add
+ * If CTMP << PP align CTMP and add 128 bits. Then compute sticky
+ * If CTMP ~= PP, align CTMP and add 128 bits. May have massive cancellation.
+ *
+ * Convert partial product and CTMP to 2's complement prior to addition
+ *
+ * After we add, we need to normalize into upper 64 bits, then compute sticky.
+ *
+ *
+ */
+
+ .text
+ .global __hexagon_fmadf4
+ .type __hexagon_fmadf4,@function
+ .global __hexagon_fmadf5
+ .type __hexagon_fmadf5,@function
+ .global fma
+ .type fma,@function
+ Q6_ALIAS(fmadf5)
+ .p2align 5
+__hexagon_fmadf4:
+__hexagon_fmadf5:
+fma:
+ {
+ P_TMP = dfclass(A,#2)
+ P_TMP = dfclass(B,#2)
+ ATMP = #0
+ BTMP = #0
+ }
+ {
+ ATMP = insert(A,#MANTBITS,#EXPBITS-3)
+ BTMP = insert(B,#MANTBITS,#EXPBITS-3)
+ PP_ODD_H = ##0x10000000
+ allocframe(#STACKSPACE)
+ }
+ {
+ PP_LL = mpyu(ATMPL,BTMPL)
+ if (!P_TMP) jump .Lfma_abnormal_ab
+ ATMPH = or(ATMPH,PP_ODD_H)
+ BTMPH = or(BTMPH,PP_ODD_H)
+ }
+ {
+ P_TMP = dfclass(C,#2)
+ if (!P_TMP.new) jump:nt .Lfma_abnormal_c
+ CTMP = combine(PP_ODD_H,#0)
+ PP_ODD = combine(#0,PP_LL_H)
+ }
+.Lfma_abnormal_c_restart:
+ {
+ PP_ODD += mpyu(BTMPL,ATMPH)
+ CTMP = insert(C,#MANTBITS,#EXPBITS-3)
+ memd(r29+#0) = PP_HH
+ memd(r29+#8) = EXPBA
+ }
+ {
+ PP_ODD += mpyu(ATMPL,BTMPH)
+ EXPBA = neg(CTMP)
+ P_TMP = cmp.gt(CH,#-1)
+ TMP = xor(AH,BH)
+ }
+ {
+ EXPA = extractu(AH,#EXPBITS,#HI_MANTBITS)
+ EXPB = extractu(BH,#EXPBITS,#HI_MANTBITS)
+ PP_HH = combine(#0,PP_ODD_H)
+ if (!P_TMP) CTMP = EXPBA
+ }
+ {
+ PP_HH += mpyu(ATMPH,BTMPH)
+ PP_LL = combine(PP_ODD_L,PP_LL_L)
+#undef PP_ODD
+#undef PP_ODD_H
+#undef PP_ODD_L
+#undef ATMP
+#undef ATMPL
+#undef ATMPH
+#undef BTMP
+#undef BTMPL
+#undef BTMPH
+#define RIGHTLEFTSHIFT r13:12
+#define RIGHTSHIFT r13
+#define LEFTSHIFT r12
+
+ EXPA = add(EXPA,EXPB)
+#undef EXPB
+#undef EXPBA
+#define EXPC r19
+#define EXPCA r19:18
+ EXPC = extractu(CH,#EXPBITS,#HI_MANTBITS)
+ }
+ /* PP_HH:PP_LL now has product */
+ /* CTMP is negated */
+ /* EXPA,B,C are extracted */
+ /*
+ * We need to negate PP
+ * Since we will be adding with carry later, if we need to negate,
+ * just invert all bits now, which we can do conditionally and in parallel
+ */
+#define PP_HH_TMP r15:14
+#define PP_LL_TMP r7:6
+ {
+ EXPA = add(EXPA,#-BIAS+(ADJUST))
+ PROD_NEG = !cmp.gt(TMP,#-1)
+ PP_LL_TMP = #0
+ PP_HH_TMP = #0
+ }
+ {
+ PP_LL_TMP = sub(PP_LL_TMP,PP_LL,PROD_NEG):carry
+ P_TMP = !cmp.gt(TMP,#-1)
+ SWAP = cmp.gt(EXPC,EXPA) // If C >> PP
+ if (SWAP.new) EXPCA = combine(EXPA,EXPC)
+ }
+ {
+ PP_HH_TMP = sub(PP_HH_TMP,PP_HH,PROD_NEG):carry
+ if (P_TMP) PP_LL = PP_LL_TMP
+#undef PP_LL_TMP
+#define CTMP2 r7:6
+#define CTMP2H r7
+#define CTMP2L r6
+ CTMP2 = #0
+ EXPC = sub(EXPA,EXPC)
+ }
+ {
+ if (P_TMP) PP_HH = PP_HH_TMP
+ P_TMP = cmp.gt(EXPC,#63)
+ if (SWAP) PP_LL = CTMP2
+ if (SWAP) CTMP2 = PP_LL
+ }
+#undef PP_HH_TMP
+//#define ONE r15:14
+//#define S_ONE r14
+#define ZERO r15:14
+#define S_ZERO r15
+#undef PROD_NEG
+#define P_CARRY p3
+ {
+ if (SWAP) PP_HH = CTMP // Swap C and PP
+ if (SWAP) CTMP = PP_HH
+ if (P_TMP) EXPC = add(EXPC,#-64)
+ TMP = #63
+ }
+ {
+ // If diff > 63, pre-shift-right by 64...
+ if (P_TMP) CTMP2 = CTMP
+ TMP = asr(CTMPH,#31)
+ RIGHTSHIFT = min(EXPC,TMP)
+ LEFTSHIFT = #0
+ }
+#undef C
+#undef CH
+#undef CL
+#define STICKIES r5:4
+#define STICKIESH r5
+#define STICKIESL r4
+ {
+ if (P_TMP) CTMP = combine(TMP,TMP) // sign extension of pre-shift-right-64
+ STICKIES = extract(CTMP2,RIGHTLEFTSHIFT)
+ CTMP2 = lsr(CTMP2,RIGHTSHIFT)
+ LEFTSHIFT = sub(#64,RIGHTSHIFT)
+ }
+ {
+ ZERO = #0
+ TMP = #-2
+ CTMP2 |= lsl(CTMP,LEFTSHIFT)
+ CTMP = asr(CTMP,RIGHTSHIFT)
+ }
+ {
+ P_CARRY = cmp.gtu(STICKIES,ZERO) // If we have sticky bits from C shift
+ if (P_CARRY.new) CTMP2L = and(CTMP2L,TMP) // make sure adding 1 == OR
+#undef ZERO
+#define ONE r15:14
+#define S_ONE r14
+ ONE = #1
+ STICKIES = #0
+ }
+ {
+ PP_LL = add(CTMP2,PP_LL,P_CARRY):carry // use the carry to add the sticky
+ }
+ {
+ PP_HH = add(CTMP,PP_HH,P_CARRY):carry
+ TMP = #62
+ }
+ /*
+ * PP_HH:PP_LL now holds the sum
+ * We may need to normalize left, up to ??? bits.
+ *
+ * I think that if we have massive cancellation, the range we normalize by
+ * is still limited
+ */
+ {
+ LEFTSHIFT = add(clb(PP_HH),#-2)
+ if (!cmp.eq(LEFTSHIFT.new,TMP)) jump:t 1f // all sign bits?
+ }
+ /* We had all sign bits, shift left by 62. */
+ {
+ CTMP = extractu(PP_LL,#62,#2)
+ PP_LL = asl(PP_LL,#62)
+ EXPA = add(EXPA,#-62) // And adjust exponent of result
+ }
+ {
+ PP_HH = insert(CTMP,#62,#0) // Then shift 63
+ }
+ {
+ LEFTSHIFT = add(clb(PP_HH),#-2)
+ }
+ .falign
+1:
+ {
+ CTMP = asl(PP_HH,LEFTSHIFT)
+ STICKIES |= asl(PP_LL,LEFTSHIFT)
+ RIGHTSHIFT = sub(#64,LEFTSHIFT)
+ EXPA = sub(EXPA,LEFTSHIFT)
+ }
+ {
+ CTMP |= lsr(PP_LL,RIGHTSHIFT)
+ EXACT = cmp.gtu(ONE,STICKIES)
+ TMP = #BIAS+BIAS-2
+ }
+ {
+ if (!EXACT) CTMPL = or(CTMPL,S_ONE)
+ // If EXPA is overflow/underflow, jump to ovf_unf
+ P_TMP = !cmp.gt(EXPA,TMP)
+ P_TMP = cmp.gt(EXPA,#1)
+ if (!P_TMP.new) jump:nt .Lfma_ovf_unf
+ }
+ {
+ // XXX: FIXME: should PP_HH for check of zero be CTMP?
+ P_TMP = cmp.gtu(ONE,CTMP) // is result true zero?
+ A = convert_d2df(CTMP)
+ EXPA = add(EXPA,#-BIAS-60)
+ PP_HH = memd(r29+#0)
+ }
+ {
+ AH += asl(EXPA,#HI_MANTBITS)
+ EXPCA = memd(r29+#8)
+ if (!P_TMP) dealloc_return // not zero, return
+ }
+.Ladd_yields_zero:
+ /* We had full cancellation. Return +/- zero (-0 when round-down) */
+ {
+ TMP = USR
+ A = #0
+ }
+ {
+ TMP = extractu(TMP,#2,#SR_ROUND_OFF)
+ PP_HH = memd(r29+#0)
+ EXPCA = memd(r29+#8)
+ }
+ {
+ p0 = cmp.eq(TMP,#2)
+ if (p0.new) AH = ##0x80000000
+ dealloc_return
+ }
+
+#undef RIGHTLEFTSHIFT
+#undef RIGHTSHIFT
+#undef LEFTSHIFT
+#undef CTMP2
+#undef CTMP2H
+#undef CTMP2L
+
+.Lfma_ovf_unf:
+ {
+ p0 = cmp.gtu(ONE,CTMP)
+ if (p0.new) jump:nt .Ladd_yields_zero
+ }
+ {
+ A = convert_d2df(CTMP)
+ EXPA = add(EXPA,#-BIAS-60)
+ TMP = EXPA
+ }
+#define NEW_EXPB r7
+#define NEW_EXPA r6
+ {
+ AH += asl(EXPA,#HI_MANTBITS)
+ NEW_EXPB = extractu(AH,#EXPBITS,#HI_MANTBITS)
+ }
+ {
+ NEW_EXPA = add(EXPA,NEW_EXPB)
+ PP_HH = memd(r29+#0)
+ EXPCA = memd(r29+#8)
+#undef PP_HH
+#undef PP_HH_H
+#undef PP_HH_L
+#undef EXPCA
+#undef EXPC
+#undef EXPA
+#undef PP_LL
+#undef PP_LL_H
+#undef PP_LL_L
+#define EXPA r6
+#define EXPB r7
+#define EXPBA r7:6
+#define ATMP r9:8
+#define ATMPH r9
+#define ATMPL r8
+#undef NEW_EXPB
+#undef NEW_EXPA
+ ATMP = abs(CTMP)
+ }
+ {
+ p0 = cmp.gt(EXPA,##BIAS+BIAS)
+ if (p0.new) jump:nt .Lfma_ovf
+ }
+ {
+ p0 = cmp.gt(EXPA,#0)
+ if (p0.new) jump:nt .Lpossible_unf
+ }
+ {
+ // TMP has original EXPA.
+ // ATMP is corresponding value
+ // Normalize ATMP and shift right to correct location
+ EXPB = add(clb(ATMP),#-2) // Amount to left shift to normalize
+ EXPA = sub(#1+5,TMP) // Amount to right shift to denormalize
+ p3 = cmp.gt(CTMPH,#-1)
+ }
+ /* Underflow */
+ /* We know that the infinte range exponent should be EXPA */
+ /* CTMP is 2's complement, ATMP is abs(CTMP) */
+ {
+ EXPA = add(EXPA,EXPB) // how much to shift back right
+ ATMP = asl(ATMP,EXPB) // shift left
+ AH = USR
+ TMP = #63
+ }
+ {
+ EXPB = min(EXPA,TMP)
+ EXPA = #0
+ AL = #0x0030
+ }
+ {
+ B = extractu(ATMP,EXPBA)
+ ATMP = asr(ATMP,EXPB)
+ }
+ {
+ p0 = cmp.gtu(ONE,B)
+ if (!p0.new) ATMPL = or(ATMPL,S_ONE)
+ ATMPH = setbit(ATMPH,#HI_MANTBITS+FUDGE2)
+ }
+ {
+ CTMP = neg(ATMP)
+ p1 = bitsclr(ATMPL,#(1<<FUDGE2)-1)
+ if (!p1.new) AH = or(AH,AL)
+ B = #0
+ }
+ {
+ if (p3) CTMP = ATMP
+ USR = AH
+ TMP = #-BIAS-(MANTBITS+FUDGE2)
+ }
+ {
+ A = convert_d2df(CTMP)
+ }
+ {
+ AH += asl(TMP,#HI_MANTBITS)
+ dealloc_return
+ }
+.Lpossible_unf:
+ {
+ TMP = ##0x7fefffff
+ ATMP = abs(CTMP)
+ }
+ {
+ p0 = cmp.eq(AL,#0)
+ p0 = bitsclr(AH,TMP)
+ if (!p0.new) dealloc_return:t
+ TMP = #0x7fff
+ }
+ {
+ p0 = bitsset(ATMPH,TMP)
+ BH = USR
+ BL = #0x0030
+ }
+ {
+ if (p0) BH = or(BH,BL)
+ }
+ {
+ USR = BH
+ }
+ {
+ p0 = dfcmp.eq(A,A)
+ dealloc_return
+ }
+.Lfma_ovf:
+ {
+ TMP = USR
+ CTMP = combine(##0x7fefffff,#-1)
+ A = CTMP
+ }
+ {
+ ATMP = combine(##0x7ff00000,#0)
+ BH = extractu(TMP,#2,#SR_ROUND_OFF)
+ TMP = or(TMP,#0x28)
+ }
+ {
+ USR = TMP
+ BH ^= lsr(AH,#31)
+ BL = BH
+ }
+ {
+ p0 = !cmp.eq(BL,#1)
+ p0 = !cmp.eq(BH,#2)
+ }
+ {
+ p0 = dfcmp.eq(ATMP,ATMP)
+ if (p0.new) CTMP = ATMP
+ }
+ {
+ A = insert(CTMP,#63,#0)
+ dealloc_return
+ }
+#undef CTMP
+#undef CTMPH
+#undef CTMPL
+#define BTMP r11:10
+#define BTMPH r11
+#define BTMPL r10
+
+#undef STICKIES
+#undef STICKIESH
+#undef STICKIESL
+#define C r5:4
+#define CH r5
+#define CL r4
+
+.Lfma_abnormal_ab:
+ {
+ ATMP = extractu(A,#63,#0)
+ BTMP = extractu(B,#63,#0)
+ deallocframe
+ }
+ {
+ p3 = cmp.gtu(ATMP,BTMP)
+ if (!p3.new) A = B // sort values
+ if (!p3.new) B = A
+ }
+ {
+ p0 = dfclass(A,#0x0f) // A NaN?
+ if (!p0.new) jump:nt .Lnan
+ if (!p3) ATMP = BTMP
+ if (!p3) BTMP = ATMP
+ }
+ {
+ p1 = dfclass(A,#0x08) // A is infinity
+ p1 = dfclass(B,#0x0e) // B is nonzero
+ }
+ {
+ p0 = dfclass(A,#0x08) // a is inf
+ p0 = dfclass(B,#0x01) // b is zero
+ }
+ {
+ if (p1) jump .Lab_inf
+ p2 = dfclass(B,#0x01)
+ }
+ {
+ if (p0) jump .Linvalid
+ if (p2) jump .Lab_true_zero
+ TMP = ##0x7c000000
+ }
+ // We are left with a normal or subnormal times a subnormal, A > B
+ // If A and B are both very small, we will go to a single sticky bit; replace
+ // A and B lower 63 bits with 0x0010_0000_0000_0000, which yields equivalent results
+ // if A and B might multiply to something bigger, decrease A exp and increase B exp
+ // and start over
+ {
+ p0 = bitsclr(AH,TMP)
+ if (p0.new) jump:nt .Lfma_ab_tiny
+ }
+ {
+ TMP = add(clb(BTMP),#-EXPBITS)
+ }
+ {
+ BTMP = asl(BTMP,TMP)
+ }
+ {
+ B = insert(BTMP,#63,#0)
+ AH -= asl(TMP,#HI_MANTBITS)
+ }
+ jump fma
+
+.Lfma_ab_tiny:
+ ATMP = combine(##0x00100000,#0)
+ {
+ A = insert(ATMP,#63,#0)
+ B = insert(ATMP,#63,#0)
+ }
+ jump fma
+
+.Lab_inf:
+ {
+ B = lsr(B,#63)
+ p0 = dfclass(C,#0x10)
+ }
+ {
+ A ^= asl(B,#63)
+ if (p0) jump .Lnan
+ }
+ {
+ p1 = dfclass(C,#0x08)
+ if (p1.new) jump:nt .Lfma_inf_plus_inf
+ }
+ /* A*B is +/- inf, C is finite. Return A */
+ {
+ jumpr r31
+ }
+ .falign
+.Lfma_inf_plus_inf:
+ { // adding infinities of different signs is invalid
+ p0 = dfcmp.eq(A,C)
+ if (!p0.new) jump:nt .Linvalid
+ }
+ {
+ jumpr r31
+ }
+
+.Lnan:
+ {
+ p0 = dfclass(B,#0x10)
+ p1 = dfclass(C,#0x10)
+ if (!p0.new) B = A
+ if (!p1.new) C = A
+ }
+ { // find sNaNs
+ BH = convert_df2sf(B)
+ BL = convert_df2sf(C)
+ }
+ {
+ BH = convert_df2sf(A)
+ A = #-1
+ jumpr r31
+ }
+
+.Linvalid:
+ {
+ TMP = ##0x7f800001 // sp snan
+ }
+ {
+ A = convert_sf2df(TMP)
+ jumpr r31
+ }
+
+.Lab_true_zero:
+ // B is zero, A is finite number
+ {
+ p0 = dfclass(C,#0x10)
+ if (p0.new) jump:nt .Lnan
+ if (p0.new) A = C
+ }
+ {
+ p0 = dfcmp.eq(B,C) // is C also zero?
+ AH = lsr(AH,#31) // get sign
+ }
+ {
+ BH ^= asl(AH,#31) // form correctly signed zero in B
+ if (!p0) A = C // If C is not zero, return C
+ if (!p0) jumpr r31
+ }
+ /* B has correctly signed zero, C is also zero */
+.Lzero_plus_zero:
+ {
+ p0 = cmp.eq(B,C) // yes, scalar equals. +0++0 or -0+-0
+ if (p0.new) jumpr:t r31
+ A = B
+ }
+ {
+ TMP = USR
+ }
+ {
+ TMP = extractu(TMP,#2,#SR_ROUND_OFF)
+ A = #0
+ }
+ {
+ p0 = cmp.eq(TMP,#2)
+ if (p0.new) AH = ##0x80000000
+ jumpr r31
+ }
+#undef BTMP
+#undef BTMPH
+#undef BTMPL
+#define CTMP r11:10
+ .falign
+.Lfma_abnormal_c:
+ /* We know that AB is normal * normal */
+ /* C is not normal: zero, subnormal, inf, or NaN. */
+ {
+ p0 = dfclass(C,#0x10) // is C NaN?
+ if (p0.new) jump:nt .Lnan
+ if (p0.new) A = C // move NaN to A
+ deallocframe
+ }
+ {
+ p0 = dfclass(C,#0x08) // is C inf?
+ if (p0.new) A = C // return C
+ if (p0.new) jumpr:nt r31
+ }
+ // zero or subnormal
+ // If we have a zero, and we know AB is normal*normal, we can just call normal multiply
+ {
+ p0 = dfclass(C,#0x01) // is C zero?
+ if (p0.new) jump:nt __hexagon_muldf3
+ TMP = #1
+ }
+ // Left with: subnormal
+ // Adjust C and jump back to restart
+ {
+ allocframe(#STACKSPACE) // oops, deallocated above, re-allocate frame
+ CTMP = #0
+ CH = insert(TMP,#EXPBITS,#HI_MANTBITS)
+ jump .Lfma_abnormal_c_restart
+ }
+END(fma)
diff --git a/lib/builtins/hexagon/dfminmax.S b/lib/builtins/hexagon/dfminmax.S
new file mode 100644
index 000000000000..41122911f183
--- /dev/null
+++ b/lib/builtins/hexagon/dfminmax.S
@@ -0,0 +1,79 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define A r1:0
+#define B r3:2
+#define ATMP r5:4
+
+
+#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG
+#define END(TAG) .size TAG,.-TAG
+
+/*
+ * Min and Max return A if B is NaN, or B if A is NaN
+ * Otherwise, they return the smaller or bigger value
+ *
+ * If values are equal, we want to favor -0.0 for min and +0.0 for max.
+ */
+
+/*
+ * Compares always return false for NaN
+ * if (isnan(A)) A = B; if (A > B) A = B will only trigger at most one of those options.
+ */
+ .text
+ .global __hexagon_mindf3
+ .global __hexagon_maxdf3
+ .global fmin
+ .type fmin,@function
+ .global fmax
+ .type fmax,@function
+ .type __hexagon_mindf3,@function
+ .type __hexagon_maxdf3,@function
+ Q6_ALIAS(mindf3)
+ Q6_ALIAS(maxdf3)
+ .p2align 5
+__hexagon_mindf3:
+fmin:
+ {
+ p0 = dfclass(A,#0x10) // If A is a number
+ p1 = dfcmp.gt(A,B) // AND B > A, don't swap
+ ATMP = A
+ }
+ {
+ if (p0) A = B // if A is NaN use B
+ if (p1) A = B // gt is always false if either is NaN
+ p2 = dfcmp.eq(A,B) // if A == B
+ if (!p2.new) jumpr:t r31
+ }
+ /* A == B, return A|B to select -0.0 over 0.0 */
+ {
+ A = or(ATMP,B)
+ jumpr r31
+ }
+END(__hexagon_mindf3)
+ .falign
+__hexagon_maxdf3:
+fmax:
+ {
+ p0 = dfclass(A,#0x10)
+ p1 = dfcmp.gt(B,A)
+ ATMP = A
+ }
+ {
+ if (p0) A = B
+ if (p1) A = B
+ p2 = dfcmp.eq(A,B)
+ if (!p2.new) jumpr:t r31
+ }
+ /* A == B, return A&B to select 0.0 over -0.0 */
+ {
+ A = and(ATMP,B)
+ jumpr r31
+ }
+END(__hexagon_maxdf3)
diff --git a/lib/builtins/hexagon/dfmul.S b/lib/builtins/hexagon/dfmul.S
new file mode 100644
index 000000000000..fde6d77bdcff
--- /dev/null
+++ b/lib/builtins/hexagon/dfmul.S
@@ -0,0 +1,418 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+/* Double Precision Multiply */
+#define A r1:0
+#define AH r1
+#define AL r0
+#define B r3:2
+#define BH r3
+#define BL r2
+
+#define BTMP r5:4
+#define BTMPH r5
+#define BTMPL r4
+
+#define PP_ODD r7:6
+#define PP_ODD_H r7
+#define PP_ODD_L r6
+
+#define ONE r9:8
+#define S_ONE r8
+#define S_ZERO r9
+
+#define PP_HH r11:10
+#define PP_HH_H r11
+#define PP_HH_L r10
+
+#define ATMP r13:12
+#define ATMPH r13
+#define ATMPL r12
+
+#define PP_LL r15:14
+#define PP_LL_H r15
+#define PP_LL_L r14
+
+#define TMP r28
+
+#define MANTBITS 52
+#define HI_MANTBITS 20
+#define EXPBITS 11
+#define BIAS 1024
+#define MANTISSA_TO_INT_BIAS 52
+
+/* Some constant to adjust normalization amount in error code */
+/* Amount to right shift the partial product to get to a denorm */
+#define FUDGE 5
+
+#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG
+#define FAST_ALIAS(TAG) .global __hexagon_fast_##TAG ; .set __hexagon_fast_##TAG, __hexagon_##TAG
+#define FAST2_ALIAS(TAG) .global __hexagon_fast2_##TAG ; .set __hexagon_fast2_##TAG, __hexagon_##TAG
+#define END(TAG) .size TAG,.-TAG
+
+#define SR_ROUND_OFF 22
+ .text
+ .global __hexagon_muldf3
+ .type __hexagon_muldf3,@function
+ Q6_ALIAS(muldf3)
+ FAST_ALIAS(muldf3)
+ FAST2_ALIAS(muldf3)
+ .p2align 5
+__hexagon_muldf3:
+ {
+ p0 = dfclass(A,#2)
+ p0 = dfclass(B,#2)
+ ATMP = combine(##0x40000000,#0)
+ }
+ {
+ ATMP = insert(A,#MANTBITS,#EXPBITS-1)
+ BTMP = asl(B,#EXPBITS-1)
+ TMP = #-BIAS
+ ONE = #1
+ }
+ {
+ PP_ODD = mpyu(BTMPL,ATMPH)
+ BTMP = insert(ONE,#2,#62)
+ }
+ /* since we know that the MSB of the H registers is zero, we should never carry */
+ /* H <= 2^31-1. L <= 2^32-1. Therefore, HL <= 2^63-2^32-2^31+1 */
+ /* Adding 2 HLs, we get 2^64-3*2^32+2 maximum. */
+ /* Therefore, we can add 3 2^32-1 values safely without carry. We only need one. */
+ {
+ PP_LL = mpyu(ATMPL,BTMPL)
+ PP_ODD += mpyu(ATMPL,BTMPH)
+ }
+ {
+ PP_ODD += lsr(PP_LL,#32)
+ PP_HH = mpyu(ATMPH,BTMPH)
+ BTMP = combine(##BIAS+BIAS-4,#0)
+ }
+ {
+ PP_HH += lsr(PP_ODD,#32)
+ if (!p0) jump .Lmul_abnormal
+ p1 = cmp.eq(PP_LL_L,#0) // 64 lsb's 0?
+ p1 = cmp.eq(PP_ODD_L,#0) // 64 lsb's 0?
+ }
+ /*
+ * PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts
+ * PP_HH can have a minimum of 0x1000_0000_0000_0000 or so
+ */
+#undef PP_ODD
+#undef PP_ODD_H
+#undef PP_ODD_L
+#define EXP10 r7:6
+#define EXP1 r7
+#define EXP0 r6
+ {
+ if (!p1) PP_HH_L = or(PP_HH_L,S_ONE)
+ EXP0 = extractu(AH,#EXPBITS,#HI_MANTBITS)
+ EXP1 = extractu(BH,#EXPBITS,#HI_MANTBITS)
+ }
+ {
+ PP_LL = neg(PP_HH)
+ EXP0 += add(TMP,EXP1)
+ TMP = xor(AH,BH)
+ }
+ {
+ if (!p2.new) PP_HH = PP_LL
+ p2 = cmp.gt(TMP,#-1)
+ p0 = !cmp.gt(EXP0,BTMPH)
+ p0 = cmp.gt(EXP0,BTMPL)
+ if (!p0.new) jump:nt .Lmul_ovf_unf
+ }
+ {
+ A = convert_d2df(PP_HH)
+ EXP0 = add(EXP0,#-BIAS-58)
+ }
+ {
+ AH += asl(EXP0,#HI_MANTBITS)
+ jumpr r31
+ }
+
+ .falign
+.Lpossible_unf:
+ /* We end up with a positive exponent */
+ /* But we may have rounded up to an exponent of 1. */
+ /* If the exponent is 1, if we rounded up to it
+ * we need to also raise underflow
+ * Fortunately, this is pretty easy to detect, we must have +/- 0x0010_0000_0000_0000
+ * And the PP should also have more than one bit set
+ */
+ /* Note: ATMP should have abs(PP_HH) */
+ /* Note: BTMPL should have 0x7FEFFFFF */
+ {
+ p0 = cmp.eq(AL,#0)
+ p0 = bitsclr(AH,BTMPL)
+ if (!p0.new) jumpr:t r31
+ BTMPH = #0x7fff
+ }
+ {
+ p0 = bitsset(ATMPH,BTMPH)
+ BTMPL = USR
+ BTMPH = #0x030
+ }
+ {
+ if (p0) BTMPL = or(BTMPL,BTMPH)
+ }
+ {
+ USR = BTMPL
+ }
+ {
+ p0 = dfcmp.eq(A,A)
+ jumpr r31
+ }
+ .falign
+.Lmul_ovf_unf:
+ {
+ A = convert_d2df(PP_HH)
+ ATMP = abs(PP_HH) // take absolute value
+ EXP1 = add(EXP0,#-BIAS-58)
+ }
+ {
+ AH += asl(EXP1,#HI_MANTBITS)
+ EXP1 = extractu(AH,#EXPBITS,#HI_MANTBITS)
+ BTMPL = ##0x7FEFFFFF
+ }
+ {
+ EXP1 += add(EXP0,##-BIAS-58)
+ //BTMPH = add(clb(ATMP),#-2)
+ BTMPH = #0
+ }
+ {
+ p0 = cmp.gt(EXP1,##BIAS+BIAS-2) // overflow
+ if (p0.new) jump:nt .Lmul_ovf
+ }
+ {
+ p0 = cmp.gt(EXP1,#0)
+ if (p0.new) jump:nt .Lpossible_unf
+ BTMPH = sub(EXP0,BTMPH)
+ TMP = #63 // max amount to shift
+ }
+ /* Underflow */
+ /*
+ * PP_HH has the partial product with sticky LSB.
+ * PP_HH can have a maximum of 0x3FFF_FFFF_FFFF_FFFF or thereabouts
+ * PP_HH can have a minimum of 0x1000_0000_0000_0000 or so
+ * The exponent of PP_HH is in EXP1, which is non-positive (0 or negative)
+ * That's the exponent that happens after the normalization
+ *
+ * EXP0 has the exponent that, when added to the normalized value, is out of range.
+ *
+ * Strategy:
+ *
+ * * Shift down bits, with sticky bit, such that the bits are aligned according
+ * to the LZ count and appropriate exponent, but not all the way to mantissa
+ * field, keep around the last few bits.
+ * * Put a 1 near the MSB
+ * * Check the LSBs for inexact; if inexact also set underflow
+ * * Convert [u]d2df -- will correctly round according to rounding mode
+ * * Replace exponent field with zero
+ *
+ *
+ */
+
+
+ {
+ BTMPL = #0 // offset for extract
+ BTMPH = sub(#FUDGE,BTMPH) // amount to right shift
+ }
+ {
+ p3 = cmp.gt(PP_HH_H,#-1) // is it positive?
+ BTMPH = min(BTMPH,TMP) // Don't shift more than 63
+ PP_HH = ATMP
+ }
+ {
+ TMP = USR
+ PP_LL = extractu(PP_HH,BTMP)
+ }
+ {
+ PP_HH = asr(PP_HH,BTMPH)
+ BTMPL = #0x0030 // underflow flag
+ AH = insert(S_ZERO,#EXPBITS,#HI_MANTBITS)
+ }
+ {
+ p0 = cmp.gtu(ONE,PP_LL) // Did we extract all zeros?
+ if (!p0.new) PP_HH_L = or(PP_HH_L,S_ONE) // add sticky bit
+ PP_HH_H = setbit(PP_HH_H,#HI_MANTBITS+3) // Add back in a bit so we can use convert instruction
+ }
+ {
+ PP_LL = neg(PP_HH)
+ p1 = bitsclr(PP_HH_L,#0x7) // Are the LSB's clear?
+ if (!p1.new) TMP = or(BTMPL,TMP) // If not, Inexact+Underflow
+ }
+ {
+ if (!p3) PP_HH = PP_LL
+ USR = TMP
+ }
+ {
+ A = convert_d2df(PP_HH) // Do rounding
+ p0 = dfcmp.eq(A,A) // realize exception
+ }
+ {
+ AH = insert(S_ZERO,#EXPBITS-1,#HI_MANTBITS+1) // Insert correct exponent
+ jumpr r31
+ }
+ .falign
+.Lmul_ovf:
+ // We get either max finite value or infinity. Either way, overflow+inexact
+ {
+ TMP = USR
+ ATMP = combine(##0x7fefffff,#-1) // positive max finite
+ A = PP_HH
+ }
+ {
+ PP_LL_L = extractu(TMP,#2,#SR_ROUND_OFF) // rounding bits
+ TMP = or(TMP,#0x28) // inexact + overflow
+ BTMP = combine(##0x7ff00000,#0) // positive infinity
+ }
+ {
+ USR = TMP
+ PP_LL_L ^= lsr(AH,#31) // Does sign match rounding?
+ TMP = PP_LL_L // unmodified rounding mode
+ }
+ {
+ p0 = !cmp.eq(TMP,#1) // If not round-to-zero and
+ p0 = !cmp.eq(PP_LL_L,#2) // Not rounding the other way,
+ if (p0.new) ATMP = BTMP // we should get infinity
+ p0 = dfcmp.eq(A,A) // Realize FP exception if enabled
+ }
+ {
+ A = insert(ATMP,#63,#0) // insert inf/maxfinite, leave sign
+ jumpr r31
+ }
+
+.Lmul_abnormal:
+ {
+ ATMP = extractu(A,#63,#0) // strip off sign
+ BTMP = extractu(B,#63,#0) // strip off sign
+ }
+ {
+ p3 = cmp.gtu(ATMP,BTMP)
+ if (!p3.new) A = B // sort values
+ if (!p3.new) B = A // sort values
+ }
+ {
+ // Any NaN --> NaN, possibly raise invalid if sNaN
+ p0 = dfclass(A,#0x0f) // A not NaN?
+ if (!p0.new) jump:nt .Linvalid_nan
+ if (!p3) ATMP = BTMP
+ if (!p3) BTMP = ATMP
+ }
+ {
+ // Infinity * nonzero number is infinity
+ p1 = dfclass(A,#0x08) // A is infinity
+ p1 = dfclass(B,#0x0e) // B is nonzero
+ }
+ {
+ // Infinity * zero --> NaN, raise invalid
+ // Other zeros return zero
+ p0 = dfclass(A,#0x08) // A is infinity
+ p0 = dfclass(B,#0x01) // B is zero
+ }
+ {
+ if (p1) jump .Ltrue_inf
+ p2 = dfclass(B,#0x01)
+ }
+ {
+ if (p0) jump .Linvalid_zeroinf
+ if (p2) jump .Ltrue_zero // so return zero
+ TMP = ##0x7c000000
+ }
+ // We are left with a normal or subnormal times a subnormal. A > B
+ // If A and B are both very small (exp(a) < BIAS-MANTBITS),
+ // we go to a single sticky bit, which we can round easily.
+ // If A and B might multiply to something bigger, decrease A exponent and increase
+ // B exponent and try again
+ {
+ p0 = bitsclr(AH,TMP)
+ if (p0.new) jump:nt .Lmul_tiny
+ }
+ {
+ TMP = cl0(BTMP)
+ }
+ {
+ TMP = add(TMP,#-EXPBITS)
+ }
+ {
+ BTMP = asl(BTMP,TMP)
+ }
+ {
+ B = insert(BTMP,#63,#0)
+ AH -= asl(TMP,#HI_MANTBITS)
+ }
+ jump __hexagon_muldf3
+.Lmul_tiny:
+ {
+ TMP = USR
+ A = xor(A,B) // get sign bit
+ }
+ {
+ TMP = or(TMP,#0x30) // Inexact + Underflow
+ A = insert(ONE,#63,#0) // put in rounded up value
+ BTMPH = extractu(TMP,#2,#SR_ROUND_OFF) // get rounding mode
+ }
+ {
+ USR = TMP
+ p0 = cmp.gt(BTMPH,#1) // Round towards pos/neg inf?
+ if (!p0.new) AL = #0 // If not, zero
+ BTMPH ^= lsr(AH,#31) // rounding my way --> set LSB
+ }
+ {
+ p0 = cmp.eq(BTMPH,#3) // if rounding towards right inf
+ if (!p0.new) AL = #0 // don't go to zero
+ jumpr r31
+ }
+.Linvalid_zeroinf:
+ {
+ TMP = USR
+ }
+ {
+ A = #-1
+ TMP = or(TMP,#2)
+ }
+ {
+ USR = TMP
+ }
+ {
+ p0 = dfcmp.uo(A,A) // force exception if enabled
+ jumpr r31
+ }
+.Linvalid_nan:
+ {
+ p0 = dfclass(B,#0x0f) // if B is not NaN
+ TMP = convert_df2sf(A) // will generate invalid if sNaN
+ if (p0.new) B = A // make it whatever A is
+ }
+ {
+ BL = convert_df2sf(B) // will generate invalid if sNaN
+ A = #-1
+ jumpr r31
+ }
+ .falign
+.Ltrue_zero:
+ {
+ A = B
+ B = A
+ }
+.Ltrue_inf:
+ {
+ BH = extract(BH,#1,#31)
+ }
+ {
+ AH ^= asl(BH,#31)
+ jumpr r31
+ }
+END(__hexagon_muldf3)
+
+#undef ATMP
+#undef ATMPL
+#undef ATMPH
+#undef BTMP
+#undef BTMPL
+#undef BTMPH
diff --git a/lib/builtins/hexagon/dfsqrt.S b/lib/builtins/hexagon/dfsqrt.S
new file mode 100644
index 000000000000..027d9e1fde43
--- /dev/null
+++ b/lib/builtins/hexagon/dfsqrt.S
@@ -0,0 +1,406 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+/* Double Precision square root */
+
+#define EXP r28
+
+#define A r1:0
+#define AH r1
+#define AL r0
+
+#define SFSH r3:2
+#define SF_S r3
+#define SF_H r2
+
+#define SFHALF_SONE r5:4
+#define S_ONE r4
+#define SFHALF r5
+#define SF_D r6
+#define SF_E r7
+#define RECIPEST r8
+#define SFRAD r9
+
+#define FRACRAD r11:10
+#define FRACRADH r11
+#define FRACRADL r10
+
+#define ROOT r13:12
+#define ROOTHI r13
+#define ROOTLO r12
+
+#define PROD r15:14
+#define PRODHI r15
+#define PRODLO r14
+
+#define P_TMP p0
+#define P_EXP1 p1
+#define NORMAL p2
+
+#define SF_EXPBITS 8
+#define SF_MANTBITS 23
+
+#define DF_EXPBITS 11
+#define DF_MANTBITS 52
+
+#define DF_BIAS 0x3ff
+
+#define DFCLASS_ZERO 0x01
+#define DFCLASS_NORMAL 0x02
+#define DFCLASS_DENORMAL 0x02
+#define DFCLASS_INFINITE 0x08
+#define DFCLASS_NAN 0x10
+
+#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG; .type __qdsp_##TAG,@function
+#define FAST_ALIAS(TAG) .global __hexagon_fast_##TAG ; .set __hexagon_fast_##TAG, __hexagon_##TAG; .type __hexagon_fast_##TAG,@function
+#define FAST2_ALIAS(TAG) .global __hexagon_fast2_##TAG ; .set __hexagon_fast2_##TAG, __hexagon_##TAG; .type __hexagon_fast2_##TAG,@function
+#define END(TAG) .size TAG,.-TAG
+
+ .text
+ .global __hexagon_sqrtdf2
+ .type __hexagon_sqrtdf2,@function
+ .global __hexagon_sqrt
+ .type __hexagon_sqrt,@function
+ Q6_ALIAS(sqrtdf2)
+ Q6_ALIAS(sqrt)
+ FAST_ALIAS(sqrtdf2)
+ FAST_ALIAS(sqrt)
+ FAST2_ALIAS(sqrtdf2)
+ FAST2_ALIAS(sqrt)
+ .type sqrt,@function
+ .p2align 5
+__hexagon_sqrtdf2:
+__hexagon_sqrt:
+ {
+ PROD = extractu(A,#SF_MANTBITS+1,#DF_MANTBITS-SF_MANTBITS)
+ EXP = extractu(AH,#DF_EXPBITS,#DF_MANTBITS-32)
+ SFHALF_SONE = combine(##0x3f000004,#1)
+ }
+ {
+ NORMAL = dfclass(A,#DFCLASS_NORMAL) // Is it normal
+ NORMAL = cmp.gt(AH,#-1) // and positive?
+ if (!NORMAL.new) jump:nt .Lsqrt_abnormal
+ SFRAD = or(SFHALF,PRODLO)
+ }
+#undef NORMAL
+.Ldenormal_restart:
+ {
+ FRACRAD = A
+ SF_E,P_TMP = sfinvsqrta(SFRAD)
+ SFHALF = and(SFHALF,#-16)
+ SFSH = #0
+ }
+#undef A
+#undef AH
+#undef AL
+#define ERROR r1:0
+#define ERRORHI r1
+#define ERRORLO r0
+ // SF_E : reciprocal square root
+ // SF_H : half rsqrt
+ // sf_S : square root
+ // SF_D : error term
+ // SFHALF: 0.5
+ {
+ SF_S += sfmpy(SF_E,SFRAD):lib // s0: root
+ SF_H += sfmpy(SF_E,SFHALF):lib // h0: 0.5*y0. Could also decrement exponent...
+ SF_D = SFHALF
+#undef SFRAD
+#define SHIFTAMT r9
+ SHIFTAMT = and(EXP,#1)
+ }
+ {
+ SF_D -= sfmpy(SF_S,SF_H):lib // d0: 0.5-H*S = 0.5-0.5*~1
+ FRACRADH = insert(S_ONE,#DF_EXPBITS+1,#DF_MANTBITS-32) // replace upper bits with hidden
+ P_EXP1 = cmp.gtu(SHIFTAMT,#0)
+ }
+ {
+ SF_S += sfmpy(SF_S,SF_D):lib // s1: refine sqrt
+ SF_H += sfmpy(SF_H,SF_D):lib // h1: refine half-recip
+ SF_D = SFHALF
+ SHIFTAMT = mux(P_EXP1,#8,#9)
+ }
+ {
+ SF_D -= sfmpy(SF_S,SF_H):lib // d1: error term
+ FRACRAD = asl(FRACRAD,SHIFTAMT) // Move fracrad bits to right place
+ SHIFTAMT = mux(P_EXP1,#3,#2)
+ }
+ {
+ SF_H += sfmpy(SF_H,SF_D):lib // d2: rsqrt
+ // cool trick: half of 1/sqrt(x) has same mantissa as 1/sqrt(x).
+ PROD = asl(FRACRAD,SHIFTAMT) // fracrad<<(2+exp1)
+ }
+ {
+ SF_H = and(SF_H,##0x007fffff)
+ }
+ {
+ SF_H = add(SF_H,##0x00800000 - 3)
+ SHIFTAMT = mux(P_EXP1,#7,#8)
+ }
+ {
+ RECIPEST = asl(SF_H,SHIFTAMT)
+ SHIFTAMT = mux(P_EXP1,#15-(1+1),#15-(1+0))
+ }
+ {
+ ROOT = mpyu(RECIPEST,PRODHI) // root = mpyu_full(recipest,hi(fracrad<<(2+exp1)))
+ }
+
+#undef SFSH // r3:2
+#undef SF_H // r2
+#undef SF_S // r3
+#undef S_ONE // r4
+#undef SFHALF // r5
+#undef SFHALF_SONE // r5:4
+#undef SF_D // r6
+#undef SF_E // r7
+
+#define HL r3:2
+#define LL r5:4
+#define HH r7:6
+
+#undef P_EXP1
+#define P_CARRY0 p1
+#define P_CARRY1 p2
+#define P_CARRY2 p3
+
+ /* Iteration 0 */
+ /* Maybe we can save a cycle by starting with ERROR=asl(fracrad), then as we multiply */
+ /* We can shift and subtract instead of shift and add? */
+ {
+ ERROR = asl(FRACRAD,#15)
+ PROD = mpyu(ROOTHI,ROOTHI)
+ P_CARRY0 = cmp.eq(r0,r0)
+ }
+ {
+ ERROR -= asl(PROD,#15)
+ PROD = mpyu(ROOTHI,ROOTLO)
+ P_CARRY1 = cmp.eq(r0,r0)
+ }
+ {
+ ERROR -= lsr(PROD,#16)
+ P_CARRY2 = cmp.eq(r0,r0)
+ }
+ {
+ ERROR = mpyu(ERRORHI,RECIPEST)
+ }
+ {
+ ROOT += lsr(ERROR,SHIFTAMT)
+ SHIFTAMT = add(SHIFTAMT,#16)
+ ERROR = asl(FRACRAD,#31) // for next iter
+ }
+ /* Iteration 1 */
+ {
+ PROD = mpyu(ROOTHI,ROOTHI)
+ ERROR -= mpyu(ROOTHI,ROOTLO) // amount is 31, no shift needed
+ }
+ {
+ ERROR -= asl(PROD,#31)
+ PROD = mpyu(ROOTLO,ROOTLO)
+ }
+ {
+ ERROR -= lsr(PROD,#33)
+ }
+ {
+ ERROR = mpyu(ERRORHI,RECIPEST)
+ }
+ {
+ ROOT += lsr(ERROR,SHIFTAMT)
+ SHIFTAMT = add(SHIFTAMT,#16)
+ ERROR = asl(FRACRAD,#47) // for next iter
+ }
+ /* Iteration 2 */
+ {
+ PROD = mpyu(ROOTHI,ROOTHI)
+ }
+ {
+ ERROR -= asl(PROD,#47)
+ PROD = mpyu(ROOTHI,ROOTLO)
+ }
+ {
+ ERROR -= asl(PROD,#16) // bidir shr 31-47
+ PROD = mpyu(ROOTLO,ROOTLO)
+ }
+ {
+ ERROR -= lsr(PROD,#17) // 64-47
+ }
+ {
+ ERROR = mpyu(ERRORHI,RECIPEST)
+ }
+ {
+ ROOT += lsr(ERROR,SHIFTAMT)
+ }
+#undef ERROR
+#undef PROD
+#undef PRODHI
+#undef PRODLO
+#define REM_HI r15:14
+#define REM_HI_HI r15
+#define REM_LO r1:0
+#undef RECIPEST
+#undef SHIFTAMT
+#define TWOROOT_LO r9:8
+ /* Adjust Root */
+ {
+ HL = mpyu(ROOTHI,ROOTLO)
+ LL = mpyu(ROOTLO,ROOTLO)
+ REM_HI = #0
+ REM_LO = #0
+ }
+ {
+ HL += lsr(LL,#33)
+ LL += asl(HL,#33)
+ P_CARRY0 = cmp.eq(r0,r0)
+ }
+ {
+ HH = mpyu(ROOTHI,ROOTHI)
+ REM_LO = sub(REM_LO,LL,P_CARRY0):carry
+ TWOROOT_LO = #1
+ }
+ {
+ HH += lsr(HL,#31)
+ TWOROOT_LO += asl(ROOT,#1)
+ }
+#undef HL
+#undef LL
+#define REM_HI_TMP r3:2
+#define REM_HI_TMP_HI r3
+#define REM_LO_TMP r5:4
+ {
+ REM_HI = sub(FRACRAD,HH,P_CARRY0):carry
+ REM_LO_TMP = sub(REM_LO,TWOROOT_LO,P_CARRY1):carry
+#undef FRACRAD
+#undef HH
+#define ZERO r11:10
+#define ONE r7:6
+ ONE = #1
+ ZERO = #0
+ }
+ {
+ REM_HI_TMP = sub(REM_HI,ZERO,P_CARRY1):carry
+ ONE = add(ROOT,ONE)
+ EXP = add(EXP,#-DF_BIAS) // subtract bias --> signed exp
+ }
+ {
+ // If carry set, no borrow: result was still positive
+ if (P_CARRY1) ROOT = ONE
+ if (P_CARRY1) REM_LO = REM_LO_TMP
+ if (P_CARRY1) REM_HI = REM_HI_TMP
+ }
+ {
+ REM_LO_TMP = sub(REM_LO,TWOROOT_LO,P_CARRY2):carry
+ ONE = #1
+ EXP = asr(EXP,#1) // divide signed exp by 2
+ }
+ {
+ REM_HI_TMP = sub(REM_HI,ZERO,P_CARRY2):carry
+ ONE = add(ROOT,ONE)
+ }
+ {
+ if (P_CARRY2) ROOT = ONE
+ if (P_CARRY2) REM_LO = REM_LO_TMP
+ // since tworoot <= 2^32, remhi must be zero
+#undef REM_HI_TMP
+#undef REM_HI_TMP_HI
+#define S_ONE r2
+#define ADJ r3
+ S_ONE = #1
+ }
+ {
+ P_TMP = cmp.eq(REM_LO,ZERO) // is the low part zero
+ if (!P_TMP.new) ROOTLO = or(ROOTLO,S_ONE) // if so, it's exact... hopefully
+ ADJ = cl0(ROOT)
+ EXP = add(EXP,#-63)
+ }
+#undef REM_LO
+#define RET r1:0
+#define RETHI r1
+ {
+ RET = convert_ud2df(ROOT) // set up mantissa, maybe set inexact flag
+ EXP = add(EXP,ADJ) // add back bias
+ }
+ {
+ RETHI += asl(EXP,#DF_MANTBITS-32) // add exponent adjust
+ jumpr r31
+ }
+#undef REM_LO_TMP
+#undef REM_HI_TMP
+#undef REM_HI_TMP_HI
+#undef REM_LO
+#undef REM_HI
+#undef TWOROOT_LO
+
+#undef RET
+#define A r1:0
+#define AH r1
+#define AL r1
+#undef S_ONE
+#define TMP r3:2
+#define TMPHI r3
+#define TMPLO r2
+#undef P_CARRY0
+#define P_NEG p1
+
+
+#define SFHALF r5
+#define SFRAD r9
+.Lsqrt_abnormal:
+ {
+ P_TMP = dfclass(A,#DFCLASS_ZERO) // zero?
+ if (P_TMP.new) jumpr:t r31
+ }
+ {
+ P_TMP = dfclass(A,#DFCLASS_NAN)
+ if (P_TMP.new) jump:nt .Lsqrt_nan
+ }
+ {
+ P_TMP = cmp.gt(AH,#-1)
+ if (!P_TMP.new) jump:nt .Lsqrt_invalid_neg
+ if (!P_TMP.new) EXP = ##0x7F800001 // sNaN
+ }
+ {
+ P_TMP = dfclass(A,#DFCLASS_INFINITE)
+ if (P_TMP.new) jumpr:nt r31
+ }
+ // If we got here, we're denormal
+ // prepare to restart
+ {
+ A = extractu(A,#DF_MANTBITS,#0) // Extract mantissa
+ }
+ {
+ EXP = add(clb(A),#-DF_EXPBITS) // how much to normalize?
+ }
+ {
+ A = asl(A,EXP) // Shift mantissa
+ EXP = sub(#1,EXP) // Form exponent
+ }
+ {
+ AH = insert(EXP,#1,#DF_MANTBITS-32) // insert lsb of exponent
+ }
+ {
+ TMP = extractu(A,#SF_MANTBITS+1,#DF_MANTBITS-SF_MANTBITS) // get sf value (mant+exp1)
+ SFHALF = ##0x3f000004 // form half constant
+ }
+ {
+ SFRAD = or(SFHALF,TMPLO) // form sf value
+ SFHALF = and(SFHALF,#-16)
+ jump .Ldenormal_restart // restart
+ }
+.Lsqrt_nan:
+ {
+ EXP = convert_df2sf(A) // if sNaN, get invalid
+ A = #-1 // qNaN
+ jumpr r31
+ }
+.Lsqrt_invalid_neg:
+ {
+ A = convert_sf2df(EXP) // Invalid,NaNval
+ jumpr r31
+ }
+END(__hexagon_sqrt)
+END(__hexagon_sqrtdf2)
diff --git a/lib/builtins/hexagon/divdi3.S b/lib/builtins/hexagon/divdi3.S
new file mode 100644
index 000000000000..49ee8104f305
--- /dev/null
+++ b/lib/builtins/hexagon/divdi3.S
@@ -0,0 +1,85 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_divdi3
+ {
+ p2 = tstbit(r1,#31)
+ p3 = tstbit(r3,#31)
+ }
+ {
+ r1:0 = abs(r1:0)
+ r3:2 = abs(r3:2)
+ }
+ {
+ r6 = cl0(r1:0) // count leading 0's of dividend (numerator)
+ r7 = cl0(r3:2) // count leading 0's of divisor (denominator)
+ r5:4 = r3:2 // divisor moved into working registers
+ r3:2 = r1:0 // dividend is the initial remainder, r3:2 contains remainder
+ }
+ {
+ p3 = xor(p2,p3)
+ r10 = sub(r7,r6) // left shift count for bit & divisor
+ r1:0 = #0 // initialize quotient to 0
+ r15:14 = #1 // initialize bit to 1
+ }
+ {
+ r11 = add(r10,#1) // loop count is 1 more than shift count
+ r13:12 = lsl(r5:4,r10) // shift divisor msb into same bit position as dividend msb
+ r15:14 = lsl(r15:14,r10) // shift the bit left by same amount as divisor
+ }
+ {
+ p0 = cmp.gtu(r5:4,r3:2) // check if divisor > dividend
+ loop0(1f,r11) // register loop
+ }
+ {
+ if (p0) jump .hexagon_divdi3_return // if divisor > dividend, we're done, so return
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r13:12,r3:2) // set predicate reg if shifted divisor > current remainder
+ }
+ {
+ r7:6 = sub(r3:2, r13:12) // subtract shifted divisor from current remainder
+ r9:8 = add(r1:0, r15:14) // save current quotient to temp (r9:8)
+ }
+ {
+ r1:0 = vmux(p0, r1:0, r9:8) // choose either current quotient or new quotient (r9:8)
+ r3:2 = vmux(p0, r3:2, r7:6) // choose either current remainder or new remainder (r7:6)
+ }
+ {
+ r15:14 = lsr(r15:14, #1) // shift bit right by 1 for next iteration
+ r13:12 = lsr(r13:12, #1) // shift "shifted divisor" right by 1 for next iteration
+ }:endloop0
+
+.hexagon_divdi3_return:
+ {
+ r3:2 = neg(r1:0)
+ }
+ {
+ r1:0 = vmux(p3,r3:2,r1:0)
+ jumpr r31
+ }
+FUNCTION_END __hexagon_divdi3
+
+ .globl __qdsp_divdi3
+ .set __qdsp_divdi3, __hexagon_divdi3
diff --git a/lib/builtins/hexagon/divsi3.S b/lib/builtins/hexagon/divsi3.S
new file mode 100644
index 000000000000..8e159baa192f
--- /dev/null
+++ b/lib/builtins/hexagon/divsi3.S
@@ -0,0 +1,84 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_divsi3
+ {
+ p0 = cmp.ge(r0,#0)
+ p1 = cmp.ge(r1,#0)
+ r1 = abs(r0)
+ r2 = abs(r1)
+ }
+ {
+ r3 = cl0(r1)
+ r4 = cl0(r2)
+ r5 = sub(r1,r2)
+ p2 = cmp.gtu(r2,r1)
+ }
+#if (__HEXAGON_ARCH__ == 60)
+ {
+ r0 = #0
+ p1 = xor(p0,p1)
+ p0 = cmp.gtu(r2,r5)
+ }
+ if (p2) jumpr r31
+#else
+ {
+ r0 = #0
+ p1 = xor(p0,p1)
+ p0 = cmp.gtu(r2,r5)
+ if (p2) jumpr r31
+ }
+#endif
+ {
+ r0 = mux(p1,#-1,#1)
+ if (p0) jumpr r31
+ r4 = sub(r4,r3)
+ r3 = #1
+ }
+ {
+ r0 = #0
+ r3:2 = vlslw(r3:2,r4)
+ loop0(1f,r4)
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r2,r1)
+ if (!p0.new) r1 = sub(r1,r2)
+ if (!p0.new) r0 = add(r0,r3)
+ r3:2 = vlsrw(r3:2,#1)
+ }:endloop0
+ {
+ p0 = cmp.gtu(r2,r1)
+ if (!p0.new) r0 = add(r0,r3)
+ if (!p1) jumpr r31
+ }
+ {
+ r0 = neg(r0)
+ jumpr r31
+ }
+FUNCTION_END __hexagon_divsi3
+
+ .globl __qdsp_divsi3
+ .set __qdsp_divsi3, __hexagon_divsi3
diff --git a/lib/builtins/hexagon/fabs_opt.S b/lib/builtins/hexagon/fabs_opt.S
new file mode 100644
index 000000000000..b09b00734d98
--- /dev/null
+++ b/lib/builtins/hexagon/fabs_opt.S
@@ -0,0 +1,37 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+.macro FUNCTION_BEGIN name
+.text
+.p2align 5
+.globl \name
+.type \name, @function
+\name:
+.endm
+
+.macro FUNCTION_END name
+.size \name, . - \name
+.endm
+
+FUNCTION_BEGIN fabs
+ {
+ r1 = clrbit(r1, #31)
+ jumpr r31
+ }
+FUNCTION_END fabs
+
+FUNCTION_BEGIN fabsf
+ {
+ r0 = clrbit(r0, #31)
+ jumpr r31
+ }
+FUNCTION_END fabsf
+
+ .globl fabsl
+ .set fabsl, fabs
diff --git a/lib/builtins/hexagon/fastmath2_dlib_asm.S b/lib/builtins/hexagon/fastmath2_dlib_asm.S
new file mode 100644
index 000000000000..9286df06c26d
--- /dev/null
+++ b/lib/builtins/hexagon/fastmath2_dlib_asm.S
@@ -0,0 +1,491 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/* ==================================================================== */
+/* FUNCTIONS Optimized double floating point operators */
+/* ==================================================================== */
+/* c = dadd_asm(a, b) */
+/* ==================================================================== *
+fast2_QDOUBLE fast2_dadd(fast2_QDOUBLE a,fast2_QDOUBLE b) {
+ fast2_QDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = Q6_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = Q6_R_sxth_R(b) ;
+ int exp, expdiff, j, k, hi, lo, cn;
+ lint mant;
+
+ expdiff = (int) Q6_P_vabsdiffh_PP(a, b);
+ expdiff = Q6_R_sxth_R(expdiff) ;
+ if (expdiff > 63) { expdiff = 62;}
+ if (expa > expb) {
+ exp = expa + 1;
+ expa = 1;
+ expb = expdiff + 1;
+ } else {
+ exp = expb + 1;
+ expb = 1;
+ expa = expdiff + 1;
+ }
+ mant = (manta>>expa) + (mantb>>expb);
+
+ hi = (int) (mant>>32);
+ lo = (int) (mant);
+
+ k = Q6_R_normamt_R(hi);
+ if(hi == 0 || hi == -1) k = 31+Q6_R_normamt_R(lo);
+
+ mant = (mant << k);
+ cn = (mant == 0x8000000000000000LL);
+ exp = exp - k + cn;
+
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global fast2_dadd_asm
+ .type fast2_dadd_asm, @function
+fast2_dadd_asm:
+#define manta R0
+#define mantexpa R1:0
+#define lmanta R1:0
+#define mantb R2
+#define mantexpb R3:2
+#define lmantb R3:2
+#define expa R4
+#define expb R5
+#define mantexpd R7:6
+#define expd R6
+#define exp R8
+#define c63 R9
+#define lmant R1:0
+#define manth R1
+#define mantl R0
+#define minmin R11:10 // exactly 0x000000000000008001LL
+#define minminl R10
+#define k R4
+#define ce P0
+ .falign
+ {
+ mantexpd = VABSDIFFH(mantexpa, mantexpb) //represented as 0x08001LL
+ c63 = #62
+ expa = SXTH(manta)
+ expb = SXTH(mantb)
+ } {
+ expd = SXTH(expd)
+ ce = CMP.GT(expa, expb);
+ if ( ce.new) exp = add(expa, #1)
+ if (!ce.new) exp = add(expb, #1)
+ } {
+ if ( ce) expa = #1
+ if (!ce) expb = #1
+ manta.L = #0
+ expd = MIN(expd, c63)
+ } {
+ if (!ce) expa = add(expd, #1)
+ if ( ce) expb = add(expd, #1)
+ mantb.L = #0
+ minmin = #0
+ } {
+ lmanta = ASR(lmanta, expa)
+ lmantb = ASR(lmantb, expb)
+ } {
+ lmant = add(lmanta, lmantb)
+ minminl.L = #0x8001
+ } {
+ k = clb(lmant)
+ c63 = #58
+ } {
+ k = add(k, #-1)
+ p0 = cmp.gt(k, c63)
+ } {
+ mantexpa = ASL(lmant, k)
+ exp = SUB(exp, k)
+ if(p0) jump .Ldenorma
+ } {
+ manta = insert(exp, #16, #0)
+ jumpr r31
+ }
+.Ldenorma:
+ {
+ mantexpa = minmin
+ jumpr r31
+ }
+/* =================================================================== *
+ fast2_QDOUBLE fast2_dsub(fast2_QDOUBLE a,fast2_QDOUBLE b) {
+ fast2_QDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = Q6_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = Q6_R_sxth_R(b) ;
+ int exp, expdiff, j, k;
+ lint mant;
+
+ expdiff = (int) Q6_P_vabsdiffh_PP(a, b);
+ expdiff = Q6_R_sxth_R(expdiff) ;
+ if (expdiff > 63) { expdiff = 62;}
+ if (expa > expb) {
+ exp = expa + 1;
+ expa = 1;
+ expb = expdiff + 1;
+ } else {
+ exp = expb + 1;
+ expb = 1;
+ expa = expdiff + 1;
+ }
+ mant = (manta>>expa) - (mantb>>expb);
+ k = Q6_R_clb_P(mant)-1;
+ mant = (mant << k);
+ exp = exp - k;
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global fast2_dsub_asm
+ .type fast2_dsub_asm, @function
+fast2_dsub_asm:
+
+#define manta R0
+#define mantexpa R1:0
+#define lmanta R1:0
+#define mantb R2
+#define mantexpb R3:2
+#define lmantb R3:2
+#define expa R4
+#define expb R5
+#define mantexpd R7:6
+#define expd R6
+#define exp R8
+#define c63 R9
+#define lmant R1:0
+#define manth R1
+#define mantl R0
+#define minmin R11:10 // exactly 0x000000000000008001LL
+#define minminl R10
+#define k R4
+#define ce P0
+ .falign
+ {
+ mantexpd = VABSDIFFH(mantexpa, mantexpb) //represented as 0x08001LL
+ c63 = #62
+ expa = SXTH(manta)
+ expb = SXTH(mantb)
+ } {
+ expd = SXTH(expd)
+ ce = CMP.GT(expa, expb);
+ if ( ce.new) exp = add(expa, #1)
+ if (!ce.new) exp = add(expb, #1)
+ } {
+ if ( ce) expa = #1
+ if (!ce) expb = #1
+ manta.L = #0
+ expd = MIN(expd, c63)
+ } {
+ if (!ce) expa = add(expd, #1)
+ if ( ce) expb = add(expd, #1)
+ mantb.L = #0
+ minmin = #0
+ } {
+ lmanta = ASR(lmanta, expa)
+ lmantb = ASR(lmantb, expb)
+ } {
+ lmant = sub(lmanta, lmantb)
+ minminl.L = #0x8001
+ } {
+ k = clb(lmant)
+ c63 = #58
+ } {
+ k = add(k, #-1)
+ p0 = cmp.gt(k, c63)
+ } {
+ mantexpa = ASL(lmant, k)
+ exp = SUB(exp, k)
+ if(p0) jump .Ldenorm
+ } {
+ manta = insert(exp, #16, #0)
+ jumpr r31
+ }
+.Ldenorm:
+ {
+ mantexpa = minmin
+ jumpr r31
+ }
+/* ==================================================================== *
+ fast2_QDOUBLE fast2_dmpy(fast2_QDOUBLE a,fast2_QDOUBLE b) {
+ fast2_QDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = Q6_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = Q6_R_sxth_R(b) ;
+ int exp, k;
+ lint mant;
+ int hia, hib, hi, lo;
+ unsigned int loa, lob;
+
+ hia = (int)(a >> 32);
+ loa = Q6_R_extractu_RII((int)manta, 31, 1);
+ hib = (int)(b >> 32);
+ lob = Q6_R_extractu_RII((int)mantb, 31, 1);
+
+ mant = Q6_P_mpy_RR(hia, lob);
+ mant = Q6_P_mpyacc_RR(mant,hib, loa);
+ mant = (mant >> 30) + (Q6_P_mpy_RR(hia, hib)<<1);
+
+ hi = (int) (mant>>32);
+
+ k = Q6_R_normamt_R(hi);
+ mant = mant << k;
+ exp = expa + expb - k;
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global fast2_dmpy_asm
+ .type fast2_dmpy_asm, @function
+fast2_dmpy_asm:
+
+#define mantal R0
+#define mantah R1
+#define mantexpa R1:0
+#define mantbl R2
+#define mantbh R3
+#define mantexpb R3:2
+#define expa R4
+#define expb R5
+#define c8001 R12
+#define mantexpd R7:6
+#define mantdh R7
+#define exp R8
+#define lmantc R11:10
+#define kb R9
+#define guard R11
+#define mantal_ R12
+#define mantbl_ R13
+#define min R15:14
+#define minh R15
+
+ .falign
+ {
+ mantbl_= lsr(mantbl, #16)
+ expb = sxth(mantbl)
+ expa = sxth(mantal)
+ mantal_= lsr(mantal, #16)
+ }
+ {
+ lmantc = mpy(mantah, mantbh)
+ mantexpd = mpy(mantah, mantbl_)
+ mantal.L = #0x0
+ min = #0
+ }
+ {
+ lmantc = add(lmantc, lmantc)
+ mantexpd+= mpy(mantbh, mantal_)
+ mantbl.L = #0x0
+ minh.H = #0x8000
+ }
+ {
+ mantexpd = asr(mantexpd, #15)
+ c8001.L = #0x8001
+ p1 = cmp.eq(mantexpa, mantexpb)
+ }
+ {
+ mantexpd = add(mantexpd, lmantc)
+ exp = add(expa, expb)
+ p2 = cmp.eq(mantexpa, min)
+ }
+ {
+ kb = clb(mantexpd)
+ mantexpb = abs(mantexpd)
+ guard = #58
+ }
+ {
+ p1 = and(p1, p2)
+ exp = sub(exp, kb)
+ kb = add(kb, #-1)
+ p0 = cmp.gt(kb, guard)
+ }
+ {
+ exp = add(exp, #1)
+ mantexpa = asl(mantexpd, kb)
+ if(p1) jump .Lsat //rarely happens
+ }
+ {
+ mantal = insert(exp,#16, #0)
+ if(!p0) jumpr r31
+ }
+ {
+ mantal = insert(c8001,#16, #0)
+ jumpr r31
+ }
+.Lsat:
+ {
+ mantexpa = #-1
+ }
+ {
+ mantexpa = lsr(mantexpa, #1)
+ }
+ {
+ mantal = insert(exp,#16, #0)
+ jumpr r31
+ }
+
+/* ==================================================================== *
+ int fast2_qd2f(fast2_QDOUBLE a) {
+ int exp;
+ long long int manta;
+ int ic, rnd, mantb;
+
+ manta = a>>32;
+ exp = Q6_R_sxth_R(a) ;
+ ic = 0x80000000 & manta;
+ manta = Q6_R_abs_R_sat(manta);
+ mantb = (manta + rnd)>>7;
+ rnd = 0x40
+ exp = (exp + 126);
+ if((manta & 0xff) == rnd) rnd = 0x00;
+ if((manta & 0x7fffffc0) == 0x7fffffc0) {
+ manta = 0x0; exp++;
+ } else {
+ manta= mantb & 0x007fffff;
+ }
+ exp = (exp << 23) & 0x7fffffc0;
+ ic = Q6_R_addacc_RR(ic, exp, manta);
+ return (ic);
+ }
+ * ==================================================================== */
+
+ .text
+ .global fast2_qd2f_asm
+ .type fast2_qd2f_asm, @function
+fast2_qd2f_asm:
+#define mantah R1
+#define mantal R0
+#define cff R0
+#define mant R3
+#define expo R4
+#define rnd R5
+#define mask R6
+#define c07f R7
+#define c80 R0
+#define mantb R2
+#define ic R0
+
+ .falign
+ {
+ mant = abs(mantah):sat
+ expo = sxth(mantal)
+ rnd = #0x40
+ mask.L = #0xffc0
+ }
+ {
+ cff = extractu(mant, #8, #0)
+ p2 = cmp.gt(expo, #126)
+ p3 = cmp.ge(expo, #-126)
+ mask.H = #0x7fff
+ }
+ {
+ p1 = cmp.eq(cff,#0x40)
+ if(p1.new) rnd = #0
+ expo = add(expo, #126)
+ if(!p3) jump .Lmin
+ }
+ {
+ p0 = bitsset(mant, mask)
+ c80.L = #0x0000
+ mantb = add(mant, rnd)
+ c07f = lsr(mask, #8)
+ }
+ {
+ if(p0) expo = add(expo, #1)
+ if(p0) mant = #0
+ mantb = lsr(mantb, #7)
+ c80.H = #0x8000
+ }
+ {
+ ic = and(c80, mantah)
+ mask &= asl(expo, #23)
+ if(!p0) mant = and(mantb, c07f)
+ if(p2) jump .Lmax
+ }
+ {
+ ic += add(mask, mant)
+ jumpr r31
+ }
+.Lmax:
+ {
+ ic.L = #0xffff;
+ }
+ {
+ ic.H = #0x7f7f;
+ jumpr r31
+ }
+.Lmin:
+ {
+ ic = #0x0
+ jumpr r31
+ }
+
+/* ==================================================================== *
+fast2_QDOUBLE fast2_f2qd(int ia) {
+ lint exp;
+ lint mant;
+ fast2_QDOUBLE c;
+
+ mant = ((ia << 7) | 0x40000000)&0x7fffff80 ;
+ if (ia & 0x80000000) mant = -mant;
+ exp = ((ia >> 23) & 0xFFLL) - 126;
+ c = (mant<<32) | Q6_R_zxth_R(exp);;
+ return(c);
+}
+ * ==================================================================== */
+ .text
+ .global fast2_f2qd_asm
+ .type fast2_f2qd_asm, @function
+fast2_f2qd_asm:
+#define ia R0
+#define mag R3
+#define mantr R1
+#define expr R0
+#define zero R2
+#define maxneg R5:4
+#define maxnegl R4
+ .falign
+ {
+ mantr = asl(ia, #7)
+ p0 = tstbit(ia, #31)
+ maxneg = #0
+ mag = add(ia,ia)
+ }
+ {
+ mantr = setbit(mantr, #30)
+ expr= extractu(ia,#8,#23)
+ maxnegl.L = #0x8001
+ p1 = cmp.eq(mag, #0)
+ }
+ {
+ mantr= extractu(mantr, #31, #0)
+ expr= add(expr, #-126)
+ zero = #0
+ if(p1) jump .Lminqd
+ }
+ {
+ expr = zxth(expr)
+ if(p0) mantr= sub(zero, mantr)
+ jumpr r31
+ }
+.Lminqd:
+ {
+ R1:0 = maxneg
+ jumpr r31
+ }
diff --git a/lib/builtins/hexagon/fastmath2_ldlib_asm.S b/lib/builtins/hexagon/fastmath2_ldlib_asm.S
new file mode 100644
index 000000000000..4192555351a6
--- /dev/null
+++ b/lib/builtins/hexagon/fastmath2_ldlib_asm.S
@@ -0,0 +1,345 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/* ==================================================================== *
+
+fast2_QLDOUBLE fast2_ldadd(fast2_QLDOUBLE a,fast2_QLDOUBLE b) {
+ fast2_QLDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = Q6_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = Q6_R_sxth_R(b) ;
+ int exp, expdiff, j, k, hi, lo, cn;
+ lint mant;
+
+ expdiff = (int) Q6_P_vabsdiffh_PP(a, b);
+ expdiff = Q6_R_sxth_R(expdiff) ;
+ if (expdiff > 63) { expdiff = 62;}
+ if (expa > expb) {
+ exp = expa + 1;
+ expa = 1;
+ expb = expdiff + 1;
+ } else {
+ exp = expb + 1;
+ expb = 1;
+ expa = expdiff + 1;
+ }
+ mant = (manta>>expa) + (mantb>>expb);
+
+ hi = (int) (mant>>32);
+ lo = (int) (mant);
+
+ k = Q6_R_normamt_R(hi);
+ if(hi == 0 || hi == -1) k = 31+Q6_R_normamt_R(lo);
+
+ mant = (mant << k);
+ cn = (mant == 0x8000000000000000LL);
+ exp = exp - k + cn;
+
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global fast2_ldadd_asm
+ .type fast2_ldadd_asm, @function
+fast2_ldadd_asm:
+#define manta R1:0
+#define lmanta R1:0
+#define mantb R3:2
+#define lmantb R3:2
+#define expa R4
+#define expb R5
+#define expd R6
+#define exp R8
+#define c63 R9
+#define lmant R1:0
+#define k R4
+#define ce P0
+#define zero R3:2
+ .falign
+ {
+ expa = memw(r29+#8)
+ expb = memw(r29+#24)
+ r7 = r0
+ }
+ {
+ expd = sub(expa, expb):sat
+ ce = CMP.GT(expa, expb);
+ if ( ce.new) exp = add(expa, #1)
+ if (!ce.new) exp = add(expb, #1)
+ } {
+ expd = abs(expd):sat
+ if ( ce) expa = #1
+ if (!ce) expb = #1
+ c63 = #62
+ } {
+ expd = MIN(expd, c63)
+ manta = memd(r29+#0)
+ mantb = memd(r29+#16)
+ } {
+ if (!ce) expa = add(expd, #1)
+ if ( ce) expb = add(expd, #1)
+ } {
+ lmanta = ASR(lmanta, expa)
+ lmantb = ASR(lmantb, expb)
+ } {
+ lmant = add(lmanta, lmantb)
+ zero = #0
+ } {
+ k = clb(lmant)
+ c63.L =#0x0001
+ } {
+ exp -= add(k, #-1) //exp = exp - (k-1)
+ k = add(k, #-1)
+ p0 = cmp.gt(k, #58)
+ c63.H =#0x8000
+ } {
+ if(!p0)memw(r7+#8) = exp
+ lmant = ASL(lmant, k)
+ if(p0) jump .Ldenorma
+ } {
+ memd(r7+#0) = lmant
+ jumpr r31
+ }
+.Ldenorma:
+ memd(r7+#0) = zero
+ {
+ memw(r7+#8) = c63
+ jumpr r31
+ }
+/* =================================================================== *
+ fast2_QLDOUBLE fast2_ldsub(fast2_QLDOUBLE a,fast2_QLDOUBLE b) {
+ fast2_QLDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = Q6_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = Q6_R_sxth_R(b) ;
+ int exp, expdiff, j, k;
+ lint mant;
+
+ expdiff = (int) Q6_P_vabsdiffh_PP(a, b);
+ expdiff = Q6_R_sxth_R(expdiff) ;
+ if (expdiff > 63) { expdiff = 62;}
+ if (expa > expb) {
+ exp = expa + 1;
+ expa = 1;
+ expb = expdiff + 1;
+ } else {
+ exp = expb + 1;
+ expb = 1;
+ expa = expdiff + 1;
+ }
+ mant = (manta>>expa) - (mantb>>expb);
+ k = Q6_R_clb_P(mant)-1;
+ mant = (mant << k);
+ exp = exp - k;
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global fast2_ldsub_asm
+ .type fast2_ldsub_asm, @function
+fast2_ldsub_asm:
+#define manta R1:0
+#define lmanta R1:0
+#define mantb R3:2
+#define lmantb R3:2
+#define expa R4
+#define expb R5
+#define expd R6
+#define exp R8
+#define c63 R9
+#define lmant R1:0
+#define k R4
+#define ce P0
+#define zero R3:2
+ .falign
+ {
+ expa = memw(r29+#8)
+ expb = memw(r29+#24)
+ r7 = r0
+ }
+ {
+ expd = sub(expa, expb):sat
+ ce = CMP.GT(expa, expb);
+ if ( ce.new) exp = add(expa, #1)
+ if (!ce.new) exp = add(expb, #1)
+ } {
+ expd = abs(expd):sat
+ if ( ce) expa = #1
+ if (!ce) expb = #1
+ c63 = #62
+ } {
+ expd = min(expd, c63)
+ manta = memd(r29+#0)
+ mantb = memd(r29+#16)
+ } {
+ if (!ce) expa = add(expd, #1)
+ if ( ce) expb = add(expd, #1)
+ } {
+ lmanta = ASR(lmanta, expa)
+ lmantb = ASR(lmantb, expb)
+ } {
+ lmant = sub(lmanta, lmantb)
+ zero = #0
+ } {
+ k = clb(lmant)
+ c63.L =#0x0001
+ } {
+ exp -= add(k, #-1) //exp = exp - (k+1)
+ k = add(k, #-1)
+ p0 = cmp.gt(k, #58)
+ c63.H =#0x8000
+ } {
+ if(!p0)memw(r7+#8) = exp
+ lmant = asl(lmant, k)
+ if(p0) jump .Ldenorma_s
+ } {
+ memd(r7+#0) = lmant
+ jumpr r31
+ }
+.Ldenorma_s:
+ memd(r7+#0) = zero
+ {
+ memw(r7+#8) = c63
+ jumpr r31
+ }
+
+/* ==================================================================== *
+ fast2_QLDOUBLE fast2_ldmpy(fast2_QLDOUBLE a,fast2_QLDOUBLE b) {
+ fast2_QLDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = Q6_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = Q6_R_sxth_R(b) ;
+ int exp, k;
+ lint mant;
+ int hia, hib, hi, lo;
+ unsigned int loa, lob;
+
+ hia = (int)(a >> 32);
+ loa = Q6_R_extractu_RII((int)manta, 31, 1);
+ hib = (int)(b >> 32);
+ lob = Q6_R_extractu_RII((int)mantb, 31, 1);
+
+ mant = Q6_P_mpy_RR(hia, lob);
+ mant = Q6_P_mpyacc_RR(mant,hib, loa);
+ mant = (mant >> 30) + (Q6_P_mpy_RR(hia, hib)<<1);
+
+ hi = (int) (mant>>32);
+
+ k = Q6_R_normamt_R(hi);
+ mant = mant << k;
+ exp = expa + expb - k;
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global fast2_ldmpy_asm
+ .type fast2_ldmpy_asm, @function
+fast2_ldmpy_asm:
+
+#define mantxl_ R9
+#define mantxl R14
+#define mantxh R15
+#define mantx R15:14
+#define mantbl R2
+#define mantbl_ R8
+#define mantbh R3
+#define mantb R3:2
+#define expa R4
+#define expb R5
+#define c8001 R8
+#define mantd R7:6
+#define lmantc R11:10
+#define kp R9
+#define min R13:12
+#define minh R13
+#define max R13:12
+#define maxh R13
+#define ret R0
+
+ .falign
+ {
+ mantx = memd(r29+#0)
+ mantb = memd(r29+#16)
+ min = #0
+ }
+ {
+ mantbl_= extractu(mantbl, #31, #1)
+ mantxl_= extractu(mantxl, #31, #1)
+ minh.H = #0x8000
+ }
+ {
+ lmantc = mpy(mantxh, mantbh)
+ mantd = mpy(mantxh, mantbl_)
+ expa = memw(r29+#8)
+ expb = memw(r29+#24)
+ }
+ {
+ lmantc = add(lmantc, lmantc)
+ mantd += mpy(mantbh, mantxl_)
+ }
+ {
+ mantd = asr(mantd, #30)
+ c8001.L = #0x0001
+ p1 = cmp.eq(mantx, mantb)
+ }
+ {
+ mantd = add(mantd, lmantc)
+ expa= add(expa, expb)
+ p2 = cmp.eq(mantb, min)
+ }
+ {
+ kp = clb(mantd)
+ c8001.H = #0x8000
+ p1 = and(p1, p2)
+ }
+ {
+ expa-= add(kp, #-1)
+ kp = add(kp, #-1)
+ if(p1) jump .Lsat
+ }
+ {
+ mantd = asl(mantd, kp)
+ memw(ret+#8) = expa
+ p0 = cmp.gt(kp, #58)
+ if(p0.new) jump:NT .Ldenorm //rarely happens
+ }
+ {
+ memd(ret+#0) = mantd
+ jumpr r31
+ }
+.Lsat:
+ {
+ max = #0
+ expa+= add(kp, #1)
+ }
+ {
+ maxh.H = #0x4000
+ memw(ret+#8) = expa
+ }
+ {
+ memd(ret+#0) = max
+ jumpr r31
+ }
+.Ldenorm:
+ {
+ memw(ret+#8) = c8001
+ mantx = #0
+ }
+ {
+ memd(ret+#0) = mantx
+ jumpr r31
+ }
diff --git a/lib/builtins/hexagon/fastmath_dlib_asm.S b/lib/builtins/hexagon/fastmath_dlib_asm.S
new file mode 100644
index 000000000000..215936b783ca
--- /dev/null
+++ b/lib/builtins/hexagon/fastmath_dlib_asm.S
@@ -0,0 +1,400 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/* ==================================================================== */
+/* FUNCTIONS Optimized double floating point operators */
+/* ==================================================================== */
+/* c = dadd_asm(a, b) */
+/* ====================================================================
+
+QDOUBLE dadd(QDOUBLE a,QDOUBLE b) {
+ QDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = HEXAGON_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = HEXAGON_R_sxth_R(b) ;
+ int exp, expdiff, j, k, hi, lo, cn;
+ lint mant;
+
+ expdiff = (int) HEXAGON_P_vabsdiffh_PP(a, b);
+ expdiff = HEXAGON_R_sxth_R(expdiff) ;
+ if (expdiff > 63) { expdiff = 62;}
+ if (expa > expb) {
+ exp = expa + 1;
+ expa = 1;
+ expb = expdiff + 1;
+ } else {
+ exp = expb + 1;
+ expb = 1;
+ expa = expdiff + 1;
+ }
+ mant = (manta>>expa) + (mantb>>expb);
+
+ hi = (int) (mant>>32);
+ lo = (int) (mant);
+
+ k = HEXAGON_R_normamt_R(hi);
+ if(hi == 0 || hi == -1) k = 31+HEXAGON_R_normamt_R(lo);
+
+ mant = (mant << k);
+ cn = (mant == 0x8000000000000000LL);
+ exp = exp - k + cn;
+
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global dadd_asm
+ .type dadd_asm, @function
+dadd_asm:
+
+#define manta R0
+#define mantexpa R1:0
+#define lmanta R1:0
+#define mantb R2
+#define mantexpb R3:2
+#define lmantb R3:2
+#define expa R4
+#define expb R5
+#define mantexpd R7:6
+#define expd R6
+#define exp R8
+#define c63 R9
+#define lmant R1:0
+#define manth R1
+#define mantl R0
+#define zero R7:6
+#define zerol R6
+#define minus R3:2
+#define minusl R2
+#define maxneg R9
+#define minmin R11:10 // exactly 0x800000000000000000LL
+#define minminh R11
+#define k R4
+#define kl R5
+#define ce P0
+ .falign
+ {
+ mantexpd = VABSDIFFH(mantexpa, mantexpb) //represented as 0x08001LL
+ c63 = #62
+ expa = SXTH(manta)
+ expb = SXTH(mantb)
+ } {
+ expd = SXTH(expd)
+ ce = CMP.GT(expa, expb);
+ if ( ce.new) exp = add(expa, #1)
+ if (!ce.new) exp = add(expb, #1)
+ } {
+ if ( ce) expa = #1
+ if (!ce) expb = #1
+ manta.L = #0
+ expd = MIN(expd, c63)
+ } {
+ if (!ce) expa = add(expd, #1)
+ if ( ce) expb = add(expd, #1)
+ mantb.L = #0
+ zero = #0
+ } {
+ lmanta = ASR(lmanta, expa)
+ lmantb = ASR(lmantb, expb)
+ minmin = #0
+ } {
+ lmant = add(lmanta, lmantb)
+ minus = #-1
+ minminh.H = #0x8000
+ } {
+ k = NORMAMT(manth)
+ kl = NORMAMT(mantl)
+ p0 = cmp.eq(manth, zerol)
+ p1 = cmp.eq(manth, minusl)
+ } {
+ p0 = OR(p0, p1)
+ if(p0.new) k = add(kl, #31)
+ maxneg.H = #0
+ } {
+ mantexpa = ASL(lmant, k)
+ exp = SUB(exp, k)
+ maxneg.L = #0x8001
+ } {
+ p0 = cmp.eq(mantexpa, zero)
+ p1 = cmp.eq(mantexpa, minus)
+ manta.L = #0
+ exp = ZXTH(exp)
+ } {
+ p2 = cmp.eq(mantexpa, minmin) //is result 0x80....0
+ if(p2.new) exp = add(exp, #1)
+ }
+#if (__HEXAGON_ARCH__ == 60)
+ {
+ p0 = OR(p0, p1)
+ if( p0.new) manta = OR(manta,maxneg)
+ if(!p0.new) manta = OR(manta,exp)
+ }
+ jumpr r31
+#else
+ {
+ p0 = OR(p0, p1)
+ if( p0.new) manta = OR(manta,maxneg)
+ if(!p0.new) manta = OR(manta,exp)
+ jumpr r31
+ }
+#endif
+/* =================================================================== *
+ QDOUBLE dsub(QDOUBLE a,QDOUBLE b) {
+ QDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = HEXAGON_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = HEXAGON_R_sxth_R(b) ;
+ int exp, expdiff, j, k, hi, lo, cn;
+ lint mant;
+
+ expdiff = (int) HEXAGON_P_vabsdiffh_PP(a, b);
+ expdiff = HEXAGON_R_sxth_R(expdiff) ;
+ if (expdiff > 63) { expdiff = 62;}
+ if (expa > expb) {
+ exp = expa + 1;
+ expa = 1;
+ expb = expdiff + 1;
+ } else {
+ exp = expb + 1;
+ expb = 1;
+ expa = expdiff + 1;
+ }
+ mant = (manta>>expa) - (mantb>>expb);
+
+ hi = (int) (mant>>32);
+ lo = (int) (mant);
+
+ k = HEXAGON_R_normamt_R(hi);
+ if(hi == 0 || hi == -1) k = 31+HEXAGON_R_normamt_R(lo);
+
+ mant = (mant << k);
+ cn = (mant == 0x8000000000000000LL);
+ exp = exp - k + cn;
+
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global dsub_asm
+ .type dsub_asm, @function
+dsub_asm:
+
+#define manta R0
+#define mantexpa R1:0
+#define lmanta R1:0
+#define mantb R2
+#define mantexpb R3:2
+#define lmantb R3:2
+#define expa R4
+#define expb R5
+#define mantexpd R7:6
+#define expd R6
+#define exp R8
+#define c63 R9
+#define lmant R1:0
+#define manth R1
+#define mantl R0
+#define zero R7:6
+#define zerol R6
+#define minus R3:2
+#define minusl R2
+#define maxneg R9
+#define minmin R11:10 // exactly 0x800000000000000000LL
+#define minminh R11
+#define k R4
+#define kl R5
+#define ce P0
+ .falign
+ {
+ mantexpd = VABSDIFFH(mantexpa, mantexpb) //represented as 0x08001LL
+ c63 = #62
+ expa = SXTH(manta)
+ expb = SXTH(mantb)
+ } {
+ expd = SXTH(expd)
+ ce = CMP.GT(expa, expb);
+ if ( ce.new) exp = add(expa, #1)
+ if (!ce.new) exp = add(expb, #1)
+ } {
+ if ( ce) expa = #1
+ if (!ce) expb = #1
+ manta.L = #0
+ expd = MIN(expd, c63)
+ } {
+ if (!ce) expa = add(expd, #1)
+ if ( ce) expb = add(expd, #1)
+ mantb.L = #0
+ zero = #0
+ } {
+ lmanta = ASR(lmanta, expa)
+ lmantb = ASR(lmantb, expb)
+ minmin = #0
+ } {
+ lmant = sub(lmanta, lmantb)
+ minus = #-1
+ minminh.H = #0x8000
+ } {
+ k = NORMAMT(manth)
+ kl = NORMAMT(mantl)
+ p0 = cmp.eq(manth, zerol)
+ p1 = cmp.eq(manth, minusl)
+ } {
+ p0 = OR(p0, p1)
+ if(p0.new) k = add(kl, #31)
+ maxneg.H = #0
+ } {
+ mantexpa = ASL(lmant, k)
+ exp = SUB(exp, k)
+ maxneg.L = #0x8001
+ } {
+ p0 = cmp.eq(mantexpa, zero)
+ p1 = cmp.eq(mantexpa, minus)
+ manta.L = #0
+ exp = ZXTH(exp)
+ } {
+ p2 = cmp.eq(mantexpa, minmin) //is result 0x80....0
+ if(p2.new) exp = add(exp, #1)
+ }
+#if (__HEXAGON_ARCH__ == 60)
+ {
+ p0 = OR(p0, p1)
+ if( p0.new) manta = OR(manta,maxneg)
+ if(!p0.new) manta = OR(manta,exp)
+ }
+ jumpr r31
+#else
+ {
+ p0 = OR(p0, p1)
+ if( p0.new) manta = OR(manta,maxneg)
+ if(!p0.new) manta = OR(manta,exp)
+ jumpr r31
+ }
+#endif
+/* ==================================================================== *
+ QDOUBLE dmpy(QDOUBLE a,QDOUBLE b) {
+ QDOUBLE c;
+ lint manta = a & MANTMASK;
+ int expa = HEXAGON_R_sxth_R(a) ;
+ lint mantb = b & MANTMASK;
+ int expb = HEXAGON_R_sxth_R(b) ;
+ int exp, k;
+ lint mant;
+ int hia, hib, hi, lo;
+ unsigned int loa, lob;
+
+ hia = (int)(a >> 32);
+ loa = HEXAGON_R_extractu_RII((int)manta, 31, 1);
+ hib = (int)(b >> 32);
+ lob = HEXAGON_R_extractu_RII((int)mantb, 31, 1);
+
+ mant = HEXAGON_P_mpy_RR(hia, lob);
+ mant = HEXAGON_P_mpyacc_RR(mant,hib, loa);
+ mant = (mant >> 30) + (HEXAGON_P_mpy_RR(hia, hib)<<1);
+
+ hi = (int) (mant>>32);
+ lo = (int) (mant);
+
+ k = HEXAGON_R_normamt_R(hi);
+ if(hi == 0 || hi == -1) k = 31+HEXAGON_R_normamt_R(lo);
+ mant = mant << k;
+ exp = expa + expb - k;
+ if (mant == 0 || mant == -1) exp = 0x8001;
+ c = (mant & MANTMASK) | (((lint) exp) & EXP_MASK);
+ return(c);
+ }
+ * ==================================================================== */
+ .text
+ .global dmpy_asm
+ .type dmpy_asm, @function
+dmpy_asm:
+
+#define mantal R0
+#define mantah R1
+#define mantexpa R1:0
+#define mantbl R2
+#define mantbh R3
+#define mantexpb R3:2
+#define expa R4
+#define expb R5
+#define mantexpd R7:6
+#define exp R8
+#define lmantc R11:10
+#define mantch R11
+#define mantcl R10
+#define zero0 R7:6
+#define zero0l R6
+#define minus1 R3:2
+#define minus1l R2
+#define maxneg R9
+#define k R4
+#define kl R5
+
+ .falign
+ {
+ mantbl = lsr(mantbl, #16)
+ mantal = lsr(mantal, #16)
+ expa = sxth(mantal)
+ expb = sxth(mantbl)
+ }
+ {
+ lmantc = mpy(mantah, mantbh)
+ mantexpd = mpy(mantah, mantbl)
+ }
+ {
+ lmantc = add(lmantc, lmantc) //<<1
+ mantexpd+= mpy(mantbh, mantal)
+ }
+ {
+ lmantc += asr(mantexpd, #15)
+ exp = add(expa, expb)
+ zero0 = #0
+ minus1 = #-1
+ }
+ {
+ k = normamt(mantch)
+ kl = normamt(mantcl)
+ p0 = cmp.eq(mantch, zero0l)
+ p1 = cmp.eq(mantch, minus1l)
+ }
+ {
+ p0 = or(p0, p1)
+ if(p0.new) k = add(kl, #31)
+ maxneg.H = #0
+ }
+ {
+ mantexpa = asl(lmantc, k)
+ exp = sub(exp, k)
+ maxneg.L = #0x8001
+ }
+ {
+ p0 = cmp.eq(mantexpa, zero0)
+ p1 = cmp.eq(mantexpa, minus1)
+ mantal.L = #0
+ exp = zxth(exp)
+ }
+#if (__HEXAGON_ARCH__ == 60)
+ {
+ p0 = or(p0, p1)
+ if( p0.new) mantal = or(mantal,maxneg)
+ if(!p0.new) mantal = or(mantal,exp)
+ }
+ jumpr r31
+#else
+ {
+ p0 = or(p0, p1)
+ if( p0.new) mantal = or(mantal,maxneg)
+ if(!p0.new) mantal = or(mantal,exp)
+ jumpr r31
+ }
+#endif
diff --git a/lib/builtins/hexagon/fma_opt.S b/lib/builtins/hexagon/fma_opt.S
new file mode 100644
index 000000000000..12378f0da04e
--- /dev/null
+++ b/lib/builtins/hexagon/fma_opt.S
@@ -0,0 +1,31 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+.macro FUNCTION_BEGIN name
+.text
+.p2align 5
+.globl \name
+.type \name, @function
+\name:
+.endm
+
+.macro FUNCTION_END name
+.size \name, . - \name
+.endm
+
+FUNCTION_BEGIN fmaf
+ r2 += sfmpy(r0, r1)
+ {
+ r0 = r2
+ jumpr r31
+ }
+FUNCTION_END fmaf
+
+ .globl fmal
+ .set fmal, fma
diff --git a/lib/builtins/hexagon/fmax_opt.S b/lib/builtins/hexagon/fmax_opt.S
new file mode 100644
index 000000000000..f3a218c9769b
--- /dev/null
+++ b/lib/builtins/hexagon/fmax_opt.S
@@ -0,0 +1,30 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+.macro FUNCTION_BEGIN name
+.text
+.p2align 5
+.globl \name
+.type \name, @function
+\name:
+.endm
+
+.macro FUNCTION_END name
+.size \name, . - \name
+.endm
+
+FUNCTION_BEGIN fmaxf
+ {
+ r0 = sfmax(r0, r1)
+ jumpr r31
+ }
+FUNCTION_END fmaxf
+
+ .globl fmaxl
+ .set fmaxl, fmax
diff --git a/lib/builtins/hexagon/fmin_opt.S b/lib/builtins/hexagon/fmin_opt.S
new file mode 100644
index 000000000000..ef9b0ff854a2
--- /dev/null
+++ b/lib/builtins/hexagon/fmin_opt.S
@@ -0,0 +1,30 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+.macro FUNCTION_BEGIN name
+.text
+.p2align 5
+.globl \name
+.type \name, @function
+\name:
+.endm
+
+.macro FUNCTION_END name
+.size \name, . - \name
+.endm
+
+FUNCTION_BEGIN fminf
+ {
+ r0 = sfmin(r0, r1)
+ jumpr r31
+ }
+FUNCTION_END fminf
+
+ .globl fminl
+ .set fminl, fmin
diff --git a/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S b/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S
new file mode 100644
index 000000000000..fbe09086cd33
--- /dev/null
+++ b/lib/builtins/hexagon/memcpy_forward_vp4cp4n2.S
@@ -0,0 +1,125 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// An optimized version of a memcpy which is equivalent to the following loop:
+//
+// volatile unsigned *dest;
+// unsigned *src;
+//
+// for (i = 0; i < num_words; ++i)
+// *dest++ = *src++;
+//
+// The corresponding C prototype for this function would be
+// void hexagon_memcpy_forward_vp4cp4n2(volatile unsigned *dest,
+// const unsigned *src,
+// unsigned num_words);
+//
+// *** Both dest and src must be aligned to 32-bit boundaries. ***
+// The code does not perform any runtime checks for this, and will fail
+// in bad ways if this requirement is not met.
+//
+// The "forward" in the name refers to the fact that the function copies
+// the words going forward in memory. It is incorrect to use this function
+// for cases where the original code copied words in any other order.
+//
+// *** This function is only for the use by the compiler. ***
+// The only indended use is for the LLVM compiler to generate calls to
+// this function, when a mem-copy loop, like the one above, is detected.
+
+ .text
+
+// Inputs:
+// r0: dest
+// r1: src
+// r2: num_words
+
+ .globl hexagon_memcpy_forward_vp4cp4n2
+ .balign 32
+ .type hexagon_memcpy_forward_vp4cp4n2,@function
+hexagon_memcpy_forward_vp4cp4n2:
+
+ // Compute r3 to be the number of words remaining in the current page.
+ // At the same time, compute r4 to be the number of 32-byte blocks
+ // remaining in the page (for prefetch).
+ {
+ r3 = sub(##4096, r1)
+ r5 = lsr(r2, #3)
+ }
+ {
+ // The word count before end-of-page is in the 12 lowest bits of r3.
+ // (If the address in r1 was already page-aligned, the bits are 0.)
+ r3 = extractu(r3, #10, #2)
+ r4 = extractu(r3, #7, #5)
+ }
+ {
+ r3 = minu(r2, r3)
+ r4 = minu(r5, r4)
+ }
+ {
+ r4 = or(r4, ##2105344) // 2105344 = 0x202000
+ p0 = cmp.eq(r3, #0)
+ if (p0.new) jump:nt .Lskipprolog
+ }
+ l2fetch(r1, r4)
+ {
+ loop0(.Lprolog, r3)
+ r2 = sub(r2, r3) // r2 = number of words left after the prolog.
+ }
+ .falign
+.Lprolog:
+ {
+ r4 = memw(r1++#4)
+ memw(r0++#4) = r4.new
+ } :endloop0
+.Lskipprolog:
+ {
+ // Let r3 = number of whole pages left (page = 1024 words).
+ r3 = lsr(r2, #10)
+ if (cmp.eq(r3.new, #0)) jump:nt .Lskipmain
+ }
+ {
+ loop1(.Lout, r3)
+ r2 = extractu(r2, #10, #0) // r2 = r2 & 1023
+ r3 = ##2105472 // r3 = 0x202080 (prefetch info)
+ }
+ // Iterate over pages.
+ .falign
+.Lout:
+ // Prefetch each individual page.
+ l2fetch(r1, r3)
+ loop0(.Lpage, #512)
+ .falign
+.Lpage:
+ r5:4 = memd(r1++#8)
+ {
+ memw(r0++#8) = r4
+ memw(r0+#4) = r5
+ } :endloop0:endloop1
+.Lskipmain:
+ {
+ r3 = ##2105344 // r3 = 0x202000 (prefetch info)
+ r4 = lsr(r2, #3) // r4 = number of 32-byte blocks remaining.
+ p0 = cmp.eq(r2, #0)
+ if (p0.new) jumpr:nt r31
+ }
+ {
+ r3 = or(r3, r4)
+ loop0(.Lepilog, r2)
+ }
+ l2fetch(r1, r3)
+ .falign
+.Lepilog:
+ {
+ r4 = memw(r1++#4)
+ memw(r0++#4) = r4.new
+ } :endloop0
+
+ jumpr r31
+
+.size hexagon_memcpy_forward_vp4cp4n2, . - hexagon_memcpy_forward_vp4cp4n2
diff --git a/lib/builtins/hexagon/memcpy_likely_aligned.S b/lib/builtins/hexagon/memcpy_likely_aligned.S
new file mode 100644
index 000000000000..bbc85c22db08
--- /dev/null
+++ b/lib/builtins/hexagon/memcpy_likely_aligned.S
@@ -0,0 +1,64 @@
+//===------------------------- memcopy routines ---------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+FUNCTION_BEGIN __hexagon_memcpy_likely_aligned_min32bytes_mult8bytes
+ {
+ p0 = bitsclr(r1,#7)
+ p0 = bitsclr(r0,#7)
+ if (p0.new) r5:4 = memd(r1)
+ r3 = #-3
+ }
+ {
+ if (!p0) jump .Lmemcpy_call
+ if (p0) memd(r0++#8) = r5:4
+ if (p0) r5:4 = memd(r1+#8)
+ r3 += lsr(r2,#3)
+ }
+ {
+ memd(r0++#8) = r5:4
+ r5:4 = memd(r1+#16)
+ r1 = add(r1,#24)
+ loop0(1f,r3)
+ }
+ .falign
+1:
+ {
+ memd(r0++#8) = r5:4
+ r5:4 = memd(r1++#8)
+ }:endloop0
+ {
+ memd(r0) = r5:4
+ r0 -= add(r2,#-8)
+ jumpr r31
+ }
+FUNCTION_END __hexagon_memcpy_likely_aligned_min32bytes_mult8bytes
+
+.Lmemcpy_call:
+#ifdef __PIC__
+ jump memcpy@PLT
+#else
+ jump memcpy
+#endif
+
+ .globl __qdsp_memcpy_likely_aligned_min32bytes_mult8bytes
+ .set __qdsp_memcpy_likely_aligned_min32bytes_mult8bytes, \
+ __hexagon_memcpy_likely_aligned_min32bytes_mult8bytes
diff --git a/lib/builtins/hexagon/moddi3.S b/lib/builtins/hexagon/moddi3.S
new file mode 100644
index 000000000000..12a0595fe465
--- /dev/null
+++ b/lib/builtins/hexagon/moddi3.S
@@ -0,0 +1,83 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_moddi3
+ {
+ p3 = tstbit(r1,#31)
+ }
+ {
+ r1:0 = abs(r1:0)
+ r3:2 = abs(r3:2)
+ }
+ {
+ r6 = cl0(r1:0) // count leading 0's of dividend (numerator)
+ r7 = cl0(r3:2) // count leading 0's of divisor (denominator)
+ r5:4 = r3:2 // divisor moved into working registers
+ r3:2 = r1:0 // dividend is the initial remainder, r3:2 contains remainder
+ }
+ {
+ r10 = sub(r7,r6) // left shift count for bit & divisor
+ r1:0 = #0 // initialize quotient to 0
+ r15:14 = #1 // initialize bit to 1
+ }
+ {
+ r11 = add(r10,#1) // loop count is 1 more than shift count
+ r13:12 = lsl(r5:4,r10) // shift divisor msb into same bit position as dividend msb
+ r15:14 = lsl(r15:14,r10) // shift the bit left by same amount as divisor
+ }
+ {
+ p0 = cmp.gtu(r5:4,r3:2) // check if divisor > dividend
+ loop0(1f,r11) // register loop
+ }
+ {
+ if (p0) jump .hexagon_moddi3_return // if divisor > dividend, we're done, so return
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r13:12,r3:2) // set predicate reg if shifted divisor > current remainder
+ }
+ {
+ r7:6 = sub(r3:2, r13:12) // subtract shifted divisor from current remainder
+ r9:8 = add(r1:0, r15:14) // save current quotient to temp (r9:8)
+ }
+ {
+ r1:0 = vmux(p0, r1:0, r9:8) // choose either current quotient or new quotient (r9:8)
+ r3:2 = vmux(p0, r3:2, r7:6) // choose either current remainder or new remainder (r7:6)
+ }
+ {
+ r15:14 = lsr(r15:14, #1) // shift bit right by 1 for next iteration
+ r13:12 = lsr(r13:12, #1) // shift "shifted divisor" right by 1 for next iteration
+ }:endloop0
+
+.hexagon_moddi3_return:
+ {
+ r1:0 = neg(r3:2)
+ }
+ {
+ r1:0 = vmux(p3,r1:0,r3:2)
+ jumpr r31
+ }
+FUNCTION_END __hexagon_moddi3
+
+ .globl __qdsp_moddi3
+ .set __qdsp_moddi3, __hexagon_moddi3
diff --git a/lib/builtins/hexagon/modsi3.S b/lib/builtins/hexagon/modsi3.S
new file mode 100644
index 000000000000..5afda9e2978b
--- /dev/null
+++ b/lib/builtins/hexagon/modsi3.S
@@ -0,0 +1,66 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_modsi3
+ {
+ p2 = cmp.ge(r0,#0)
+ r2 = abs(r0)
+ r1 = abs(r1)
+ }
+ {
+ r3 = cl0(r2)
+ r4 = cl0(r1)
+ p0 = cmp.gtu(r1,r2)
+ }
+ {
+ r3 = sub(r4,r3)
+ if (p0) jumpr r31
+ }
+ {
+ p1 = cmp.eq(r3,#0)
+ loop0(1f,r3)
+ r0 = r2
+ r2 = lsl(r1,r3)
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r2,r0)
+ if (!p0.new) r0 = sub(r0,r2)
+ r2 = lsr(r2,#1)
+ if (p1) r1 = #0
+ }:endloop0
+ {
+ p0 = cmp.gtu(r2,r0)
+ if (!p0.new) r0 = sub(r0,r1)
+ if (p2) jumpr r31
+ }
+ {
+ r0 = neg(r0)
+ jumpr r31
+ }
+FUNCTION_END __hexagon_modsi3
+
+ .globl __qdsp_modsi3
+ .set __qdsp_modsi3, __hexagon_modsi3
diff --git a/lib/builtins/hexagon/sfdiv_opt.S b/lib/builtins/hexagon/sfdiv_opt.S
new file mode 100644
index 000000000000..6bdd4808c2b8
--- /dev/null
+++ b/lib/builtins/hexagon/sfdiv_opt.S
@@ -0,0 +1,66 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG
+#define FAST_ALIAS(TAG) .global __hexagon_fast_##TAG ; .set __hexagon_fast_##TAG, __hexagon_##TAG
+#define FAST2_ALIAS(TAG) .global __hexagon_fast2_##TAG ; .set __hexagon_fast2_##TAG, __hexagon_##TAG
+
+FUNCTION_BEGIN __hexagon_divsf3
+ {
+ r2,p0 = sfrecipa(r0,r1)
+ r4 = sffixupd(r0,r1)
+ r3 = ##0x3f800000 // 1.0
+ }
+ {
+ r5 = sffixupn(r0,r1)
+ r3 -= sfmpy(r4,r2):lib // 1-(den/recip) yields error?
+ r6 = ##0x80000000
+ r7 = r3
+ }
+ {
+ r2 += sfmpy(r3,r2):lib
+ r3 = r7
+ r6 = r5
+ r0 = and(r6,r5)
+ }
+ {
+ r3 -= sfmpy(r4,r2):lib
+ r0 += sfmpy(r5,r2):lib
+ }
+ {
+ r2 += sfmpy(r3,r2):lib
+ r6 -= sfmpy(r0,r4):lib
+ }
+ {
+ r0 += sfmpy(r6,r2):lib
+ }
+ {
+ r5 -= sfmpy(r0,r4):lib
+ }
+ {
+ r0 += sfmpy(r5,r2,p0):scale
+ jumpr r31
+ }
+FUNCTION_END __hexagon_divsf3
+
+Q6_ALIAS(divsf3)
+FAST_ALIAS(divsf3)
+FAST2_ALIAS(divsf3)
diff --git a/lib/builtins/hexagon/sfsqrt_opt.S b/lib/builtins/hexagon/sfsqrt_opt.S
new file mode 100644
index 000000000000..7f619002774f
--- /dev/null
+++ b/lib/builtins/hexagon/sfsqrt_opt.S
@@ -0,0 +1,82 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+#define RIN r0
+#define S r0
+#define H r1
+#define D r2
+#define E r3
+#define HALF r4
+#define R r5
+
+#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG
+#define FAST_ALIAS(TAG) .global __hexagon_fast_##TAG ; .set __hexagon_fast_##TAG, __hexagon_##TAG
+#define FAST2_ALIAS(TAG) .global __hexagon_fast2_##TAG ; .set __hexagon_fast2_##TAG, __hexagon_##TAG
+
+FUNCTION_BEGIN __hexagon_sqrtf
+ {
+ E,p0 = sfinvsqrta(RIN)
+ R = sffixupr(RIN)
+ HALF = ##0x3f000000 // 0.5
+ r1:0 = combine(#0,#0) // clear S/H
+ }
+ {
+ S += sfmpy(E,R):lib // S0
+ H += sfmpy(E,HALF):lib // H0
+ D = HALF
+ E = R
+ }
+ {
+ D -= sfmpy(S,H):lib // d0
+ p1 = sfclass(R,#1) // is zero?
+ //E -= sfmpy(S,S):lib // e0
+ }
+ {
+ S += sfmpy(S,D):lib // S1
+ H += sfmpy(H,D):lib // H1
+ D = HALF
+ E = R
+ }
+ {
+ D -= sfmpy(S,H):lib // d0
+ E -= sfmpy(S,S):lib // e0
+ }
+ {
+ S += sfmpy(H,E):lib // S2
+ H += sfmpy(H,D):lib // H2
+ D = HALF
+ E = R
+ }
+ {
+ //D -= sfmpy(S,H):lib // d2
+ E -= sfmpy(S,S):lib // e2
+ if (p1) r0 = or(r0,R) // sqrt(-0.0) = -0.0
+ }
+ {
+ S += sfmpy(H,E,p0):scale // S3
+ jumpr r31
+ }
+
+FUNCTION_END __hexagon_sqrtf
+
+Q6_ALIAS(sqrtf)
+FAST_ALIAS(sqrtf)
+FAST2_ALIAS(sqrtf)
diff --git a/lib/builtins/hexagon/udivdi3.S b/lib/builtins/hexagon/udivdi3.S
new file mode 100644
index 000000000000..1ca326b75208
--- /dev/null
+++ b/lib/builtins/hexagon/udivdi3.S
@@ -0,0 +1,71 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_udivdi3
+ {
+ r6 = cl0(r1:0) // count leading 0's of dividend (numerator)
+ r7 = cl0(r3:2) // count leading 0's of divisor (denominator)
+ r5:4 = r3:2 // divisor moved into working registers
+ r3:2 = r1:0 // dividend is the initial remainder, r3:2 contains remainder
+ }
+ {
+ r10 = sub(r7,r6) // left shift count for bit & divisor
+ r1:0 = #0 // initialize quotient to 0
+ r15:14 = #1 // initialize bit to 1
+ }
+ {
+ r11 = add(r10,#1) // loop count is 1 more than shift count
+ r13:12 = lsl(r5:4,r10) // shift divisor msb into same bit position as dividend msb
+ r15:14 = lsl(r15:14,r10) // shift the bit left by same amount as divisor
+ }
+ {
+ p0 = cmp.gtu(r5:4,r3:2) // check if divisor > dividend
+ loop0(1f,r11) // register loop
+ }
+ {
+ if (p0) jumpr r31 // if divisor > dividend, we're done, so return
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r13:12,r3:2) // set predicate reg if shifted divisor > current remainder
+ }
+ {
+ r7:6 = sub(r3:2, r13:12) // subtract shifted divisor from current remainder
+ r9:8 = add(r1:0, r15:14) // save current quotient to temp (r9:8)
+ }
+ {
+ r1:0 = vmux(p0, r1:0, r9:8) // choose either current quotient or new quotient (r9:8)
+ r3:2 = vmux(p0, r3:2, r7:6) // choose either current remainder or new remainder (r7:6)
+ }
+ {
+ r15:14 = lsr(r15:14, #1) // shift bit right by 1 for next iteration
+ r13:12 = lsr(r13:12, #1) // shift "shifted divisor" right by 1 for next iteration
+ }:endloop0
+ {
+ jumpr r31 // return
+ }
+FUNCTION_END __hexagon_udivdi3
+
+ .globl __qdsp_udivdi3
+ .set __qdsp_udivdi3, __hexagon_udivdi3
diff --git a/lib/builtins/hexagon/udivmoddi4.S b/lib/builtins/hexagon/udivmoddi4.S
new file mode 100644
index 000000000000..deb5aae0924d
--- /dev/null
+++ b/lib/builtins/hexagon/udivmoddi4.S
@@ -0,0 +1,71 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_udivmoddi4
+ {
+ r6 = cl0(r1:0) // count leading 0's of dividend (numerator)
+ r7 = cl0(r3:2) // count leading 0's of divisor (denominator)
+ r5:4 = r3:2 // divisor moved into working registers
+ r3:2 = r1:0 // dividend is the initial remainder, r3:2 contains remainder
+ }
+ {
+ r10 = sub(r7,r6) // left shift count for bit & divisor
+ r1:0 = #0 // initialize quotient to 0
+ r15:14 = #1 // initialize bit to 1
+ }
+ {
+ r11 = add(r10,#1) // loop count is 1 more than shift count
+ r13:12 = lsl(r5:4,r10) // shift divisor msb into same bit position as dividend msb
+ r15:14 = lsl(r15:14,r10) // shift the bit left by same amount as divisor
+ }
+ {
+ p0 = cmp.gtu(r5:4,r3:2) // check if divisor > dividend
+ loop0(1f,r11) // register loop
+ }
+ {
+ if (p0) jumpr r31 // if divisor > dividend, we're done, so return
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r13:12,r3:2) // set predicate reg if shifted divisor > current remainder
+ }
+ {
+ r7:6 = sub(r3:2, r13:12) // subtract shifted divisor from current remainder
+ r9:8 = add(r1:0, r15:14) // save current quotient to temp (r9:8)
+ }
+ {
+ r1:0 = vmux(p0, r1:0, r9:8) // choose either current quotient or new quotient (r9:8)
+ r3:2 = vmux(p0, r3:2, r7:6) // choose either current remainder or new remainder (r7:6)
+ }
+ {
+ r15:14 = lsr(r15:14, #1) // shift bit right by 1 for next iteration
+ r13:12 = lsr(r13:12, #1) // shift "shifted divisor" right by 1 for next iteration
+ }:endloop0
+ {
+ jumpr r31 // return
+ }
+FUNCTION_END __hexagon_udivmoddi4
+
+ .globl __qdsp_udivmoddi4
+ .set __qdsp_udivmoddi4, __hexagon_udivmoddi4
diff --git a/lib/builtins/hexagon/udivmodsi4.S b/lib/builtins/hexagon/udivmodsi4.S
new file mode 100644
index 000000000000..25bbe7cd5925
--- /dev/null
+++ b/lib/builtins/hexagon/udivmodsi4.S
@@ -0,0 +1,60 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_udivmodsi4
+ {
+ r2 = cl0(r0)
+ r3 = cl0(r1)
+ r5:4 = combine(#1,#0)
+ p0 = cmp.gtu(r1,r0)
+ }
+ {
+ r6 = sub(r3,r2)
+ r4 = r1
+ r1:0 = combine(r0,r4)
+ if (p0) jumpr r31
+ }
+ {
+ r3:2 = vlslw(r5:4,r6)
+ loop0(1f,r6)
+ p0 = cmp.eq(r6,#0)
+ if (p0.new) r4 = #0
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r2,r1)
+ if (!p0.new) r1 = sub(r1,r2)
+ if (!p0.new) r0 = add(r0,r3)
+ r3:2 = vlsrw(r3:2,#1)
+ }:endloop0
+ {
+ p0 = cmp.gtu(r2,r1)
+ if (!p0.new) r1 = sub(r1,r4)
+ if (!p0.new) r0 = add(r0,r3)
+ jumpr r31
+ }
+FUNCTION_END __hexagon_udivmodsi4
+
+ .globl __qdsp_udivmodsi4
+ .set __qdsp_udivmodsi4, __hexagon_udivmodsi4
diff --git a/lib/builtins/hexagon/udivsi3.S b/lib/builtins/hexagon/udivsi3.S
new file mode 100644
index 000000000000..54f0aa409f93
--- /dev/null
+++ b/lib/builtins/hexagon/udivsi3.S
@@ -0,0 +1,56 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_udivsi3
+ {
+ r2 = cl0(r0)
+ r3 = cl0(r1)
+ r5:4 = combine(#1,#0)
+ p0 = cmp.gtu(r1,r0)
+ }
+ {
+ r6 = sub(r3,r2)
+ r4 = r1
+ r1:0 = combine(r0,r4)
+ if (p0) jumpr r31
+ }
+ {
+ r3:2 = vlslw(r5:4,r6)
+ loop0(1f,r6)
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r2,r1)
+ if (!p0.new) r1 = sub(r1,r2)
+ if (!p0.new) r0 = add(r0,r3)
+ r3:2 = vlsrw(r3:2,#1)
+ }:endloop0
+ {
+ p0 = cmp.gtu(r2,r1)
+ if (!p0.new) r0 = add(r0,r3)
+ jumpr r31
+ }
+FUNCTION_END __hexagon_udivsi3
+
+ .globl __qdsp_udivsi3
+ .set __qdsp_udivsi3, __hexagon_udivsi3
diff --git a/lib/builtins/hexagon/umoddi3.S b/lib/builtins/hexagon/umoddi3.S
new file mode 100644
index 000000000000..f091521414af
--- /dev/null
+++ b/lib/builtins/hexagon/umoddi3.S
@@ -0,0 +1,74 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_umoddi3
+ {
+ r6 = cl0(r1:0) // count leading 0's of dividend (numerator)
+ r7 = cl0(r3:2) // count leading 0's of divisor (denominator)
+ r5:4 = r3:2 // divisor moved into working registers
+ r3:2 = r1:0 // dividend is the initial remainder, r3:2 contains remainder
+ }
+ {
+ r10 = sub(r7,r6) // left shift count for bit & divisor
+ r1:0 = #0 // initialize quotient to 0
+ r15:14 = #1 // initialize bit to 1
+ }
+ {
+ r11 = add(r10,#1) // loop count is 1 more than shift count
+ r13:12 = lsl(r5:4,r10) // shift divisor msb into same bit position as dividend msb
+ r15:14 = lsl(r15:14,r10) // shift the bit left by same amount as divisor
+ }
+ {
+ p0 = cmp.gtu(r5:4,r3:2) // check if divisor > dividend
+ loop0(1f,r11) // register loop
+ }
+ {
+ if (p0) jump .hexagon_umoddi3_return // if divisor > dividend, we're done, so return
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r13:12,r3:2) // set predicate reg if shifted divisor > current remainder
+ }
+ {
+ r7:6 = sub(r3:2, r13:12) // subtract shifted divisor from current remainder
+ r9:8 = add(r1:0, r15:14) // save current quotient to temp (r9:8)
+ }
+ {
+ r1:0 = vmux(p0, r1:0, r9:8) // choose either current quotient or new quotient (r9:8)
+ r3:2 = vmux(p0, r3:2, r7:6) // choose either current remainder or new remainder (r7:6)
+ }
+ {
+ r15:14 = lsr(r15:14, #1) // shift bit right by 1 for next iteration
+ r13:12 = lsr(r13:12, #1) // shift "shifted divisor" right by 1 for next iteration
+ }:endloop0
+
+.hexagon_umoddi3_return:
+ {
+ r1:0 = r3:2
+ jumpr r31
+ }
+FUNCTION_END __hexagon_umoddi3
+
+ .globl __qdsp_umoddi3
+ .set __qdsp_umoddi3, __hexagon_umoddi3
diff --git a/lib/builtins/hexagon/umodsi3.S b/lib/builtins/hexagon/umodsi3.S
new file mode 100644
index 000000000000..a8270c2030d5
--- /dev/null
+++ b/lib/builtins/hexagon/umodsi3.S
@@ -0,0 +1,55 @@
+//===----------------------Hexagon builtin routine ------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+ .macro FUNCTION_BEGIN name
+ .text
+ .p2align 5
+ .globl \name
+ .type \name, @function
+\name:
+ .endm
+
+ .macro FUNCTION_END name
+ .size \name, . - \name
+ .endm
+
+
+FUNCTION_BEGIN __hexagon_umodsi3
+ {
+ r2 = cl0(r0)
+ r3 = cl0(r1)
+ p0 = cmp.gtu(r1,r0)
+ }
+ {
+ r2 = sub(r3,r2)
+ if (p0) jumpr r31
+ }
+ {
+ loop0(1f,r2)
+ p1 = cmp.eq(r2,#0)
+ r2 = lsl(r1,r2)
+ }
+ .falign
+1:
+ {
+ p0 = cmp.gtu(r2,r0)
+ if (!p0.new) r0 = sub(r0,r2)
+ r2 = lsr(r2,#1)
+ if (p1) r1 = #0
+ }:endloop0
+ {
+ p0 = cmp.gtu(r2,r0)
+ if (!p0.new) r0 = sub(r0,r1)
+ jumpr r31
+ }
+FUNCTION_END __hexagon_umodsi3
+
+ .globl __qdsp_umodsi3
+ .set __qdsp_umodsi3, __hexagon_umodsi3
diff --git a/lib/builtins/int_types.h b/lib/builtins/int_types.h
index a92238c5b730..f53f343d35d5 100644
--- a/lib/builtins/int_types.h
+++ b/lib/builtins/int_types.h
@@ -60,7 +60,7 @@ typedef union
}s;
} udwords;
-#if (defined(__LP64__) || defined(__wasm__) || defined(__mips64))
+#if (defined(__LP64__) || defined(__wasm__) || defined(__mips64)) || defined(__riscv)
#define CRT_HAS_128BIT
#endif
diff --git a/lib/builtins/os_version_check.c b/lib/builtins/os_version_check.c
index 74ade2f5b966..772e33333c0f 100644
--- a/lib/builtins/os_version_check.c
+++ b/lib/builtins/os_version_check.c
@@ -16,8 +16,8 @@
#ifdef __APPLE__
#include <CoreFoundation/CoreFoundation.h>
-#include <dispatch/dispatch.h>
#include <TargetConditionals.h>
+#include <dispatch/dispatch.h>
#include <dlfcn.h>
#include <stdint.h>
#include <stdio.h>
@@ -28,6 +28,26 @@
static int32_t GlobalMajor, GlobalMinor, GlobalSubminor;
static dispatch_once_t DispatchOnceCounter;
+typedef CFDataRef (*CFDataCreateWithBytesNoCopyFuncTy)(CFAllocatorRef,
+ const UInt8 *, CFIndex,
+ CFAllocatorRef);
+typedef CFPropertyListRef (*CFPropertyListCreateWithDataFuncTy)(
+ CFAllocatorRef, CFDataRef, CFOptionFlags, CFPropertyListFormat *,
+ CFErrorRef *);
+typedef CFPropertyListRef (*CFPropertyListCreateFromXMLDataFuncTy)(
+ CFAllocatorRef, CFDataRef, CFOptionFlags, CFStringRef *);
+typedef CFStringRef (*CFStringCreateWithCStringNoCopyFuncTy)(CFAllocatorRef,
+ const char *,
+ CFStringEncoding,
+ CFAllocatorRef);
+typedef const void *(*CFDictionaryGetValueFuncTy)(CFDictionaryRef,
+ const void *);
+typedef CFTypeID (*CFGetTypeIDFuncTy)(CFTypeRef);
+typedef CFTypeID (*CFStringGetTypeIDFuncTy)(void);
+typedef Boolean (*CFStringGetCStringFuncTy)(CFStringRef, char *, CFIndex,
+ CFStringEncoding);
+typedef void (*CFReleaseFuncTy)(CFTypeRef);
+
/* Find and parse the SystemVersion.plist file. */
static void parseSystemVersionPList(void *Unused) {
(void)Unused;
@@ -37,50 +57,49 @@ static void parseSystemVersionPList(void *Unused) {
return;
const CFAllocatorRef kCFAllocatorNull =
*(const CFAllocatorRef *)NullAllocator;
- typeof(CFDataCreateWithBytesNoCopy) *CFDataCreateWithBytesNoCopyFunc =
- (typeof(CFDataCreateWithBytesNoCopy) *)dlsym(
- RTLD_DEFAULT, "CFDataCreateWithBytesNoCopy");
+ CFDataCreateWithBytesNoCopyFuncTy CFDataCreateWithBytesNoCopyFunc =
+ (CFDataCreateWithBytesNoCopyFuncTy)dlsym(RTLD_DEFAULT,
+ "CFDataCreateWithBytesNoCopy");
if (!CFDataCreateWithBytesNoCopyFunc)
return;
- typeof(CFPropertyListCreateWithData) *CFPropertyListCreateWithDataFunc =
- (typeof(CFPropertyListCreateWithData) *)dlsym(
+ CFPropertyListCreateWithDataFuncTy CFPropertyListCreateWithDataFunc =
+ (CFPropertyListCreateWithDataFuncTy)dlsym(
RTLD_DEFAULT, "CFPropertyListCreateWithData");
- /* CFPropertyListCreateWithData was introduced only in macOS 10.6+, so it
- * will be NULL on earlier OS versions. */
+/* CFPropertyListCreateWithData was introduced only in macOS 10.6+, so it
+ * will be NULL on earlier OS versions. */
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
- typeof(CFPropertyListCreateFromXMLData) *CFPropertyListCreateFromXMLDataFunc =
- (typeof(CFPropertyListCreateFromXMLData) *)dlsym(
+ CFPropertyListCreateFromXMLDataFuncTy CFPropertyListCreateFromXMLDataFunc =
+ (CFPropertyListCreateFromXMLDataFuncTy)dlsym(
RTLD_DEFAULT, "CFPropertyListCreateFromXMLData");
#pragma clang diagnostic pop
/* CFPropertyListCreateFromXMLDataFunc is deprecated in macOS 10.10, so it
* might be NULL in future OS versions. */
if (!CFPropertyListCreateWithDataFunc && !CFPropertyListCreateFromXMLDataFunc)
return;
- typeof(CFStringCreateWithCStringNoCopy) *CFStringCreateWithCStringNoCopyFunc =
- (typeof(CFStringCreateWithCStringNoCopy) *)dlsym(
+ CFStringCreateWithCStringNoCopyFuncTy CFStringCreateWithCStringNoCopyFunc =
+ (CFStringCreateWithCStringNoCopyFuncTy)dlsym(
RTLD_DEFAULT, "CFStringCreateWithCStringNoCopy");
if (!CFStringCreateWithCStringNoCopyFunc)
return;
- typeof(CFDictionaryGetValue) *CFDictionaryGetValueFunc =
- (typeof(CFDictionaryGetValue) *)dlsym(RTLD_DEFAULT,
- "CFDictionaryGetValue");
+ CFDictionaryGetValueFuncTy CFDictionaryGetValueFunc =
+ (CFDictionaryGetValueFuncTy)dlsym(RTLD_DEFAULT, "CFDictionaryGetValue");
if (!CFDictionaryGetValueFunc)
return;
- typeof(CFGetTypeID) *CFGetTypeIDFunc =
- (typeof(CFGetTypeID) *)dlsym(RTLD_DEFAULT, "CFGetTypeID");
+ CFGetTypeIDFuncTy CFGetTypeIDFunc =
+ (CFGetTypeIDFuncTy)dlsym(RTLD_DEFAULT, "CFGetTypeID");
if (!CFGetTypeIDFunc)
return;
- typeof(CFStringGetTypeID) *CFStringGetTypeIDFunc =
- (typeof(CFStringGetTypeID) *)dlsym(RTLD_DEFAULT, "CFStringGetTypeID");
+ CFStringGetTypeIDFuncTy CFStringGetTypeIDFunc =
+ (CFStringGetTypeIDFuncTy)dlsym(RTLD_DEFAULT, "CFStringGetTypeID");
if (!CFStringGetTypeIDFunc)
return;
- typeof(CFStringGetCString) *CFStringGetCStringFunc =
- (typeof(CFStringGetCString) *)dlsym(RTLD_DEFAULT, "CFStringGetCString");
+ CFStringGetCStringFuncTy CFStringGetCStringFunc =
+ (CFStringGetCStringFuncTy)dlsym(RTLD_DEFAULT, "CFStringGetCString");
if (!CFStringGetCStringFunc)
return;
- typeof(CFRelease) *CFReleaseFunc =
- (typeof(CFRelease) *)dlsym(RTLD_DEFAULT, "CFRelease");
+ CFReleaseFuncTy CFReleaseFunc =
+ (CFReleaseFuncTy)dlsym(RTLD_DEFAULT, "CFRelease");
if (!CFReleaseFunc)
return;
@@ -163,10 +182,14 @@ int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) {
/* Populate the global version variables, if they haven't already. */
dispatch_once_f(&DispatchOnceCounter, NULL, parseSystemVersionPList);
- if (Major < GlobalMajor) return 1;
- if (Major > GlobalMajor) return 0;
- if (Minor < GlobalMinor) return 1;
- if (Minor > GlobalMinor) return 0;
+ if (Major < GlobalMajor)
+ return 1;
+ if (Major > GlobalMajor)
+ return 0;
+ if (Minor < GlobalMinor)
+ return 1;
+ if (Minor > GlobalMinor)
+ return 0;
return Subminor <= GlobalSubminor;
}
diff --git a/lib/builtins/riscv/mulsi3.S b/lib/builtins/riscv/mulsi3.S
new file mode 100644
index 000000000000..a58d237040b6
--- /dev/null
+++ b/lib/builtins/riscv/mulsi3.S
@@ -0,0 +1,28 @@
+//===--- mulsi3.S - Integer multiplication routines routines ---===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#if !defined(__riscv_mul) && __riscv_xlen == 32
+ .text
+ .align 2
+
+ .globl __mulsi3
+ .type __mulsi3, @function
+__mulsi3:
+ mv a2, a0
+ mv a0, zero
+.L1:
+ andi a3, a1, 1
+ beqz a3, .L2
+ add a0, a0, a2
+.L2:
+ srli a1, a1, 1
+ slli a2, a2, 1
+ bnez a1, .L1
+ ret
+#endif
diff --git a/lib/cfi/CMakeLists.txt b/lib/cfi/CMakeLists.txt
index 6c531445626a..7ed72bca5d23 100644
--- a/lib/cfi/CMakeLists.txt
+++ b/lib/cfi/CMakeLists.txt
@@ -30,6 +30,8 @@ if(OS_NAME MATCHES "Linux")
OBJECT_LIBS RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
+ RTSanitizerCommonCoverage
+ RTSanitizerCommonSymbolizer
RTUbsan
CFLAGS ${CFI_CFLAGS} ${CFI_DIAG_CFLAGS}
PARENT_TARGET cfi)
diff --git a/lib/cfi/cfi.cc b/lib/cfi/cfi.cc
index f7693b37d29c..a2f127f93cda 100644
--- a/lib/cfi/cfi.cc
+++ b/lib/cfi/cfi.cc
@@ -132,7 +132,11 @@ void ShadowBuilder::Start() {
void ShadowBuilder::AddUnchecked(uptr begin, uptr end) {
uint16_t *shadow_begin = MemToShadow(begin, shadow_);
uint16_t *shadow_end = MemToShadow(end - 1, shadow_) + 1;
- memset(shadow_begin, kUncheckedShadow,
+ // memset takes a byte, so our unchecked shadow value requires both bytes to
+ // be the same. Make sure we're ok during compilation.
+ static_assert((kUncheckedShadow & 0xff) == ((kUncheckedShadow >> 8) & 0xff),
+ "Both bytes of the 16-bit value must be the same!");
+ memset(shadow_begin, kUncheckedShadow & 0xff,
(shadow_end - shadow_begin) * sizeof(*shadow_begin));
}
@@ -379,6 +383,8 @@ __cfi_slowpath_diag(u64 CallSiteTypeId, void *Ptr, void *DiagData) {
}
#endif
+static void EnsureInterceptorsInitialized();
+
// Setup shadow for dlopen()ed libraries.
// The actual shadow setup happens after dlopen() returns, which means that
// a library can not be a target of any CFI checks while its constructors are
@@ -388,6 +394,7 @@ __cfi_slowpath_diag(u64 CallSiteTypeId, void *Ptr, void *DiagData) {
// We could insert a high-priority constructor into the library, but that would
// not help with the uninstrumented libraries.
INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
+ EnsureInterceptorsInitialized();
EnterLoader();
void *handle = REAL(dlopen)(filename, flag);
ExitLoader();
@@ -395,12 +402,27 @@ INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
}
INTERCEPTOR(int, dlclose, void *handle) {
+ EnsureInterceptorsInitialized();
EnterLoader();
int res = REAL(dlclose)(handle);
ExitLoader();
return res;
}
+static BlockingMutex interceptor_init_lock(LINKER_INITIALIZED);
+static bool interceptors_inited = false;
+
+static void EnsureInterceptorsInitialized() {
+ BlockingMutexLock lock(&interceptor_init_lock);
+ if (interceptors_inited)
+ return;
+
+ INTERCEPT_FUNCTION(dlopen);
+ INTERCEPT_FUNCTION(dlclose);
+
+ interceptors_inited = true;
+}
+
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
#if !SANITIZER_CAN_USE_PREINIT_ARRAY
// On ELF platforms, the constructor is invoked using .preinit_array (see below)
@@ -411,9 +433,6 @@ void __cfi_init() {
InitializeFlags();
InitShadow();
- INTERCEPT_FUNCTION(dlopen);
- INTERCEPT_FUNCTION(dlclose);
-
#ifdef CFI_ENABLE_DIAG
__ubsan::InitAsPlugin();
#endif
diff --git a/lib/cfi/cfi_blacklist.txt b/lib/cfi/cfi_blacklist.txt
index d8c9d49e3cdd..3d73508f5707 100644
--- a/lib/cfi/cfi_blacklist.txt
+++ b/lib/cfi/cfi_blacklist.txt
@@ -1,7 +1,9 @@
[cfi-unrelated-cast]
-# std::get_temporary_buffer, likewise (libstdc++, libc++).
+# The specification of std::get_temporary_buffer mandates a cast to
+# uninitialized T* (libstdc++, libc++, MSVC stdlib).
fun:_ZSt20get_temporary_buffer*
fun:_ZNSt3__120get_temporary_buffer*
+fun:*get_temporary_buffer@.*@std@@*
# STL address-of magic (libstdc++, libc++).
fun:*__addressof*
diff --git a/lib/dfsan/.clang-format b/lib/dfsan/.clang-format
index f6cb8ad931f5..560308c91dee 100644
--- a/lib/dfsan/.clang-format
+++ b/lib/dfsan/.clang-format
@@ -1 +1,2 @@
BasedOnStyle: Google
+AllowShortIfStatementsOnASingleLine: false
diff --git a/lib/dfsan/CMakeLists.txt b/lib/dfsan/CMakeLists.txt
index 2c486bff821b..b3ae713cf02c 100644
--- a/lib/dfsan/CMakeLists.txt
+++ b/lib/dfsan/CMakeLists.txt
@@ -5,6 +5,12 @@ set(DFSAN_RTL_SOURCES
dfsan.cc
dfsan_custom.cc
dfsan_interceptors.cc)
+
+set(DFSAN_RTL_HEADERS
+ dfsan.h
+ dfsan_flags.inc
+ dfsan_platform.h)
+
set(DFSAN_COMMON_CFLAGS ${SANITIZER_COMMON_CFLAGS})
append_rtti_flag(OFF DFSAN_COMMON_CFLAGS)
# Prevent clang from generating libc calls.
@@ -23,6 +29,7 @@ foreach(arch ${DFSAN_SUPPORTED_ARCH})
$<TARGET_OBJECTS:RTInterception.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ ADDITIONAL_HEADERS ${DFSAN_RTL_HEADERS}
CFLAGS ${DFSAN_CFLAGS}
PARENT_TARGET dfsan)
add_sanitizer_rt_symbols(clang_rt.dfsan
@@ -32,16 +39,19 @@ foreach(arch ${DFSAN_SUPPORTED_ARCH})
clang_rt.dfsan-${arch}-symbols)
endforeach()
-set(dfsan_abilist_filename ${COMPILER_RT_OUTPUT_DIR}/dfsan_abilist.txt)
+set(dfsan_abilist_dir ${COMPILER_RT_OUTPUT_DIR}/share)
+set(dfsan_abilist_filename ${dfsan_abilist_dir}/dfsan_abilist.txt)
add_custom_target(dfsan_abilist ALL
DEPENDS ${dfsan_abilist_filename})
add_custom_command(OUTPUT ${dfsan_abilist_filename}
VERBATIM
COMMAND
+ ${CMAKE_COMMAND} -E make_directory ${dfsan_abilist_dir}
+ COMMAND
cat ${CMAKE_CURRENT_SOURCE_DIR}/done_abilist.txt
${CMAKE_CURRENT_SOURCE_DIR}/libc_ubuntu1404_abilist.txt
> ${dfsan_abilist_filename}
DEPENDS done_abilist.txt libc_ubuntu1404_abilist.txt)
add_dependencies(dfsan dfsan_abilist)
install(FILES ${dfsan_abilist_filename}
- DESTINATION ${COMPILER_RT_INSTALL_PATH})
+ DESTINATION ${COMPILER_RT_INSTALL_PATH}/share)
diff --git a/lib/dfsan/dfsan.cc b/lib/dfsan/dfsan.cc
index 9e360c959fa7..d4dbebc43a78 100644
--- a/lib/dfsan/dfsan.cc
+++ b/lib/dfsan/dfsan.cc
@@ -425,7 +425,8 @@ static void dfsan_init(int argc, char **argv, char **envp) {
InitializePlatformEarly();
- MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr());
+ if (!MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr()))
+ Die();
// Protect the region of memory we don't use, to preserve the one-to-one
// mapping from application to shadow memory. But if ASLR is disabled, Linux
diff --git a/lib/dfsan/dfsan_custom.cc b/lib/dfsan/dfsan_custom.cc
index e0cd16ab695c..022aa9a9ab8e 100644
--- a/lib/dfsan/dfsan_custom.cc
+++ b/lib/dfsan/dfsan_custom.cc
@@ -1132,4 +1132,26 @@ int __dfsw_snprintf(char *str, size_t size, const char *format,
va_end(ap);
return ret;
}
-} // extern "C"
+
+// Default empty implementations (weak). Users should redefine them.
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32 *) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init, u32 *,
+ u32 *) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_cmp, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_cmp1, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_cmp2, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_cmp4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_cmp8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_const_cmp1,
+ void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_const_cmp2,
+ void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_const_cmp4,
+ void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_const_cmp8,
+ void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_switch, void) {}
+} // extern "C"
diff --git a/lib/dfsan/done_abilist.txt b/lib/dfsan/done_abilist.txt
index a560cd7b477c..9765626ad3ff 100644
--- a/lib/dfsan/done_abilist.txt
+++ b/lib/dfsan/done_abilist.txt
@@ -264,8 +264,6 @@ fun:reflect.makeFuncStub=discard
# lib/Fuzzer
###############################################################################
# Replaces __sanitizer_cov_trace_cmp with __dfsw___sanitizer_cov_trace_cmp
-fun:__sanitizer_cov_trace_cmp=custom
-fun:__sanitizer_cov_trace_cmp=uninstrumented
fun:__sanitizer_cov_trace_cmp1=custom
fun:__sanitizer_cov_trace_cmp1=uninstrumented
fun:__sanitizer_cov_trace_cmp2=custom
@@ -274,6 +272,14 @@ fun:__sanitizer_cov_trace_cmp4=custom
fun:__sanitizer_cov_trace_cmp4=uninstrumented
fun:__sanitizer_cov_trace_cmp8=custom
fun:__sanitizer_cov_trace_cmp8=uninstrumented
+fun:__sanitizer_cov_trace_const_cmp1=custom
+fun:__sanitizer_cov_trace_const_cmp1=uninstrumented
+fun:__sanitizer_cov_trace_const_cmp2=custom
+fun:__sanitizer_cov_trace_const_cmp2=uninstrumented
+fun:__sanitizer_cov_trace_const_cmp4=custom
+fun:__sanitizer_cov_trace_const_cmp4=uninstrumented
+fun:__sanitizer_cov_trace_const_cmp8=custom
+fun:__sanitizer_cov_trace_const_cmp8=uninstrumented
# Similar for __sanitizer_cov_trace_switch
fun:__sanitizer_cov_trace_switch=custom
fun:__sanitizer_cov_trace_switch=uninstrumented
@@ -289,10 +295,11 @@ fun:__sanitizer_set_death_callback=uninstrumented
fun:__sanitizer_set_death_callback=discard
fun:__sanitizer_update_counter_bitset_and_clear_counters=uninstrumented
fun:__sanitizer_update_counter_bitset_and_clear_counters=discard
+fun:__sanitizer_cov_trace_pc*=uninstrumented
+fun:__sanitizer_cov_trace_pc*=discard
+fun:__sanitizer_cov_pcs_init=uninstrumented
+fun:__sanitizer_cov_pcs_init=discard
# Ignores the dfsan wrappers.
fun:__dfsw_*=uninstrumented
fun:__dfsw_*=discard
-
-# Don't add extra parameters to the Fuzzer callback.
-fun:LLVMFuzzerTestOneInput=uninstrumented
diff --git a/lib/dfsan/scripts/check_custom_wrappers.sh b/lib/dfsan/scripts/check_custom_wrappers.sh
index 99bf50cbbd08..9a80cb9c6ffd 100755
--- a/lib/dfsan/scripts/check_custom_wrappers.sh
+++ b/lib/dfsan/scripts/check_custom_wrappers.sh
@@ -23,6 +23,7 @@ grep -E "^fun:.*=custom" ${DFSAN_ABI_LIST} \
| grep -v "dfsan_get_label\|__sanitizer_cov_trace" \
| sed "s/^fun:\(.*\)=custom.*/\1/" | sort > $DIFF_A
grep -E "__dfsw.*\(" ${DFSAN_CUSTOM_WRAPPERS} \
+ | grep -v "__sanitizer_cov_trace" \
| sed "s/.*__dfsw_\(.*\)(.*/\1/" | sort > $DIFF_B
diff -u $DIFF_A $DIFF_B > ${DIFFOUT}
if [ $? -ne 0 ]
@@ -33,6 +34,7 @@ then
fi
grep -E __dfsw_ ${DFSAN_CUSTOM_WRAPPERS} \
+ | grep -v "__sanitizer_cov_trace" \
| sed "s/.*__dfsw_\([^(]*\).*/\1/" | sort > $DIFF_A
grep -E "^[[:space:]]*test_.*\(\);" ${DFSAN_CUSTOM_TESTS} \
| sed "s/.*test_\(.*\)();/\1/" | sort > $DIFF_B
diff --git a/lib/esan/CMakeLists.txt b/lib/esan/CMakeLists.txt
index 2012ab642bf1..4de5c0205c1d 100644
--- a/lib/esan/CMakeLists.txt
+++ b/lib/esan/CMakeLists.txt
@@ -18,6 +18,18 @@ set(ESAN_SOURCES
working_set.cpp
working_set_posix.cpp)
+set(ESAN_HEADERS
+ cache_frag.h
+ esan.h
+ esan_circular_buffer.h
+ esan_flags.h
+ esan_flags.inc
+ esan_hashtable.h
+ esan_interface_internal.h
+ esan_shadow.h
+ esan_sideline.h
+ working_set.h)
+
foreach (arch ${ESAN_SUPPORTED_ARCH})
add_compiler_rt_runtime(clang_rt.esan
STATIC
@@ -26,6 +38,8 @@ foreach (arch ${ESAN_SUPPORTED_ARCH})
$<TARGET_OBJECTS:RTInterception.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
+ ADDITIONAL_HEADERS ${ESAN_HEADERS}
CFLAGS ${ESAN_RTL_CFLAGS})
add_sanitizer_rt_symbols(clang_rt.esan
ARCHS ${arch}
diff --git a/lib/esan/esan.cpp b/lib/esan/esan.cpp
index 09b530b6645f..44b8032caa1a 100644
--- a/lib/esan/esan.cpp
+++ b/lib/esan/esan.cpp
@@ -163,15 +163,15 @@ static void initializeShadow() {
VPrintf(1, "Shadow #%d: [%zx-%zx) (%zuGB)\n", i, ShadowStart, ShadowEnd,
(ShadowEnd - ShadowStart) >> 30);
- uptr Map;
+ uptr Map = 0;
if (__esan_which_tool == ESAN_WorkingSet) {
// We want to identify all shadow pages that are touched so we start
// out inaccessible.
Map = (uptr)MmapFixedNoAccess(ShadowStart, ShadowEnd- ShadowStart,
"shadow");
} else {
- Map = (uptr)MmapFixedNoReserve(ShadowStart, ShadowEnd - ShadowStart,
- "shadow");
+ if (MmapFixedNoReserve(ShadowStart, ShadowEnd - ShadowStart, "shadow"))
+ Map = ShadowStart;
}
if (Map != ShadowStart) {
Printf("FATAL: EfficiencySanitizer failed to map its shadow memory.\n");
diff --git a/lib/esan/esan_interceptors.cpp b/lib/esan/esan_interceptors.cpp
index 62fa13c83822..0c596f1cf37e 100644
--- a/lib/esan/esan_interceptors.cpp
+++ b/lib/esan/esan_interceptors.cpp
@@ -175,6 +175,15 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr)
do { \
} while (false)
+#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
+ off) \
+ do { \
+ if (!fixMmapAddr(&addr, sz, flags)) \
+ return (void *)-1; \
+ void *result = REAL(mmap)(addr, sz, prot, flags, fd, off); \
+ return (void *)checkMmapResult((uptr)result, sz); \
+ } while (false)
+
#include "sanitizer_common/sanitizer_common_interceptors.inc"
//===----------------------------------------------------------------------===//
@@ -232,6 +241,7 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr)
} while (false)
#include "sanitizer_common/sanitizer_common_syscalls.inc"
+#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
//===----------------------------------------------------------------------===//
// Custom interceptors
@@ -306,13 +316,6 @@ INTERCEPTOR(int, unlink, char *path) {
return REAL(unlink)(path);
}
-INTERCEPTOR(int, puts, const char *s) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, puts, s);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s));
- return REAL(puts)(s);
-}
-
INTERCEPTOR(int, rmdir, char *path) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, rmdir, path);
@@ -321,44 +324,6 @@ INTERCEPTOR(int, rmdir, char *path) {
}
//===----------------------------------------------------------------------===//
-// Shadow-related interceptors
-//===----------------------------------------------------------------------===//
-
-// These are candidates for sharing with all sanitizers if shadow memory
-// support is also standardized.
-
-INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags,
- int fd, OFF_T off) {
- if (UNLIKELY(REAL(mmap) == nullptr)) {
- // With esan init during interceptor init and a static libc preventing
- // our early-calloc from triggering, we can end up here before our
- // REAL pointer is set up.
- return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
- }
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, mmap, addr, sz, prot, flags, fd, off);
- if (!fixMmapAddr(&addr, sz, flags))
- return (void *)-1;
- void *result = REAL(mmap)(addr, sz, prot, flags, fd, off);
- return (void *)checkMmapResult((uptr)result, sz);
-}
-
-#if SANITIZER_LINUX
-INTERCEPTOR(void *, mmap64, void *addr, SIZE_T sz, int prot, int flags,
- int fd, OFF64_T off) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, mmap64, addr, sz, prot, flags, fd, off);
- if (!fixMmapAddr(&addr, sz, flags))
- return (void *)-1;
- void *result = REAL(mmap64)(addr, sz, prot, flags, fd, off);
- return (void *)checkMmapResult((uptr)result, sz);
-}
-#define ESAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64)
-#else
-#define ESAN_MAYBE_INTERCEPT_MMAP64
-#endif
-
-//===----------------------------------------------------------------------===//
// Signal-related interceptors
//===----------------------------------------------------------------------===//
@@ -521,14 +486,8 @@ void initializeInterceptors() {
INTERCEPT_FUNCTION(creat);
ESAN_MAYBE_INTERCEPT_CREAT64;
INTERCEPT_FUNCTION(unlink);
- INTERCEPT_FUNCTION(fread);
- INTERCEPT_FUNCTION(fwrite);
- INTERCEPT_FUNCTION(puts);
INTERCEPT_FUNCTION(rmdir);
- INTERCEPT_FUNCTION(mmap);
- ESAN_MAYBE_INTERCEPT_MMAP64;
-
ESAN_MAYBE_INTERCEPT_SIGNAL;
ESAN_MAYBE_INTERCEPT_SIGACTION;
ESAN_MAYBE_INTERCEPT_SIGPROCMASK;
diff --git a/lib/esan/esan_sideline_linux.cpp b/lib/esan/esan_sideline_linux.cpp
index 4a96910ece35..2de25fba700d 100644
--- a/lib/esan/esan_sideline_linux.cpp
+++ b/lib/esan/esan_sideline_linux.cpp
@@ -70,7 +70,7 @@ int SidelineThread::runSideline(void *Arg) {
internal_prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
// Set up a signal handler on an alternate stack for safety.
- InternalScopedBuffer<char> StackMap(SigAltStackSize);
+ InternalMmapVector<char> StackMap(SigAltStackSize);
stack_t SigAltStack;
SigAltStack.ss_sp = StackMap.data();
SigAltStack.ss_size = SigAltStackSize;
diff --git a/lib/fuzzer/CMakeLists.txt b/lib/fuzzer/CMakeLists.txt
index 9769be52ae01..679318e460b5 100644
--- a/lib/fuzzer/CMakeLists.txt
+++ b/lib/fuzzer/CMakeLists.txt
@@ -1,6 +1,6 @@
set(LIBFUZZER_SOURCES
- FuzzerClangCounters.cpp
FuzzerCrossOver.cpp
+ FuzzerDataFlowTrace.cpp
FuzzerDriver.cpp
FuzzerExtFunctionsDlsym.cpp
FuzzerExtFunctionsDlsymWin.cpp
@@ -13,6 +13,7 @@ set(LIBFUZZER_SOURCES
FuzzerMerge.cpp
FuzzerMutate.cpp
FuzzerSHA1.cpp
+ FuzzerShmemFuchsia.cpp
FuzzerShmemPosix.cpp
FuzzerShmemWindows.cpp
FuzzerTracePC.cpp
@@ -21,8 +22,29 @@ set(LIBFUZZER_SOURCES
FuzzerUtilFuchsia.cpp
FuzzerUtilLinux.cpp
FuzzerUtilPosix.cpp
- FuzzerUtilWindows.cpp
- )
+ FuzzerUtilWindows.cpp)
+
+set(LIBFUZZER_HEADERS
+ FuzzerCommand.h
+ FuzzerCorpus.h
+ FuzzerDataFlowTrace.h
+ FuzzerDefs.h
+ FuzzerDictionary.h
+ FuzzerExtFunctions.def
+ FuzzerExtFunctions.h
+ FuzzerFlags.def
+ FuzzerIO.h
+ FuzzerInterface.h
+ FuzzerInternal.h
+ FuzzerMerge.h
+ FuzzerMutate.h
+ FuzzerOptions.h
+ FuzzerRandom.h
+ FuzzerSHA1.h
+ FuzzerShmem.h
+ FuzzerTracePC.h
+ FuzzerUtil.h
+ FuzzerValueBitMap.h)
CHECK_CXX_SOURCE_COMPILES("
static thread_local int blah;
@@ -33,6 +55,14 @@ CHECK_CXX_SOURCE_COMPILES("
set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH)
+ list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer)
+ # Remove -stdlib= which is unused when passing -nostdinc++.
+ string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+elseif(TARGET cxx-headers OR HAVE_LIBCXX)
+ set(LIBFUZZER_DEPS cxx-headers)
+endif()
+
append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer LIBFUZZER_CFLAGS)
if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage")
@@ -43,21 +73,22 @@ if(NOT HAS_THREAD_LOCAL)
list(APPEND LIBFUZZER_CFLAGS -Dthread_local=__thread)
endif()
-if(APPLE)
- set(FUZZER_SUPPORTED_OS osx)
-endif()
+set(FUZZER_SUPPORTED_OS ${SANITIZER_COMMON_SUPPORTED_OS})
add_compiler_rt_object_libraries(RTfuzzer
OS ${FUZZER_SUPPORTED_OS}
ARCHS ${FUZZER_SUPPORTED_ARCH}
SOURCES ${LIBFUZZER_SOURCES}
- CFLAGS ${LIBFUZZER_CFLAGS})
+ ADDITIONAL_HEADERS ${LIBFUZZER_HEADERS}
+ CFLAGS ${LIBFUZZER_CFLAGS}
+ DEPS ${LIBFUZZER_DEPS})
add_compiler_rt_object_libraries(RTfuzzer_main
OS ${FUZZER_SUPPORTED_OS}
ARCHS ${FUZZER_SUPPORTED_ARCH}
SOURCES FuzzerMain.cpp
- CFLAGS ${LIBFUZZER_CFLAGS})
+ CFLAGS ${LIBFUZZER_CFLAGS}
+ DEPS ${LIBFUZZER_DEPS})
add_compiler_rt_runtime(clang_rt.fuzzer
STATIC
@@ -75,6 +106,40 @@ add_compiler_rt_runtime(clang_rt.fuzzer_no_main
CFLAGS ${LIBFUZZER_CFLAGS}
PARENT_TARGET fuzzer)
+if(OS_NAME MATCHES "Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH)
+ macro(partially_link_libcxx name dir arch)
+ set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
+ file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
+ add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
+ COMMAND ${CMAKE_LINKER} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
+ COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
+ COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
+ COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
+ WORKING_DIRECTORY ${cxx_${arch}_merge_dir}
+ )
+ endmacro()
+
+ foreach(arch ${FUZZER_SUPPORTED_ARCH})
+ get_target_flags_for_arch(${arch} TARGET_CFLAGS)
+ set(LIBCXX_${arch}_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libcxx_fuzzer_${arch})
+ add_custom_libcxx(libcxx_fuzzer_${arch} ${LIBCXX_${arch}_PREFIX}
+ CFLAGS ${TARGET_CFLAGS}
+ -D_LIBCPP_ABI_VERSION=Fuzzer
+ -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS=1
+ -fvisibility=hidden
+ CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON
+ -DLIBCXX_ENABLE_EXCEPTIONS=OFF
+ -DLIBCXX_ENABLE_SHARED=OFF
+ -DLIBCXX_CXX_ABI=none)
+ target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
+ add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build)
+ target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
+ add_dependencies(RTfuzzer_main.${arch} libcxx_fuzzer_${arch}-build)
+ partially_link_libcxx(fuzzer_no_main ${LIBCXX_${arch}_PREFIX} ${arch})
+ partially_link_libcxx(fuzzer ${LIBCXX_${arch}_PREFIX} ${arch})
+ endforeach()
+endif()
+
if(COMPILER_RT_INCLUDE_TESTS)
add_subdirectory(tests)
endif()
diff --git a/lib/fuzzer/FuzzerClangCounters.cpp b/lib/fuzzer/FuzzerClangCounters.cpp
deleted file mode 100644
index f69e922cf004..000000000000
--- a/lib/fuzzer/FuzzerClangCounters.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-//===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// Coverage counters from Clang's SourceBasedCodeCoverage.
-//===----------------------------------------------------------------------===//
-
-// Support for SourceBasedCodeCoverage is experimental:
-// * Works only for the main binary, not DSOs yet.
-// * Works only on Linux.
-// * Does not implement print_pcs/print_coverage yet.
-// * Is not fully evaluated for performance and sensitivity.
-// We expect large performance drop due to 64-bit counters,
-// and *maybe* better sensitivity due to more fine-grained counters.
-// Preliminary comparison on a single benchmark (RE2) shows
-// a bit worse sensitivity though.
-
-#include "FuzzerDefs.h"
-
-#if LIBFUZZER_LINUX
-__attribute__((weak)) extern uint64_t __start___llvm_prf_cnts;
-__attribute__((weak)) extern uint64_t __stop___llvm_prf_cnts;
-namespace fuzzer {
-uint64_t *ClangCountersBegin() { return &__start___llvm_prf_cnts; }
-uint64_t *ClangCountersEnd() { return &__stop___llvm_prf_cnts; }
-} // namespace fuzzer
-#else
-// TODO: Implement on Mac (if the data shows it's worth it).
-//__attribute__((visibility("hidden")))
-//extern uint64_t CountersStart __asm("section$start$__DATA$__llvm_prf_cnts");
-//__attribute__((visibility("hidden")))
-//extern uint64_t CountersEnd __asm("section$end$__DATA$__llvm_prf_cnts");
-namespace fuzzer {
-uint64_t *ClangCountersBegin() { return nullptr; }
-uint64_t *ClangCountersEnd() { return nullptr; }
-} // namespace fuzzer
-#endif
-
-namespace fuzzer {
-ATTRIBUTE_NO_SANITIZE_ALL
-void ClearClangCounters() { // hand-written memset, don't asan-ify.
- for (auto P = ClangCountersBegin(); P < ClangCountersEnd(); P++)
- *P = 0;
-}
-}
diff --git a/lib/fuzzer/FuzzerCommand.h b/lib/fuzzer/FuzzerCommand.h
index c5500ed21fc1..255f571ecf31 100644
--- a/lib/fuzzer/FuzzerCommand.h
+++ b/lib/fuzzer/FuzzerCommand.h
@@ -29,8 +29,7 @@ public:
// is immutable, meaning this flag effectively marks the end of the mutable
// argument list.
static inline const char *ignoreRemainingArgs() {
- static const char *kIgnoreRemaining = "-ignore_remaining_args=1";
- return kIgnoreRemaining;
+ return "-ignore_remaining_args=1";
}
Command() : CombinedOutAndErr(false) {}
diff --git a/lib/fuzzer/FuzzerCorpus.h b/lib/fuzzer/FuzzerCorpus.h
index 2da929835f45..8ad14656cffc 100644
--- a/lib/fuzzer/FuzzerCorpus.h
+++ b/lib/fuzzer/FuzzerCorpus.h
@@ -12,6 +12,7 @@
#ifndef LLVM_FUZZER_CORPUS
#define LLVM_FUZZER_CORPUS
+#include "FuzzerDataFlowTrace.h"
#include "FuzzerDefs.h"
#include "FuzzerIO.h"
#include "FuzzerRandom.h"
@@ -35,8 +36,9 @@ struct InputInfo {
size_t NumSuccessfullMutations = 0;
bool MayDeleteFile = false;
bool Reduced = false;
+ bool HasFocusFunction = false;
Vector<uint32_t> UniqFeatureSet;
- float FeatureFrequencyScore = 1.0;
+ Vector<uint8_t> DataFlowTraceForFocusFunction;
};
class InputCorpus {
@@ -45,7 +47,6 @@ class InputCorpus {
InputCorpus(const std::string &OutputCorpus) : OutputCorpus(OutputCorpus) {
memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature));
memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature));
- memset(FeatureFrequency, 0, sizeof(FeatureFrequency));
}
~InputCorpus() {
for (auto II : Inputs)
@@ -70,10 +71,24 @@ class InputCorpus {
Res = std::max(Res, II->U.size());
return Res;
}
+
+ size_t NumInputsThatTouchFocusFunction() {
+ return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) {
+ return II->HasFocusFunction;
+ });
+ }
+
+ size_t NumInputsWithDataFlowTrace() {
+ return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) {
+ return !II->DataFlowTraceForFocusFunction.empty();
+ });
+ }
+
bool empty() const { return Inputs.empty(); }
const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
- const Vector<uint32_t> &FeatureSet) {
+ bool HasFocusFunction, const Vector<uint32_t> &FeatureSet,
+ const DataFlowTrace &DFT, const InputInfo *BaseII) {
assert(!U.empty());
if (FeatureDebug)
Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures);
@@ -83,9 +98,19 @@ class InputCorpus {
II.NumFeatures = NumFeatures;
II.MayDeleteFile = MayDeleteFile;
II.UniqFeatureSet = FeatureSet;
+ II.HasFocusFunction = HasFocusFunction;
std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end());
ComputeSHA1(U.data(), U.size(), II.Sha1);
- Hashes.insert(Sha1ToString(II.Sha1));
+ auto Sha1Str = Sha1ToString(II.Sha1);
+ Hashes.insert(Sha1Str);
+ if (HasFocusFunction)
+ if (auto V = DFT.Get(Sha1Str))
+ II.DataFlowTraceForFocusFunction = *V;
+ // This is a gross heuristic.
+ // Ideally, when we add an element to a corpus we need to know its DFT.
+ // But if we don't, we'll use the DFT of its base input.
+ if (II.DataFlowTraceForFocusFunction.empty() && BaseII)
+ II.DataFlowTraceForFocusFunction = BaseII->DataFlowTraceForFocusFunction;
UpdateCorpusDistribution();
PrintCorpus();
// ValidateFeatureSet();
@@ -157,9 +182,9 @@ class InputCorpus {
void PrintStats() {
for (size_t i = 0; i < Inputs.size(); i++) {
const auto &II = *Inputs[i];
- Printf(" [%zd %s]\tsz: %zd\truns: %zd\tsucc: %zd\n", i,
+ Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i,
Sha1ToString(II.Sha1).c_str(), II.U.size(),
- II.NumExecutedMutations, II.NumSuccessfullMutations);
+ II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction);
}
}
@@ -213,18 +238,10 @@ class InputCorpus {
return false;
}
- void UpdateFeatureFrequency(size_t Idx) {
- FeatureFrequency[Idx % kFeatureSetSize]++;
- }
- float GetFeatureFrequency(size_t Idx) const {
- return FeatureFrequency[Idx % kFeatureSetSize];
- }
- void UpdateFeatureFrequencyScore(InputInfo *II) {
- const float kMin = 0.01, kMax = 100.;
- II->FeatureFrequencyScore = kMin;
- for (auto Idx : II->UniqFeatureSet)
- II->FeatureFrequencyScore += 1. / (GetFeatureFrequency(Idx) + 1.);
- II->FeatureFrequencyScore = Min(II->FeatureFrequencyScore, kMax);
+ bool IsFeatureNew(size_t Idx, uint32_t NewSize, bool Shrink) {
+ assert(NewSize);
+ uint32_t OldSize = GetFeature(Idx % kFeatureSetSize);
+ return OldSize == 0 || (Shrink && OldSize > NewSize);
}
size_t NumFeatures() const { return NumAddedFeatures; }
@@ -264,14 +281,11 @@ private:
std::iota(Intervals.begin(), Intervals.end(), 0);
for (size_t i = 0; i < N; i++)
Weights[i] = Inputs[i]->NumFeatures
- ? (i + 1) * Inputs[i]->FeatureFrequencyScore
+ ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1)
: 0.;
if (FeatureDebug) {
for (size_t i = 0; i < N; i++)
Printf("%zd ", Inputs[i]->NumFeatures);
- Printf("NUM\n");
- for (size_t i = 0; i < N; i++)
- Printf("%f ", Inputs[i]->FeatureFrequencyScore);
Printf("SCORE\n");
for (size_t i = 0; i < N; i++)
Printf("%f ", Weights[i]);
@@ -292,7 +306,6 @@ private:
size_t NumUpdatedFeatures = 0;
uint32_t InputSizesPerFeature[kFeatureSetSize];
uint32_t SmallestElementPerFeature[kFeatureSetSize];
- float FeatureFrequency[kFeatureSetSize];
std::string OutputCorpus;
};
diff --git a/lib/fuzzer/FuzzerDataFlowTrace.cpp b/lib/fuzzer/FuzzerDataFlowTrace.cpp
new file mode 100644
index 000000000000..764f3e49fd2d
--- /dev/null
+++ b/lib/fuzzer/FuzzerDataFlowTrace.cpp
@@ -0,0 +1,91 @@
+//===- FuzzerDataFlowTrace.cpp - DataFlowTrace ---*- C++ -* ===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// fuzzer::DataFlowTrace
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerDataFlowTrace.h"
+#include "FuzzerIO.h"
+
+#include <cstdlib>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace fuzzer {
+
+void DataFlowTrace::Init(const std::string &DirPath,
+ const std::string &FocusFunction) {
+ if (DirPath.empty()) return;
+ const char *kFunctionsTxt = "functions.txt";
+ Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str());
+ Vector<SizedFile> Files;
+ GetSizedFilesFromDir(DirPath, &Files);
+ std::string L;
+
+ // Read functions.txt
+ std::ifstream IF(DirPlusFile(DirPath, kFunctionsTxt));
+ size_t FocusFuncIdx = SIZE_MAX;
+ size_t NumFunctions = 0;
+ while (std::getline(IF, L, '\n')) {
+ NumFunctions++;
+ if (FocusFunction == L)
+ FocusFuncIdx = NumFunctions - 1;
+ }
+ if (!NumFunctions || FocusFuncIdx == SIZE_MAX || Files.size() <= 1)
+ return;
+ // Read traces.
+ size_t NumTraceFiles = 0;
+ size_t NumTracesWithFocusFunction = 0;
+ for (auto &SF : Files) {
+ auto Name = Basename(SF.File);
+ if (Name == kFunctionsTxt) continue;
+ auto ParseError = [&](const char *Err) {
+ Printf("DataFlowTrace: parse error: %s\n File: %s\n Line: %s\n", Err,
+ Name.c_str(), L.c_str());
+ };
+ NumTraceFiles++;
+ // Printf("=== %s\n", Name.c_str());
+ std::ifstream IF(SF.File);
+ while (std::getline(IF, L, '\n')) {
+ size_t SpacePos = L.find(' ');
+ if (SpacePos == std::string::npos)
+ return ParseError("no space in the trace line");
+ if (L.empty() || L[0] != 'F')
+ return ParseError("the trace line doesn't start with 'F'");
+ size_t N = std::atol(L.c_str() + 1);
+ if (N >= NumFunctions)
+ return ParseError("N is greater than the number of functions");
+ if (N == FocusFuncIdx) {
+ NumTracesWithFocusFunction++;
+ const char *Beg = L.c_str() + SpacePos + 1;
+ const char *End = L.c_str() + L.size();
+ assert(Beg < End);
+ size_t Len = End - Beg;
+ Vector<uint8_t> V(Len);
+ for (size_t I = 0; I < Len; I++) {
+ if (Beg[I] != '0' && Beg[I] != '1')
+ ParseError("the trace should contain only 0 or 1");
+ V[I] = Beg[I] == '1';
+ }
+ Traces[Name] = V;
+ // Print just a few small traces.
+ if (NumTracesWithFocusFunction <= 3 && Len <= 16)
+ Printf("%s => |%s|\n", Name.c_str(), L.c_str() + SpacePos + 1);
+ break; // No need to parse the following lines.
+ }
+ }
+ }
+ assert(NumTraceFiles == Files.size() - 1);
+ Printf("INFO: DataFlowTrace: %zd trace files, %zd functions, "
+ "%zd traces with focus function\n",
+ NumTraceFiles, NumFunctions, NumTracesWithFocusFunction);
+}
+
+} // namespace fuzzer
+
diff --git a/lib/fuzzer/FuzzerDataFlowTrace.h b/lib/fuzzer/FuzzerDataFlowTrace.h
new file mode 100644
index 000000000000..ad4faeab7b2f
--- /dev/null
+++ b/lib/fuzzer/FuzzerDataFlowTrace.h
@@ -0,0 +1,56 @@
+//===- FuzzerDataFlowTrace.h - Internal header for the Fuzzer ---*- C++ -* ===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// fuzzer::DataFlowTrace; reads and handles a data-flow trace.
+//
+// A data flow trace is generated by e.g. dataflow/DataFlow.cpp
+// and is stored on disk in a separate directory.
+//
+// The trace dir contains a file 'functions.txt' which lists function names,
+// oner per line, e.g.
+// ==> functions.txt <==
+// Func2
+// LLVMFuzzerTestOneInput
+// Func1
+//
+// All other files in the dir are the traces, see dataflow/DataFlow.cpp.
+// The name of the file is sha1 of the input used to generate the trace.
+//
+// Current status:
+// the data is parsed and the summary is printed, but the data is not yet
+// used in any other way.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FUZZER_DATA_FLOW_TRACE
+#define LLVM_FUZZER_DATA_FLOW_TRACE
+
+#include "FuzzerDefs.h"
+
+#include <unordered_map>
+#include <vector>
+#include <string>
+
+namespace fuzzer {
+class DataFlowTrace {
+ public:
+ void Init(const std::string &DirPath, const std::string &FocusFunction);
+ void Clear() { Traces.clear(); }
+ const Vector<uint8_t> *Get(const std::string &InputSha1) const {
+ auto It = Traces.find(InputSha1);
+ if (It != Traces.end())
+ return &It->second;
+ return nullptr;
+ }
+
+ private:
+ // Input's sha1 => DFT for the FocusFunction.
+ std::unordered_map<std::string, Vector<uint8_t> > Traces;
+};
+} // namespace fuzzer
+
+#endif // LLVM_FUZZER_DATA_FLOW_TRACE
diff --git a/lib/fuzzer/FuzzerDefs.h b/lib/fuzzer/FuzzerDefs.h
index 5942efc47a4a..a35c7a181b74 100644
--- a/lib/fuzzer/FuzzerDefs.h
+++ b/lib/fuzzer/FuzzerDefs.h
@@ -27,30 +27,56 @@
#define LIBFUZZER_FUCHSIA 0
#define LIBFUZZER_LINUX 1
#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 0
#elif __APPLE__
#define LIBFUZZER_APPLE 1
#define LIBFUZZER_FUCHSIA 0
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 0
#elif __NetBSD__
#define LIBFUZZER_APPLE 0
#define LIBFUZZER_FUCHSIA 0
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 1
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 0
+#define LIBFUZZER_WINDOWS 0
+#elif __FreeBSD__
+#define LIBFUZZER_APPLE 0
+#define LIBFUZZER_FUCHSIA 0
+#define LIBFUZZER_LINUX 0
+#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 1
+#define LIBFUZZER_OPENBSD 0
+#define LIBFUZZER_WINDOWS 0
+#elif __OpenBSD__
+#define LIBFUZZER_APPLE 0
+#define LIBFUZZER_FUCHSIA 0
+#define LIBFUZZER_LINUX 0
+#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 1
#define LIBFUZZER_WINDOWS 0
#elif _WIN32
#define LIBFUZZER_APPLE 0
#define LIBFUZZER_FUCHSIA 0
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 1
#elif __Fuchsia__
#define LIBFUZZER_APPLE 0
#define LIBFUZZER_FUCHSIA 1
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 0
#else
#error "Support for your platform has not been implemented"
@@ -60,7 +86,9 @@
# define __has_attribute(x) 0
#endif
-#define LIBFUZZER_POSIX (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD)
+#define LIBFUZZER_POSIX \
+ (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || \
+ LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD)
#ifdef __x86_64
# if __has_attribute(target)
@@ -127,6 +155,11 @@ extern ExternalFunctions *EF;
template<typename T>
class fuzzer_allocator: public std::allocator<T> {
public:
+ fuzzer_allocator() = default;
+
+ template<class U>
+ fuzzer_allocator(const fuzzer_allocator<U>&) {}
+
template<class Other>
struct rebind { typedef fuzzer_allocator<Other> other; };
};
@@ -143,12 +176,6 @@ typedef int (*UserCallback)(const uint8_t *Data, size_t Size);
int FuzzerDriver(int *argc, char ***argv, UserCallback Callback);
-struct ScopedDoingMyOwnMemOrStr {
- ScopedDoingMyOwnMemOrStr() { DoingMyOwnMemOrStr++; }
- ~ScopedDoingMyOwnMemOrStr() { DoingMyOwnMemOrStr--; }
- static int DoingMyOwnMemOrStr;
-};
-
inline uint8_t Bswap(uint8_t x) { return x; }
inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); }
inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); }
@@ -158,9 +185,7 @@ uint8_t *ExtraCountersBegin();
uint8_t *ExtraCountersEnd();
void ClearExtraCounters();
-uint64_t *ClangCountersBegin();
-uint64_t *ClangCountersEnd();
-void ClearClangCounters();
+extern bool RunningUserCallback;
} // namespace fuzzer
diff --git a/lib/fuzzer/FuzzerDictionary.h b/lib/fuzzer/FuzzerDictionary.h
index daf7d003ea91..0d9d91bcd2f1 100644
--- a/lib/fuzzer/FuzzerDictionary.h
+++ b/lib/fuzzer/FuzzerDictionary.h
@@ -33,17 +33,9 @@ public:
}
bool operator==(const FixedWord<kMaxSize> &w) const {
- ScopedDoingMyOwnMemOrStr scoped_doing_my_own_mem_os_str;
return Size == w.Size && 0 == memcmp(Data, w.Data, Size);
}
- bool operator<(const FixedWord<kMaxSize> &w) const {
- ScopedDoingMyOwnMemOrStr scoped_doing_my_own_mem_os_str;
- if (Size != w.Size)
- return Size < w.Size;
- return memcmp(Data, w.Data, Size) < 0;
- }
-
static size_t GetMaxSize() { return kMaxSize; }
const uint8_t *data() const { return Data; }
uint8_t size() const { return Size; }
@@ -115,11 +107,11 @@ private:
};
// Parses one dictionary entry.
-// If successfull, write the enty to Unit and returns true,
+// If successful, write the enty to Unit and returns true,
// otherwise returns false.
bool ParseOneDictionaryEntry(const std::string &Str, Unit *U);
// Parses the dictionary file, fills Units, returns true iff all lines
-// were parsed succesfully.
+// were parsed successfully.
bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units);
} // namespace fuzzer
diff --git a/lib/fuzzer/FuzzerDriver.cpp b/lib/fuzzer/FuzzerDriver.cpp
index f6b642daeda7..783474a39e16 100644
--- a/lib/fuzzer/FuzzerDriver.cpp
+++ b/lib/fuzzer/FuzzerDriver.cpp
@@ -537,6 +537,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
EF = new ExternalFunctions();
if (EF->LLVMFuzzerInitialize)
EF->LLVMFuzzerInitialize(argc, argv);
+ if (EF->__msan_scoped_disable_interceptor_checks)
+ EF->__msan_scoped_disable_interceptor_checks();
const Vector<std::string> Args(*argv, *argv + *argc);
assert(!Args.empty());
ProgName = new std::string(Args[0]);
@@ -567,7 +569,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
FuzzingOptions Options;
Options.Verbosity = Flags.verbosity;
Options.MaxLen = Flags.max_len;
- Options.ExperimentalLenControl = Flags.experimental_len_control;
+ Options.LenControl = Flags.len_control;
Options.UnitTimeoutSec = Flags.timeout;
Options.ErrorExitCode = Flags.error_exitcode;
Options.TimeoutExitCode = Flags.timeout_exitcode;
@@ -613,15 +615,22 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
Options.PrintNewCovPcs = Flags.print_pcs;
Options.PrintNewCovFuncs = Flags.print_funcs;
Options.PrintFinalStats = Flags.print_final_stats;
+ Options.PrintMutationStats = Flags.print_mutation_stats;
Options.PrintCorpusStats = Flags.print_corpus_stats;
Options.PrintCoverage = Flags.print_coverage;
+ Options.PrintUnstableStats = Flags.print_unstable_stats;
+ if (Flags.handle_unstable == TracePC::MinUnstable ||
+ Flags.handle_unstable == TracePC::ZeroUnstable)
+ Options.HandleUnstable = Flags.handle_unstable;
Options.DumpCoverage = Flags.dump_coverage;
- Options.UseClangCoverage = Flags.use_clang_coverage;
- Options.UseFeatureFrequency = Flags.use_feature_frequency;
if (Flags.exit_on_src_pos)
Options.ExitOnSrcPos = Flags.exit_on_src_pos;
if (Flags.exit_on_item)
Options.ExitOnItem = Flags.exit_on_item;
+ if (Flags.focus_function)
+ Options.FocusFunction = Flags.focus_function;
+ if (Flags.data_flow_trace)
+ Options.DataFlowTrace = Flags.data_flow_trace;
unsigned Seed = Flags.seed;
// Initialize Seed.
@@ -665,6 +674,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
if (Flags.cleanse_crash)
return CleanseCrashInput(Args, Options);
+#if 0 // deprecated, to be removed.
if (auto Name = Flags.run_equivalence_server) {
SMR.Destroy(Name);
if (!SMR.Create(Name)) {
@@ -690,6 +700,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
}
Printf("INFO: EQUIVALENCE CLIENT UP\n");
}
+#endif
if (DoPlainRun) {
Options.SaveArtifacts = false;
@@ -747,7 +758,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
Printf("Dictionary analysis failed\n");
exit(1);
}
- Printf("Dictionary analysis suceeded\n");
+ Printf("Dictionary analysis succeeded\n");
exit(0);
}
diff --git a/lib/fuzzer/FuzzerExtFunctions.def b/lib/fuzzer/FuzzerExtFunctions.def
index 25a655bfd71d..8bfffdde56d4 100644
--- a/lib/fuzzer/FuzzerExtFunctions.def
+++ b/lib/fuzzer/FuzzerExtFunctions.def
@@ -29,6 +29,7 @@ EXT_FUNC(LLVMFuzzerCustomCrossOver, size_t,
EXT_FUNC(__lsan_enable, void, (), false);
EXT_FUNC(__lsan_disable, void, (), false);
EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false);
+EXT_FUNC(__sanitizer_acquire_crash_state, bool, (), true);
EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int,
(void (*malloc_hook)(const volatile void *, size_t),
void (*free_hook)(const volatile void *)),
@@ -45,3 +46,6 @@ EXT_FUNC(__sanitizer_set_death_callback, void, (void (*)(void)), true);
EXT_FUNC(__sanitizer_set_report_fd, void, (void*), false);
EXT_FUNC(__sanitizer_dump_coverage, void, (const uintptr_t *, uintptr_t),
false);
+EXT_FUNC(__msan_scoped_disable_interceptor_checks, void, (), false);
+EXT_FUNC(__msan_scoped_enable_interceptor_checks, void, (), false);
+EXT_FUNC(__msan_unpoison, void, (const volatile void *, size_t size), false);
diff --git a/lib/fuzzer/FuzzerExtFunctionsWeak.cpp b/lib/fuzzer/FuzzerExtFunctionsWeak.cpp
index 5a90723986af..a4e56fc27b8d 100644
--- a/lib/fuzzer/FuzzerExtFunctionsWeak.cpp
+++ b/lib/fuzzer/FuzzerExtFunctionsWeak.cpp
@@ -13,7 +13,8 @@
// to clients right now.
//===----------------------------------------------------------------------===//
#include "FuzzerDefs.h"
-#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA
+#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA || \
+ LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD
#include "FuzzerExtFunctions.h"
#include "FuzzerIO.h"
@@ -51,4 +52,4 @@ ExternalFunctions::ExternalFunctions() {
} // namespace fuzzer
-#endif // LIBFUZZER_LINUX || LIBFUZZER_NETBSD
+#endif
diff --git a/lib/fuzzer/FuzzerExtraCounters.cpp b/lib/fuzzer/FuzzerExtraCounters.cpp
index 0e7a7761bf80..c99cd89be293 100644
--- a/lib/fuzzer/FuzzerExtraCounters.cpp
+++ b/lib/fuzzer/FuzzerExtraCounters.cpp
@@ -11,7 +11,8 @@
#include "FuzzerDefs.h"
-#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD
+#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \
+ LIBFUZZER_OPENBSD
__attribute__((weak)) extern uint8_t __start___libfuzzer_extra_counters;
__attribute__((weak)) extern uint8_t __stop___libfuzzer_extra_counters;
diff --git a/lib/fuzzer/FuzzerFlags.def b/lib/fuzzer/FuzzerFlags.def
index a32102a7da07..ba04bc25fd45 100644
--- a/lib/fuzzer/FuzzerFlags.def
+++ b/lib/fuzzer/FuzzerFlags.def
@@ -17,7 +17,10 @@ FUZZER_FLAG_INT(runs, -1,
FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. "
"If 0, libFuzzer tries to guess a good value based on the corpus "
"and reports it. ")
-FUZZER_FLAG_INT(experimental_len_control, 0, "experimental flag")
+FUZZER_FLAG_INT(len_control, 1000, "Try generating small inputs first, "
+ "then try larger inputs over time. Specifies the rate at which the length "
+ "limit is increased (smaller == faster). If 0, immediately try inputs with "
+ "size up to max_len.")
FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.")
FUZZER_FLAG_INT(mutate_depth, 5,
"Apply this number of consecutive mutations to each input.")
@@ -42,7 +45,7 @@ FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
"This flag can be used to minimize a corpus.")
FUZZER_FLAG_STRING(merge_inner, "internal flag")
FUZZER_FLAG_STRING(merge_control_file,
- "Specify a control file used for the merge proccess. "
+ "Specify a control file used for the merge process. "
"If a merge process gets killed it tries to leave this file "
"in a state suitable for resuming the merge. "
"By default a temporary file will be used.")
@@ -107,6 +110,15 @@ FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text"
FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated."
" If 1, dump coverage information as a"
" .sancov file at exit.")
+FUZZER_FLAG_INT(handle_unstable, 0, "Experimental."
+ " Executes every input 3 times in total if a unique feature"
+ " is found during the first execution."
+ " If 1, we only use the minimum hit count from the 3 runs"
+ " to determine whether an input is interesting."
+ " If 2, we disregard edges that are found unstable for"
+ " feature collection.")
+FUZZER_FLAG_INT(print_unstable_stats, 0, "Experimental."
+ " If 1, print unstable statistics at exit.")
FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.")
FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.")
FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.")
@@ -142,9 +154,12 @@ FUZZER_FLAG_STRING(exit_on_item, "Exit if an item with a given sha1 sum"
FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed "
"after this one. Useful for fuzzers that need to do their own "
"argument parsing.")
+FUZZER_FLAG_STRING(focus_function, "Experimental. "
+ "Fuzzing will focus on inputs that trigger calls to this function")
-FUZZER_FLAG_STRING(run_equivalence_server, "Experimental")
-FUZZER_FLAG_STRING(use_equivalence_server, "Experimental")
+FUZZER_DEPRECATED_FLAG(run_equivalence_server)
+FUZZER_DEPRECATED_FLAG(use_equivalence_server)
FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
-FUZZER_FLAG_INT(use_clang_coverage, 0, "Experimental")
-FUZZER_FLAG_INT(use_feature_frequency, 0, "Experimental/internal")
+FUZZER_DEPRECATED_FLAG(use_clang_coverage)
+FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
+FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental")
diff --git a/lib/fuzzer/FuzzerIO.cpp b/lib/fuzzer/FuzzerIO.cpp
index dac5ec658f1c..f3ead0ec5357 100644
--- a/lib/fuzzer/FuzzerIO.cpp
+++ b/lib/fuzzer/FuzzerIO.cpp
@@ -100,6 +100,14 @@ std::string DirPlusFile(const std::string &DirPath,
return DirPath + GetSeparator() + FileName;
}
+std::string Basename(const std::string &Path, char Separator) {
+ size_t Pos = Path.rfind(Separator);
+ if (Pos == std::string::npos)
+ return Path;
+ assert(Pos < Path.size());
+ return Path.substr(Pos + 1);
+}
+
void DupAndCloseStderr() {
int OutputFd = DuplicateFile(2);
if (OutputFd > 0) {
diff --git a/lib/fuzzer/FuzzerIO.h b/lib/fuzzer/FuzzerIO.h
index ea9f0d5a6703..6d7757435b7b 100644
--- a/lib/fuzzer/FuzzerIO.h
+++ b/lib/fuzzer/FuzzerIO.h
@@ -67,6 +67,8 @@ struct SizedFile {
void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V);
char GetSeparator();
+// Similar to the basename utility: returns the file name w/o the dir prefix.
+std::string Basename(const std::string &Path, char Separator = GetSeparator());
FILE* OpenFile(int Fd, const char *Mode);
diff --git a/lib/fuzzer/FuzzerIOPosix.cpp b/lib/fuzzer/FuzzerIOPosix.cpp
index 2257751c662d..17e884d3c4c3 100644
--- a/lib/fuzzer/FuzzerIOPosix.cpp
+++ b/lib/fuzzer/FuzzerIOPosix.cpp
@@ -54,7 +54,7 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
DIR *D = opendir(Dir.c_str());
if (!D) {
- Printf("No such directory: %s; exiting\n", Dir.c_str());
+ Printf("%s: %s; exiting\n", strerror(errno), Dir.c_str());
exit(1);
}
while (auto E = readdir(D)) {
diff --git a/lib/fuzzer/FuzzerInterface.h b/lib/fuzzer/FuzzerInterface.h
index c2c0a39843c0..0f7effb2ab6a 100644
--- a/lib/fuzzer/FuzzerInterface.h
+++ b/lib/fuzzer/FuzzerInterface.h
@@ -30,35 +30,39 @@ extern "C" {
// Executes the code under test with [Data, Data+Size) as the input.
// libFuzzer will invoke this function *many* times with different inputs.
// Must return 0.
-int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
+__attribute__((visibility("default"))) int
+LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
// Optional user-provided initialization function.
// If provided, this function will be called by libFuzzer once at startup.
// It may read and modify argc/argv.
// Must return 0.
-int LLVMFuzzerInitialize(int *argc, char ***argv);
+__attribute__((visibility("default"))) int LLVMFuzzerInitialize(int *argc,
+ char ***argv);
// Optional user-provided custom mutator.
// Mutates raw data in [Data, Data+Size) inplace.
// Returns the new size, which is not greater than MaxSize.
// Given the same Seed produces the same mutation.
-size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize,
- unsigned int Seed);
+__attribute__((visibility("default"))) size_t
+LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize,
+ unsigned int Seed);
// Optional user-provided custom cross-over function.
// Combines pieces of Data1 & Data2 together into Out.
// Returns the new size, which is not greater than MaxOutSize.
// Should produce the same mutation given the same Seed.
-size_t LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1,
- const uint8_t *Data2, size_t Size2,
- uint8_t *Out, size_t MaxOutSize,
- unsigned int Seed);
+__attribute__((visibility("default"))) size_t
+LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1,
+ const uint8_t *Data2, size_t Size2, uint8_t *Out,
+ size_t MaxOutSize, unsigned int Seed);
// Experimental, may go away in future.
// libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator.
// Mutates raw data in [Data, Data+Size) inplace.
// Returns the new size, which is not greater than MaxSize.
-size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize);
+__attribute__((visibility("default"))) size_t
+LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize);
#ifdef __cplusplus
} // extern "C"
diff --git a/lib/fuzzer/FuzzerInternal.h b/lib/fuzzer/FuzzerInternal.h
index 2b2638f1f8f2..bfc898248adb 100644
--- a/lib/fuzzer/FuzzerInternal.h
+++ b/lib/fuzzer/FuzzerInternal.h
@@ -12,6 +12,7 @@
#ifndef LLVM_FUZZER_INTERNAL_H
#define LLVM_FUZZER_INTERNAL_H
+#include "FuzzerDataFlowTrace.h"
#include "FuzzerDefs.h"
#include "FuzzerExtFunctions.h"
#include "FuzzerInterface.h"
@@ -66,6 +67,7 @@ public:
static void StaticGracefulExitCallback();
void ExecuteCallback(const uint8_t *Data, size_t Size);
+ void CheckForUnstableCounters(const uint8_t *Data, size_t Size);
bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false,
InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr);
@@ -116,7 +118,6 @@ private:
uint8_t *CurrentUnitData = nullptr;
std::atomic<size_t> CurrentUnitSize;
uint8_t BaseSha1[kSHA1NumBytes]; // Checksum of the base unit.
- bool RunningCB = false;
bool GracefulExitRequested = false;
@@ -134,6 +135,7 @@ private:
InputCorpus &Corpus;
MutationDispatcher &MD;
FuzzingOptions Options;
+ DataFlowTrace DFT;
system_clock::time_point ProcessStartTime = system_clock::now();
system_clock::time_point UnitStartTime, UnitStopTime;
@@ -150,6 +152,28 @@ private:
static thread_local bool IsMyThread;
};
+struct ScopedEnableMsanInterceptorChecks {
+ ScopedEnableMsanInterceptorChecks() {
+ if (EF->__msan_scoped_enable_interceptor_checks)
+ EF->__msan_scoped_enable_interceptor_checks();
+ }
+ ~ScopedEnableMsanInterceptorChecks() {
+ if (EF->__msan_scoped_disable_interceptor_checks)
+ EF->__msan_scoped_disable_interceptor_checks();
+ }
+};
+
+struct ScopedDisableMsanInterceptorChecks {
+ ScopedDisableMsanInterceptorChecks() {
+ if (EF->__msan_scoped_disable_interceptor_checks)
+ EF->__msan_scoped_disable_interceptor_checks();
+ }
+ ~ScopedDisableMsanInterceptorChecks() {
+ if (EF->__msan_scoped_enable_interceptor_checks)
+ EF->__msan_scoped_enable_interceptor_checks();
+ }
+};
+
} // namespace fuzzer
#endif // LLVM_FUZZER_INTERNAL_H
diff --git a/lib/fuzzer/FuzzerLoop.cpp b/lib/fuzzer/FuzzerLoop.cpp
index 5b451ca122d7..4bc88365a0b9 100644
--- a/lib/fuzzer/FuzzerLoop.cpp
+++ b/lib/fuzzer/FuzzerLoop.cpp
@@ -43,6 +43,8 @@ thread_local bool Fuzzer::IsMyThread;
SharedMemoryRegion SMR;
+bool RunningUserCallback = false;
+
// Only one Fuzzer per process.
static Fuzzer *F;
@@ -105,7 +107,7 @@ void MallocHook(const volatile void *ptr, size_t size) {
return;
Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
if (TraceLevel >= 2 && EF)
- EF->__sanitizer_print_stack_trace();
+ PrintStackTrace();
}
}
@@ -118,7 +120,7 @@ void FreeHook(const volatile void *ptr) {
return;
Printf("FREE[%zd] %p\n", N, ptr);
if (TraceLevel >= 2 && EF)
- EF->__sanitizer_print_stack_trace();
+ PrintStackTrace();
}
}
@@ -129,8 +131,7 @@ void Fuzzer::HandleMalloc(size_t Size) {
Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
Size);
Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
- if (EF->__sanitizer_print_stack_trace)
- EF->__sanitizer_print_stack_trace();
+ PrintStackTrace();
DumpCurrentUnit("oom-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
PrintFinalStats();
@@ -149,8 +150,7 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
if (Options.DetectLeaks && EF->__sanitizer_install_malloc_and_free_hooks)
EF->__sanitizer_install_malloc_and_free_hooks(MallocHook, FreeHook);
TPC.SetUseCounters(Options.UseCounters);
- TPC.SetUseValueProfile(Options.UseValueProfile);
- TPC.SetUseClangCoverage(Options.UseClangCoverage);
+ TPC.SetUseValueProfileMask(Options.UseValueProfile);
if (Options.Verbosity)
TPC.PrintModuleInfo();
@@ -161,6 +161,8 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
AllocateCurrentUnitData();
CurrentUnitSize = 0;
memset(BaseSha1, 0, sizeof(BaseSha1));
+ TPC.SetFocusFunction(Options.FocusFunction);
+ DFT.Init(Options.DataFlowTrace, Options.FocusFunction);
}
Fuzzer::~Fuzzer() {}
@@ -179,6 +181,7 @@ void Fuzzer::StaticDeathCallback() {
void Fuzzer::DumpCurrentUnit(const char *Prefix) {
if (!CurrentUnitData)
return; // Happens when running individual inputs.
+ ScopedDisableMsanInterceptorChecks S;
MD.PrintMutationSequence();
Printf("; base unit: %s\n", Sha1ToString(BaseSha1).c_str());
size_t UnitSize = CurrentUnitSize;
@@ -228,9 +231,10 @@ void Fuzzer::StaticFileSizeExceedCallback() {
}
void Fuzzer::CrashCallback() {
+ if (EF->__sanitizer_acquire_crash_state)
+ EF->__sanitizer_acquire_crash_state();
Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid());
- if (EF->__sanitizer_print_stack_trace)
- EF->__sanitizer_print_stack_trace();
+ PrintStackTrace();
Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
" Combine libFuzzer with AddressSanitizer or similar for better "
"crash reports.\n");
@@ -241,11 +245,13 @@ void Fuzzer::CrashCallback() {
}
void Fuzzer::ExitCallback() {
- if (!RunningCB)
+ if (!RunningUserCallback)
return; // This exit did not come from the user callback
+ if (EF->__sanitizer_acquire_crash_state &&
+ !EF->__sanitizer_acquire_crash_state())
+ return;
Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid());
- if (EF->__sanitizer_print_stack_trace)
- EF->__sanitizer_print_stack_trace();
+ PrintStackTrace();
Printf("SUMMARY: libFuzzer: fuzz target exited\n");
DumpCurrentUnit("crash-");
PrintFinalStats();
@@ -273,7 +279,7 @@ void Fuzzer::AlarmCallback() {
if (!InFuzzingThread())
return;
#endif
- if (!RunningCB)
+ if (!RunningUserCallback)
return; // We have not started running units yet.
size_t Seconds =
duration_cast<seconds>(system_clock::now() - UnitStartTime).count();
@@ -282,14 +288,16 @@ void Fuzzer::AlarmCallback() {
if (Options.Verbosity >= 2)
Printf("AlarmCallback %zd\n", Seconds);
if (Seconds >= (size_t)Options.UnitTimeoutSec) {
+ if (EF->__sanitizer_acquire_crash_state &&
+ !EF->__sanitizer_acquire_crash_state())
+ return;
Printf("ALARM: working on the last Unit for %zd seconds\n", Seconds);
Printf(" and the timeout value is %d (use -timeout=N to change)\n",
Options.UnitTimeoutSec);
DumpCurrentUnit("timeout-");
Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
Seconds);
- if (EF->__sanitizer_print_stack_trace)
- EF->__sanitizer_print_stack_trace();
+ PrintStackTrace();
Printf("SUMMARY: libFuzzer: timeout\n");
PrintFinalStats();
_Exit(Options.TimeoutExitCode); // Stop right now.
@@ -297,12 +305,14 @@ void Fuzzer::AlarmCallback() {
}
void Fuzzer::RssLimitCallback() {
+ if (EF->__sanitizer_acquire_crash_state &&
+ !EF->__sanitizer_acquire_crash_state())
+ return;
Printf(
"==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
- if (EF->__sanitizer_print_memory_profile)
- EF->__sanitizer_print_memory_profile(95, 8);
+ PrintMemoryProfile();
DumpCurrentUnit("oom-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
PrintFinalStats();
@@ -328,7 +338,11 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
else
Printf("/%zdMb", N >> 20);
}
+ if (size_t FF = Corpus.NumInputsThatTouchFocusFunction())
+ Printf(" focus: %zd", FF);
}
+ if (TmpMaxMutationLen)
+ Printf(" lim: %zd", TmpMaxMutationLen);
if (Units)
Printf(" units: %zd", Units);
@@ -340,10 +354,13 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
void Fuzzer::PrintFinalStats() {
if (Options.PrintCoverage)
TPC.PrintCoverage();
+ if (Options.PrintUnstableStats)
+ TPC.PrintUnstableStats();
if (Options.DumpCoverage)
TPC.DumpCoverage();
if (Options.PrintCorpusStats)
Corpus.PrintStats();
+ if (Options.PrintMutationStats) MD.PrintMutationStats();
if (!Options.PrintFinalStats)
return;
size_t ExecPerSec = execPerSec();
@@ -432,6 +449,34 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) {
}
}
+void Fuzzer::CheckForUnstableCounters(const uint8_t *Data, size_t Size) {
+ auto CBSetupAndRun = [&]() {
+ ScopedEnableMsanInterceptorChecks S;
+ UnitStartTime = system_clock::now();
+ TPC.ResetMaps();
+ RunningUserCallback = true;
+ CB(Data, Size);
+ RunningUserCallback = false;
+ UnitStopTime = system_clock::now();
+ };
+
+ // Copy original run counters into our unstable counters
+ TPC.InitializeUnstableCounters();
+
+ // First Rerun
+ CBSetupAndRun();
+ TPC.UpdateUnstableCounters(Options.HandleUnstable);
+
+ // Second Rerun
+ CBSetupAndRun();
+ TPC.UpdateUnstableCounters(Options.HandleUnstable);
+
+ // Move minimum hit counts back to ModuleInline8bitCounters
+ if (Options.HandleUnstable == TracePC::MinUnstable ||
+ Options.HandleUnstable == TracePC::ZeroUnstable)
+ TPC.ApplyUnstableCounters();
+}
+
bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
InputInfo *II, bool *FoundUniqFeatures) {
if (!Size)
@@ -442,9 +487,18 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
UniqFeatureSetTmp.clear();
size_t FoundUniqFeaturesOfII = 0;
size_t NumUpdatesBefore = Corpus.NumFeatureUpdates();
+ bool NewFeaturesUnstable = false;
+
+ if (Options.HandleUnstable || Options.PrintUnstableStats) {
+ TPC.CollectFeatures([&](size_t Feature) {
+ if (Corpus.IsFeatureNew(Feature, Size, Options.Shrink))
+ NewFeaturesUnstable = true;
+ });
+ if (NewFeaturesUnstable)
+ CheckForUnstableCounters(Data, Size);
+ }
+
TPC.CollectFeatures([&](size_t Feature) {
- if (Options.UseFeatureFrequency)
- Corpus.UpdateFeatureFrequency(Feature);
if (Corpus.AddFeature(Feature, Size, Options.Shrink))
UniqFeatureSetTmp.push_back(Feature);
if (Options.ReduceInputs && II)
@@ -452,17 +506,20 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
II->UniqFeatureSet.end(), Feature))
FoundUniqFeaturesOfII++;
});
+
if (FoundUniqFeatures)
*FoundUniqFeatures = FoundUniqFeaturesOfII;
PrintPulseAndReportSlowInput(Data, Size);
size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore;
+
if (NumNewFeatures) {
TPC.UpdateObservedPCs();
Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile,
- UniqFeatureSetTmp);
+ TPC.ObservedFocusFunction(), UniqFeatureSetTmp, DFT, II);
return true;
}
if (II && FoundUniqFeaturesOfII &&
+ II->DataFlowTraceForFocusFunction.empty() &&
FoundUniqFeaturesOfII == II->UniqFeatureSet.size() &&
II->U.size() > Size) {
Corpus.Replace(II, {Data, Data + Size});
@@ -505,19 +562,24 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
// so that we reliably find buffer overflows in it.
uint8_t *DataCopy = new uint8_t[Size];
memcpy(DataCopy, Data, Size);
+ if (EF->__msan_unpoison)
+ EF->__msan_unpoison(DataCopy, Size);
if (CurrentUnitData && CurrentUnitData != Data)
memcpy(CurrentUnitData, Data, Size);
CurrentUnitSize = Size;
- AllocTracer.Start(Options.TraceMalloc);
- UnitStartTime = system_clock::now();
- TPC.ResetMaps();
- RunningCB = true;
- int Res = CB(DataCopy, Size);
- RunningCB = false;
- UnitStopTime = system_clock::now();
- (void)Res;
- assert(Res == 0);
- HasMoreMallocsThanFrees = AllocTracer.Stop();
+ {
+ ScopedEnableMsanInterceptorChecks S;
+ AllocTracer.Start(Options.TraceMalloc);
+ UnitStartTime = system_clock::now();
+ TPC.ResetMaps();
+ RunningUserCallback = true;
+ int Res = CB(DataCopy, Size);
+ RunningUserCallback = false;
+ UnitStopTime = system_clock::now();
+ (void)Res;
+ assert(Res == 0);
+ HasMoreMallocsThanFrees = AllocTracer.Stop();
+ }
if (!LooseMemeq(DataCopy, Data, Size))
CrashOnOverwrittenData();
CurrentUnitSize = 0;
@@ -618,8 +680,6 @@ void Fuzzer::MutateAndTestOne() {
MD.StartMutationSequence();
auto &II = Corpus.ChooseUnitToMutate(MD.GetRand());
- if (Options.UseFeatureFrequency)
- Corpus.UpdateFeatureFrequencyScore(&II);
const auto &U = II.U;
memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1));
assert(CurrentUnitData);
@@ -638,7 +698,12 @@ void Fuzzer::MutateAndTestOne() {
break;
MaybeExitGracefully();
size_t NewSize = 0;
- NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen);
+ if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() &&
+ Size <= CurrentMaxMutationLen)
+ NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size,
+ II.DataFlowTraceForFocusFunction);
+ else
+ NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen);
assert(NewSize > 0 && "Mutator returned empty unit");
assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit");
Size = NewSize;
@@ -728,7 +793,14 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) {
}
PrintStats("INITED");
- if (Corpus.empty()) {
+ if (!Options.FocusFunction.empty())
+ Printf("INFO: %zd/%zd inputs touch the focus function\n",
+ Corpus.NumInputsThatTouchFocusFunction(), Corpus.size());
+ if (!Options.DataFlowTrace.empty())
+ Printf("INFO: %zd/%zd inputs have the Data Flow Trace\n",
+ Corpus.NumInputsWithDataFlowTrace(), Corpus.size());
+
+ if (Corpus.empty() && Options.MaxNumberOfRuns) {
Printf("ERROR: no interesting inputs were found. "
"Is the code instrumented for coverage? Exiting.\n");
exit(1);
@@ -737,6 +809,7 @@ void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) {
void Fuzzer::Loop(const Vector<std::string> &CorpusDirs) {
ReadAndExecuteSeedCorpora(CorpusDirs);
+ DFT.Clear(); // No need for DFT any more.
TPC.SetPrintNewPCs(Options.PrintNewCovPcs);
TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs);
system_clock::time_point LastCorpusReload = system_clock::now();
@@ -755,16 +828,12 @@ void Fuzzer::Loop(const Vector<std::string> &CorpusDirs) {
break;
// Update TmpMaxMutationLen
- if (Options.ExperimentalLenControl) {
+ if (Options.LenControl) {
if (TmpMaxMutationLen < MaxMutationLen &&
TotalNumberOfRuns - LastCorpusUpdateRun >
- Options.ExperimentalLenControl * Log(TmpMaxMutationLen)) {
+ Options.LenControl * Log(TmpMaxMutationLen)) {
TmpMaxMutationLen =
Min(MaxMutationLen, TmpMaxMutationLen + Log(TmpMaxMutationLen));
- if (TmpMaxMutationLen <= MaxMutationLen)
- Printf("#%zd\tTEMP_MAX_LEN: %zd (%zd %zd)\n", TotalNumberOfRuns,
- TmpMaxMutationLen, Options.ExperimentalLenControl,
- LastCorpusUpdateRun);
LastCorpusUpdateRun = TotalNumberOfRuns;
}
} else {
@@ -826,13 +895,15 @@ void Fuzzer::AnnounceOutput(const uint8_t *Data, size_t Size) {
extern "C" {
-size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) {
+__attribute__((visibility("default"))) size_t
+LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) {
assert(fuzzer::F);
return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize);
}
// Experimental
-void LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) {
+__attribute__((visibility("default"))) void
+LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) {
assert(fuzzer::F);
fuzzer::F->AnnounceOutput(Data, Size);
}
diff --git a/lib/fuzzer/FuzzerMain.cpp b/lib/fuzzer/FuzzerMain.cpp
index af8657200be2..f2c8e9c7bb11 100644
--- a/lib/fuzzer/FuzzerMain.cpp
+++ b/lib/fuzzer/FuzzerMain.cpp
@@ -16,6 +16,6 @@ extern "C" {
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
} // extern "C"
-int main(int argc, char **argv) {
+__attribute__((visibility("default"))) int main(int argc, char **argv) {
return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput);
}
diff --git a/lib/fuzzer/FuzzerMutate.cpp b/lib/fuzzer/FuzzerMutate.cpp
index 9ee5299f1b60..ff076cca683f 100644
--- a/lib/fuzzer/FuzzerMutate.cpp
+++ b/lib/fuzzer/FuzzerMutate.cpp
@@ -30,39 +30,41 @@ MutationDispatcher::MutationDispatcher(Random &Rand,
DefaultMutators.insert(
DefaultMutators.begin(),
{
- {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"},
- {&MutationDispatcher::Mutate_InsertByte, "InsertByte"},
+ {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 0, 0},
+ {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 0, 0},
{&MutationDispatcher::Mutate_InsertRepeatedBytes,
- "InsertRepeatedBytes"},
- {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"},
- {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"},
- {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"},
- {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"},
- {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"},
- {&MutationDispatcher::Mutate_CopyPart, "CopyPart"},
- {&MutationDispatcher::Mutate_CrossOver, "CrossOver"},
+ "InsertRepeatedBytes", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 0, 0},
+ {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 0,
+ 0},
+ {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 0,
+ 0},
+ {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 0, 0},
+ {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 0, 0},
{&MutationDispatcher::Mutate_AddWordFromManualDictionary,
- "ManualDict"},
+ "ManualDict", 0, 0},
{&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary,
- "PersAutoDict"},
+ "PersAutoDict", 0, 0},
});
if(Options.UseCmp)
DefaultMutators.push_back(
- {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"});
+ {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 0, 0});
if (EF->LLVMFuzzerCustomMutator)
- Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
+ Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 0, 0});
else
Mutators = DefaultMutators;
if (EF->LLVMFuzzerCustomCrossOver)
Mutators.push_back(
- {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"});
+ {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 0, 0});
}
static char RandCh(Random &Rand) {
if (Rand.RandBool()) return Rand(256);
- const char *Special = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00";
+ const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00";
return Special[Rand(sizeof(Special) - 1)];
}
@@ -195,7 +197,6 @@ DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP(
const void *Arg1Mutation, const void *Arg2Mutation,
size_t ArgSize, const uint8_t *Data,
size_t Size) {
- ScopedDoingMyOwnMemOrStr scoped_doing_my_own_mem_os_str;
bool HandleFirst = Rand.RandBool();
const void *ExistingBytes, *DesiredBytes;
Word W;
@@ -339,7 +340,9 @@ size_t MutationDispatcher::InsertPartOf(const uint8_t *From, size_t FromSize,
size_t MutationDispatcher::Mutate_CopyPart(uint8_t *Data, size_t Size,
size_t MaxSize) {
if (Size > MaxSize || Size == 0) return 0;
- if (Rand.RandBool())
+ // If Size == MaxSize, `InsertPartOf(...)` will
+ // fail so there's no point using it in this case.
+ if (Size == MaxSize || Rand.RandBool())
return CopyPartOf(Data, Size, Data, Size);
else
return InsertPartOf(Data, Size, Data, Size, MaxSize);
@@ -463,6 +466,7 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() {
if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
PersistentAutoDictionary.push_back({DE->GetW(), 1});
}
+ RecordUsefulMutations();
}
void MutationDispatcher::PrintRecommendedDictionary() {
@@ -483,8 +487,7 @@ void MutationDispatcher::PrintRecommendedDictionary() {
void MutationDispatcher::PrintMutationSequence() {
Printf("MS: %zd ", CurrentMutatorSequence.size());
- for (auto M : CurrentMutatorSequence)
- Printf("%s-", M.Name);
+ for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name);
if (!CurrentDictionaryEntrySequence.empty()) {
Printf(" DE: ");
for (auto DE : CurrentDictionaryEntrySequence) {
@@ -512,12 +515,13 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
// in which case they will return 0.
// Try several times before returning un-mutated data.
for (int Iter = 0; Iter < 100; Iter++) {
- auto M = Mutators[Rand(Mutators.size())];
- size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize);
+ auto M = &Mutators[Rand(Mutators.size())];
+ size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize);
if (NewSize && NewSize <= MaxSize) {
if (Options.OnlyASCII)
ToASCII(Data, NewSize);
CurrentMutatorSequence.push_back(M);
+ M->TotalCount++;
return NewSize;
}
}
@@ -525,9 +529,54 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
return 1; // Fallback, should not happen frequently.
}
+// Mask represents the set of Data bytes that are worth mutating.
+size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size,
+ size_t MaxSize,
+ const Vector<uint8_t> &Mask) {
+ assert(Size <= Mask.size());
+ // * Copy the worthy bytes into a temporary array T
+ // * Mutate T
+ // * Copy T back.
+ // This is totally unoptimized.
+ auto &T = MutateWithMaskTemp;
+ if (T.size() < Size)
+ T.resize(Size);
+ size_t OneBits = 0;
+ for (size_t I = 0; I < Size; I++)
+ if (Mask[I])
+ T[OneBits++] = Data[I];
+
+ assert(!T.empty());
+ size_t NewSize = Mutate(T.data(), OneBits, OneBits);
+ assert(NewSize <= OneBits);
+ (void)NewSize;
+ // Even if NewSize < OneBits we still use all OneBits bytes.
+ for (size_t I = 0, J = 0; I < Size; I++)
+ if (Mask[I])
+ Data[I] = T[J++];
+ return Size;
+}
+
void MutationDispatcher::AddWordToManualDictionary(const Word &W) {
ManualDictionary.push_back(
{W, std::numeric_limits<size_t>::max()});
}
+void MutationDispatcher::RecordUsefulMutations() {
+ for (auto M : CurrentMutatorSequence) M->UsefulCount++;
+}
+
+void MutationDispatcher::PrintMutationStats() {
+ Printf("\nstat::mutation_usefulness: ");
+ for (size_t i = 0; i < Mutators.size(); i++) {
+ double UsefulPercentage =
+ Mutators[i].TotalCount
+ ? (100.0 * Mutators[i].UsefulCount) / Mutators[i].TotalCount
+ : 0;
+ Printf("%.3f", UsefulPercentage);
+ if (i < Mutators.size() - 1) Printf(",");
+ }
+ Printf("\n");
+}
+
} // namespace fuzzer
diff --git a/lib/fuzzer/FuzzerMutate.h b/lib/fuzzer/FuzzerMutate.h
index 4aa58af9902d..828ecc13d866 100644
--- a/lib/fuzzer/FuzzerMutate.h
+++ b/lib/fuzzer/FuzzerMutate.h
@@ -27,7 +27,7 @@ public:
void StartMutationSequence();
/// Print the current sequence of mutations.
void PrintMutationSequence();
- /// Indicate that the current sequence of mutations was successfull.
+ /// Indicate that the current sequence of mutations was successful.
void RecordSuccessfulMutationSequence();
/// Mutates data by invoking user-provided mutator.
size_t Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize);
@@ -70,6 +70,13 @@ public:
/// Applies one of the configured mutations.
/// Returns the new size of data which could be up to MaxSize.
size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize);
+
+ /// Applies one of the configured mutations to the bytes of Data
+ /// that have '1' in Mask.
+ /// Mask.size() should be >= Size.
+ size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize,
+ const Vector<uint8_t> &Mask);
+
/// Applies one of the default mutations. Provided as a service
/// to mutation authors.
size_t DefaultMutate(uint8_t *Data, size_t Size, size_t MaxSize);
@@ -86,11 +93,16 @@ public:
Random &GetRand() { return Rand; }
-private:
+ void PrintMutationStats();
+
+ void RecordUsefulMutations();
+ private:
struct Mutator {
size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max);
const char *Name;
+ uint64_t UsefulCount;
+ uint64_t TotalCount;
};
size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
@@ -125,11 +137,11 @@ private:
// recreated periodically.
Dictionary TempAutoDictionary;
// Persistent dictionary modified by the fuzzer, consists of
- // entries that led to successfull discoveries in the past mutations.
+ // entries that led to successful discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
- Vector<Mutator> CurrentMutatorSequence;
Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+ Vector<Mutator *> CurrentMutatorSequence;
static const size_t kCmpDictionaryEntriesDequeSize = 16;
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
@@ -137,6 +149,7 @@ private:
const InputCorpus *Corpus = nullptr;
Vector<uint8_t> MutateInPlaceHere;
+ Vector<uint8_t> MutateWithMaskTemp;
// CustomCrossOver needs its own buffer as a custom implementation may call
// LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere.
Vector<uint8_t> CustomCrossOverInPlaceHere;
diff --git a/lib/fuzzer/FuzzerOptions.h b/lib/fuzzer/FuzzerOptions.h
index 15a378020b85..ce39c0876cd7 100644
--- a/lib/fuzzer/FuzzerOptions.h
+++ b/lib/fuzzer/FuzzerOptions.h
@@ -18,7 +18,7 @@ namespace fuzzer {
struct FuzzingOptions {
int Verbosity = 1;
size_t MaxLen = 0;
- size_t ExperimentalLenControl = 0;
+ size_t LenControl = 1000;
int UnitTimeoutSec = 300;
int TimeoutExitCode = 77;
int ErrorExitCode = 77;
@@ -31,7 +31,7 @@ struct FuzzingOptions {
bool UseCounters = false;
bool UseMemmem = true;
bool UseCmp = false;
- bool UseValueProfile = false;
+ int UseValueProfile = false;
bool Shrink = false;
bool ReduceInputs = false;
int ReloadIntervalSec = 1;
@@ -45,18 +45,21 @@ struct FuzzingOptions {
std::string ExactArtifactPath;
std::string ExitOnSrcPos;
std::string ExitOnItem;
+ std::string FocusFunction;
+ std::string DataFlowTrace;
bool SaveArtifacts = true;
bool PrintNEW = true; // Print a status line when new units are found;
bool PrintNewCovPcs = false;
int PrintNewCovFuncs = 0;
bool PrintFinalStats = false;
+ bool PrintMutationStats = false;
bool PrintCorpusStats = false;
bool PrintCoverage = false;
+ bool PrintUnstableStats = false;
+ int HandleUnstable = 0;
bool DumpCoverage = false;
- bool UseClangCoverage = false;
bool DetectLeaks = true;
int PurgeAllocatorIntervalSec = 1;
- int UseFeatureFrequency = false;
int TraceMalloc = 0;
bool HandleAbrt = false;
bool HandleBus = false;
diff --git a/lib/fuzzer/FuzzerShmemPosix.cpp b/lib/fuzzer/FuzzerShmemPosix.cpp
index 50cdcfb509dc..41a93f61004b 100644
--- a/lib/fuzzer/FuzzerShmemPosix.cpp
+++ b/lib/fuzzer/FuzzerShmemPosix.cpp
@@ -32,6 +32,11 @@ std::string SharedMemoryRegion::Path(const char *Name) {
std::string SharedMemoryRegion::SemName(const char *Name, int Idx) {
std::string Res(Name);
+ // When passing a name without a leading <slash> character to
+ // sem_open, the behaviour is unspecified in POSIX. Add a leading
+ // <slash> character for the name if there is no such one.
+ if (!Res.empty() && Res[0] != '/')
+ Res.insert(Res.begin(), '/');
return Res + (char)('0' + Idx);
}
@@ -52,7 +57,7 @@ bool SharedMemoryRegion::Create(const char *Name) {
for (int i = 0; i < 2; i++) {
sem_unlink(SemName(Name, i).c_str());
Semaphore[i] = sem_open(SemName(Name, i).c_str(), O_CREAT, 0644, 0);
- if (Semaphore[i] == (void *)-1)
+ if (Semaphore[i] == SEM_FAILED)
return false;
}
IAmServer = true;
@@ -70,7 +75,7 @@ bool SharedMemoryRegion::Open(const char *Name) {
return false;
for (int i = 0; i < 2; i++) {
Semaphore[i] = sem_open(SemName(Name, i).c_str(), 0);
- if (Semaphore[i] == (void *)-1)
+ if (Semaphore[i] == SEM_FAILED)
return false;
}
IAmServer = false;
diff --git a/lib/fuzzer/FuzzerTracePC.cpp b/lib/fuzzer/FuzzerTracePC.cpp
index 5e9f9f2f6dcc..29ffc8e34fc0 100644
--- a/lib/fuzzer/FuzzerTracePC.cpp
+++ b/lib/fuzzer/FuzzerTracePC.cpp
@@ -39,8 +39,6 @@ namespace fuzzer {
TracePC TPC;
-int ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr;
-
uint8_t *TracePC::Counters() const {
return __sancov_trace_pc_guard_8bit_counters;
}
@@ -59,6 +57,49 @@ size_t TracePC::GetTotalPCCoverage() {
return Res;
}
+template<class CallBack>
+void TracePC::IterateInline8bitCounters(CallBack CB) const {
+ if (NumInline8bitCounters && NumInline8bitCounters == NumPCsInPCTables) {
+ size_t CounterIdx = 0;
+ for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) {
+ uint8_t *Beg = ModuleCounters[i].Start;
+ size_t Size = ModuleCounters[i].Stop - Beg;
+ assert(Size == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start));
+ for (size_t j = 0; j < Size; j++, CounterIdx++)
+ CB(i, j, CounterIdx);
+ }
+ }
+}
+
+// Initializes unstable counters by copying Inline8bitCounters to unstable
+// counters.
+void TracePC::InitializeUnstableCounters() {
+ IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
+ UnstableCounters[UnstableIdx].Counter = ModuleCounters[i].Start[j];
+ });
+}
+
+// Compares the current counters with counters from previous runs
+// and records differences as unstable edges.
+void TracePC::UpdateUnstableCounters(int UnstableMode) {
+ IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
+ if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) {
+ UnstableCounters[UnstableIdx].IsUnstable = true;
+ if (UnstableMode == ZeroUnstable)
+ UnstableCounters[UnstableIdx].Counter = 0;
+ else if (UnstableMode == MinUnstable)
+ UnstableCounters[UnstableIdx].Counter = std::min(
+ ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter);
+ }
+ });
+}
+
+// Moves the minimum hit counts to ModuleCounters.
+void TracePC::ApplyUnstableCounters() {
+ IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
+ ModuleCounters[i].Start[j] = UnstableCounters[UnstableIdx].Counter;
+ });
+}
void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) {
if (Start == Stop) return;
@@ -132,8 +173,8 @@ void TracePC::PrintModuleInfo() {
_Exit(1);
}
}
- if (size_t NumClangCounters = ClangCountersEnd() - ClangCountersBegin())
- Printf("INFO: %zd Clang Coverage Counters\n", NumClangCounters);
+ if (size_t NumExtraCounters = ExtraCountersEnd() - ExtraCountersBegin())
+ Printf("INFO: %zd Extra Counters\n", NumExtraCounters);
}
ATTRIBUTE_NO_SANITIZE_ALL
@@ -147,28 +188,25 @@ void TracePC::HandleCallerCallee(uintptr_t Caller, uintptr_t Callee) {
void TracePC::UpdateObservedPCs() {
Vector<uintptr_t> CoveredFuncs;
auto ObservePC = [&](uintptr_t PC) {
- if (ObservedPCs.insert(PC).second && DoPrintNewPCs)
- PrintPC("\tNEW_PC: %p %F %L\n", "\tNEW_PC: %p\n", PC + 1);
+ if (ObservedPCs.insert(PC).second && DoPrintNewPCs) {
+ PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", PC + 1);
+ Printf("\n");
+ }
};
auto Observe = [&](const PCTableEntry &TE) {
if (TE.PCFlags & 1)
- if (ObservedFuncs.insert(TE.PC).second && NumPrintNewFuncs)
+ if (++ObservedFuncs[TE.PC] == 1 && NumPrintNewFuncs)
CoveredFuncs.push_back(TE.PC);
ObservePC(TE.PC);
};
if (NumPCsInPCTables) {
if (NumInline8bitCounters == NumPCsInPCTables) {
- for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) {
- uint8_t *Beg = ModuleCounters[i].Start;
- size_t Size = ModuleCounters[i].Stop - Beg;
- assert(Size ==
- (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start));
- for (size_t j = 0; j < Size; j++)
- if (Beg[j])
- Observe(ModulePCTable[i].Start[j]);
- }
+ IterateInline8bitCounters([&](int i, int j, int CounterIdx) {
+ if (ModuleCounters[i].Start[j])
+ Observe(ModulePCTable[i].Start[j]);
+ });
} else if (NumGuards == NumPCsInPCTables) {
size_t GuardIdx = 1;
for (size_t i = 0; i < NumModules; i++) {
@@ -182,17 +220,12 @@ void TracePC::UpdateObservedPCs() {
}
}
}
- if (size_t NumClangCounters =
- ClangCountersEnd() - ClangCountersBegin()) {
- auto P = ClangCountersBegin();
- for (size_t Idx = 0; Idx < NumClangCounters; Idx++)
- if (P[Idx])
- ObservePC((uintptr_t)Idx);
- }
- for (size_t i = 0, N = Min(CoveredFuncs.size(), NumPrintNewFuncs); i < N; i++) {
- Printf("\tNEW_FUNC[%zd/%zd]: ", i, CoveredFuncs.size());
- PrintPC("%p %F %L\n", "%p\n", CoveredFuncs[i] + 1);
+ for (size_t i = 0, N = Min(CoveredFuncs.size(), NumPrintNewFuncs); i < N;
+ i++) {
+ Printf("\tNEW_FUNC[%zd/%zd]: ", i + 1, CoveredFuncs.size());
+ PrintPC("%p %F %L", "%p", CoveredFuncs[i] + 1);
+ Printf("\n");
}
}
@@ -218,6 +251,57 @@ static std::string GetModuleName(uintptr_t PC) {
return ModulePathRaw;
}
+template<class CallBack>
+void TracePC::IterateCoveredFunctions(CallBack CB) {
+ for (size_t i = 0; i < NumPCTables; i++) {
+ auto &M = ModulePCTable[i];
+ assert(M.Start < M.Stop);
+ auto ModuleName = GetModuleName(M.Start->PC);
+ for (auto NextFE = M.Start; NextFE < M.Stop; ) {
+ auto FE = NextFE;
+ assert((FE->PCFlags & 1) && "Not a function entry point");
+ do {
+ NextFE++;
+ } while (NextFE < M.Stop && !(NextFE->PCFlags & 1));
+ if (ObservedFuncs.count(FE->PC))
+ CB(FE, NextFE, ObservedFuncs[FE->PC]);
+ }
+ }
+}
+
+void TracePC::SetFocusFunction(const std::string &FuncName) {
+ // This function should be called once.
+ assert(FocusFunction.first > NumModulesWithInline8bitCounters);
+ if (FuncName.empty())
+ return;
+ for (size_t M = 0; M < NumModulesWithInline8bitCounters; M++) {
+ auto &PCTE = ModulePCTable[M];
+ size_t N = PCTE.Stop - PCTE.Start;
+ for (size_t I = 0; I < N; I++) {
+ if (!(PCTE.Start[I].PCFlags & 1)) continue; // not a function entry.
+ auto Name = DescribePC("%F", GetNextInstructionPc(PCTE.Start[I].PC));
+ if (Name[0] == 'i' && Name[1] == 'n' && Name[2] == ' ')
+ Name = Name.substr(3, std::string::npos);
+ if (FuncName != Name) continue;
+ Printf("INFO: Focus function is set to '%s'\n", Name.c_str());
+ FocusFunction = {M, I};
+ return;
+ }
+ }
+}
+
+bool TracePC::ObservedFocusFunction() {
+ size_t I = FocusFunction.first;
+ size_t J = FocusFunction.second;
+ if (I >= NumModulesWithInline8bitCounters)
+ return false;
+ auto &MC = ModuleCounters[I];
+ size_t Size = MC.Stop - MC.Start;
+ if (J >= Size)
+ return false;
+ return MC.Start[J] != 0;
+}
+
void TracePC::PrintCoverage() {
if (!EF->__sanitizer_symbolize_pc ||
!EF->__sanitizer_get_module_and_offset_for_pc) {
@@ -227,53 +311,33 @@ void TracePC::PrintCoverage() {
return;
}
Printf("COVERAGE:\n");
- std::string LastFunctionName = "";
- std::string LastFileStr = "";
- Set<size_t> UncoveredLines;
- Set<size_t> CoveredLines;
-
- auto FunctionEndCallback = [&](const std::string &CurrentFunc,
- const std::string &CurrentFile) {
- if (LastFunctionName != CurrentFunc) {
- if (CoveredLines.empty() && !UncoveredLines.empty()) {
- Printf("UNCOVERED_FUNC: %s\n", LastFunctionName.c_str());
- } else {
- for (auto Line : UncoveredLines) {
- if (!CoveredLines.count(Line))
- Printf("UNCOVERED_LINE: %s %s:%zd\n", LastFunctionName.c_str(),
- LastFileStr.c_str(), Line);
- }
- }
-
- UncoveredLines.clear();
- CoveredLines.clear();
- LastFunctionName = CurrentFunc;
- LastFileStr = CurrentFile;
- }
+ auto CoveredFunctionCallback = [&](const PCTableEntry *First,
+ const PCTableEntry *Last,
+ uintptr_t Counter) {
+ assert(First < Last);
+ auto VisualizePC = GetNextInstructionPc(First->PC);
+ std::string FileStr = DescribePC("%s", VisualizePC);
+ if (!IsInterestingCoverageFile(FileStr))
+ return;
+ std::string FunctionStr = DescribePC("%F", VisualizePC);
+ if (FunctionStr.find("in ") == 0)
+ FunctionStr = FunctionStr.substr(3);
+ std::string LineStr = DescribePC("%l", VisualizePC);
+ size_t Line = std::stoul(LineStr);
+ size_t NumEdges = Last - First;
+ Vector<uintptr_t> UncoveredPCs;
+ for (auto TE = First; TE < Last; TE++)
+ if (!ObservedPCs.count(TE->PC))
+ UncoveredPCs.push_back(TE->PC);
+ Printf("COVERED_FUNC: hits: %zd", Counter);
+ Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges);
+ Printf(" %s %s:%zd\n", FunctionStr.c_str(), FileStr.c_str(), Line);
+ for (auto PC: UncoveredPCs)
+ Printf(" UNCOVERED_PC: %s\n",
+ DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str());
};
- for (size_t i = 0; i < NumPCTables; i++) {
- auto &M = ModulePCTable[i];
- assert(M.Start < M.Stop);
- auto ModuleName = GetModuleName(M.Start->PC);
- for (auto Ptr = M.Start; Ptr < M.Stop; Ptr++) {
- auto PC = Ptr->PC;
- auto VisualizePC = GetNextInstructionPc(PC);
- bool IsObserved = ObservedPCs.count(PC);
- std::string FileStr = DescribePC("%s", VisualizePC);
- if (!IsInterestingCoverageFile(FileStr)) continue;
- std::string FunctionStr = DescribePC("%F", VisualizePC);
- FunctionEndCallback(FunctionStr, FileStr);
- std::string LineStr = DescribePC("%l", VisualizePC);
- size_t Line = std::stoul(LineStr);
- if (IsObserved && CoveredLines.insert(Line).second)
- Printf("COVERED: %s %s:%zd\n", FunctionStr.c_str(), FileStr.c_str(),
- Line);
- else
- UncoveredLines.insert(Line);
- }
- }
- FunctionEndCallback("", "");
+ IterateCoveredFunctions(CoveredFunctionCallback);
}
void TracePC::DumpCoverage() {
@@ -285,6 +349,15 @@ void TracePC::DumpCoverage() {
}
}
+void TracePC::PrintUnstableStats() {
+ size_t count = 0;
+ for (size_t i = 0; i < NumInline8bitCounters; i++)
+ if (UnstableCounters[i].IsUnstable)
+ count++;
+ Printf("stat::stability_rate: %.2f\n",
+ 100 - static_cast<float>(count * 100) / NumInline8bitCounters);
+}
+
// Value profile.
// We keep track of various values that affect control flow.
// These values are inserted into a bit-set-based hash map.
@@ -334,7 +407,14 @@ void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) {
TORC4.Insert(ArgXor, Arg1, Arg2);
else if (sizeof(T) == 8)
TORC8.Insert(ArgXor, Arg1, Arg2);
- ValueProfileMap.AddValue(Idx);
+ // TODO: remove these flags and instead use all metrics at once.
+ if (UseValueProfileMask & 1)
+ ValueProfileMap.AddValue(Idx);
+ if (UseValueProfileMask & 2)
+ ValueProfileMap.AddValue(
+ PC * 64 + (Arg1 == Arg2 ? 0 : __builtin_clzll(Arg1 - Arg2) + 1));
+ if (UseValueProfileMask & 4) // alternative way to use the hamming distance
+ ValueProfileMap.AddValue(PC * 64 + ArgDistance);
}
static size_t InternalStrnlen(const char *S, size_t MaxLen) {
@@ -536,7 +616,7 @@ void __sanitizer_cov_trace_gep(uintptr_t Idx) {
ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1,
const void *s2, size_t n, int result) {
- if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
+ if (!fuzzer::RunningUserCallback) return;
if (result == 0) return; // No reason to mutate.
if (n <= 1) return; // Not interesting.
fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/false);
@@ -545,7 +625,7 @@ void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1,
ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1,
const char *s2, size_t n, int result) {
- if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
+ if (!fuzzer::RunningUserCallback) return;
if (result == 0) return; // No reason to mutate.
size_t Len1 = fuzzer::InternalStrnlen(s1, n);
size_t Len2 = fuzzer::InternalStrnlen(s2, n);
@@ -558,7 +638,7 @@ void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1,
ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1,
const char *s2, int result) {
- if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
+ if (!fuzzer::RunningUserCallback) return;
if (result == 0) return; // No reason to mutate.
size_t N = fuzzer::InternalStrnlen2(s1, s2);
if (N <= 1) return; // Not interesting.
@@ -568,35 +648,35 @@ void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1,
ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1,
const char *s2, size_t n, int result) {
- if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
+ if (!fuzzer::RunningUserCallback) return;
return __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result);
}
ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1,
const char *s2, int result) {
- if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
+ if (!fuzzer::RunningUserCallback) return;
return __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result);
}
ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1,
const char *s2, char *result) {
- if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
+ if (!fuzzer::RunningUserCallback) return;
fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), strlen(s2));
}
ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1,
const char *s2, char *result) {
- if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
+ if (!fuzzer::RunningUserCallback) return;
fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), strlen(s2));
}
ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1,
const void *s2, size_t len2, void *result) {
- if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
+ if (!fuzzer::RunningUserCallback) return;
fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), len2);
}
} // extern "C"
diff --git a/lib/fuzzer/FuzzerTracePC.h b/lib/fuzzer/FuzzerTracePC.h
index c3f241b905b1..097ba69bdc08 100644
--- a/lib/fuzzer/FuzzerTracePC.h
+++ b/lib/fuzzer/FuzzerTracePC.h
@@ -17,6 +17,7 @@
#include "FuzzerValueBitMap.h"
#include <set>
+#include <unordered_map>
namespace fuzzer {
@@ -73,6 +74,11 @@ class TracePC {
// How many bits of PC are used from __sanitizer_cov_trace_pc.
static const size_t kTracePcBits = 18;
+ enum HandleUnstableOptions {
+ MinUnstable = 1,
+ ZeroUnstable = 2,
+ };
+
void HandleInit(uint32_t *Start, uint32_t *Stop);
void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop);
void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop);
@@ -80,8 +86,7 @@ class TracePC {
template <class T> void HandleCmp(uintptr_t PC, T Arg1, T Arg2);
size_t GetTotalPCCoverage();
void SetUseCounters(bool UC) { UseCounters = UC; }
- void SetUseClangCoverage(bool UCC) { UseClangCoverage = UCC; }
- void SetUseValueProfile(bool VP) { UseValueProfile = VP; }
+ void SetUseValueProfileMask(uint32_t VPMask) { UseValueProfileMask = VPMask; }
void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; }
void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; }
void UpdateObservedPCs();
@@ -93,8 +98,6 @@ class TracePC {
memset(Counters(), 0, GetNumPCs());
ClearExtraCounters();
ClearInlineCounters();
- if (UseClangCoverage)
- ClearClangCounters();
}
void ClearInlineCounters();
@@ -106,6 +109,10 @@ class TracePC {
void PrintCoverage();
void DumpCoverage();
+ void PrintUnstableStats();
+
+ template<class CallBack>
+ void IterateCoveredFunctions(CallBack CB);
void AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2,
size_t n, bool StopAtZero);
@@ -132,10 +139,23 @@ class TracePC {
CB(PC);
}
+ void SetFocusFunction(const std::string &FuncName);
+ bool ObservedFocusFunction();
+
+ void InitializeUnstableCounters();
+ void UpdateUnstableCounters(int UnstableMode);
+ void ApplyUnstableCounters();
+
private:
+ struct UnstableEdge {
+ uint8_t Counter;
+ bool IsUnstable;
+ };
+
+ UnstableEdge UnstableCounters[kNumPCs];
+
bool UseCounters = false;
- bool UseValueProfile = false;
- bool UseClangCoverage = false;
+ uint32_t UseValueProfileMask = false;
bool DoPrintNewPCs = false;
size_t NumPrintNewFuncs = 0;
@@ -163,7 +183,12 @@ private:
uintptr_t *PCs() const;
Set<uintptr_t> ObservedPCs;
- Set<uintptr_t> ObservedFuncs;
+ std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter.
+
+ template <class Callback>
+ void IterateInline8bitCounters(Callback CB) const;
+
+ std::pair<size_t, size_t> FocusFunction = {-1, -1}; // Module and PC IDs.
ValueBitMap ValueProfileMap;
uintptr_t InitialStack;
@@ -251,23 +276,11 @@ void TracePC::CollectFeatures(Callback HandleFeature) const {
}
}
- if (size_t NumClangCounters = ClangCountersEnd() - ClangCountersBegin()) {
- auto P = ClangCountersBegin();
- for (size_t Idx = 0; Idx < NumClangCounters; Idx++)
- if (auto Cnt = P[Idx]) {
- if (UseCounters)
- HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Cnt));
- else
- HandleFeature(FirstFeature + Idx);
- }
- FirstFeature += NumClangCounters;
- }
-
ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), FirstFeature,
Handle8bitCounter);
FirstFeature += (ExtraCountersEnd() - ExtraCountersBegin()) * 8;
- if (UseValueProfile) {
+ if (UseValueProfileMask) {
ValueProfileMap.ForEach([&](size_t Idx) {
HandleFeature(FirstFeature + Idx);
});
diff --git a/lib/fuzzer/FuzzerUtil.cpp b/lib/fuzzer/FuzzerUtil.cpp
index 96b37d34815d..6286f9a718ad 100644
--- a/lib/fuzzer/FuzzerUtil.cpp
+++ b/lib/fuzzer/FuzzerUtil.cpp
@@ -16,6 +16,7 @@
#include <chrono>
#include <cstring>
#include <errno.h>
+#include <mutex>
#include <signal.h>
#include <sstream>
#include <stdio.h>
@@ -179,8 +180,12 @@ std::string Base64(const Unit &U) {
return Res;
}
+static std::mutex SymbolizeMutex;
+
std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC) {
- if (!EF->__sanitizer_symbolize_pc) return "<can not symbolize>";
+ std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+ if (!EF->__sanitizer_symbolize_pc || !l.owns_lock())
+ return "<can not symbolize>";
char PcDescr[1024] = {};
EF->__sanitizer_symbolize_pc(reinterpret_cast<void*>(PC),
SymbolizedFMT, PcDescr, sizeof(PcDescr));
@@ -195,6 +200,18 @@ void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC) {
Printf(FallbackFMT, PC);
}
+void PrintStackTrace() {
+ std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+ if (EF->__sanitizer_print_stack_trace && l.owns_lock())
+ EF->__sanitizer_print_stack_trace();
+}
+
+void PrintMemoryProfile() {
+ std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+ if (EF->__sanitizer_print_memory_profile && l.owns_lock())
+ EF->__sanitizer_print_memory_profile(95, 8);
+}
+
unsigned NumberOfCpuCores() {
unsigned N = std::thread::hardware_concurrency();
if (!N) {
diff --git a/lib/fuzzer/FuzzerUtil.h b/lib/fuzzer/FuzzerUtil.h
index f2ed028ce78e..8c5c57c3ab83 100644
--- a/lib/fuzzer/FuzzerUtil.h
+++ b/lib/fuzzer/FuzzerUtil.h
@@ -40,6 +40,10 @@ void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC);
std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC);
+void PrintStackTrace();
+
+void PrintMemoryProfile();
+
unsigned NumberOfCpuCores();
// Platform specific functions.
diff --git a/lib/fuzzer/FuzzerUtilFuchsia.cpp b/lib/fuzzer/FuzzerUtilFuchsia.cpp
index a5fdc37fb5a6..cd2bb7438e9d 100644
--- a/lib/fuzzer/FuzzerUtilFuchsia.cpp
+++ b/lib/fuzzer/FuzzerUtilFuchsia.cpp
@@ -17,29 +17,54 @@
#include <cerrno>
#include <cinttypes>
#include <cstdint>
-#include <fbl/unique_fd.h>
#include <fcntl.h>
-#include <launchpad/launchpad.h>
+#include <lib/fdio/spawn.h>
#include <string>
+#include <sys/select.h>
#include <thread>
+#include <unistd.h>
#include <zircon/errors.h>
+#include <zircon/process.h>
+#include <zircon/sanitizer.h>
#include <zircon/status.h>
#include <zircon/syscalls.h>
+#include <zircon/syscalls/debug.h>
+#include <zircon/syscalls/exception.h>
#include <zircon/syscalls/port.h>
#include <zircon/types.h>
-#include <zx/object.h>
-#include <zx/port.h>
-#include <zx/process.h>
-#include <zx/time.h>
namespace fuzzer {
+// Given that Fuchsia doesn't have the POSIX signals that libFuzzer was written
+// around, the general approach is to spin up dedicated threads to watch for
+// each requested condition (alarm, interrupt, crash). Of these, the crash
+// handler is the most involved, as it requires resuming the crashed thread in
+// order to invoke the sanitizers to get the needed state.
+
+// Forward declaration of assembly trampoline needed to resume crashed threads.
+// This appears to have external linkage to C++, which is why it's not in the
+// anonymous namespace. The assembly definition inside MakeTrampoline()
+// actually defines the symbol with internal linkage only.
+void CrashTrampolineAsm() __asm__("CrashTrampolineAsm");
+
namespace {
+// TODO(phosek): remove this and replace it with ZX_TIME_INFINITE
+#define ZX_TIME_INFINITE_OLD INT64_MAX
+
// A magic value for the Zircon exception port, chosen to spell 'FUZZING'
// when interpreted as a byte sequence on little-endian platforms.
const uint64_t kFuzzingCrash = 0x474e495a5a5546;
+// Helper function to handle Zircon syscall failures.
+void ExitOnErr(zx_status_t Status, const char *Syscall) {
+ if (Status != ZX_OK) {
+ Printf("libFuzzer: %s failed: %s\n", Syscall,
+ _zx_status_get_string(Status));
+ exit(1);
+ }
+}
+
void AlarmHandler(int Seconds) {
while (true) {
SleepSeconds(Seconds);
@@ -48,37 +73,226 @@ void AlarmHandler(int Seconds) {
}
void InterruptHandler() {
+ fd_set readfds;
// Ctrl-C sends ETX in Zircon.
- while (getchar() != 0x03);
+ do {
+ FD_ZERO(&readfds);
+ FD_SET(STDIN_FILENO, &readfds);
+ select(STDIN_FILENO + 1, &readfds, nullptr, nullptr, nullptr);
+ } while(!FD_ISSET(STDIN_FILENO, &readfds) || getchar() != 0x03);
Fuzzer::StaticInterruptCallback();
}
-void CrashHandler(zx::port *Port) {
- std::unique_ptr<zx::port> ExceptionPort(Port);
- zx_port_packet_t Packet;
- ExceptionPort->wait(ZX_TIME_INFINITE, &Packet, 0);
- // Unbind as soon as possible so we don't receive exceptions from this thread.
- if (zx_task_bind_exception_port(ZX_HANDLE_INVALID, ZX_HANDLE_INVALID,
- kFuzzingCrash, 0) != ZX_OK) {
- // Shouldn't happen; if it does the safest option is to just exit.
- Printf("libFuzzer: unable to unbind exception port; aborting!\n");
- exit(1);
- }
- if (Packet.key != kFuzzingCrash) {
- Printf("libFuzzer: invalid crash key: %" PRIx64 "; aborting!\n",
- Packet.key);
- exit(1);
- }
- // CrashCallback should not return from this call
+// For the crash handler, we need to call Fuzzer::StaticCrashSignalCallback
+// without POSIX signal handlers. To achieve this, we use an assembly function
+// to add the necessary CFI unwinding information and a C function to bridge
+// from that back into C++.
+
+// FIXME: This works as a short-term solution, but this code really shouldn't be
+// architecture dependent. A better long term solution is to implement remote
+// unwinding and expose the necessary APIs through sanitizer_common and/or ASAN
+// to allow the exception handling thread to gather the crash state directly.
+//
+// Alternatively, Fuchsia may in future actually implement basic signal
+// handling for the machine trap signals.
+#if defined(__x86_64__)
+#define FOREACH_REGISTER(OP_REG, OP_NUM) \
+ OP_REG(rax) \
+ OP_REG(rbx) \
+ OP_REG(rcx) \
+ OP_REG(rdx) \
+ OP_REG(rsi) \
+ OP_REG(rdi) \
+ OP_REG(rbp) \
+ OP_REG(rsp) \
+ OP_REG(r8) \
+ OP_REG(r9) \
+ OP_REG(r10) \
+ OP_REG(r11) \
+ OP_REG(r12) \
+ OP_REG(r13) \
+ OP_REG(r14) \
+ OP_REG(r15) \
+ OP_REG(rip)
+
+#elif defined(__aarch64__)
+#define FOREACH_REGISTER(OP_REG, OP_NUM) \
+ OP_NUM(0) \
+ OP_NUM(1) \
+ OP_NUM(2) \
+ OP_NUM(3) \
+ OP_NUM(4) \
+ OP_NUM(5) \
+ OP_NUM(6) \
+ OP_NUM(7) \
+ OP_NUM(8) \
+ OP_NUM(9) \
+ OP_NUM(10) \
+ OP_NUM(11) \
+ OP_NUM(12) \
+ OP_NUM(13) \
+ OP_NUM(14) \
+ OP_NUM(15) \
+ OP_NUM(16) \
+ OP_NUM(17) \
+ OP_NUM(18) \
+ OP_NUM(19) \
+ OP_NUM(20) \
+ OP_NUM(21) \
+ OP_NUM(22) \
+ OP_NUM(23) \
+ OP_NUM(24) \
+ OP_NUM(25) \
+ OP_NUM(26) \
+ OP_NUM(27) \
+ OP_NUM(28) \
+ OP_NUM(29) \
+ OP_NUM(30) \
+ OP_REG(sp)
+
+#else
+#error "Unsupported architecture for fuzzing on Fuchsia"
+#endif
+
+// Produces a CFI directive for the named or numbered register.
+#define CFI_OFFSET_REG(reg) ".cfi_offset " #reg ", %c[" #reg "]\n"
+#define CFI_OFFSET_NUM(num) CFI_OFFSET_REG(r##num)
+
+// Produces an assembler input operand for the named or numbered register.
+#define ASM_OPERAND_REG(reg) \
+ [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg)),
+#define ASM_OPERAND_NUM(num) \
+ [r##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num])),
+
+// Trampoline to bridge from the assembly below to the static C++ crash
+// callback.
+__attribute__((noreturn))
+static void StaticCrashHandler() {
Fuzzer::StaticCrashSignalCallback();
+ for (;;) {
+ _Exit(1);
+ }
+}
+
+// Creates the trampoline with the necessary CFI information to unwind through
+// to the crashing call stack. The attribute is necessary because the function
+// is never called; it's just a container around the assembly to allow it to
+// use operands for compile-time computed constants.
+__attribute__((used))
+void MakeTrampoline() {
+ __asm__(".cfi_endproc\n"
+ ".pushsection .text.CrashTrampolineAsm\n"
+ ".type CrashTrampolineAsm,STT_FUNC\n"
+"CrashTrampolineAsm:\n"
+ ".cfi_startproc simple\n"
+ ".cfi_signal_frame\n"
+#if defined(__x86_64__)
+ ".cfi_return_column rip\n"
+ ".cfi_def_cfa rsp, 0\n"
+ FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
+ "call %c[StaticCrashHandler]\n"
+ "ud2\n"
+#elif defined(__aarch64__)
+ ".cfi_return_column 33\n"
+ ".cfi_def_cfa sp, 0\n"
+ ".cfi_offset 33, %c[pc]\n"
+ FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
+ "bl %[StaticCrashHandler]\n"
+#else
+#error "Unsupported architecture for fuzzing on Fuchsia"
+#endif
+ ".cfi_endproc\n"
+ ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n"
+ ".popsection\n"
+ ".cfi_startproc\n"
+ : // No outputs
+ : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM)
+#if defined(__aarch64__)
+ ASM_OPERAND_REG(pc)
+#endif
+ [StaticCrashHandler] "i" (StaticCrashHandler));
+}
+
+void CrashHandler(zx_handle_t *Event) {
+ // This structure is used to ensure we close handles to objects we create in
+ // this handler.
+ struct ScopedHandle {
+ ~ScopedHandle() { _zx_handle_close(Handle); }
+ zx_handle_t Handle = ZX_HANDLE_INVALID;
+ };
+
+ // Create and bind the exception port. We need to claim to be a "debugger" so
+ // the kernel will allow us to modify and resume dying threads (see below).
+ // Once the port is set, we can signal the main thread to continue and wait
+ // for the exception to arrive.
+ ScopedHandle Port;
+ ExitOnErr(_zx_port_create(0, &Port.Handle), "_zx_port_create");
+ zx_handle_t Self = _zx_process_self();
+
+ ExitOnErr(_zx_task_bind_exception_port(Self, Port.Handle, kFuzzingCrash,
+ ZX_EXCEPTION_PORT_DEBUGGER),
+ "_zx_task_bind_exception_port");
+
+ ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0),
+ "_zx_object_signal");
+
+ zx_port_packet_t Packet;
+ ExitOnErr(_zx_port_wait(Port.Handle, ZX_TIME_INFINITE_OLD, &Packet),
+ "_zx_port_wait");
+
+ // At this point, we want to get the state of the crashing thread, but
+ // libFuzzer and the sanitizers assume this will happen from that same thread
+ // via a POSIX signal handler. "Resurrecting" the thread in the middle of the
+ // appropriate callback is as simple as forcibly setting the instruction
+ // pointer/program counter, provided we NEVER EVER return from that function
+ // (since otherwise our stack will not be valid).
+ ScopedHandle Thread;
+ ExitOnErr(_zx_object_get_child(Self, Packet.exception.tid,
+ ZX_RIGHT_SAME_RIGHTS, &Thread.Handle),
+ "_zx_object_get_child");
+
+ zx_thread_state_general_regs_t GeneralRegisters;
+ ExitOnErr(_zx_thread_read_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS,
+ &GeneralRegisters, sizeof(GeneralRegisters)),
+ "_zx_thread_read_state");
+
+ // To unwind properly, we need to push the crashing thread's register state
+ // onto the stack and jump into a trampoline with CFI instructions on how
+ // to restore it.
+#if defined(__x86_64__)
+ uintptr_t StackPtr =
+ (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) &
+ -(uintptr_t)16;
+ __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters,
+ sizeof(GeneralRegisters));
+ GeneralRegisters.rsp = StackPtr;
+ GeneralRegisters.rip = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm);
+
+#elif defined(__aarch64__)
+ uintptr_t StackPtr =
+ (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16;
+ __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters,
+ sizeof(GeneralRegisters));
+ GeneralRegisters.sp = StackPtr;
+ GeneralRegisters.pc = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm);
+
+#else
+#error "Unsupported architecture for fuzzing on Fuchsia"
+#endif
+
+ // Now force the crashing thread's state.
+ ExitOnErr(_zx_thread_write_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS,
+ &GeneralRegisters, sizeof(GeneralRegisters)),
+ "_zx_thread_write_state");
+
+ ExitOnErr(_zx_task_resume_from_exception(Thread.Handle, Port.Handle, 0),
+ "_zx_task_resume_from_exception");
}
} // namespace
// Platform specific functions.
void SetSignalHandler(const FuzzingOptions &Options) {
- zx_status_t rc;
-
// Set up alarm handler if needed.
if (Options.UnitTimeoutSec > 0) {
std::thread T(AlarmHandler, Options.UnitTimeoutSec / 2 + 1);
@@ -96,37 +310,30 @@ void SetSignalHandler(const FuzzingOptions &Options) {
!Options.HandleFpe && !Options.HandleAbrt)
return;
- // Create an exception port
- zx::port *ExceptionPort = new zx::port();
- if ((rc = zx::port::create(0, ExceptionPort)) != ZX_OK) {
- Printf("libFuzzer: zx_port_create failed: %s\n", zx_status_get_string(rc));
- exit(1);
- }
+ // Set up the crash handler and wait until it is ready before proceeding.
+ zx_handle_t Event;
+ ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create");
- // Bind the port to receive exceptions from our process
- if ((rc = zx_task_bind_exception_port(zx_process_self(), ExceptionPort->get(),
- kFuzzingCrash, 0)) != ZX_OK) {
- Printf("libFuzzer: unable to bind exception port: %s\n",
- zx_status_get_string(rc));
- exit(1);
- }
+ std::thread T(CrashHandler, &Event);
+ zx_status_t Status = _zx_object_wait_one(Event, ZX_USER_SIGNAL_0,
+ ZX_TIME_INFINITE_OLD, nullptr);
+ _zx_handle_close(Event);
+ ExitOnErr(Status, "_zx_object_wait_one");
- // Set up the crash handler.
- std::thread T(CrashHandler, ExceptionPort);
T.detach();
}
void SleepSeconds(int Seconds) {
- zx::nanosleep(zx::deadline_after(ZX_SEC(Seconds)));
+ _zx_nanosleep(_zx_deadline_after(ZX_SEC(Seconds)));
}
unsigned long GetPid() {
zx_status_t rc;
zx_info_handle_basic_t Info;
- if ((rc = zx_object_get_info(zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info,
- sizeof(Info), NULL, NULL)) != ZX_OK) {
+ if ((rc = _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info,
+ sizeof(Info), NULL, NULL)) != ZX_OK) {
Printf("libFuzzer: unable to get info about self: %s\n",
- zx_status_get_string(rc));
+ _zx_status_get_string(rc));
exit(1);
}
return Info.koid;
@@ -135,15 +342,30 @@ unsigned long GetPid() {
size_t GetPeakRSSMb() {
zx_status_t rc;
zx_info_task_stats_t Info;
- if ((rc = zx_object_get_info(zx_process_self(), ZX_INFO_TASK_STATS, &Info,
- sizeof(Info), NULL, NULL)) != ZX_OK) {
+ if ((rc = _zx_object_get_info(_zx_process_self(), ZX_INFO_TASK_STATS, &Info,
+ sizeof(Info), NULL, NULL)) != ZX_OK) {
Printf("libFuzzer: unable to get info about self: %s\n",
- zx_status_get_string(rc));
+ _zx_status_get_string(rc));
exit(1);
}
return (Info.mem_private_bytes + Info.mem_shared_bytes) >> 20;
}
+template <typename Fn>
+class RunOnDestruction {
+ public:
+ explicit RunOnDestruction(Fn fn) : fn_(fn) {}
+ ~RunOnDestruction() { fn_(); }
+
+ private:
+ Fn fn_;
+};
+
+template <typename Fn>
+RunOnDestruction<Fn> at_scope_exit(Fn fn) {
+ return RunOnDestruction<Fn>(fn);
+}
+
int ExecuteCommand(const Command &Cmd) {
zx_status_t rc;
@@ -151,30 +373,24 @@ int ExecuteCommand(const Command &Cmd) {
auto Args = Cmd.getArguments();
size_t Argc = Args.size();
assert(Argc != 0);
- std::unique_ptr<const char *[]> Argv(new const char *[Argc]);
+ std::unique_ptr<const char *[]> Argv(new const char *[Argc + 1]);
for (size_t i = 0; i < Argc; ++i)
Argv[i] = Args[i].c_str();
-
- // Create the basic launchpad. Clone everything except stdio.
- launchpad_t *lp;
- launchpad_create(ZX_HANDLE_INVALID, Argv[0], &lp);
- launchpad_load_from_file(lp, Argv[0]);
- launchpad_set_args(lp, Argc, Argv.get());
- launchpad_clone(lp, LP_CLONE_ALL & (~LP_CLONE_FDIO_STDIO));
+ Argv[Argc] = nullptr;
// Determine stdout
int FdOut = STDOUT_FILENO;
- fbl::unique_fd OutputFile;
+
if (Cmd.hasOutputFile()) {
auto Filename = Cmd.getOutputFile();
- OutputFile.reset(open(Filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0));
- if (!OutputFile) {
+ FdOut = open(Filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0);
+ if (FdOut == -1) {
Printf("libFuzzer: failed to open %s: %s\n", Filename.c_str(),
strerror(errno));
return ZX_ERR_IO;
}
- FdOut = OutputFile.get();
}
+ auto CloseFdOut = at_scope_exit([&]() { close(FdOut); } );
// Determine stderr
int FdErr = STDERR_FILENO;
@@ -182,36 +398,59 @@ int ExecuteCommand(const Command &Cmd) {
FdErr = FdOut;
// Clone the file descriptors into the new process
- if ((rc = launchpad_clone_fd(lp, STDIN_FILENO, STDIN_FILENO)) != ZX_OK ||
- (rc = launchpad_clone_fd(lp, FdOut, STDOUT_FILENO)) != ZX_OK ||
- (rc = launchpad_clone_fd(lp, FdErr, STDERR_FILENO)) != ZX_OK) {
- Printf("libFuzzer: failed to clone FDIO: %s\n", zx_status_get_string(rc));
- return rc;
- }
-
- // Start the process
+ fdio_spawn_action_t SpawnAction[] = {
+ {
+ .action = FDIO_SPAWN_ACTION_CLONE_FD,
+ .fd =
+ {
+ .local_fd = STDIN_FILENO,
+ .target_fd = STDIN_FILENO,
+ },
+ },
+ {
+ .action = FDIO_SPAWN_ACTION_CLONE_FD,
+ .fd =
+ {
+ .local_fd = FdOut,
+ .target_fd = STDOUT_FILENO,
+ },
+ },
+ {
+ .action = FDIO_SPAWN_ACTION_CLONE_FD,
+ .fd =
+ {
+ .local_fd = FdErr,
+ .target_fd = STDERR_FILENO,
+ },
+ },
+ };
+
+ // Start the process.
+ char ErrorMsg[FDIO_SPAWN_ERR_MSG_MAX_LENGTH];
zx_handle_t ProcessHandle = ZX_HANDLE_INVALID;
- const char *ErrorMsg = nullptr;
- if ((rc = launchpad_go(lp, &ProcessHandle, &ErrorMsg)) != ZX_OK) {
+ rc = fdio_spawn_etc(
+ ZX_HANDLE_INVALID, FDIO_SPAWN_CLONE_ALL & (~FDIO_SPAWN_CLONE_STDIO),
+ Argv[0], Argv.get(), nullptr, 3, SpawnAction, &ProcessHandle, ErrorMsg);
+ if (rc != ZX_OK) {
Printf("libFuzzer: failed to launch '%s': %s, %s\n", Argv[0], ErrorMsg,
- zx_status_get_string(rc));
+ _zx_status_get_string(rc));
return rc;
}
- zx::process Process(ProcessHandle);
+ auto CloseHandle = at_scope_exit([&]() { _zx_handle_close(ProcessHandle); });
// Now join the process and return the exit status.
- if ((rc = Process.wait_one(ZX_PROCESS_TERMINATED, ZX_TIME_INFINITE,
- nullptr)) != ZX_OK) {
+ if ((rc = _zx_object_wait_one(ProcessHandle, ZX_PROCESS_TERMINATED,
+ ZX_TIME_INFINITE_OLD, nullptr)) != ZX_OK) {
Printf("libFuzzer: failed to join '%s': %s\n", Argv[0],
- zx_status_get_string(rc));
+ _zx_status_get_string(rc));
return rc;
}
zx_info_process_t Info;
- if ((rc = Process.get_info(ZX_INFO_PROCESS, &Info, sizeof(Info), nullptr,
- nullptr)) != ZX_OK) {
+ if ((rc = _zx_object_get_info(ProcessHandle, ZX_INFO_PROCESS, &Info,
+ sizeof(Info), nullptr, nullptr)) != ZX_OK) {
Printf("libFuzzer: unable to get return code from '%s': %s\n", Argv[0],
- zx_status_get_string(rc));
+ _zx_status_get_string(rc));
return rc;
}
diff --git a/lib/fuzzer/FuzzerUtilLinux.cpp b/lib/fuzzer/FuzzerUtilLinux.cpp
index c7cf2c0a778b..c103fd230b0c 100644
--- a/lib/fuzzer/FuzzerUtilLinux.cpp
+++ b/lib/fuzzer/FuzzerUtilLinux.cpp
@@ -9,7 +9,8 @@
// Misc utils for Linux.
//===----------------------------------------------------------------------===//
#include "FuzzerDefs.h"
-#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD
+#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \
+ LIBFUZZER_OPENBSD
#include "FuzzerCommand.h"
#include <stdlib.h>
@@ -23,4 +24,4 @@ int ExecuteCommand(const Command &Cmd) {
} // namespace fuzzer
-#endif // LIBFUZZER_LINUX || LIBFUZZER_NETBSD
+#endif
diff --git a/lib/fuzzer/FuzzerUtilPosix.cpp b/lib/fuzzer/FuzzerUtilPosix.cpp
index 934b7aa98ff1..bc64d3293702 100644
--- a/lib/fuzzer/FuzzerUtilPosix.cpp
+++ b/lib/fuzzer/FuzzerUtilPosix.cpp
@@ -118,7 +118,8 @@ size_t GetPeakRSSMb() {
struct rusage usage;
if (getrusage(RUSAGE_SELF, &usage))
return 0;
- if (LIBFUZZER_LINUX) {
+ if (LIBFUZZER_LINUX || LIBFUZZER_FREEBSD || LIBFUZZER_NETBSD ||
+ LIBFUZZER_OPENBSD) {
// ru_maxrss is in KiB
return usage.ru_maxrss >> 10;
} else if (LIBFUZZER_APPLE) {
diff --git a/lib/fuzzer/afl/afl_driver.cpp b/lib/fuzzer/afl/afl_driver.cpp
index bbe5be795ed4..fa494c03bde0 100644
--- a/lib/fuzzer/afl/afl_driver.cpp
+++ b/lib/fuzzer/afl/afl_driver.cpp
@@ -69,20 +69,38 @@ statistics from the file. If that fails then the process will quit.
#define LIBFUZZER_LINUX 1
#define LIBFUZZER_APPLE 0
#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 0
#elif __APPLE__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_APPLE 1
#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 0
#elif __NetBSD__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_APPLE 0
#define LIBFUZZER_NETBSD 1
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 0
+#elif __FreeBSD__
+#define LIBFUZZER_LINUX 0
+#define LIBFUZZER_APPLE 0
+#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 1
+#define LIBFUZZER_OPENBSD 0
+#elif __OpenBSD__
+#define LIBFUZZER_LINUX 0
+#define LIBFUZZER_APPLE 0
+#define LIBFUZZER_NETBSD 0
+#define LIBFUZZER_FREEBSD 0
+#define LIBFUZZER_OPENBSD 1
#else
#error "Support for your platform has not been implemented"
#endif
// Used to avoid repeating error checking boilerplate. If cond is false, a
-// fatal error has occured in the program. In this event print error_message
+// fatal error has occurred in the program. In this event print error_message
// to stderr and abort(). Otherwise do nothing. Note that setting
// AFL_DRIVER_STDERR_DUPLICATE_FILENAME may cause error_message to be appended
// to the file as well, if the error occurs after the duplication is performed.
@@ -120,12 +138,24 @@ static const int kNumExtraStats = 2;
static const char *kExtraStatsFormatString = "peak_rss_mb : %u\n"
"slowest_unit_time_sec : %u\n";
+// Experimental feature to use afl_driver without AFL's deferred mode.
+// Needs to run before __afl_auto_init.
+__attribute__((constructor(0))) void __decide_deferred_forkserver(void) {
+ if (getenv("AFL_DRIVER_DONT_DEFER")) {
+ if (unsetenv("__AFL_DEFER_FORKSRV")) {
+ perror("Failed to unset __AFL_DEFER_FORKSRV");
+ abort();
+ }
+ }
+}
+
// Copied from FuzzerUtil.cpp.
size_t GetPeakRSSMb() {
struct rusage usage;
if (getrusage(RUSAGE_SELF, &usage))
return 0;
- if (LIBFUZZER_LINUX || LIBFUZZER_NETBSD) {
+ if (LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD ||
+ LIBFUZZER_OPENBSD) {
// ru_maxrss is in KiB
return usage.ru_maxrss >> 10;
} else if (LIBFUZZER_APPLE) {
@@ -270,7 +300,7 @@ int ExecuteFilesOnyByOne(int argc, char **argv) {
assert(in);
LLVMFuzzerTestOneInput(reinterpret_cast<const uint8_t *>(bytes.data()),
bytes.size());
- std::cout << "Execution successfull" << std::endl;
+ std::cout << "Execution successful" << std::endl;
}
return 0;
}
@@ -296,7 +326,8 @@ int main(int argc, char **argv) {
maybe_duplicate_stderr();
maybe_initialize_extra_stats();
- __afl_manual_init();
+ if (!getenv("AFL_DRIVER_DONT_DEFER"))
+ __afl_manual_init();
int N = 1000;
if (argc == 2 && argv[1][0] == '-')
diff --git a/lib/fuzzer/build.sh b/lib/fuzzer/build.sh
index 4556af5daf7d..504e54e3a819 100755
--- a/lib/fuzzer/build.sh
+++ b/lib/fuzzer/build.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
LIBFUZZER_SRC_DIR=$(dirname $0)
CXX="${CXX:-clang}"
for f in $LIBFUZZER_SRC_DIR/*.cpp; do
diff --git a/lib/fuzzer/dataflow/DataFlow.cpp b/lib/fuzzer/dataflow/DataFlow.cpp
new file mode 100644
index 000000000000..a79c796ac456
--- /dev/null
+++ b/lib/fuzzer/dataflow/DataFlow.cpp
@@ -0,0 +1,217 @@
+/*===- DataFlow.cpp - a standalone DataFlow tracer -------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// An experimental data-flow tracer for fuzz targets.
+// It is based on DFSan and SanitizerCoverage.
+// https://clang.llvm.org/docs/DataFlowSanitizer.html
+// https://clang.llvm.org/docs/SanitizerCoverage.html#tracing-data-flow
+//
+// It executes the fuzz target on the given input while monitoring the
+// data flow for every instrumented comparison instruction.
+//
+// The output shows which functions depend on which bytes of the input.
+//
+// Build:
+// 1. Compile this file with -fsanitize=dataflow
+// 2. Build the fuzz target with -g -fsanitize=dataflow
+// -fsanitize-coverage=trace-pc-guard,pc-table,func,trace-cmp
+// 3. Link those together with -fsanitize=dataflow
+//
+// -fsanitize-coverage=trace-cmp inserts callbacks around every comparison
+// instruction, DFSan modifies the calls to pass the data flow labels.
+// The callbacks update the data flow label for the current function.
+// See e.g. __dfsw___sanitizer_cov_trace_cmp1 below.
+//
+// -fsanitize-coverage=trace-pc-guard,pc-table,func instruments function
+// entries so that the comparison callback knows that current function.
+//
+//
+// Run:
+// # Collect data flow for INPUT_FILE, write to OUTPUT_FILE (default: stdout)
+// ./a.out INPUT_FILE [OUTPUT_FILE]
+//
+// # Print all instrumented functions. llvm-symbolizer must be present in PATH
+// ./a.out
+//
+// Example output:
+// ===============
+// F0 11111111111111
+// F1 10000000000000
+// ===============
+// "FN xxxxxxxxxx": tells what bytes of the input does the function N depend on.
+// The byte string is LEN+1 bytes. The last byte is set if the function
+// depends on the input length.
+//===----------------------------------------------------------------------===*/
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+
+#include <execinfo.h> // backtrace_symbols_fd
+
+#include <sanitizer/dfsan_interface.h>
+
+extern "C" {
+extern int LLVMFuzzerTestOneInput(const unsigned char *Data, size_t Size);
+__attribute__((weak)) extern int LLVMFuzzerInitialize(int *argc, char ***argv);
+} // extern "C"
+
+static size_t InputLen;
+static size_t NumFuncs;
+static const uintptr_t *FuncsBeg;
+static __thread size_t CurrentFunc;
+static dfsan_label *FuncLabels; // Array of NumFuncs elements.
+static char *PrintableStringForLabel; // InputLen + 2 bytes.
+static bool LabelSeen[1 << 8 * sizeof(dfsan_label)];
+
+// Prints all instrumented functions.
+static int PrintFunctions() {
+ // We don't have the symbolizer integrated with dfsan yet.
+ // So use backtrace_symbols_fd and pipe it through llvm-symbolizer.
+ // TODO(kcc): this is pretty ugly and may break in lots of ways.
+ // We'll need to make a proper in-process symbolizer work with DFSan.
+ FILE *Pipe = popen("sed 's/(+/ /g; s/).*//g' "
+ "| llvm-symbolizer "
+ "| grep 'dfs\\$' "
+ "| sed 's/dfs\\$//g'", "w");
+ for (size_t I = 0; I < NumFuncs; I++) {
+ uintptr_t PC = FuncsBeg[I * 2];
+ void *const Buf[1] = {(void*)PC};
+ backtrace_symbols_fd(Buf, 1, fileno(Pipe));
+ }
+ pclose(Pipe);
+ return 0;
+}
+
+extern "C"
+void SetBytesForLabel(dfsan_label L, char *Bytes) {
+ if (LabelSeen[L])
+ return;
+ LabelSeen[L] = true;
+ assert(L);
+ if (L <= InputLen + 1) {
+ Bytes[L - 1] = '1';
+ } else {
+ auto *DLI = dfsan_get_label_info(L);
+ SetBytesForLabel(DLI->l1, Bytes);
+ SetBytesForLabel(DLI->l2, Bytes);
+ }
+}
+
+static char *GetPrintableStringForLabel(dfsan_label L) {
+ memset(PrintableStringForLabel, '0', InputLen + 1);
+ PrintableStringForLabel[InputLen + 1] = 0;
+ memset(LabelSeen, 0, sizeof(LabelSeen));
+ SetBytesForLabel(L, PrintableStringForLabel);
+ return PrintableStringForLabel;
+}
+
+static void PrintDataFlow(FILE *Out) {
+ for (size_t I = 0; I < NumFuncs; I++)
+ if (FuncLabels[I])
+ fprintf(Out, "F%zd %s\n", I, GetPrintableStringForLabel(FuncLabels[I]));
+}
+
+int main(int argc, char **argv) {
+ if (LLVMFuzzerInitialize)
+ LLVMFuzzerInitialize(&argc, &argv);
+ if (argc == 1)
+ return PrintFunctions();
+ assert(argc == 4 || argc == 5);
+ size_t Beg = atoi(argv[1]);
+ size_t End = atoi(argv[2]);
+ assert(Beg < End);
+
+ const char *Input = argv[3];
+ fprintf(stderr, "INFO: reading '%s'\n", Input);
+ FILE *In = fopen(Input, "r");
+ assert(In);
+ fseek(In, 0, SEEK_END);
+ InputLen = ftell(In);
+ fseek(In, 0, SEEK_SET);
+ unsigned char *Buf = (unsigned char*)malloc(InputLen);
+ size_t NumBytesRead = fread(Buf, 1, InputLen, In);
+ assert(NumBytesRead == InputLen);
+ PrintableStringForLabel = (char*)malloc(InputLen + 2);
+ fclose(In);
+
+ fprintf(stderr, "INFO: running '%s'\n", Input);
+ for (size_t I = 1; I <= InputLen; I++) {
+ dfsan_label L = dfsan_create_label("", nullptr);
+ assert(L == I);
+ size_t Idx = I - 1;
+ if (Idx >= Beg && Idx < End)
+ dfsan_set_label(L, Buf + Idx, 1);
+ }
+ dfsan_label SizeL = dfsan_create_label("", nullptr);
+ assert(SizeL == InputLen + 1);
+ dfsan_set_label(SizeL, &InputLen, sizeof(InputLen));
+
+ LLVMFuzzerTestOneInput(Buf, InputLen);
+ free(Buf);
+
+ bool OutIsStdout = argc == 4;
+ fprintf(stderr, "INFO: writing dataflow to %s\n",
+ OutIsStdout ? "<stdout>" : argv[4]);
+ FILE *Out = OutIsStdout ? stdout : fopen(argv[4], "w");
+ PrintDataFlow(Out);
+ if (!OutIsStdout) fclose(Out);
+}
+
+extern "C" {
+
+void __sanitizer_cov_trace_pc_guard_init(uint32_t *start,
+ uint32_t *stop) {
+ assert(NumFuncs == 0 && "This tool does not support DSOs");
+ assert(start < stop && "The code is not instrumented for coverage");
+ if (start == stop || *start) return; // Initialize only once.
+ for (uint32_t *x = start; x < stop; x++)
+ *x = ++NumFuncs; // The first index is 1.
+ FuncLabels = (dfsan_label*)calloc(NumFuncs, sizeof(dfsan_label));
+ fprintf(stderr, "INFO: %zd instrumented function(s) observed\n", NumFuncs);
+}
+
+void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg,
+ const uintptr_t *pcs_end) {
+ assert(NumFuncs == (pcs_end - pcs_beg) / 2);
+ FuncsBeg = pcs_beg;
+}
+
+void __sanitizer_cov_trace_pc_indir(uint64_t x){} // unused.
+
+void __sanitizer_cov_trace_pc_guard(uint32_t *guard){
+ uint32_t FuncNum = *guard - 1; // Guards start from 1.
+ assert(FuncNum < NumFuncs);
+ CurrentFunc = FuncNum;
+}
+
+void __dfsw___sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases,
+ dfsan_label L1, dfsan_label UnusedL) {
+ assert(CurrentFunc < NumFuncs);
+ FuncLabels[CurrentFunc] = dfsan_union(FuncLabels[CurrentFunc], L1);
+}
+
+#define HOOK(Name, Type) \
+ void Name(Type Arg1, Type Arg2, dfsan_label L1, dfsan_label L2) { \
+ assert(CurrentFunc < NumFuncs); \
+ FuncLabels[CurrentFunc] = \
+ dfsan_union(FuncLabels[CurrentFunc], dfsan_union(L1, L2)); \
+ }
+
+HOOK(__dfsw___sanitizer_cov_trace_const_cmp1, uint8_t)
+HOOK(__dfsw___sanitizer_cov_trace_const_cmp2, uint16_t)
+HOOK(__dfsw___sanitizer_cov_trace_const_cmp4, uint32_t)
+HOOK(__dfsw___sanitizer_cov_trace_const_cmp8, uint64_t)
+HOOK(__dfsw___sanitizer_cov_trace_cmp1, uint8_t)
+HOOK(__dfsw___sanitizer_cov_trace_cmp2, uint16_t)
+HOOK(__dfsw___sanitizer_cov_trace_cmp4, uint32_t)
+HOOK(__dfsw___sanitizer_cov_trace_cmp8, uint64_t)
+
+} // extern "C"
diff --git a/lib/fuzzer/scripts/collect_data_flow.py b/lib/fuzzer/scripts/collect_data_flow.py
new file mode 100755
index 000000000000..3edff66bb9d1
--- /dev/null
+++ b/lib/fuzzer/scripts/collect_data_flow.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+#===- lib/fuzzer/scripts/collect_data_flow.py ------------------------------===#
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===------------------------------------------------------------------------===#
+# Runs the data-flow tracer several times on the same input in order to collect
+# the complete trace for all input bytes (running it on all bytes at once
+# may fail if DFSan runs out of labels).
+# Usage:
+#
+# # Collect dataflow for one input, store it in OUTPUT (default is stdout)
+# collect_data_flow.py BINARY INPUT [OUTPUT]
+#
+# # Collect dataflow for all inputs in CORPUS_DIR, store them in OUTPUT_DIR
+# collect_data_flow.py BINARY CORPUS_DIR OUTPUT_DIR
+#===------------------------------------------------------------------------===#
+import atexit
+import hashlib
+import sys
+import os
+import subprocess
+import tempfile
+import shutil
+
+tmpdir = ""
+
+def cleanup(d):
+ print("removing: %s" % d)
+ shutil.rmtree(d)
+
+def collect_dataflow_for_corpus(self, exe, corpus_dir, output_dir):
+ print("Collecting dataflow for corpus: %s output_dir: %s" % (corpus_dir,
+ output_dir))
+ assert not os.path.exists(output_dir)
+ os.mkdir(output_dir)
+ for root, dirs, files in os.walk(corpus_dir):
+ for f in files:
+ path = os.path.join(root, f)
+ sha1 = hashlib.sha1(open(path).read()).hexdigest()
+ output = os.path.join(output_dir, sha1)
+ subprocess.call([self, exe, path, output])
+ functions_txt = open(os.path.join(output_dir, "functions.txt"), "w")
+ subprocess.call([exe], stdout=functions_txt)
+
+
+def main(argv):
+ exe = argv[1]
+ inp = argv[2]
+ if os.path.isdir(inp):
+ return collect_dataflow_for_corpus(argv[0], exe, inp, argv[3])
+ size = os.path.getsize(inp)
+ q = [[0, size]]
+ tmpdir = tempfile.mkdtemp(prefix="libfuzzer-tmp-")
+ atexit.register(cleanup, tmpdir)
+ print "tmpdir: ", tmpdir
+ outputs = []
+ while len(q):
+ r = q.pop()
+ print "******* Trying: ", r
+ tmpfile = os.path.join(tmpdir, str(r[0]) + "-" + str(r[1]))
+ ret = subprocess.call([exe, str(r[0]), str(r[1]), inp, tmpfile])
+ if ret and r[1] - r[0] >= 2:
+ q.append([r[0], (r[1] + r[0]) / 2])
+ q.append([(r[1] + r[0]) / 2, r[1]])
+ else:
+ outputs.append(tmpfile)
+ print "******* Success: ", r
+ f = sys.stdout
+ if len(argv) >= 4:
+ f = open(argv[3], "w")
+ merge = os.path.join(os.path.dirname(argv[0]), "merge_data_flow.py")
+ subprocess.call([merge] + outputs, stdout=f)
+
+if __name__ == '__main__':
+ main(sys.argv)
diff --git a/lib/fuzzer/scripts/merge_data_flow.py b/lib/fuzzer/scripts/merge_data_flow.py
new file mode 100755
index 000000000000..d2f5081e7b8c
--- /dev/null
+++ b/lib/fuzzer/scripts/merge_data_flow.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+#===- lib/fuzzer/scripts/merge_data_flow.py ------------------------------===#
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===------------------------------------------------------------------------===#
+# Merge several data flow traces into one.
+# Usage:
+# merge_data_flow.py trace1 trace2 ... > result
+#===------------------------------------------------------------------------===#
+import sys
+import fileinput
+from array import array
+
+def Merge(a, b):
+ res = array('b')
+ for i in range(0, len(a)):
+ res.append(ord('1' if a[i] == '1' or b[i] == '1' else '0'))
+ return res.tostring()
+
+def main(argv):
+ D = {}
+ for line in fileinput.input():
+ [F,BV] = line.strip().split(' ')
+ if F in D:
+ D[F] = Merge(D[F], BV)
+ else:
+ D[F] = BV;
+ for F in D.keys():
+ print("%s %s" % (F, D[F]))
+
+if __name__ == '__main__':
+ main(sys.argv)
diff --git a/lib/fuzzer/scripts/unbalanced_allocs.py b/lib/fuzzer/scripts/unbalanced_allocs.py
index a4ce187679d7..74478ad55af0 100755
--- a/lib/fuzzer/scripts/unbalanced_allocs.py
+++ b/lib/fuzzer/scripts/unbalanced_allocs.py
@@ -24,9 +24,9 @@ def PrintStack(line, stack):
global _skip
if _skip > 0:
return
- print 'Unbalanced ' + line.rstrip();
+ print('Unbalanced ' + line.rstrip());
for l in stack:
- print l.rstrip()
+ print(l.rstrip())
def ProcessStack(line, f):
stack = []
@@ -63,15 +63,15 @@ def ProcessRun(line, f):
return ProcessMalloc(line, f, {})
allocs = {}
- print line.rstrip()
+ print(line.rstrip())
line = f.readline()
while line:
if line.startswith('MallocFreeTracer: STOP'):
global _skip
_skip = _skip - 1
- for _, (l, s) in allocs.iteritems():
+ for _, (l, s) in allocs.items():
PrintStack(l, s)
- print line.rstrip()
+ print(line.rstrip())
return f.readline()
line = ProcessMalloc(line, f, allocs)
return line
diff --git a/lib/fuzzer/tests/CMakeLists.txt b/lib/fuzzer/tests/CMakeLists.txt
index dac8773597e8..ed5807168301 100644
--- a/lib/fuzzer/tests/CMakeLists.txt
+++ b/lib/fuzzer/tests/CMakeLists.txt
@@ -3,22 +3,32 @@ set(LIBFUZZER_UNITTEST_CFLAGS
${COMPILER_RT_GTEST_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/lib/fuzzer
-fno-rtti
- -Werror
-O2)
+if (APPLE)
+ set(FUZZER_SUPPORTED_OS osx)
+endif()
+
add_custom_target(FuzzerUnitTests)
set_target_properties(FuzzerUnitTests PROPERTIES FOLDER "Compiler-RT Tests")
set(LIBFUZZER_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS})
list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS --driver-mode=g++)
-if(APPLE)
- list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lc++)
+if(APPLE OR CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+ list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lc++ -lpthread)
else()
list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lstdc++ -lpthread)
endif()
-foreach(arch ${FUZZER_SUPPORTED_ARCH})
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
+ list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer)
+endif()
+
+if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH)
+ # libFuzzer unit tests are only run on the host machine.
+ set(arch ${COMPILER_RT_DEFAULT_TARGET_ARCH})
+
set(LIBFUZZER_TEST_RUNTIME RTFuzzerTest.${arch})
if(APPLE)
set(LIBFUZZER_TEST_RUNTIME_OBJECTS
@@ -33,14 +43,20 @@ foreach(arch ${FUZZER_SUPPORTED_ARCH})
ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
FOLDER "Compiler-RT Runtime tests")
+ if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
+ set(LIBFUZZER_TEST_RUNTIME_DEPS libcxx_fuzzer_${arch}-build)
+ set(LIBFUZZER_TEST_RUNTIME_CFLAGS -isystem ${COMPILER_RT_LIBCXX_PATH}/include)
+ set(LIBFUZZER_TEST_RUNTIME_LINK_FLAGS ${LIBCXX_${arch}_PREFIX}/lib/libc++.a)
+ endif()
+
set(FuzzerTestObjects)
generate_compiler_rt_tests(FuzzerTestObjects
FuzzerUnitTests "Fuzzer-${arch}-Test" ${arch}
SOURCES FuzzerUnittest.cpp ${COMPILER_RT_GTEST_SOURCE}
RUNTIME ${LIBFUZZER_TEST_RUNTIME}
- DEPS gtest
- CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS}
- LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS})
+ DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS}
+ CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS}
+ LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS})
set_target_properties(FuzzerUnitTests PROPERTIES
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-endforeach()
+endif()
diff --git a/lib/fuzzer/tests/FuzzerUnittest.cpp b/lib/fuzzer/tests/FuzzerUnittest.cpp
index 97ec3b4bb6af..e3b06702603d 100644
--- a/lib/fuzzer/tests/FuzzerUnittest.cpp
+++ b/lib/fuzzer/tests/FuzzerUnittest.cpp
@@ -28,6 +28,14 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
abort();
}
+TEST(Fuzzer, Basename) {
+ EXPECT_EQ(Basename("foo/bar"), "bar");
+ EXPECT_EQ(Basename("bar"), "bar");
+ EXPECT_EQ(Basename("/bar"), "bar");
+ EXPECT_EQ(Basename("foo/x"), "x");
+ EXPECT_EQ(Basename("foo/"), "");
+}
+
TEST(Fuzzer, CrossOver) {
std::unique_ptr<ExternalFunctions> t(new ExternalFunctions());
fuzzer::EF = t.get();
@@ -328,7 +336,7 @@ void TestShuffleBytes(Mutator M, int NumIter) {
}
TEST(FuzzerMutate, ShuffleBytes1) {
- TestShuffleBytes(&MutationDispatcher::Mutate_ShuffleBytes, 1 << 16);
+ TestShuffleBytes(&MutationDispatcher::Mutate_ShuffleBytes, 1 << 17);
}
TEST(FuzzerMutate, ShuffleBytes2) {
TestShuffleBytes(&MutationDispatcher::Mutate, 1 << 20);
@@ -381,6 +389,21 @@ TEST(FuzzerMutate, CopyPart1) {
TEST(FuzzerMutate, CopyPart2) {
TestCopyPart(&MutationDispatcher::Mutate, 1 << 13);
}
+TEST(FuzzerMutate, CopyPartNoInsertAtMaxSize) {
+ // This (non exhaustively) tests if `Mutate_CopyPart` tries to perform an
+ // insert on an input of size `MaxSize`. Performing an insert in this case
+ // will lead to the mutation failing.
+ std::unique_ptr<ExternalFunctions> t(new ExternalFunctions());
+ fuzzer::EF = t.get();
+ Random Rand(0);
+ std::unique_ptr<MutationDispatcher> MD(new MutationDispatcher(Rand, {}));
+ uint8_t Data[8] = {0x00, 0x11, 0x22, 0x33, 0x44, 0x00, 0x11, 0x22};
+ size_t MaxSize = sizeof(Data);
+ for (int count = 0; count < (1 << 18); ++count) {
+ size_t NewSize = MD->Mutate_CopyPart(Data, MaxSize, MaxSize);
+ ASSERT_EQ(NewSize, MaxSize);
+ }
+}
void TestAddWordFromDictionary(Mutator M, int NumIter) {
std::unique_ptr<ExternalFunctions> t(new ExternalFunctions());
@@ -559,12 +582,14 @@ TEST(FuzzerUtil, Base64) {
}
TEST(Corpus, Distribution) {
+ DataFlowTrace DFT;
Random Rand(0);
std::unique_ptr<InputCorpus> C(new InputCorpus(""));
size_t N = 10;
size_t TriesPerUnit = 1<<16;
for (size_t i = 0; i < N; i++)
- C->AddToCorpus(Unit{ static_cast<uint8_t>(i) }, 1, false, {});
+ C->AddToCorpus(Unit{static_cast<uint8_t>(i)}, 1, false, false, {}, DFT,
+ nullptr);
Vector<size_t> Hist(N);
for (size_t i = 0; i < N * TriesPerUnit; i++) {
diff --git a/lib/hwasan/.clang-format b/lib/hwasan/.clang-format
index f6cb8ad931f5..560308c91dee 100644
--- a/lib/hwasan/.clang-format
+++ b/lib/hwasan/.clang-format
@@ -1 +1,2 @@
BasedOnStyle: Google
+AllowShortIfStatementsOnASingleLine: false
diff --git a/lib/hwasan/CMakeLists.txt b/lib/hwasan/CMakeLists.txt
index 3f3a6155590c..42bf4366f192 100644
--- a/lib/hwasan/CMakeLists.txt
+++ b/lib/hwasan/CMakeLists.txt
@@ -4,39 +4,38 @@ include_directories(..)
set(HWASAN_RTL_SOURCES
hwasan.cc
hwasan_allocator.cc
+ hwasan_dynamic_shadow.cc
hwasan_interceptors.cc
hwasan_linux.cc
+ hwasan_poisoning.cc
hwasan_report.cc
hwasan_thread.cc
- hwasan_poisoning.cc
)
set(HWASAN_RTL_CXX_SOURCES
hwasan_new_delete.cc)
+set(HWASAN_RTL_HEADERS
+ hwasan.h
+ hwasan_allocator.h
+ hwasan_dynamic_shadow.h
+ hwasan_flags.h
+ hwasan_flags.inc
+ hwasan_interface_internal.h
+ hwasan_mapping.h
+ hwasan_poisoning.h
+ hwasan_report.h
+ hwasan_thread.h)
+
set(HWASAN_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS})
append_rtti_flag(OFF HWASAN_RTL_CFLAGS)
-append_list_if(COMPILER_RT_HAS_FPIE_FLAG -fPIE HWASAN_RTL_CFLAGS)
+append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC HWASAN_RTL_CFLAGS)
# Prevent clang from generating libc calls.
append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding HWASAN_RTL_CFLAGS)
set(HWASAN_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
-if(ANDROID)
-# On Android, -z global does not do what it is documented to do.
-# On Android, -z global moves the library ahead in the lookup order,
-# placing it right after the LD_PRELOADs. This is used to compensate for the fact
-# that Android linker does not look at the dependencies of the main executable
-# that aren't dependencies of the current DSO when resolving symbols from said DSO.
-# As a net result, this allows running ASan executables without LD_PRELOAD-ing the
-# ASan runtime library.
-# The above is applicable to L MR1 or newer.
- if (COMPILER_RT_HAS_Z_GLOBAL)
- list(APPEND HWASAN_DYNAMIC_LINK_FLAGS -Wl,-z,global)
- endif()
-endif()
-
set(HWASAN_DYNAMIC_CFLAGS ${HWASAN_RTL_CFLAGS})
append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC
-ftls-model=initial-exec HWASAN_DYNAMIC_CFLAGS)
@@ -55,13 +54,18 @@ add_compiler_rt_component(hwasan)
add_compiler_rt_object_libraries(RTHwasan
ARCHS ${HWASAN_SUPPORTED_ARCH}
- SOURCES ${HWASAN_RTL_SOURCES} CFLAGS ${HWASAN_RTL_CFLAGS})
+ SOURCES ${HWASAN_RTL_SOURCES}
+ ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+ CFLAGS ${HWASAN_RTL_CFLAGS})
add_compiler_rt_object_libraries(RTHwasan_cxx
ARCHS ${HWASAN_SUPPORTED_ARCH}
- SOURCES ${HWASAN_RTL_CXX_SOURCES} CFLAGS ${HWASAN_RTL_CFLAGS})
+ SOURCES ${HWASAN_RTL_CXX_SOURCES}
+ ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+ CFLAGS ${HWASAN_RTL_CFLAGS})
add_compiler_rt_object_libraries(RTHwasan_dynamic
ARCHS ${HWASAN_SUPPORTED_ARCH}
- SOURCES ${HWASAN_RTL_SOURCES} ${TSAN_RTL_CXX_SOURCES}
+ SOURCES ${HWASAN_RTL_SOURCES} ${HWASAN_RTL_CXX_SOURCES}
+ ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
CFLAGS ${HWASAN_DYNAMIC_CFLAGS})
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc "")
@@ -78,7 +82,9 @@ foreach(arch ${HWASAN_SUPPORTED_ARCH})
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
- RTUbsan
+ RTSanitizerCommonCoverage
+ RTSanitizerCommonSymbolizer
+ RTUbsan
CFLAGS ${HWASAN_RTL_CFLAGS}
PARENT_TARGET hwasan)
add_compiler_rt_runtime(clang_rt.hwasan_cxx
@@ -112,7 +118,9 @@ foreach(arch ${HWASAN_SUPPORTED_ARCH})
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
- RTUbsan
+ RTSanitizerCommonCoverage
+ RTSanitizerCommonSymbolizer
+ RTUbsan
# The only purpose of RTHWAsan_dynamic_version_script_dummy is to
# carry a dependency of the shared runtime on the version script.
# Replacing it with a straightforward
@@ -126,7 +134,7 @@ foreach(arch ${HWASAN_SUPPORTED_ARCH})
DEFS ${ASAN_DYNAMIC_DEFINITIONS}
PARENT_TARGET hwasan)
- if(UNIX)
+ if(SANITIZER_USE_SYMBOLS)
add_sanitizer_rt_symbols(clang_rt.hwasan
ARCHS ${arch}
EXTRA hwasan.syms.extra)
diff --git a/lib/hwasan/hwasan.cc b/lib/hwasan/hwasan.cc
index 8b1e5a7846af..7dab8249e3f7 100644
--- a/lib/hwasan/hwasan.cc
+++ b/lib/hwasan/hwasan.cc
@@ -1,4 +1,4 @@
-//===-- hwasan.cc -----------------------------------------------------------===//
+//===-- hwasan.cc ---------------------------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
@@ -13,8 +13,10 @@
//===----------------------------------------------------------------------===//
#include "hwasan.h"
-#include "hwasan_thread.h"
+#include "hwasan_mapping.h"
#include "hwasan_poisoning.h"
+#include "hwasan_report.h"
+#include "hwasan_thread.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_flags.h"
@@ -84,7 +86,7 @@ static void InitializeFlags() {
cf.check_printf = false;
cf.intercept_tls_get_addr = true;
cf.exitcode = 99;
- cf.handle_sigill = kHandleSignalExclusive;
+ cf.handle_sigtrap = kHandleSignalExclusive;
OverrideCommonFlags(cf);
}
@@ -143,12 +145,22 @@ void PrintWarning(uptr pc, uptr bp) {
ReportInvalidAccess(&stack, 0);
}
+static void HWAsanCheckFailed(const char *file, int line, const char *cond,
+ u64 v1, u64 v2) {
+ Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
+ line, cond, (uptr)v1, (uptr)v2);
+ PRINT_CURRENT_STACK_CHECK();
+ Die();
+}
+
} // namespace __hwasan
// Interface.
using namespace __hwasan;
+uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol.
+
void __hwasan_init() {
CHECK(!hwasan_init_is_running);
if (hwasan_inited) return;
@@ -160,23 +172,28 @@ void __hwasan_init() {
CacheBinaryName();
InitializeFlags();
- __sanitizer_set_report_path(common_flags()->log_path);
+ // Install tool-specific callbacks in sanitizer_common.
+ SetCheckFailedCallback(HWAsanCheckFailed);
- InitializeInterceptors();
- InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
- InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
+ __sanitizer_set_report_path(common_flags()->log_path);
DisableCoreDumperIfNecessary();
if (!InitShadow()) {
- Printf("FATAL: HWAddressSanitizer can not mmap the shadow memory.\n");
- Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
- Printf("FATAL: Disabling ASLR is known to cause this error.\n");
- Printf("FATAL: If running under GDB, try "
- "'set disable-randomization off'.\n");
+ Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
+ if (HWASAN_FIXED_MAPPING) {
+ Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
+ Printf("FATAL: Disabling ASLR is known to cause this error.\n");
+ Printf("FATAL: If running under GDB, try "
+ "'set disable-randomization off'.\n");
+ }
DumpProcessMap();
Die();
}
+ InitializeInterceptors();
+ InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
+ InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
+
Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
@@ -240,11 +257,23 @@ void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
template<unsigned X>
__attribute__((always_inline))
-static void SigIll() {
+static void SigTrap(uptr p) {
#if defined(__aarch64__)
- asm("hlt %0\n\t" ::"n"(X));
-#elif defined(__x86_64__) || defined(__i386__)
- asm("ud2\n\t");
+ (void)p;
+ // 0x900 is added to do not interfere with the kernel use of lower values of
+ // brk immediate.
+ // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch.
+ asm("brk %0\n\t" ::"n"(0x900 + X));
+#elif defined(__x86_64__)
+ // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes
+ // total. The pointer is passed via rdi.
+ // 0x40 is added as a safeguard, to help distinguish our trap from others and
+ // to avoid 0 offsets in the command (otherwise it'll be reduced to a
+ // different nop command, the three bytes one).
+ asm volatile(
+ "int3\n"
+ "nopl %c0(%%rax)\n"
+ :: "n"(0x40 + X), "D"(p));
#else
// FIXME: not always sigill.
__builtin_trap();
@@ -261,8 +290,8 @@ __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
uptr ptr_raw = p & ~kAddressTagMask;
tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
if (UNLIKELY(ptr_tag != mem_tag)) {
- SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) +
- 0x10 * (AT == AccessType::Store) + LogSize>();
+ SigTrap<0x20 * (EA == ErrorAction::Recover) +
+ 0x10 * (AT == AccessType::Store) + LogSize>(p);
if (EA == ErrorAction::Abort) __builtin_unreachable();
}
}
@@ -277,13 +306,13 @@ __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
for (tag_t *t = shadow_first; t <= shadow_last; ++t)
if (UNLIKELY(ptr_tag != *t)) {
- SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) +
- 0x10 * (AT == AccessType::Store) + 0xf>();
+ SigTrap<0x20 * (EA == ErrorAction::Recover) +
+ 0x10 * (AT == AccessType::Store) + 0xf>(p);
if (EA == ErrorAction::Abort) __builtin_unreachable();
}
}
-void __hwasan_load(uptr p, uptr sz) {
+void __hwasan_loadN(uptr p, uptr sz) {
CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
}
void __hwasan_load1(uptr p) {
@@ -302,7 +331,7 @@ void __hwasan_load16(uptr p) {
CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p);
}
-void __hwasan_load_noabort(uptr p, uptr sz) {
+void __hwasan_loadN_noabort(uptr p, uptr sz) {
CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz);
}
void __hwasan_load1_noabort(uptr p) {
@@ -321,7 +350,7 @@ void __hwasan_load16_noabort(uptr p) {
CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
}
-void __hwasan_store(uptr p, uptr sz) {
+void __hwasan_storeN(uptr p, uptr sz) {
CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
}
void __hwasan_store1(uptr p) {
@@ -340,7 +369,7 @@ void __hwasan_store16(uptr p) {
CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p);
}
-void __hwasan_store_noabort(uptr p, uptr sz) {
+void __hwasan_storeN_noabort(uptr p, uptr sz) {
CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz);
}
void __hwasan_store1_noabort(uptr p) {
@@ -359,6 +388,18 @@ void __hwasan_store16_noabort(uptr p) {
CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
}
+void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) {
+ TagMemoryAligned(p, sz, tag);
+}
+
+static const u8 kFallbackTag = 0xBB;
+
+u8 __hwasan_generate_tag() {
+ HwasanThread *t = GetCurrentThread();
+ if (!t) return kFallbackTag;
+ return t->GenerateRandomTag();
+}
+
#if !SANITIZER_SUPPORTS_WEAK_HOOKS
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
diff --git a/lib/hwasan/hwasan.h b/lib/hwasan/hwasan.h
index bcf5282dce7c..47d1d057a0dd 100644
--- a/lib/hwasan/hwasan.h
+++ b/lib/hwasan/hwasan.h
@@ -1,4 +1,4 @@
-//===-- hwasan.h --------------------------------------------------*- C++ -*-===//
+//===-- hwasan.h ------------------------------------------------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -32,16 +32,6 @@
typedef u8 tag_t;
-// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
-const uptr kShadowScale = 4;
-const uptr kShadowAlignment = 1UL << kShadowScale;
-
-#define MEM_TO_SHADOW_OFFSET(mem) ((uptr)(mem) >> kShadowScale)
-#define MEM_TO_SHADOW(mem) ((uptr)(mem) >> kShadowScale)
-#define SHADOW_TO_MEM(shadow) ((uptr)(shadow) << kShadowScale)
-
-#define MEM_IS_APP(mem) true
-
// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
// translation and can be used to store a tag.
const unsigned kAddressTagShift = 56;
@@ -107,15 +97,6 @@ void PrintWarning(uptr pc, uptr bp);
void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
void *context, bool request_fast_unwind);
-void ReportInvalidAccess(StackTrace *stack, u32 origin);
-void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
- bool is_store);
-void ReportStats();
-void ReportAtExitStatistics();
-void DescribeMemoryRange(const void *x, uptr size);
-void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, uptr size,
- uptr offset);
-
// Returns a "chained" origin id, pointing to the given stack trace followed by
// the previous origin id.
u32 ChainOrigin(u32 id, StackTrace *stack);
@@ -135,6 +116,15 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1;
GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \
common_flags()->fast_unwind_on_fatal)
+#define GET_FATAL_STACK_TRACE_HERE \
+ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME())
+
+#define PRINT_CURRENT_STACK_CHECK() \
+ { \
+ GET_FATAL_STACK_TRACE_HERE; \
+ stack.Print(); \
+ }
+
class ScopedThreadLocalStateBackup {
public:
ScopedThreadLocalStateBackup() { Backup(); }
diff --git a/lib/hwasan/hwasan_allocator.cc b/lib/hwasan/hwasan_allocator.cc
index fbcaf78b88f0..c2b9b0b69589 100644
--- a/lib/hwasan/hwasan_allocator.cc
+++ b/lib/hwasan/hwasan_allocator.cc
@@ -1,4 +1,4 @@
-//===-- hwasan_allocator.cc --------------------------- ---------------------===//
+//===-- hwasan_allocator.cc ------------------------- ---------------------===//
//
// The LLVM Compiler Infrastructure
//
@@ -15,11 +15,13 @@
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include "hwasan.h"
#include "hwasan_allocator.h"
+#include "hwasan_mapping.h"
#include "hwasan_thread.h"
#include "hwasan_poisoning.h"
@@ -70,8 +72,8 @@ struct HwasanMapUnmapCallback {
}
};
-#if !defined(__aarch64__)
-#error unsupported platform
+#if !defined(__aarch64__) && !defined(__x86_64__)
+#error Unsupported platform
#endif
static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
@@ -100,6 +102,9 @@ static AllocatorCache fallback_allocator_cache;
static SpinMutex fallback_mutex;
static atomic_uint8_t hwasan_allocator_tagging_enabled;
+static const tag_t kFallbackAllocTag = 0xBB;
+static const tag_t kFallbackFreeTag = 0xBC;
+
void HwasanAllocatorInit() {
atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
!flags()->disable_allocator_tagging);
@@ -123,9 +128,12 @@ static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment,
size = RoundUpTo(size, kShadowAlignment);
if (size > kMaxAllowedMallocSize) {
- Report("WARNING: HWAddressSanitizer failed to allocate %p bytes\n",
- (void *)size);
- return Allocator::FailureHandler::OnBadRequest();
+ if (AllocatorMayReturnNull()) {
+ Report("WARNING: HWAddressSanitizer failed to allocate 0x%zx bytes\n",
+ size);
+ return nullptr;
+ }
+ ReportAllocationSizeTooBig(size, kMaxAllowedMallocSize, stack);
}
HwasanThread *t = GetCurrentThread();
void *allocated;
@@ -137,6 +145,12 @@ static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment,
AllocatorCache *cache = &fallback_allocator_cache;
allocated = allocator.Allocate(cache, size, alignment);
}
+ if (UNLIKELY(!allocated)) {
+ SetAllocatorOutOfMemory();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportOutOfMemory(size, stack);
+ }
Metadata *meta =
reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
meta->state = CHUNK_ALLOCATED;
@@ -145,10 +159,11 @@ static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment,
if (zeroise)
internal_memset(allocated, 0, size);
- void *user_ptr = (flags()->tag_in_malloc &&
- atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
- ? (void *)TagMemoryAligned((uptr)allocated, size, 0xBB)
- : allocated;
+ void *user_ptr = allocated;
+ if (flags()->tag_in_malloc &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ user_ptr = (void *)TagMemoryAligned(
+ (uptr)user_ptr, size, t ? t->GenerateRandomTag() : kFallbackAllocTag);
HWASAN_MALLOC_HOOK(user_ptr, size);
return user_ptr;
@@ -166,10 +181,11 @@ void HwasanDeallocate(StackTrace *stack, void *user_ptr) {
meta->free_context_id = StackDepotPut(*stack);
// This memory will not be reused by anyone else, so we are free to keep it
// poisoned.
+ HwasanThread *t = GetCurrentThread();
if (flags()->tag_in_free &&
atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
- TagMemoryAligned((uptr)p, size, 0xBC);
- HwasanThread *t = GetCurrentThread();
+ TagMemoryAligned((uptr)p, size,
+ t ? t->GenerateRandomTag() : kFallbackFreeTag);
if (t) {
AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
allocator.Deallocate(cache, p);
@@ -195,8 +211,12 @@ void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size,
meta->requested_size = new_size;
if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
return user_old_p;
- if (flags()->retag_in_realloc)
- return (void *)TagMemoryAligned((uptr)old_p, new_size, 0xCC);
+ if (flags()->retag_in_realloc) {
+ HwasanThread *t = GetCurrentThread();
+ return (void *)TagMemoryAligned(
+ (uptr)old_p, new_size,
+ t ? t->GenerateRandomTag() : kFallbackAllocTag);
+ }
if (new_size > old_size) {
tag_t tag = GetTagFromPointer((uptr)user_old_p);
TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag);
@@ -212,6 +232,15 @@ void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size,
return new_p;
}
+void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportCallocOverflow(nmemb, size, stack);
+ }
+ return HwasanAllocate(stack, nmemb * size, sizeof(u64), true);
+}
+
HwasanChunkView FindHeapChunkByAddress(uptr address) {
void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
if (!block)
@@ -235,9 +264,7 @@ void *hwasan_malloc(uptr size, StackTrace *stack) {
}
void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
- if (UNLIKELY(CheckForCallocOverflow(size, nmemb)))
- return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest());
- return SetErrnoOnNull(HwasanAllocate(stack, nmemb * size, sizeof(u64), true));
+ return SetErrnoOnNull(HwasanCalloc(stack, nmemb, size));
}
void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
@@ -251,14 +278,17 @@ void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
}
void *hwasan_valloc(uptr size, StackTrace *stack) {
- return SetErrnoOnNull(HwasanAllocate(stack, size, GetPageSizeCached(), false));
+ return SetErrnoOnNull(
+ HwasanAllocate(stack, size, GetPageSizeCached(), false));
}
void *hwasan_pvalloc(uptr size, StackTrace *stack) {
uptr PageSize = GetPageSizeCached();
if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
errno = errno_ENOMEM;
- return Allocator::FailureHandler::OnBadRequest();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportPvallocOverflow(size, stack);
}
// pvalloc(0) should allocate one page.
size = size ? RoundUpTo(size, PageSize) : PageSize;
@@ -268,7 +298,9 @@ void *hwasan_pvalloc(uptr size, StackTrace *stack) {
void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
errno = errno_EINVAL;
- return Allocator::FailureHandler::OnBadRequest();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportInvalidAlignedAllocAlignment(size, alignment, stack);
}
return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
}
@@ -276,7 +308,9 @@ void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
if (UNLIKELY(!IsPowerOfTwo(alignment))) {
errno = errno_EINVAL;
- return Allocator::FailureHandler::OnBadRequest();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportInvalidAllocationAlignment(alignment, stack);
}
return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
}
@@ -284,18 +318,20 @@ void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
StackTrace *stack) {
if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
- Allocator::FailureHandler::OnBadRequest();
- return errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return errno_EINVAL;
+ ReportInvalidPosixMemalignAlignment(alignment, stack);
}
void *ptr = HwasanAllocate(stack, size, alignment, false);
if (UNLIKELY(!ptr))
+ // OOM error is already taken care of by HwasanAllocate.
return errno_ENOMEM;
CHECK(IsAligned((uptr)ptr, alignment));
*memptr = ptr;
return 0;
}
-} // namespace __hwasan
+} // namespace __hwasan
using namespace __hwasan;
diff --git a/lib/hwasan/hwasan_dynamic_shadow.cc b/lib/hwasan/hwasan_dynamic_shadow.cc
new file mode 100644
index 000000000000..17338003aa65
--- /dev/null
+++ b/lib/hwasan/hwasan_dynamic_shadow.cc
@@ -0,0 +1,132 @@
+//===-- hwasan_dynamic_shadow.cc --------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer. It reserves dynamic shadow memory
+/// region and handles ifunc resolver case, when necessary.
+///
+//===----------------------------------------------------------------------===//
+
+#include "hwasan_dynamic_shadow.h"
+#include "hwasan_mapping.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_posix.h"
+
+// The code in this file needs to run in an unrelocated binary. It should not
+// access any external symbol, including its own non-hidden globals.
+
+namespace __hwasan {
+
+static void UnmapFromTo(uptr from, uptr to) {
+ if (to == from)
+ return;
+ CHECK(to >= from);
+ uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from);
+ if (UNLIKELY(internal_iserror(res))) {
+ Report("ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",
+ SanitizerToolName, to - from, to - from, from);
+ CHECK("unable to unmap" && 0);
+ }
+}
+
+// Returns an address aligned to 8 pages, such that one page on the left and
+// shadow_size_bytes bytes on the right of it are mapped r/o.
+static uptr MapDynamicShadow(uptr shadow_size_bytes) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr alignment = granularity * SHADOW_GRANULARITY;
+ const uptr left_padding = granularity;
+ const uptr shadow_size =
+ RoundUpTo(shadow_size_bytes, granularity);
+ const uptr map_size = shadow_size + left_padding + alignment;
+
+ const uptr map_start = (uptr)MmapNoAccess(map_size);
+ CHECK_NE(map_start, ~(uptr)0);
+
+ const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
+
+ UnmapFromTo(map_start, shadow_start - left_padding);
+ UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
+
+ return shadow_start;
+}
+
+} // namespace __hwasan
+
+#if HWASAN_PREMAP_SHADOW
+
+extern "C" {
+
+INTERFACE_ATTRIBUTE void __hwasan_shadow();
+decltype(__hwasan_shadow)* __hwasan_premap_shadow();
+
+} // extern "C"
+
+namespace __hwasan {
+
+// Conservative upper limit.
+static uptr PremapShadowSize() {
+ return RoundUpTo(GetMaxVirtualAddress() >> kShadowScale,
+ GetMmapGranularity());
+}
+
+static uptr PremapShadow() {
+ return MapDynamicShadow(PremapShadowSize());
+}
+
+static bool IsPremapShadowAvailable() {
+ const uptr shadow = reinterpret_cast<uptr>(&__hwasan_shadow);
+ const uptr resolver = reinterpret_cast<uptr>(&__hwasan_premap_shadow);
+ // shadow == resolver is how Android KitKat and older handles ifunc.
+ // shadow == 0 just in case.
+ return shadow != 0 && shadow != resolver;
+}
+
+static uptr FindPremappedShadowStart(uptr shadow_size_bytes) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr shadow_start = reinterpret_cast<uptr>(&__hwasan_shadow);
+ const uptr premap_shadow_size = PremapShadowSize();
+ const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity);
+
+ // We may have mapped too much. Release extra memory.
+ UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size);
+ return shadow_start;
+}
+
+} // namespace __hwasan
+
+extern "C" {
+
+decltype(__hwasan_shadow)* __hwasan_premap_shadow() {
+ // The resolver might be called multiple times. Map the shadow just once.
+ static __sanitizer::uptr shadow = 0;
+ if (!shadow)
+ shadow = __hwasan::PremapShadow();
+ return reinterpret_cast<decltype(__hwasan_shadow)*>(shadow);
+}
+
+// __hwasan_shadow is a "function" that has the same address as the first byte
+// of the shadow mapping.
+INTERFACE_ATTRIBUTE __attribute__((ifunc("__hwasan_premap_shadow")))
+void __hwasan_shadow();
+
+} // extern "C"
+
+#endif // HWASAN_PREMAP_SHADOW
+
+namespace __hwasan {
+
+uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
+#if HWASAN_PREMAP_SHADOW
+ if (IsPremapShadowAvailable())
+ return FindPremappedShadowStart(shadow_size_bytes);
+#endif
+ return MapDynamicShadow(shadow_size_bytes);
+}
+
+} // namespace __hwasan
diff --git a/lib/hwasan/hwasan_dynamic_shadow.h b/lib/hwasan/hwasan_dynamic_shadow.h
new file mode 100644
index 000000000000..b5e9e1dd6a06
--- /dev/null
+++ b/lib/hwasan/hwasan_dynamic_shadow.h
@@ -0,0 +1,27 @@
+//===-- hwasan_dynamic_shadow.h ---------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer. It reserves dynamic shadow memory
+/// region.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_PREMAP_SHADOW_H
+#define HWASAN_PREMAP_SHADOW_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+namespace __hwasan {
+
+uptr FindDynamicShadowStart(uptr shadow_size_bytes);
+
+} // namespace __hwasan
+
+#endif // HWASAN_PREMAP_SHADOW_H
diff --git a/lib/hwasan/hwasan_flags.inc b/lib/hwasan/hwasan_flags.inc
index a2cb701ae93d..c45781168d6c 100644
--- a/lib/hwasan/hwasan_flags.inc
+++ b/lib/hwasan/hwasan_flags.inc
@@ -27,3 +27,7 @@ HWASAN_FLAG(bool, atexit, false, "")
// Test only flag to disable malloc/realloc/free memory tagging on startup.
// Tagging can be reenabled with __hwasan_enable_allocator_tagging().
HWASAN_FLAG(bool, disable_allocator_tagging, false, "")
+
+// If false, use simple increment of a thread local counter to generate new
+// tags.
+HWASAN_FLAG(bool, random_tags, true, "")
diff --git a/lib/hwasan/hwasan_interceptors.cc b/lib/hwasan/hwasan_interceptors.cc
index fb39e9fdacf9..66aab95db56f 100644
--- a/lib/hwasan/hwasan_interceptors.cc
+++ b/lib/hwasan/hwasan_interceptors.cc
@@ -17,8 +17,10 @@
#include "interception/interception.h"
#include "hwasan.h"
+#include "hwasan_mapping.h"
#include "hwasan_thread.h"
#include "hwasan_poisoning.h"
+#include "hwasan_report.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
@@ -258,34 +260,17 @@ INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
INTERCEPTOR(void *, malloc, SIZE_T size) {
GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_init_is_running))
+ ENSURE_HWASAN_INITED();
if (UNLIKELY(!hwasan_inited))
// Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
return AllocateFromLocalPool(size);
return hwasan_malloc(size, &stack);
}
-
-INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
- int fd, OFF_T offset) {
- if (hwasan_init_is_running)
- return REAL(mmap)(addr, length, prot, flags, fd, offset);
- ENSURE_HWASAN_INITED();
- if (addr && !MEM_IS_APP(addr)) {
- if (flags & map_fixed) {
- errno = errno_EINVAL;
- return (void *)-1;
- } else {
- addr = nullptr;
- }
- }
- void *res = REAL(mmap)(addr, length, prot, flags, fd, offset);
- return res;
-}
-
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
-INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
- int fd, OFF64_T offset) {
- ENSURE_HWASAN_INITED();
+template <class Mmap>
+static void *mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T sz, int prot,
+ int flags, int fd, OFF64_T off) {
if (addr && !MEM_IS_APP(addr)) {
if (flags & map_fixed) {
errno = errno_EINVAL;
@@ -294,13 +279,8 @@ INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
addr = nullptr;
}
}
- void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset);
- return res;
+ return real_mmap(addr, sz, prot, flags, fd, off);
}
-#define HWASAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64)
-#else
-#define HWASAN_MAYBE_INTERCEPT_MMAP64
-#endif
extern "C" int pthread_attr_init(void *attr);
extern "C" int pthread_attr_destroy(void *attr);
@@ -427,6 +407,22 @@ int OnExit() {
*begin = *end = 0; \
}
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
+ { \
+ COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \
+ if (common_flags()->intercept_intrin && \
+ MEM_IS_APP(GetAddressFromPointer(dst))) \
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
+ return REAL(memset)(dst, v, size); \
+ }
+
+#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, fd, \
+ offset) \
+ do { \
+ return mmap_interceptor(REAL(mmap), addr, length, prot, flags, fd, \
+ offset); \
+ } while (false)
+
#include "sanitizer_common/sanitizer_platform_interceptors.h"
#include "sanitizer_common/sanitizer_common_interceptors.inc"
#include "sanitizer_common/sanitizer_signal_interceptors.inc"
@@ -448,6 +444,7 @@ int OnExit() {
(void)(s); \
} while (false)
#include "sanitizer_common/sanitizer_common_syscalls.inc"
+#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
@@ -459,8 +456,6 @@ void InitializeInterceptors() {
InitializeCommonInterceptors();
InitializeSignalInterceptors();
- INTERCEPT_FUNCTION(mmap);
- HWASAN_MAYBE_INTERCEPT_MMAP64;
INTERCEPT_FUNCTION(posix_memalign);
HWASAN_MAYBE_INTERCEPT_MEMALIGN;
INTERCEPT_FUNCTION(__libc_memalign);
diff --git a/lib/hwasan/hwasan_interface_internal.h b/lib/hwasan/hwasan_interface_internal.h
index 7e95271ac0af..b4e5c80904df 100644
--- a/lib/hwasan/hwasan_interface_internal.h
+++ b/lib/hwasan/hwasan_interface_internal.h
@@ -18,6 +18,7 @@
#include "sanitizer_common/sanitizer_internal_defs.h"
extern "C" {
+
SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_init();
@@ -32,7 +33,10 @@ using __sanitizer::u16;
using __sanitizer::u8;
SANITIZER_INTERFACE_ATTRIBUTE
-void __hwasan_load(uptr, uptr);
+extern uptr __hwasan_shadow_memory_dynamic_address;
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_loadN(uptr, uptr);
SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_load1(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
@@ -45,7 +49,7 @@ SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_load16(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
-void __hwasan_load_noabort(uptr, uptr);
+void __hwasan_loadN_noabort(uptr, uptr);
SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_load1_noabort(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
@@ -58,7 +62,7 @@ SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_load16_noabort(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
-void __hwasan_store(uptr, uptr);
+void __hwasan_storeN(uptr, uptr);
SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_store1(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
@@ -71,7 +75,7 @@ SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_store16(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
-void __hwasan_store_noabort(uptr, uptr);
+void __hwasan_storeN_noabort(uptr, uptr);
SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_store1_noabort(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
@@ -83,6 +87,12 @@ void __hwasan_store8_noabort(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_store16_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_tag_memory(uptr p, u8 tag, uptr sz);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u8 __hwasan_generate_tag();
+
// Returns the offset of the first tag mismatch or -1 if the whole range is
// good.
SANITIZER_INTERFACE_ATTRIBUTE
diff --git a/lib/hwasan/hwasan_linux.cc b/lib/hwasan/hwasan_linux.cc
index 48dea8eb6ff4..5ab98dca594d 100644
--- a/lib/hwasan/hwasan_linux.cc
+++ b/lib/hwasan/hwasan_linux.cc
@@ -1,4 +1,4 @@
-//===-- hwasan_linux.cc -----------------------------------------------------===//
+//===-- hwasan_linux.cc -----------------------------------------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -6,41 +6,45 @@
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
-//
-// This file is a part of HWAddressSanitizer.
-//
-// Linux-, NetBSD- and FreeBSD-specific code.
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
+/// FreeBSD-specific code.
+///
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
#include "hwasan.h"
+#include "hwasan_dynamic_shadow.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_mapping.h"
+#include "hwasan_report.h"
#include "hwasan_thread.h"
#include <elf.h>
#include <link.h>
#include <pthread.h>
+#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
-#include <signal.h>
+#include <sys/resource.h>
+#include <sys/time.h>
#include <unistd.h>
#include <unwind.h>
-#include <sys/time.h>
-#include <sys/resource.h>
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_procmaps.h"
namespace __hwasan {
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
+static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
CHECK_EQ((beg % GetMmapGranularity()), 0);
CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
uptr size = end - beg + 1;
DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
- void *res = MmapFixedNoReserve(beg, size, name);
- if (res != (void *)beg) {
+ if (!MmapFixedNoReserve(beg, size, name)) {
Report(
"ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
"Perhaps you're using ulimit -v\n",
@@ -52,8 +56,11 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
}
static void ProtectGap(uptr addr, uptr size) {
+ if (!size)
+ return;
void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res) return;
+ if (addr == (uptr)res)
+ return;
// A few pages at the start of the address space can not be protected.
// But we really want to protect as much as possible, to prevent this memory
// being returned as a result of a non-FIXED mmap().
@@ -63,63 +70,160 @@ static void ProtectGap(uptr addr, uptr size) {
addr += step;
size -= step;
void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res) return;
+ if (addr == (uptr)res)
+ return;
}
}
Report(
- "ERROR: Failed to protect the shadow gap. "
- "ASan cannot proceed correctly. ABORTING.\n");
+ "ERROR: Failed to protect shadow gap [%p, %p]. "
+ "HWASan cannot proceed correctly. ABORTING.\n", (void *)addr,
+ (void *)(addr + size));
DumpProcessMap();
Die();
}
-bool InitShadow() {
- const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
+static uptr kLowMemStart;
+static uptr kLowMemEnd;
+static uptr kLowShadowEnd;
+static uptr kLowShadowStart;
+static uptr kHighShadowStart;
+static uptr kHighShadowEnd;
+static uptr kHighMemStart;
+static uptr kHighMemEnd;
+
+static void PrintRange(uptr start, uptr end, const char *name) {
+ Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
+}
- // LowMem covers as much of the first 4GB as possible.
- const uptr kLowMemEnd = 1UL<<32;
- const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale;
- const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale;
+static void PrintAddressSpaceLayout() {
+ PrintRange(kHighMemStart, kHighMemEnd, "HighMem");
+ if (kHighShadowEnd + 1 < kHighMemStart)
+ PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap");
+ else
+ CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
+ PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow");
+ if (SHADOW_OFFSET) {
+ if (kLowShadowEnd + 1 < kHighShadowStart)
+ PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap");
+ else
+ CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
+ PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
+ if (kLowMemEnd + 1 < kLowShadowStart)
+ PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap");
+ else
+ CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
+ PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
+ CHECK_EQ(0, kLowMemStart);
+ } else {
+ if (kLowMemEnd + 1 < kHighShadowStart)
+ PrintRange(kLowMemEnd + 1, kHighShadowStart - 1, "ShadowGap");
+ else
+ CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
+ PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
+ CHECK_EQ(kLowShadowEnd + 1, kLowMemStart);
+ PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
+ PrintRange(0, kLowShadowStart - 1, "ShadowGap");
+ }
+}
+static uptr GetHighMemEnd() {
// HighMem covers the upper part of the address space.
- const uptr kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1;
- const uptr kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale);
- CHECK(kHighShadowStart < kHighShadowEnd);
-
- const uptr kHighMemStart = kHighShadowStart << kShadowScale;
- CHECK(kHighShadowEnd <= kHighMemStart);
-
- if (Verbosity()) {
- Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemStart,
- (void *)maxVirtualAddress);
- if (kHighMemStart > kHighShadowEnd)
- Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd,
- (void *)kHighMemStart);
- Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowStart,
- (void *)kHighShadowEnd);
- if (kHighShadowStart > kLowMemEnd)
- Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd,
- (void *)kHighMemStart);
- Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowShadowEnd,
- (void *)kLowMemEnd);
- Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowStart,
- (void *)kLowShadowEnd);
- Printf("|| `[%p, %p]` || ShadowGap1 ||\n", (void *)0,
- (void *)kLowShadowStart);
+ uptr max_address = GetMaxUserVirtualAddress();
+ if (SHADOW_OFFSET)
+ // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
+ // properly aligned:
+ max_address |= SHADOW_GRANULARITY * GetMmapGranularity() - 1;
+ return max_address;
+}
+
+static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
+ // Set the shadow memory address to uninitialized.
+ __hwasan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
+ uptr shadow_start = SHADOW_OFFSET;
+ // Detect if a dynamic shadow address must be used and find the available
+ // location when necessary. When dynamic address is used, the macro
+ // kLowShadowBeg expands to __hwasan_shadow_memory_dynamic_address which
+ // was just set to kDefaultShadowSentinel.
+ if (shadow_start == kDefaultShadowSentinel) {
+ __hwasan_shadow_memory_dynamic_address = 0;
+ CHECK_EQ(0, SHADOW_OFFSET);
+ shadow_start = FindDynamicShadowStart(shadow_size_bytes);
}
+ // Update the shadow memory address (potentially) used by instrumentation.
+ __hwasan_shadow_memory_dynamic_address = shadow_start;
+}
- ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd - 1, "low shadow");
- ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd - 1, "high shadow");
- ProtectGap(0, kLowShadowStart);
- if (kHighShadowStart > kLowMemEnd)
- ProtectGap(kLowMemEnd, kHighShadowStart - kLowMemEnd);
- if (kHighMemStart > kHighShadowEnd)
- ProtectGap(kHighShadowEnd, kHighMemStart - kHighShadowEnd);
+bool InitShadow() {
+ // Define the entire memory range.
+ kHighMemEnd = GetHighMemEnd();
+
+ // Determine shadow memory base offset.
+ InitializeShadowBaseAddress(MEM_TO_SHADOW_SIZE(kHighMemEnd));
+
+ // Place the low memory first.
+ if (SHADOW_OFFSET) {
+ kLowMemEnd = SHADOW_OFFSET - 1;
+ kLowMemStart = 0;
+ } else {
+ // LowMem covers as much of the first 4GB as possible.
+ kLowMemEnd = (1UL << 32) - 1;
+ kLowMemStart = MEM_TO_SHADOW(kLowMemEnd) + 1;
+ }
+
+ // Define the low shadow based on the already placed low memory.
+ kLowShadowEnd = MEM_TO_SHADOW(kLowMemEnd);
+ kLowShadowStart = SHADOW_OFFSET ? SHADOW_OFFSET : MEM_TO_SHADOW(kLowMemStart);
+
+ // High shadow takes whatever memory is left up there (making sure it is not
+ // interfering with low memory in the fixed case).
+ kHighShadowEnd = MEM_TO_SHADOW(kHighMemEnd);
+ kHighShadowStart = Max(kLowMemEnd, MEM_TO_SHADOW(kHighShadowEnd)) + 1;
+
+ // High memory starts where allocated shadow allows.
+ kHighMemStart = SHADOW_TO_MEM(kHighShadowStart);
+
+ // Check the sanity of the defined memory ranges (there might be gaps).
+ CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
+ CHECK_GT(kHighMemStart, kHighShadowEnd);
+ CHECK_GT(kHighShadowEnd, kHighShadowStart);
+ CHECK_GT(kHighShadowStart, kLowMemEnd);
+ CHECK_GT(kLowMemEnd, kLowMemStart);
+ CHECK_GT(kLowShadowEnd, kLowShadowStart);
+ if (SHADOW_OFFSET)
+ CHECK_GT(kLowShadowStart, kLowMemEnd);
+ else
+ CHECK_GT(kLowMemEnd, kLowShadowStart);
+
+ if (Verbosity())
+ PrintAddressSpaceLayout();
+
+ // Reserve shadow memory.
+ ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow");
+ ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow");
+
+ // Protect all the gaps.
+ ProtectGap(0, Min(kLowMemStart, kLowShadowStart));
+ if (SHADOW_OFFSET) {
+ if (kLowMemEnd + 1 < kLowShadowStart)
+ ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1);
+ if (kLowShadowEnd + 1 < kHighShadowStart)
+ ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1);
+ } else {
+ if (kLowMemEnd + 1 < kHighShadowStart)
+ ProtectGap(kLowMemEnd + 1, kHighShadowStart - kLowMemEnd - 1);
+ }
+ if (kHighShadowEnd + 1 < kHighMemStart)
+ ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1);
return true;
}
+bool MemIsApp(uptr p) {
+ CHECK(GetTagFromPointer(p) == 0);
+ return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
+}
+
static void HwasanAtExit(void) {
if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
ReportStats();
@@ -177,50 +281,65 @@ struct AccessInfo {
bool recover;
};
-#if defined(__aarch64__)
static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
- // Access type is encoded in HLT immediate as 0x1XY,
- // where X&1 is 1 for store, 0 for load,
- // and X&2 is 1 if the error is recoverable.
- // Valid values of Y are 0 to 4, which are interpreted as log2(access_size),
- // and 0xF, which means that access size is stored in X1 register.
- // Access address is always in X0 register.
- AccessInfo ai;
+ // Access type is passed in a platform dependent way (see below) and encoded
+ // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
+ // recoverable. Valid values of Y are 0 to 4, which are interpreted as
+ // log2(access_size), and 0xF, which means that access size is passed via
+ // platform dependent register (see below).
+#if defined(__aarch64__)
+ // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
+ // access size is stored in X1 register. Access address is always in X0
+ // register.
uptr pc = (uptr)info->si_addr;
- unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
- if ((code & 0xff00) != 0x100)
- return AccessInfo{0, 0, false, false}; // Not ours.
- bool is_store = code & 0x10;
- bool recover = code & 0x20;
- unsigned size_log = code & 0xf;
+ const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
+ if ((code & 0xff00) != 0x900)
+ return AccessInfo{}; // Not ours.
+
+ const bool is_store = code & 0x10;
+ const bool recover = code & 0x20;
+ const uptr addr = uc->uc_mcontext.regs[0];
+ const unsigned size_log = code & 0xf;
+ if (size_log > 4 && size_log != 0xf)
+ return AccessInfo{}; // Not ours.
+ const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
+
+#elif defined(__x86_64__)
+ // Access type is encoded in the instruction following INT3 as
+ // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
+ // RSI register. Access address is always in RDI register.
+ uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
+ uint8_t *nop = (uint8_t*)pc;
+ if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
+ *(nop + 3) < 0x40)
+ return AccessInfo{}; // Not ours.
+ const unsigned code = *(nop + 3);
+
+ const bool is_store = code & 0x10;
+ const bool recover = code & 0x20;
+ const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
+ const unsigned size_log = code & 0xf;
if (size_log > 4 && size_log != 0xf)
- return AccessInfo{0, 0, false, false}; // Not ours.
+ return AccessInfo{}; // Not ours.
+ const uptr size =
+ size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
- ai.is_store = is_store;
- ai.is_load = !is_store;
- ai.addr = uc->uc_mcontext.regs[0];
- if (size_log == 0xf)
- ai.size = uc->uc_mcontext.regs[1];
- else
- ai.size = 1U << size_log;
- ai.recover = recover;
- return ai;
-}
#else
-static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
- return AccessInfo{0, 0, false, false};
-}
+# error Unsupported architecture
#endif
-static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
- SignalContext sig{info, uc};
+ return AccessInfo{addr, size, is_store, !is_store, recover};
+}
+
+static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
AccessInfo ai = GetAccessInfo(info, uc);
if (!ai.is_store && !ai.is_load)
return false;
- InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
+ InternalMmapVector<BufferedStackTrace> stack_buffer(1);
BufferedStackTrace *stack = stack_buffer.data();
stack->Reset();
+ SignalContext sig{info, uc};
GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc,
common_flags()->fast_unwind_on_fatal);
@@ -230,7 +349,12 @@ static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
if (flags()->halt_on_error || !ai.recover)
Die();
+#if defined(__aarch64__)
uc->uc_mcontext.pc += 4;
+#elif defined(__x86_64__)
+#else
+# error Unsupported architecture
+#endif
return true;
}
@@ -242,8 +366,8 @@ static void OnStackUnwind(const SignalContext &sig, const void *,
void HwasanOnDeadlySignal(int signo, void *info, void *context) {
// Probably a tag mismatch.
- if (signo == SIGILL)
- if (HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context))
+ if (signo == SIGTRAP)
+ if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context))
return;
HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
diff --git a/lib/hwasan/hwasan_mapping.h b/lib/hwasan/hwasan_mapping.h
new file mode 100644
index 000000000000..650a5aefcb2d
--- /dev/null
+++ b/lib/hwasan/hwasan_mapping.h
@@ -0,0 +1,85 @@
+//===-- hwasan_mapping.h ----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and defines memory mapping.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_MAPPING_H
+#define HWASAN_MAPPING_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+// Typical mapping on Linux/x86_64 with fixed shadow mapping:
+// || [0x080000000000, 0x7fffffffffff] || HighMem ||
+// || [0x008000000000, 0x07ffffffffff] || HighShadow ||
+// || [0x000100000000, 0x007fffffffff] || ShadowGap ||
+// || [0x000010000000, 0x0000ffffffff] || LowMem ||
+// || [0x000001000000, 0x00000fffffff] || LowShadow ||
+// || [0x000000000000, 0x000000ffffff] || ShadowGap ||
+//
+// and with dynamic shadow mapped at [0x770d59f40000, 0x7f0d59f40000]:
+// || [0x7f0d59f40000, 0x7fffffffffff] || HighMem ||
+// || [0x7efe2f934000, 0x7f0d59f3ffff] || HighShadow ||
+// || [0x7e7e2f934000, 0x7efe2f933fff] || ShadowGap ||
+// || [0x770d59f40000, 0x7e7e2f933fff] || LowShadow ||
+// || [0x000000000000, 0x770d59f3ffff] || LowMem ||
+
+// Typical mapping on Android/AArch64 (39-bit VMA):
+// || [0x001000000000, 0x007fffffffff] || HighMem ||
+// || [0x000800000000, 0x000fffffffff] || ShadowGap ||
+// || [0x000100000000, 0x0007ffffffff] || HighShadow ||
+// || [0x000010000000, 0x0000ffffffff] || LowMem ||
+// || [0x000001000000, 0x00000fffffff] || LowShadow ||
+// || [0x000000000000, 0x000000ffffff] || ShadowGap ||
+//
+// and with dynamic shadow mapped: [0x007477480000, 0x007c77480000]:
+// || [0x007c77480000, 0x007fffffffff] || HighMem ||
+// || [0x007c3ebc8000, 0x007c7747ffff] || HighShadow ||
+// || [0x007bbebc8000, 0x007c3ebc7fff] || ShadowGap ||
+// || [0x007477480000, 0x007bbebc7fff] || LowShadow ||
+// || [0x000000000000, 0x00747747ffff] || LowMem ||
+
+static constexpr __sanitizer::u64 kDefaultShadowSentinel = ~(__sanitizer::u64)0;
+
+// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
+constexpr __sanitizer::uptr kShadowScale = 4;
+constexpr __sanitizer::uptr kShadowAlignment = 1ULL << kShadowScale;
+
+#if SANITIZER_ANDROID
+# define HWASAN_FIXED_MAPPING 0
+#else
+# define HWASAN_FIXED_MAPPING 1
+#endif
+
+#if HWASAN_FIXED_MAPPING
+# define SHADOW_OFFSET (0)
+# define HWASAN_PREMAP_SHADOW 0
+#else
+# define SHADOW_OFFSET (__hwasan_shadow_memory_dynamic_address)
+# define HWASAN_PREMAP_SHADOW 1
+#endif
+
+#define SHADOW_GRANULARITY (1ULL << kShadowScale)
+
+#define MEM_TO_SHADOW(mem) (((uptr)(mem) >> kShadowScale) + SHADOW_OFFSET)
+#define SHADOW_TO_MEM(shadow) (((uptr)(shadow) - SHADOW_OFFSET) << kShadowScale)
+
+#define MEM_TO_SHADOW_SIZE(size) ((uptr)(size) >> kShadowScale)
+
+#define MEM_IS_APP(mem) MemIsApp((uptr)(mem))
+
+namespace __hwasan {
+
+bool MemIsApp(uptr p);
+
+} // namespace __hwasan
+
+#endif // HWASAN_MAPPING_H
diff --git a/lib/hwasan/hwasan_new_delete.cc b/lib/hwasan/hwasan_new_delete.cc
index 3ccc26734bf6..63ca74edd481 100644
--- a/lib/hwasan/hwasan_new_delete.cc
+++ b/lib/hwasan/hwasan_new_delete.cc
@@ -1,4 +1,4 @@
-//===-- hwasan_new_delete.cc ------------------------------------------------===//
+//===-- hwasan_new_delete.cc ----------------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
@@ -15,6 +15,7 @@
#include "hwasan.h"
#include "interception/interception.h"
#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
@@ -32,7 +33,7 @@ namespace std {
#define OPERATOR_NEW_BODY(nothrow) \
GET_MALLOC_STACK_TRACE; \
void *res = hwasan_malloc(size, &stack);\
- if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+ if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\
return res
INTERCEPTOR_ATTRIBUTE
diff --git a/lib/hwasan/hwasan_poisoning.cc b/lib/hwasan/hwasan_poisoning.cc
index 411fd05b10d5..b99d8ed0be79 100644
--- a/lib/hwasan/hwasan_poisoning.cc
+++ b/lib/hwasan/hwasan_poisoning.cc
@@ -13,6 +13,7 @@
#include "hwasan_poisoning.h"
+#include "hwasan_mapping.h"
#include "interception/interception.h"
#include "sanitizer_common/sanitizer_common.h"
@@ -22,7 +23,7 @@ uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
CHECK(IsAligned(p, kShadowAlignment));
CHECK(IsAligned(size, kShadowAlignment));
uptr shadow_start = MEM_TO_SHADOW(p);
- uptr shadow_size = MEM_TO_SHADOW_OFFSET(size);
+ uptr shadow_size = MEM_TO_SHADOW_SIZE(size);
internal_memset((void *)shadow_start, tag, shadow_size);
return AddTagToPointer(p, tag);
}
diff --git a/lib/hwasan/hwasan_report.cc b/lib/hwasan/hwasan_report.cc
index a3c6709491d3..16e9016ea35b 100644
--- a/lib/hwasan/hwasan_report.cc
+++ b/lib/hwasan/hwasan_report.cc
@@ -1,4 +1,4 @@
-//===-- hwasan_report.cc ----------------------------------------------------===//
+//===-- hwasan_report.cc --------------------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
@@ -14,6 +14,7 @@
#include "hwasan.h"
#include "hwasan_allocator.h"
+#include "hwasan_mapping.h"
#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_flags.h"
@@ -36,9 +37,9 @@ static StackTrace GetStackTraceFromId(u32 id) {
class Decorator: public __sanitizer::SanitizerCommonDecorator {
public:
Decorator() : SanitizerCommonDecorator() { }
- const char *Allocation() { return Magenta(); }
- const char *Origin() { return Magenta(); }
- const char *Name() { return Green(); }
+ const char *Allocation() const { return Magenta(); }
+ const char *Origin() const { return Magenta(); }
+ const char *Name() const { return Green(); }
};
struct HeapAddressDescription {
@@ -129,5 +130,4 @@ void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
ReportErrorSummary("tag-mismatch", stack);
}
-
} // namespace __hwasan
diff --git a/lib/hwasan/hwasan_report.h b/lib/hwasan/hwasan_report.h
new file mode 100644
index 000000000000..bb33f1a87308
--- /dev/null
+++ b/lib/hwasan/hwasan_report.h
@@ -0,0 +1,36 @@
+//===-- hwasan_report.h -----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer. HWASan-private header for error
+/// reporting functions.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_REPORT_H
+#define HWASAN_REPORT_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+
+namespace __hwasan {
+
+void ReportInvalidAccess(StackTrace *stack, u32 origin);
+void ReportStats();
+void ReportInvalidAccessInsideAddressRange(const char *what, const void *start,
+ uptr size, uptr offset);
+void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
+ bool is_store);
+
+void ReportAtExitStatistics();
+
+
+} // namespace __hwasan
+
+#endif // HWASAN_REPORT_H
diff --git a/lib/hwasan/hwasan_thread.cc b/lib/hwasan/hwasan_thread.cc
index d6551ffc6fdc..b50c0fc76be4 100644
--- a/lib/hwasan/hwasan_thread.cc
+++ b/lib/hwasan/hwasan_thread.cc
@@ -1,5 +1,6 @@
#include "hwasan.h"
+#include "hwasan_mapping.h"
#include "hwasan_thread.h"
#include "hwasan_poisoning.h"
#include "hwasan_interface_internal.h"
@@ -8,6 +9,19 @@
namespace __hwasan {
+static u32 RandomSeed() {
+ u32 seed;
+ do {
+ if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(&seed), sizeof(seed),
+ /*blocking=*/false))) {
+ seed = static_cast<u32>(
+ (NanoTime() >> 12) ^
+ (reinterpret_cast<uptr>(__builtin_frame_address(0)) >> 4));
+ }
+ } while (!seed);
+ return seed;
+}
+
HwasanThread *HwasanThread::Create(thread_callback_t start_routine,
void *arg) {
uptr PageSize = GetPageSizeCached();
@@ -16,6 +30,7 @@ HwasanThread *HwasanThread::Create(thread_callback_t start_routine,
thread->start_routine_ = start_routine;
thread->arg_ = arg;
thread->destructor_iterations_ = GetPthreadDestructorIterations();
+ thread->random_state_ = flags()->random_tags ? RandomSeed() : 0;
return thread;
}
@@ -72,4 +87,28 @@ thread_return_t HwasanThread::ThreadStart() {
return res;
}
+static u32 xorshift(u32 state) {
+ state ^= state << 13;
+ state ^= state >> 17;
+ state ^= state << 5;
+ return state;
+}
+
+// Generate a (pseudo-)random non-zero tag.
+tag_t HwasanThread::GenerateRandomTag() {
+ tag_t tag;
+ do {
+ if (flags()->random_tags) {
+ if (!random_buffer_)
+ random_buffer_ = random_state_ = xorshift(random_state_);
+ CHECK(random_buffer_);
+ tag = random_buffer_ & 0xFF;
+ random_buffer_ >>= 8;
+ } else {
+ tag = random_state_ = (random_state_ + 1) & 0xFF;
+ }
+ } while (!tag);
+ return tag;
+}
+
} // namespace __hwasan
diff --git a/lib/hwasan/hwasan_thread.h b/lib/hwasan/hwasan_thread.h
index 96f1bb813adf..1e482adeac84 100644
--- a/lib/hwasan/hwasan_thread.h
+++ b/lib/hwasan/hwasan_thread.h
@@ -52,6 +52,8 @@ class HwasanThread {
HwasanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
+ tag_t GenerateRandomTag();
+
int destructor_iterations_;
private:
@@ -70,6 +72,9 @@ class HwasanThread {
unsigned in_symbolizer_;
unsigned in_interceptor_scope_;
+ u32 random_state_;
+ u32 random_buffer_;
+
HwasanThreadLocalMallocStorage malloc_storage_;
};
diff --git a/lib/interception/.clang-format b/lib/interception/.clang-format
index f6cb8ad931f5..560308c91dee 100644
--- a/lib/interception/.clang-format
+++ b/lib/interception/.clang-format
@@ -1 +1,2 @@
BasedOnStyle: Google
+AllowShortIfStatementsOnASingleLine: false
diff --git a/lib/interception/CMakeLists.txt b/lib/interception/CMakeLists.txt
index 18d25948105d..c0ac974d726a 100644
--- a/lib/interception/CMakeLists.txt
+++ b/lib/interception/CMakeLists.txt
@@ -4,8 +4,13 @@ set(INTERCEPTION_SOURCES
interception_linux.cc
interception_mac.cc
interception_win.cc
- interception_type_test.cc
- )
+ interception_type_test.cc)
+
+set(INTERCEPTION_HEADERS
+ interception.h
+ interception_linux.h
+ interception_mac.h
+ interception_win.h)
include_directories(..)
@@ -16,6 +21,7 @@ add_compiler_rt_object_libraries(RTInterception
OS ${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
SOURCES ${INTERCEPTION_SOURCES}
+ ADDITIONAL_HEADERS ${INTERCEPTION_HEADERS}
CFLAGS ${INTERCEPTION_CFLAGS})
if(COMPILER_RT_INCLUDE_TESTS)
diff --git a/lib/interception/interception.h b/lib/interception/interception.h
index cba484936eac..ddd6ec20979b 100644
--- a/