aboutsummaryrefslogtreecommitdiff
path: root/databases/clickhouse/files
diff options
context:
space:
mode:
authorRuslan Makhmatkhanov <rm@FreeBSD.org>2020-09-10 15:21:26 +0000
committerRuslan Makhmatkhanov <rm@FreeBSD.org>2020-09-10 15:21:26 +0000
commit5ac71a446cf8860129777cf7f0f72990dcf8dbc5 (patch)
tree310fa6f9a2a7d66a0359f044d480ddedcaca2f1c /databases/clickhouse/files
parent6385c18d93f05ce3e19750ead71b5d06ba0a057e (diff)
downloadports-5ac71a446cf8860129777cf7f0f72990dcf8dbc5.tar.gz
ports-5ac71a446cf8860129777cf7f0f72990dcf8dbc5.zip
databases/clickhouse: apply patch from #13859: MemoryTracking support
Clickhouse for FreeBSD updating no memory usage stats. In the case build without NDEBUG, allocator use 4KB chunk, those dont account when freeing and MemoryTracking metric grow up to ~6TB, after this clickhouse stop query processing with error "Memory limit (total) exceeded: would use 6.24 TiB" due skipping calc allocation < 4MB Patch still not upstreamed, obtained from: https://github.com/ClickHouse/ClickHouse/issues/13859 + https://github.com/ClickHouse/ClickHouse/pull/13869 reported and tested by: Vyacheslav (via Clickhouse telegram) PR: 249205 Submitted by: olevole@olevole.ru (maintainer) Obtained from: clickhouse repository
Notes
Notes: svn path=/head/; revision=548207
Diffstat (limited to 'databases/clickhouse/files')
-rw-r--r--databases/clickhouse/files/patch-cmake_freebsd_default__libs.cmake22
-rw-r--r--databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.cpp127
-rw-r--r--databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.h22
-rw-r--r--databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.cpp11
-rw-r--r--databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.h11
5 files changed, 193 insertions, 0 deletions
diff --git a/databases/clickhouse/files/patch-cmake_freebsd_default__libs.cmake b/databases/clickhouse/files/patch-cmake_freebsd_default__libs.cmake
new file mode 100644
index 000000000000..92eb1ef0be45
--- /dev/null
+++ b/databases/clickhouse/files/patch-cmake_freebsd_default__libs.cmake
@@ -0,0 +1,22 @@
+--- cmake/freebsd/default_libs.cmake.orig 2020-08-31 16:22:57 UTC
++++ cmake/freebsd/default_libs.cmake
+@@ -4,13 +4,13 @@ if (NOT COMPILER_CLANG)
+ message (FATAL_ERROR "FreeBSD build is supported only for Clang")
+ endif ()
+
+-if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64")
+- execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-x86_64.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+-else ()
+- execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+-endif ()
++#if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64")
++# execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-x86_64.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
++#else ()
++# execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
++#endif ()
+
+-set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread")
++set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -lcompiler_rt -lprocstat")
+
+ message(STATUS "Default libraries: ${DEFAULT_LIBS}")
+
diff --git a/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.cpp b/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.cpp
new file mode 100644
index 000000000000..a4e9ad63dbcc
--- /dev/null
+++ b/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.cpp
@@ -0,0 +1,127 @@
+--- src/Common/MemoryStatisticsOS.cpp.orig 2020-08-31 16:22:57 UTC
++++ src/Common/MemoryStatisticsOS.cpp
+@@ -1,7 +1,13 @@
+-#if defined(OS_LINUX)
+-
+ #include <sys/types.h>
+ #include <sys/stat.h>
++
++#ifdef OS_FREEBSD
++#include <sys/param.h>
++#include <sys/sysctl.h>
++#include <sys/user.h>
++#include <libprocstat.h>
++#endif
++
+ #include <fcntl.h>
+ #include <unistd.h>
+ #include <cassert>
+@@ -11,9 +17,10 @@
+ #include <Common/Exception.h>
+ #include <IO/ReadBufferFromMemory.h>
+ #include <IO/ReadHelpers.h>
+-#include <common/logger_useful.h>
+
+
++
++
+ namespace DB
+ {
+
+@@ -23,21 +30,40 @@ namespace ErrorCodes
+ extern const int CANNOT_OPEN_FILE;
+ extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
+ extern const int CANNOT_CLOSE_FILE;
++#ifdef OS_FREEBSD
++ extern const int CANNOT_ALLOCATE_MEMORY;
++#endif
+ }
+
++#ifndef OS_FREEBSD
+ static constexpr auto filename = "/proc/self/statm";
+ static constexpr size_t PAGE_SIZE = 4096;
++#endif
+
+ MemoryStatisticsOS::MemoryStatisticsOS()
+ {
++#ifdef OS_FREEBSD
++ pstat = ::procstat_open_sysctl();
++ if (NULL == pstat)
++ {
++ throwFromErrno("Cannot open sysctl", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
++ }
++#else
+ fd = ::open(filename, O_RDONLY | O_CLOEXEC);
+
+ if (-1 == fd)
+ throwFromErrno("Cannot open file " + std::string(filename), errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
++#endif
+ }
+
+ MemoryStatisticsOS::~MemoryStatisticsOS()
+ {
++#ifdef OS_FREEBSD
++ if (NULL != pstat)
++ {
++ ::procstat_close(pstat);
++ }
++#else
+ if (0 != ::close(fd))
+ {
+ try
+@@ -51,12 +77,43 @@ MemoryStatisticsOS::~MemoryStatisticsOS()
+ DB::tryLogCurrentException(__PRETTY_FUNCTION__);
+ }
+ }
++#endif
+ }
+
+ MemoryStatisticsOS::Data MemoryStatisticsOS::get() const
+ {
+ Data data;
+
++#ifdef OS_FREEBSD
++ size_t pagesize = ::getpagesize();
++ unsigned int count = 0;
++
++ struct kinfo_proc *kp;
++ struct kinfo_vmentry *kve;
++
++ kp = ::procstat_getprocs(pstat, KERN_PROC_PID, ::getpid(), &count);
++ if (NULL == kp)
++ {
++ throwFromErrno("Cannot get proc info", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
++ }
++
++ kve = ::procstat_getvmmap(pstat, kp, &count);
++ if (NULL == kve)
++ {
++ ::procstat_freeprocs(pstat, kp);
++ throwFromErrno("Cannot get vmmap info", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
++ }
++
++ data.virt = kp->ki_size;
++ data.resident = kp->ki_rssize * pagesize;
++ data.shared = (kp->ki_rssize - kve->kve_private_resident) * pagesize;
++ data.code = kp->ki_tsize * pagesize;
++ data.data_and_stack = (kp->ki_dsize + kp->ki_ssize) * pagesize;
++
++ ::procstat_freevmmap(pstat, kve);
++ ::procstat_freeprocs(pstat, kp);
++#else
++
+ constexpr size_t buf_size = 1024;
+ char buf[buf_size];
+
+@@ -98,10 +155,8 @@ MemoryStatisticsOS::Data MemoryStatisticsOS::get() con
+ data.shared *= PAGE_SIZE;
+ data.code *= PAGE_SIZE;
+ data.data_and_stack *= PAGE_SIZE;
+-
++#endif
+ return data;
+ }
+
+ }
+-
+-#endif
diff --git a/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.h b/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.h
new file mode 100644
index 000000000000..053ce988d569
--- /dev/null
+++ b/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.h
@@ -0,0 +1,22 @@
+--- src/Common/MemoryStatisticsOS.h.orig 2020-08-31 16:22:57 UTC
++++ src/Common/MemoryStatisticsOS.h
+@@ -1,5 +1,4 @@
+ #pragma once
+-#if defined(OS_LINUX)
+ #include <cstdint>
+
+
+@@ -35,9 +34,12 @@ class MemoryStatisticsOS (public)
+ Data get() const;
+
+ private:
++#ifdef OS_FREEBSD
++ struct procstat * pstat;
++#else
+ int fd;
++#endif
+ };
+
+ }
+
+-#endif
diff --git a/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.cpp b/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.cpp
new file mode 100644
index 000000000000..0f7ea8b8ce75
--- /dev/null
+++ b/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.cpp
@@ -0,0 +1,11 @@
+--- src/Interpreters/AsynchronousMetrics.cpp.orig 2020-08-31 16:22:57 UTC
++++ src/Interpreters/AsynchronousMetrics.cpp
+@@ -194,7 +194,7 @@ void AsynchronousMetrics::update()
+ new_values["Uptime"] = context.getUptimeSeconds();
+
+ /// Process memory usage according to OS
+-#if defined(OS_LINUX)
++#if defined(OS_LINUX) || defined(OS_FREEBSD)
+ {
+ MemoryStatisticsOS::Data data = memory_stat.get();
+
diff --git a/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.h b/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.h
new file mode 100644
index 000000000000..5fa3e02fcb07
--- /dev/null
+++ b/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.h
@@ -0,0 +1,11 @@
+--- src/Interpreters/AsynchronousMetrics.h.orig 2020-08-31 16:22:57 UTC
++++ src/Interpreters/AsynchronousMetrics.h
+@@ -50,7 +50,7 @@ class AsynchronousMetrics (private)
+ bool quit {false};
+ AsynchronousMetricValues values;
+
+-#if defined(OS_LINUX)
++#if defined(OS_LINUX) || defined(OS_FREEBSD)
+ MemoryStatisticsOS memory_stat;
+ #endif
+