From c8f0c4e67de2cd5bc10dbb740e66a7ed720c36a6 Mon Sep 17 00:00:00 2001 From: Xwg Date: Wed, 26 Jul 2023 11:44:56 +0800 Subject: [PATCH] feat(log): add glog library and logging capabilities. (#26) --- cpp/CMakeLists.txt | 2 + cpp/conanfile.py | 1 + cpp/src/common/log.cpp | 166 +++++++++++++++++++++++++++++++++++++ cpp/src/common/log.h | 78 +++++++++++++++++ cpp/src/storage/schema.cpp | 3 +- cpp/test/CMakeLists.txt | 3 +- 6 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 cpp/src/common/log.cpp create mode 100644 cpp/src/common/log.h diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 761e2ee4..334549a1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -11,12 +11,14 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) find_package(Boost REQUIRED) find_package(Arrow REQUIRED) find_package(protobuf REQUIRED) +find_package(glog REQUIRED) file(GLOB_RECURSE SRC_FILES src/*.cpp src/*.cc) message(STATUS "SRC_FILES: ${SRC_FILES}") add_library(storage ${SRC_FILES}) target_include_directories(storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) target_link_libraries(storage PUBLIC arrow::libarrow arrow::libparquet Boost::boost protobuf::protobuf) +target_link_libraries(storage PUBLIC glog::glog) if (WITH_UT) enable_testing() diff --git a/cpp/conanfile.py b/cpp/conanfile.py index 5cfda68a..b4b4e7fe 100644 --- a/cpp/conanfile.py +++ b/cpp/conanfile.py @@ -76,6 +76,7 @@ def requirements(self): self.requires("boost/1.81.0") self.requires("arrow/12.0.0") self.requires("protobuf/3.21.9") + self.requires("glog/0.6.0") if self.options.with_ut: self.requires("gtest/1.13.0") diff --git a/cpp/src/common/log.cpp b/cpp/src/common/log.cpp new file mode 100644 index 00000000..3e1138c0 --- /dev/null +++ b/cpp/src/common/log.cpp @@ -0,0 +1,166 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "log.h" + +/* + * INITIALIZE_EASYLOGGINGPP will create a global variable whose name is same to that already created in knowhere, + * which will lead a `double-free` bug when the program exits. + * For why this issue happened please refer to + * https://gcc-help.gcc.gnu.narkive.com/KZGaXRNr/global-variable-in-static-library-double-free-or-corruption-error. + */ +// INITIALIZE_EASYLOGGINGPP + +#ifdef WIN32 +#include +#endif +#include +#include +#include +#include +#include +#include + +// namespace milvus { + +std::string +LogOut(const char* pattern, ...) { + size_t len = strnlen(pattern, 1024) + 256; + auto str_p = std::make_unique(len); + memset(str_p.get(), 0, len); + + va_list vl; + va_start(vl, pattern); + vsnprintf(str_p.get(), len, pattern, vl); // NOLINT + va_end(vl); + + return std::string(str_p.get()); +} + +void +SetThreadName(const std::string_view name) { + // Note: the name cannot exceed 16 bytes +#ifdef __APPLE__ + pthread_setname_np(name.data()); +#elif defined(__linux__) || defined(__MINGW64__) + pthread_setname_np(pthread_self(), name.data()); +#else +#error "Unsupported SetThreadName"; +#endif +} + +std::string +GetThreadName() { + std::string thread_name = "unnamed"; + char name[16]; + size_t len = 16; + auto err = pthread_getname_np(pthread_self(), name, len); + if (not err) { + thread_name = name; + } + + return thread_name; +} + +int64_t +get_now_timestamp() { + auto now = std::chrono::system_clock::now().time_since_epoch(); + return std::chrono::duration_cast(now).count(); +} + +#ifndef WIN32 + +int64_t +get_system_boottime() { + FILE* uptime = fopen("/proc/uptime", "r"); + float since_sys_boot, _; + auto ret = fscanf(uptime, "%f %f", &since_sys_boot, &_); + fclose(uptime); + if (ret != 2) { + throw std::runtime_error("read /proc/uptime failed."); + } + return static_cast(since_sys_boot); +} + +int64_t +get_thread_starttime() { +#ifdef __APPLE__ + uint64_t tid; + pthread_threadid_np(nullptr, &tid); +#elif __linux__ + int64_t tid = gettid(); +#else +#error "Unsupported SetThreadName"; +#endif + + int64_t pid = getpid(); + char filename[256]; + snprintf(filename, + sizeof(filename), + "/proc/%lld/task/%lld/stat", + (long long)pid, // NOLINT, TODO: How to solve this? + (long long)tid); // NOLINT + + int64_t val = 0; + char comm[16], state; + FILE* thread_stat = fopen(filename, "r"); + auto ret = fscanf( + thread_stat, "%lld %s %s ", (long long*)&val, comm, &state); // NOLINT + + for (auto i = 4; i < 23; i++) { + ret = fscanf(thread_stat, "%lld ", (long long*)&val); // NOLINT + if (i == 22) { + break; + } + } + fclose(thread_stat); + if (ret != 1) { + throw std::runtime_error("read " + std::string(filename) + " failed."); + } + return val / sysconf(_SC_CLK_TCK); +} + +int64_t +get_thread_start_timestamp() { + try { + return get_now_timestamp() - get_system_boottime() + + get_thread_starttime(); + } catch (...) { + return 0; + } +} + +#else + +#define WINDOWS_TICK 10000000 +#define SEC_TO_UNIX_EPOCH 11644473600LL + +int64_t +get_thread_start_timestamp() { + FILETIME dummy; + FILETIME ret; + + if (GetThreadTimes(GetCurrentThread(), &ret, &dummy, &dummy, &dummy)) { + auto ticks = Int64ShllMod32(ret.dwHighDateTime, 32) | ret.dwLowDateTime; + auto thread_started = ticks / WINDOWS_TICK - SEC_TO_UNIX_EPOCH; + return get_now_timestamp() - thread_started; + } + return 0; +} + +#endif + +// } // namespace milvus diff --git a/cpp/src/common/log.h b/cpp/src/common/log.h new file mode 100644 index 00000000..bfcf9559 --- /dev/null +++ b/cpp/src/common/log.h @@ -0,0 +1,78 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include "glog/logging.h" + +// namespace milvus { + +#if __GLIBC__ == 2 && __GLIBC_MINOR__ < 30 +#include +#define gettid() syscall(SYS_gettid) +#endif + +// Log message format: %timestamp | %request_id | %level | %collection_name | %client_id | %client_tag | %client_ipport +// | %thread_id | %thread_start_timestamp | %command_tag | %module | %error_code | %message + +#define VAR_REQUEST_ID (context->request_id()) +#define VAR_COLLECTION_NAME (context->collection_name()) +#define VAR_CLIENT_ID ("") +#define VAR_CLIENT_TAG (context->client_tag()) +#define VAR_CLIENT_IPPORT (context->client_ipport()) +#define VAR_THREAD_ID (gettid()) +#define VAR_THREAD_START_TIMESTAMP (get_thread_start_timestamp()) +#define VAR_COMMAND_TAG (context->command_tag()) + +///////////////////////////////////////////////////////////////////////////////////////////////// +#define STORAGE_MODULE_NAME "STORAGE" +#define STORAGE_MODULE_CLASS_FUNCTION \ + LogOut("[%s][%s::%s][%s] ", \ + STORAGE_MODULE_NAME, \ + (typeid(*this).name()), \ + __FUNCTION__, \ + GetThreadName().c_str()) +#define STORAGE_MODULE_FUNCTION \ + LogOut("[%s][%s][%s] ", \ + STORAGE_MODULE_NAME, \ + __FUNCTION__, \ + GetThreadName().c_str()) + +#define LOG_STORAGE_TRACE_ DLOG(INFO) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_DEBUG_ DLOG(INFO) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_MODULE_FUNCTION +#define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_MODULE_FUNCTION + +///////////////////////////////////////////////////////////////////////////////////////////////// + +std::string +LogOut(const char* pattern, ...); + +void +SetThreadName(const std::string_view name); + +std::string +GetThreadName(); + +int64_t +get_thread_start_timestamp(); + +// } // namespace milvus diff --git a/cpp/src/storage/schema.cpp b/cpp/src/storage/schema.cpp index abfac23e..43f51aec 100644 --- a/cpp/src/storage/schema.cpp +++ b/cpp/src/storage/schema.cpp @@ -1,8 +1,8 @@ #include "storage/schema.h" #include - #include "common/macro.h" #include "common/utils.h" +#include "common/log.h" namespace milvus_storage { Schema::Schema(std::shared_ptr schema, std::shared_ptr options) @@ -13,6 +13,7 @@ Status Schema::Validate() { RETURN_NOT_OK(BuildScalarSchema()); RETURN_NOT_OK(BuildVectorSchema()); RETURN_NOT_OK(BuildDeleteSchema()); + LOG_STORAGE_DEBUG_ << "Schema validate success"; return Status::OK(); } diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index fc9f005f..48a2e777 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -5,7 +5,8 @@ add_executable( options_test.cpp schema_test.cpp manifest_test.cpp - space_test.cpp) + space_test.cpp +) target_link_libraries( milvus_test storage GTest::gtest_main