feat(log): add glog library and logging capabilities. (#26)

cpp/CMakeLists.txt

@@ -11,12 +11,14 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 find_package(Boost REQUIRED)
 find_package(Arrow REQUIRED)
 find_package(protobuf REQUIRED)
+find_package(glog REQUIRED)
 
 file(GLOB_RECURSE SRC_FILES src/*.cpp src/*.cc)
 message(STATUS "SRC_FILES: ${SRC_FILES}")
 add_library(storage ${SRC_FILES})
 target_include_directories(storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src)
 target_link_libraries(storage PUBLIC arrow::libarrow arrow::libparquet Boost::boost protobuf::protobuf)
+target_link_libraries(storage PUBLIC glog::glog)
 
 if (WITH_UT)
   enable_testing()

cpp/conanfile.py

@@ -76,6 +76,7 @@ def requirements(self):
         self.requires("boost/1.81.0")
         self.requires("arrow/12.0.0")
         self.requires("protobuf/3.21.9")
+        self.requires("glog/0.6.0")
         if self.options.with_ut:
             self.requires("gtest/1.13.0")
 

cpp/src/common/log.cpp

@@ -0,0 +1,166 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "log.h"
+
+/*
+ * INITIALIZE_EASYLOGGINGPP will create a global variable whose name is same to that already created in knowhere,
+ * which will lead a `double-free` bug when the program exits.
+ * For why this issue happened please refer to
+ * https://gcc-help.gcc.gnu.narkive.com/KZGaXRNr/global-variable-in-static-library-double-free-or-corruption-error.
+ */
+// INITIALIZE_EASYLOGGINGPP
+
+#ifdef WIN32
+#include <Windows.h>
+#endif
+#include <chrono>
+#include <cstdarg>
+#include <cstdio>
+#include <memory>
+#include <stdexcept>
+#include <string>
+
+// namespace milvus {
+
+std::string
+LogOut(const char* pattern, ...) {
+  size_t len = strnlen(pattern, 1024) + 256;
+  auto str_p = std::make_unique<char[]>(len);
+  memset(str_p.get(), 0, len);
+
+  va_list vl;
+  va_start(vl, pattern);
+  vsnprintf(str_p.get(), len, pattern, vl);  // NOLINT
+  va_end(vl);
+
+  return std::string(str_p.get());
+}
+
+void
+SetThreadName(const std::string_view name) {
+  // Note: the name cannot exceed 16 bytes
+#ifdef __APPLE__
+  pthread_setname_np(name.data());
+#elif defined(__linux__) || defined(__MINGW64__)
+  pthread_setname_np(pthread_self(), name.data());
+#else
+#error "Unsupported SetThreadName";
+#endif
+}
+
+std::string
+GetThreadName() {
+  std::string thread_name = "unnamed";
+  char name[16];
+  size_t len = 16;
+  auto err = pthread_getname_np(pthread_self(), name, len);
+  if (not err) {
+    thread_name = name;
+  }
+
+  return thread_name;
+}
+
+int64_t
+get_now_timestamp() {
+  auto now = std::chrono::system_clock::now().time_since_epoch();
+  return std::chrono::duration_cast<std::chrono::seconds>(now).count();
+}
+
+#ifndef WIN32
+
+int64_t
+get_system_boottime() {
+  FILE* uptime = fopen("/proc/uptime", "r");
+  float since_sys_boot, _;
+  auto ret = fscanf(uptime, "%f %f", &since_sys_boot, &_);
+  fclose(uptime);
+  if (ret != 2) {
+    throw std::runtime_error("read /proc/uptime failed.");
+  }
+  return static_cast<int64_t>(since_sys_boot);
+}
+
+int64_t
+get_thread_starttime() {
+#ifdef __APPLE__
+  uint64_t tid;
+  pthread_threadid_np(nullptr, &tid);
+#elif __linux__
+  int64_t tid = gettid();
+#else
+#error "Unsupported SetThreadName";
+#endif
+
+  int64_t pid = getpid();
+  char filename[256];
+  snprintf(filename,
+           sizeof(filename),
+           "/proc/%lld/task/%lld/stat",
+           (long long)pid,   // NOLINT, TODO: How to solve this?
+           (long long)tid);  // NOLINT
+
+  int64_t val = 0;
+  char comm[16], state;
+  FILE* thread_stat = fopen(filename, "r");
+  auto ret = fscanf(
+      thread_stat, "%lld %s %s ", (long long*)&val, comm, &state);  // NOLINT
+
+  for (auto i = 4; i < 23; i++) {
+    ret = fscanf(thread_stat, "%lld ", (long long*)&val);  // NOLINT
+    if (i == 22) {
+      break;
+    }
+  }
+  fclose(thread_stat);
+  if (ret != 1) {
+    throw std::runtime_error("read " + std::string(filename) + " failed.");
+  }
+  return val / sysconf(_SC_CLK_TCK);
+}
+
+int64_t
+get_thread_start_timestamp() {
+  try {
+    return get_now_timestamp() - get_system_boottime() +
+           get_thread_starttime();
+  } catch (...) {
+    return 0;
+  }
+}
+
+#else
+
+#define WINDOWS_TICK 10000000
+#define SEC_TO_UNIX_EPOCH 11644473600LL
+
+int64_t
+get_thread_start_timestamp() {
+  FILETIME dummy;
+  FILETIME ret;
+
+  if (GetThreadTimes(GetCurrentThread(), &ret, &dummy, &dummy, &dummy)) {
+    auto ticks = Int64ShllMod32(ret.dwHighDateTime, 32) | ret.dwLowDateTime;
+    auto thread_started = ticks / WINDOWS_TICK - SEC_TO_UNIX_EPOCH;
+    return get_now_timestamp() - thread_started;
+  }
+  return 0;
+}
+
+#endif
+
+// }  // namespace milvus

cpp/src/common/log.h

@@ -0,0 +1,78 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <sys/types.h>
+#include <unistd.h>
+#include "glog/logging.h"
+
+// namespace milvus {
+
+#if __GLIBC__ == 2 && __GLIBC_MINOR__ < 30
+#include <sys/syscall.h>
+#define gettid() syscall(SYS_gettid)
+#endif
+
+// Log message format: %timestamp | %request_id | %level | %collection_name | %client_id | %client_tag | %client_ipport
+// | %thread_id | %thread_start_timestamp | %command_tag | %module | %error_code | %message
+
+#define VAR_REQUEST_ID (context->request_id())
+#define VAR_COLLECTION_NAME (context->collection_name())
+#define VAR_CLIENT_ID ("")
+#define VAR_CLIENT_TAG (context->client_tag())
+#define VAR_CLIENT_IPPORT (context->client_ipport())
+#define VAR_THREAD_ID (gettid())
+#define VAR_THREAD_START_TIMESTAMP (get_thread_start_timestamp())
+#define VAR_COMMAND_TAG (context->command_tag())
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define STORAGE_MODULE_NAME "STORAGE"
+#define STORAGE_MODULE_CLASS_FUNCTION \
+    LogOut("[%s][%s::%s][%s] ",       \
+           STORAGE_MODULE_NAME,       \
+           (typeid(*this).name()),    \
+           __FUNCTION__,              \
+           GetThreadName().c_str())
+#define STORAGE_MODULE_FUNCTION \
+    LogOut("[%s][%s][%s] ",     \
+           STORAGE_MODULE_NAME, \
+           __FUNCTION__,        \
+           GetThreadName().c_str())
+
+#define LOG_STORAGE_TRACE_ DLOG(INFO) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_DEBUG_ DLOG(INFO) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_MODULE_FUNCTION
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+std::string
+LogOut(const char* pattern, ...);
+
+void
+SetThreadName(const std::string_view name);
+
+std::string
+GetThreadName();
+
+int64_t
+get_thread_start_timestamp();
+
+// }  // namespace milvus

cpp/src/storage/schema.cpp

@@ -1,8 +1,8 @@
 #include "storage/schema.h"
 #include <memory>
-
 #include "common/macro.h"
 #include "common/utils.h"
+#include "common/log.h"
 namespace milvus_storage {
 
 Schema::Schema(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<SchemaOptions> options)
@@ -13,6 +13,7 @@ Status Schema::Validate() {
   RETURN_NOT_OK(BuildScalarSchema());
   RETURN_NOT_OK(BuildVectorSchema());
   RETURN_NOT_OK(BuildDeleteSchema());
+  LOG_STORAGE_DEBUG_ << "Schema validate success";
   return Status::OK();
 }
 

cpp/test/CMakeLists.txt

@@ -5,7 +5,8 @@ add_executable(
   options_test.cpp
   schema_test.cpp
   manifest_test.cpp
-  space_test.cpp)
+  space_test.cpp
+)
 
 target_link_libraries(
   milvus_test storage GTest::gtest_main