diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f59c9b847e..04461d658c 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -129,7 +129,7 @@ set(SUB_DIRECTORIES_LIST file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling prometheus prometheus/labels prometheus/schedulers prometheus/async prometheus/component ebpf ebpf/observer ebpf/security ebpf/handler - parser sls_control sdk + parser ) if (LINUX) if (ENABLE_ENTERPRISE) diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index 1410714e3a..7ec06437c0 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -124,8 +124,6 @@ DECLARE_FLAG_INT32(reader_close_unused_file_time); DECLARE_FLAG_INT32(batch_send_interval); DECLARE_FLAG_INT32(batch_send_metric_size); -DECLARE_FLAG_BOOL(send_prefer_real_ip); -DECLARE_FLAG_INT32(send_switch_real_ip_interval); DECLARE_FLAG_INT32(truncate_pos_skip_bytes); DECLARE_FLAG_INT32(default_tail_limit_kb); @@ -989,26 +987,11 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) { mCheckPointFilePath = AbsolutePath(mCheckPointFilePath, mProcessExecutionDir); LOG_INFO(sLogger, ("logtail checkpoint path", mCheckPointFilePath)); - if (confJson.isMember("send_prefer_real_ip") && confJson["send_prefer_real_ip"].isBool()) { - BOOL_FLAG(send_prefer_real_ip) = confJson["send_prefer_real_ip"].asBool(); - } - - if (confJson.isMember("send_switch_real_ip_interval") && confJson["send_switch_real_ip_interval"].isInt()) { - INT32_FLAG(send_switch_real_ip_interval) = confJson["send_switch_real_ip_interval"].asInt(); - } - LoadInt32Parameter(INT32_FLAG(truncate_pos_skip_bytes), confJson, "truncate_pos_skip_bytes", "ALIYUN_LOGTAIL_TRUNCATE_POS_SKIP_BYTES"); - if (BOOL_FLAG(send_prefer_real_ip)) { - LOG_INFO(sLogger, - ("change send policy, prefer use real ip, switch interval seconds", - INT32_FLAG(send_switch_real_ip_interval))("truncate skip read offset", - INT32_FLAG(truncate_pos_skip_bytes))); - } - if (confJson.isMember("ignore_dir_inode_changed") && confJson["ignore_dir_inode_changed"].isBool()) { mIgnoreDirInodeChanged = confJson["ignore_dir_inode_changed"].asBool(); } diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 71a8d2df49..4c555f5b8b 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -316,7 +316,7 @@ class AppConfig { public: AppConfig(); - ~AppConfig(){}; + ~AppConfig() {}; void LoadInstanceConfig(const std::map>&); @@ -533,6 +533,8 @@ class AppConfig { friend class InputPrometheusUnittest; friend class InputContainerStdioUnittest; friend class BatcherUnittest; + friend class EnterpriseSLSClientManagerUnittest; + friend class FlusherRunnerUnittest; friend class PipelineUpdateUnittest; #endif }; diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 88cb9e5a31..a32acb08a8 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -46,6 +46,7 @@ #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/SenderQueueManager.h" #include "plugin/flusher/sls/DiskBufferWriter.h" +#include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/input/InputFeedbackInterfaceRegistry.h" #include "prometheus/PrometheusInputRunner.h" #include "runner/FlusherRunner.h" @@ -73,9 +74,6 @@ DEFINE_FLAG_INT32(queue_check_gc_interval_sec, "30s", 30); DEFINE_FLAG_BOOL(enable_cgroup, "", false); #endif -DECLARE_FLAG_BOOL(send_prefer_real_ip); -DECLARE_FLAG_BOOL(global_network_success); - using namespace std; namespace logtail { @@ -199,11 +197,13 @@ void Application::Start() { // GCOVR_EXCL_START #if defined(__ENTERPRISE__) && defined(_MSC_VER) InitWindowsSignalObject(); #endif - BoundedSenderQueueInterface::SetFeedback(ProcessQueueManager::GetInstance()); - HttpSink::GetInstance()->Init(); - FlusherRunner::GetInstance()->Init(); + // resource monitor + // TODO: move metric related initialization to input Init + LoongCollectorMonitor::GetInstance()->Init(); + LogtailMonitor::GetInstance()->Init(); + // config provider { // add local config dir filesystem::path localConfigPath = filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir()) @@ -217,7 +217,6 @@ void Application::Start() { // GCOVR_EXCL_START } PipelineConfigWatcher::GetInstance()->AddSource(localConfigPath.string()); } - #ifdef __ENTERPRISE__ EnterpriseConfigProvider::GetInstance()->Start(); LegacyConfigProvider::GetInstance()->Init("legacy"); @@ -225,10 +224,16 @@ void Application::Start() { // GCOVR_EXCL_START InitRemoteConfigProviders(); #endif - AlarmManager::GetInstance()->Init(); - LoongCollectorMonitor::GetInstance()->Init(); - LogtailMonitor::GetInstance()->Init(); + // runner + BoundedSenderQueueInterface::SetFeedback(ProcessQueueManager::GetInstance()); + HttpSink::GetInstance()->Init(); + FlusherRunner::GetInstance()->Init(); + ProcessorRunner::GetInstance()->Init(); + // flusher_sls resource should be explicitly initialized to allow internal metrics and alarms to be sent + FlusherSLS::InitResource(); + + // plugin registration PluginRegistry::GetInstance()->LoadPlugins(); InputFeedbackInterfaceRegistry::GetInstance()->LoadFeedbackInterfaces(); @@ -258,10 +263,10 @@ void Application::Start() { // GCOVR_EXCL_START LogtailPlugin::GetInstance()->LoadPluginBase(); } - ProcessorRunner::GetInstance()->Init(); + // TODO: this should be refactored to internal pipeline + AlarmManager::GetInstance()->Init(); - time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, - lastCheckTagsTime = 0, lastQueueGCTime = 0; + time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, lastCheckTagsTime = 0, lastQueueGCTime = 0; #ifndef LOGTAIL_NO_TC_MALLOC time_t lastTcmallocReleaseMemTime = 0; #endif @@ -393,16 +398,6 @@ void Application::CheckCriticalCondition(int32_t curTime) { _exit(1); } #endif - // if network is fail in 2 hours, force exit (for ant only) - // work around for no network when docker start - if (BOOL_FLAG(send_prefer_real_ip) && !BOOL_FLAG(global_network_success) && curTime - mStartTime > 7200) { - LOG_ERROR(sLogger, ("network is fail", "prepare force exit")); - AlarmManager::GetInstance()->SendAlarm(LOGTAIL_CRASH_ALARM, - "network is fail since " + ToString(mStartTime) + " force exit"); - AlarmManager::GetInstance()->ForceToSend(); - sleep(10); - _exit(1); - } } bool Application::GetUUIDThread() { diff --git a/core/common/CompressTools.cpp b/core/common/CompressTools.cpp index fc7d753ee5..4aacdfe487 100644 --- a/core/common/CompressTools.cpp +++ b/core/common/CompressTools.cpp @@ -15,141 +15,9 @@ #include "CompressTools.h" #include -#ifdef __ANDROID__ -#include -#else -#include -#endif -#include - -#include - -#include "protobuf/sls/sls_logs.pb.h" namespace logtail { -const int32_t ZSTD_DEFAULT_LEVEL = 1; - -bool UncompressData(sls_logs::SlsCompressType compressType, - const std::string& src, - uint32_t rawSize, - std::string& dst) { - switch (compressType) { - case sls_logs::SLS_CMP_NONE: - dst = src; - return true; - case sls_logs::SLS_CMP_LZ4: - return UncompressLz4(src, rawSize, dst); - case sls_logs::SLS_CMP_DEFLATE: - return UncompressDeflate(src, rawSize, dst); - case sls_logs::SLS_CMP_ZSTD: - return UncompressZstd(src, rawSize, dst); - default: - return false; - } -} - -bool CompressData(sls_logs::SlsCompressType compressType, const std::string& src, std::string& dst) { - switch (compressType) { - case sls_logs::SLS_CMP_NONE: - dst = src; - return true; - case sls_logs::SLS_CMP_LZ4: - return CompressLz4(src, dst); - case sls_logs::SLS_CMP_DEFLATE: - return CompressDeflate(src, dst); - case sls_logs::SLS_CMP_ZSTD: - return CompressZstd(src, dst, ZSTD_DEFAULT_LEVEL); - default: - return false; - } -} - -bool CompressData(sls_logs::SlsCompressType compressType, const char* src, uint32_t size, std::string& dst) { - switch (compressType) { - case sls_logs::SLS_CMP_NONE: { - dst.assign(src, size); - return true; - } - case sls_logs::SLS_CMP_LZ4: - return CompressLz4(src, size, dst); - case sls_logs::SLS_CMP_DEFLATE: - return CompressDeflate(src, size, dst); - case sls_logs::SLS_CMP_ZSTD: - return CompressZstd(src, size, dst, ZSTD_DEFAULT_LEVEL); - default: - return false; - } -} - -bool UncompressLz4(const std::string& src, const uint32_t rawSize, char* dst) { - uint32_t length = 0; - try { - length = LZ4_decompress_safe(src.c_str(), dst, src.length(), rawSize); - } catch (...) { - return false; - } - if (length != rawSize) { - return false; - } - return true; -} - -bool UncompressLz4(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst) { - dst.resize(rawSize); - char* unCompressed = const_cast(dst.c_str()); - uint32_t length = 0; - try { - length = LZ4_decompress_safe(srcPtr, unCompressed, srcSize, rawSize); - } catch (...) { - return false; - } - if (length != rawSize) { - return false; - } - return true; -} -bool CompressDeflate(const char* srcPtr, const uint32_t srcSize, std::string& dst) { - int64_t dstLen = compressBound(srcSize); - dst.resize(dstLen); - if (compress((Bytef*)(dst.c_str()), (uLongf*)&dstLen, (const Bytef*)srcPtr, srcSize) == Z_OK) { - dst.resize(dstLen); - return true; - } - return false; -} - -bool CompressDeflate(const std::string& src, std::string& dst) { - int64_t dstLen = compressBound(src.size()); - dst.resize(dstLen); - if (compress((Bytef*)(dst.c_str()), (uLongf*)&dstLen, (const Bytef*)(src.c_str()), src.size()) == Z_OK) { - dst.resize(dstLen); - return true; - } - return false; -} - -bool UncompressDeflate(const char* srcPtr, const uint32_t srcSize, const int64_t rawSize, std::string& dst) { - static const int64_t MAX_UMCOMPRESS_SIZE = 128 * 1024 * 1024; - if (rawSize > MAX_UMCOMPRESS_SIZE) { - return false; - } - dst.resize(rawSize); - if (uncompress((Bytef*)(dst.c_str()), (uLongf*)&rawSize, (const Bytef*)(srcPtr), srcSize) != Z_OK) { - return false; - } - return true; -} - - -bool UncompressDeflate(const std::string& src, const int64_t rawSize, std::string& dst) { - return UncompressDeflate(src.c_str(), src.size(), rawSize, dst); -} - - -bool UncompressLz4(const std::string& src, const uint32_t rawSize, std::string& dst) { - return UncompressLz4(src.c_str(), src.length(), rawSize, dst); -} bool CompressLz4(const char* srcPtr, const uint32_t srcSize, std::string& dst) { uint32_t encodingSize = LZ4_compressBound(srcSize); dst.resize(encodingSize); @@ -169,43 +37,4 @@ bool CompressLz4(const std::string& src, std::string& dst) { return CompressLz4(src.c_str(), src.length(), dst); } -bool UncompressZstd(const std::string& src, const uint32_t rawSize, std::string& dst) { - return UncompressZstd(src.c_str(), src.length(), rawSize, dst); -} - -bool UncompressZstd(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst) { - dst.resize(rawSize); - char* unCompressed = const_cast(dst.c_str()); - uint32_t length = 0; - try { - length = ZSTD_decompress(unCompressed, rawSize, srcPtr, srcSize); - } catch (...) { - return false; - } - if (length != rawSize) { - return false; - } - return true; -} - -bool CompressZstd(const char* srcPtr, const uint32_t srcSize, std::string& dst, int32_t level) { - uint32_t encodingSize = ZSTD_compressBound(srcSize); - dst.resize(encodingSize); - char* compressed = const_cast(dst.c_str()); - try { - size_t const cmp_size = ZSTD_compress(compressed, encodingSize, srcPtr, srcSize, level); - if (ZSTD_isError(cmp_size)) { - return false; - } - dst.resize(cmp_size); - return true; - } catch (...) { - } - return false; -} - -bool CompressZstd(const std::string& src, std::string& dst, int32_t level) { - return CompressZstd(src.c_str(), src.length(), dst, level); -} - } // namespace logtail diff --git a/core/common/CompressTools.h b/core/common/CompressTools.h index 9293150ff9..1ccf1fe041 100644 --- a/core/common/CompressTools.h +++ b/core/common/CompressTools.h @@ -15,36 +15,14 @@ */ #pragma once -#include -#include -#include "protobuf/sls/sls_logs.pb.h" - -namespace logtail { - -extern const int32_t ZSTD_DEFAULT_LEVEL; - -bool UncompressData(sls_logs::SlsCompressType compressType, const std::string& src, uint32_t rawSize, std::string& dst); -bool CompressData(sls_logs::SlsCompressType compressType, const std::string& src, std::string& dst); -bool CompressData(sls_logs::SlsCompressType compressType, const char* src, uint32_t size, std::string& dst); - -bool UncompressDeflate(const std::string& src, const int64_t rawSize, std::string& dst); -bool UncompressDeflate(const char* srcPtr, const uint32_t srcSize, const int64_t rawSize, std::string& dst); +#include -bool CompressDeflate(const std::string& src, std::string& dst); -bool CompressDeflate(const char* srcPtr, const uint32_t srcSize, std::string& dst); +#include -bool UncompressLz4(const std::string& src, const uint32_t rawSize, std::string& dst); -bool UncompressLz4(const std::string& src, const uint32_t rawSize, char* dst); -bool UncompressLz4(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst); +namespace logtail { bool CompressLz4(const std::string& src, std::string& dst); bool CompressLz4(const char* srcPtr, const uint32_t srcSize, std::string& dest); -bool UncompressZstd(const std::string& src, const uint32_t rawSize, std::string& dst); -bool UncompressZstd(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst); - -bool CompressZstd(const char* srcPtr, const uint32_t srcSize, std::string& dst, int32_t level); -bool CompressZstd(const std::string& src, std::string& dst, int32_t level); - } // namespace logtail \ No newline at end of file diff --git a/core/common/DNSCache.cpp b/core/common/DNSCache.cpp index 8c1ce33f76..a5fc00480f 100644 --- a/core/common/DNSCache.cpp +++ b/core/common/DNSCache.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "DNSCache.h" + #include #if defined(__linux__) #include @@ -22,86 +23,91 @@ #include #endif +DEFINE_FLAG_INT32(dns_cache_ttl_sec, "", 600); + namespace logtail { - // ParseHost only supports IPv4 now. - bool DnsCache::ParseHost(const char* host, std::string& ip) { +DnsCache::DnsCache(const int32_t ttlSeconds) : mUpdateTime(time(NULL)), mDnsTTL(ttlSeconds) { +} + +// ParseHost only supports IPv4 now. +bool DnsCache::ParseHost(const char* host, std::string& ip) { #if defined(__linux__) - struct sockaddr_in addr; - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; - char* buffer = NULL; - if (host && host[0]) { - if (IsRawIp(host)) { - if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) - return false; - } else { - int bufferLen = 2048; - int rc, res; - struct hostent* hp = NULL; - struct hostent h; - while (true) { - buffer = new char[bufferLen]; - res = gethostbyname_r(host, &h, buffer, bufferLen, &hp, &rc); - if (res == ERANGE) { - if (buffer != NULL) - delete[] buffer; - bufferLen *= 4; - if (bufferLen > 32768) // 32KB - return false; - continue; - } - if (res != 0 || hp == NULL || hp->h_addr == NULL) { - if (buffer != NULL) - delete[] buffer; + char* buffer = NULL; + if (host && host[0]) { + if (IsRawIp(host)) { + if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) + return false; + } else { + int bufferLen = 2048; + int rc, res; + struct hostent* hp = NULL; + struct hostent h; + while (true) { + buffer = new char[bufferLen]; + res = gethostbyname_r(host, &h, buffer, bufferLen, &hp, &rc); + if (res == ERANGE) { + if (buffer != NULL) + delete[] buffer; + bufferLen *= 4; + if (bufferLen > 32768) // 32KB return false; - } else - break; + continue; } - addr.sin_addr.s_addr = *((in_addr_t*)(hp->h_addr)); + if (res != 0 || hp == NULL || hp->h_addr == NULL) { + if (buffer != NULL) + delete[] buffer; + return false; + } else + break; } - } else { - addr.sin_addr.s_addr = htonl(INADDR_ANY); + addr.sin_addr.s_addr = *((in_addr_t*)(hp->h_addr)); } - ip = inet_ntoa(addr.sin_addr); - if (buffer != NULL) - delete[] buffer; - return true; + } else { + addr.sin_addr.s_addr = htonl(INADDR_ANY); + } + ip = inet_ntoa(addr.sin_addr); + if (buffer != NULL) + delete[] buffer; + return true; #elif defined(_MSC_VER) - struct sockaddr_in addr; - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - if (host && host[0]) { - if (IsRawIp(host)) { - if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) - return false; - } else { - addrinfo hints; - struct addrinfo* result = NULL; - std::memset(&hints, 0, sizeof(hints)); - auto ret = ::getaddrinfo(host, NULL, &hints, &result); - if (ret != 0) { - return false; - } + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + if (host && host[0]) { + if (IsRawIp(host)) { + if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) + return false; + } else { + addrinfo hints; + struct addrinfo* result = NULL; + std::memset(&hints, 0, sizeof(hints)); + auto ret = ::getaddrinfo(host, NULL, &hints, &result); + if (ret != 0) { + return false; + } - bool found = false; - for (auto ptr = result; ptr != NULL; ptr = ptr->ai_next) { - if (AF_INET == ptr->ai_family) { - addr.sin_addr = ((struct sockaddr_in*)ptr->ai_addr)->sin_addr; - found = true; - break; - } + bool found = false; + for (auto ptr = result; ptr != NULL; ptr = ptr->ai_next) { + if (AF_INET == ptr->ai_family) { + addr.sin_addr = ((struct sockaddr_in*)ptr->ai_addr)->sin_addr; + found = true; + break; } - freeaddrinfo(result); - if (!found) - return false; } - } else { - addr.sin_addr.s_addr = htonl(INADDR_ANY); + freeaddrinfo(result); + if (!found) + return false; } - ip = inet_ntoa(addr.sin_addr); - return true; -#endif + } else { + addr.sin_addr.s_addr = htonl(INADDR_ANY); } + ip = inet_ntoa(addr.sin_addr); + return true; +#endif +} -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/common/DNSCache.h b/core/common/DNSCache.h index aec08786b0..90395ed187 100644 --- a/core/common/DNSCache.h +++ b/core/common/DNSCache.h @@ -15,11 +15,16 @@ */ #pragma once + #include #include #include #include +#include "common/Flags.h" + +DECLARE_FLAG_INT32(dns_cache_ttl_sec); + namespace logtail { class DnsCache { @@ -75,7 +80,7 @@ class DnsCache { } private: - DnsCache(const int32_t ttlSeconds = 60 * 10) : mUpdateTime(time(NULL)), mDnsTTL(ttlSeconds) {} + DnsCache(const int32_t ttlSeconds = INT32_FLAG(dns_cache_ttl_sec)); ~DnsCache() = default; bool IsRawIp(const char* host) { diff --git a/core/common/EncodingUtil.cpp b/core/common/EncodingUtil.cpp new file mode 100644 index 0000000000..32c8d836c7 --- /dev/null +++ b/core/common/EncodingUtil.cpp @@ -0,0 +1,75 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/EncodingUtil.h" + +#include + +using namespace std; + +namespace logtail { + +static void Base64Encoding(istream& is, ostream& os, char makeupChar, const char* alphabet) { + int out[4]; + int remain = 0; + while (!is.eof()) { + int byte1 = is.get(); + if (byte1 < 0) { + break; + } + int byte2 = is.get(); + int byte3; + if (byte2 < 0) { + byte2 = 0; + byte3 = 0; + remain = 1; + } else { + byte3 = is.get(); + if (byte3 < 0) { + byte3 = 0; + remain = 2; + } + } + out[0] = static_cast(byte1) >> 2; + out[1] = ((byte1 & 0x03) << 4) + (static_cast(byte2) >> 4); + out[2] = ((byte2 & 0x0F) << 2) + (static_cast(byte3) >> 6); + out[3] = byte3 & 0x3F; + + if (remain == 1) { + os.put(out[0] = alphabet[out[0]]); + os.put(out[1] = alphabet[out[1]]); + os.put(makeupChar); + os.put(makeupChar); + } else if (remain == 2) { + os.put(out[0] = alphabet[out[0]]); + os.put(out[1] = alphabet[out[1]]); + os.put(out[2] = alphabet[out[2]]); + os.put(makeupChar); + } else { + os.put(out[0] = alphabet[out[0]]); + os.put(out[1] = alphabet[out[1]]); + os.put(out[2] = alphabet[out[2]]); + os.put(out[3] = alphabet[out[3]]); + } + } +} + +string Base64Enconde(const string& message) { + istringstream iss(message); + ostringstream oss; + Base64Encoding(iss, oss, '=', "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"); + return oss.str(); +} + +} // namespace logtail diff --git a/core/common/EncodingUtil.h b/core/common/EncodingUtil.h new file mode 100644 index 0000000000..f467818627 --- /dev/null +++ b/core/common/EncodingUtil.h @@ -0,0 +1,25 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace logtail { + +std::string Base64Enconde(const std::string& message); + +} // namespace logtail diff --git a/core/common/EndpointUtil.cpp b/core/common/EndpointUtil.cpp index 67e4b34333..c466ab2eb3 100644 --- a/core/common/EndpointUtil.cpp +++ b/core/common/EndpointUtil.cpp @@ -16,6 +16,7 @@ #include "common/EndpointUtil.h" +#include "common/StringTools.h" #include "logger/Logger.h" using namespace std; @@ -23,10 +24,11 @@ using namespace std; namespace logtail { bool IsHttpsEndpoint(const string& endpoint) { - return endpoint.find("https://") == 0; + string trimmedEndpoint = TrimString(endpoint); + return trimmedEndpoint.find("https://") == 0; } - -string StandardizeEndpoint(const string& endpoint, const string& defaultEndpoint) { + +string StandardizeHost(const string& endpoint, const string& defaultEndpoint) { string res = endpoint; if (endpoint.find("https://") == 0) { if (endpoint.size() < string("https://x").size()) { @@ -49,6 +51,22 @@ string StandardizeEndpoint(const string& endpoint, const string& defaultEndpoint return res; } +string ExtractEndpoint(const string& endpoint) { + string trimmedEndpoint = TrimString(endpoint); + auto bpos = trimmedEndpoint.find("://"); + if (bpos == string::npos) { + bpos = 0; + } else { + bpos += strlen("://"); + } + + auto epos = trimmedEndpoint.find("/", bpos); + if (epos == string::npos) { + epos = trimmedEndpoint.length(); + } + return trimmedEndpoint.substr(bpos, epos - bpos); +} + string GetHostFromEndpoint(const std::string& endpoint) { static size_t httpSchemaLen = strlen("http://"), httpsSchemaLen = strlen("https://"); if (endpoint.find("https://") == 0) { diff --git a/core/common/EndpointUtil.h b/core/common/EndpointUtil.h index b678dc6792..3651a10e48 100644 --- a/core/common/EndpointUtil.h +++ b/core/common/EndpointUtil.h @@ -22,7 +22,9 @@ namespace logtail { bool IsHttpsEndpoint(const std::string& endpoint); -std::string StandardizeEndpoint(const std::string& endpoint, const std::string& defaultEndpoint); +std::string ExtractEndpoint(const std::string& endpoint); + +std::string StandardizeHost(const std::string& endpoint, const std::string& defaultEndpoint); std::string GetHostFromEndpoint(const std::string& endpoint); diff --git a/core/common/HashUtil.cpp b/core/common/HashUtil.cpp index 3b58726971..206c872f25 100644 --- a/core/common/HashUtil.cpp +++ b/core/common/HashUtil.cpp @@ -24,6 +24,8 @@ namespace logtail { +static constexpr uint32_t MD5_BYTES = 16; + /////////////////////////////////////////////// MACRO ////////////////////////////////////////////////// #define SHIFT_LEFT(a, b) ((a) << (b) | (a) >> (32 - b)) @@ -310,6 +312,22 @@ void DoMd5(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t md5[16]) } } /// DoMd5 +static std::string HexToString(const uint8_t md5[16]) { + static const char* table = "0123456789ABCDEF"; + std::string ss(32, 'a'); + for (int i = 0; i < 16; ++i) { + ss[i * 2] = table[md5[i] >> 4]; + ss[i * 2 + 1] = table[md5[i] & 0x0F]; + } + return ss; +} + +std::string CalcMD5(const std::string& message) { + uint8_t md5[MD5_BYTES]; + DoMd5((const uint8_t*)message.data(), message.length(), md5); + return HexToString(md5); +} + bool SignatureToHash(const std::string& signature, uint64_t& sigHash, uint32_t& sigSize) { sigSize = (uint32_t)signature.size(); sigHash = (uint64_t)HashSignatureString(signature.c_str(), signature.size()); diff --git a/core/common/HashUtil.h b/core/common/HashUtil.h index 1612ffeee8..5bd7204c74 100644 --- a/core/common/HashUtil.h +++ b/core/common/HashUtil.h @@ -24,6 +24,7 @@ namespace logtail { // Hash(string(@poolIn, @inputBytesNum)) => @md5. // TODO: Same implementation in sdk module, merge them. void DoMd5(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t md5[16]); +std::string CalcMD5(const std::string& message); bool SignatureToHash(const std::string& signature, uint64_t& sigHash, uint32_t& sigSize); bool CheckAndUpdateSignature(const std::string& signature, uint64_t& sigHash, uint32_t& sigSize); diff --git a/core/common/LogtailCommonFlags.cpp b/core/common/LogtailCommonFlags.cpp index 1d70138f27..ca328dce6c 100644 --- a/core/common/LogtailCommonFlags.cpp +++ b/core/common/LogtailCommonFlags.cpp @@ -96,10 +96,7 @@ DEFINE_FLAG_INT32(ilogtail_epoll_wait_events, "epoll_wait event number", 100); DEFINE_FLAG_INT32(ilogtail_max_epoll_events, "the max events number in epoll", 10000); // sls sender -DEFINE_FLAG_INT32(sls_client_send_timeout, "timeout time of one operation for SlsClient", 15); DEFINE_FLAG_BOOL(sls_client_send_compress, "whether compresses the data or not when put data", true); -DEFINE_FLAG_INT32(send_retrytimes, "how many times should retry if PostLogStoreLogs operation fail", 3); -DEFINE_FLAG_DOUBLE(loggroup_bytes_inflation, "", 1.2); DEFINE_FLAG_STRING(default_region_name, "for compatible with old user_log_config.json or old config server", "__default_region__"); diff --git a/core/common/LogtailCommonFlags.h b/core/common/LogtailCommonFlags.h index 2c980742a0..60bf0d1de9 100644 --- a/core/common/LogtailCommonFlags.h +++ b/core/common/LogtailCommonFlags.h @@ -34,10 +34,7 @@ DECLARE_FLAG_INT32(ilogtail_epoll_wait_events); DECLARE_FLAG_INT32(ilogtail_max_epoll_events); // sls sender -DECLARE_FLAG_INT32(sls_client_send_timeout); DECLARE_FLAG_BOOL(sls_client_send_compress); -DECLARE_FLAG_INT32(send_retrytimes); -DECLARE_FLAG_DOUBLE(loggroup_bytes_inflation); DECLARE_FLAG_STRING(default_region_name); // profile diff --git a/core/common/StringPiece.h b/core/common/StringPiece.h deleted file mode 100644 index 9e59362249..0000000000 --- a/core/common/StringPiece.h +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -namespace logtail { -namespace detail { - - template - class StringPieceDetail { - public: - // for standard STL container - typedef size_t size_type; - typedef CharT value_type; - typedef const value_type* pointer; - typedef const value_type& reference; - typedef const value_type& const_reference; - typedef ptrdiff_t difference_type; - typedef const value_type* const_iterator; - typedef std::reverse_iterator const_reverse_iterator; - - static const size_type npos; - - typedef std::basic_string string_type; - typedef typename string_type::traits_type traits_type; - - public: - StringPieceDetail(const value_type* str = NULL) - : mStr(str), mLength((str == NULL) ? 0 : traits_type::length(str)) {} - StringPieceDetail(const string_type& str) : mStr(str.data()), mLength(str.size()) {} - StringPieceDetail(const value_type* str, size_type len) : mStr(str), mLength(len) {} - StringPieceDetail(const typename string_type::const_iterator& begin, - const typename string_type::const_iterator& end) - : mStr((end > begin) ? &(*begin) : NULL), mLength((end > begin) ? (size_type)(end - begin) : 0) {} - - bool operator==(const StringPieceDetail& s) const { return this == &s || this->compare(s) == 0; } - - // Iterators - const_iterator begin() const { return data(); } - const_iterator end() const { return data() + size(); } - const_reverse_iterator rbegin() const { return const_reverse_iterator(data() + size()); } - const_reverse_iterator rend() const { return const_reverse_iterator(data()); } - - // Capacity - size_type size() const { return mLength; } - size_type length() const { return size(); } - size_type max_size() const { return size(); } - size_type capacity() const { return size(); } - void clear() { - mStr = NULL; - mLength = 0; - } - bool empty() const { return size() == 0; } - - // Element access - value_type operator[](size_type pos) const { return data()[pos]; } - value_type at(size_type pos) const { - if (pos >= size()) { - throw std::out_of_range("pos is out of range"); - } - return data()[pos]; - } - - // Modifiers - void swap(StringPieceDetail& sp) { - std::swap(sp.mStr, mStr); - std::swap(sp.mLength, mLength); - } - - // String Operations - const value_type* c_str() const { return mStr; } - // data() may return a pointer to a buffer with embedded NULs, and the - // returned buffer may or may not be null terminated. Therefore it is - // typically a mistake to pass data() to a routine that expects a NUL - // terminated string. - const value_type* data() const { return mStr; } - - // find - size_type find(const StringPieceDetail& sp, size_type pos = 0) const { return find(sp.data(), pos, sp.size()); } - size_type find(const value_type* s, size_type pos = 0) const { - StringPieceDetail sp(s); - return find(sp, pos); - } - size_type find(const value_type* s, size_type pos, size_type n) const { - size_type ret = npos; - const size_type size = this->size(); - if (pos <= size && n <= size - pos) { - const value_type* data = this->data(); - const value_type* p = std::search(data + pos, data + size, s, s + n); - if (p != data + size || n == 0) { - ret = p - data; - } - } - return ret; - } - size_type find(const value_type c, size_type pos = 0) const { - size_type ret = npos; - const size_type size = this->size(); - if (pos < size) { - const value_type* data = this->data(); - const size_type n = size - pos; - const value_type* p = traits_type::find(data + pos, n, c); - if (p != 0) { - ret = p - data; - } - } - return ret; - } - // rfind - size_type rfind(const StringPieceDetail& sp, size_type pos = npos) const { - return rfind(sp.data(), pos, sp.size()); - } - size_type rfind(const value_type* s, size_type pos = npos) const { - StringPieceDetail sp(s); - return rfind(sp, pos); - } - size_type rfind(const value_type* s, size_type pos, size_type n) const { - const size_type size = this->size(); - if (n <= size) { - pos = std::min(size_type(size - n), pos); - const value_type* data = this->data(); - do { - if (traits_type::compare(data + pos, s, n) == 0) { - return pos; - } - } while (pos-- > 0); - } - return npos; - } - size_type rfind(const value_type c, size_type pos = npos) const { - size_type size = this->size(); - if (size) { - if (--size > pos) { - size = pos; - } - for (++size; size-- > 0;) { - if (traits_type::eq(data()[size], c)) { - return size; - } - } - } - return npos; - } - // find_first_of - size_type find_first_of(const StringPieceDetail& s, size_type pos = 0) const { - return find_first_of(s.data(), pos, s.size()); - } - size_type find_first_of(const value_type* s, size_type pos = 0) const { - StringPieceDetail sp(s); - return find_first_of(sp, pos); - } - size_type find_first_of(const value_type* s, size_type pos, size_type n) const { - for (; n && pos < size(); ++pos) { - const value_type* p = traits_type::find(s, n, data()[pos]); - if (p) { - return pos; - } - } - return npos; - } - size_type find_first_of(value_type c, size_type pos = 0) const { return find(c, pos); } - // find_last_of - size_type find_last_of(const StringPieceDetail& s, size_type pos = npos) const { - return find_last_of(s.data(), pos, s.size()); - } - size_type find_last_of(const value_type* s, size_type pos = npos) const { - StringPieceDetail sp(s); - return find_last_of(sp, pos); - } - size_type find_last_of(const value_type* s, size_type pos, size_type n) const { - size_type index = size(); - if (index && n) { - if (--index > pos) { - index = pos; - } - do { - if (traits_type::find(s, n, data()[index])) { - return index; - } - } while (index-- != 0); - } - return npos; - } - size_type find_last_of(value_type c, size_type pos = npos) const { return rfind(c, pos); } - // find_first_not_of - size_type find_first_not_of(const StringPieceDetail& s, size_type pos = 0) const { - return find_first_not_of(s.data(), pos, s.size()); - } - size_type find_first_not_of(const value_type* s, size_type pos = 0) const { - StringPieceDetail sp(s); - return find_first_not_of(sp, pos); - } - size_type find_first_not_of(const value_type* s, size_type pos, size_type n) const { - for (; pos < size(); ++pos) { - if (!traits_type::find(s, n, data()[pos])) { - return pos; - } - } - return npos; - } - size_type find_first_not_of(value_type c, size_type pos = 0) const { - StringPieceDetail sp(&c, 1); - return find_first_not_of(sp, pos); - } - // find_last_not_of - size_type find_last_not_of(const StringPieceDetail& s, size_type pos = npos) const { - return find_last_not_of(s.data(), pos, s.size()); - } - size_type find_last_not_of(const value_type* s, size_type pos = npos) const { - StringPieceDetail sp(s); - return find_last_not_of(s, pos); - } - size_type find_last_not_of(const value_type* s, size_type pos, size_type n) const { - size_type index = size(); - if (index) { - if (--index > pos) { - index = pos; - } - do { - if (!traits_type::find(s, n, data()[index])) { - return index; - } - } while (index--); - } - return npos; - } - size_type find_last_not_of(value_type c, size_type pos = npos) const { - StringPieceDetail sp(&c, 1); - return find_last_not_of(sp, pos); - } - - void set(const value_type* data, size_type len) { - mStr = data; - mLength = len; - } - void set(const value_type* str) { - mStr = str; - mLength = str ? traits_type::length(str) : 0; - } - void set(const string_type& str) { - mStr = str.data(); - mLength = str.size(); - } - - void remove_prefix(size_type n) { - if (mLength < n) { - throw std::out_of_range("invalid parameter"); - } - mStr += n; - mLength -= n; - } - void remove_suffix(size_type n) { - if (mLength < n) { - throw std::out_of_range("invalid parameter"); - } - mLength -= n; - } - - StringPieceDetail substr(size_t pos = 0, size_t len = npos) const { - if (pos > size()) { - throw std::out_of_range("pos is out of range"); - } - const value_type* p = data() + pos; - if (len == npos) { - len = size() - pos; - } else { - len = std::min(size() - pos, len); - } - return StringPieceDetail(p, len); - } - int compare(const StringPieceDetail& s) const { - const size_type this_size = size(); - const size_type other_size = s.size(); - const size_type len = std::min(this_size, other_size); - - int r = traits_type::compare(data(), s.data(), len); - if (r == 0) { - r = this_size - other_size; - } - return r; - } - - string_type as_string() const { return empty() ? string_type() : string_type(data(), size()); } - - private: - const value_type* mStr; - size_type mLength; - }; - - template - const size_t StringPieceDetail::npos = -1; - -} // namespace detail - -typedef detail::StringPieceDetail StringPiece; -// typedef detail::StringPieceDetail StringPiece16; - -} // namespace logtail diff --git a/core/common/common.cmake b/core/common/common.cmake index cd5e9401c6..5cc621bd51 100644 --- a/core/common/common.cmake +++ b/core/common/common.cmake @@ -28,7 +28,7 @@ endif () list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES}) # add memory in common list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h) -list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpRequest.cpp) +list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpRequest.cpp ${CMAKE_SOURCE_DIR}/common/http/Constant.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/compression/Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/CompressorFactory.cpp ${CMAKE_SOURCE_DIR}/common/compression/LZ4Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/ZstdCompressor.cpp) # remove several files in common diff --git a/core/common/http/AsynCurlRunner.cpp b/core/common/http/AsynCurlRunner.cpp index ee9e514137..6ab352d997 100644 --- a/core/common/http/AsynCurlRunner.cpp +++ b/core/common/http/AsynCurlRunner.cpp @@ -16,7 +16,6 @@ #include -#include "app_config/AppConfig.h" #include "common/StringTools.h" #include "common/http/Curl.h" #include "logger/Logger.h" @@ -61,7 +60,7 @@ void AsynCurlRunner::Run() { LOG_DEBUG( sLogger, ("got request from queue, request address", request.get())("try cnt", ToString(request->mTryCnt))); - if (!AddRequestToClient(std::move(request))) { + if (!AddRequestToMultiCurlHandler(mClient, std::move(request))) { continue; } } else if (mIsFlush && mQueue.Empty()) { @@ -77,49 +76,6 @@ void AsynCurlRunner::Run() { } } -bool AsynCurlRunner::AddRequestToClient(unique_ptr&& request) { - curl_slist* headers = nullptr; - CURL* curl = CreateCurlHandler(request->mMethod, - request->mHTTPSFlag, - request->mHost, - request->mPort, - request->mUrl, - request->mQueryString, - request->mHeader, - request->mBody, - request->mResponse, - headers, - request->mTimeout, - AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(), - AppConfig::GetInstance()->GetBindInterface(), - request->mFollowRedirects, - request->mTls); - - if (curl == nullptr) { - LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get())); - request->mResponse.SetNetworkStatus(CURLE_FAILED_INIT); - request->OnSendDone(request->mResponse); - return false; - } - - request->mPrivateData = headers; - curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); - request->mLastSendTime = std::chrono::system_clock::now(); - auto res = curl_multi_add_handle(mClient, curl); - if (res != CURLM_OK) { - LOG_ERROR(sLogger, - ("failed to send request", "failed to add the easy curl handle to multi_handle")( - "errMsg", curl_multi_strerror(res))("request address", request.get())); - request->mResponse.SetNetworkStatus(CURLE_FAILED_INIT); - request->OnSendDone(request->mResponse); - curl_easy_cleanup(curl); - return false; - } - // let runner destruct the request - request.release(); - return true; -} - void AsynCurlRunner::DoRun() { CURLMcode mc; int runningHandlers = 1; @@ -131,14 +87,14 @@ void AsynCurlRunner::DoRun() { this_thread::sleep_for(chrono::milliseconds(100)); continue; } - HandleCompletedRequests(runningHandlers); + HandleCompletedAsynRequests(mClient, runningHandlers); unique_ptr request; if (mQueue.TryPop(request)) { LOG_DEBUG(sLogger, ("got item from flusher runner, request address", request.get())("try cnt", ToString(request->mTryCnt))); - if (AddRequestToClient(std::move(request))) { + if (AddRequestToMultiCurlHandler(mClient, std::move(request))) { ++runningHandlers; } } @@ -179,68 +135,4 @@ void AsynCurlRunner::DoRun() { } } -void AsynCurlRunner::HandleCompletedRequests(int& runningHandlers) { - int msgsLeft = 0; - CURLMsg* msg = curl_multi_info_read(mClient, &msgsLeft); - while (msg) { - if (msg->msg == CURLMSG_DONE) { - bool requestReused = false; - CURL* handler = msg->easy_handle; - AsynHttpRequest* request = nullptr; - curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); - auto responseTime - = chrono::duration_cast(chrono::system_clock::now() - request->mLastSendTime) - .count(); - switch (msg->data.result) { - case CURLE_OK: { - long statusCode = 0; - curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); - request->mResponse.SetNetworkStatus(CURLE_OK); - request->mResponse.SetStatusCode(statusCode); - request->OnSendDone(request->mResponse); - LOG_DEBUG(sLogger, - ("send http request succeeded, request address", - request)("response time", ToString(responseTime) + "ms")("try cnt", - ToString(request->mTryCnt))); - break; - } - default: - // considered as network error - if (request->mTryCnt < request->mMaxTryCnt) { - LOG_WARNING(sLogger, - ("failed to send http request", "retry immediately")("request address", request)( - "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); - // free first，becase mPrivateData will be reset in AddRequestToClient - if (request->mPrivateData) { - curl_slist_free_all((curl_slist*)request->mPrivateData); - request->mPrivateData = nullptr; - } - ++request->mTryCnt; - AddRequestToClient(unique_ptr(request)); - ++runningHandlers; - requestReused = true; - } else { - request->mResponse.SetNetworkStatus(msg->data.result); - request->OnSendDone(request->mResponse); - LOG_DEBUG( - sLogger, - ("failed to send http request", "abort")("request address", request)( - "response time", ToString(responseTime) + "ms")("try cnt", ToString(request->mTryCnt))); - } - break; - } - - curl_multi_remove_handle(mClient, handler); - curl_easy_cleanup(handler); - if (!requestReused) { - if (request->mPrivateData) { - curl_slist_free_all((curl_slist*)request->mPrivateData); - } - delete request; - } - } - msg = curl_multi_info_read(mClient, &msgsLeft); - } -} - } // namespace logtail diff --git a/core/common/http/AsynCurlRunner.h b/core/common/http/AsynCurlRunner.h index a5c02eb4d6..16a3dc3f93 100644 --- a/core/common/http/AsynCurlRunner.h +++ b/core/common/http/AsynCurlRunner.h @@ -48,9 +48,7 @@ class AsynCurlRunner { ~AsynCurlRunner() = default; void Run(); - bool AddRequestToClient(std::unique_ptr&& request); void DoRun(); - void HandleCompletedRequests(int& runningHandlers); CURLM* mClient = nullptr; SafeQueue> mQueue; diff --git a/core/common/http/Constant.cpp b/core/common/http/Constant.cpp new file mode 100644 index 0000000000..0a1421182c --- /dev/null +++ b/core/common/http/Constant.cpp @@ -0,0 +1,34 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/http/Constant.h" + +using namespace std; + +namespace logtail { + +const string HTTP_POST = "POST"; +const string HTTP_GET = "GET"; + +const string HOST = "Host"; +const string DATE = "Date"; +const string USER_AGENT = "User-Agent"; +const string CONTENT_TYPE = "Content-Type"; +const string CONTENT_LENGTH = "Content-Length"; +const string AUTHORIZATION = "Authorization"; +const string SIGNATURE = "Signature"; + +const string TYPE_LOG_PROTOBUF = "application/x-protobuf"; + +} // namespace logtail diff --git a/core/common/http/Constant.h b/core/common/http/Constant.h new file mode 100644 index 0000000000..e96716c9a2 --- /dev/null +++ b/core/common/http/Constant.h @@ -0,0 +1,36 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace logtail { + +extern const std::string HTTP_POST; +extern const std::string HTTP_GET; + +extern const std::string HOST; +extern const std::string DATE; +extern const std::string USER_AGENT; +extern const std::string CONTENT_LENGTH; +extern const std::string CONTENT_TYPE; +extern const std::string AUTHORIZATION; +extern const std::string SIGNATURE; + +extern const std::string TYPE_LOG_PROTOBUF; + +} // namespace logtail diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp index 19778c2abc..075ae6629d 100644 --- a/core/common/http/Curl.cpp +++ b/core/common/http/Curl.cpp @@ -20,6 +20,7 @@ #include "app_config/AppConfig.h" #include "common/DNSCache.h" +#include "common/StringTools.h" #include "common/http/HttpResponse.h" #include "logger/Logger.h" @@ -27,6 +28,46 @@ using namespace std; namespace logtail { +NetworkCode GetNetworkStatus(CURLcode code) { + // please refer to https://curl.se/libcurl/c/libcurl-errors.html + switch (code) { + case CURLE_OK: + return NetworkCode::Ok; + case CURLE_COULDNT_CONNECT: + return NetworkCode::ConnectionFailed; + case CURLE_LOGIN_DENIED: + case CURLE_REMOTE_ACCESS_DENIED: + return NetworkCode::RemoteAccessDenied; + case CURLE_OPERATION_TIMEDOUT: + return NetworkCode::Timeout; + case CURLE_SSL_CONNECT_ERROR: + return NetworkCode::SSLConnectError; + case CURLE_SSL_CERTPROBLEM: + case CURLE_SSL_CACERT: + return NetworkCode::SSLCertError; + case CURLE_SEND_ERROR: + case CURLE_SEND_FAIL_REWIND: + return NetworkCode::SendDataFailed; + case CURLE_RECV_ERROR: + return NetworkCode::RecvDataFailed; + case CURLE_SSL_PINNEDPUBKEYNOTMATCH: + case CURLE_SSL_INVALIDCERTSTATUS: + case CURLE_SSL_CACERT_BADFILE: + case CURLE_SSL_CIPHER: + case CURLE_SSL_ENGINE_NOTFOUND: + case CURLE_SSL_ENGINE_SETFAILED: + case CURLE_USE_SSL_FAILED: + case CURLE_SSL_ENGINE_INITFAILED: + case CURLE_SSL_CRL_BADFILE: + case CURLE_SSL_ISSUER_ERROR: + case CURLE_SSL_SHUTDOWN_FAILED: + return NetworkCode::SSLOtherProblem; + case CURLE_FAILED_INIT: + default: + return NetworkCode::Other; + } +} + static size_t header_write_callback(char* buffer, size_t size, size_t nmemb, @@ -57,21 +98,21 @@ static size_t header_write_callback(char* buffer, return sizes; } -CURL* CreateCurlHandler(const std::string& method, +CURL* CreateCurlHandler(const string& method, bool httpsFlag, - const std::string& host, + const string& host, int32_t port, - const std::string& url, - const std::string& queryString, - const std::map& header, - const std::string& body, + const string& url, + const string& queryString, + const map& header, + const string& body, HttpResponse& response, curl_slist*& headers, uint32_t timeout, bool replaceHostWithIp, - const std::string& intf, + const string& intf, bool followRedirects, - std::optional tls) { + optional tls) { static DnsCache* dnsCache = DnsCache::GetInstance(); CURL* curl = curl_easy_init(); @@ -80,7 +121,7 @@ CURL* CreateCurlHandler(const std::string& method, } string totalUrl = httpsFlag ? "https://" : "http://"; - std::string hostIP; + string hostIP; if (replaceHostWithIp && dnsCache->GetIPFromDnsCache(host, hostIP)) { totalUrl.append(hostIP); } else { @@ -144,7 +185,7 @@ CURL* CreateCurlHandler(const std::string& method, return curl; } -bool SendHttpRequest(std::unique_ptr&& request, HttpResponse& response) { +bool SendHttpRequest(unique_ptr&& request, HttpResponse& response) { curl_slist* headers = NULL; CURL* curl = CreateCurlHandler(request->mMethod, request->mHTTPSFlag, @@ -170,17 +211,30 @@ bool SendHttpRequest(std::unique_ptr&& request, HttpResponse& respo while (true) { CURLcode res = curl_easy_perform(curl); if (res == CURLE_OK) { - long http_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); - response.SetStatusCode(http_code); + long statusCode = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode); + curl_off_t responseTime; + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME_T, &responseTime); + auto responseTimeMs = responseTime / 1000; + response.SetNetworkStatus(NetworkCode::Ok, ""); + response.SetStatusCode(statusCode); + response.SetResponseTime(chrono::milliseconds(responseTimeMs)); success = true; + LOG_DEBUG(sLogger, + ("send http request succeeded, host", request->mHost)( + "response time", ToString(responseTimeMs) + "ms")("try cnt", ToString(request->mTryCnt))); break; } else if (request->mTryCnt < request->mMaxTryCnt) { - LOG_WARNING(sLogger, - ("failed to send http request", "retry immediately")("request address", request.get())( - "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(res))); + LOG_DEBUG(sLogger, + ("failed to send http request", "retry immediately")("host", request->mHost)( + "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(res))); ++request->mTryCnt; } else { + auto errMsg = curl_easy_strerror(res); + response.SetNetworkStatus(GetNetworkStatus(res), errMsg); + LOG_DEBUG(sLogger, + ("failed to send http request", + "abort")("host", request->mHost)("try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); break; } } @@ -191,4 +245,161 @@ bool SendHttpRequest(std::unique_ptr&& request, HttpResponse& respo return success; } +bool AddRequestToMultiCurlHandler(CURLM* multiCurl, unique_ptr&& request) { + curl_slist* headers = NULL; + CURL* curl = CreateCurlHandler(request->mMethod, + request->mHTTPSFlag, + request->mHost, + request->mPort, + request->mUrl, + request->mQueryString, + request->mHeader, + request->mBody, + request->mResponse, + headers, + request->mTimeout, + AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(), + AppConfig::GetInstance()->GetBindInterface(), + request->mFollowRedirects, + request->mTls); + if (curl == NULL) { + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to init curl handler"); + LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get())); + request->OnSendDone(request->mResponse); + return false; + } + + request->mPrivateData = headers; + curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); + request->mLastSendTime = chrono::system_clock::now(); + + CURLMcode res = curl_multi_add_handle(multiCurl, curl); + if (res != CURLM_OK) { + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to add the easy curl handle to multi_handle"); + LOG_ERROR(sLogger, + ("failed to send request", "failed to add the easy curl handle to multi_handle")( + "errMsg", curl_multi_strerror(res))("request address", request.get())); + request->OnSendDone(request->mResponse); + curl_easy_cleanup(curl); + return false; + } + // let callback destruct the request + request.release(); + return true; +} + +void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { + int msgsLeft = 0; + CURLMsg* msg = curl_multi_info_read(multiCurl, &msgsLeft); + while (msg) { + if (msg->msg == CURLMSG_DONE) { + bool requestReused = false; + CURL* handler = msg->easy_handle; + AsynHttpRequest* request = nullptr; + curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); + switch (msg->data.result) { + case CURLE_OK: { + long statusCode = 0; + curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); + curl_off_t responseTime; + curl_easy_getinfo(handler, CURLINFO_TOTAL_TIME_T, &responseTime); + auto responseTimeMs = responseTime / 1000; + request->mResponse.SetNetworkStatus(NetworkCode::Ok, ""); + request->mResponse.SetStatusCode(statusCode); + request->mResponse.SetResponseTime(chrono::milliseconds(responseTimeMs)); + LOG_DEBUG( + sLogger, + ("send http request succeeded, request address", + request)("host", request->mHost)("response time", ToString(responseTimeMs) + "ms")( + "try cnt", ToString(request->mTryCnt))("errMsg", curl_easy_strerror(msg->data.result))); + request->OnSendDone(request->mResponse); + break; + } + default: + // considered as network error + if (request->mTryCnt < request->mMaxTryCnt) { + LOG_DEBUG(sLogger, + ("failed to send http request", "retry immediately")("request address", + request)("host", request->mHost)( + "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); + // free first，becase mPrivateData will be reset in AddRequestToMultiCurlHandler + if (request->mPrivateData) { + curl_slist_free_all((curl_slist*)request->mPrivateData); + request->mPrivateData = nullptr; + } + ++request->mTryCnt; + AddRequestToMultiCurlHandler(multiCurl, unique_ptr(request)); + ++runningHandlers; + requestReused = true; + } else { + auto errMsg = curl_easy_strerror(msg->data.result); + request->mResponse.SetNetworkStatus(GetNetworkStatus(msg->data.result), errMsg); + LOG_DEBUG(sLogger, + ("failed to send http request", "abort")("request address", request)( + "host", request->mHost)("try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); + request->OnSendDone(request->mResponse); + } + break; + } + + curl_multi_remove_handle(multiCurl, handler); + curl_easy_cleanup(handler); + if (!requestReused) { + if (request->mPrivateData) { + curl_slist_free_all((curl_slist*)request->mPrivateData); + } + delete request; + } + } + msg = curl_multi_info_read(multiCurl, &msgsLeft); + } +} + +void SendAsynRequests(CURLM* multiCurl) { + CURLMcode mc; + int runningHandlers = 0; + do { + if ((mc = curl_multi_perform(multiCurl, &runningHandlers)) != CURLM_OK) { + LOG_ERROR( + sLogger, + ("failed to call curl_multi_perform", "sleep 100ms and retry")("errMsg", curl_multi_strerror(mc))); + this_thread::sleep_for(chrono::milliseconds(100)); + continue; + } + HandleCompletedAsynRequests(multiCurl, runningHandlers); + + long curlTimeout = -1; + if ((mc = curl_multi_timeout(multiCurl, &curlTimeout)) != CURLM_OK) { + LOG_WARNING( + sLogger, + ("failed to call curl_multi_timeout", "use default timeout 1s")("errMsg", curl_multi_strerror(mc))); + } + struct timeval timeout { + 1, 0 + }; + if (curlTimeout >= 0) { + timeout.tv_sec = curlTimeout / 1000; + timeout.tv_usec = (curlTimeout % 1000) * 1000; + } + + int maxfd = -1; + fd_set fdread; + fd_set fdwrite; + fd_set fdexcep; + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcep); + if ((mc = curl_multi_fdset(multiCurl, &fdread, &fdwrite, &fdexcep, &maxfd)) != CURLM_OK) { + LOG_ERROR(sLogger, ("failed to call curl_multi_fdset", "sleep 100ms")("errMsg", curl_multi_strerror(mc))); + } + if (maxfd == -1) { + // sleep min(timeout, 100ms) according to libcurl + int64_t sleepMs = (curlTimeout >= 0 && curlTimeout < 100) ? curlTimeout : 100; + this_thread::sleep_for(chrono::milliseconds(sleepMs)); + } else { + select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); + } + } while (runningHandlers); +} + } // namespace logtail diff --git a/core/common/http/Curl.h b/core/common/http/Curl.h index e1f311c017..ad21f876e8 100644 --- a/core/common/http/Curl.h +++ b/core/common/http/Curl.h @@ -28,6 +28,8 @@ namespace logtail { +NetworkCode GetNetworkStatus(CURLcode code); + CURL* CreateCurlHandler(const std::string& method, bool httpsFlag, const std::string& host, @@ -46,4 +48,8 @@ CURL* CreateCurlHandler(const std::string& method, bool SendHttpRequest(std::unique_ptr&& request, HttpResponse& response); +bool AddRequestToMultiCurlHandler(CURLM* multiCurl, std::unique_ptr&& request); +void SendAsynRequests(CURLM* multiCurl); +void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers); + } // namespace logtail diff --git a/core/common/http/HttpRequest.cpp b/core/common/http/HttpRequest.cpp index 3f6fd215d7..c108032283 100644 --- a/core/common/http/HttpRequest.cpp +++ b/core/common/http/HttpRequest.cpp @@ -14,9 +14,45 @@ #include "common/http/HttpRequest.h" -DEFINE_FLAG_INT32(default_http_request_timeout_secs, "", 15); +DEFINE_FLAG_INT32(default_http_request_timeout_sec, "", 15); DEFINE_FLAG_INT32(default_http_request_max_try_cnt, "", 3); using namespace std; -namespace logtail {} // namespace logtail +namespace logtail { + +static unsigned char ToHex(unsigned char x) { + return x > 9 ? x + 55 : x + 48; +} + +static string UrlEncode(const string& str) { + string strTemp; + size_t length = str.length(); + for (size_t i = 0; i < length; i++) { + if (isalnum((unsigned char)str[i]) || (str[i] == '-') || (str[i] == '_') || (str[i] == '.') || (str[i] == '~')) + strTemp += str[i]; + else if (str[i] == ' ') + strTemp += "+"; + else { + strTemp += '%'; + strTemp += ToHex((unsigned char)str[i] >> 4); + strTemp += ToHex((unsigned char)str[i] % 16); + } + } + return strTemp; +} + +string GetQueryString(const map& parameters) { + string res; + for (auto it = parameters.begin(); it != parameters.end(); ++it) { + if (it != parameters.begin()) { + res.append("&"); + } + res.append(it->first); + res.append("="); + res.append(UrlEncode(it->second)); + } + return res; +} + +} // namespace logtail diff --git a/core/common/http/HttpRequest.h b/core/common/http/HttpRequest.h index 13ef2db70c..2218568344 100644 --- a/core/common/http/HttpRequest.h +++ b/core/common/http/HttpRequest.h @@ -25,7 +25,7 @@ #include "common/Flags.h" #include "common/http/HttpResponse.h" -DECLARE_FLAG_INT32(default_http_request_timeout_secs); +DECLARE_FLAG_INT32(default_http_request_timeout_sec); DECLARE_FLAG_INT32(default_http_request_max_try_cnt); namespace logtail { @@ -49,7 +49,7 @@ struct HttpRequest { std::string mBody; std::string mHost; int32_t mPort; - uint32_t mTimeout = static_cast(INT32_FLAG(default_http_request_timeout_secs)); + uint32_t mTimeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)); uint32_t mMaxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)); bool mFollowRedirects = false; std::optional mTls = std::nullopt; @@ -65,7 +65,7 @@ struct HttpRequest { const std::string& query, const std::map& header, const std::string& body, - uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_secs)), + uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)), uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)), bool followRedirects = false, std::optional tls = std::nullopt) @@ -98,17 +98,28 @@ struct AsynHttpRequest : public HttpRequest { const std::map& header, const std::string& body, HttpResponse&& response = HttpResponse(), - uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_secs)), + uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)), uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)), bool followRedirects = false, std::optional tls = std::nullopt) - : HttpRequest( - method, httpsFlag, host, port, url, query, header, body, timeout, maxTryCnt, followRedirects, std::move(tls)), + : HttpRequest(method, + httpsFlag, + host, + port, + url, + query, + header, + body, + timeout, + maxTryCnt, + followRedirects, + std::move(tls)), mResponse(std::move(response)) {} virtual bool IsContextValid() const = 0; virtual void OnSendDone(HttpResponse& response) = 0; }; +std::string GetQueryString(const std::map& parameters); } // namespace logtail diff --git a/core/common/http/HttpResponse.h b/core/common/http/HttpResponse.h index c4282df428..88bdaab1ab 100644 --- a/core/common/http/HttpResponse.h +++ b/core/common/http/HttpResponse.h @@ -16,8 +16,7 @@ #pragma once -#include - +#include #include #include #include @@ -82,6 +81,8 @@ class HttpResponse { : mHeader(compareHeader), mBody(body, bodyDeleter), mWriteCallback(callback) {} int32_t GetStatusCode() const { return mStatusCode; } + void SetStatusCode(int32_t code) { mStatusCode = code; } + const std::map& GetHeader() const { return mHeader; } template @@ -94,67 +95,31 @@ class HttpResponse { return static_cast(mBody.get()); } - void SetStatusCode(int32_t code) { mStatusCode = code; } + void SetResponseTime(const std::chrono::milliseconds& time) { mResponseTime = time; } + std::chrono::milliseconds GetResponseTime() const { return mResponseTime; } - void SetNetworkStatus(CURLcode code) { - mNetworkStatus.mMessage = curl_easy_strerror(code); - // please refer to https://curl.se/libcurl/c/libcurl-errors.html - switch (code) { - case CURLE_OK: - mNetworkStatus.mCode = NetworkCode::Ok; - break; - case CURLE_COULDNT_CONNECT: - mNetworkStatus.mCode = NetworkCode::ConnectionFailed; - break; - case CURLE_LOGIN_DENIED: - case CURLE_REMOTE_ACCESS_DENIED: - mNetworkStatus.mCode = NetworkCode::RemoteAccessDenied; - break; - case CURLE_OPERATION_TIMEDOUT: - mNetworkStatus.mCode = NetworkCode::Timeout; - break; - case CURLE_SSL_CONNECT_ERROR: - mNetworkStatus.mCode = NetworkCode::SSLConnectError; - break; - case CURLE_SSL_CERTPROBLEM: - case CURLE_SSL_CACERT: - mNetworkStatus.mCode = NetworkCode::SSLCertError; - break; - case CURLE_SEND_ERROR: - case CURLE_SEND_FAIL_REWIND: - mNetworkStatus.mCode = NetworkCode::SendDataFailed; - break; - case CURLE_RECV_ERROR: - mNetworkStatus.mCode = NetworkCode::RecvDataFailed; - break; - case CURLE_SSL_PINNEDPUBKEYNOTMATCH: - case CURLE_SSL_INVALIDCERTSTATUS: - case CURLE_SSL_CACERT_BADFILE: - case CURLE_SSL_CIPHER: - case CURLE_SSL_ENGINE_NOTFOUND: - case CURLE_SSL_ENGINE_SETFAILED: - case CURLE_USE_SSL_FAILED: - case CURLE_SSL_ENGINE_INITFAILED: - case CURLE_SSL_CRL_BADFILE: - case CURLE_SSL_ISSUER_ERROR: - case CURLE_SSL_SHUTDOWN_FAILED: - mNetworkStatus.mCode = NetworkCode::SSLOtherProblem; - break; - case CURLE_FAILED_INIT: - default: - mNetworkStatus.mCode = NetworkCode::Other; - break; - } + const NetworkStatus& GetNetworkStatus() { return mNetworkStatus; } + void SetNetworkStatus(NetworkCode code, const std::string& msg) { + mNetworkStatus.mCode = code; + mNetworkStatus.mMessage = msg; } - const NetworkStatus& GetNetworkStatus() { return mNetworkStatus; } +#ifdef APSARA_UNIT_TEST_MAIN + template + void SetBody(const T& body) { + *mBody = body; + } + + void AddHeader(const std::string& key, const std::string& value) { mHeader[key] = value; } +#endif private: int32_t mStatusCode = 0; // 0 means no response from server - NetworkStatus mNetworkStatus; // 0 means no error + NetworkStatus mNetworkStatus; std::map mHeader; std::unique_ptr> mBody; size_t (*mWriteCallback)(char*, size_t, size_t, void*) = nullptr; + std::chrono::milliseconds mResponseTime = std::chrono::milliseconds::max(); #ifdef APSARA_UNIT_TEST_MAIN friend class HttpSinkMock; diff --git a/core/config/common_provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp index e86808d7a2..2e278c6ecf 100644 --- a/core/config/common_provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -26,6 +26,8 @@ #include "common/StringTools.h" #include "common/UUIDUtil.h" #include "common/YamlUtil.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/version.h" #include "config/ConfigUtil.h" #include "config/PipelineConfig.h" @@ -33,9 +35,6 @@ #include "constants/Constants.h" #include "logger/Logger.h" #include "monitor/Monitor.h" -#include "sdk/Common.h" -#include "sdk/CurlImp.h" -#include "sdk/Exception.h" using namespace std; @@ -43,7 +42,9 @@ DEFINE_FLAG_INT32(heartbeat_interval, "second", 10); namespace logtail { -std::string CommonConfigProvider::configVersion = "version"; +const string AGENT = "/Agent"; + +string CommonConfigProvider::configVersion = "version"; void CommonConfigProvider::Init(const string& dir) { sName = "common config provider"; @@ -300,7 +301,7 @@ configserver::proto::v2::HeartbeatRequest CommonConfigProvider::PrepareHeartbeat bool CommonConfigProvider::SendHeartbeat(const configserver::proto::v2::HeartbeatRequest& heartbeatReq, configserver::proto::v2::HeartbeatResponse& heartbeatResponse) { - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/").append("Heartbeat"); string reqBody; heartbeatReq.SerializeToString(&reqBody); @@ -323,31 +324,25 @@ bool CommonConfigProvider::SendHttpRequest(const string& operation, // LCOV_EXCL_START ConfigServerAddress configServerAddress = GetOneConfigServerAddress(false); map httpHeader; - httpHeader[sdk::CONTENT_TYPE] = sdk::TYPE_LOG_PROTOBUF; - sdk::HttpMessage httpResponse; - httpResponse.header[sdk::X_LOG_REQUEST_ID] = requestId; - sdk::CurlClient client; - - try { - client.Send(sdk::HTTP_POST, - configServerAddress.host, - configServerAddress.port, - operation, - "", - httpHeader, - reqBody, - INT32_FLAG(sls_client_send_timeout), - httpResponse, - "", - false); - resp.swap(httpResponse.content); - return true; - } catch (const sdk::LOGException& e) { + httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; + + HttpResponse httpResponse; + if (!logtail::SendHttpRequest(make_unique(HTTP_POST, + false, + configServerAddress.host, + configServerAddress.port, + operation, + "", + httpHeader, + reqBody), + httpResponse)) { LOG_WARNING(sLogger, - (configType, "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())("errMsg", e.GetMessage())( - "host", configServerAddress.host)("port", configServerAddress.port)); + (configType, "fail")("reqBody", + reqBody)("host", configServerAddress.host)("port", configServerAddress.port)); return false; } + resp = *httpResponse.GetBody(); + return true; // LCOV_EXCL_STOP } @@ -498,7 +493,7 @@ bool CommonConfigProvider::FetchInstanceConfigFromServer( reqConfig->set_name(config.name()); reqConfig->set_version(config.version()); } - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/FetchInstanceConfig"); string reqBody; fetchConfigRequest.SerializeToString(&reqBody); @@ -525,7 +520,7 @@ bool CommonConfigProvider::FetchPipelineConfigFromServer( reqConfig->set_name(config.name()); reqConfig->set_version(config.version()); } - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/FetchPipelineConfig"); string reqBody; fetchConfigRequest.SerializeToString(&reqBody); diff --git a/core/config/common_provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp index 334b4dc85b..9231cee6db 100644 --- a/core/config/common_provider/LegacyCommonConfigProvider.cpp +++ b/core/config/common_provider/LegacyCommonConfigProvider.cpp @@ -22,14 +22,14 @@ #include "app_config/AppConfig.h" #include "application/Application.h" +#include "common/EncodingUtil.h" #include "common/LogtailCommonFlags.h" #include "common/StringTools.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/version.h" #include "logger/Logger.h" #include "monitor/Monitor.h" -#include "sdk/Common.h" -#include "sdk/CurlImp.h" -#include "sdk/Exception.h" using namespace std; @@ -37,6 +37,8 @@ DEFINE_FLAG_INT32(config_update_interval, "second", 10); namespace logtail { +const string AGENT = "/Agent"; + void LegacyCommonConfigProvider::Init(const string& dir) { ConfigProvider::Init(dir); @@ -157,7 +159,7 @@ google::protobuf::RepeatedPtrField LegacyCommonConfigProvider::SendHeartbeat(const ConfigServerAddress& configServerAddress) { configserver::proto::HeartBeatRequest heartBeatReq; configserver::proto::AgentAttributes attributes; - string requestID = sdk::Base64Enconde(string("heartbeat").append(to_string(time(NULL)))); + string requestID = Base64Enconde(string("heartbeat").append(to_string(time(NULL)))); heartBeatReq.set_request_id(requestID); heartBeatReq.set_agent_id(Application::GetInstance()->GetInstanceId()); heartBeatReq.set_agent_type("iLogtail"); @@ -179,46 +181,41 @@ LegacyCommonConfigProvider::SendHeartbeat(const ConfigServerAddress& configServe } heartBeatReq.mutable_pipeline_configs()->MergeFrom(pipelineConfigs); - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/").append("HeartBeat"); map httpHeader; - httpHeader[sdk::CONTENT_TYPE] = sdk::TYPE_LOG_PROTOBUF; + httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; string reqBody; heartBeatReq.SerializeToString(&reqBody); - sdk::HttpMessage httpResponse; - httpResponse.header[sdk::X_LOG_REQUEST_ID] = "ConfigServer"; - sdk::CurlClient client; google::protobuf::RepeatedPtrField emptyResult; - try { - client.Send(sdk::HTTP_POST, - configServerAddress.host, - configServerAddress.port, - operation, - "", - httpHeader, - reqBody, - INT32_FLAG(sls_client_send_timeout), - httpResponse, - "", - false); - configserver::proto::HeartBeatResponse heartBeatResp; - heartBeatResp.ParseFromString(httpResponse.content); - - if (0 != strcmp(heartBeatResp.request_id().c_str(), requestID.c_str())) - return emptyResult; - - LOG_DEBUG(sLogger, - ("SendHeartbeat", "success")("reqBody", reqBody)("requestId", heartBeatResp.request_id())( - "statusCode", heartBeatResp.code())); - - return heartBeatResp.pipeline_check_results(); - } catch (const sdk::LOGException& e) { + HttpResponse httpResponse; + if (!logtail::SendHttpRequest(make_unique(HTTP_POST, + false, + configServerAddress.host, + configServerAddress.port, + operation, + "", + httpHeader, + reqBody), + httpResponse)) { LOG_WARNING(sLogger, - ("SendHeartbeat", "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())( - "errMsg", e.GetMessage())("host", configServerAddress.host)("port", configServerAddress.port)); + ("SendHeartbeat", + "fail")("reqBody", reqBody)("host", configServerAddress.host)("port", configServerAddress.port)); return emptyResult; } + + configserver::proto::HeartBeatResponse heartBeatResp; + heartBeatResp.ParseFromString(*httpResponse.GetBody()); + + if (0 != strcmp(heartBeatResp.request_id().c_str(), requestID.c_str())) + return emptyResult; + + LOG_DEBUG(sLogger, + ("SendHeartbeat", "success")("reqBody", reqBody)("requestId", heartBeatResp.request_id())( + "statusCode", heartBeatResp.code())); + + return heartBeatResp.pipeline_check_results(); } google::protobuf::RepeatedPtrField LegacyCommonConfigProvider::FetchPipelineConfig( @@ -226,7 +223,7 @@ google::protobuf::RepeatedPtrField LegacyComm const google::protobuf::RepeatedPtrField& requestConfigs) { configserver::proto::FetchPipelineConfigRequest fetchConfigReq; string requestID - = sdk::Base64Enconde(Application::GetInstance()->GetInstanceId().append("_").append(to_string(time(NULL)))); + = Base64Enconde(Application::GetInstance()->GetInstanceId().append("_").append(to_string(time(NULL)))); fetchConfigReq.set_request_id(requestID); fetchConfigReq.set_agent_id(Application::GetInstance()->GetInstanceId()); @@ -242,47 +239,39 @@ google::protobuf::RepeatedPtrField LegacyComm } fetchConfigReq.mutable_req_configs()->MergeFrom(configInfos); - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/").append("FetchPipelineConfig"); map httpHeader; - httpHeader[sdk::CONTENT_TYPE] = sdk::TYPE_LOG_PROTOBUF; + httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; string reqBody; fetchConfigReq.SerializeToString(&reqBody); - sdk::HttpMessage httpResponse; - httpResponse.header[sdk::X_LOG_REQUEST_ID] = "ConfigServer"; - sdk::CurlClient client; google::protobuf::RepeatedPtrField emptyResult; - try { - client.Send(sdk::HTTP_POST, - configServerAddress.host, - configServerAddress.port, - operation, - "", - httpHeader, - reqBody, - INT32_FLAG(sls_client_send_timeout), - httpResponse, - "", - false); - - configserver::proto::FetchPipelineConfigResponse fetchConfigResp; - fetchConfigResp.ParseFromString(httpResponse.content); - - if (0 != strcmp(fetchConfigResp.request_id().c_str(), requestID.c_str())) - return emptyResult; - - LOG_DEBUG(sLogger, - ("GetConfigUpdateInfos", "success")("reqBody", reqBody)("requestId", fetchConfigResp.request_id())( - "statusCode", fetchConfigResp.code())); - - return fetchConfigResp.config_details(); - } catch (const sdk::LOGException& e) { - LOG_WARNING(sLogger, - ("GetConfigUpdateInfos", "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())("errMsg", - e.GetMessage())); + HttpResponse httpResponse; + if (!logtail::SendHttpRequest(make_unique(HTTP_POST, + false, + configServerAddress.host, + configServerAddress.port, + operation, + "", + httpHeader, + reqBody), + httpResponse)) { + LOG_WARNING(sLogger, ("GetConfigUpdateInfos", "fail")("reqBody", reqBody)); return emptyResult; } + + configserver::proto::FetchPipelineConfigResponse fetchConfigResp; + fetchConfigResp.ParseFromString(*httpResponse.GetBody()); + + if (0 != strcmp(fetchConfigResp.request_id().c_str(), requestID.c_str())) + return emptyResult; + + LOG_DEBUG(sLogger, + ("GetConfigUpdateInfos", "success")("reqBody", reqBody)("requestId", fetchConfigResp.request_id())( + "statusCode", fetchConfigResp.code())); + + return fetchConfigResp.config_details(); } void LegacyCommonConfigProvider::UpdateRemoteConfig( diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index fb914c4d55..0cd8c00570 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -440,7 +440,7 @@ bool PipelineConfigWatcher::CheckUnchangedConfig(const std::string& configName, return false; } if (!IsConfigEnabled(configName, *detail)) { - LOG_INFO(sLogger, ("unchanged config found and disabled", "skip current object")("config", configName)); + LOG_DEBUG(sLogger, ("unchanged config found and disabled", "skip current object")("config", configName)); return false; } PipelineConfig config(configName, std::move(detail)); diff --git a/core/file_server/event_handler/LogInput.cpp b/core/file_server/event_handler/LogInput.cpp index 012d83967c..6f0dbacebf 100644 --- a/core/file_server/event_handler/LogInput.cpp +++ b/core/file_server/event_handler/LogInput.cpp @@ -39,9 +39,6 @@ #include "logger/Logger.h" #include "monitor/AlarmManager.h" #include "monitor/Monitor.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "file_server/FileServer.h" using namespace std; @@ -60,8 +57,6 @@ DEFINE_FLAG_BOOL(force_close_file_on_container_stopped, "whether close file handler immediately when associate container stopped", false); -DECLARE_FLAG_BOOL(send_prefer_real_ip); - namespace logtail { LogInput::LogInput() : mAccessMainThreadRWL(ReadWriteLock::PREFER_WRITER) { diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index 08b7693354..d5024ca48c 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -53,7 +53,6 @@ #include "pipeline/queue/QueueKeyManager.h" #include "plugin/processor/inner/ProcessorParseContainerLogNative.h" #include "rapidjson/document.h" -#include "sdk/Common.h" using namespace sls_logs; using namespace std; @@ -90,6 +89,8 @@ namespace logtail { size_t LogFileReader::BUFFER_SIZE = 1024 * 512; // 512KB +const int64_t kFirstHashKeySeqID = 1; + LogFileReader* LogFileReader::CreateLogFileReader(const string& hostLogPathDir, const string& hostLogPathFile, const DevInode& devInode, @@ -439,7 +440,7 @@ void LogFileReader::initExactlyOnce(uint32_t concurrency) { auto partitionRange = detail::getPartitionRange(partIdx, mEOOption->concurrency, kPartitionCount); auto partitionID = partitionRange.first + rand() % (partitionRange.second - partitionRange.first + 1); rangeCpt->data.set_hash_key(GenerateHashKey(baseHashKey, partitionID, kPartitionCount)); - rangeCpt->data.set_sequence_id(sdk::kFirstHashKeySeqID); + rangeCpt->data.set_sequence_id(kFirstHashKeySeqID); rangeCpt->data.set_committed(false); } LOG_DEBUG(sLogger, diff --git a/core/models/EventPool.cpp b/core/models/EventPool.cpp index 5e50091750..0c045438b5 100644 --- a/core/models/EventPool.cpp +++ b/core/models/EventPool.cpp @@ -19,7 +19,7 @@ #include "common/Flags.h" #include "logger/Logger.h" -DEFINE_FLAG_INT32(event_pool_gc_interval_secs, "", 60); +DEFINE_FLAG_INT32(event_pool_gc_interval_sec, "", 60); using namespace std; @@ -143,7 +143,7 @@ void DoGC(vector& pool, vector& poolBak, size_t& minUnusedCnt, mutex* mu } void EventPool::CheckGC() { - if (time(nullptr) - mLastGCTime > INT32_FLAG(event_pool_gc_interval_secs)) { + if (time(nullptr) - mLastGCTime > INT32_FLAG(event_pool_gc_interval_sec)) { if (mEnableLock) { lock_guard lock(mPoolMux); DoGC(mLogEventPool, mLogEventPoolBak, mMinUnusedLogEventsCnt, &mPoolBakMux, "log"); diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 34fabe9059..b00bbb9594 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -41,7 +41,6 @@ #include "plugin/flusher/sls/FlusherSLS.h" #include "protobuf/sls/sls_logs.pb.h" #include "runner/FlusherRunner.h" -#include "sdk/Common.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif @@ -52,7 +51,6 @@ using namespace std; using namespace sls_logs; DEFINE_FLAG_BOOL(logtail_dump_monitor_info, "enable to dump Logtail monitor info (CPU, mem)", false); -DECLARE_FLAG_BOOL(send_prefer_real_ip); DECLARE_FLAG_BOOL(check_profile_region); namespace logtail { diff --git a/core/monitor/profile_sender/ProfileSender.cpp b/core/monitor/profile_sender/ProfileSender.cpp index 2c6b942c67..ec2d50ea30 100644 --- a/core/monitor/profile_sender/ProfileSender.cpp +++ b/core/monitor/profile_sender/ProfileSender.cpp @@ -27,7 +27,6 @@ #endif #include "app_config/AppConfig.h" #include "plugin/flusher/sls/SLSClientManager.h" -#include "sdk/Exception.h" // TODO: temporarily used #include "common/compression/CompressorFactory.h" @@ -153,7 +152,7 @@ void ProfileSender::SendRunningStatus(sls_logs::LogGroup& logGroup) { string region = "cn-shanghai"; string project = "ilogtail-community-edition"; string logstore = "ilogtail-online"; - string endpoint = region + ".log.aliyuncs.com"; + string host = project + "." + region + ".log.aliyuncs.com"; Json::Value logtailStatus; logtailStatus["__topic__"] = "logtail_status_profile"; @@ -171,26 +170,14 @@ void ProfileSender::SendRunningStatus(sls_logs::LogGroup& logGroup) { } logtailStatus["__logs__"][0] = status; string logBody = logtailStatus.toStyledString(); - sdk::Client client("", endpoint, INT32_FLAG(sls_client_send_timeout)); - client.SetPort(AppConfig::GetInstance()->GetDataServerPort()); - try { - string res; - if (!CompressLz4(logBody, res)) { - LOG_ERROR(sLogger, ("lz4 compress data", "fail")); - return; - } - - sdk::PostLogStoreLogsResponse resp - = client.PostLogUsingWebTracking(project, logstore, sls_logs::SLS_CMP_LZ4, res, logBody.size()); - LOG_DEBUG(sLogger, - ("SendToProfileProject", - "success")("logBody", logBody)("requestId", resp.requestId)("statusCode", resp.statusCode)); - } catch (const sdk::LOGException& e) { - LOG_DEBUG(sLogger, - ("SendToProfileProject", "fail")("logBody", logBody)("errCode", e.GetErrorCode())("errMsg", - e.GetMessage())); + string res; + if (!CompressLz4(logBody, res)) { + LOG_ERROR(sLogger, ("lz4 compress data", "fail")); + return; } + + PutWebTracking(host, true, logstore, "lz4", res, logBody.size()); } } // namespace logtail diff --git a/core/pipeline/GlobalConfig.cpp b/core/pipeline/GlobalConfig.cpp index 15e98c10bd..91a3ed43bd 100644 --- a/core/pipeline/GlobalConfig.cpp +++ b/core/pipeline/GlobalConfig.cpp @@ -94,7 +94,7 @@ bool GlobalConfig::Init(const Json::Value& config, const PipelineContext& ctx, J } // Priority - uint32_t priority = 0; + uint32_t priority = 1; if (!GetOptionalUIntParam(config, "Priority", priority, errorMsg)) { PARAM_WARNING_DEFAULT(ctx.GetLogger(), ctx.GetAlarm(), diff --git a/core/pipeline/plugin/interface/HttpFlusher.h b/core/pipeline/plugin/interface/HttpFlusher.h index a8bba21296..f68eab2bd2 100644 --- a/core/pipeline/plugin/interface/HttpFlusher.h +++ b/core/pipeline/plugin/interface/HttpFlusher.h @@ -16,6 +16,8 @@ #pragma once +#include + #include "common/http/HttpResponse.h" #include "pipeline/plugin/interface/Flusher.h" #include "pipeline/queue/SenderQueueItem.h" @@ -27,7 +29,7 @@ class HttpFlusher : public Flusher { public: virtual ~HttpFlusher() = default; - virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const = 0; + virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) = 0; virtual void OnSendDone(const HttpResponse& response, SenderQueueItem* item) = 0; virtual SinkType GetSinkType() override { return SinkType::HTTP; } diff --git a/core/pipeline/queue/SLSSenderQueueItem.h b/core/pipeline/queue/SLSSenderQueueItem.h index 1d34efe40f..2ee185dbb0 100644 --- a/core/pipeline/queue/SLSSenderQueueItem.h +++ b/core/pipeline/queue/SLSSenderQueueItem.h @@ -32,7 +32,7 @@ struct SLSSenderQueueItem : public SenderQueueItem { std::string mLogstore; RangeCheckpointPtr mExactlyOnceCheckpoint; - std::string mCurrentEndpoint; + std::string mCurrentHost; bool mRealIpFlag = false; int32_t mLastLogWarningTime = 0; // temporaily used diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 2d8e6e22f4..f613da5f30 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -28,10 +28,14 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "plugin/flusher/sls/FlusherSLS.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif #include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/SLSConstant.h" +#include "plugin/flusher/sls/SendResult.h" #include "protobuf/sls/sls_logs.pb.h" #include "provider/Provider.h" -#include "sdk/Exception.h" DEFINE_FLAG_INT32(write_secondary_wait_timeout, "interval of dump seconary buffer from memory to file, seconds", 2); DEFINE_FLAG_INT32(buffer_file_alive_interval, "the max alive time of a bufferfile, 5 minutes", 300); @@ -41,13 +45,60 @@ DEFINE_FLAG_INT32(secondary_buffer_count_limit, "data ready for write buffer fil DEFINE_FLAG_INT32(send_retry_sleep_interval, "sleep microseconds when sync send fail, 50ms", 50000); DEFINE_FLAG_INT32(buffer_check_period, "check logtail local storage buffer period", 60); DEFINE_FLAG_INT32(unauthorized_wait_interval, "", 1); +DEFINE_FLAG_INT32(send_retrytimes, "how many times should retry if PostLogStoreLogs operation fail", 3); DECLARE_FLAG_INT32(discard_send_fail_interval); using namespace std; - namespace logtail { +#ifdef __ENTERPRISE__ +static EndpointMode GetEndpointMode(sls_logs::EndpointMode mode) { + switch (mode) { + case sls_logs::EndpointMode::DEFAULT: + return EndpointMode::DEFAULT; + case sls_logs::EndpointMode::ACCELERATE: + return EndpointMode::ACCELERATE; + case sls_logs::EndpointMode::CUSTOM: + return EndpointMode::CUSTOM; + } + return EndpointMode::DEFAULT; +} + +static sls_logs::EndpointMode GetEndpointMode(EndpointMode mode) { + switch (mode) { + case EndpointMode::DEFAULT: + return sls_logs::EndpointMode::DEFAULT; + case EndpointMode::ACCELERATE: + return sls_logs::EndpointMode::ACCELERATE; + case EndpointMode::CUSTOM: + return sls_logs::EndpointMode::CUSTOM; + } + return sls_logs::EndpointMode::DEFAULT; +} + +static const string kAKErrorMsg = "can not get valid access key"; +#endif + +static const string kNoHostErrorMsg = "can not get available host"; + +static const string& GetSLSCompressTypeString(sls_logs::SlsCompressType compressType) { + switch (compressType) { + case sls_logs::SLS_CMP_NONE: { + static string none = ""; + return none; + } + case sls_logs::SLS_CMP_ZSTD: { + static string zstd = "zstd"; + return zstd; + } + default: { + static string lz4 = "lz4"; + return lz4; + } + } +} + const int32_t DiskBufferWriter::BUFFER_META_BASE_SIZE = 65536; void DiskBufferWriter::Init() { @@ -146,13 +197,6 @@ void DiskBufferWriter::BufferSenderThread() { LOG_INFO(sLogger, ("disk buffer sender", "started")); unique_lock lock(mBufferSenderThreadRunningMux); while (mIsSendBufferThreadRunning) { - if (!SLSClientManager::GetInstance()->HasNetworkAvailable()) { - if (mStopCV.wait_for( - lock, chrono::seconds(mCheckPeriod), [this]() { return !mIsSendBufferThreadRunning; })) { - break; - } - continue; - } vector filesToSend; if (!LoadFileToSend(mBufferDivideTime, filesToSend)) { if (mStopCV.wait_for( @@ -200,6 +244,9 @@ void DiskBufferWriter::BufferSenderThread() { "check header of buffer file failed, delete file: " + fileName); } } +#ifdef __ENTERPRISE__ + mCandidateHostsInfos.clear(); +#endif // mIsSendingBuffer = false; lock.lock(); if (mStopCV.wait_for(lock, chrono::seconds(mCheckPeriod), [this]() { return !mIsSendBufferThreadRunning; })) { @@ -386,7 +433,7 @@ bool DiskBufferWriter::ReadNextEncryption(int32_t& pos, } } else { bufferMeta.set_project(encodedInfo); - bufferMeta.set_endpoint(FlusherSLS::GetDefaultRegion()); // new mode + bufferMeta.set_region(FlusherSLS::GetDefaultRegion()); // new mode bufferMeta.set_aliuid(""); } if (!bufferMeta.has_compresstype()) { @@ -395,6 +442,14 @@ bool DiskBufferWriter::ReadNextEncryption(int32_t& pos, if (!bufferMeta.has_telemetrytype()) { bufferMeta.set_telemetrytype(sls_logs::SLS_TELEMETRY_TYPE_LOGS); } +#ifdef __ENTERPRISE__ + if (!bufferMeta.has_endpointmode()) { + bufferMeta.set_endpointmode(sls_logs::EndpointMode::DEFAULT); + } +#endif + if (!bufferMeta.has_endpoint()) { + bufferMeta.set_endpoint(""); + } buffer = new char[meta.mEncryptionSize + 1]; nbytes = fread(buffer, sizeof(char), meta.mEncryptionSize, fin); @@ -480,20 +535,72 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t } } if (!sendResult) { - string errorCode; - SendResult res = SendBufferFileData(bufferMeta, logData, errorCode); - if (res == SEND_OK) - sendResult = true; - else if (res == SEND_DISCARD_ERROR || res == SEND_PARAMETER_INVALID) { - AlarmManager::GetInstance()->SendAlarm(SEND_DATA_FAIL_ALARM, - string("send buffer file fail, rawsize:") - + ToString(bufferMeta.rawsize()) - + "errorCode: " + errorCode, - bufferMeta.project(), - bufferMeta.logstore(), - ""); - sendResult = true; - discardCount++; + time_t beginTime = time(nullptr); + while (true) { + string host; + auto response = SendBufferFileData(bufferMeta, logData, host); + SendResult sendRes = SEND_OK; + if (response.mStatusCode != 200) { + sendRes = ConvertErrorCode(response.mErrorCode); + } + switch (sendRes) { + case SEND_OK: + sendResult = true; + break; + case SEND_NETWORK_ERROR: + case SEND_SERVER_ERROR: + if (response.mErrorMsg != kNoHostErrorMsg) { + LOG_WARNING( + sLogger, + ("send data to SLS fail", "retry later")("request id", response.mRequestId)( + "error_code", response.mErrorCode)("error_message", response.mErrorMsg)( + "endpoint", host)("projectName", bufferMeta.project())( + "logstore", bufferMeta.logstore())("rawsize", bufferMeta.rawsize())); + } + usleep(INT32_FLAG(send_retry_sleep_interval)); + break; + case SEND_QUOTA_EXCEED: + AlarmManager::GetInstance()->SendAlarm(SEND_QUOTA_EXCEED_ALARM, + "error_code: " + response.mErrorCode + + ", error_message: " + response.mErrorMsg, + bufferMeta.project(), + bufferMeta.logstore(), + ""); + // no region + if (!GetProfileSender()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) + LOG_WARNING( + sLogger, + ("send data to SLS fail", "retry later")("request id", response.mRequestId)( + "error_code", response.mErrorCode)("error_message", response.mErrorMsg)( + "endpoint", host)("projectName", bufferMeta.project())( + "logstore", bufferMeta.logstore())("rawsize", bufferMeta.rawsize())); + usleep(INT32_FLAG(quota_exceed_wait_interval)); + break; + case SEND_UNAUTHORIZED: + usleep(INT32_FLAG(unauthorized_wait_interval)); + break; + default: + sendResult = true; + discardCount++; + break; + } +#ifdef __ENTERPRISE__ + if (sendRes != SEND_NETWORK_ERROR && sendRes != SEND_SERVER_ERROR) { + bool hasAuthError + = sendRes == SEND_UNAUTHORIZED && response.mErrorMsg != kAKErrorMsg; + EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), + !hasAuthError); + EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus( + bufferMeta.project(), !hasAuthError); + } +#endif + if (time(nullptr) - beginTime >= INT32_FLAG(discard_send_fail_interval)) { + sendResult = true; + discardCount++; + } + if (sendResult) { + break; + } } } } @@ -656,7 +763,7 @@ bool DiskBufferWriter::SendToBufferFile(SenderQueueItem* dataPtr) { sls_logs::LogtailBufferMeta bufferMeta; bufferMeta.set_project(flusher->mProject); - bufferMeta.set_endpoint(flusher->mRegion); + bufferMeta.set_region(flusher->mRegion); bufferMeta.set_aliuid(flusher->mAliuid); bufferMeta.set_logstore(data->mLogstore); bufferMeta.set_datatype(int32_t(data->mType)); @@ -664,6 +771,10 @@ bool DiskBufferWriter::SendToBufferFile(SenderQueueItem* dataPtr) { bufferMeta.set_shardhashkey(data->mShardHashKey); bufferMeta.set_compresstype(ConvertCompressType(flusher->GetCompressType())); bufferMeta.set_telemetrytype(flusher->mTelemetryType); +#ifdef __ENTERPRISE__ + bufferMeta.set_endpointmode(GetEndpointMode(flusher->mEndpointMode)); +#endif + bufferMeta.set_endpoint(flusher->mEndpoint); string encodedInfo; bufferMeta.SerializeToString(&encodedInfo); @@ -702,138 +813,85 @@ bool DiskBufferWriter::SendToBufferFile(SenderQueueItem* dataPtr) { return true; } -SendResult DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMeta& bufferMeta, - const std::string& logData, - std::string& errorCode) { +SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMeta& bufferMeta, + const std::string& logData, + std::string& host) { RateLimiter::FlowControl(bufferMeta.rawsize(), mSendLastTime, mSendLastByte, false); - string region = bufferMeta.endpoint(); - if (region.find("http://") == 0) // old buffer file which record the endpoint - region = SLSClientManager::GetInstance()->GetRegionFromEndpoint(region); - - sdk::Client* sendClient = SLSClientManager::GetInstance()->GetClient(region, bufferMeta.aliuid()); - SendResult sendRes; - const string& endpoint = sendClient->GetRawSlsHost(); - if (endpoint.empty()) - sendRes = SEND_NETWORK_ERROR; - else { - sendRes = SendToNetSync(sendClient, region, endpoint, bufferMeta, logData, errorCode); - } - return sendRes; -} + string region = bufferMeta.region(); +#ifdef __ENTERPRISE__ + // old buffer file which record the endpoint + if (region.find("http://") == 0) { + region = EnterpriseSLSClientManager::GetInstance()->GetRegionFromEndpoint(region); + } +#endif -SendResult DiskBufferWriter::SendToNetSync(sdk::Client* sendClient, - const std::string& region, - const std::string& endpoint, - const sls_logs::LogtailBufferMeta& bufferMeta, - const std::string& logData, - std::string& errorCode) { - int32_t retryTimes = 0; - time_t beginTime = time(NULL); - while (true) { - ++retryTimes; - try { - if (bufferMeta.datatype() == int(RawDataType::EVENT_GROUP)) { - if (bufferMeta.has_telemetrytype() - && bufferMeta.telemetrytype() == sls_logs::SLS_TELEMETRY_TYPE_METRICS) { - sendClient->PostMetricStoreLogs(bufferMeta.project(), - bufferMeta.logstore(), - bufferMeta.compresstype(), - logData, - bufferMeta.rawsize()); - } else if (bufferMeta.has_shardhashkey() && !bufferMeta.shardhashkey().empty()) - sendClient->PostLogStoreLogs(bufferMeta.project(), - bufferMeta.logstore(), - bufferMeta.compresstype(), - logData, - bufferMeta.rawsize(), - bufferMeta.shardhashkey()); - else - sendClient->PostLogStoreLogs(bufferMeta.project(), - bufferMeta.logstore(), - bufferMeta.compresstype(), - logData, - bufferMeta.rawsize()); - } else { - if (bufferMeta.has_shardhashkey() && !bufferMeta.shardhashkey().empty()) - sendClient->PostLogStoreLogPackageList(bufferMeta.project(), - bufferMeta.logstore(), - bufferMeta.compresstype(), - logData, - bufferMeta.shardhashkey()); - else - sendClient->PostLogStoreLogPackageList( - bufferMeta.project(), bufferMeta.logstore(), bufferMeta.compresstype(), logData); - } - return SEND_OK; - } catch (sdk::LOGException& ex) { - errorCode = ex.GetErrorCode(); - SendResult sendRes = ConvertErrorCode(errorCode); - bool hasAuthError = false; - switch (sendRes) { - case SEND_NETWORK_ERROR: - case SEND_SERVER_ERROR: - SLSClientManager::GetInstance()->UpdateEndpointStatus(region, endpoint, false); - SLSClientManager::GetInstance()->ResetClientEndpoint(bufferMeta.aliuid(), region, time(NULL)); - LOG_WARNING(sLogger, - ("send data to SLS fail", "retry later")("error_code", errorCode)( - "error_message", ex.GetMessage())("endpoint", sendClient->GetRawSlsHost())( - "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( - "RetryTimes", retryTimes)("rawsize", bufferMeta.rawsize())); - usleep(INT32_FLAG(send_retry_sleep_interval)); - break; - case SEND_QUOTA_EXCEED: - AlarmManager::GetInstance()->SendAlarm(SEND_QUOTA_EXCEED_ALARM, - "error_code: " + errorCode - + ", error_message: " + ex.GetMessage(), - bufferMeta.project(), - bufferMeta.logstore(), - ""); - // no region - if (!GetProfileSender()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) - LOG_WARNING(sLogger, - ("send data to SLS fail, error_code", errorCode)("error_message", ex.GetMessage())( - "endpoint", sendClient->GetRawSlsHost())("projectName", bufferMeta.project())( - "logstore", bufferMeta.logstore())("RetryTimes", retryTimes)( - "rawsize", bufferMeta.rawsize())); - usleep(INT32_FLAG(quota_exceed_wait_interval)); - break; - case SEND_UNAUTHORIZED: - hasAuthError = true; - usleep(INT32_FLAG(unauthorized_wait_interval)); - break; - default: - break; - } - SLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), !hasAuthError); - if (time(nullptr) - beginTime >= INT32_FLAG(discard_send_fail_interval)) { - sendRes = SEND_DISCARD_ERROR; - } - if (sendRes != SEND_NETWORK_ERROR && sendRes != SEND_SERVER_ERROR && sendRes != SEND_QUOTA_EXCEED - && sendRes != SEND_UNAUTHORIZED) { - return sendRes; - } - { - lock_guard lock(mBufferSenderThreadRunningMux); - if (!mIsSendBufferThreadRunning) { - return sendRes; - } - } - } catch (...) { - if (retryTimes >= INT32_FLAG(send_retrytimes)) { - LOG_ERROR(sLogger, - ("send data fail", "unknown excepiton")("endpoint", sendClient->GetRawSlsHost())( - "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( - "rawsize", bufferMeta.rawsize())); - return SEND_DISCARD_ERROR; - } else { - LOG_DEBUG(sLogger, - ("send data fail", "unknown excepiton, retry later")("endpoint", sendClient->GetRawSlsHost())( - "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( - "rawsize", bufferMeta.rawsize())); - usleep(INT32_FLAG(send_retry_sleep_interval)); - } + SLSClientManager::AuthType type; + string accessKeyId, accessKeySecret; + if (!SLSClientManager::GetInstance()->GetAccessKey(bufferMeta.aliuid(), type, accessKeyId, accessKeySecret)) { +#ifdef __ENTERPRISE__ + if (!EnterpriseSLSClientManager::GetInstance()->GetAccessKeyIfProjectSupportsAnonymousWrite( + bufferMeta.project(), type, accessKeyId, accessKeySecret)) { + SLSResponse response; + response.mErrorCode = LOGE_UNAUTHORIZED; + response.mErrorMsg = kAKErrorMsg; + return response; } +#endif + } + +#ifdef __ENTERPRISE__ + if (bufferMeta.endpointmode() == sls_logs::EndpointMode::DEFAULT) { + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + region, {bufferMeta.endpoint()}, EnterpriseSLSClientManager::RemoteEndpointUpdateAction::CREATE); + } + auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo( + region, bufferMeta.project(), GetEndpointMode(bufferMeta.endpointmode())); + mCandidateHostsInfos.insert(info); + + host = info->GetCurrentHost(); + if (host.empty()) { + SLSResponse response; + response.mErrorCode = LOGE_REQUEST_ERROR; + response.mErrorMsg = kNoHostErrorMsg; + return response; + } +#else + host = bufferMeta.project() + "." + bufferMeta.endpoint(); +#endif + + bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(region); + + RawDataType dataType; + if (bufferMeta.datatype() == 0) { + dataType = RawDataType::EVENT_GROUP_LIST; + } else { + dataType = RawDataType::EVENT_GROUP; + } + if (bufferMeta.has_telemetrytype() && bufferMeta.telemetrytype() == sls_logs::SLS_TELEMETRY_TYPE_METRICS) { + return PostMetricStoreLogs(accessKeyId, + accessKeySecret, + type, + host, + httpsFlag, + bufferMeta.project(), + bufferMeta.logstore(), + GetSLSCompressTypeString(bufferMeta.compresstype()), + logData, + bufferMeta.rawsize()); + } else { + return PostLogStoreLogs(accessKeyId, + accessKeySecret, + type, + host, + httpsFlag, + bufferMeta.project(), + bufferMeta.logstore(), + GetSLSCompressTypeString(bufferMeta.compresstype()), + dataType, + logData, + bufferMeta.rawsize(), + bufferMeta.has_shardhashkey() ? bufferMeta.shardhashkey() : ""); } } -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/plugin/flusher/sls/DiskBufferWriter.h b/core/plugin/flusher/sls/DiskBufferWriter.h index 0ed9367c5e..6075ffa763 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.h +++ b/core/plugin/flusher/sls/DiskBufferWriter.h @@ -16,18 +16,27 @@ #pragma once +#include #include #include +#include #include +#include #include #include +#ifdef __ENTERPRIRSE__ +#include +#endif #include #include "common/SafeQueue.h" #include "pipeline/queue/SenderQueueItem.h" -#include "plugin/flusher/sls/SendResult.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif +#include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/SLSResponse.h" #include "protobuf/sls/logtail_buffer_meta.pb.h" -#include "sdk/Client.h" namespace logtail { @@ -64,15 +73,8 @@ class DiskBufferWriter { void BufferWriterThread(); void BufferSenderThread(); - SendResult SendToNetSync(sdk::Client* sendClient, - const std::string& region, - const std::string& endpoint, - const sls_logs::LogtailBufferMeta& bufferMeta, - const std::string& logData, - std::string& errorCode); - SendResult SendBufferFileData(const sls_logs::LogtailBufferMeta& bufferMeta, - const std::string& logData, - std::string& errorCode); + SLSResponse + SendBufferFileData(const sls_logs::LogtailBufferMeta& bufferMeta, const std::string& logData, std::string& host); bool SendToBufferFile(SenderQueueItem* dataPtr); bool LoadFileToSend(time_t timeLine, std::vector& filesToSend); bool CreateNewFile(); @@ -100,12 +102,28 @@ class DiskBufferWriter { bool mIsSendBufferThreadRunning = true; mutable std::condition_variable mStopCV; +#ifdef __ENTERPRISE__ + struct PointerHash { + std::size_t operator()(const std::shared_ptr& ptr) const { + return std::hash()(ptr.get()); + } + }; + + struct PointerEqual { + bool operator()(const std::shared_ptr& lhs, + const std::shared_ptr& rhs) const { + return lhs.get() == rhs.get(); + } + }; + + std::unordered_set, PointerHash, PointerEqual> mCandidateHostsInfos; +#endif + mutable std::mutex mBufferFileLock; std::string mBufferFilePath; std::string mBufferFileName; volatile time_t mBufferDivideTime = 0; - // volatile bool mIsSendingBuffer = false; int64_t mCheckPeriod = 0; int64_t mSendLastTime = 0; diff --git a/core/sdk/Exception.h b/core/plugin/flusher/sls/Exception.h similarity index 100% rename from core/sdk/Exception.h rename to core/plugin/flusher/sls/Exception.h diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 3ecb22a0be..affabd4aae 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -21,6 +21,7 @@ #include "common/ParamExtractor.h" #include "common/TimeUtil.h" #include "common/compression/CompressorFactory.h" +#include "common/http/Constant.h" #include "sls_logs.pb.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" @@ -30,13 +31,17 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/SLSConstant.h" #include "plugin/flusher/sls/SLSResponse.h" +#include "plugin/flusher/sls/SLSUtil.h" #include "plugin/flusher/sls/SendResult.h" #include "provider/Provider.h" #include "runner/FlusherRunner.h" -#include "sdk/Common.h" // TODO: temporarily used here #include "pipeline/PipelineManager.h" #include "plugin/flusher/sls/DiskBufferWriter.h" @@ -58,7 +63,6 @@ DEFINE_FLAG_INT32(unauthorized_allowed_delay_after_reset, "allowed delay to retr DEFINE_FLAG_INT32(discard_send_fail_interval, "discard data when send fail after 6 * 3600 seconds", 6 * 3600); DEFINE_FLAG_INT32(profile_data_send_retrytimes, "how many times should retry if profile data send fail", 5); DEFINE_FLAG_INT32(unknow_error_try_max, "discard data when try times > this value", 5); -DEFINE_FLAG_BOOL(global_network_success, "global network success flag, default false", false); DEFINE_FLAG_BOOL(enable_metricstore_channel, "only works for metrics data for enhance metrics query performance", true); DEFINE_FLAG_INT32(max_send_log_group_size, "bytes", 10 * 1024 * 1024); DEFINE_FLAG_DOUBLE(sls_serialize_size_expansion_ratio, "", 1.2); @@ -71,6 +75,8 @@ enum class OperationOnFail { RETRY_IMMEDIATELY, RETRY_LATER, DISCARD }; static const int ON_FAIL_LOG_WARNING_INTERVAL_SECOND = 10; +static constexpr int64_t kInvalidHashKeySeqID = 0; + static const char* GetOperationString(OperationOnFail op) { switch (op) { case OperationOnFail::RETRY_IMMEDIATELY: @@ -126,12 +132,14 @@ shared_ptr FlusherSLS::GetLogstoreConcurrencyLimiter(const s auto iter = sLogstoreConcurrencyLimiterMap.find(key); if (iter == sLogstoreConcurrencyLimiterMap.end()) { - auto limiter = make_shared(sName + "#quota#logstore#" + key, AppConfig::GetInstance()->GetSendRequestConcurrency()); + auto limiter = make_shared(sName + "#quota#logstore#" + key, + AppConfig::GetInstance()->GetSendRequestConcurrency()); sLogstoreConcurrencyLimiterMap.try_emplace(key, limiter); return limiter; } if (iter->second.expired()) { - auto limiter = make_shared(sName + "#quota#logstore#" + key, AppConfig::GetInstance()->GetSendRequestConcurrency()); + auto limiter = make_shared(sName + "#quota#logstore#" + key, + AppConfig::GetInstance()->GetSendRequestConcurrency()); iter->second = limiter; return limiter; } @@ -142,12 +150,14 @@ shared_ptr FlusherSLS::GetProjectConcurrencyLimiter(const st lock_guard lock(sMux); auto iter = sProjectConcurrencyLimiterMap.find(project); if (iter == sProjectConcurrencyLimiterMap.end()) { - auto limiter = make_shared(sName + "#quota#project#" + project, AppConfig::GetInstance()->GetSendRequestConcurrency()); + auto limiter = make_shared(sName + "#quota#project#" + project, + AppConfig::GetInstance()->GetSendRequestConcurrency()); sProjectConcurrencyLimiterMap.try_emplace(project, limiter); return limiter; } if (iter->second.expired()) { - auto limiter = make_shared(sName + "#quota#project#" + project, AppConfig::GetInstance()->GetSendRequestConcurrency()); + auto limiter = make_shared(sName + "#quota#project#" + project, + AppConfig::GetInstance()->GetSendRequestConcurrency()); iter->second = limiter; return limiter; } @@ -158,12 +168,20 @@ shared_ptr FlusherSLS::GetRegionConcurrencyLimiter(const str lock_guard lock(sMux); auto iter = sRegionConcurrencyLimiterMap.find(region); if (iter == sRegionConcurrencyLimiterMap.end()) { - auto limiter = make_shared(sName + "#network#region#" + region, AppConfig::GetInstance()->GetSendRequestConcurrency(), AppConfig::GetInstance()->GetSendRequestConcurrency()*AppConfig::GetInstance()->GetGlobalConcurrencyFreePercentageForOneRegion()); + auto limiter = make_shared( + sName + "#network#region#" + region, + AppConfig::GetInstance()->GetSendRequestConcurrency(), + AppConfig::GetInstance()->GetSendRequestConcurrency() + * AppConfig::GetInstance()->GetGlobalConcurrencyFreePercentageForOneRegion()); sRegionConcurrencyLimiterMap.try_emplace(region, limiter); return limiter; } if (iter->second.expired()) { - auto limiter = make_shared(sName + "#network#region#" + region, AppConfig::GetInstance()->GetSendRequestConcurrency(), AppConfig::GetInstance()->GetSendRequestConcurrency()*AppConfig::GetInstance()->GetGlobalConcurrencyFreePercentageForOneRegion()); + auto limiter = make_shared( + sName + "#network#region#" + region, + AppConfig::GetInstance()->GetSendRequestConcurrency(), + AppConfig::GetInstance()->GetSendRequestConcurrency() + * AppConfig::GetInstance()->GetGlobalConcurrencyFreePercentageForOneRegion()); iter->second = limiter; return limiter; } @@ -213,7 +231,6 @@ void FlusherSLS::SetDefaultRegion(const string& region) { mutex FlusherSLS::sProjectRegionMapLock; unordered_map FlusherSLS::sProjectRefCntMap; -unordered_map FlusherSLS::sRegionRefCntMap; unordered_map FlusherSLS::sProjectRegionMap; string FlusherSLS::GetAllProjects() { @@ -225,11 +242,6 @@ string FlusherSLS::GetAllProjects() { return result; } -bool FlusherSLS::IsRegionContainingConfig(const string& region) { - lock_guard lock(sProjectRegionMapLock); - return sRegionRefCntMap.find(region) != sRegionRefCntMap.end(); -} - std::string FlusherSLS::GetProjectRegion(const std::string& project) { lock_guard lock(sProjectRegionMapLock); auto iter = sProjectRegionMap.find(project); @@ -242,7 +254,6 @@ std::string FlusherSLS::GetProjectRegion(const std::string& project) { void FlusherSLS::IncreaseProjectRegionReferenceCnt(const string& project, const string& region) { lock_guard lock(sProjectRegionMapLock); ++sProjectRefCntMap[project]; - ++sRegionRefCntMap[region]; sProjectRegionMap[project] = region; } @@ -255,32 +266,6 @@ void FlusherSLS::DecreaseProjectRegionReferenceCnt(const string& project, const sProjectRegionMap.erase(project); } } - - auto regionRefCnt = sRegionRefCntMap.find(region); - if (regionRefCnt != sRegionRefCntMap.end()) { - if (--regionRefCnt->second == 0) { - sRegionRefCntMap.erase(regionRefCnt); - } - } -} - -mutex FlusherSLS::sRegionStatusLock; -unordered_map FlusherSLS::sAllRegionStatus; - -void FlusherSLS::UpdateRegionStatus(const string& region, bool status) { - LOG_DEBUG(sLogger, ("update region status, region", region)("is network in good condition", ToString(status))); - lock_guard lock(sRegionStatusLock); - sAllRegionStatus[region] = status; -} - -bool FlusherSLS::GetRegionStatus(const string& region) { - lock_guard lock(sRegionStatusLock); - auto rst = sAllRegionStatus.find(region); - if (rst == sAllRegionStatus.end()) { - return true; - } else { - return rst->second; - } } bool FlusherSLS::sIsResourceInited = false; @@ -291,10 +276,10 @@ const unordered_set FlusherSLS::sNativeParam = {"Project", "Logstore", "Region", "Endpoint", + "EndpointMode", "Aliuid", "CompressType", "TelemetryType", - "FlowControlExpireTime", "MaxSendRate", "ShardHashKeys", "Batch"}; @@ -329,26 +314,66 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mContext->GetRegion()); } + // Region + if ( #ifdef __ENTERPRISE__ - if (EnterpriseConfigProvider::GetInstance()->IsDataServerPrivateCloud()) { - mRegion = STRING_FLAG(default_region_name); - } else { + !EnterpriseConfigProvider::GetInstance()->IsDataServerPrivateCloud() && #endif - // Region - if (!GetOptionalStringParam(config, "Region", mRegion, errorMsg)) { - PARAM_WARNING_DEFAULT(mContext->GetLogger(), - mContext->GetAlarm(), - errorMsg, - mRegion, - sName, - mContext->GetConfigName(), - mContext->GetProjectName(), - mContext->GetLogstoreName(), - mContext->GetRegion()); - } + !GetOptionalStringParam(config, "Region", mRegion, errorMsg)) { + PARAM_WARNING_DEFAULT(mContext->GetLogger(), + mContext->GetAlarm(), + errorMsg, + mRegion, + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } - // Endpoint #ifdef __ENTERPRISE__ + // Aliuid + if (!GetOptionalStringParam(config, "Aliuid", mAliuid, errorMsg)) { + PARAM_WARNING_IGNORE(mContext->GetLogger(), + mContext->GetAlarm(), + errorMsg, + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } + + // EndpointMode + string endpointMode = "default"; + if (!GetOptionalStringParam(config, "EndpointMode", endpointMode, errorMsg)) { + PARAM_WARNING_DEFAULT(mContext->GetLogger(), + mContext->GetAlarm(), + errorMsg, + endpointMode, + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } + if (endpointMode == "accelerate") { + mEndpointMode = EndpointMode::ACCELERATE; + } else if (endpointMode != "default") { + PARAM_WARNING_DEFAULT(mContext->GetLogger(), + mContext->GetAlarm(), + "string param EndpointMode is not valid", + "default", + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } + if (mEndpointMode == EndpointMode::DEFAULT) { + // for local pipeline whose flusher region is neither specified in local info nor included by config provider, + // param Endpoint should be used, and the mode is set to default. + // warning: if inconsistency exists among configs, only the first config would be considered in this situation. if (!GetOptionalStringParam(config, "Endpoint", mEndpoint, errorMsg)) { PARAM_WARNING_IGNORE(mContext->GetLogger(), mContext->GetAlarm(), @@ -358,8 +383,17 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mContext->GetProjectName(), mContext->GetLogstoreName(), mContext->GetRegion()); - } else { + } + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + mRegion, {mEndpoint}, EnterpriseSLSClientManager::RemoteEndpointUpdateAction::CREATE); + } + mCandidateHostsInfo + = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); + LOG_INFO(mContext->GetLogger(), + ("get candidate hosts info, region", mRegion)("project", mProject)("logstore", mLogstore)( + "endpoint mode", EndpointModeToString(mCandidateHostsInfo->GetMode()))); #else + // Endpoint if (!GetMandatoryStringParam(config, "Endpoint", mEndpoint, errorMsg)) { PARAM_ERROR_RETURN(mContext->GetLogger(), mContext->GetAlarm(), @@ -369,35 +403,22 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mContext->GetProjectName(), mContext->GetLogstoreName(), mContext->GetRegion()); - } else { -#endif - mEndpoint = TrimString(mEndpoint); - if (!mEndpoint.empty()) { - SLSClientManager::GetInstance()->AddEndpointEntry(mRegion, - StandardizeEndpoint(mEndpoint, mEndpoint), - false, - SLSClientManager::EndpointSourceType::REMOTE); - } - } -#ifdef __ENTERPRISE__ } - - // Aliuid - if (!GetOptionalStringParam(config, "Aliuid", mAliuid, errorMsg)) { - PARAM_WARNING_IGNORE(mContext->GetLogger(), - mContext->GetAlarm(), - errorMsg, - sName, - mContext->GetConfigName(), - mContext->GetProjectName(), - mContext->GetLogstoreName(), - mContext->GetRegion()); + mEndpoint = TrimString(mEndpoint); + if (mEndpoint.empty()) { + PARAM_ERROR_RETURN(mContext->GetLogger(), + mContext->GetAlarm(), + "param Endpoint is empty", + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); } #endif // TelemetryType string telemetryType; - if (!GetOptionalStringParam(config, "TelemetryType", telemetryType, errorMsg)) { PARAM_WARNING_DEFAULT(mContext->GetLogger(), mContext->GetAlarm(), @@ -541,10 +562,8 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline bool FlusherSLS::Start() { Flusher::Start(); - InitResource(); IncreaseProjectRegionReferenceCnt(mProject, mRegion); - SLSClientManager::GetInstance()->IncreaseAliuidReferenceCntForRegion(mRegion, mAliuid); return true; } @@ -552,7 +571,6 @@ bool FlusherSLS::Stop(bool isPipelineRemoving) { Flusher::Stop(isPipelineRemoving); DecreaseProjectRegionReferenceCnt(mProject, mRegion); - SLSClientManager::GetInstance()->DecreaseAliuidReferenceCntForRegion(mRegion, mAliuid); return true; } @@ -579,71 +597,82 @@ bool FlusherSLS::FlushAll() { return SerializeAndPush(std::move(res)); } -bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr& req, bool* keepItem) const { - auto data = static_cast(item); - sdk::Client* sendClient = SLSClientManager::GetInstance()->GetClient(mRegion, mAliuid); - - int32_t curTime = time(NULL); - static int32_t lastResetEndpointTime = 0; - data->mCurrentEndpoint = sendClient->GetRawSlsHost(); - if (data->mCurrentEndpoint.empty()) { - if (curTime - lastResetEndpointTime >= 30) { - SLSClientManager::GetInstance()->ResetClientEndpoint(mAliuid, mRegion, curTime); - data->mCurrentEndpoint = sendClient->GetRawSlsHost(); - lastResetEndpointTime = curTime; - } - } +bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr& req, bool* keepItem, string* errMsg) { if (mSendCnt) { mSendCnt->Add(1); } - if (BOOL_FLAG(send_prefer_real_ip)) { - if (curTime - sendClient->GetSlsRealIpUpdateTime() >= INT32_FLAG(send_check_real_ip_interval)) { - SLSClientManager::GetInstance()->UpdateSendClientRealIp(sendClient, mRegion); + + SLSClientManager::AuthType type; + string accessKeyId, accessKeySecret; + if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { +#ifdef __ENTERPRISE__ + if (!EnterpriseSLSClientManager::GetInstance()->GetAccessKeyIfProjectSupportsAnonymousWrite( + mProject, type, accessKeyId, accessKeySecret)) { + *keepItem = true; + *errMsg = "failed to get access key"; + return false; } - data->mRealIpFlag = sendClient->GetRawSlsHostFlag(); +#endif } - if (data->mType == RawDataType::EVENT_GROUP) { - if (mTelemetryType == sls_logs::SLS_TELEMETRY_TYPE_METRICS) { - req = sendClient->CreatePostMetricStoreLogsRequest( - mProject, data->mLogstore, ConvertCompressType(GetCompressType()), data->mData, data->mRawSize, item); - } else { - if (data->mShardHashKey.empty()) { - req = sendClient->CreatePostLogStoreLogsRequest(mProject, - data->mLogstore, - ConvertCompressType(GetCompressType()), - data->mData, - data->mRawSize, - item); - } else { - auto& exactlyOnceCpt = data->mExactlyOnceCheckpoint; - int64_t hashKeySeqID = exactlyOnceCpt ? exactlyOnceCpt->data.sequence_id() : sdk::kInvalidHashKeySeqID; - req = sendClient->CreatePostLogStoreLogsRequest(mProject, - data->mLogstore, - ConvertCompressType(GetCompressType()), - data->mData, - data->mRawSize, - item, - data->mShardHashKey, - hashKeySeqID); + auto data = static_cast(item); +#ifdef __ENTERPRISE__ + if (BOOL_FLAG(send_prefer_real_ip)) { + data->mCurrentHost = EnterpriseSLSClientManager::GetInstance()->GetRealIp(mRegion); + if (data->mCurrentHost.empty()) { + auto info + = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); + if (mCandidateHostsInfo.get() != info.get()) { + LOG_INFO(sLogger, + ("update candidate hosts info, region", mRegion)("project", mProject)("logstore", mLogstore)( + "from", EndpointModeToString(mCandidateHostsInfo->GetMode()))( + "to", EndpointModeToString(info->GetMode()))); + mCandidateHostsInfo = info; } + data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); + data->mRealIpFlag = false; + } else { + data->mRealIpFlag = true; } } else { - if (data->mShardHashKey.empty()) - req = sendClient->CreatePostLogStoreLogPackageListRequest( - mProject, data->mLogstore, ConvertCompressType(GetCompressType()), data->mData, item); - else - req = sendClient->CreatePostLogStoreLogPackageListRequest(mProject, - data->mLogstore, - ConvertCompressType(GetCompressType()), - data->mData, - item, - data->mShardHashKey); + // in case local region endpoint mode is changed, we should always check before sending + auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); + if (mCandidateHostsInfo == nullptr) { + // TODO: temporarily used here, for send logtail alarm only, should be removed after alarm is refactored + mCandidateHostsInfo = info; + } + if (mCandidateHostsInfo.get() != info.get()) { + LOG_INFO(sLogger, + ("update candidate hosts info, region", mRegion)("project", mProject)("logstore", mLogstore)( + "from", EndpointModeToString(mCandidateHostsInfo->GetMode()))( + "to", EndpointModeToString(info->GetMode()))); + mCandidateHostsInfo = info; + } + data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); } - if (!req) { + if (data->mCurrentHost.empty()) { + if (mCandidateHostsInfo->IsInitialized()) { + GetRegionConcurrencyLimiter(mRegion)->OnFail(chrono::system_clock::now()); + } + *errMsg = "failed to get available host"; *keepItem = true; return false; } +#else + static string host = mProject + "." + mEndpoint; + data->mCurrentHost = host; +#endif + + switch (mTelemetryType) { + case sls_logs::SLS_TELEMETRY_TYPE_LOGS: + req = CreatePostLogStoreLogsRequest(accessKeyId, accessKeySecret, type, data); + break; + case sls_logs::SLS_TELEMETRY_TYPE_METRICS: + req = CreatePostMetricStoreLogsRequest(accessKeyId, accessKeySecret, type, data); + break; + default: + break; + } return true; } @@ -651,31 +680,14 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) if (mSendDoneCnt) { mSendDoneCnt->Add(1); } - SLSResponse slsResponse; - if (AppConfig::GetInstance()->IsResponseVerificationEnabled() && !IsSLSResponse(response)) { - slsResponse.mStatusCode = 0; - slsResponse.mErrorCode = sdk::LOGE_REQUEST_ERROR; - slsResponse.mErrorMsg = "invalid response body"; - } else { - slsResponse.Parse(response); - - if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) { - static uint32_t sCount = 0; - if (sCount++ % 10000 == 0 || slsResponse.mErrorCode == sdk::LOGE_REQUEST_TIME_EXPIRED) { - time_t serverTime = GetServerTime(response); - if (serverTime > 0) { - UpdateTimeDelta(serverTime); - } - } - } - } + SLSResponse slsResponse = ParseHttpResponse(response); auto data = static_cast(item); string configName = HasContext() ? GetContext().GetConfigName() : ""; bool isProfileData = GetProfileSender()->IsProfileData(mRegion, mProject, data->mLogstore); int32_t curTime = time(NULL); auto curSystemTime = chrono::system_clock::now(); - bool hasAuthError = false; + SendResult sendResult = SEND_OK; if (slsResponse.mStatusCode == 200) { auto& cpt = data->mExactlyOnceCheckpoint; if (cpt) { @@ -691,8 +703,8 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) + "ms")( "total send time", ToString(chrono::duration_cast(curSystemTime - item->mFirstEnqueTime).count()) - + "ms")("try cnt", data->mTryCnt)("endpoint", data->mCurrentEndpoint)("is profile data", - isProfileData)); + + "ms")("try cnt", data->mTryCnt)("endpoint", data->mCurrentHost)("is profile data", + isProfileData)); GetRegionConcurrencyLimiter(mRegion)->OnSuccess(curSystemTime); GetProjectConcurrencyLimiter(mProject)->OnSuccess(curSystemTime); GetLogstoreConcurrencyLimiter(mProject, mLogstore)->OnSuccess(curSystemTime); @@ -703,9 +715,8 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) DealSenderQueueItemAfterSend(item, false); } else { OperationOnFail operation; - SendResult sendResult = ConvertErrorCode(slsResponse.mErrorCode); + sendResult = ConvertErrorCode(slsResponse.mErrorCode); ostringstream failDetail, suggestion; - string failEndpoint = data->mCurrentEndpoint; if (sendResult == SEND_NETWORK_ERROR || sendResult == SEND_SERVER_ERROR) { if (sendResult == SEND_NETWORK_ERROR) { failDetail << "network error"; @@ -719,29 +730,19 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) } } suggestion << "check network connection to endpoint"; - if (BOOL_FLAG(send_prefer_real_ip) && data->mRealIpFlag) { +#ifdef __ENTERPRISE__ + if (data->mRealIpFlag) { // connect refused, use vip directly failDetail << ", real ip may be stale, force update"; - // just set force update flag - SLSClientManager::GetInstance()->ForceUpdateRealIp(mRegion); - } - if (sendResult == SEND_NETWORK_ERROR) { - // only set network stat when no real ip - if (!BOOL_FLAG(send_prefer_real_ip) || !data->mRealIpFlag) { - SLSClientManager::GetInstance()->UpdateEndpointStatus(mRegion, data->mCurrentEndpoint, false); - if (SLSClientManager::GetInstance()->GetServerSwitchPolicy() - == SLSClientManager::EndpointSwitchPolicy::DESIGNATED_FIRST) { - SLSClientManager::GetInstance()->ResetClientEndpoint(mAliuid, mRegion, curTime); - } - } + EnterpriseSLSClientManager::GetInstance()->UpdateOutdatedRealIpRegions(mRegion); } +#endif operation = data->mBufferOrNot ? OperationOnFail::RETRY_LATER : OperationOnFail::DISCARD; GetRegionConcurrencyLimiter(mRegion)->OnFail(curSystemTime); GetProjectConcurrencyLimiter(mProject)->OnSuccess(curSystemTime); GetLogstoreConcurrencyLimiter(mProject, mLogstore)->OnSuccess(curSystemTime); } else if (sendResult == SEND_QUOTA_EXCEED) { - BOOL_FLAG(global_network_success) = true; - if (slsResponse.mErrorCode == sdk::LOGE_SHARD_WRITE_QUOTA_EXCEED) { + if (slsResponse.mErrorCode == LOGE_SHARD_WRITE_QUOTA_EXCEED) { failDetail << "shard write quota exceed"; suggestion << "Split logstore shards. https://help.aliyun.com/zh/sls/user-guide/expansion-of-resources"; GetLogstoreConcurrencyLimiter(mProject, mLogstore)->OnFail(curSystemTime); @@ -773,8 +774,6 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) failDetail << "write unauthorized"; suggestion << "check access keys provided"; operation = OperationOnFail::RETRY_LATER; - BOOL_FLAG(global_network_success) = true; - hasAuthError = true; if (mUnauthErrorCnt) { mUnauthErrorCnt->Add(1); } @@ -824,7 +823,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) cpt->IncreaseSequenceID(); } while (0); } else if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust() - && sdk::LOGE_REQUEST_TIME_EXPIRED == slsResponse.mErrorCode) { + && LOGE_REQUEST_TIME_EXPIRED == slsResponse.mErrorCode) { failDetail << "write request expired, will retry"; suggestion << "check local system time"; operation = OperationOnFail::RETRY_IMMEDIATELY; @@ -861,7 +860,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) ToString(chrono::duration_cast(curSystemTime - data->mLastSendTime).count()) \ + "ms")("total send time", \ ToString(chrono::duration_cast(curSystemTime - data->mFirstEnqueTime).count()) \ - + "ms")("endpoint", data->mCurrentEndpoint)("is profile data", isProfileData) + + "ms")("endpoint", data->mCurrentHost)("is profile data", isProfileData) switch (operation) { case OperationOnFail::RETRY_IMMEDIATELY: @@ -869,7 +868,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) FlusherRunner::GetInstance()->PushToHttpSink(item, false); break; case OperationOnFail::RETRY_LATER: - if (slsResponse.mErrorCode == sdk::LOGE_REQUEST_TIMEOUT + if (slsResponse.mErrorCode == LOGE_REQUEST_TIMEOUT || curTime - data->mLastLogWarningTime > ON_FAIL_LOG_WARNING_INTERVAL_SECOND) { LOG_WARNING(sLogger, LOG_PATTERN); data->mLastLogWarningTime = curTime; @@ -887,7 +886,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) + "\trequestId: " + slsResponse.mRequestId + "\tstatusCode: " + ToString(slsResponse.mStatusCode) + "\terrorCode: " + slsResponse.mErrorCode + "\terrorMessage: " + slsResponse.mErrorMsg - + "\tconfig: " + configName + "\tendpoint: " + data->mCurrentEndpoint, + + "\tconfig: " + configName + "\tendpoint: " + data->mCurrentHost, mProject, data->mLogstore, mRegion); @@ -897,10 +896,17 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) break; } } - SLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); #ifdef __ENTERPRISE__ - static auto manager = static_cast(SLSClientManager::GetInstance()); - manager->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); + bool hasNetworkError = (sendResult == SEND_NETWORK_ERROR || sendResult == SEND_SERVER_ERROR); + EnterpriseSLSClientManager::GetInstance()->UpdateHostStatus( + mProject, mCandidateHostsInfo->GetMode(), data->mCurrentHost, !hasNetworkError); + mCandidateHostsInfo->SelectBestHost(); + + if (!hasNetworkError) { + bool hasAuthError = sendResult == SEND_UNAUTHORIZED; + EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); + EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); + } #endif } @@ -1155,7 +1161,7 @@ string FlusherSLS::GetShardHashKey(const BatchedEvents& g) const { key += "_"; } } - return sdk::CalcMD5(key); + return CalcMD5(key); } void FlusherSLS::AddPackId(BatchedEvents& g) const { @@ -1167,6 +1173,78 @@ void FlusherSLS::AddPackId(BatchedEvents& g) const { g.mTags.Insert(LOG_RESERVED_KEY_PACKAGE_ID, StringView(packId.data, packId.size)); } +unique_ptr FlusherSLS::CreatePostLogStoreLogsRequest(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + SLSSenderQueueItem* item) const { + optional seqId; + if (item->mExactlyOnceCheckpoint) { + seqId = item->mExactlyOnceCheckpoint->data.sequence_id(); + } + string path, query; + map header; + PreparePostLogStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + item->mCurrentHost, + item->mRealIpFlag, + mProject, + item->mLogstore, + CompressTypeToString(mCompressor->GetCompressType()), + item->mType, + item->mData, + item->mRawSize, + item->mShardHashKey, + seqId, + path, + query, + header); + bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(mRegion); + return make_unique(HTTP_POST, + httpsFlag, + item->mCurrentHost, + httpsFlag ? 443 : 80, + path, + query, + header, + item->mData, + item, + INT32_FLAG(default_http_request_timeout_sec), + 1); +} + +unique_ptr FlusherSLS::CreatePostMetricStoreLogsRequest(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + SLSSenderQueueItem* item) const { + string path; + map header; + PreparePostMetricStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + item->mCurrentHost, + item->mRealIpFlag, + mProject, + item->mLogstore, + CompressTypeToString(mCompressor->GetCompressType()), + item->mData, + item->mRawSize, + path, + header); + bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(mRegion); + return make_unique(HTTP_POST, + httpsFlag, + item->mCurrentHost, + httpsFlag ? 443 : 80, + path, + "", + header, + item->mData, + item, + INT32_FLAG(default_http_request_timeout_sec), + 1); +} + sls_logs::SlsCompressType ConvertCompressType(CompressType type) { sls_logs::SlsCompressType compressType = sls_logs::SLS_CMP_NONE; switch (type) { diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 6a71b0f526..0d83f49ba0 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -30,7 +30,12 @@ #include "pipeline/batch/Batcher.h" #include "pipeline/limiter/ConcurrencyLimiter.h" #include "pipeline/plugin/interface/HttpFlusher.h" +#include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/serializer/SLSSerializer.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif +#include "plugin/flusher/sls/SLSClientManager.h" #include "protobuf/sls/sls_logs.pb.h" namespace logtail { @@ -43,18 +48,14 @@ class FlusherSLS : public HttpFlusher { static std::shared_ptr GetRegionConcurrencyLimiter(const std::string& region); static void ClearInvalidConcurrencyLimiters(); + static void InitResource(); static void RecycleResourceIfNotUsed(); static std::string GetDefaultRegion(); static void SetDefaultRegion(const std::string& region); static std::string GetAllProjects(); - static bool IsRegionContainingConfig(const std::string& region); static std::string GetProjectRegion(const std::string& project); - // TODO: should be moved to enterprise config provider - static bool GetRegionStatus(const std::string& region); - static void UpdateRegionStatus(const std::string& region, bool status); - static const std::string sName; FlusherSLS(); @@ -66,7 +67,7 @@ class FlusherSLS : public HttpFlusher { bool Send(PipelineEventGroup&& g) override; bool Flush(size_t key) override; bool FlushAll() override; - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const override; + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override; void OnSendDone(const HttpResponse& response, SenderQueueItem* item) override; CompressType GetCompressType() const { return mCompressor ? mCompressor->GetCompressType() : CompressType::NONE; } @@ -77,8 +78,11 @@ class FlusherSLS : public HttpFlusher { std::string mProject; std::string mLogstore; std::string mRegion; - std::string mEndpoint; std::string mAliuid; +#ifdef __ENTERPRISE__ + EndpointMode mEndpointMode = EndpointMode::DEFAULT; +#endif + std::string mEndpoint; sls_logs::SlsTelemetryType mTelemetryType = sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS; std::vector mShardHashKeys; uint32_t mMaxSendRate = 0; // preserved only for exactly once @@ -87,10 +91,6 @@ class FlusherSLS : public HttpFlusher { std::unique_ptr mCompressor; private: - static const std::unordered_set sNativeParam; - - static void InitResource(); - static void IncreaseProjectRegionReferenceCnt(const std::string& project, const std::string& region); static void DecreaseProjectRegionReferenceCnt(const std::string& project, const std::string& region); @@ -99,18 +99,15 @@ class FlusherSLS : public HttpFlusher { static std::unordered_map> sRegionConcurrencyLimiterMap; static std::unordered_map> sLogstoreConcurrencyLimiterMap; + static const std::unordered_set sNativeParam; + static std::mutex sDefaultRegionLock; static std::string sDefaultRegion; static std::mutex sProjectRegionMapLock; static std::unordered_map sProjectRefCntMap; - static std::unordered_map sRegionRefCntMap; static std::unordered_map sProjectRegionMap; - // TODO: should be moved to enterprise config provider - static std::mutex sRegionStatusLock; - static std::unordered_map sAllRegionStatus; - static bool sIsResourceInited; void GenerateGoPlugin(const Json::Value& config, Json::Value& res) const; @@ -121,9 +118,23 @@ class FlusherSLS : public HttpFlusher { std::string GetShardHashKey(const BatchedEvents& g) const; void AddPackId(BatchedEvents& g) const; + std::unique_ptr CreatePostLogStoreLogsRequest(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + SLSSenderQueueItem* item) const; + std::unique_ptr CreatePostMetricStoreLogsRequest(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + SLSSenderQueueItem* item) const; + Batcher mBatcher; std::unique_ptr mGroupSerializer; std::unique_ptr>> mGroupListSerializer; +#ifdef __ENTERPRISE__ + // This may not be cached. However, this provides a simple way to control the lifetime of a CandidateHostsInfo. + // Otherwise, timeout machanisim must be emplyed to clean up unused CandidateHostsInfo. + std::shared_ptr mCandidateHostsInfo; +#endif CounterPtr mSendCnt; CounterPtr mSendDoneCnt; diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 7e7930a547..be9e57291e 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -18,658 +18,52 @@ #include #endif -#include - #include "app_config/AppConfig.h" -#include "common/EndpointUtil.h" #include "common/Flags.h" -#include "common/LogtailCommonFlags.h" +#include "common/HashUtil.h" #include "common/StringTools.h" -#include "common/TimeUtil.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/version.h" #include "logger/Logger.h" #include "monitor/Monitor.h" #ifdef __ENTERPRISE__ #include "plugin/flusher/sls/EnterpriseSLSClientManager.h" #endif -#include "plugin/flusher/sls/FlusherSLS.h" -#include "plugin/flusher/sls/SendResult.h" -#include "sdk/Exception.h" - -// for windows compatability, to avoid conflict with the same function defined in windows.h -#ifdef SetPort -#undef SetPort -#endif +#include "plugin/flusher/sls/SLSConstant.h" +#include "plugin/flusher/sls/SLSUtil.h" -DEFINE_FLAG_STRING(data_endpoint_policy, - "policy for switching between data server endpoints, possible options include " - "'designated_first'(default) and 'designated_locked'", - "designated_first"); -DEFINE_FLAG_INT32(sls_host_update_interval, "seconds", 5); -DEFINE_FLAG_INT32(send_client_timeout_interval, "recycle clients avoid memory increment", 12 * 3600); -DEFINE_FLAG_INT32(test_network_normal_interval, "if last check is normal, test network again after seconds ", 30); -DEFINE_FLAG_INT32(test_unavailable_endpoint_interval, "test unavailable endpoint interval", 60); -DEFINE_FLAG_INT32(send_switch_real_ip_interval, "seconds", 60); -DEFINE_FLAG_BOOL(send_prefer_real_ip, "use real ip to send data", false); +DEFINE_FLAG_STRING(custom_user_agent, "custom user agent appended at the end of the exsiting ones", ""); DEFINE_FLAG_STRING(default_access_key_id, "", ""); DEFINE_FLAG_STRING(default_access_key, "", ""); -DEFINE_FLAG_STRING(custom_user_agent, "custom user agent appended at the end of the exsiting ones", ""); using namespace std; namespace logtail { -bool SLSClientManager::RegionEndpointsInfo::AddDefaultEndpoint(const std::string& endpoint, - const EndpointSourceType& endpointType, - bool& isDefault) { - if (mDefaultEndpoint.empty() - || (endpointType == EndpointSourceType::LOCAL && mDefaultEndpointType == EndpointSourceType::REMOTE)) { - mDefaultEndpoint = endpoint; - mDefaultEndpointType = endpointType; - isDefault = true; - } - return AddEndpoint(endpoint, true, false); -} - -bool SLSClientManager::RegionEndpointsInfo::AddEndpoint(const std::string& endpoint, bool status, bool proxy) { - if (mEndpointInfoMap.find(endpoint) == mEndpointInfoMap.end()) { - mEndpointInfoMap.emplace(std::make_pair(endpoint, EndpointInfo(status, proxy))); - return true; - } - return false; -} - -void SLSClientManager::RegionEndpointsInfo::UpdateEndpointInfo(const std::string& endpoint, - bool status, - std::optional latency, - bool createFlag) { - auto iter = mEndpointInfoMap.find(endpoint); - if (iter == mEndpointInfoMap.end()) { - if (createFlag) { - AddEndpoint(endpoint, status); - } - } else { - (iter->second).UpdateInfo(status, latency); - } -} - -void SLSClientManager::RegionEndpointsInfo::RemoveEndpoint(const std::string& endpoint) { - mEndpointInfoMap.erase(endpoint); - if (mDefaultEndpoint == endpoint) { - mDefaultEndpoint.clear(); - } -} - -std::string SLSClientManager::RegionEndpointsInfo::GetAvailableEndpointWithTopPriority() const { - if (!mDefaultEndpoint.empty()) { - auto iter = mEndpointInfoMap.find(mDefaultEndpoint); - if (iter != mEndpointInfoMap.end() && (iter->second).mValid) { - return mDefaultEndpoint; - } - } - std::string proxyEndpoint; - for (auto iter = mEndpointInfoMap.begin(); iter != mEndpointInfoMap.end(); ++iter) { - if (!(iter->second).mValid) { - continue; - } - if ((iter->second).mValid && !(iter->second).mProxy) { - return iter->first; - } - proxyEndpoint = iter->first; - } - if (!proxyEndpoint.empty()) { - return proxyEndpoint; - } - if (!mDefaultEndpoint.empty()) { - return mDefaultEndpoint; - } - if (!mEndpointInfoMap.empty()) { - return mEndpointInfoMap.begin()->first; - } - return mDefaultEndpoint; -} - SLSClientManager* SLSClientManager::GetInstance() { #ifdef __ENTERPRISE__ - static auto ptr = unique_ptr(new EnterpriseSLSClientManager()); + return EnterpriseSLSClientManager::GetInstance(); #else - static auto ptr = unique_ptr(new SLSClientManager()); + static SLSClientManager instance; + return &instance; #endif - return ptr.get(); } void SLSClientManager::Init() { - InitEndpointSwitchPolicy(); GenerateUserAgent(); - if (mDataServerSwitchPolicy == EndpointSwitchPolicy::DESIGNATED_FIRST) { - mProbeNetworkClient.reset(new sdk::Client("", - "", - INT32_FLAG(sls_client_send_timeout))); - mProbeNetworkClient->SetPort(AppConfig::GetInstance()->GetDataServerPort()); - mProbeNetworkThreadRes = async(launch::async, &SLSClientManager::ProbeNetworkThread, this); - } - if (BOOL_FLAG(send_prefer_real_ip)) { - mUpdateRealIpClient.reset(new sdk::Client("", - "", - INT32_FLAG(sls_client_send_timeout))); - mUpdateRealIpClient->SetPort(AppConfig::GetInstance()->GetDataServerPort()); - mUpdateRealIpThreadRes = async(launch::async, &SLSClientManager::UpdateRealIpThread, this); - } -} - -void SLSClientManager::Stop() { - if (mDataServerSwitchPolicy == EndpointSwitchPolicy::DESIGNATED_FIRST) { - lock_guard lock(mProbeNetworkThreadRunningMux); - mIsProbeNetworkThreadRunning = false; - } - if (BOOL_FLAG(send_prefer_real_ip)) { - lock_guard lock(mUpdateRealIpThreadRunningMux); - mIsUpdateRealIpThreadRunning = false; - } - mStopCV.notify_all(); - if (mDataServerSwitchPolicy == EndpointSwitchPolicy::DESIGNATED_FIRST && mProbeNetworkThreadRes.valid()) { - future_status s = mProbeNetworkThreadRes.wait_for(chrono::seconds(1)); - if (s == future_status::ready) { - LOG_INFO(sLogger, ("sls endpoint probe", "stopped successfully")); - } else { - LOG_WARNING(sLogger, ("sls endpoint probe", "forced to stopped")); - } - } - if (BOOL_FLAG(send_prefer_real_ip) && mUpdateRealIpThreadRes.valid()) { - future_status s = mUpdateRealIpThreadRes.wait_for(chrono::seconds(1)); - if (s == future_status::ready) { - LOG_INFO(sLogger, ("sls real ip update", "stopped successfully")); - } else { - LOG_WARNING(sLogger, ("sls real ip update", "forced to stopped")); - } - } -} - -void SLSClientManager::InitEndpointSwitchPolicy() { - if (STRING_FLAG(data_endpoint_policy) == "designated_locked") { - mDataServerSwitchPolicy = EndpointSwitchPolicy::DESIGNATED_LOCKED; - } else if (STRING_FLAG(data_endpoint_policy) == "designated_first") { - mDataServerSwitchPolicy = EndpointSwitchPolicy::DESIGNATED_FIRST; - } else { - LOG_WARNING(sLogger, - ("param data_endpoint_policy is invalid, action", "use default value instead")("default value", - "designated_first")); - } -} - -vector SLSClientManager::GetRegionAliuids(const std::string& region) { - lock_guard lock(mRegionAliuidRefCntMapLock); - vector aliuids; - for (const auto& item : mRegionAliuidRefCntMap[region]) { - aliuids.push_back(item.first); - } - return aliuids; -} - -void SLSClientManager::IncreaseAliuidReferenceCntForRegion(const std::string& region, const std::string& aliuid) { - lock_guard lock(mRegionAliuidRefCntMapLock); - ++mRegionAliuidRefCntMap[region][aliuid]; -} - -void SLSClientManager::DecreaseAliuidReferenceCntForRegion(const std::string& region, const std::string& aliuid) { - lock_guard lock(mRegionAliuidRefCntMapLock); - auto outerIter = mRegionAliuidRefCntMap.find(region); - if (outerIter == mRegionAliuidRefCntMap.end()) { - // should not happen - return; - } - auto innerIter = outerIter->second.find(aliuid); - if (innerIter == outerIter->second.end()) { - // should not happen - return; - } - if (--innerIter->second == 0) { - outerIter->second.erase(innerIter); - } - if (outerIter->second.empty()) { - mRegionAliuidRefCntMap.erase(outerIter); - } -} - -sdk::Client* SLSClientManager::GetClient(const string& region, const string& aliuid, bool createIfNotFound) { - string key = region + "_" + aliuid; - { - lock_guard lock(mClientMapMux); - auto iter = mClientMap.find(key); - if (iter != mClientMap.end()) { - (iter->second).second = time(NULL); - return (iter->second).first.get(); - } - } - if (!createIfNotFound) { - return nullptr; - } - - string endpoint = GetAvailableEndpointWithTopPriority(region); - auto client = make_unique(aliuid, - endpoint, - INT32_FLAG(sls_client_send_timeout)); - ResetClientPort(region, client.get()); - LOG_INFO(sLogger, - ("init endpoint for sender, region", region)("uid", aliuid)("hostname", GetHostFromEndpoint(endpoint))( - "use https", ToString(client->IsUsingHTTPS()))); - auto ptr = client.get(); - { - lock_guard lock(mClientMapMux); - mClientMap.insert(make_pair(key, make_pair(std::move(client), time(nullptr)))); - } - return ptr; -} - -bool SLSClientManager::ResetClientEndpoint(const string& aliuid, const string& region, time_t curTime) { - sdk::Client* sendClient = GetClient(region, aliuid, false); - if (sendClient == nullptr) { - return false; - } - if (curTime - sendClient->GetSlsHostUpdateTime() < INT32_FLAG(sls_host_update_interval)) { - return false; - } - sendClient->SetSlsHostUpdateTime(curTime); - string endpoint = GetAvailableEndpointWithTopPriority(region); - if (endpoint.empty()) { - return false; - } - string originalEndpoint = sendClient->GetRawSlsHost(); - if (originalEndpoint == endpoint) { - return false; - } - sendClient->SetSlsHost(endpoint); - ResetClientPort(region, sendClient); - LOG_INFO( - sLogger, - ("reset endpoint for sender, region", region)("uid", aliuid)("from", GetHostFromEndpoint(originalEndpoint))( - "to", GetHostFromEndpoint(endpoint))("use https", ToString(sendClient->IsUsingHTTPS()))); - return true; -} - -void SLSClientManager::ResetClientPort(const string& region, sdk::Client* sendClient) { - sendClient->SetPort(AppConfig::GetInstance()->GetDataServerPort()); - if (AppConfig::GetInstance()->GetDataServerPort() == 80) { - lock_guard lock(mRegionEndpointEntryMapLock); - auto iter = mRegionEndpointEntryMap.find(region); - if (iter != mRegionEndpointEntryMap.end()) { - const string& defaultEndpoint = iter->second.mDefaultEndpoint; - if (!defaultEndpoint.empty()) { - if (IsHttpsEndpoint(defaultEndpoint)) { - sendClient->SetPort(443); - } - } else { - if (IsHttpsEndpoint(sendClient->GetRawSlsHost())) { - sendClient->SetPort(443); - } - } - } - } -} - -void SLSClientManager::CleanTimeoutClient() { - lock_guard lock(mClientMapMux); - time_t curTime = time(nullptr); - for (auto iter = mClientMap.begin(); iter != mClientMap.end();) { - if ((curTime - (iter->second).second) > INT32_FLAG(send_client_timeout_interval)) { - iter = mClientMap.erase(iter); - } else { - ++iter; - } - } } -bool SLSClientManager::GetAccessKey(const std::string& aliuid, +bool SLSClientManager::GetAccessKey(const string& aliuid, AuthType& type, - std::string& accessKeyId, - std::string& accessKeySecret) { + string& accessKeyId, + string& accessKeySecret) { accessKeyId = STRING_FLAG(default_access_key_id); accessKeySecret = STRING_FLAG(default_access_key); type = AuthType::AK; return true; } -void SLSClientManager::AddEndpointEntry(const string& region, - const string& endpoint, - bool isProxy, - const EndpointSourceType& endpointType) { - lock_guard lock(mRegionEndpointEntryMapLock); - RegionEndpointsInfo& info = mRegionEndpointEntryMap[region]; - if (!isProxy) { - bool isDefault = false; - if (info.AddDefaultEndpoint(endpoint, endpointType, isDefault)) { - LOG_INFO(sLogger, - ("add data server endpoint, region", region)("endpoint", endpoint)( - "isDefault", isDefault ? "yes" : "no")("isProxy", "false")("#endpoint", - info.mEndpointInfoMap.size())); - } - } else { - if (info.AddEndpoint(endpoint, true, isProxy)) { - LOG_INFO(sLogger, - ("add data server endpoint, region", region)("endpoint", endpoint)("isProxy", ToString(isProxy))( - "#endpoint", info.mEndpointInfoMap.size())); - } - } -} - -void SLSClientManager::UpdateEndpointStatus(const string& region, - const string& endpoint, - bool status, - optional latency) { - lock_guard lock(mRegionEndpointEntryMapLock); - auto iter = mRegionEndpointEntryMap.find(region); - if (iter != mRegionEndpointEntryMap.end()) { - (iter->second).UpdateEndpointInfo(endpoint, status, latency, false); - } -} - -string SLSClientManager::GetAvailableEndpointWithTopPriority(const string& region) const { - static string emptyStr = ""; - lock_guard lock(mRegionEndpointEntryMapLock); - auto iter = mRegionEndpointEntryMap.find(region); - if (iter != mRegionEndpointEntryMap.end()) { - return (iter->second).GetAvailableEndpointWithTopPriority(); - } - return emptyStr; -} - -string SLSClientManager::GetRegionFromEndpoint(const string& endpoint) { - lock_guard lock(mRegionEndpointEntryMapLock); - for (auto iter = mRegionEndpointEntryMap.begin(); iter != mRegionEndpointEntryMap.end(); ++iter) { - for (auto epIter = ((iter->second).mEndpointInfoMap).begin(); epIter != ((iter->second).mEndpointInfoMap).end(); - ++epIter) { - if (epIter->first == endpoint) - return iter->first; - } - } - return STRING_FLAG(default_region_name); -} - -bool SLSClientManager::HasNetworkAvailable() { - static time_t lastCheckTime = time(nullptr); - time_t curTime = time(nullptr); - if (curTime - lastCheckTime >= 3600) { - lastCheckTime = curTime; - return true; - } - { - lock_guard lock(mRegionEndpointEntryMapLock); - for (auto iter = mRegionEndpointEntryMap.begin(); iter != mRegionEndpointEntryMap.end(); ++iter) { - for (auto epIter = ((iter->second).mEndpointInfoMap).begin(); - epIter != ((iter->second).mEndpointInfoMap).end(); - ++epIter) { - if ((epIter->second).mValid) { - return true; - } - } - } - } - return false; -} - -void SLSClientManager::ProbeNetworkThread() { - LOG_INFO(sLogger, ("sls endpoint probe", "started")); - // pair represents the weight of each endpoint - map>> unavaliableEndpoints; - set unavaliableRegions; - int32_t lastCheckAllTime = 0; - unique_lock lock(mProbeNetworkThreadRunningMux); - while (mIsProbeNetworkThreadRunning) { - unavaliableEndpoints.clear(); - unavaliableRegions.clear(); - { - lock_guard lock(mRegionEndpointEntryMapLock); - for (auto iter = mRegionEndpointEntryMap.begin(); iter != mRegionEndpointEntryMap.end(); ++iter) { - auto& endpoints = unavaliableEndpoints[iter->first]; - bool unavaliable = true; - for (auto epIter = ((iter->second).mEndpointInfoMap).begin(); - epIter != ((iter->second).mEndpointInfoMap).end(); - ++epIter) { - if (!(epIter->second).mValid) { - if (epIter->first == iter->second.mDefaultEndpoint) { - endpoints.emplace_back(0, epIter->first); - } else { - endpoints.emplace_back(10, epIter->first); - } - } else { - unavaliable = false; - } - } - sort(endpoints.begin(), endpoints.end()); - if (unavaliable) { - unavaliableRegions.insert(iter->first); - } - } - } - if (unavaliableEndpoints.empty()) { - if (mStopCV.wait_for(lock, chrono::seconds(INT32_FLAG(test_network_normal_interval)), [this]() { - return !mIsProbeNetworkThreadRunning; - })) { - break; - } - continue; - } - int32_t curTime = time(NULL); - // bool wakeUp = false; - for (const auto& value : unavaliableEndpoints) { - const string& region = value.first; - vector uids = GetRegionAliuids(region); - bool endpointChanged = false; - for (const auto& item : value.second) { - const string& endpoint = item.second; - const int32_t priority = item.first; - if (unavaliableRegions.find(region) == unavaliableRegions.end()) { - if (!endpointChanged && priority != 10) { - if (TestEndpoint(region, endpoint)) { - for (const auto& uid : uids) { - ResetClientEndpoint(uid, region, curTime); - } - endpointChanged = true; - } - } else { - if (curTime - lastCheckAllTime >= 1800) { - TestEndpoint(region, endpoint); - } - } - } else { - if (TestEndpoint(region, endpoint)) { - // wakeUp = true; - // Sender::GetInstance()->OnRegionRecover(region); - if (!endpointChanged) { - for (const auto& uid : uids) { - ResetClientEndpoint(uid, region, curTime); - } - endpointChanged = true; - } - } - } - } - } - // if (wakeUp && (!mIsSendingBuffer)) { - // mSenderQueue.Signal(); - // } - if (curTime - lastCheckAllTime >= 1800) { - lastCheckAllTime = curTime; - } - if (mStopCV.wait_for(lock, chrono::seconds(INT32_FLAG(test_unavailable_endpoint_interval)), [this]() { - return !mIsProbeNetworkThreadRunning; - })) { - break; - } - } -} - -bool SLSClientManager::TestEndpoint(const string& region, const string& endpoint) { - // TODO: this should be removed, since control-plane status is not the same as data-plane - if (!FlusherSLS::GetRegionStatus(region)) { - return false; - } - if (endpoint.empty()) { - return false; - } - mProbeNetworkClient->SetSlsHost(endpoint); - ResetClientPort(region, mProbeNetworkClient.get()); - - bool status = true; - uint64_t beginTime = GetCurrentTimeInMicroSeconds(); - try { - status = mProbeNetworkClient->TestNetwork(); - } catch (sdk::LOGException& ex) { - if (ConvertErrorCode(ex.GetErrorCode()) == SEND_NETWORK_ERROR) { - status = false; - } - } catch (...) { - LOG_ERROR(sLogger, ("test network", "send fail")("exception", "unknown")); - status = false; - } - uint32_t latency = (GetCurrentTimeInMicroSeconds() - beginTime) / 1000; - LOG_DEBUG(sLogger, ("TestEndpoint, region", region)("endpoint", endpoint)("status", status)("latency", latency)); - UpdateEndpointStatus(region, endpoint, status, latency); - return status; -} - -void SLSClientManager::ForceUpdateRealIp(const string& region) { - lock_guard lock(mRegionRealIpLock); - auto iter = mRegionRealIpMap.find(region); - if (iter != mRegionRealIpMap.end()) { - iter->second->mForceFlushFlag = true; - } -} - -void SLSClientManager::UpdateSendClientRealIp(sdk::Client* client, const string& region) { - string realIp; - RealIpInfo* pInfo = NULL; - { - lock_guard lock(mRegionRealIpLock); - auto iter = mRegionRealIpMap.find(region); - if (iter != mRegionRealIpMap.end()) { - pInfo = iter->second; - } else { - pInfo = new RealIpInfo; - mRegionRealIpMap.insert(make_pair(region, pInfo)); - } - realIp = pInfo->mRealIp; - } - if (!realIp.empty()) { - client->SetSlsHost(realIp); - client->SetSlsRealIpUpdateTime(time(NULL)); - } else if (pInfo->mLastUpdateTime >= client->GetSlsRealIpUpdateTime()) { - const string& defaultEndpoint = GetAvailableEndpointWithTopPriority(region); - if (!defaultEndpoint.empty()) { - client->SetSlsHost(defaultEndpoint); - client->SetSlsRealIpUpdateTime(time(NULL)); - } - } -} - -void SLSClientManager::UpdateRealIpThread() { - LOG_INFO(sLogger, ("sls real ip update", "started")); - int32_t lastUpdateRealIpTime = 0; - vector regionEndpointArray; - vector regionArray; - unique_lock lock(mUpdateRealIpThreadRunningMux); - while (mIsUpdateRealIpThreadRunning) { - int32_t curTime = time(NULL); - bool updateFlag = curTime - lastUpdateRealIpTime > INT32_FLAG(send_switch_real_ip_interval); - { - // check force update - lock_guard lock(mRegionRealIpLock); - auto iter = mRegionRealIpMap.begin(); - for (; iter != mRegionRealIpMap.end(); ++iter) { - if (iter->second->mForceFlushFlag) { - iter->second->mForceFlushFlag = false; - updateFlag = true; - LOG_INFO(sLogger, ("force update real ip", iter->first)); - } - } - } - if (updateFlag) { - LOG_DEBUG(sLogger, ("start update real ip", "")); - regionEndpointArray.clear(); - regionArray.clear(); - { - lock_guard lock(mRegionEndpointEntryMapLock); - auto iter = mRegionEndpointEntryMap.begin(); - for (; iter != mRegionEndpointEntryMap.end(); ++iter) { - regionEndpointArray.push_back((iter->second).GetAvailableEndpointWithTopPriority()); - regionArray.push_back(iter->first); - } - } - for (size_t i = 0; i < regionEndpointArray.size(); ++i) { - // no available endpoint - if (regionEndpointArray[i].empty()) { - continue; - } - - EndpointStatus status = UpdateRealIp(regionArray[i], regionEndpointArray[i]); - if (status == EndpointStatus::STATUS_ERROR) { - UpdateEndpointStatus(regionArray[i], regionEndpointArray[i], false); - } - } - lastUpdateRealIpTime = time(NULL); - } - if (mStopCV.wait_for(lock, chrono::seconds(1), [this]() { return !mIsUpdateRealIpThreadRunning; })) { - break; - } - } -} - -SLSClientManager::EndpointStatus SLSClientManager::UpdateRealIp(const string& region, const string& endpoint) { - mUpdateRealIpClient->SetSlsHost(endpoint); - EndpointStatus status = EndpointStatus::STATUS_ERROR; - int64_t beginTime = GetCurrentTimeInMicroSeconds(); - try { - sdk::GetRealIpResponse rsp; - rsp = mUpdateRealIpClient->GetRealIp(); - - if (!rsp.realIp.empty()) { - SetRealIp(region, rsp.realIp); - status = EndpointStatus::STATUS_OK_WITH_IP; - } else { - status = EndpointStatus::STATUS_OK_WITH_ENDPOINT; - static int32_t sUpdateRealIpWarningCount = 0; - if (sUpdateRealIpWarningCount++ % 100 == 0) { - sUpdateRealIpWarningCount %= 100; - LOG_WARNING(sLogger, - ("get real ip request succeeded but server did not give real ip, region", - region)("endpoint", endpoint)); - } - - // we should set real ip to empty string if server did not give real ip - SetRealIp(region, ""); - } - } - // GetRealIp's implement should not throw LOGException, but we catch it to hold implement changing - catch (sdk::LOGException& ex) { - const string& errorCode = ex.GetErrorCode(); - LOG_DEBUG(sLogger, ("get real ip", "send fail")("errorCode", errorCode)("errorMessage", ex.GetMessage())); - SendResult sendRst = ConvertErrorCode(errorCode); - if (sendRst == SEND_NETWORK_ERROR) - status = EndpointStatus::STATUS_ERROR; - } catch (...) { - LOG_ERROR(sLogger, ("get real ip", "send fail")("exception", "unknown")); - } - int64_t endTime = GetCurrentTimeInMicroSeconds(); - int32_t latency = int32_t((endTime - beginTime) / 1000); // ms - LOG_DEBUG(sLogger, - ("Get real ip, region", region)("endpoint", endpoint)("status", int(status))("latency", latency)); - return status; -} - -void SLSClientManager::SetRealIp(const string& region, const string& ip) { - lock_guard lock(mRegionRealIpLock); - RealIpInfo* pInfo = NULL; - auto iter = mRegionRealIpMap.find(region); - if (iter != mRegionRealIpMap.end()) { - pInfo = iter->second; - } else { - pInfo = new RealIpInfo; - mRegionRealIpMap.insert(make_pair(region, pInfo)); - } - LOG_DEBUG(sLogger, ("set real ip, last", pInfo->mRealIp)("now", ip)("region", region)); - pInfo->SetRealIp(ip); -} - void SLSClientManager::GenerateUserAgent() { string os; #if defined(__linux__) @@ -714,7 +108,7 @@ string SLSClientManager::GetRunningEnvironment() { // containers in K8S will possess the above env if (AppConfig::GetInstance()->IsPurageContainerMode()) { env = "K8S-Daemonset"; - } else if (TryCurlEndpoint("http://100.100.100.200/latest/meta-data")) { + } else if (PingEndpoint("100.100.100.200", "/latest/meta-data")) { // containers in ACK can be connected to the above address, see // https://help.aliyun.com/document_detail/108460.html#section-akf-lwh-1gb. // Note: we can not distinguish ACK from K8S built on ECS @@ -724,7 +118,7 @@ string SLSClientManager::GetRunningEnvironment() { } } else if (AppConfig::GetInstance()->IsPurageContainerMode() || getenv("ALIYUN_LOGTAIL_CONFIG")) { env = "Docker"; - } else if (TryCurlEndpoint("http://100.100.100.200/latest/meta-data")) { + } else if (PingEndpoint("100.100.100.200", "/latest/meta-data")) { env = "ECS"; } else { env = "Others"; @@ -732,37 +126,207 @@ string SLSClientManager::GetRunningEnvironment() { return env; } -bool SLSClientManager::TryCurlEndpoint(const string& endpoint) { - CURL* curl; - for (size_t retryTimes = 1; retryTimes <= 5; retryTimes++) { - curl = curl_easy_init(); - if (curl) { - break; - } - this_thread::sleep_for(chrono::seconds(1)); +bool SLSClientManager::PingEndpoint(const string& host, const string& path) { + map header; + HttpResponse response; + return SendHttpRequest(make_unique(HTTP_GET, false, host, 80, path, "", header, "", 3, 1, true), + response); +} + +void PreparePostLogStoreLogsRequest(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + const string& host, + bool isHostIp, + const string& project, + const string& logstore, + const string& compressType, + RawDataType dataType, + const string& body, + size_t rawSize, + const string& shardHashKey, + optional seqId, + string& path, + string& query, + map& header) { + path = LOGSTORES; + path.append("/").append(logstore); + if (shardHashKey.empty()) { + path.append("/shards/lb"); + } else { + path.append("/shards/route"); } - if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, endpoint.c_str()); - curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); + if (isHostIp) { + header[HOST] = project + "." + host; + } else { + header[HOST] = host; + } + header[USER_AGENT] = SLSClientManager::GetInstance()->GetUserAgent(); + header[DATE] = GetDateString(); + header[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; + header[CONTENT_LENGTH] = to_string(body.size()); + header[CONTENT_MD5] = CalcMD5(body); + header[X_LOG_APIVERSION] = LOG_API_VERSION; + header[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; + if (!compressType.empty()) { + header[X_LOG_COMPRESSTYPE] = compressType; + } + if (dataType == RawDataType::EVENT_GROUP) { + header[X_LOG_BODYRAWSIZE] = to_string(rawSize); + } else { + header[X_LOG_BODYRAWSIZE] = to_string(body.size()); + header[X_LOG_MODE] = LOG_MODE_BATCH_GROUP; + } + if (type == SLSClientManager::AuthType::ANONYMOUS) { + header[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; + } - if (curl_easy_perform(curl) != CURLE_OK) { - curl_easy_cleanup(curl); - return false; + map parameterList; + if (!shardHashKey.empty()) { + parameterList["key"] = shardHashKey; + if (seqId.has_value()) { + parameterList["seqid"] = to_string(seqId.value()); } - curl_easy_cleanup(curl); - return true; } - - LOG_WARNING( - sLogger, - ("curl handler cannot be initialized during user environment identification", "user agent may be mislabeled")); - return false; + query = GetQueryString(parameterList); + + string signature = GetUrlSignature(HTTP_POST, path, header, parameterList, body, accessKeySecret); + header[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; +} + +void PreparePostMetricStoreLogsRequest(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + const string& host, + bool isHostIp, + const string& project, + const string& logstore, + const string& compressType, + const string& body, + size_t rawSize, + string& path, + map& header) { + path = METRICSTORES; + path.append("/").append(project).append("/").append(logstore).append("/api/v1/write"); + + if (isHostIp) { + header[HOST] = project + "." + host; + } else { + header[HOST] = host; + } + header[USER_AGENT] = SLSClientManager::GetInstance()->GetUserAgent(); + header[DATE] = GetDateString(); + header[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; + header[CONTENT_LENGTH] = to_string(body.size()); + header[CONTENT_MD5] = CalcMD5(body); + header[X_LOG_APIVERSION] = LOG_API_VERSION; + header[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; + if (!compressType.empty()) { + header[X_LOG_COMPRESSTYPE] = compressType; + } + header[X_LOG_BODYRAWSIZE] = to_string(rawSize); + if (type == SLSClientManager::AuthType::ANONYMOUS) { + header[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; + } + + map parameterList; + string signature = GetUrlSignature(HTTP_POST, path, header, parameterList, body, accessKeySecret); + header[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; +} + +SLSResponse PostLogStoreLogs(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + const string& host, + bool httpsFlag, + const string& project, + const string& logstore, + const string& compressType, + RawDataType dataType, + const string& body, + size_t rawSize, + const string& shardHashKey) { + string path, query; + map header; + PreparePostLogStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + host, + false, // sync request always uses vip + project, + logstore, + compressType, + dataType, + body, + rawSize, + shardHashKey, + nullopt, // sync request does not support exactly-once + path, + query, + header); + HttpResponse response; + SendHttpRequest( + make_unique(HTTP_POST, httpsFlag, host, httpsFlag ? 443 : 80, path, query, header, body), + response); + return ParseHttpResponse(response); +} + +SLSResponse PostMetricStoreLogs(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + const string& host, + bool httpsFlag, + const string& project, + const string& logstore, + const string& compressType, + const string& body, + size_t rawSize) { + string path; + map header; + PreparePostMetricStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + host, + false, // sync request always uses vip + project, + logstore, + compressType, + body, + rawSize, + path, + header); + HttpResponse response; + SendHttpRequest(make_unique(HTTP_POST, httpsFlag, host, httpsFlag ? 443 : 80, path, "", header, body), + response); + return ParseHttpResponse(response); +} + +SLSResponse PutWebTracking(const string& host, + bool httpsFlag, + const string& logstore, + const string& compressType, + const string& body, + size_t rawSize) { + string path = LOGSTORES; + path.append("/").append(logstore).append("/track"); + + map header; + header[HOST] = host; + header[USER_AGENT] = SLSClientManager::GetInstance()->GetUserAgent(); + header[DATE] = GetDateString(); + header[CONTENT_LENGTH] = to_string(body.size()); + header[X_LOG_APIVERSION] = LOG_API_VERSION; + // header[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; + if (!compressType.empty()) { + header[X_LOG_COMPRESSTYPE] = compressType; + } + header[X_LOG_BODYRAWSIZE] = to_string(rawSize); + + HttpResponse response; + SendHttpRequest(make_unique(HTTP_POST, httpsFlag, host, httpsFlag ? 443 : 80, path, "", header, body), + response); + return ParseHttpResponse(response); } } // namespace logtail diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index c82a8a8cb1..a49f3123a9 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -16,24 +16,18 @@ #pragma once -#include #include -#include -#include -#include +#include #include #include -#include -#include -#include "sdk/Client.h" +#include "pipeline/queue/SenderQueueItem.h" +#include "plugin/flusher/sls/SLSResponse.h" namespace logtail { class SLSClientManager { public: - enum class EndpointSourceType { LOCAL, REMOTE }; - enum class EndpointSwitchPolicy { DESIGNATED_FIRST, DESIGNATED_LOCKED }; enum class AuthType { ANONYMOUS, AK }; virtual ~SLSClientManager() = default; @@ -42,131 +36,87 @@ class SLSClientManager { static SLSClientManager* GetInstance(); - void Init(); - void Stop(); + virtual void Init(); + virtual void Stop() {}; - EndpointSwitchPolicy GetServerSwitchPolicy() const { return mDataServerSwitchPolicy; } const std::string& GetUserAgent() const { return mUserAgent; } - void IncreaseAliuidReferenceCntForRegion(const std::string& region, const std::string& aliuid); - void DecreaseAliuidReferenceCntForRegion(const std::string& region, const std::string& aliuid); - - sdk::Client* GetClient(const std::string& region, const std::string& aliuid, bool createIfNotFound = true); - bool ResetClientEndpoint(const std::string& aliuid, const std::string& region, time_t curTime); - void CleanTimeoutClient(); virtual bool GetAccessKey(const std::string& aliuid, AuthType& type, std::string& accessKeyId, std::string& accessKeySecret); - virtual void UpdateAccessKeyStatus(const std::string& aliuid, bool success) {} - - void AddEndpointEntry(const std::string& region, - const std::string& endpoint, - bool isProxy, - const EndpointSourceType& endpointType); - void UpdateEndpointStatus(const std::string& region, - const std::string& endpoint, - bool status, - std::optional latency = std::optional()); - void ForceUpdateRealIp(const std::string& region); - void UpdateSendClientRealIp(sdk::Client* client, const std::string& region); - - std::string GetRegionFromEndpoint(const std::string& endpoint); // for backward compatibility - bool HasNetworkAvailable(); // TODO: remove this function + virtual bool UsingHttps(const std::string& region) const { return true; } protected: SLSClientManager() = default; virtual std::string GetRunningEnvironment(); - bool TryCurlEndpoint(const std::string& endpoint); + bool PingEndpoint(const std::string& host, const std::string& path); std::string mUserAgent; private: - enum class EndpointStatus { STATUS_OK_WITH_IP, STATUS_OK_WITH_ENDPOINT, STATUS_ERROR }; - - struct EndpointInfo { - bool mValid = true; - std::optional mLatencyMs; - bool mProxy = false; - - EndpointInfo(bool valid, bool proxy) : mValid(valid), mProxy(proxy) {} - - void UpdateInfo(bool valid, std::optional latency) { - mValid = valid; - mLatencyMs = latency; - } - }; - - struct RegionEndpointsInfo { - std::unordered_map mEndpointInfoMap; - std::string mDefaultEndpoint; - EndpointSourceType mDefaultEndpointType; - - bool AddDefaultEndpoint(const std::string& endpoint, const EndpointSourceType& endpointType, bool& isDefault); - bool AddEndpoint(const std::string& endpoint, bool status, bool proxy = false); - void UpdateEndpointInfo(const std::string& endpoint, - bool status, - std::optional latency, - bool createFlag = true); - void RemoveEndpoint(const std::string& endpoint); - std::string GetAvailableEndpointWithTopPriority() const; - }; - - struct RealIpInfo { - std::string mRealIp; - time_t mLastUpdateTime = 0; - bool mForceFlushFlag = false; - - void SetRealIp(const std::string& realIp) { - mRealIp = realIp; - mLastUpdateTime = time(NULL); - mForceFlushFlag = false; - } - }; - virtual void GenerateUserAgent(); - void InitEndpointSwitchPolicy(); - std::vector GetRegionAliuids(const std::string& region); - - void ResetClientPort(const std::string& region, sdk::Client* sendClient); - std::string GetAvailableEndpointWithTopPriority(const std::string& region) const; - - void ProbeNetworkThread(); - bool TestEndpoint(const std::string& region, const std::string& endpoint); - - void UpdateRealIpThread(); - EndpointStatus UpdateRealIp(const std::string& region, const std::string& endpoint); - void SetRealIp(const std::string& region, const std::string& ip); - - mutable std::mutex mRegionAliuidRefCntMapLock; - std::unordered_map> mRegionAliuidRefCntMap; - - mutable std::mutex mClientMapMux; - std::unordered_map, time_t>> mClientMap; - // int32_t mLastCheckSendClientTime; - - mutable std::mutex mRegionEndpointEntryMapLock; - std::unordered_map mRegionEndpointEntryMap; - EndpointSwitchPolicy mDataServerSwitchPolicy = EndpointSwitchPolicy::DESIGNATED_FIRST; - std::unique_ptr mProbeNetworkClient; - - std::future mProbeNetworkThreadRes; - mutable std::mutex mProbeNetworkThreadRunningMux; - bool mIsProbeNetworkThreadRunning = true; - - mutable std::mutex mRegionRealIpLock; - std::unordered_map mRegionRealIpMap; - std::unique_ptr mUpdateRealIpClient; - - std::future mUpdateRealIpThreadRes; - mutable std::mutex mUpdateRealIpThreadRunningMux; - bool mIsUpdateRealIpThreadRunning = true; - - mutable std::condition_variable mStopCV; #ifdef APSARA_UNIT_TEST_MAIN - friend class FlusherSLSUnittest; + friend class SLSClientManagerUnittest; #endif }; +void PreparePostLogStoreLogsRequest(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + const std::string& host, + bool isHostIp, + const std::string& project, + const std::string& logstore, + const std::string& compressType, + RawDataType dataType, + const std::string& body, + size_t rawSize, + const std::string& shardHashKey, + std::optional seqId, + std::string& path, + std::string& query, + std::map& header); +void PreparePostMetricStoreLogsRequest(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + const std::string& host, + bool isHostIp, + const std::string& project, + const std::string& logstore, + const std::string& compressType, + const std::string& body, + size_t rawSize, + std::string& path, + std::map& header); +SLSResponse PostLogStoreLogs(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + const std::string& host, + bool httpsFlag, + const std::string& project, + const std::string& logstore, + const std::string& compressType, + RawDataType dataType, + const std::string& body, + size_t rawSize, + const std::string& shardHashKey); +SLSResponse PostMetricStoreLogs(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + const std::string& host, + bool httpsFlag, + const std::string& project, + const std::string& logstore, + const std::string& compressType, + const std::string& body, + size_t rawSize); +SLSResponse PutWebTracking(const std::string& host, + bool httpsFlag, + const std::string& logstore, + const std::string& compressType, + const std::string& body, + size_t rawSize); + } // namespace logtail diff --git a/core/plugin/flusher/sls/SLSConstant.cpp b/core/plugin/flusher/sls/SLSConstant.cpp new file mode 100644 index 0000000000..d81dbf284e --- /dev/null +++ b/core/plugin/flusher/sls/SLSConstant.cpp @@ -0,0 +1,98 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "plugin/flusher/sls/SLSConstant.h" + +using namespace std; + +namespace logtail { + +const string LOGSTORES = "/logstores"; +const string METRICSTORES = "/prometheus"; +const string HEALTH = "/health"; + +const string LOGTAIL_USER_AGENT = "ali-log-logtail"; + +const string CONTENT_MD5 = "Content-MD5"; + +const string LOG_HEADER_PREFIX = "x-log-"; +const string LOG_OLD_HEADER_PREFIX = "x-sls-"; +const string ACS_HEADER_PREFIX = "x-acs-"; +const string X_LOG_KEYPROVIDER = "x-log-keyprovider"; +const string X_LOG_APIVERSION = "x-log-apiversion"; +const string X_LOG_COMPRESSTYPE = "x-log-compresstype"; +const string X_LOG_BODYRAWSIZE = "x-log-bodyrawsize"; +const string X_LOG_SIGNATUREMETHOD = "x-log-signaturemethod"; +const string X_LOG_MODE = "x-log-mode"; +const string X_LOG_HOSTIP = "x-log-hostip"; +const string X_LOG_REQUEST_ID = "x-log-requestid"; +const string HMAC_SHA1 = "hmac-sha1"; +const string LOG_HEADSIGNATURE_PREFIX = "LOG "; +const string LOG_API_VERSION = "0.6.0"; +const string LOG_MODE_BATCH_GROUP = "batch_group"; + +const string MD5_SHA1_SALT_KEYPROVIDER = "md5-sha1-salt"; + +const string LOGE_REQUEST_ERROR = "RequestError"; +const string LOGE_INVALID_HOST = "InvalidHost"; +const string LOGE_UNKNOWN_ERROR = "UnknownError"; +const string LOGE_NOT_IMPLEMENTED = "NotImplemented"; +const string LOGE_SERVER_BUSY = "ServerBusy"; +const string LOGE_INTERNAL_SERVER_ERROR = "InternalServerError"; +const string LOGE_RESPONSE_SIG_ERROR = "ResponseSignatureError"; +const string LOGE_PARAMETER_INVALID = "ParameterInvalid"; +const string LOGE_MISSING_PARAMETER = "MissingParameter"; +const string LOGE_INVALID_METHOD = "InvalidMethod"; +const string LOGE_BAD_RESPONSE = "BadResponse"; +const string LOGE_UNAUTHORIZED = "Unauthorized"; +const string LOGE_QUOTA_EXCEED = "ExceedQuota"; +const string LOGE_REQUEST_TIMEOUT = "RequestTimeout"; +const string LOGE_CLIENT_OPERATION_TIMEOUT = "ClientOpertaionTimeout"; +const string LOGE_CLIENT_NETWORK_ERROR = "ClientNetworkError"; +const string LOGE_USER_NOT_EXIST = "UserNotExist"; +const string LOGE_CATEGORY_NOT_EXIST = "CategoryNotExist"; +const string LOGE_TOPIC_NOT_EXIST = "TopicNotExist"; +const string LOGE_POST_BODY_INVALID = "PostBodyInvalid"; +const string LOGE_INVALID_CONTENTTYPE = "InvalidContentType"; +const string LOGE_INVALID_CONTENLENGTH = "InvalidContentLength"; +const string LOGE_INVALID_APIVERSION = "InvalidAPIVersion"; +const string LOGE_PROJECT_NOT_EXIST = "ProjectNotExist"; +const string LOGE_MACHINEGROUP_NOT_EXIST = "MachineGroupNotExist"; +const string LOGE_MACHINEGROUP_ALREADY_EXIST = "MachineGroupAlreadyExist"; +const string LOGE_CONFIG_NOT_EXIST = "ConfigNotExist"; +const string LOGE_CONFIG_ALREADY_EXIST = "ConfigAlreadyExist"; +const string LOGE_LOGSTORE_NOT_EXIST = "LogStoreNotExist"; +const string LOGE_INVALID_ACCESSKEYID = "InvalidAccessKeyId"; +const string LOGE_SIGNATURE_NOT_MATCH = "SignatureNotMatch"; +const string LOGE_PROJECT_FORBIDDEN = "ProjectForbidden"; +const string LOGE_WRITE_QUOTA_EXCEED = "WriteQuotaExceed"; +const string LOGE_READ_QUOTA_EXCEED = "ReadQuotaExceed"; +const string LOGE_REQUEST_TIME_EXPIRED = "RequestTimeExpired"; +const string LOGE_INVALID_REQUEST_TIME = "InvalidRequestTime"; +const string LOGE_POST_BODY_TOO_LARGE = "PostBodyTooLarge"; +const string LOGE_INVALID_TIME_RANGE = "InvalidTimeRange"; +const string LOGE_INVALID_REVERSE = "InvalidReverse"; +const string LOGE_LOGSTORE_WITHOUT_SHARD = "LogStoreWithoutShard"; +const string LOGE_SHARD_WRITE_QUOTA_EXCEED = "ShardWriteQuotaExceed"; +const string LOGE_SHARD_READ_QUOTA_EXCEED = "ShardReadQuotaExceed"; +const string LOGE_INVALID_SEQUENCE_ID = "InvalidSequenceId"; +const string LOGE_NOT_SUPPORTED_ACCEPT_CONTENT_TYPE = "InvalidAcceptContentType"; +const string LOGE_NOT_SUPPORTED_ACCEPT_ENCODING = "InvalidAcceptEncoding"; +const string LOGE_SHARD_NOT_EXIST = "ShardNotExist"; +const string LOGE_INVALID_CURSOR = "InvalidCursor"; + +const string LOG_ERROR_CODE = "errorCode"; +const string LOG_ERROR_MESSAGE = "errorMessage"; + +} // namespace logtail diff --git a/core/plugin/flusher/sls/SLSConstant.h b/core/plugin/flusher/sls/SLSConstant.h new file mode 100644 index 0000000000..5874d5f2ec --- /dev/null +++ b/core/plugin/flusher/sls/SLSConstant.h @@ -0,0 +1,101 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace logtail { + +extern const std::string LOGSTORES; +extern const std::string METRICSTORES; +extern const std::string HEALTH; + +extern const std::string CONTENT_MD5; + +extern const std::string LOGTAIL_USER_AGENT; + +extern const std::string LOG_HEADER_PREFIX; +extern const std::string LOG_OLD_HEADER_PREFIX; +extern const std::string ACS_HEADER_PREFIX; +extern const std::string X_LOG_KEYPROVIDER; +extern const std::string X_LOG_APIVERSION; +extern const std::string X_LOG_COMPRESSTYPE; +extern const std::string X_LOG_BODYRAWSIZE; +extern const std::string X_LOG_SIGNATUREMETHOD; +extern const std::string X_LOG_MODE; +extern const std::string X_LOG_HOSTIP; +extern const std::string X_LOG_REQUEST_ID; + +extern const std::string LOG_HEADSIGNATURE_PREFIX; +extern const std::string LOG_API_VERSION; +extern const std::string LOG_MODE_BATCH_GROUP; +extern const std::string HMAC_SHA1; +extern const std::string MD5_SHA1_SALT_KEYPROVIDER; + +extern const std::string LOGE_REQUEST_ERROR; +extern const std::string LOGE_INVALID_HOST; +extern const std::string LOGE_UNKNOWN_ERROR; +extern const std::string LOGE_NOT_IMPLEMENTED; +extern const std::string LOGE_SERVER_BUSY; +extern const std::string LOGE_INTERNAL_SERVER_ERROR; +extern const std::string LOGE_RESPONSE_SIG_ERROR; +extern const std::string LOGE_PARAMETER_INVALID; +extern const std::string LOGE_MISSING_PARAMETER; +extern const std::string LOGE_INVALID_METHOD; +extern const std::string LOGE_INVALID_CONTENTTYPE; +extern const std::string LOGE_INVALID_CONTENTLENGTH; +extern const std::string LOGE_BAD_RESPONSE; +extern const std::string LOGE_UNAUTHORIZED; +extern const std::string LOGE_QUOTA_EXCEED; +extern const std::string LOGE_REQUEST_TIMEOUT; +extern const std::string LOGE_CLIENT_OPERATION_TIMEOUT; +extern const std::string LOGE_CLIENT_NETWORK_ERROR; +extern const std::string LOGE_USER_NOT_EXIST; +extern const std::string LOGE_CATEGORY_NOT_EXIST; +extern const std::string LOGE_TOPIC_NOT_EXIST; +extern const std::string LOGE_POST_BODY_INVALID; +extern const std::string LOGE_INVALID_HOST; +extern const std::string LOGE_INVALID_APIVERSION; +extern const std::string LOGE_PROJECT_NOT_EXIST; +extern const std::string LOGE_MACHINEGROUP_NOT_EXIST; +extern const std::string LOGE_MACHINEGROUP_ALREADY_EXIST; +extern const std::string LOGE_CONFIG_NOT_EXIST; +extern const std::string LOGE_CONFIG_ALREADY_EXIST; +extern const std::string LOGE_LOGSTORE_NOT_EXIST; +extern const std::string LOGE_INVALID_ACCESSKEYID; +extern const std::string LOGE_SIGNATURE_NOT_MATCH; +extern const std::string LOGE_PROJECT_FORBIDDEN; +extern const std::string LOGE_WRITE_QUOTA_EXCEED; +extern const std::string LOGE_READ_QUOTA_EXCEED; +extern const std::string LOGE_REQUEST_TIME_EXPIRED; +extern const std::string LOGE_INVALID_REQUEST_TIME; +extern const std::string LOGE_POST_BODY_TOO_LARGE; +extern const std::string LOGE_INVALID_TIME_RANGE; +extern const std::string LOGE_INVALID_REVERSE; +extern const std::string LOGE_LOGSTORE_WITHOUT_SHARD; +extern const std::string LOGE_INVALID_SEQUENCE_ID; +extern const std::string LOGE_NOT_SUPPORTED_ACCEPT_CONTENT_TYPE; +extern const std::string LOGE_NOT_SUPPORTED_ACCEPT_ENCODING; +extern const std::string LOGE_SHARD_NOT_EXIST; +extern const std::string LOGE_INVALID_CURSOR; +extern const std::string LOGE_SHARD_WRITE_QUOTA_EXCEED; +extern const std::string LOGE_SHARD_READ_QUOTA_EXCEED; + +extern const std::string LOG_ERROR_CODE; +extern const std::string LOG_ERROR_MESSAGE; + +} // namespace logtail diff --git a/core/plugin/flusher/sls/SLSResponse.cpp b/core/plugin/flusher/sls/SLSResponse.cpp index d9fa405479..e13847618a 100644 --- a/core/plugin/flusher/sls/SLSResponse.cpp +++ b/core/plugin/flusher/sls/SLSResponse.cpp @@ -14,31 +14,80 @@ #include "plugin/flusher/sls/SLSResponse.h" +#include + +#include "app_config/AppConfig.h" #include "common/ErrorUtil.h" #include "common/StringTools.h" #include "common/TimeUtil.h" #include "logger/Logger.h" -#include "sdk/Common.h" -#include "sdk/Exception.h" -#include "sdk/Result.h" +#include "plugin/flusher/sls/SLSConstant.h" +#include "Exception.h" using namespace std; +using namespace logtail::sdk; namespace logtail { +void ExtractJsonResult(const string& response, rapidjson::Document& document) { + document.Parse(response.c_str()); + if (document.HasParseError()) { + throw JsonException("ParseException", "Fail to parse from json string"); + } +} + +void JsonMemberCheck(const rapidjson::Value& value, const char* name) { + if (!value.IsObject()) { + throw JsonException("InvalidObjectException", "response is not valid JSON object"); + } + if (!value.HasMember(name)) { + throw JsonException("NoMemberException", string("Member ") + name + " does not exist"); + } +} + +void ExtractJsonResult(const rapidjson::Value& value, const char* name, string& dst) { + JsonMemberCheck(value, name); + if (value[name].IsString()) { + dst = value[name].GetString(); + } else { + throw JsonException("ValueTypeException", string("Member ") + name + " is not string type"); + } +} + +void ErrorCheck(const string& response, const string& requestId, const int32_t httpCode) { + rapidjson::Document document; + try { + ExtractJsonResult(response, document); + + string errorCode; + ExtractJsonResult(document, LOG_ERROR_CODE.c_str(), errorCode); + + string errorMessage; + ExtractJsonResult(document, LOG_ERROR_MESSAGE.c_str(), errorMessage); + + throw LOGException(errorCode, errorMessage, requestId, httpCode); + } catch (JsonException& e) { + if (httpCode >= 500) { + throw LOGException(LOGE_INTERNAL_SERVER_ERROR, response, requestId, httpCode); + } else { + throw LOGException(LOGE_BAD_RESPONSE, string("Unextractable error:") + response, requestId, httpCode); + } + } +} + bool SLSResponse::Parse(const HttpResponse& response) { - const auto iter = response.GetHeader().find(sdk::X_LOG_REQUEST_ID); + const auto iter = response.GetHeader().find(X_LOG_REQUEST_ID); if (iter != response.GetHeader().end()) { mRequestId = iter->second; } mStatusCode = response.GetStatusCode(); if (mStatusCode == 0) { - mErrorCode = sdk::LOGE_REQUEST_TIMEOUT; + mErrorCode = LOGE_REQUEST_TIMEOUT; mErrorMsg = "Request timeout"; } else if (mStatusCode != 200) { try { - sdk::ErrorCheck(*response.GetBody(), mRequestId, response.GetStatusCode()); + ErrorCheck(*response.GetBody(), mRequestId, response.GetStatusCode()); } catch (sdk::LOGException& e) { mErrorCode = e.GetErrorCode(); mErrorMsg = e.GetMessage_(); @@ -47,8 +96,30 @@ bool SLSResponse::Parse(const HttpResponse& response) { return true; } +SLSResponse ParseHttpResponse(const HttpResponse& response) { + SLSResponse slsResponse; + if (AppConfig::GetInstance()->IsResponseVerificationEnabled() && !IsSLSResponse(response)) { + slsResponse.mStatusCode = 0; + slsResponse.mErrorCode = LOGE_REQUEST_ERROR; + slsResponse.mErrorMsg = "invalid response body"; + } else { + slsResponse.Parse(response); + + if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) { + static uint32_t sCount = 0; + if (sCount++ % 10000 == 0 || slsResponse.mErrorCode == LOGE_REQUEST_TIME_EXPIRED) { + time_t serverTime = GetServerTime(response); + if (serverTime > 0) { + UpdateTimeDelta(serverTime); + } + } + } + } + return slsResponse; +} + bool IsSLSResponse(const HttpResponse& response) { - const auto iter = response.GetHeader().find(sdk::X_LOG_REQUEST_ID); + const auto iter = response.GetHeader().find(X_LOG_REQUEST_ID); if (iter == response.GetHeader().end()) { return false; } diff --git a/core/plugin/flusher/sls/SLSResponse.h b/core/plugin/flusher/sls/SLSResponse.h index 1388c7ef56..887c6f07f1 100644 --- a/core/plugin/flusher/sls/SLSResponse.h +++ b/core/plugin/flusher/sls/SLSResponse.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include "common/http/HttpResponse.h" @@ -32,6 +33,7 @@ struct SLSResponse { bool Parse(const HttpResponse& response); }; +SLSResponse ParseHttpResponse(const HttpResponse& response); bool IsSLSResponse(const HttpResponse& response); time_t GetServerTime(const HttpResponse& response); diff --git a/core/plugin/flusher/sls/SLSUtil.cpp b/core/plugin/flusher/sls/SLSUtil.cpp new file mode 100644 index 0000000000..21e30d4ce6 --- /dev/null +++ b/core/plugin/flusher/sls/SLSUtil.cpp @@ -0,0 +1,307 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "plugin/flusher/sls/SLSUtil.h" + +#include "app_config/AppConfig.h" +#include "common/EncodingUtil.h" +#include "common/HashUtil.h" +#include "common/TimeUtil.h" +#include "common/http/Constant.h" +#include "plugin/flusher/sls/SLSConstant.h" + +using namespace std; + +namespace logtail { + +static string DATE_FORMAT_RFC822 = "%a, %d %b %Y %H:%M:%S GMT"; + +#define BIT_COUNT_WORDS 2 +#define BIT_COUNT_BYTES (BIT_COUNT_WORDS * sizeof(uint32_t)) + +/* + * define the rotate left (circular left shift) operation + */ +#define rotl(v, b) (((v) << (b)) | ((v) >> (32 - (b)))) + +/* + * Define the basic SHA-1 functions F1 ~ F4. Note that the exclusive-OR + * operation (^) in F1 and F3 may be replaced by a bitwise OR operation + * (|), which produce identical results. + * + * F1 is used in ROUND 0~19, F2 is used in ROUND 20~39 + * F3 is used in ROUND 40~59, F4 is used in ROUND 60~79 + */ +#define F1(B, C, D) (((B) & (C)) ^ (~(B) & (D))) +#define F2(B, C, D) ((B) ^ (C) ^ (D)) +#define F3(B, C, D) (((B) & (C)) ^ ((B) & (D)) ^ ((C) & (D))) +#define F4(B, C, D) ((B) ^ (C) ^ (D)) + +/* + * Use different K in different ROUND + */ +#define K00_19 0x5A827999 +#define K20_39 0x6ED9EBA1 +#define K40_59 0x8F1BBCDC +#define K60_79 0xCA62C1D6 + +/* + * Another implementation of the ROUND transformation: + * (here the T is a temp variable) + * For t=0 to 79: + * { + * T=rotl(A,5)+Func(B,C,D)+K+W[t]+E; + * E=D; D=C; C=rotl(B,30); B=A; A=T; + * } + */ +#define ROUND(t, A, B, C, D, E, Func, K) \ + E += rotl(A, 5) + Func(B, C, D) + W[t] + K; \ + B = rotl(B, 30); + +#define ROUND5(t, Func, K) \ + ROUND(t, A, B, C, D, E, Func, K); \ + ROUND(t + 1, E, A, B, C, D, Func, K); \ + ROUND(t + 2, D, E, A, B, C, Func, K); \ + ROUND(t + 3, C, D, E, A, B, Func, K); \ + ROUND(t + 4, B, C, D, E, A, Func, K) + +#define ROUND20(t, Func, K) \ + ROUND5(t, Func, K); \ + ROUND5(t + 5, Func, K); \ + ROUND5(t + 10, Func, K); \ + ROUND5(t + 15, Func, K) + +/* + * Define constant of the initial vector + */ +const uint32_t SHA1::IV[SHA1_DIGEST_WORDS] = {0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0}; + +/* + * the message must be the big-endian32 (or left-most word) + * before calling the transform() function. + */ +const static uint32_t iii = 1; +const static bool littleEndian = *(uint8_t*)&iii != 0; + +inline uint32_t littleEndianToBig(uint32_t d) { + uint8_t* data = (uint8_t*)&d; + return data[0] << 24 | data[1] << 16 | data[2] << 8 | data[3]; +} + +inline void make_big_endian32(uint32_t* data, unsigned n) { + if (!littleEndian) { + return; + } + for (; n > 0; ++data, --n) { + *data = littleEndianToBig(*data); + } +} + +inline size_t min(size_t a, size_t b) { + return a < b ? a : b; +} + +void SHA1::transform() { + uint32_t W[80]; + memcpy(W, M, SHA1_INPUT_BYTES); + memset((uint8_t*)W + SHA1_INPUT_BYTES, 0, sizeof(W) - SHA1_INPUT_BYTES); + for (unsigned t = 16; t < 80; t++) { + W[t] = rotl(W[t - 16] ^ W[t - 14] ^ W[t - 8] ^ W[t - 3], 1); + } + + uint32_t A = H[0]; + uint32_t B = H[1]; + uint32_t C = H[2]; + uint32_t D = H[3]; + uint32_t E = H[4]; + + ROUND20(0, F1, K00_19); + ROUND20(20, F2, K20_39); + ROUND20(40, F3, K40_59); + ROUND20(60, F4, K60_79); + + H[0] += A; + H[1] += B; + H[2] += C; + H[3] += D; + H[4] += E; +} + +void SHA1::add(const uint8_t* data, size_t data_len) { + unsigned mlen = (unsigned)((bits >> 3) % SHA1_INPUT_BYTES); + bits += (uint64_t)data_len << 3; + unsigned use = (unsigned)min((size_t)(SHA1_INPUT_BYTES - mlen), data_len); + memcpy(M + mlen, data, use); + mlen += use; + + while (mlen == SHA1_INPUT_BYTES) { + data_len -= use; + data += use; + make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS); + transform(); + use = (unsigned)min((size_t)SHA1_INPUT_BYTES, data_len); + memcpy(M, data, use); + mlen = use; + } +} + +uint8_t* SHA1::result() { + unsigned mlen = (unsigned)((bits >> 3) % SHA1_INPUT_BYTES), padding = SHA1_INPUT_BYTES - mlen; + M[mlen++] = 0x80; + if (padding > BIT_COUNT_BYTES) { + memset(M + mlen, 0x00, padding - BIT_COUNT_BYTES); + make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS - BIT_COUNT_WORDS); + } else { + memset(M + mlen, 0x00, SHA1_INPUT_BYTES - mlen); + make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS); + transform(); + memset(M, 0x00, SHA1_INPUT_BYTES - BIT_COUNT_BYTES); + } + + uint64_t temp = littleEndian ? bits << 32 | bits >> 32 : bits; + memcpy(M + SHA1_INPUT_BYTES - BIT_COUNT_BYTES, &temp, BIT_COUNT_BYTES); + transform(); + make_big_endian32(H, SHA1_DIGEST_WORDS); + return (uint8_t*)H; +} + +template +inline void axor(T* p1, const T* p2, size_t len) { + for (; len != 0; --len) + *p1++ ^= *p2++; +} + +HMAC::HMAC(const uint8_t* key, size_t lkey) { + init(key, lkey); +} + +void HMAC::init(const uint8_t* key, size_t lkey) { + in.init(); + out.init(); + + uint8_t ipad[SHA1_INPUT_BYTES]; + uint8_t opad[SHA1_INPUT_BYTES]; + memset(ipad, 0x36, sizeof(ipad)); + memset(opad, 0x5c, sizeof(opad)); + + if (lkey <= SHA1_INPUT_BYTES) { + axor(ipad, key, lkey); + axor(opad, key, lkey); + } else { + SHA1 tmp; + tmp.add(key, lkey); + const uint8_t* key2 = tmp.result(); + axor(ipad, key2, SHA1_DIGEST_BYTES); + axor(opad, key2, SHA1_DIGEST_BYTES); + } + + in.add((uint8_t*)ipad, sizeof(ipad)); + out.add((uint8_t*)opad, sizeof(opad)); +} + +string GetDateString() { + time_t now_time; + time(&now_time); + if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) { + now_time += GetTimeDelta(); + } + char buffer[128] = {'\0'}; + tm timeInfo; +#if defined(__linux__) + gmtime_r(&now_time, &timeInfo); +#elif defined(_MSC_VER) + gmtime_s(&timeInfo, &now_time); +#endif + strftime(buffer, 128, DATE_FORMAT_RFC822.c_str(), &timeInfo); + return string(buffer); +} + +static bool StartWith(const std::string& input, const std::string& pattern) { + if (input.length() < pattern.length()) { + return false; + } + + size_t i = 0; + while (i < pattern.length() && input[i] == pattern[i]) { + i++; + } + + return i == pattern.length(); +} + +static std::string CalcSHA1(const std::string& message, const std::string& key) { + HMAC hmac(reinterpret_cast(key.data()), key.size()); + hmac.add(reinterpret_cast(message.data()), message.size()); + return string(reinterpret_cast(hmac.result()), SHA1_DIGEST_BYTES); +} + +string GetUrlSignature(const string& httpMethod, + const string& operationType, + map& httpHeader, + const map& parameterList, + const string& content, + const string& signKey) { + string contentMd5; + string signature; + string osstream; + if (!content.empty()) { + contentMd5 = CalcMD5(content); + } + string contentType; + map::iterator iter = httpHeader.find(CONTENT_TYPE); + if (iter != httpHeader.end()) { + contentType = iter->second; + } + map endingMap; + osstream.append(httpMethod); + osstream.append("\n"); + osstream.append(contentMd5); + osstream.append("\n"); + osstream.append(contentType); + osstream.append("\n"); + osstream.append(httpHeader[DATE]); + osstream.append("\n"); + for (map::const_iterator iter = httpHeader.begin(); iter != httpHeader.end(); ++iter) { + if (StartWith(iter->first, LOG_OLD_HEADER_PREFIX)) { + string key = iter->first; + endingMap.insert(make_pair(key.replace(0, LOG_OLD_HEADER_PREFIX.size(), LOG_HEADER_PREFIX), iter->second)); + } else if (StartWith(iter->first, LOG_HEADER_PREFIX) || StartWith(iter->first, ACS_HEADER_PREFIX)) { + endingMap.insert(make_pair(iter->first, iter->second)); + } + } + for (map::const_iterator it = endingMap.begin(); it != endingMap.end(); ++it) { + osstream.append(it->first); + osstream.append(":"); + osstream.append(it->second); + osstream.append("\n"); + } + osstream.append(operationType); + if (parameterList.size() > 0) { + osstream.append("?"); + for (map::const_iterator iter = parameterList.begin(); iter != parameterList.end(); ++iter) { + if (iter != parameterList.begin()) { + osstream.append("&"); + } + osstream.append(iter->first); + osstream.append("="); + osstream.append(iter->second); + } + } + + signature = Base64Enconde(CalcSHA1(osstream, signKey)); + + return signature; +} + +} // namespace logtail diff --git a/core/plugin/flusher/sls/SLSUtil.h b/core/plugin/flusher/sls/SLSUtil.h new file mode 100644 index 0000000000..a30288c90a --- /dev/null +++ b/core/plugin/flusher/sls/SLSUtil.h @@ -0,0 +1,82 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace logtail { + +#define SHA1_INPUT_WORDS 16 +#define SHA1_DIGEST_WORDS 5 +#define SHA1_INPUT_BYTES (SHA1_INPUT_WORDS * sizeof(uint32_t)) +#define SHA1_DIGEST_BYTES (SHA1_DIGEST_WORDS * sizeof(uint32_t)) + +class SHA1 { +public: + SHA1() : bits(0) { memcpy(H, IV, sizeof(H)); } + SHA1(const SHA1& s) { + bits = s.bits; + memcpy(H, s.H, sizeof(H)); + memcpy(M, s.M, sizeof(M)); + } + void init() { + bits = 0; + memcpy(H, IV, sizeof(H)); + } + void add(const uint8_t* data, size_t len); + uint8_t* result(); + +private: + uint64_t bits; + uint32_t H[SHA1_DIGEST_WORDS]; + uint8_t M[SHA1_INPUT_BYTES]; + + static const uint32_t IV[SHA1_DIGEST_WORDS]; + void transform(); +}; + +class HMAC { +public: + HMAC(const uint8_t* key, size_t lkey); + HMAC(const HMAC& hm) : in(hm.in), out(hm.out) {} + + void init(const uint8_t* key, size_t lkey); + + void add(const uint8_t* data, size_t len) { in.add(data, len); } + + uint8_t* result() { + out.add(in.result(), SHA1_DIGEST_BYTES); + return out.result(); + } + +private: + SHA1 in, out; +}; + +std::string GetDateString(); + +std::string GetUrlSignature(const std::string& httpMethod, + const std::string& operationType, + std::map& httpHeader, + const std::map& parameterList, + const std::string& content, + const std::string& signKey); + +} // namespace logtail diff --git a/core/plugin/flusher/sls/SendResult.cpp b/core/plugin/flusher/sls/SendResult.cpp index 955bff6a52..1f168ab261 100644 --- a/core/plugin/flusher/sls/SendResult.cpp +++ b/core/plugin/flusher/sls/SendResult.cpp @@ -14,23 +14,23 @@ #include "plugin/flusher/sls/SendResult.h" -#include "sdk/Common.h" +#include "plugin/flusher/sls/SLSConstant.h" namespace logtail { SendResult ConvertErrorCode(const std::string& errorCode) { - if (errorCode == sdk::LOGE_REQUEST_ERROR || errorCode == sdk::LOGE_CLIENT_OPERATION_TIMEOUT - || errorCode == sdk::LOGE_REQUEST_TIMEOUT) { + if (errorCode == LOGE_REQUEST_ERROR || errorCode == LOGE_CLIENT_OPERATION_TIMEOUT + || errorCode == LOGE_REQUEST_TIMEOUT) { return SEND_NETWORK_ERROR; - } else if (errorCode == sdk::LOGE_SERVER_BUSY || errorCode == sdk::LOGE_INTERNAL_SERVER_ERROR) { + } else if (errorCode == LOGE_SERVER_BUSY || errorCode == LOGE_INTERNAL_SERVER_ERROR) { return SEND_SERVER_ERROR; - } else if (errorCode == sdk::LOGE_WRITE_QUOTA_EXCEED || errorCode == sdk::LOGE_SHARD_WRITE_QUOTA_EXCEED) { + } else if (errorCode == LOGE_WRITE_QUOTA_EXCEED || errorCode == LOGE_SHARD_WRITE_QUOTA_EXCEED) { return SEND_QUOTA_EXCEED; - } else if (errorCode == sdk::LOGE_UNAUTHORIZED) { + } else if (errorCode == LOGE_UNAUTHORIZED) { return SEND_UNAUTHORIZED; - } else if (errorCode == sdk::LOGE_INVALID_SEQUENCE_ID) { + } else if (errorCode == LOGE_INVALID_SEQUENCE_ID) { return SEND_INVALID_SEQUENCE_ID; - } else if (errorCode == sdk::LOGE_PARAMETER_INVALID) { + } else if (errorCode == LOGE_PARAMETER_INVALID) { return SEND_PARAMETER_INVALID; } else { return SEND_DISCARD_ERROR; diff --git a/core/plugin/processor/ProcessorDesensitizeNative.cpp b/core/plugin/processor/ProcessorDesensitizeNative.cpp index 5cef5e64c9..47c94bfcf8 100644 --- a/core/plugin/processor/ProcessorDesensitizeNative.cpp +++ b/core/plugin/processor/ProcessorDesensitizeNative.cpp @@ -20,7 +20,7 @@ #include "models/LogEvent.h" #include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" -#include "sdk/Common.h" +#include "common/HashUtil.h" namespace logtail { @@ -230,7 +230,7 @@ void ProcessorDesensitizeNative::CastOneSensitiveWord(std::string* value) { // add : xxxx, psw destStr.append(pVal->substr(beginPos, beginOffset - beginPos)); // md5: 123abc - destStr.append(sdk::CalcMD5(pVal->substr(beginOffset, endOffset - beginOffset))); + destStr.append(CalcMD5(pVal->substr(beginOffset, endOffset - beginOffset))); beginPos = endOffset; // refine for : xxxx. psw=123abc if (endOffset >= maxSize) { diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index 052959bb15..cdb4ba46bf 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -26,14 +26,14 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "common/http/AsynCurlRunner.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/timer/Timer.h" #include "logger/Logger.h" #include "monitor/metric_constants/MetricConstants.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "prometheus/Constants.h" #include "prometheus/Utils.h" -#include "sdk/Common.h" -#include "sdk/Exception.h" using namespace std; @@ -49,7 +49,6 @@ PrometheusInputRunner::PrometheusInputRunner() mPodName(STRING_FLAG(_pod_name_)), mEventPool(true), mUnRegisterMs(0) { - mClient = std::make_unique(); mTimer = std::make_shared(); // self monitor @@ -151,17 +150,18 @@ void PrometheusInputRunner::Init() { int retry = 0; while (mIsThreadRunning.load()) { ++retry; - sdk::HttpMessage httpResponse = SendRegisterMessage(prometheus::REGISTER_COLLECTOR_PATH); - if (httpResponse.statusCode != 200) { + auto httpResponse = SendRegisterMessage(prometheus::REGISTER_COLLECTOR_PATH); + if (httpResponse.GetStatusCode() != 200) { mPromRegisterRetryTotal->Add(1); if (retry % 10 == 0) { - LOG_INFO(sLogger, ("register failed, retried", retry)("statusCode", httpResponse.statusCode)); + LOG_INFO(sLogger, + ("register failed, retried", retry)("statusCode", httpResponse.GetStatusCode())); } } else { // register success // response will be { "unRegisterMs": 30000 } - if (!httpResponse.content.empty()) { - string responseStr = httpResponse.content; + if (!httpResponse.GetBody()->empty()) { + string responseStr = *httpResponse.GetBody(); string errMsg; Json::Value responseJson; if (!ParseJsonTable(responseStr, responseJson, errMsg)) { @@ -222,9 +222,9 @@ void PrometheusInputRunner::Stop() { auto res = std::async(launch::async, [this]() { std::lock_guard lock(mRegisterMutex); for (int retry = 0; retry < 3; ++retry) { - sdk::HttpMessage httpResponse = SendRegisterMessage(prometheus::UNREGISTER_COLLECTOR_PATH); - if (httpResponse.statusCode != 200) { - LOG_ERROR(sLogger, ("unregister failed, statusCode", httpResponse.statusCode)); + auto httpResponse = SendRegisterMessage(prometheus::UNREGISTER_COLLECTOR_PATH); + if (httpResponse.GetStatusCode() != 200) { + LOG_ERROR(sLogger, ("unregister failed, statusCode", httpResponse.GetStatusCode())); } else { LOG_INFO(sLogger, ("Unregister Success", mPodName)); mPromRegisterState->Set(0); @@ -242,29 +242,18 @@ bool PrometheusInputRunner::HasRegisteredPlugins() const { return !mTargetSubscriberSchedulerMap.empty(); } -sdk::HttpMessage PrometheusInputRunner::SendRegisterMessage(const string& url) const { - map httpHeader; - httpHeader[sdk::X_LOG_REQUEST_ID] = prometheus::PROMETHEUS_PREFIX + mPodName; - sdk::HttpMessage httpResponse; - httpResponse.header[sdk::X_LOG_REQUEST_ID] = prometheus::PROMETHEUS_PREFIX + mPodName; +HttpResponse PrometheusInputRunner::SendRegisterMessage(const string& url) const { + HttpResponse httpResponse; #ifdef APSARA_UNIT_TEST_MAIN - httpResponse.statusCode = 200; + httpResponse.SetStatusCode(200); return httpResponse; #endif - try { - mClient->Send(sdk::HTTP_GET, - mServiceHost, - mServicePort, - url, - "pod_name=" + mPodName, - httpHeader, - "", - 10, - httpResponse, - "", - false); - } catch (const sdk::LOGException& e) { - LOG_ERROR(sLogger, ("curl error", e.what())("url", url)("pod_name", mPodName)); + map httpHeader; + if (!SendHttpRequest( + make_unique( + HTTP_GET, false, mServiceHost, mServicePort, url, "pod_name=" + mPodName, httpHeader, "", 10), + httpResponse)) { + LOG_ERROR(sLogger, ("curl error", "")("url", url)("pod_name", mPodName)); } return httpResponse; } diff --git a/core/prometheus/PrometheusInputRunner.h b/core/prometheus/PrometheusInputRunner.h index 996caf163e..b8e7719171 100644 --- a/core/prometheus/PrometheusInputRunner.h +++ b/core/prometheus/PrometheusInputRunner.h @@ -21,12 +21,11 @@ #include #include "common/Lock.h" +#include "common/http/HttpResponse.h" #include "common/timer/Timer.h" #include "monitor/metric_models/MetricTypes.h" #include "prometheus/schedulers/TargetSubscriberScheduler.h" #include "runner/InputRunner.h" -#include "sdk/Common.h" -#include "sdk/CurlImp.h" namespace logtail { @@ -56,7 +55,7 @@ class PrometheusInputRunner : public InputRunner { private: PrometheusInputRunner(); - sdk::HttpMessage SendRegisterMessage(const std::string& url) const; + HttpResponse SendRegisterMessage(const std::string& url) const; void CancelAllTargetSubscriber(); void SubscribeOnce(); @@ -74,7 +73,6 @@ class PrometheusInputRunner : public InputRunner { int32_t mServicePort; std::string mPodName; - std::unique_ptr mClient; std::shared_ptr mTimer; EventPool mEventPool; diff --git a/core/prometheus/schedulers/ScrapeConfig.cpp b/core/prometheus/schedulers/ScrapeConfig.cpp index 3946bf54a2..3bc05e9f65 100644 --- a/core/prometheus/schedulers/ScrapeConfig.cpp +++ b/core/prometheus/schedulers/ScrapeConfig.cpp @@ -10,7 +10,7 @@ #include "logger/Logger.h" #include "prometheus/Constants.h" #include "prometheus/Utils.h" -#include "sdk/Common.h" +#include "common/EncodingUtil.h" using namespace std; @@ -223,7 +223,7 @@ bool ScrapeConfig::InitBasicAuth(const Json::Value& basicAuth) { } auto token = username + ":" + password; - auto token64 = sdk::Base64Enconde(token); + auto token64 = Base64Enconde(token); mRequestHeaders[prometheus::A_UTHORIZATION] = prometheus::BASIC_PREFIX + token64; return true; } diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index 8596d6f36b..c37ebaf23f 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -23,6 +23,7 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" +#include "common/http/Constant.h" #include "common/timer/HttpRequestTimerEvent.h" #include "logger/Logger.h" #include "pipeline/queue/ProcessQueueManager.h" @@ -32,7 +33,6 @@ #include "prometheus/async/PromFuture.h" #include "prometheus/async/PromHttpRequest.h" #include "prometheus/component/StreamScraper.h" -#include "sdk/Common.h" using namespace std; @@ -168,7 +168,7 @@ std::unique_ptr ScrapeScheduler::BuildScrapeTimerEvent(std::chrono:: } mPromStreamScraper.SetScrapeTime(mLatestScrapeTime); auto request = std::make_unique( - sdk::HTTP_GET, + HTTP_GET, mScrapeConfigPtr->mScheme == prometheus::HTTPS, mHost, mPort, diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index ca8a3ad97c..a5540c8958 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -20,10 +20,10 @@ #include #include -#include "Common.h" -#include "TimeUtil.h" #include "common/JsonUtil.h" #include "common/StringTools.h" +#include "common/TimeUtil.h" +#include "common/http/Constant.h" #include "common/timer/HttpRequestTimerEvent.h" #include "common/timer/Timer.h" #include "logger/Logger.h" @@ -295,7 +295,7 @@ TargetSubscriberScheduler::BuildSubscriberTimerEvent(std::chrono::steady_clock:: if (!mETag.empty()) { httpHeader[prometheus::IF_NONE_MATCH] = mETag; } - auto request = std::make_unique(sdk::HTTP_GET, + auto request = std::make_unique(HTTP_GET, false, mServiceHost, mServicePort, diff --git a/core/protobuf/sls/logtail_buffer_meta.proto b/core/protobuf/sls/logtail_buffer_meta.proto index dc2639e997..131e4099d7 100644 --- a/core/protobuf/sls/logtail_buffer_meta.proto +++ b/core/protobuf/sls/logtail_buffer_meta.proto @@ -17,10 +17,17 @@ package sls_logs; import "sls_logs.proto"; +enum EndpointMode +{ + DEFAULT = 0; + ACCELERATE = 1; + CUSTOM = 2; +} + message LogtailBufferMeta { required string project = 1; - required string endpoint = 2; + required string region = 2; required string aliuid = 3; optional string logstore = 4; optional int32 datatype = 5; @@ -28,4 +35,6 @@ message LogtailBufferMeta optional string shardhashkey = 7; optional SlsCompressType compresstype = 8; optional SlsTelemetryType telemetrytype = 9; + optional EndpointMode endpointmode = 10; + optional string endpoint = 11; } diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 4a0ea254dc..1e985cecb2 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -26,12 +26,9 @@ #include "pipeline/queue/SenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" #include "plugin/flusher/sls/DiskBufferWriter.h" -// TODO: temporarily used here -#include "plugin/flusher/sls/PackIdManager.h" -#include "plugin/flusher/sls/SLSClientManager.h" +#include "runner/sink/http/HttpSink.h" -DEFINE_FLAG_INT32(flusher_runner_exit_timeout_secs, "", 60); -DEFINE_FLAG_INT32(check_send_client_timeout_interval, "", 600); +DEFINE_FLAG_INT32(flusher_runner_exit_timeout_sec, "", 60); DECLARE_FLAG_INT32(discard_send_fail_interval); @@ -99,7 +96,7 @@ void FlusherRunner::Stop() { if (!mThreadRes.valid()) { return; } - future_status s = mThreadRes.wait_for(chrono::seconds(INT32_FLAG(flusher_runner_exit_timeout_secs))); + future_status s = mThreadRes.wait_for(chrono::seconds(INT32_FLAG(flusher_runner_exit_timeout_sec))); if (s == future_status::ready) { LOG_INFO(sLogger, ("flusher runner", "stopped successfully")); } else { @@ -121,7 +118,8 @@ void FlusherRunner::PushToHttpSink(SenderQueueItem* item, bool withLimit) { unique_ptr req; bool keepItem = false; - if (!static_cast(item->mFlusher)->BuildRequest(item, req, &keepItem)) { + string errMsg; + if (!static_cast(item->mFlusher)->BuildRequest(item, req, &keepItem, &errMsg)) { if (keepItem && chrono::duration_cast(chrono::system_clock::now() - item->mFirstEnqueTime).count() < INT32_FLAG(discard_send_fail_interval)) { @@ -129,10 +127,12 @@ void FlusherRunner::PushToHttpSink(SenderQueueItem* item, bool withLimit) { LOG_DEBUG(sLogger, ("failed to build request", "retry later")("item address", item)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(item->mQueueKey))); + SenderQueueManager::GetInstance()->DecreaseConcurrencyLimiterInSendingCnt(item->mQueueKey); } else { LOG_WARNING(sLogger, ("failed to build request", "discard item")("item address", item)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(item->mQueueKey))); + SenderQueueManager::GetInstance()->DecreaseConcurrencyLimiterInSendingCnt(item->mQueueKey); SenderQueueManager::GetInstance()->RemoveItem(item->mQueueKey, item); } return; @@ -189,12 +189,6 @@ void FlusherRunner::Run() { mTotalDelayMs->Add(chrono::system_clock::now() - curTime); } - // TODO: move the following logic to scheduler - if ((time(NULL) - mLastCheckSendClientTime) > INT32_FLAG(check_send_client_timeout_interval)) { - SLSClientManager::GetInstance()->CleanTimeoutClient(); - PackIdManager::GetInstance()->CleanTimeoutEntry(); - mLastCheckSendClientTime = time(NULL); - } if (mIsFlush && SenderQueueManager::GetInstance()->IsAllQueueEmpty()) { break; } diff --git a/core/runner/FlusherRunner.h b/core/runner/FlusherRunner.h index 3390a021b6..d433d6138b 100644 --- a/core/runner/FlusherRunner.h +++ b/core/runner/FlusherRunner.h @@ -24,7 +24,6 @@ #include "pipeline/plugin/interface/Flusher.h" #include "pipeline/queue/SenderQueueItem.h" #include "runner/sink/SinkType.h" -#include "runner/sink/http/HttpSink.h" namespace logtail { diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index ab88c517c6..aae72adad0 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -26,7 +26,7 @@ #include "queue/QueueKeyManager.h" DEFINE_FLAG_INT32(default_flush_merged_buffer_interval, "default flush merged buffer, seconds", 1); -DEFINE_FLAG_INT32(processor_runner_exit_timeout_secs, "", 60); +DEFINE_FLAG_INT32(processor_runner_exit_timeout_sec, "", 60); DECLARE_FLAG_INT32(max_send_log_group_size); @@ -59,7 +59,7 @@ void ProcessorRunner::Stop() { continue; } future_status s - = mThreadRes[threadNo].wait_for(chrono::seconds(INT32_FLAG(processor_runner_exit_timeout_secs))); + = mThreadRes[threadNo].wait_for(chrono::seconds(INT32_FLAG(processor_runner_exit_timeout_sec))); if (s == future_status::ready) { LOG_INFO(sLogger, ("processor runner", "stopped successfully")("threadNo", threadNo)); } else { diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index 0213edd166..49724c42c8 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -28,7 +28,7 @@ #include "unittest/pipeline/HttpSinkMock.h" #endif -DEFINE_FLAG_INT32(http_sink_exit_timeout_secs, "", 5); +DEFINE_FLAG_INT32(http_sink_exit_timeout_sec, "", 5); using namespace std; @@ -77,7 +77,7 @@ void HttpSink::Stop() { if (!mThreadRes.valid()) { return; } - future_status s = mThreadRes.wait_for(chrono::seconds(INT32_FLAG(http_sink_exit_timeout_secs))); + future_status s = mThreadRes.wait_for(chrono::seconds(INT32_FLAG(http_sink_exit_timeout_sec))); if (s == future_status::ready) { LOG_INFO(sLogger, ("http sink", "stopped successfully")); } else { @@ -134,6 +134,7 @@ bool HttpSink::AddRequestToClient(unique_ptr&& request) { AppConfig::GetInstance()->GetBindInterface()); if (curl == nullptr) { request->mItem->mStatus = SendingStatus::IDLE; + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to init curl handler"); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); mOutFailedItemsTotal->Add(1); LOG_ERROR(sLogger, @@ -146,11 +147,11 @@ bool HttpSink::AddRequestToClient(unique_ptr&& request) { request->mPrivateData = headers; curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); - request->mLastSendTime = chrono::system_clock::now(); auto res = curl_multi_add_handle(mClient, curl); if (res != CURLM_OK) { request->mItem->mStatus = SendingStatus::IDLE; + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to add the easy curl handle to multi_handle"); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); curl_easy_cleanup(curl); mOutFailedItemsTotal->Add(1); @@ -252,18 +253,21 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); auto pipelinePlaceHolder = request->mItem->mPipeline; // keep pipeline alive auto responseTime = chrono::system_clock::now() - request->mLastSendTime; - auto responseTimeMs = chrono::duration_cast(responseTime).count(); + auto responseTimeMs = chrono::duration_cast(responseTime); switch (msg->data.result) { case CURLE_OK: { long statusCode = 0; curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); + request->mResponse.SetNetworkStatus(NetworkCode::Ok, ""); request->mResponse.SetStatusCode(statusCode); - LOG_DEBUG( - sLogger, - ("send http request succeeded, item address", request->mItem)( - "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( - "response time", ToString(responseTimeMs) + "ms")("try cnt", ToString(request->mTryCnt))( - "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); + request->mResponse.SetResponseTime(responseTimeMs); + LOG_DEBUG(sLogger, + ("send http request succeeded, item address", + request->mItem)("config-flusher-dst", + QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( + "response time", ToString(responseTimeMs.count()) + "ms")("try cnt", + ToString(request->mTryCnt))( + "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); mOutSuccessfulItemsTotal->Add(1); @@ -274,12 +278,11 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { default: // considered as network error if (request->mTryCnt <= request->mMaxTryCnt) { - LOG_WARNING( - sLogger, - ("failed to send http request", "retry immediately")("item address", request->mItem)( - "config-flusher-dst", - QueueKeyManager::GetInstance()->GetName(request->mItem->mFlusher->GetQueueKey()))( - "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); + LOG_DEBUG(sLogger, + ("failed to send http request", "retry immediately")("item address", request->mItem)( + "config-flusher-dst", + QueueKeyManager::GetInstance()->GetName(request->mItem->mFlusher->GetQueueKey()))( + "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); // free first，becase mPrivateData will be reset in AddRequestToClient if (request->mPrivateData) { curl_slist_free_all((curl_slist*)request->mPrivateData); @@ -290,12 +293,14 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { ++runningHandlers; requestReused = true; } else { + auto errMsg = curl_easy_strerror(msg->data.result); + request->mResponse.SetNetworkStatus(GetNetworkStatus(msg->data.result), errMsg); LOG_DEBUG(sLogger, ("failed to send http request", "abort")("item address", request->mItem)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( - "response time", ToString(responseTimeMs) + "ms")("try cnt", - ToString(request->mTryCnt))( + "response time", ToString(responseTimeMs.count()) + "ms")( + "try cnt", ToString(request->mTryCnt))("errMsg", errMsg)( "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); static_cast(request->mItem->mFlusher) ->OnSendDone(request->mResponse, request->mItem); diff --git a/core/runner/sink/http/HttpSinkRequest.h b/core/runner/sink/http/HttpSinkRequest.h index f8220f7722..d9ed07ce41 100644 --- a/core/runner/sink/http/HttpSinkRequest.h +++ b/core/runner/sink/http/HttpSinkRequest.h @@ -33,7 +33,7 @@ struct HttpSinkRequest : public AsynHttpRequest { const std::map& header, const std::string& body, SenderQueueItem* item, - uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_secs)), + uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)), uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)) ) : AsynHttpRequest(method, httpsFlag, host, port, url, query, header, body, HttpResponse(), timeout, maxTryCnt), mItem(item) {} diff --git a/core/sdk/Client.cpp b/core/sdk/Client.cpp deleted file mode 100644 index 053e6da40a..0000000000 --- a/core/sdk/Client.cpp +++ /dev/null @@ -1,483 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "Client.h" - -#include "Common.h" -#include "CurlImp.h" -#include "Exception.h" -#include "Result.h" -#include "app_config/AppConfig.h" -#include "common/Flags.h" -#include "logger/Logger.h" -#include "monitor/Monitor.h" -#include "plugin/flusher/sls/SLSClientManager.h" -#ifdef __ENTERPRISE__ -#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" -#endif - -namespace logtail { -namespace sdk { - - using namespace std; - - Client::Client(const string& aliuid, const string& slsHost, int32_t timeout) - : mTimeout(timeout), mHostFieldSuffix(""), mIsHostRawIp(false), mPort(80), mUsingHTTPS(false), mAliuid(aliuid) { - mClient = new CurlClient(); - mSlsHostUpdateTime = 0; - mSlsRealIpUpdateTime = 0; - SetSlsHost(slsHost); - if (mTimeout <= 0) { - mTimeout = LOG_REQUEST_TIMEOUT; - } - } - - Client::~Client() throw() { - if (mClient != NULL) { - delete mClient; - } - } - - void Client::SetPort(int32_t port) { - mPort = port; - mUsingHTTPS = (443 == mPort); - } - - string Client::GetSlsHost() { - mSpinLock.lock(); - string slsHost = mSlsHost; - mSpinLock.unlock(); - return slsHost; - } - - string Client::GetRawSlsHost() { - mSpinLock.lock(); - string rawSlsHost = mRawSlsHost; - mSpinLock.unlock(); - return rawSlsHost; - } - - string Client::GetHostFieldSuffix() { - mSpinLock.lock(); - string hostFieldSuffix = mHostFieldSuffix; - mSpinLock.unlock(); - return hostFieldSuffix; - } - - bool Client::GetRawSlsHostFlag() { - return mIsHostRawIp; - } - - void Client::SetSlsHost(const string& slsHost) { - mSpinLock.lock(); - if (slsHost == mRawSlsHost) { - mSpinLock.unlock(); - return; - } - mRawSlsHost = slsHost; - size_t bpos = slsHost.find("://"); - if (bpos == string::npos) - bpos = 0; - else - bpos += 3; - string tmpstr = slsHost.substr(bpos); - size_t epos = tmpstr.find_first_of("/"); - if (epos == string::npos) - epos = tmpstr.length(); - string host = tmpstr.substr(0, epos); - - mSlsHost = host; - - mHostFieldSuffix = "." + host; - size_t i = 0; - for (; i < host.length(); ++i) { - if ((host[i] >= 'a' && host[i] <= 'z') || (host[i] >= 'A' && host[i] <= 'Z')) - break; - } - if (i == host.length()) - mIsHostRawIp = true; - else - mIsHostRawIp = false; - mSpinLock.unlock(); - } - - - string Client::GetHost(const string& project) { - if (mIsHostRawIp || project.empty()) { - return GetSlsHost(); - } else { - return project + GetHostFieldSuffix(); - } - } - - GetRealIpResponse Client::GetRealIp() { - static string project = "logtail-real-ip-project"; - static string logstore = "logtail-real-ip-logstore"; - GetRealIpResponse rsp; - try { - PingSLSServer(project, logstore, &rsp.realIp); - } catch (const LOGException&) { - } - return rsp; - } - - bool Client::TestNetwork() { - static string project = "logtail-test-network-project"; - static string logstore = "logtail-test-network-logstore"; - PingSLSServer(project, logstore); - return true; - } - - PostLogStoreLogsResponse Client::PostLogStoreLogs(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - const std::string& hashKey, - bool isTimeSeries) { - map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(rawSize); - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - if (isTimeSeries) { - return SynPostMetricStoreLogs(project, logstore, compressedLogGroup, httpHeader); - } else { - return SynPostLogStoreLogs(project, logstore, compressedLogGroup, httpHeader, hashKey); - } - } - - PostLogStoreLogsResponse Client::PostLogStoreLogPackageList(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& packageListData, - const std::string& hashKey) { - map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_MODE] = LOG_MODE_BATCH_GROUP; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(packageListData.size()); - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - return SynPostLogStoreLogs(project, logstore, packageListData, httpHeader, hashKey); - } - - unique_ptr Client::CreatePostLogStoreLogsRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - SenderQueueItem* item, - const std::string& hashKey, - int64_t hashKeySeqID, - bool isTimeSeries) { - map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(rawSize); - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - if (isTimeSeries) { - return CreateAsynPostMetricStoreLogsRequest(project, logstore, compressedLogGroup, httpHeader, item); - } else { - return CreateAsynPostLogStoreLogsRequest( - project, logstore, compressedLogGroup, httpHeader, hashKey, hashKeySeqID, item); - } - } - - - unique_ptr Client::CreatePostLogStoreLogPackageListRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& packageListData, - SenderQueueItem* item, - const std::string& hashKey) { - map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_MODE] = LOG_MODE_BATCH_GROUP; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(packageListData.size()); - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - return CreateAsynPostLogStoreLogsRequest( - project, logstore, packageListData, httpHeader, hashKey, kInvalidHashKeySeqID, item); - } - - void Client::SendRequest(const std::string& project, - const std::string& httpMethod, - const std::string& url, - const std::string& body, - const std::map& parameterList, - std::map& header, - HttpMessage& httpMessage, - std::string* realIpPtr) { - SLSClientManager::AuthType type; - string accessKeyId, accessKeySecret; - if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { -#ifdef __ENTERPRISE__ - static auto* manager = static_cast(SLSClientManager::GetInstance()); - if (!manager->GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)) { - throw LOGException(LOGE_UNAUTHORIZED, ""); - } -#endif - } - if (type == SLSClientManager::AuthType::ANONYMOUS) { - header[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; - } - - string host = GetHost(project); - SetCommonHeader(header, (int32_t)(body.length()), project); - string signature = GetUrlSignature(httpMethod, url, header, parameterList, body, accessKeySecret); - header[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; - - string queryString; - GetQueryString(parameterList, queryString); - - int32_t port = mPort; - if (mPort == 80 && mUsingHTTPS) { - port = 443; - } - mClient->Send(httpMethod, - host, - port, - url, - queryString, - header, - body, - mTimeout, - httpMessage, - AppConfig::GetInstance()->GetBindInterface(), - mUsingHTTPS); - - if (httpMessage.statusCode != 200) { - if (realIpPtr != NULL) { - *realIpPtr = httpMessage.header[X_LOG_HOSTIP]; - } - ErrorCheck(httpMessage.content, httpMessage.header[X_LOG_REQUEST_ID], httpMessage.statusCode); - } - } - - std::unique_ptr - Client::CreateAsynPostMetricStoreLogsRequest(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - SenderQueueItem* item) { - SLSClientManager::AuthType type; - string accessKeyId, accessKeySecret; - if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { -#ifdef __ENTERPRISE__ - static auto* manager = static_cast(SLSClientManager::GetInstance()); - if (!manager->GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)) { - return nullptr; - } -#endif - } - if (type == SLSClientManager::AuthType::ANONYMOUS) { - httpHeader[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; - } - - string operation = METRICSTORES; - operation.append("/").append(project).append("/").append(logstore).append("/api/v1/write"); - httpHeader[CONTENT_MD5] = CalcMD5(body); - map parameterList; - string host = GetSlsHost(); - SetCommonHeader(httpHeader, (int32_t)(body.length()), ""); - string signature = GetUrlSignature(HTTP_POST, operation, httpHeader, parameterList, body, accessKeySecret); - httpHeader[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; - return make_unique(HTTP_POST, mUsingHTTPS, host, mPort, operation, "", httpHeader, body, item, INT32_FLAG(default_http_request_timeout_secs), LOG_REQUEST_TRY_TIMES); - } - - unique_ptr - Client::CreateAsynPostLogStoreLogsRequest(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - const std::string& hashKey, - int64_t hashKeySeqID, - SenderQueueItem* item) { - SLSClientManager::AuthType type; - string accessKeyId, accessKeySecret; - if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { -#ifdef __ENTERPRISE__ - static auto* manager = static_cast(SLSClientManager::GetInstance()); - if (!manager->GetAccessKeyIfProjectSupportsAnonymousWrite(project, type, accessKeyId, accessKeySecret)) { - return nullptr; - } -#endif - } - if (type == SLSClientManager::AuthType::ANONYMOUS) { - httpHeader[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; - } - - string operation = LOGSTORES; - operation.append("/").append(logstore); - if (hashKey.empty()) - operation.append("/shards/lb"); - else - operation.append("/shards/route"); - - httpHeader[CONTENT_MD5] = CalcMD5(body); - - map parameterList; - if (!hashKey.empty()) { - parameterList["key"] = hashKey; - if (hashKeySeqID != kInvalidHashKeySeqID) { - parameterList["seqid"] = std::to_string(hashKeySeqID); - } - } - - string host = GetHost(project); - SetCommonHeader(httpHeader, (int32_t)(body.length()), project); - string signature = GetUrlSignature(HTTP_POST, operation, httpHeader, parameterList, body, accessKeySecret); - httpHeader[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; - - string queryString; - GetQueryString(parameterList, queryString); - - return make_unique( - HTTP_POST, mUsingHTTPS, host, mPort, operation, queryString, httpHeader, body, item, INT32_FLAG(default_http_request_timeout_secs), LOG_REQUEST_TRY_TIMES); - } - - PostLogStoreLogsResponse - Client::PingSLSServer(const std::string& project, const std::string& logstore, std::string* realIpPtr) { - sls_logs::LogGroup logGroup; - logGroup.set_source(LoongCollectorMonitor::mIpAddr); - auto serializeData = logGroup.SerializeAsString(); - - std::map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(serializeData.size()); - return SynPostLogStoreLogs(project, logstore, serializeData, httpHeader, "", realIpPtr); - } - - PostLogStoreLogsResponse Client::SynPostLogStoreLogs(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - const std::string& hashKey, - std::string* realIpPtr) { - string operation = LOGSTORES; - operation.append("/").append(logstore); - if (hashKey.empty()) - operation.append("/shards/lb"); - else - operation.append("/shards/route"); - - httpHeader[CONTENT_MD5] = CalcMD5(body); - - map parameterList; - if (!hashKey.empty()) - parameterList["key"] = hashKey; - - HttpMessage httpResponse; - SendRequest(project, HTTP_POST, operation, body, parameterList, httpHeader, httpResponse, realIpPtr); - - PostLogStoreLogsResponse ret; - ret.bodyBytes = (int32_t)body.size(); - ret.statusCode = httpResponse.statusCode; - ret.requestId = httpResponse.header[X_LOG_REQUEST_ID]; - return ret; - } - - PostLogStoreLogsResponse Client::SynPostMetricStoreLogs(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - std::string* realIpPtr) { - string operation = METRICSTORES; - operation.append("/").append(project).append("/").append(logstore).append("/api/v1/write"); - httpHeader[CONTENT_MD5] = CalcMD5(body); - map parameterList; - HttpMessage httpResponse; - SendRequest(project, HTTP_POST, operation, body, parameterList, httpHeader, httpResponse, realIpPtr); - PostLogStoreLogsResponse ret; - ret.bodyBytes = (int32_t)body.size(); - ret.statusCode = httpResponse.statusCode; - ret.requestId = httpResponse.header[X_LOG_REQUEST_ID]; - return ret; - } - - PostLogStoreLogsResponse Client::PostLogUsingWebTracking(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLog, - uint32_t rawSize) { - map httpHeader; - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(rawSize); - SetCommonHeader(httpHeader, (int32_t)(compressedLog.length()), project); - - string operation = LOGSTORES; - operation.append("/").append(logstore).append("/track"); - - string host = GetHost(project); - int32_t port = mPort; - if (mPort == 80 && mUsingHTTPS) { - port = 443; - } - - HttpMessage httpResponse; - mClient->Send(HTTP_POST, - host, - port, - operation, - "", - httpHeader, - compressedLog, - mTimeout, - httpResponse, - AppConfig::GetInstance()->GetBindInterface(), - mUsingHTTPS); - - PostLogStoreLogsResponse ret; - ret.bodyBytes = (int32_t)compressedLog.length(); - ret.statusCode = httpResponse.statusCode; - ret.requestId = httpResponse.header[X_LOG_REQUEST_ID]; - return ret; - } - - void Client::SetCommonHeader(map& httpHeader, int32_t contentLength, const string& project) { - if (!project.empty()) { - httpHeader[HOST] = project + GetHostFieldSuffix(); - } else { - httpHeader[HOST] = GetSlsHost(); - } - - httpHeader[USER_AGENT] = SLSClientManager::GetInstance()->GetUserAgent(); - httpHeader[X_LOG_APIVERSION] = LOG_API_VERSION; - httpHeader[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; - httpHeader[DATE] = GetDateString(); - httpHeader[CONTENT_LENGTH] = std::to_string(contentLength); - } - - std::string Client::GetCompressTypeString(sls_logs::SlsCompressType compressType) { - switch (compressType) { - case sls_logs::SLS_CMP_NONE: - return ""; - case sls_logs::SLS_CMP_LZ4: - return LOG_LZ4; - case sls_logs::SLS_CMP_ZSTD: - return LOG_ZSTD; - default: - return LOG_LZ4; - } - } - - sls_logs::SlsCompressType Client::GetCompressType(std::string compressTypeString, - sls_logs::SlsCompressType defaultType) { - if (compressTypeString == "none") { - return sls_logs::SLS_CMP_NONE; - } else if (compressTypeString == LOG_LZ4) { - return sls_logs::SLS_CMP_LZ4; - } else if (compressTypeString == LOG_ZSTD) { - return sls_logs::SLS_CMP_ZSTD; - } - return defaultType; - } -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/Client.h b/core/sdk/Client.h deleted file mode 100644 index 073f39b657..0000000000 --- a/core/sdk/Client.h +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -#include "Common.h" -#include "CurlImp.h" -#include "protobuf/sls/sls_logs.pb.h" -#include "runner/sink/http/HttpSinkRequest.h" - -namespace logtail { -namespace sdk { - - class Client { - public: - /** Constructor needs at least three parameters. - * @param LOGHost LOG service address, for example:http://cn-hangzhou.log.aliyuncs.com. - * @param timeout Timeout time of one operation. - */ - Client(const std::string& aliuid, - const std::string& slsHost, - int32_t timeout = LOG_REQUEST_TIMEOUT); - ~Client() throw(); - - void SetPort(int32_t port); - - GetRealIpResponse GetRealIp(); - bool TestNetwork(); - - std::string GetHost(const std::string& project); - - void SetSlsHost(const std::string& slsHost); - std::string GetSlsHost(); - std::string GetRawSlsHost(); - std::string GetHostFieldSuffix(); - bool GetRawSlsHostFlag(); - - void SetSlsHostUpdateTime(int32_t uptime) { mSlsHostUpdateTime = uptime; } - int32_t GetSlsHostUpdateTime() { return mSlsHostUpdateTime; } - - void SetSlsRealIpUpdateTime(int32_t uptime) { mSlsRealIpUpdateTime = uptime; } - int32_t GetSlsRealIpUpdateTime() { return mSlsRealIpUpdateTime; } - bool IsUsingHTTPS() { return mUsingHTTPS; } - - /////////////////////////////////////Internal Interface For Logtail//////////////////////////////////////// - /** Sync Put data to LOG service. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param compressedLogGroup serialized data of logGroup, LZ4 or ZSTD comressed - * @param rawSize before compress - * @param compressType compression type - * @return request_id. - */ - PostLogStoreLogsResponse PostLogStoreLogs(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - const std::string& hashKey = "", - bool isTimeSeries = false); - - PostLogStoreLogsResponse PostMetricStoreLogs(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize) { - return PostLogStoreLogs(project, logstore, compressType, compressedLogGroup, rawSize, "", true); - } - - - /** Sync Put data to LOG service. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param packageListData data of logPackageList, consist of several LogGroup - * @return request_id. - */ - PostLogStoreLogsResponse PostLogStoreLogPackageList(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& packageListData, - const std::string& hashKey = ""); - /** Async Put data to LOG service. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param compressedLogGroup data of logGroup, LZ4 comressed - * @param rawSize before compress - * @param compressType compression type - * @return request_id. - */ - std::unique_ptr CreatePostLogStoreLogsRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - SenderQueueItem* item, - const std::string& hashKey = "", - int64_t hashKeySeqID = kInvalidHashKeySeqID, - bool isTimeSeries = false); - /** Async Put metrics data to SLS metricstore. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param compressedLogGroup data of logGroup, LZ4 comressed - * @param rawSize before compress - * @param compressType compression type - * @return request_id. - */ - std::unique_ptr CreatePostMetricStoreLogsRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - SenderQueueItem* item) { - return CreatePostLogStoreLogsRequest( - project, logstore, compressType, compressedLogGroup, rawSize, item, "", kInvalidHashKeySeqID, true); - } - - - /** Async Put data to LOG service. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param packageListData data of logPackageList, consist of several LogGroup - * @return request_id. - */ - std::unique_ptr CreatePostLogStoreLogPackageListRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& packageListData, - SenderQueueItem* item, - const std::string& hashKey = ""); - - PostLogStoreLogsResponse PostLogUsingWebTracking(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize); - ///////////////////////////////////////////////////////////////////////////////////////////////// - - static std::string GetCompressTypeString(sls_logs::SlsCompressType compressType); - static sls_logs::SlsCompressType GetCompressType(std::string compressTypeString, - sls_logs::SlsCompressType defaultType = sls_logs::SLS_CMP_LZ4); - - protected: - void SendRequest(const std::string& project, - const std::string& httpMethod, - const std::string& url, - const std::string& body, - const std::map& parameterList, - std::map& header, - HttpMessage& httpMessage, - std::string* realIpPtr = NULL); - - std::unique_ptr - CreateAsynPostLogStoreLogsRequest(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - const std::string& hashKey, - int64_t hashKeySeqID, - SenderQueueItem* item); - - std::unique_ptr - CreateAsynPostMetricStoreLogsRequest(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - SenderQueueItem* item); - - // PingSLSServer sends a trivial data packet to SLS for some inner purposes. - PostLogStoreLogsResponse - PingSLSServer(const std::string& project, const std::string& logstore, std::string* realIpPtr = NULL); - - PostLogStoreLogsResponse SynPostLogStoreLogs(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - const std::string& hashKey, - std::string* realIpPtr = NULL); - - PostLogStoreLogsResponse SynPostMetricStoreLogs(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - std::string* realIpPtr = NULL); - - void SetCommonHeader(std::map& httpHeader, - int32_t contentLength, - const std::string& project = ""); - - protected: - int32_t mSlsHostUpdateTime; - int32_t mSlsRealIpUpdateTime; - std::string mRawSlsHost; - std::string mSlsHost; - int32_t mTimeout; - std::string mHostFieldSuffix; - bool mIsHostRawIp; - int32_t mPort; - bool mUsingHTTPS; - std::string mAliuid; - - SpinLock mSpinLock; - - CurlClient* mClient; - }; - -} // namespace sdk - -} // namespace logtail diff --git a/core/sdk/Common.cpp b/core/sdk/Common.cpp deleted file mode 100644 index 8f3a1a7890..0000000000 --- a/core/sdk/Common.cpp +++ /dev/null @@ -1,883 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "Common.h" -#include "app_config/AppConfig.h" -#include "common/TimeUtil.h" -#include "common/StringTools.h" -#include "common/ErrorUtil.h" -#include "logger/Logger.h" - -using namespace std; -using namespace logtail::sdk; - -namespace logtail { -namespace sdk { - const char* const LOG_HEADSIGNATURE_PREFIX = "LOG "; - const char* const LOGE_REQUEST_ERROR = "RequestError"; - const char* const LOGE_INVALID_HOST = "InvalidHost"; - const char* const LOGE_UNKNOWN_ERROR = "UnknownError"; - const char* const LOGE_NOT_IMPLEMENTED = "NotImplemented"; - const char* const LOGE_SERVER_BUSY = "ServerBusy"; - const char* const LOGE_INTERNAL_SERVER_ERROR = "InternalServerError"; - const char* const LOGE_RESPONSE_SIG_ERROR = "ResponseSignatureError"; - const char* const LOGE_PARAMETER_INVALID = "ParameterInvalid"; - const char* const LOGE_MISSING_PARAMETER = "MissingParameter"; - const char* const LOGE_INVALID_METHOD = "InvalidMethod"; - const char* const LOGE_BAD_RESPONSE = "BadResponse"; - const char* const LOGE_UNAUTHORIZED = "Unauthorized"; - const char* const LOGE_QUOTA_EXCEED = "ExceedQuota"; - const char* const LOGE_REQUEST_TIMEOUT = "RequestTimeout"; - const char* const LOGE_CLIENT_OPERATION_TIMEOUT = "ClientOpertaionTimeout"; - const char* const LOGE_CLIENT_NETWORK_ERROR = "ClientNetworkError"; - const char* const LOGE_USER_NOT_EXIST = "UserNotExist"; - const char* const LOGE_CATEGORY_NOT_EXIST = "CategoryNotExist"; - const char* const LOGE_TOPIC_NOT_EXIST = "TopicNotExist"; - const char* const LOGE_POST_BODY_INVALID = "PostBodyInvalid"; - const char* const LOGE_INVALID_CONTENTTYPE = "InvalidContentType"; - const char* const LOGE_INVALID_CONTENLENGTH = "InvalidContentLength"; - const char* const LOGE_INVALID_APIVERSION = "InvalidAPIVersion"; - const char* const LOGE_PROJECT_NOT_EXIST = "ProjectNotExist"; - const char* const LOGE_MACHINEGROUP_NOT_EXIST = "MachineGroupNotExist"; - const char* const LOGE_MACHINEGROUP_ALREADY_EXIST = "MachineGroupAlreadyExist"; - const char* const LOGE_CONFIG_NOT_EXIST = "ConfigNotExist"; - const char* const LOGE_CONFIG_ALREADY_EXIST = "ConfigAlreadyExist"; - const char* const LOGE_LOGSTORE_NOT_EXIST = "LogStoreNotExist"; - const char* const LOGE_INVALID_ACCESSKEYID = "InvalidAccessKeyId"; - const char* const LOGE_SIGNATURE_NOT_MATCH = "SignatureNotMatch"; - const char* const LOGE_PROJECT_FORBIDDEN = "ProjectForbidden"; - const char* const LOGE_WRITE_QUOTA_EXCEED = "WriteQuotaExceed"; - const char* const LOGE_READ_QUOTA_EXCEED = "ReadQuotaExceed"; - const char* const LOGE_REQUEST_TIME_EXPIRED = "RequestTimeExpired"; - const char* const LOGE_INVALID_REQUEST_TIME = "InvalidRequestTime"; - const char* const LOGE_POST_BODY_TOO_LARGE = "PostBodyTooLarge"; - const char* const LOGE_INVALID_TIME_RANGE = "InvalidTimeRange"; - const char* const LOGE_INVALID_REVERSE = "InvalidReverse"; - const char* const LOGE_LOGSTORE_WITHOUT_SHARD = "LogStoreWithoutShard"; - const char* const LOGE_SHARD_WRITE_QUOTA_EXCEED = "ShardWriteQuotaExceed"; - const char* const LOGE_SHARD_READ_QUOTA_EXCEED = "ShardReadQuotaExceed"; - const char* const LOGE_INVALID_SEQUENCE_ID = "InvalidSequenceId"; - - const char* const LOGSTORES = "/logstores"; - const char* const METRICSTORES = "/prometheus"; - const char* const SHARDS = "/shards"; - const char* const INDEX = "/index"; - const char* const CONFIGS = "/configs"; - const char* const MACHINES = "/machines"; - const char* const MACHINEGROUPS = "/machinegroups"; - const char* const ACLS = "/acls"; - const char* const CONFIGSERVERAGENT = "/Agent"; - - const char* const HTTP_GET = "GET"; - const char* const HTTP_POST = "POST"; - const char* const HTTP_PUT = "PUT"; - const char* const HTTP_DELETE = "DELETE"; - - const char* const HOST = "Host"; - const char* const DATE = "Date"; - const char* const USER_AGENT = "User-Agent"; - const char* const LOG_OLD_HEADER_PREFIX = "x-sls-"; - const char* const LOG_HEADER_PREFIX = "x-log-"; - const char* const ACS_HEADER_PREFIX = "x-acs-"; - const char* const X_LOG_KEYPROVIDER = "x-log-keyprovider"; - const char* const X_LOG_APIVERSION = "x-log-apiversion"; - const char* const X_LOG_COMPRESSTYPE = "x-log-compresstype"; - const char* const X_LOG_BODYRAWSIZE = "x-log-bodyrawsize"; - const char* const X_LOG_SIGNATUREMETHOD = "x-log-signaturemethod"; - const char* const X_ACS_SECURITY_TOKEN = "x-acs-security-token"; - const char* const X_LOG_CURSOR = "x-log-cursor"; - const char* const X_LOG_REQUEST_ID = "x-log-requestid"; - const char* const X_LOG_MODE = "x-log-mode"; - - const char* const X_LOG_PROGRESS = "x-log-progress"; - const char* const X_LOG_COUNT = "x-log-count"; - const char* const X_LOG_HOSTIP = "x-log-hostip"; - - const char* const HTTP_ACCEPT = "accept"; - const char* const DEFLATE = "deflate"; - const char* const HMAC_SHA1 = "hmac-sha1"; - const char* const CONTENT_TYPE = "Content-Type"; - const char* const CONTENT_LENGTH = "Content-Length"; - const char* const CONTENT_MD5 = "Content-MD5"; - const char* const AUTHORIZATION = "Authorization"; - const char* const SIGNATURE = "Signature"; - const char* const ACCEPT_ENCODING = "Accept-Encoding"; - const char* const ENCONDING_GZIP = "gzip"; - const char* const TYPE_LOG_PROTOBUF = "application/x-protobuf"; - const char* const TYPE_LOG_JSON = "application/json"; - const char* const LOG_MODE_BATCH_GROUP = "batch_group"; - const char* const LOGITEM_TIME_STAMP_LABEL = "__time__"; - const char* const LOGITEM_SOURCE_LABEL = "__source__"; - const char* const LOG_API_VERSION = "0.6.0"; - const char* const LOGTAIL_USER_AGENT = "ali-log-logtail"; - const char* const MD5_SHA1_SALT_KEYPROVIDER = "md5-sha1-salt"; - const char* const LOG_TYPE_CURSOR = "cursor"; - const char* const LOG_TYPE = "type"; - const char* const LOGE_NOT_SUPPORTED_ACCEPT_CONTENT_TYPE = "InvalidAcceptContentType"; - const char* const LOGE_NOT_SUPPORTED_ACCEPT_ENCODING = "InvalidAcceptEncoding"; - const char* const LOGE_SHARD_NOT_EXIST = "ShardNotExist"; - const char* const LOGE_INVALID_CURSOR = "InvalidCursor"; - const char* const LOG_LZ4 = "lz4"; - const char* const LOG_DEFLATE = "deflate"; - const char* const LOG_ZSTD = "zstd"; - const char* const LOG_ERROR_CODE = "errorCode"; - const char* const LOG_ERROR_MESSAGE = "errorMessage"; - - const char* const LOG_SHARD_STATUS_READWRITE = "readwrite"; - const char* const LOG_SHARD_STATUS_READONLY = "readonly"; - - bool caseInsensitiveComp(const char lhs, const char rhs) { - return tolower(lhs) < tolower(rhs); - } - - bool compareHeader(const std::string& lhs, const std::string& rhs) { - return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), caseInsensitiveComp); - } - - bool HttpMessage::IsLogServiceResponse() const { - if (!AppConfig::GetInstance()->IsResponseVerificationEnabled()) { - return true; - } - - const auto iter = header.find(X_LOG_REQUEST_ID); - if (iter == header.end()) { - return false; - } - return !iter->second.empty(); - } - - static unsigned char ToHex(unsigned char x) { - return x > 9 ? x + 55 : x + 48; - } - - static unsigned char FromHex(unsigned char x) { - unsigned char y = '\0'; - if (x >= 'A' && x <= 'Z') - y = x - 'A' + 10; - else if (x >= 'a' && x <= 'z') - y = x - 'a' + 10; - else if (x >= '0' && x <= '9') - y = x - '0'; - else - assert(0); - return y; - } - - - static std::string HexToString(const uint8_t md5[16]) { - static const char* table = "0123456789ABCDEF"; - std::string ss(32, 'a'); - for (int i = 0; i < 16; ++i) { - ss[i * 2] = table[md5[i] >> 4]; - ss[i * 2 + 1] = table[md5[i] & 0x0F]; - } - return ss; - } - - std::string CalcMD5(const std::string& message) { - uint8_t md5[MD5_BYTES]; - DoMd5((const uint8_t*)message.data(), message.length(), md5); - return HexToString(md5); - } - - std::string CalcSHA1(const std::string& message, const std::string& key) { - HMAC hmac(reinterpret_cast(key.data()), key.size()); - hmac.add(reinterpret_cast(message.data()), message.size()); - return string(reinterpret_cast(hmac.result()), SHA1_DIGEST_BYTES); - } - - - void Base64Encoding(std::istream& is, std::ostream& os, char makeupChar, const char* alphabet) { - int out[4]; - int remain = 0; - while (!is.eof()) { - int byte1 = is.get(); - if (byte1 < 0) { - break; - } - int byte2 = is.get(); - int byte3; - if (byte2 < 0) { - byte2 = 0; - byte3 = 0; - remain = 1; - } else { - byte3 = is.get(); - if (byte3 < 0) { - byte3 = 0; - remain = 2; - } - } - out[0] = static_cast(byte1) >> 2; - out[1] = ((byte1 & 0x03) << 4) + (static_cast(byte2) >> 4); - out[2] = ((byte2 & 0x0F) << 2) + (static_cast(byte3) >> 6); - out[3] = byte3 & 0x3F; - - if (remain == 1) { - os.put(out[0] = alphabet[out[0]]); - os.put(out[1] = alphabet[out[1]]); - os.put(makeupChar); - os.put(makeupChar); - } else if (remain == 2) { - os.put(out[0] = alphabet[out[0]]); - os.put(out[1] = alphabet[out[1]]); - os.put(out[2] = alphabet[out[2]]); - os.put(makeupChar); - } else { - os.put(out[0] = alphabet[out[0]]); - os.put(out[1] = alphabet[out[1]]); - os.put(out[2] = alphabet[out[2]]); - os.put(out[3] = alphabet[out[3]]); - } - } - } - - - std::string Base64Enconde(const std::string& message) { - std::istringstream iss(message); - std::ostringstream oss; - Base64Encoding(iss, oss); - return oss.str(); - } - - - std::string UrlEncode(const std::string& str) { - std::string strTemp; - size_t length = str.length(); - for (size_t i = 0; i < length; i++) { - if (isalnum((unsigned char)str[i]) || (str[i] == '-') || (str[i] == '_') || (str[i] == '.') - || (str[i] == '~')) - strTemp += str[i]; - else if (str[i] == ' ') - strTemp += "+"; - else { - strTemp += '%'; - strTemp += ToHex((unsigned char)str[i] >> 4); - strTemp += ToHex((unsigned char)str[i] % 16); - } - } - return strTemp; - } - - - std::string UrlDecode(const std::string& str) { - std::string strTemp = ""; - size_t length = str.length(); - for (size_t i = 0; i < length; i++) { - if (str[i] == '+') - strTemp += ' '; - else if (str[i] == '%') { - assert(i + 2 < length); - unsigned char high = FromHex((unsigned char)str[++i]); - unsigned char low = FromHex((unsigned char)str[++i]); - strTemp += high * 16 + low; - } else - strTemp += str[i]; - } - return strTemp; - } - - std::string GetDateString(const std::string& dateFormat) { - time_t now_time; - time(&now_time); - if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) { - now_time += GetTimeDelta(); - } - char buffer[128] = {'\0'}; - tm timeInfo; -#if defined(__linux__) - gmtime_r(&now_time, &timeInfo); -#elif defined(_MSC_VER) - gmtime_s(&timeInfo, &now_time); -#endif - strftime(buffer, 128, dateFormat.c_str(), &timeInfo); - return string(buffer); - } - - std::string GetDateString() { - return GetDateString(DATE_FORMAT_RFC822); - } - - time_t DecodeDateString(const std::string dateString, const std::string& dateFormat) { - return 0; - // struct tm t; - // memset(&t, 0, sizeof(t)); - // t.tm_sec = -1; - // strptime(dateString.c_str(), dateFormat.c_str(),&t); - // if(t.tm_sec == -1) - // { - // throw LOGException(LOGE_PARAMETER_INVALID, string("Invalid date string:") + dateString + ",format:" + - // dateFormat); - // } - // struct timezone tz; - // struct timeval tv; - // gettimeofday(&tv, &tz); - // return mktime(&t)-tz.tz_minuteswest*60; - } - - bool StartWith(const std::string& input, const std::string& pattern) { - if (input.length() < pattern.length()) { - return false; - } - - size_t i = 0; - while (i < pattern.length() && input[i] == pattern[i]) { - i++; - } - - return i == pattern.length(); - } - - void GetQueryString(const map& parameterList, string& queryString) { - queryString.clear(); - for (map::const_iterator iter = parameterList.begin(); iter != parameterList.end(); ++iter) { - if (iter != parameterList.begin()) { - queryString.append("&"); - } - queryString.append(iter->first); - queryString.append("="); - queryString.append(UrlEncode(iter->second)); - } - } - - string GetUrlSignature(const string& httpMethod, - const string& operationType, - map& httpHeader, - const map& parameterList, - const string& content, - const string& signKey) { - string contentMd5; - string signature; - string osstream; - if (!content.empty()) { - contentMd5 = CalcMD5(content); - } - string contentType; - map::iterator iter = httpHeader.find(CONTENT_TYPE); - if (iter != httpHeader.end()) { - contentType = iter->second; - } - std::map endingMap; - osstream.append(httpMethod); - osstream.append("\n"); - osstream.append(contentMd5); - osstream.append("\n"); - osstream.append(contentType); - osstream.append("\n"); - osstream.append(httpHeader[DATE]); - osstream.append("\n"); - for (map::const_iterator iter = httpHeader.begin(); iter != httpHeader.end(); ++iter) { - if (StartWith(iter->first, LOG_OLD_HEADER_PREFIX)) { - std::string key = iter->first; - endingMap.insert(std::make_pair(key.replace(0, std::strlen(LOG_OLD_HEADER_PREFIX), LOG_HEADER_PREFIX), - iter->second)); - } else if (StartWith(iter->first, LOG_HEADER_PREFIX) || StartWith(iter->first, ACS_HEADER_PREFIX)) { - endingMap.insert(std::make_pair(iter->first, iter->second)); - } - } - for (map::const_iterator it = endingMap.begin(); it != endingMap.end(); ++it) { - osstream.append(it->first); - osstream.append(":"); - osstream.append(it->second); - osstream.append("\n"); - } - osstream.append(operationType); - if (parameterList.size() > 0) { - osstream.append("?"); - for (map::const_iterator iter = parameterList.begin(); iter != parameterList.end(); - ++iter) { - if (iter != parameterList.begin()) { - osstream.append("&"); - } - osstream.append(iter->first); - osstream.append("="); - osstream.append(iter->second); - } - } - - signature = Base64Enconde(CalcSHA1(osstream, signKey)); - - return signature; - } - - -/////////////////////////////////////////////// MACRO ////////////////////////////////////////////////// -#define SHIFT_LEFT(a, b) ((a) << (b) | (a) >> (32 - b)) - -/** - * each operation - */ -#define F(b, c, d) (((b) & (c)) | ((~(b)) & (d))) -#define G(b, c, d) (((d) & (b)) | ((~(d)) & (c))) -#define H(b, c, d) ((b) ^ (c) ^ (d)) -#define I(b, c, d) ((c) ^ ((b) | (~(d)))) - -/** - * each round - */ -#define FF(a, b, c, d, word, shift, k) \ - { \ - (a) += F((b), (c), (d)) + (word) + (k); \ - (a) = SHIFT_LEFT((a), (shift)); \ - (a) += (b); \ - } -#define GG(a, b, c, d, word, shift, k) \ - { \ - (a) += G((b), (c), (d)) + (word) + (k); \ - (a) = SHIFT_LEFT((a), (shift)); \ - (a) += (b); \ - } -#define HH(a, b, c, d, word, shift, k) \ - { \ - (a) += H((b), (c), (d)) + (word) + (k); \ - (a) = SHIFT_LEFT((a), (shift)); \ - (a) += (b); \ - } -#define II(a, b, c, d, word, shift, k) \ - { \ - (a) += I((b), (c), (d)) + (word) + (k); \ - (a) = SHIFT_LEFT((a), (shift)); \ - (a) += (b); \ - } - ////////////////////////////////////////////////////////// GLOBAL VARIABLE - //////////////////////////////////////////////////////////// - const uint8_t gPadding[64] = {0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; - - - //////////////////////////////////////////////////////// LOCAL DECLEARATION - ///////////////////////////////////////////////////////////// - struct Md5Block { - uint32_t word[16]; - }; - /** - * copy a pool into a block, using little endian - */ - void CopyBytesToBlock(const uint8_t* poolIn, struct Md5Block& block) { - uint32_t j = 0; - for (uint32_t i = 0; i < 16; ++i, j += 4) { - block.word[i] = ((uint32_t)poolIn[j]) | (((uint32_t)poolIn[j + 1]) << 8) | (((uint32_t)poolIn[j + 2]) << 16) - | (((uint32_t)poolIn[j + 3]) << 24); - } - } - - /** - * calculate md5 hash value from a block - */ - void CalMd5(struct Md5Block block, uint32_t h[4]) { - uint32_t a = h[0]; - uint32_t b = h[1]; - uint32_t c = h[2]; - uint32_t d = h[3]; - - // Round 1 - FF(a, b, c, d, block.word[0], 7, 0xd76aa478); - FF(d, a, b, c, block.word[1], 12, 0xe8c7b756); - FF(c, d, a, b, block.word[2], 17, 0x242070db); - FF(b, c, d, a, block.word[3], 22, 0xc1bdceee); - FF(a, b, c, d, block.word[4], 7, 0xf57c0faf); - FF(d, a, b, c, block.word[5], 12, 0x4787c62a); - FF(c, d, a, b, block.word[6], 17, 0xa8304613); - FF(b, c, d, a, block.word[7], 22, 0xfd469501); - FF(a, b, c, d, block.word[8], 7, 0x698098d8); - FF(d, a, b, c, block.word[9], 12, 0x8b44f7af); - FF(c, d, a, b, block.word[10], 17, 0xffff5bb1); - FF(b, c, d, a, block.word[11], 22, 0x895cd7be); - FF(a, b, c, d, block.word[12], 7, 0x6b901122); - FF(d, a, b, c, block.word[13], 12, 0xfd987193); - FF(c, d, a, b, block.word[14], 17, 0xa679438e); - FF(b, c, d, a, block.word[15], 22, 0x49b40821); - - // Round 2 - GG(a, b, c, d, block.word[1], 5, 0xf61e2562); - GG(d, a, b, c, block.word[6], 9, 0xc040b340); - GG(c, d, a, b, block.word[11], 14, 0x265e5a51); - GG(b, c, d, a, block.word[0], 20, 0xe9b6c7aa); - GG(a, b, c, d, block.word[5], 5, 0xd62f105d); - GG(d, a, b, c, block.word[10], 9, 0x2441453); - GG(c, d, a, b, block.word[15], 14, 0xd8a1e681); - GG(b, c, d, a, block.word[4], 20, 0xe7d3fbc8); - GG(a, b, c, d, block.word[9], 5, 0x21e1cde6); - GG(d, a, b, c, block.word[14], 9, 0xc33707d6); - GG(c, d, a, b, block.word[3], 14, 0xf4d50d87); - GG(b, c, d, a, block.word[8], 20, 0x455a14ed); - GG(a, b, c, d, block.word[13], 5, 0xa9e3e905); - GG(d, a, b, c, block.word[2], 9, 0xfcefa3f8); - GG(c, d, a, b, block.word[7], 14, 0x676f02d9); - GG(b, c, d, a, block.word[12], 20, 0x8d2a4c8a); - - // Round 3 - HH(a, b, c, d, block.word[5], 4, 0xfffa3942); - HH(d, a, b, c, block.word[8], 11, 0x8771f681); - HH(c, d, a, b, block.word[11], 16, 0x6d9d6122); - HH(b, c, d, a, block.word[14], 23, 0xfde5380c); - HH(a, b, c, d, block.word[1], 4, 0xa4beea44); - HH(d, a, b, c, block.word[4], 11, 0x4bdecfa9); - HH(c, d, a, b, block.word[7], 16, 0xf6bb4b60); - HH(b, c, d, a, block.word[10], 23, 0xbebfbc70); - HH(a, b, c, d, block.word[13], 4, 0x289b7ec6); - HH(d, a, b, c, block.word[0], 11, 0xeaa127fa); - HH(c, d, a, b, block.word[3], 16, 0xd4ef3085); - HH(b, c, d, a, block.word[6], 23, 0x4881d05); - HH(a, b, c, d, block.word[9], 4, 0xd9d4d039); - HH(d, a, b, c, block.word[12], 11, 0xe6db99e5); - HH(c, d, a, b, block.word[15], 16, 0x1fa27cf8); - HH(b, c, d, a, block.word[2], 23, 0xc4ac5665); - - // Round 4 - II(a, b, c, d, block.word[0], 6, 0xf4292244); - II(d, a, b, c, block.word[7], 10, 0x432aff97); - II(c, d, a, b, block.word[14], 15, 0xab9423a7); - II(b, c, d, a, block.word[5], 21, 0xfc93a039); - II(a, b, c, d, block.word[12], 6, 0x655b59c3); - II(d, a, b, c, block.word[3], 10, 0x8f0ccc92); - II(c, d, a, b, block.word[10], 15, 0xffeff47d); - II(b, c, d, a, block.word[1], 21, 0x85845dd1); - II(a, b, c, d, block.word[8], 6, 0x6fa87e4f); - II(d, a, b, c, block.word[15], 10, 0xfe2ce6e0); - II(c, d, a, b, block.word[6], 15, 0xa3014314); - II(b, c, d, a, block.word[13], 21, 0x4e0811a1); - II(a, b, c, d, block.word[4], 6, 0xf7537e82); - II(d, a, b, c, block.word[11], 10, 0xbd3af235); - II(c, d, a, b, block.word[2], 15, 0x2ad7d2bb); - II(b, c, d, a, block.word[9], 21, 0xeb86d391); - - // Add to hash value - h[0] += a; - h[1] += b; - h[2] += c; - h[3] += d; - } - - - void DoMd5Little(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t hash[16]) { - struct Md5Block block; - - /// initialize hash value - uint32_t h[4]; - h[0] = 0x67452301; - h[1] = 0xEFCDAB89; - h[2] = 0x98BADCFE; - h[3] = 0x10325476; - - /// padding and divide input data into blocks - uint64_t fullLen = (inputBytesNum >> 6) << 6; /// complete blocked length - uint64_t partLen = inputBytesNum & 0x3F; /// length remained - - uint32_t i; - for (i = 0; i < fullLen; i += 64) { - /// copy input data into block - memcpy(block.word, &(poolIn[i]), 64); - - /// calculate Md5 - CalMd5(block, h); - } - - - if (partLen > 55) /// append two more blocks - { - /// copy input data into block and pad - memcpy(block.word, &(poolIn[i]), partLen); - memcpy(((uint8_t*)&(block.word[partLen >> 2])) + (partLen & 0x3), gPadding, (64 - partLen)); - - /// calculate Md5 - CalMd5(block, h); - - /// set rest byte to 0x0 - memset(block.word, 0x0, 64); - } else /// append one more block - { - /// copy input data into block and pad - memcpy(block.word, &(poolIn[i]), partLen); - memcpy(((uint8_t*)&(block.word[partLen >> 2])) + (partLen & 0x3), gPadding, (64 - partLen)); - } - - /// append length (bits) - uint64_t bitsNum = inputBytesNum * 8; - memcpy(&(block.word[14]), &bitsNum, 8); - - /// calculate Md5 - CalMd5(block, h); - - /// clear sensitive information - memset(block.word, 0, 64); - - /// fill hash value - memcpy(&(hash[0]), &(h[0]), 16); - } /// DoMd5Little - - void DoMd5Big(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t hash[16]) { - struct Md5Block block; - uint8_t tempBlock[64]; - - /// initialize hash value - uint32_t h[4]; - h[0] = 0x67452301; - h[1] = 0xEFCDAB89; - h[2] = 0x98BADCFE; - h[3] = 0x10325476; - - /// padding and divide input data into blocks - uint64_t fullLen = (inputBytesNum >> 6) << 6; - uint64_t partLen = inputBytesNum & 0x3F; - - uint32_t i; - for (i = 0; i < fullLen; i += 64) { - /// copy input data into block, in little endian - CopyBytesToBlock(&(poolIn[i]), block); - - /// calculate Md5 - CalMd5(block, h); - } - - /// append two more blocks - if (partLen > 55) { - /// put input data into a temporary block - memcpy(tempBlock, &(poolIn[i]), partLen); - memcpy(&(tempBlock[partLen]), gPadding, (64 - partLen)); - - /// copy temporary data into block, in little endian - CopyBytesToBlock(tempBlock, block); - - /// calculate Md5 - CalMd5(block, h); - - memset(tempBlock, 0x0, 64); - } - /// append one more block - else { - memcpy(tempBlock, &(poolIn[i]), partLen); - memcpy(&(tempBlock[partLen]), gPadding, (64 - partLen)); - } - /// append length (bits) - uint64_t bitsNum = inputBytesNum * 8; - memcpy(&(tempBlock[56]), &bitsNum, 8); - - /// copy temporary data into block, in little endian - CopyBytesToBlock(tempBlock, block); - - /// calculate Md5 - CalMd5(block, h); - - /// clear sensitive information - memset(block.word, 0, 64); - memset(tempBlock, 0, 64); - - /// fill hash value - memcpy(&(hash[0]), &(h[0]), 16); - } /// DoMd5Big - - void DoMd5(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t md5[16]) { - /// detect big or little endian - union { - uint32_t a; - uint8_t b; - } symbol; - - symbol.a = 1; - - /// for little endian - if (symbol.b == 1) { - DoMd5Little(poolIn, inputBytesNum, md5); - } - /// for big endian - else { - DoMd5Big(poolIn, inputBytesNum, md5); - } - } /// DoMd5 - -/* - * define the rotate left (circular left shift) operation - */ -#define rotl(v, b) (((v) << (b)) | ((v) >> (32 - (b)))) - -/* - * Define the basic SHA-1 functions F1 ~ F4. Note that the exclusive-OR - * operation (^) in F1 and F3 may be replaced by a bitwise OR operation - * (|), which produce identical results. - * - * F1 is used in ROUND 0~19, F2 is used in ROUND 20~39 - * F3 is used in ROUND 40~59, F4 is used in ROUND 60~79 - */ -#define F1(B, C, D) (((B) & (C)) ^ (~(B) & (D))) -#define F2(B, C, D) ((B) ^ (C) ^ (D)) -#define F3(B, C, D) (((B) & (C)) ^ ((B) & (D)) ^ ((C) & (D))) -#define F4(B, C, D) ((B) ^ (C) ^ (D)) - -/* - * Use different K in different ROUND - */ -#define K00_19 0x5A827999 -#define K20_39 0x6ED9EBA1 -#define K40_59 0x8F1BBCDC -#define K60_79 0xCA62C1D6 - -/* - * Another implementation of the ROUND transformation: - * (here the T is a temp variable) - * For t=0 to 79: - * { - * T=rotl(A,5)+Func(B,C,D)+K+W[t]+E; - * E=D; D=C; C=rotl(B,30); B=A; A=T; - * } - */ -#define ROUND(t, A, B, C, D, E, Func, K) \ - E += rotl(A, 5) + Func(B, C, D) + W[t] + K; \ - B = rotl(B, 30); - -#define ROUND5(t, Func, K) \ - ROUND(t, A, B, C, D, E, Func, K); \ - ROUND(t + 1, E, A, B, C, D, Func, K); \ - ROUND(t + 2, D, E, A, B, C, Func, K); \ - ROUND(t + 3, C, D, E, A, B, Func, K); \ - ROUND(t + 4, B, C, D, E, A, Func, K) - -#define ROUND20(t, Func, K) \ - ROUND5(t, Func, K); \ - ROUND5(t + 5, Func, K); \ - ROUND5(t + 10, Func, K); \ - ROUND5(t + 15, Func, K) - - /* - * Define constant of the initial vector - */ - const uint32_t SHA1::IV[SHA1_DIGEST_WORDS] = {0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0}; - - /* - * the message must be the big-endian32 (or left-most word) - * before calling the transform() function. - */ - const static uint32_t iii = 1; - const static bool littleEndian = *(uint8_t*)&iii != 0; - - inline uint32_t littleEndianToBig(uint32_t d) { - uint8_t* data = (uint8_t*)&d; - return data[0] << 24 | data[1] << 16 | data[2] << 8 | data[3]; - } - - inline void make_big_endian32(uint32_t* data, unsigned n) { - if (!littleEndian) { - return; - } - for (; n > 0; ++data, --n) { - *data = littleEndianToBig(*data); - } - } - - inline size_t min(size_t a, size_t b) { - return a < b ? a : b; - } - - void SHA1::transform() { - uint32_t W[80]; - memcpy(W, M, SHA1_INPUT_BYTES); - memset((uint8_t*)W + SHA1_INPUT_BYTES, 0, sizeof(W) - SHA1_INPUT_BYTES); - for (unsigned t = 16; t < 80; t++) { - W[t] = rotl(W[t - 16] ^ W[t - 14] ^ W[t - 8] ^ W[t - 3], 1); - } - - uint32_t A = H[0]; - uint32_t B = H[1]; - uint32_t C = H[2]; - uint32_t D = H[3]; - uint32_t E = H[4]; - - ROUND20(0, F1, K00_19); - ROUND20(20, F2, K20_39); - ROUND20(40, F3, K40_59); - ROUND20(60, F4, K60_79); - - H[0] += A; - H[1] += B; - H[2] += C; - H[3] += D; - H[4] += E; - } - - void SHA1::add(const uint8_t* data, size_t data_len) { - unsigned mlen = (unsigned)((bits >> 3) % SHA1_INPUT_BYTES); - bits += (uint64_t)data_len << 3; - unsigned use = (unsigned)min((size_t)(SHA1_INPUT_BYTES - mlen), data_len); - memcpy(M + mlen, data, use); - mlen += use; - - while (mlen == SHA1_INPUT_BYTES) { - data_len -= use; - data += use; - make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS); - transform(); - use = (unsigned)min((size_t)SHA1_INPUT_BYTES, data_len); - memcpy(M, data, use); - mlen = use; - } - } - - uint8_t* SHA1::result() { - unsigned mlen = (unsigned)((bits >> 3) % SHA1_INPUT_BYTES), padding = SHA1_INPUT_BYTES - mlen; - M[mlen++] = 0x80; - if (padding > BIT_COUNT_BYTES) { - memset(M + mlen, 0x00, padding - BIT_COUNT_BYTES); - make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS - BIT_COUNT_WORDS); - } else { - memset(M + mlen, 0x00, SHA1_INPUT_BYTES - mlen); - make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS); - transform(); - memset(M, 0x00, SHA1_INPUT_BYTES - BIT_COUNT_BYTES); - } - - uint64_t temp = littleEndian ? bits << 32 | bits >> 32 : bits; - memcpy(M + SHA1_INPUT_BYTES - BIT_COUNT_BYTES, &temp, BIT_COUNT_BYTES); - transform(); - make_big_endian32(H, SHA1_DIGEST_WORDS); - return (uint8_t*)H; - } - - template - inline void axor(T* p1, const T* p2, size_t len) { - for (; len != 0; --len) - *p1++ ^= *p2++; - } - - HMAC::HMAC(const uint8_t* key, size_t lkey) { - init(key, lkey); - } - - void HMAC::init(const uint8_t* key, size_t lkey) { - in.init(); - out.init(); - - uint8_t ipad[SHA1_INPUT_BYTES]; - uint8_t opad[SHA1_INPUT_BYTES]; - memset(ipad, 0x36, sizeof(ipad)); - memset(opad, 0x5c, sizeof(opad)); - - if (lkey <= SHA1_INPUT_BYTES) { - axor(ipad, key, lkey); - axor(opad, key, lkey); - } else { - SHA1 tmp; - tmp.add(key, lkey); - const uint8_t* key2 = tmp.result(); - axor(ipad, key2, SHA1_DIGEST_BYTES); - axor(opad, key2, SHA1_DIGEST_BYTES); - } - - in.add((uint8_t*)ipad, sizeof(ipad)); - out.add((uint8_t*)opad, sizeof(opad)); - } - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/Common.h b/core/sdk/Common.h deleted file mode 100644 index e37d2efa61..0000000000 --- a/core/sdk/Common.h +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace logtail { -namespace sdk { - - const int64_t kInvalidHashKeySeqID = 0; - const int64_t kFirstHashKeySeqID = 1; - - const uint32_t LOG_REQUEST_TIMEOUT = 20; - const uint32_t LOG_REQUEST_TRY_TIMES = 1; - - const uint32_t MD5_BYTES = 16; -#define DATE_FORMAT_RFC822 "%a, %d %b %Y %H:%M:%S GMT" ///< RFC822 date formate, GMT time. - - extern const char* const LOG_HEADSIGNATURE_PREFIX; ///< ""; - - extern const char* const LOGE_REQUEST_ERROR; //= "RequestError"; - extern const char* const LOGE_INVALID_HOST; //= "InvalidHost" - extern const char* const LOGE_UNKNOWN_ERROR; //= "UnknownError"; - extern const char* const LOGE_NOT_IMPLEMENTED; //= "NotImplemented"; - extern const char* const LOGE_SERVER_BUSY; //= "ServerBusy"; - extern const char* const LOGE_INTERNAL_SERVER_ERROR; //= "InternalServerError"; - extern const char* const LOGE_RESPONSE_SIG_ERROR; //= "ResponseSignatureError"; - extern const char* const LOGE_PARAMETER_INVALID; //= "ParameterInvalid"; - extern const char* const LOGE_MISSING_PARAMETER; //= "MissingParameter"; - extern const char* const LOGE_INVALID_METHOD; //= "InvalidMethod"; - extern const char* const LOGE_INVALID_CONTENTTYPE; //= "InvalidContentType"; - extern const char* const LOGE_INVALID_CONTENTLENGTH; //= "InvalidContentLength"; - extern const char* const LOGE_BAD_RESPONSE; //= "BadResponse"; - extern const char* const LOGE_UNAUTHORIZED; //= "Unauthorized"; - extern const char* const LOGE_QUOTA_EXCEED; //= "ExceedQuota"; - extern const char* const LOGE_REQUEST_TIMEOUT; //= "RequestTimeout"; - extern const char* const LOGE_CLIENT_OPERATION_TIMEOUT; //= "ClientOpertaionTimeout"; - extern const char* const LOGE_CLIENT_NETWORK_ERROR; //= "ClientNetworkError"; - extern const char* const LOGE_USER_NOT_EXIST; //= "UserNotExist"; - extern const char* const LOGE_CATEGORY_NOT_EXIST; //= "CategoryNotExist"; - extern const char* const LOGE_TOPIC_NOT_EXIST; //= "TopicNotExist"; - extern const char* const LOGE_POST_BODY_INVALID; //= "PostBodyInvalid"; - extern const char* const LOGE_INVALID_HOST; //= "InvalidHost"; - extern const char* const LOGE_INVALID_APIVERSION; //="InvalidAPIVersion"; - extern const char* const LOGE_PROJECT_NOT_EXIST; //="ProjectNotExist"; - extern const char* const LOGE_MACHINEGROUP_NOT_EXIST; //="MachineGroupNotExist"; - extern const char* const LOGE_MACHINEGROUP_ALREADY_EXIST; //="MachineGroupAlreadyExist"; - extern const char* const LOGE_CONFIG_NOT_EXIST; //="ConfigNotExist"; - extern const char* const LOGE_CONFIG_ALREADY_EXIST; //="ConfigAlreadyExist"; - extern const char* const LOGE_LOGSTORE_NOT_EXIST; //="LogStoreNotExist"; - extern const char* const LOGE_INVALID_ACCESSKEYID; //="InvalidAccessKeyId"; - extern const char* const LOGE_SIGNATURE_NOT_MATCH; //="SignatureNotMatch"; - extern const char* const LOGE_PROJECT_FORBIDDEN; //="ProjectForbidden"; - extern const char* const LOGE_WRITE_QUOTA_EXCEED; //="WriteQuotaExceed"; - extern const char* const LOGE_READ_QUOTA_EXCEED; //="ReadQuotaExceed"; - extern const char* const LOGE_REQUEST_TIME_EXPIRED; //="RequestTimeExpired"; - extern const char* const LOGE_INVALID_REQUEST_TIME; //="InvalidRequestTime"; - extern const char* const LOGE_POST_BODY_TOO_LARGE; //="PostBodyTooLarge"; - extern const char* const LOGE_INVALID_TIME_RANGE; //="InvalidTimeRange"; - extern const char* const LOGE_INVALID_REVERSE; //="InvalidReverse"; - extern const char* const LOGE_LOGSTORE_WITHOUT_SHARD; //="LogStoreWithoutShard"; - extern const char* const LOGE_INVALID_SEQUENCE_ID; //="InvalidSequenceId"; - - extern const char* const LOGSTORES; //= "/logstores" - extern const char* const METRICSTORES; //= "/prometheus" - extern const char* const SHARDS; //= "/shards" - extern const char* const INDEX; //= "/index" - extern const char* const CONFIGS; //= "/configs" - extern const char* const MACHINES; //= "/machines" - extern const char* const MACHINEGROUPS; //= "/machinegroups" - extern const char* const ACLS; //= "/acls" - extern const char* const CONFIGSERVERAGENT; //= "/Agent" - - extern const char* const HTTP_GET; //= "GET"; - extern const char* const HTTP_POST; //= "POST"; - extern const char* const HTTP_PUT; //= "PUT"; - extern const char* const HTTP_DELETE; //= "DELETE"; - - extern const char* const HOST; //= "Host"; - extern const char* const DATE; //= "Date"; - extern const char* const USER_AGENT; //= "User-Agent"; - extern const char* const LOG_HEADER_PREFIX; //= "x-log-"; - extern const char* const LOG_OLD_HEADER_PREFIX; //= "x-sls-"; - extern const char* const X_LOG_KEYPROVIDER; // = "x-log-keyprovider"; - extern const char* const X_LOG_APIVERSION; // = "x-log-apiversion"; - extern const char* const X_LOG_COMPRESSTYPE; // = "x-log-compresstype"; - extern const char* const X_LOG_BODYRAWSIZE; // = "x-log-bodyrawsize"; - extern const char* const X_LOG_SIGNATUREMETHOD; // = "x-log-signaturemethod"; - extern const char* const X_ACS_SECURITY_TOKEN; // = "x-acs-security-token"; - extern const char* const X_LOG_CURSOR; // = "cursor"; - extern const char* const X_LOG_REQUEST_ID; // = "x-log-requestid"; - extern const char* const X_LOG_MODE; // = "x-log-mode"; - - extern const char* const X_LOG_PROGRESS; // = "x-log-progress"; - extern const char* const X_LOG_COUNT; // = "x-log-count"; - extern const char* const X_LOG_HOSTIP; // = "x-log-hostip" - - extern const char* const HTTP_ACCEPT; // = "accept"; - extern const char* const DEFLATE; //= "deflate"; - extern const char* const HMAC_SHA1; //= "hmac-sha1"; - extern const char* const CONTENT_LENGTH; //= "Content-Length"; - extern const char* const CONTENT_TYPE; //= "Content-Type"; - extern const char* const CONTENT_MD5; //= "Content-MD5"; - extern const char* const AUTHORIZATION; //= "Authorization"; - extern const char* const SIGNATURE; //= "Signature"; - extern const char* const ACCEPT_ENCODING; // = "Accept-Encoding"; - extern const char* const ENCONDING_GZIP; // = "gzip"; - extern const char* const TYPE_LOG_PROTOBUF; //="application/x-protobuf"; - extern const char* const TYPE_LOG_JSON; //="application/json"; - extern const char* const LOG_MODE_BATCH_GROUP; //="batch_group"; - extern const char* const LOGITEM_TIME_STAMP_LABEL; //="__time__"; - extern const char* const LOGITEM_SOURCE_LABEL; //="__source__"; - extern const char* const LOG_API_VERSION; // = "0.6.0"; - extern const char* const LOGTAIL_USER_AGENT; // = "ali-sls-logtail"; - extern const char* const MD5_SHA1_SALT_KEYPROVIDER; // = "md5-shal-salt"; - extern const char* const LOG_TYPE_CURSOR; // = "cursor"; - extern const char* const LOG_TYPE; // = "type"; - extern const char* const LOGE_NOT_SUPPORTED_ACCEPT_CONTENT_TYPE; - extern const char* const LOGE_NOT_SUPPORTED_ACCEPT_ENCODING; - extern const char* const LOGE_SHARD_NOT_EXIST; - extern const char* const LOGE_INVALID_CURSOR; - extern const char* const LOGE_SHARD_WRITE_QUOTA_EXCEED; - extern const char* const LOGE_SHARD_READ_QUOTA_EXCEED; - extern const char* const LOG_LZ4; //= "lz4"; - extern const char* const LOG_DEFLATE; //= "deflate"; - extern const char* const LOG_ZSTD; //= "zstd"; - - extern const char* const LOG_ERROR_CODE; //= "errorCode"; - extern const char* const LOG_ERROR_MESSAGE; //= "errorMessage"; - - extern const char* const LOG_SHARD_STATUS_READWRITE; // "readwrite"; - extern const char* const LOG_SHARD_STATUS_READONLY; // "readonly"; - - bool caseInsensitiveComp(const char lhs, const char rhs); - - bool compareHeader(const std::string& lhs, const std::string& rhs); - - /** - * HTTP message structure includes three parts: http status code, http header, and http content. - */ - struct HttpMessage { - int32_t statusCode = 0; ///< Http status code - std::map - header; ///< Only contains the header lines which have key:value pair - std::string content; ///< Http content - /** Constructor with no parameter. - * @param void None. - * @return The objcect pointer. - */ - HttpMessage() : header(compareHeader) {} - /** Constructor with header and content. - * @param para_header A map structure which contains the key:value pair of http header lines. - Those header lines which do not contains key:value pair are not included. - * @param para_content A string which contains the content of http request. - * @return The objcect pointer. - */ - HttpMessage(const std::map& para_header, const std::string& para_content) - : header(para_header.begin(), para_header.end(), compareHeader), content(para_content) {} - /** Constructor with status code, header and content. - * @param para_statusCode Http status code. - * @param para_header A map structure which contains the key:value pair of http header lines. - Those header lines which do not contains key:value pair are not included. - * @param para_content A string which contains the http content of http content. - * @return The objcect pointer. - */ - HttpMessage(const int32_t para_statusCode, - const std::map& para_header, - const std::string& para_content) - : statusCode(para_statusCode), - header(para_header.begin(), para_header.end(), compareHeader), - content(para_content) {} - - bool IsLogServiceResponse() const; - }; - - /* - * Responses - */ - struct Response { - int32_t statusCode = 0; ///< Http status code - std::string requestId; - - virtual ~Response() {} - }; - - struct PostLogStoreLogsResponse : public Response { - int32_t bodyBytes; - }; - - struct GetRealIpResponse : public Response { - std::string realIp; - }; - -#define SHA1_INPUT_WORDS 16 -#define SHA1_DIGEST_WORDS 5 -#define SHA1_INPUT_BYTES (SHA1_INPUT_WORDS * sizeof(uint32_t)) -#define SHA1_DIGEST_BYTES (SHA1_DIGEST_WORDS * sizeof(uint32_t)) - -#define BIT_COUNT_WORDS 2 -#define BIT_COUNT_BYTES (BIT_COUNT_WORDS * sizeof(uint32_t)) - - /** Calculate Md5 for a byte stream, - * result is stored in md5[16] - * - * @param poolIn Input data - * @param inputBytesNum Length of input data - * @param md5[16] A 128-bit pool for storing md5 - */ - void DoMd5(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t md5[16]); - void Base64Encoding(std::istream&, - std::ostream&, - char makeupChar = '=', - const char* alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"); - - std::string CalcMD5(const std::string& message); - std::string CalcSHA1(const std::string& message, const std::string& key); - std::string Base64Enconde(const std::string& message); - - std::string UrlEncode(const std::string& str); - std::string UrlDecode(const std::string& str); - - std::string GetDateString(const std::string& dateFormat); - std::string GetDateString(); - time_t DecodeDateString(const std::string dateString, const std::string& dateFormat = DATE_FORMAT_RFC822); - - bool StartWith(const std::string& input, const std::string& pattern); - - void GetQueryString(const std::map& parameterList, std::string& queryString); - - std::string GetUrlSignature(const std::string& httpMethod, - const std::string& operationType, - std::map& httpHeader, - const std::map& parameterList, - const std::string& content, - const std::string& signKey); - - class SHA1 { - public: - SHA1() : bits(0) { memcpy(H, IV, sizeof(H)); } - SHA1(const SHA1& s) { - bits = s.bits; - memcpy(H, s.H, sizeof(H)); - memcpy(M, s.M, sizeof(M)); - } - void init() { - bits = 0; - memcpy(H, IV, sizeof(H)); - } - void add(const uint8_t* data, size_t len); - uint8_t* result(); - - private: - uint64_t bits; - uint32_t H[SHA1_DIGEST_WORDS]; - uint8_t M[SHA1_INPUT_BYTES]; - - static const uint32_t IV[SHA1_DIGEST_WORDS]; - void transform(); - }; - - class HMAC { - public: - HMAC(const uint8_t* key, size_t lkey); - HMAC(const HMAC& hm) : in(hm.in), out(hm.out) {} - - void init(const uint8_t* key, size_t lkey); - - void add(const uint8_t* data, size_t len) { in.add(data, len); } - - uint8_t* result() { - out.add(in.result(), SHA1_DIGEST_BYTES); - return out.result(); - } - - private: - SHA1 in, out; - }; - - class SpinLock { - std::atomic_flag locked = ATOMIC_FLAG_INIT; - - SpinLock(const SpinLock&) = delete; - SpinLock& operator=(const SpinLock&) = delete; - - public: - SpinLock() {} - - void lock() { - while (locked.test_and_set(std::memory_order_acquire)) { - ; - } - } - - void unlock() { locked.clear(std::memory_order_release); } - }; - - using ScopedSpinLock = std::lock_guard; - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/CurlImp.cpp b/core/sdk/CurlImp.cpp deleted file mode 100644 index 9cea4bb9d6..0000000000 --- a/core/sdk/CurlImp.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "CurlImp.h" - -#include - -#include "DNSCache.h" -#include "Exception.h" -#include "app_config/AppConfig.h" -#include "common/http/Curl.h" - -using namespace std; - -namespace logtail { -namespace sdk { - - static CURLcode globalInitCode = curl_global_init(CURL_GLOBAL_ALL); - - // callback function to store the response - static size_t data_write_callback(char* buffer, size_t size, size_t nmemb, string* write_buf) { - unsigned long sizes = size * nmemb; - - if (buffer == NULL) { - return 0; - } - - write_buf->append(buffer, sizes); - return sizes; - } - - static size_t header_write_callback(char* buffer, - size_t size, - size_t nmemb, - map* write_buf) { - unsigned long sizes = size * nmemb; - - if (buffer == NULL) { - return 0; - } - unsigned long colonIndex; - for (colonIndex = 1; colonIndex < sizes - 2; colonIndex++) { - if (buffer[colonIndex] == ':') - break; - } - if (colonIndex < sizes - 2) { - unsigned long leftSpaceNum, rightSpaceNum; - for (leftSpaceNum = 0; leftSpaceNum < colonIndex - 1; leftSpaceNum++) { - if (buffer[colonIndex - leftSpaceNum - 1] != ' ') - break; - } - for (rightSpaceNum = 0; rightSpaceNum < sizes - colonIndex - 1 - 2; rightSpaceNum++) { - if (buffer[colonIndex + rightSpaceNum + 1] != ' ') - break; - } - (*write_buf)[string(buffer, 0, colonIndex - leftSpaceNum)] - = string(buffer, colonIndex + 1 + rightSpaceNum, sizes - colonIndex - 1 - 2 - rightSpaceNum); - } - return sizes; - } - - CURL* PackCurlRequest(const std::string& httpMethod, - const std::string& host, - const int32_t port, - const std::string& url, - const std::string& queryString, - const std::map& header, - const std::string& body, - const int32_t timeout, - HttpMessage& httpMessage, - const std::string& intf, - const bool httpsFlag, - curl_slist*& headers) { - static DnsCache* dnsCache = DnsCache::GetInstance(); - - CURL* curl = curl_easy_init(); - if (curl == NULL) - return NULL; - - string totalUrl = httpsFlag ? "https://" : "http://"; - std::string hostIP; - if (AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled() && dnsCache->GetIPFromDnsCache(host, hostIP)) { - totalUrl.append(hostIP); - } else { - totalUrl.append(host); - } - totalUrl.append(url); - if (!queryString.empty()) { - totalUrl.append("?").append(queryString); - } - curl_easy_setopt(curl, CURLOPT_URL, totalUrl.c_str()); - for (const auto& iter : header) { - headers = curl_slist_append(headers, (iter.first + ":" + iter.second).c_str()); - } - if (headers != NULL) { - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - } - curl_easy_setopt(curl, CURLOPT_PORT, port); - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, httpMethod.c_str()); - if (!body.empty()) { - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (void*)body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); - } - - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); - curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 1); - curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_IGNORED); - - if (httpsFlag) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - } - curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); - if (!intf.empty()) { - curl_easy_setopt(curl, CURLOPT_INTERFACE, intf.c_str()); - } - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &(httpMessage.content)); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, data_write_callback); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, &(httpMessage.header)); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_write_callback); - return curl; - } - - void CurlClient::Send(const std::string& httpMethod, - const std::string& host, - const int32_t port, - const std::string& url, - const std::string& queryString, - const std::map& header, - const std::string& body, - const int32_t timeout, - HttpMessage& httpMessage, - const std::string& intf, - const bool httpsFlag) { - curl_slist* headers = NULL; - CURL* curl = PackCurlRequest( - httpMethod, host, port, url, queryString, header, body, timeout, httpMessage, intf, httpsFlag, headers); - if (curl == NULL) { - throw LOGException(LOGE_UNKNOWN_ERROR, "Init curl instance error."); - } - - CURLcode res = curl_easy_perform(curl); - if (headers != NULL) { - curl_slist_free_all(headers); - } - - switch (res) { - case CURLE_OK: - break; - case CURLE_OPERATION_TIMEDOUT: - curl_easy_cleanup(curl); - throw LOGException(LOGE_CLIENT_OPERATION_TIMEOUT, "Request operation timeout."); - break; - case CURLE_COULDNT_CONNECT: - curl_easy_cleanup(curl); - throw LOGException(LOGE_REQUEST_TIMEOUT, "Can not connect to server."); - break; - default: - curl_easy_cleanup(curl); - throw LOGException(LOGE_REQUEST_ERROR, - string("Request operation failed, curl error code : ") + curl_easy_strerror(res)); - break; - } - - long http_code = 0; - if ((res = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code)) != CURLE_OK) { - curl_easy_cleanup(curl); - throw LOGException(LOGE_UNKNOWN_ERROR, - string("Get curl response code error, curl error code : ") + curl_easy_strerror(res)); - } - httpMessage.statusCode = (int32_t)http_code; - curl_easy_cleanup(curl); - if (!httpMessage.IsLogServiceResponse()) { - throw LOGException(LOGE_REQUEST_ERROR, "Get invalid response"); - } - } - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/CurlImp.h b/core/sdk/CurlImp.h deleted file mode 100644 index cf020e6669..0000000000 --- a/core/sdk/CurlImp.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -#include "Common.h" - -namespace logtail { -namespace sdk { - - class CurlClient { - public: - void Send(const std::string& httpMethod, - const std::string& host, - const int32_t port, - const std::string& url, - const std::string& queryString, - const std::map& header, - const std::string& body, - const int32_t timeout, - HttpMessage& httpMessage, - const std::string& intf, - const bool httpsFlag); - }; - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/Result.cpp b/core/sdk/Result.cpp deleted file mode 100644 index 4caf9fc4c5..0000000000 --- a/core/sdk/Result.cpp +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "Result.h" -#include "Exception.h" - -namespace logtail { - -namespace sdk { - - using namespace std; - - /************************ common method ***********************/ - /************************ json method *************************/ - void ExtractJsonResult(const string& response, rapidjson::Document& document) { - document.Parse(response.c_str()); - if (document.HasParseError()) { - throw JsonException("ParseException", "Fail to parse from json string"); - } - } - - void JsonMemberCheck(const rapidjson::Value& value, const char* name) { - if (!value.IsObject()) { - throw JsonException("InvalidObjectException", "response is not valid JSON object"); - } - if (!value.HasMember(name)) { - throw JsonException("NoMemberException", string("Member ") + name + " does not exist"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, int32_t& number) { - JsonMemberCheck(value, name); - if (value[name].IsInt()) { - number = value[name].GetInt(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not int type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, uint32_t& number) { - JsonMemberCheck(value, name); - if (value[name].IsUint()) { - number = value[name].GetUint(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not uint type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, int64_t& number) { - JsonMemberCheck(value, name); - if (value[name].IsInt64()) { - number = value[name].GetInt64(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not int type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, uint64_t& number) { - JsonMemberCheck(value, name); - if (value[name].IsUint64()) { - number = value[name].GetUint64(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not uint type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, bool& boolean) { - JsonMemberCheck(value, name); - if (value[name].IsBool()) { - boolean = value[name].GetBool(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not boolean type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, string& dst) { - JsonMemberCheck(value, name); - if (value[name].IsString()) { - dst = value[name].GetString(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not string type"); - } - } - - const rapidjson::Value& GetJsonValue(const rapidjson::Value& value, const char* name) { - JsonMemberCheck(value, name); - if (value[name].IsObject() || value[name].IsArray()) { - return value[name]; - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not json value type"); - } - } - - - void ErrorCheck(const string& response, const string& requestId, const int32_t httpCode) { - rapidjson::Document document; - try { - ExtractJsonResult(response, document); - - string errorCode; - ExtractJsonResult(document, LOG_ERROR_CODE, errorCode); - - string errorMessage; - ExtractJsonResult(document, LOG_ERROR_MESSAGE, errorMessage); - - throw LOGException(errorCode, errorMessage, requestId, httpCode); - } catch (JsonException& e) { - if (httpCode >= 500) { - throw LOGException(LOGE_INTERNAL_SERVER_ERROR, response, requestId, httpCode); - } else { - throw LOGException(LOGE_BAD_RESPONSE, string("Unextractable error:") + response, requestId, httpCode); - } - } - } - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/Result.h b/core/sdk/Result.h deleted file mode 100644 index d99b2fc135..0000000000 --- a/core/sdk/Result.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include "Common.h" -#include "rapidjson/document.h" - -namespace logtail { -namespace sdk { - - void ExtractJsonResult(const std::string& response, rapidjson::Document& document); - - void JsonMemberCheck(const rapidjson::Value& value, const char* name); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, int32_t& number); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, uint32_t& number); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, int64_t& number); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, uint64_t& number); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, bool& boolean); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, std::string& dst); - - const rapidjson::Value& GetJsonValue(const rapidjson::Value& value, const char* name); - - void ErrorCheck(const std::string& response, const std::string& requestId, const int32_t httpCode); - -} // namespace sdk -} // namespace logtail diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 0c7f5b504a..6e51752723 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -51,7 +51,6 @@ macro(add_core_subdir) add_subdirectory(provider) add_subdirectory(queue) add_subdirectory(reader) - add_subdirectory(sdk) add_subdirectory(sender) add_subdirectory(serializer) add_subdirectory(prometheus) diff --git a/core/unittest/common/CMakeLists.txt b/core/unittest/common/CMakeLists.txt index 6ddd095c73..80de5c490b 100644 --- a/core/unittest/common/CMakeLists.txt +++ b/core/unittest/common/CMakeLists.txt @@ -54,9 +54,6 @@ target_link_libraries(timer_unittest ${UT_BASE_TARGET}) add_executable(curl_unittest http/CurlUnittest.cpp) target_link_libraries(curl_unittest ${UT_BASE_TARGET}) -add_executable(http_response_unittest http/HttpResponseUnittest.cpp) -target_link_libraries(http_response_unittest ${UT_BASE_TARGET}) - include(GoogleTest) gtest_discover_tests(common_simple_utils_unittest) gtest_discover_tests(common_logfileoperator_unittest) @@ -69,5 +66,3 @@ gtest_discover_tests(safe_queue_unittest) gtest_discover_tests(http_request_timer_event_unittest) gtest_discover_tests(timer_unittest) gtest_discover_tests(curl_unittest) -gtest_discover_tests(http_response_unittest) - diff --git a/core/unittest/common/http/HttpResponseUnittest.cpp b/core/unittest/common/http/HttpResponseUnittest.cpp deleted file mode 100644 index 2df46b7d86..0000000000 --- a/core/unittest/common/http/HttpResponseUnittest.cpp +++ /dev/null @@ -1,27 +0,0 @@ - -#include "common/http/HttpResponse.h" -#include "unittest/Unittest.h" - -using namespace std; - -namespace logtail { -class HttpResponseUnittest : public ::testing::Test { -public: - void TestNetworkStatus(); -}; - -void HttpResponseUnittest::TestNetworkStatus() { - HttpResponse resp; - resp.SetNetworkStatus(CURLE_OK); - APSARA_TEST_EQUAL(resp.GetNetworkStatus().mCode, NetworkCode::Ok); - - resp.SetNetworkStatus(CURLE_RECV_ERROR); - APSARA_TEST_EQUAL(resp.GetNetworkStatus().mCode, NetworkCode::RecvDataFailed); - - resp.SetNetworkStatus(CURLE_FAILED_INIT); - APSARA_TEST_EQUAL(resp.GetNetworkStatus().mCode, NetworkCode::Other); -} - -UNIT_TEST_CASE(HttpResponseUnittest, TestNetworkStatus); -} // namespace logtail -UNIT_TEST_MAIN diff --git a/core/unittest/flusher/CMakeLists.txt b/core/unittest/flusher/CMakeLists.txt index 76ac1afd6d..177a86fcc1 100644 --- a/core/unittest/flusher/CMakeLists.txt +++ b/core/unittest/flusher/CMakeLists.txt @@ -16,14 +16,21 @@ cmake_minimum_required(VERSION 3.22) project(flusher_unittest) add_executable(flusher_sls_unittest FlusherSLSUnittest.cpp) +if (ENABLE_ENTERPRISE) + target_sources(flusher_sls_unittest PRIVATE SLSNetworkRequestMock.cpp) +endif () target_link_libraries(flusher_sls_unittest ${UT_BASE_TARGET}) add_executable(pack_id_manager_unittest PackIdManagerUnittest.cpp) target_link_libraries(pack_id_manager_unittest ${UT_BASE_TARGET}) +add_executable(sls_client_manager_unittest SLSClientManagerUnittest.cpp) +target_link_libraries(sls_client_manager_unittest ${UT_BASE_TARGET}) + if (ENABLE_ENTERPRISE) - add_executable(enterprise_sls_client_manager_unittest EnterpriseSLSClientManagerUnittest.cpp) + add_executable(enterprise_sls_client_manager_unittest EnterpriseSLSClientManagerUnittest.cpp SLSNetworkRequestMock.cpp) target_link_libraries(enterprise_sls_client_manager_unittest ${UT_BASE_TARGET}) + add_executable(enterprise_flusher_sls_monitor_unittest EnterpriseFlusherSLSMonitorUnittest.cpp) target_link_libraries(enterprise_flusher_sls_monitor_unittest ${UT_BASE_TARGET}) endif () @@ -31,6 +38,7 @@ endif () include(GoogleTest) gtest_discover_tests(flusher_sls_unittest) gtest_discover_tests(pack_id_manager_unittest) +gtest_discover_tests(sls_client_manager_unittest) if (ENABLE_ENTERPRISE) gtest_discover_tests(enterprise_sls_client_manager_unittest) gtest_discover_tests(enterprise_flusher_sls_monitor_unittest) diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 73dc1e97e3..3639b7b44c 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -17,8 +17,10 @@ #include #include +#include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" +#include "common/http/Constant.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif @@ -30,16 +32,26 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif #include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/SLSConstant.h" #include "unittest/Unittest.h" +#ifdef __ENTERPRISE__ +#include "unittest/flusher/SLSNetworkRequestMock.h" +#endif DECLARE_FLAG_INT32(batch_send_interval); DECLARE_FLAG_INT32(merge_log_count_limit); DECLARE_FLAG_INT32(batch_send_metric_size); DECLARE_FLAG_INT32(max_send_log_group_size); DECLARE_FLAG_DOUBLE(sls_serialize_size_expansion_ratio); +DECLARE_FLAG_BOOL(send_prefer_real_ip); +DECLARE_FLAG_STRING(default_access_key_id); +DECLARE_FLAG_STRING(default_access_key); using namespace std; @@ -50,6 +62,7 @@ class FlusherSLSUnittest : public testing::Test { void OnSuccessfulInit(); void OnFailedInit(); void OnPipelineUpdate(); + void TestBuildRequest(); void TestSend(); void TestFlush(); void TestFlushAll(); @@ -57,6 +70,14 @@ class FlusherSLSUnittest : public testing::Test { void OnGoPipelineSend(); protected: + static void SetUpTestCase() { +#ifdef __ENTERPRISE__ + EnterpriseSLSClientManager::GetInstance()->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; + EnterpriseSLSClientManager::GetInstance()->mGetEndpointRealIp = GetRealIpMock::GetEndpointRealIp; + EnterpriseSLSClientManager::GetInstance()->mGetAccessKeyFromSLS = GetAccessKeyMock::DoGetAccessKey; +#endif + } + void SetUp() override { ctx.SetConfigName("test_config"); ctx.SetPipeline(pipeline); @@ -67,6 +88,9 @@ class FlusherSLSUnittest : public testing::Test { QueueKeyManager::GetInstance()->Clear(); SenderQueueManager::GetInstance()->Clear(); ExactlyOnceQueueManager::GetInstance()->Clear(); +#ifdef __ENTERPRISE__ + EnterpriseSLSClientManager::GetInstance()->Clear(); +#endif } private: @@ -89,7 +113,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); #ifndef __ENTERPRISE__ - configJson["Endpoint"] = "cn-hangzhou.log.aliyuncs.com"; + configJson["Endpoint"] = "test_region.log.aliyuncs.com"; #endif flusher.reset(new FlusherSLS()); flusher->SetContext(ctx); @@ -100,17 +124,16 @@ void FlusherSLSUnittest::OnSuccessfulInit() { APSARA_TEST_EQUAL("test_logstore", flusher->mLogstore); APSARA_TEST_EQUAL(STRING_FLAG(default_region_name), flusher->mRegion); #ifndef __ENTERPRISE__ - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); -#else - APSARA_TEST_EQUAL("", flusher->mEndpoint); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", flusher->mEndpoint); #endif APSARA_TEST_EQUAL("", flusher->mAliuid); APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS, flusher->mTelemetryType); APSARA_TEST_TRUE(flusher->mShardHashKeys.empty()); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(merge_log_count_limit)), flusher->mBatcher.GetEventFlushStrategy().GetMinCnt()); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(max_send_log_group_size) / DOUBLE_FLAG(sls_serialize_size_expansion_ratio)), - flusher->mBatcher.GetEventFlushStrategy().GetMaxSizeBytes()); + APSARA_TEST_EQUAL( + static_cast(INT32_FLAG(max_send_log_group_size) / DOUBLE_FLAG(sls_serialize_size_expansion_ratio)), + flusher->mBatcher.GetEventFlushStrategy().GetMaxSizeBytes()); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(batch_send_metric_size)), flusher->mBatcher.GetEventFlushStrategy().GetMinSizeBytes()); uint32_t timeout = static_cast(INT32_FLAG(batch_send_interval)) / 2; @@ -136,8 +159,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789", "TelemetryType": "metrics", "ShardHashKeys": [ @@ -146,19 +169,24 @@ void FlusherSLSUnittest::OnSuccessfulInit() { } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); +#ifndef __ENTERPRISE__ + configJson["EndpointMode"] = "default"; +#endif flusher.reset(new FlusherSLS()); flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); - APSARA_TEST_EQUAL("cn-hangzhou", flusher->mRegion); + APSARA_TEST_EQUAL("test_region", flusher->mRegion); #ifdef __ENTERPRISE__ APSARA_TEST_EQUAL("123456789", flusher->mAliuid); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); #else - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); APSARA_TEST_EQUAL("", flusher->mAliuid); #endif + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", flusher->mEndpoint); APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_METRICS, flusher->mTelemetryType); APSARA_TEST_EQUAL(1U, flusher->mShardHashKeys.size()); + APSARA_TEST_EQUAL("__source__", flusher->mShardHashKeys[0]); SenderQueueManager::GetInstance()->Clear(); // invalid optional param @@ -175,9 +203,10 @@ void FlusherSLSUnittest::OnSuccessfulInit() { )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); #ifdef __ENTERPRISE__ + configJson["EndpointMode"] = true; configJson["Endpoint"] = true; #else - configJson["Endpoint"] = "cn-hangzhou.log.aliyuncs.com"; + configJson["Endpoint"] = "test_region.log.aliyuncs.com"; #endif flusher.reset(new FlusherSLS()); flusher->SetContext(ctx); @@ -185,6 +214,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(STRING_FLAG(default_region_name), flusher->mRegion); #ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); APSARA_TEST_EQUAL("", flusher->mEndpoint); #endif APSARA_TEST_EQUAL("", flusher->mAliuid); @@ -200,8 +230,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -214,14 +244,94 @@ void FlusherSLSUnittest::OnSuccessfulInit() { SenderQueueManager::GetInstance()->Clear(); #endif +#ifdef __ENTERPRISE__ + // EndpointMode && Endpoint + EnterpriseSLSClientManager::GetInstance()->Clear(); + // Endpoint ignored in acclerate mode + configStr = R"( + { + "Type": "flusher_sls", + "Project": "test_project", + "Logstore": "test_logstore", + "Region": "test_region", + "EndpointMode": "accelerate", + "Endpoint": " test_region.log.aliyuncs.com " + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + flusher.reset(new FlusherSLS()); + flusher->SetContext(ctx); + flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); + APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, flusher->mEndpointMode); + APSARA_TEST_EQUAL(EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap.end(), + EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap.find("test_region")); + APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); + APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, flusher->mCandidateHostsInfo->GetMode()); + SenderQueueManager::GetInstance()->Clear(); + + // Endpoint should be added to region remote endpoints if not existed + configStr = R"( + { + "Type": "flusher_sls", + "Project": "test_project", + "Logstore": "test_logstore", + "Region": "test_region", + "EndpointMode": "unknown", + "Endpoint": " test_region.log.aliyuncs.com " + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + flusher.reset(new FlusherSLS()); + flusher->SetContext(ctx); + flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); + APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); + auto& endpoints + = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["test_region"].mRemoteEndpoints; + APSARA_TEST_EQUAL(1U, endpoints.size()); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", endpoints[0]); + APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); + APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mCandidateHostsInfo->GetMode()); + SenderQueueManager::GetInstance()->Clear(); + + // Endpoint should be ignored when region remote endpoints existed + configStr = R"( + { + "Type": "flusher_sls", + "Project": "test_project", + "Logstore": "test_logstore", + "Region": "test_region", + "EndpointMode": "default", + "Endpoint": " test_region-intranet.log.aliyuncs.com " + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + flusher.reset(new FlusherSLS()); + flusher->SetContext(ctx); + flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); + APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); + endpoints = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["test_region"].mRemoteEndpoints; + APSARA_TEST_EQUAL(1U, endpoints.size()); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", endpoints[0]); + APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); + APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mCandidateHostsInfo->GetMode()); + SenderQueueManager::GetInstance()->Clear(); +#endif + +#ifndef __ENTERPRISE__ // Endpoint configStr = R"( { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": " cn-hangzhou.log.aliyuncs.com " + "Region": "test_region", + "Endpoint": " test_region.log.aliyuncs.com " } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -229,12 +339,9 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); - auto iter = SLSClientManager::GetInstance()->mRegionEndpointEntryMap.find("cn-hangzhou"); - APSARA_TEST_NOT_EQUAL(SLSClientManager::GetInstance()->mRegionEndpointEntryMap.end(), iter); - APSARA_TEST_NOT_EQUAL(iter->second.mEndpointInfoMap.end(), - iter->second.mEndpointInfoMap.find("http://cn-hangzhou.log.aliyuncs.com")); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", flusher->mEndpoint); SenderQueueManager::GetInstance()->Clear(); +#endif // TelemetryType configStr = R"( @@ -242,9 +349,9 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", - "TelemetryType": "logs" + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", + "TelemetryType": "metrics" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -252,7 +359,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); - APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS, flusher->mTelemetryType); + APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_METRICS, flusher->mTelemetryType); SenderQueueManager::GetInstance()->Clear(); configStr = R"( @@ -260,8 +367,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "TelemetryType": "unknown" } )"; @@ -279,8 +386,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "ShardHashKeys": [ "__source__" ] @@ -302,8 +409,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "ShardHashKeys": [ "__source__" ] @@ -322,8 +429,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -343,8 +450,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "EnableShardHash": false } )"; @@ -381,7 +488,7 @@ void FlusherSLSUnittest::OnFailedInit() { { "Type": "flusher_sls", "Logstore": "test_logstore", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -395,7 +502,7 @@ void FlusherSLSUnittest::OnFailedInit() { "Type": "flusher_sls", "Project": true, "Logstore": "test_logstore", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -409,7 +516,7 @@ void FlusherSLSUnittest::OnFailedInit() { { "Type": "flusher_sls", "Project": "test_project", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -423,7 +530,7 @@ void FlusherSLSUnittest::OnFailedInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": true, - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -478,16 +585,14 @@ void FlusherSLSUnittest::OnPipelineUpdate() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); APSARA_TEST_TRUE(flusher1.Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(flusher1.Start()); APSARA_TEST_EQUAL(1U, FlusherSLS::sProjectRefCntMap.size()); - APSARA_TEST_TRUE(FlusherSLS::IsRegionContainingConfig("cn-hangzhou")); - APSARA_TEST_EQUAL(1U, SLSClientManager::GetInstance()->GetRegionAliuids("cn-hangzhou").size()); { PipelineContext ctx2; @@ -500,8 +605,8 @@ void FlusherSLSUnittest::OnPipelineUpdate() { "Type": "flusher_sls", "Project": "test_project_2", "Logstore": "test_logstore_2", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -510,14 +615,10 @@ void FlusherSLSUnittest::OnPipelineUpdate() { APSARA_TEST_TRUE(flusher1.Stop(false)); APSARA_TEST_TRUE(FlusherSLS::sProjectRefCntMap.empty()); - APSARA_TEST_FALSE(FlusherSLS::IsRegionContainingConfig("cn-hangzhou")); - APSARA_TEST_TRUE(SLSClientManager::GetInstance()->GetRegionAliuids("cn-hangzhou").empty()); APSARA_TEST_TRUE(SenderQueueManager::GetInstance()->IsQueueMarkedDeleted(flusher1.GetQueueKey())); APSARA_TEST_TRUE(flusher2.Start()); APSARA_TEST_EQUAL(1U, FlusherSLS::sProjectRefCntMap.size()); - APSARA_TEST_TRUE(FlusherSLS::IsRegionContainingConfig("cn-hangzhou")); - APSARA_TEST_EQUAL(1U, SLSClientManager::GetInstance()->GetRegionAliuids("cn-hangzhou").size()); APSARA_TEST_TRUE(SenderQueueManager::GetInstance()->IsQueueMarkedDeleted(flusher1.GetQueueKey())); APSARA_TEST_FALSE(SenderQueueManager::GetInstance()->IsQueueMarkedDeleted(flusher2.GetQueueKey())); flusher2.Stop(true); @@ -534,8 +635,8 @@ void FlusherSLSUnittest::OnPipelineUpdate() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -552,6 +653,486 @@ void FlusherSLSUnittest::OnPipelineUpdate() { } } +void FlusherSLSUnittest::TestBuildRequest() { +#ifdef __ENTERPRISE__ + EnterpriseSLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo("test_region", + {kAccelerationDataEndpoint}); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + "test_region", {"test_region-intranet.log.aliyuncs.com", "test_region.log.aliyuncs.com"}); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + "test_region-b", {"test_region-b-intranet.log.aliyuncs.com", "test_region-b.log.aliyuncs.com"}); +#endif + Json::Value configJson, optionalGoPipeline; + string errorMsg; + string configStr = R"( + { + "Type": "flusher_sls", + "Project": "test_project", + "Logstore": "test_logstore", + "Region": "test_region-b", + "Aliuid": "1234567890", + "Endpoint": "test_endpoint" + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + FlusherSLS flusher; + flusher.SetContext(ctx); + flusher.SetMetricsRecordRef(FlusherSLS::sName, "1"); + APSARA_TEST_TRUE(flusher.Init(configJson, optionalGoPipeline)); + + string body = "hello, world!"; + string bodyLenStr = to_string(body.size()); + uint32_t rawSize = 100; + string rawSizeStr = "100"; + + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + unique_ptr req; + bool keepItem = false; + string errMsg; +#ifdef __ENTERPRISE__ + { + // empty ak, first try + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + // empty ak, second try + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(nullptr, req); + APSARA_TEST_TRUE(keepItem); + } + EnterpriseSLSClientManager::GetInstance()->SetAccessKey( + "1234567890", SLSClientManager::AuthType::ANONYMOUS, "test_ak", "test_sk"); + { + // no available host, uninitialized + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(nullptr, req); + APSARA_TEST_TRUE(keepItem); + APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()), + FlusherSLS::GetRegionConcurrencyLimiter(flusher.mRegion)->GetCurrentLimit()); + } + { + // no available host, initialized + flusher.mCandidateHostsInfo->SetInitialized(); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(nullptr, req); + APSARA_TEST_TRUE(keepItem); + APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()), + FlusherSLS::GetRegionConcurrencyLimiter(flusher.mRegion)->GetCurrentLimit()); + } + EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency("test_project", + EndpointMode::DEFAULT, + "test_project.test_region-b.log.aliyuncs.com", + chrono::milliseconds(100)); + flusher.mCandidateHostsInfo->SelectBestHost(); +#endif + // log telemetry type + { + // normal + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/lb", req->mUrl); + APSARA_TEST_EQUAL("", req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(11U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + { + // event group list + SLSSenderQueueItem item("hello, world!", + rawSize, + &flusher, + flusher.GetQueueKey(), + flusher.mLogstore, + RawDataType::EVENT_GROUP_LIST); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/lb", req->mUrl); + APSARA_TEST_EQUAL("", req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(13U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[X_LOG_BODYRAWSIZE]); + APSARA_TEST_EQUAL(LOG_MODE_BATCH_GROUP, req->mHeader[X_LOG_MODE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + { + // shard hash + SLSSenderQueueItem item("hello, world!", + rawSize, + &flusher, + flusher.GetQueueKey(), + flusher.mLogstore, + RawDataType::EVENT_GROUP, + "hash_key"); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/route", req->mUrl); + map params{{"key", "hash_key"}}; + APSARA_TEST_EQUAL(GetQueryString(params), req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(11U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + { + // exactly once + auto cpt = make_shared(); + cpt->index = 0; + cpt->data.set_hash_key("hash_key_0"); + cpt->data.set_sequence_id(1); + SLSSenderQueueItem item("hello, world!", + rawSize, + &flusher, + flusher.GetQueueKey(), + flusher.mLogstore, + RawDataType::EVENT_GROUP, + "hash_key_0", + std::move(cpt)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/route", req->mUrl); + map params{{"key", "hash_key_0"}, {"seqid", "1"}}; + APSARA_TEST_EQUAL(GetQueryString(params), req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(11U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + // metric telemtery type + flusher.mTelemetryType = sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_METRICS; + { + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/prometheus/test_project/test_logstore/api/v1/write", req->mUrl); + APSARA_TEST_EQUAL("", req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(11U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + flusher.mTelemetryType = sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS; +#ifdef __ENTERPRISE__ + { + // region mode changed + EnterpriseSLSClientManager::GetInstance()->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted("test_region", + "test_region-b"); + auto old = flusher.mCandidateHostsInfo.get(); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); + + EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency("test_project", + EndpointMode::ACCELERATE, + "test_project." + kAccelerationDataEndpoint, + chrono::milliseconds(10)); + flusher.mCandidateHostsInfo->SelectBestHost(); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHost); + } + // real ip + BOOL_FLAG(send_prefer_real_ip) = true; + { + // ip not empty + EnterpriseSLSClientManager::GetInstance()->SetRealIp("test_region-b", "192.168.0.1"); + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); + APSARA_TEST_FALSE(req->mHTTPSFlag); + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/lb", req->mUrl); + APSARA_TEST_EQUAL("", req->mQueryString); + APSARA_TEST_EQUAL(12U, req->mHeader.size()); + APSARA_TEST_EQUAL("test_project.192.168.0.1", req->mHeader[HOST]); + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); + APSARA_TEST_EQUAL("192.168.0.1", req->mHost); + APSARA_TEST_EQUAL(80, req->mPort); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_TRUE(item.mRealIpFlag); + APSARA_TEST_EQUAL("192.168.0.1", item.mCurrentHost); + } + { + // ip empty + EnterpriseSLSClientManager::GetInstance()->SetRealIp("test_region-b", ""); + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHeader[HOST]); + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); + APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHost); + APSARA_TEST_EQUAL(80, req->mPort); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); + APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, item.mCurrentHost); + } + { + // ip empty, and region mode changed + auto& endpoints = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["test_region-b"]; + endpoints.mMode = EndpointMode::CUSTOM; + endpoints.mLocalEndpoints = {"custom.endpoint"}; + + auto old = flusher.mCandidateHostsInfo.get(); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); + + EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency( + "test_project", EndpointMode::CUSTOM, "test_project.custom.endpoint", chrono::milliseconds(10)); + flusher.mCandidateHostsInfo->SelectBestHost(); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); + APSARA_TEST_EQUAL("test_project.custom.endpoint", req->mHost); + } + BOOL_FLAG(send_prefer_real_ip) = false; +#endif +} + void FlusherSLSUnittest::TestSend() { { // exactly once enabled @@ -563,8 +1144,8 @@ void FlusherSLSUnittest::TestSend() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -710,8 +1291,8 @@ void FlusherSLSUnittest::TestSend() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789", "ShardHashKeys": [ "tag_key" @@ -754,7 +1335,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_TRUE(item->mBufferOrNot); APSARA_TEST_EQUAL(&flusher, item->mFlusher); APSARA_TEST_EQUAL(flusher.mQueueKey, item->mQueueKey); - APSARA_TEST_EQUAL(sdk::CalcMD5("tag_value"), item->mShardHashKey); + APSARA_TEST_EQUAL(CalcMD5("tag_value"), item->mShardHashKey); APSARA_TEST_EQUAL(flusher.mLogstore, item->mLogstore); auto compressor @@ -805,8 +1386,8 @@ void FlusherSLSUnittest::TestSend() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -907,8 +1488,8 @@ void FlusherSLSUnittest::TestFlush() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -951,8 +1532,8 @@ void FlusherSLSUnittest::TestFlushAll() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -1003,8 +1584,8 @@ void FlusherSLSUnittest::OnGoPipelineSend() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -1084,13 +1665,13 @@ void FlusherSLSUnittest::OnGoPipelineSend() { UNIT_TEST_CASE(FlusherSLSUnittest, OnSuccessfulInit) UNIT_TEST_CASE(FlusherSLSUnittest, OnFailedInit) UNIT_TEST_CASE(FlusherSLSUnittest, OnPipelineUpdate) +UNIT_TEST_CASE(FlusherSLSUnittest, TestBuildRequest) UNIT_TEST_CASE(FlusherSLSUnittest, TestSend) UNIT_TEST_CASE(FlusherSLSUnittest, TestFlush) UNIT_TEST_CASE(FlusherSLSUnittest, TestFlushAll) UNIT_TEST_CASE(FlusherSLSUnittest, TestAddPackId) UNIT_TEST_CASE(FlusherSLSUnittest, OnGoPipelineSend) - } // namespace logtail UNIT_TEST_MAIN diff --git a/core/unittest/flusher/SLSClientManagerUnittest.cpp b/core/unittest/flusher/SLSClientManagerUnittest.cpp new file mode 100644 index 0000000000..71dec9fec5 --- /dev/null +++ b/core/unittest/flusher/SLSClientManagerUnittest.cpp @@ -0,0 +1,46 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "plugin/flusher/sls/SLSClientManager.h" +#include "unittest/Unittest.h" + +DECLARE_FLAG_STRING(default_access_key_id); +DECLARE_FLAG_STRING(default_access_key); + +using namespace std; + +namespace logtail { + +class SLSClientManagerUnittest : public ::testing::Test { +public: + void TestAccessKeyManagement(); + +private: + SLSClientManager mManager; +}; + +void SLSClientManagerUnittest::TestAccessKeyManagement() { + string accessKeyId, accessKeySecret; + SLSClientManager::AuthType type; + mManager.GetAccessKey("", type, accessKeyId, accessKeySecret); + APSARA_TEST_EQUAL(SLSClientManager::AuthType::AK, type); + APSARA_TEST_EQUAL(STRING_FLAG(default_access_key_id), accessKeyId); + APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); +} + +UNIT_TEST_CASE(SLSClientManagerUnittest, TestAccessKeyManagement) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/models/MetricEventUnittest.cpp b/core/unittest/models/MetricEventUnittest.cpp index ada8af3239..b148cce39b 100644 --- a/core/unittest/models/MetricEventUnittest.cpp +++ b/core/unittest/models/MetricEventUnittest.cpp @@ -346,7 +346,7 @@ void MetricEventUnittest::TestTagsIterator() { void MetricEventUnittest::TestCopy() { MetricEvent* oldMetricEvent = mEventGroup->AddMetricEvent(); oldMetricEvent->SetValue(map{{"test-1", 10.0}, {"test-2", 2.0}}); - APSARA_TEST_EQUAL(1, mEventGroup->GetEvents().size()); + APSARA_TEST_EQUAL(1U, mEventGroup->GetEvents().size()); PipelineEventGroup newGroup = mEventGroup->Copy(); MetricEvent newMetricEvent = newGroup.GetEvents().at(0).Cast(); diff --git a/core/unittest/pipeline/GlobalConfigUnittest.cpp b/core/unittest/pipeline/GlobalConfigUnittest.cpp index 7c7bc555bd..d646d17251 100644 --- a/core/unittest/pipeline/GlobalConfigUnittest.cpp +++ b/core/unittest/pipeline/GlobalConfigUnittest.cpp @@ -43,6 +43,8 @@ void GlobalConfigUnittest::OnSuccessfulInit() const { // only mandatory param config.reset(new GlobalConfig()); + APSARA_TEST_TRUE(config->Init(Json::Value(Json::ValueType::objectValue), ctx, extendedParams)); + APSARA_TEST_TRUE(extendedParams.isNull()); APSARA_TEST_EQUAL(GlobalConfig::TopicType::NONE, config->mTopicType); APSARA_TEST_EQUAL("", config->mTopicFormat); APSARA_TEST_EQUAL(1U, config->mPriority); diff --git a/core/unittest/pipeline/HttpSinkMock.h b/core/unittest/pipeline/HttpSinkMock.h index 3016f56e1d..38a3a9882e 100644 --- a/core/unittest/pipeline/HttpSinkMock.h +++ b/core/unittest/pipeline/HttpSinkMock.h @@ -22,7 +22,7 @@ #include "plugin/flusher/sls/FlusherSLS.h" #include "runner/FlusherRunner.h" #include "runner/sink/http/HttpSink.h" -#include "sdk/Common.h" +#include "plugin/flusher/sls/SLSConstant.h" namespace logtail { class HttpSinkMock : public HttpSink { @@ -64,8 +64,11 @@ class HttpSinkMock : public HttpSink { std::lock_guard lock(mMutex); mRequests.push_back(*(request->mItem)); } + request->mResponse.SetNetworkStatus(NetworkCode::Ok, ""); request->mResponse.SetStatusCode(200); - request->mResponse.mHeader[sdk::X_LOG_REQUEST_ID] = "request_id"; + request->mResponse.SetResponseTime(std::chrono::milliseconds(10)); + // for sls only + request->mResponse.mHeader[X_LOG_REQUEST_ID] = "request_id"; static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); request.reset(); diff --git a/core/unittest/pipeline/PipelineUpdateUnittest.cpp b/core/unittest/pipeline/PipelineUpdateUnittest.cpp index 6e5a4be888..1301e4046d 100644 --- a/core/unittest/pipeline/PipelineUpdateUnittest.cpp +++ b/core/unittest/pipeline/PipelineUpdateUnittest.cpp @@ -33,6 +33,9 @@ #include "unittest/pipeline/HttpSinkMock.h" #include "unittest/pipeline/LogtailPluginMock.h" #include "unittest/plugin/PluginMock.h" +#ifdef __ENTERPRISE__ +#include "config/provider/EnterpriseConfigProvider.h" +#endif using namespace std; @@ -63,7 +66,7 @@ class FlusherSLSMock : public FlusherSLS { public: static const std::string sName; - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const override { + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override { auto data = static_cast(item); std::map header; req = std::make_unique( @@ -372,10 +375,12 @@ class PipelineUpdateUnittest : public testing::Test { "Type": "flusher_stdout2" })"; - size_t builtinPipelineCnt = 0; + static size_t builtinPipelineCnt; bool isFileServerStart = false; }; +size_t PipelineUpdateUnittest::builtinPipelineCnt = 0; + void PipelineUpdateUnittest::TestFileServerStart() { isFileServerStart = true; Json::Value nativePipelineConfigJson diff --git a/core/unittest/plugin/PluginMock.h b/core/unittest/plugin/PluginMock.h index 154d1930be..e61410df75 100644 --- a/core/unittest/plugin/PluginMock.h +++ b/core/unittest/plugin/PluginMock.h @@ -151,7 +151,7 @@ class FlusherHttpMock : public HttpFlusher { return true; } bool FlushAll() override { return mIsValid; } - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const override { + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override { if (item->mData == "invalid_keep") { *keepItem = true; return false; diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index dc159d3cb1..1eb84623a3 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -14,11 +14,11 @@ * limitations under the License. */ - #include #include #include "common/StringTools.h" +#include "common/http/Curl.h" #include "common/http/HttpResponse.h" #include "common/timer/Timer.h" #include "models/RawEvent.h" @@ -80,13 +80,13 @@ void ScrapeSchedulerUnittest::TestProcess() { // if status code is not 200, no data will be processed // but will continue running, sending self-monitoring metrics httpResponse.SetStatusCode(503); - httpResponse.SetNetworkStatus(CURLE_OK); + httpResponse.SetNetworkStatus(NetworkCode::Ok, ""); event.OnMetricResult(httpResponse, 0); APSARA_TEST_EQUAL(1UL, event.mPromStreamScraper.mItem.size()); event.mPromStreamScraper.mItem.clear(); httpResponse.SetStatusCode(503); - httpResponse.SetNetworkStatus(CURLE_COULDNT_CONNECT); + httpResponse.SetNetworkStatus(GetNetworkStatus(CURLE_COULDNT_CONNECT), ""); event.OnMetricResult(httpResponse, 0); APSARA_TEST_EQUAL(event.mPromStreamScraper.mItem[0] ->mEventGroup.GetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_STATE) @@ -96,7 +96,7 @@ void ScrapeSchedulerUnittest::TestProcess() { event.mPromStreamScraper.mItem.clear(); httpResponse.SetStatusCode(200); - httpResponse.SetNetworkStatus(CURLE_OK); + httpResponse.SetNetworkStatus(NetworkCode::Ok, ""); string body1 = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n" "# TYPE go_gc_duration_seconds summary\n" "go_gc_duration_seconds{quantile=\"0\"} 1.5531e-05\n" @@ -284,4 +284,4 @@ UNIT_TEST_CASE(ScrapeSchedulerUnittest, TestExactlyScrape) } // namespace logtail -UNIT_TEST_MAIN \ No newline at end of file +UNIT_TEST_MAIN diff --git a/core/unittest/sdk/CMakeLists.txt b/core/unittest/sdk/CMakeLists.txt deleted file mode 100644 index f052c1a847..0000000000 --- a/core/unittest/sdk/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2022 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 3.22) -project(sdk_unittest) - -# add_executable(sdk_common_unittest SDKCommonUnittest.cpp) -# target_link_libraries(sdk_common_unittest ${UT_BASE_TARGET}) diff --git a/core/unittest/sdk/SDKCommonUnittest.cpp b/core/unittest/sdk/SDKCommonUnittest.cpp deleted file mode 100644 index 315a3b450d..0000000000 --- a/core/unittest/sdk/SDKCommonUnittest.cpp +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "unittest/Unittest.h" -#include "sdk/Common.h" -#include "sdk/Client.h" -#include "sdk/Exception.h" -#include "common/CompressTools.h" -#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" - -DECLARE_FLAG_STRING(default_access_key_id); -DECLARE_FLAG_STRING(default_access_key); - -namespace logtail { - -class HttpMessageUnittest : public ::testing::Test { -public: - void TestGetServerTimeFromHeader(); -}; - -UNIT_TEST_CASE(HttpMessageUnittest, TestGetServerTimeFromHeader); - -void HttpMessageUnittest::TestGetServerTimeFromHeader() { - sdk::HttpMessage httpMsg; - EXPECT_EQ(0, httpMsg.GetServerTimeFromHeader()); - - auto& header = httpMsg.header; - header["Date"] = "Thu, 18 Feb 2021 10:11:10 GMT"; - EXPECT_EQ(1613643070, httpMsg.GetServerTimeFromHeader()); - - const time_t kTimestamp = 1613588970; - header["x-log-time"] = std::to_string(kTimestamp); - EXPECT_EQ(kTimestamp, httpMsg.GetServerTimeFromHeader()); -} - -class SDKClientUnittest : public ::testing::Test {}; - -TEST_F(SDKClientUnittest, TestNetwork) { - sdk::Client client("log-global.aliyuncs.com", - STRING_FLAG(default_access_key_id), - STRING_FLAG(default_access_key), - INT32_FLAG(sls_client_send_timeout), - "192.168.1.1", - ""); - try { - client.TestNetwork(); - ASSERT_TRUE(false); - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - ASSERT_EQ(errorCode, sdk::LOGE_REQUEST_ERROR); - std::cout << "ErrorMessage: " << e.GetMessage() << std::endl; - } - - // Machine to run the test might have accesibility to Internet. - client.SetSlsHost("cn-hangzhou.log.aliyuncs.com"); - try { - client.TestNetwork(); - ASSERT_TRUE(false); - } catch (const sdk::LOGException& e) { - const std::string errorCode = e.GetErrorCode(); - std::cout << errorCode << std::endl; - std::cout << e.GetMessage() << std::endl; - if (e.GetHttpCode() == 404) { - EXPECT_EQ(errorCode, sdk::LOGE_PROJECT_NOT_EXIST); - } else if (e.GetHttpCode() == 401) { - EXPECT_EQ(ConvertErrorCode(errorCode), SEND_UNAUTHORIZED); - } else if (e.GetHttpCode() == 400) { - EXPECT_EQ(ConvertErrorCode(errorCode), SEND_PARAMETER_INVALID); - } else { - std::cout << "HttpCode: " << e.GetHttpCode() << std::endl; - EXPECT_EQ(ConvertErrorCode(errorCode), SEND_NETWORK_ERROR); - } - } -} - -TEST_F(SDKClientUnittest, TestGetRealIp) { - sdk::Client client("cn-shanghai-corp.sls.aliyuncs.com", - STRING_FLAG(default_access_key_id), - STRING_FLAG(default_access_key), - INT32_FLAG(sls_client_send_timeout), - "192.168.1.1", - ""); - logtail::sdk::GetRealIpResponse resp = client.GetRealIp(); - std::cout << "realIp: " << resp.realIp << std::endl; - EXPECT_GT(resp.realIp.size(), 0L); - - client.SetSlsHost("cn-shanghai.sls.aliyuncs.com"); - resp = client.GetRealIp(); - std::cout << "realIp: " << resp.realIp << std::endl; - EXPECT_EQ(resp.realIp.size(), 0L); -} - -/* -TEST_F(SDKClientUnittest, PostLogstoreLogsSuccessOpenSource) { - std::string uid = ""; - std::string accessKeyId = ""; - std::string accessKey = ""; - std::string region = "cn-wulanchabu"; - std::string project = ""; - std::string logstore = ""; - sdk::Client client("cn-wulanchabu.log.aliyuncs.com", - accessKeyId, - accessKey, - INT32_FLAG(sls_client_send_timeout), - "192.168.1.1", - ""); - SLSControl::Instance()->SetSlsSendClientCommonParam(&client); - client.SetKeyProvider(""); - sls_logs::LogGroup logGroup; - - logGroup.set_source("192.168.1.1"); - logGroup.set_category(logstore); - logGroup.set_topic("unittest"); - - sls_logs::Log* log = logGroup.add_logs(); - log->set_time(time(NULL)); - sls_logs::Log_Content* content = nullptr; - content = log->add_contents(); - content->set_key("kk1"); - content->set_value("vv1"); - content = log->add_contents(); - content->set_key("kk2"); - content->set_value("vv2"); - - std::string oriData; - logGroup.SerializeToString(&oriData); - int32_t logSize = (int32_t)logGroup.logs_size(); - time_t curTime = time(NULL); - sls_logs::SlsCompressType compressType = sls_logs::SLS_CMP_ZSTD; - - LogGroupContext logGroupContext(region, project, logstore, compressType); - - LoggroupTimeValue* data = new LoggroupTimeValue(project, - logstore, - "ut-config", - "ut.log", - false, - uid, - "cn-huhehaote", - LOGGROUP_COMPRESSED, - logSize, - oriData.size(), - curTime, - "", - 0, - logGroupContext); - - ASSERT_TRUE(CompressData(compressType, oriData, data->mLogData)); - - try { - sdk::PostLogStoreLogsResponse resp = client.PostLogStoreLogs( - data->mProjectName, data->mLogstore, data->mLogGroupContext.mCompressType, data->mLogData, data->mRawSize); - std::cout << resp.requestId << "," << resp.statusCode << "," << resp.bodyBytes << std::endl; - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - std::cerr << "errorCode:" << errorCode << " errorMessage: " << e.GetMessage() << std::endl; - if (e.GetMessage().find("x-log-compresstype : zstd") == std::string::npos) { // ignore compresstype error - ASSERT_TRUE(false); - } - std::cerr << "compresstype zstd is not supported, fallback to lz4" << std::endl; - } - - // fallback to lz4 - ASSERT_TRUE(UncompressData(compressType, data->mLogData, data->mRawSize, oriData)); - - compressType = sls_logs::SLS_CMP_LZ4; - - logGroupContext.mCompressType = compressType; - - data->mLogGroupContext = logGroupContext; - - ASSERT_TRUE(CompressData(compressType, oriData, data->mLogData)); - - try { - sdk::PostLogStoreLogsResponse resp = client.PostLogStoreLogs( - data->mProjectName, data->mLogstore, data->mLogGroupContext.mCompressType, data->mLogData, data->mRawSize); - std::cout << resp.requestId << "," << resp.statusCode << "," << resp.bodyBytes << std::endl; - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - std::cerr << "errorCode:" << errorCode << " errorMessage: " << e.GetMessage() << std::endl; - ASSERT_TRUE(false); - } -} - -TEST_F(SDKClientUnittest, PostLogstoreLogsSuccessClosedSource) { - std::string uid = ""; - std::string accessKeyId = ""; // start with ## - std::string accessKey = ""; - std::string region = "cn-wulanchabu"; - std::string project = ""; - std::string logstore = ""; - sdk::Client client("cn-wulanchabu.log.aliyuncs.com", - accessKeyId, - accessKey, - INT32_FLAG(sls_client_send_timeout), - "192.168.1.1", - ""); - SLSControl::Instance()->SetSlsSendClientCommonParam(&client); - client.SetKeyProvider(sdk::MD5_SHA1_SALT_KEYPROVIDER); - sls_logs::LogGroup logGroup; - - logGroup.set_source("192.168.1.1"); - logGroup.set_category(logstore); - logGroup.set_topic("unittest"); - - sls_logs::Log* log = logGroup.add_logs(); - log->set_time(time(NULL)); - sls_logs::Log_Content* content = nullptr; - content = log->add_contents(); - content->set_key("kk1"); - content->set_value("vv1"); - content = log->add_contents(); - content->set_key("kk2"); - content->set_value("vv2"); - - std::string oriData; - logGroup.SerializeToString(&oriData); - int32_t logSize = (int32_t)logGroup.logs_size(); - time_t curTime = time(NULL); - - // try zstd first - sls_logs::SlsCompressType compressType = sls_logs::SLS_CMP_ZSTD; - - LogGroupContext logGroupContext(region, project, logstore, compressType); - - LoggroupTimeValue* data = new LoggroupTimeValue(project, - logstore, - "ut-config", - "ut.log", - false, - uid, - "cn-huhehaote", - LOGGROUP_COMPRESSED, - logSize, - oriData.size(), - curTime, - "", - 0, - logGroupContext); - - ASSERT_TRUE(CompressData(compressType, oriData, data->mLogData)); - try { - sdk::PostLogStoreLogsResponse resp = client.PostLogStoreLogs( - data->mProjectName, data->mLogstore, data->mLogGroupContext.mCompressType, data->mLogData, data->mRawSize); - std::cout << resp.requestId << "," << resp.statusCode << "," << resp.bodyBytes << std::endl; - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - std::cerr << "errorCode:" << errorCode << " errorMessage: " << e.GetMessage() << std::endl; - if (e.GetMessage().find("x-log-compresstype : zstd") == std::string::npos) { // ignore compresstype error - ASSERT_TRUE(false); - } - std::cerr << "compresstype zstd is not supported, fallback to lz4" << std::endl; - } - - // fallback to lz4 - ASSERT_TRUE(UncompressData(compressType, data->mLogData, data->mRawSize, oriData)); - - compressType = sls_logs::SLS_CMP_LZ4; - - logGroupContext.mCompressType = compressType; - - data->mLogGroupContext = logGroupContext; - - ASSERT_TRUE(CompressData(compressType, oriData, data->mLogData)); - - try { - sdk::PostLogStoreLogsResponse resp = client.PostLogStoreLogs( - data->mProjectName, data->mLogstore, data->mLogGroupContext.mCompressType, data->mLogData, data->mRawSize); - std::cout << resp.requestId << "," << resp.statusCode << "," << resp.bodyBytes << std::endl; - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - std::cerr << "errorCode:" << errorCode << " errorMessage: " << e.GetMessage() << std::endl; - ASSERT_TRUE(false); - } -} -*/ -} // namespace logtail - -UNIT_TEST_MAIN diff --git a/core/unittest/sender/FlusherRunnerUnittest.cpp b/core/unittest/sender/FlusherRunnerUnittest.cpp index e0ce09cf66..eac4f5ff70 100644 --- a/core/unittest/sender/FlusherRunnerUnittest.cpp +++ b/core/unittest/sender/FlusherRunnerUnittest.cpp @@ -32,6 +32,10 @@ class FlusherRunnerUnittest : public ::testing::Test { void TestPushToHttpSink(); protected: + static void SetUpTestCase() { + AppConfig::GetInstance()->mSendRequestGlobalConcurrency = 10; + } + void TearDown() override { SenderQueueManager::GetInstance()->Clear(); HttpSink::GetInstance()->mQueue.Clear(); @@ -48,8 +52,6 @@ void FlusherRunnerUnittest::TestDispatch() { flusher->SetMetricsRecordRef("name", "1"); flusher->Init(Json::Value(), tmp); - AppConfig::GetInstance()->mSendRequestGlobalConcurrency = 10; - auto item = make_unique("content", 10, flusher.get(), flusher->GetQueueKey()); auto realItem = item.get(); flusher->PushToQueue(std::move(item)); diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index 75a36a307d..d542ec95cb 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -218,7 +218,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // span string res, errorMsg; auto events = CreateBatchedSpanEvents(); - APSARA_TEST_EQUAL(events.mEvents.size(), 1); + APSARA_TEST_EQUAL(events.mEvents.size(), 1U); APSARA_TEST_TRUE(events.mEvents[0]->GetType() == PipelineEvent::Type::SPAN); APSARA_TEST_TRUE(serializer.DoSerialize(std::move(events), res, errorMsg)); sls_logs::LogGroup logGroup; @@ -256,7 +256,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { std::istringstream s(attrs); bool ret = Json::parseFromStream(readerBuilder, s, &jsonVal, &errs); APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 10); + APSARA_TEST_EQUAL(jsonVal.size(), 10U); APSARA_TEST_EQUAL(jsonVal["rpcType"].asString(), "25"); APSARA_TEST_EQUAL(jsonVal["scope-tag-0"].asString(), "scope-value-0"); // APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).value(), ""); @@ -268,7 +268,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { std::istringstream ss(linksStr); ret = Json::parseFromStream(readerBuilder, ss, &jsonVal, &errs); APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 1); + APSARA_TEST_EQUAL(jsonVal.size(), 1U); for (auto& link : jsonVal) { APSARA_TEST_EQUAL(link["spanId"].asString(), "inner-link-spanid"); APSARA_TEST_EQUAL(link["traceId"].asString(), "inner-link-traceid"); @@ -280,7 +280,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { std::istringstream sss(eventsStr); ret = Json::parseFromStream(readerBuilder, sss, &jsonVal, &errs); APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 1); + APSARA_TEST_EQUAL(jsonVal.size(), 1U); for (auto& event : jsonVal) { APSARA_TEST_EQUAL(event["name"].asString(), "inner-event"); APSARA_TEST_EQUAL(event["timestamp"].asString(), "1000"); diff --git a/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md b/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md index 031b1ed76c..4b62fe91f6 100644 --- a/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md +++ b/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md @@ -24,7 +24,7 @@ public: class HttpFlusher : public Flusher { public: // 用于将待发送数据打包成http请求 - virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const = 0; + virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) = 0; // 用于发送完成后进行记录和处理 virtual void OnSendDone(const HttpResponse& response, SenderQueueItem* item) = 0; };